diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1830412..ef29de7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,9 +1,12 @@ name: Release builds -# Pushing a vX.Y.Z tag builds the binaries for every published target and drafts the GitHub release -# with them. Publishing that draft (by hand, after testing) triggers publish-r2.yml. +# A merge to main whose workspace version has no vX.Y.Z tag yet builds the binaries for every +# published target, tags that commit and drafts the GitHub release with them; when the tag exists +# the run stops after the version check. Pushing a vX.Y.Z tag by hand does the same for that tag. +# Publishing the draft (by hand, after testing) triggers publish-r2.yml. on: push: + branches: [main] tags: ['v*'] workflow_dispatch: @@ -22,29 +25,50 @@ jobs: runs-on: ubuntu-24.04 outputs: version: ${{ steps.version.outputs.version }} + # true when this run builds and drafts a release + release: ${{ steps.version.outputs.release }} steps: - uses: actions/checkout@v4 - id: version name: Validate release version shell: bash + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} run: | - python3 - <<'PY' + # on main, a version already tagged is already released (or being released): stop here + tagged=false + if [ "$GITHUB_REF_TYPE" = branch ]; then + version=$(python3 -c "import tomllib; print(tomllib.load(open('Cargo.toml', 'rb'))['workspace']['package']['version'])") + if gh api "repos/$GH_REPO/git/ref/tags/v$version" >/dev/null 2>&1; then + tagged=true + echo "v$version is already tagged; nothing to release." + fi + fi + TAGGED=$tagged python3 - <<'PY' import os, re, tomllib with open('Cargo.toml', 'rb') as f: version = tomllib.load(f)['workspace']['package']['version'] - if not re.fullmatch(r'\d+\.\d+\.\d+', version): - raise SystemExit('Only stable X.Y.Z versions are released') - if os.environ['GITHUB_REF_TYPE'] == 'tag' and os.environ['GITHUB_REF_NAME'] != f'v{version}': - raise SystemExit(f'Tag {os.environ["GITHUB_REF_NAME"]} does not match the workspace version {version}') - notes = f'docs/releases/v{version}.md' - if os.environ['GITHUB_REF_TYPE'] == 'tag' and not (os.path.isfile(notes) and os.path.getsize(notes)): - raise SystemExit(f'No release notes for v{version}: add {notes}') + ref_type, ref_name = os.environ['GITHUB_REF_TYPE'], os.environ['GITHUB_REF_NAME'] + pushed_main = os.environ['GITHUB_EVENT_NAME'] == 'push' and ref_type == 'branch' and ref_name == 'main' + release = ref_type == 'tag' or (pushed_main and os.environ['TAGGED'] != 'true') + if release: + if not re.fullmatch(r'\d+\.\d+\.\d+', version): + raise SystemExit('Only stable X.Y.Z versions are released') + if ref_type == 'tag' and ref_name != f'v{version}': + raise SystemExit(f'Tag {ref_name} does not match the workspace version {version}') + notes = f'docs/releases/v{version}.md' + if not (os.path.isfile(notes) and os.path.getsize(notes)): + raise SystemExit(f'No release notes for v{version}: add {notes} and merge it to main') with open(os.environ['GITHUB_OUTPUT'], 'a') as f: f.write(f'version={version}\n') + f.write(f'release={str(release).lower()}\n') PY build: needs: version + # a manual run on a branch builds without tagging or drafting, to try the build + if: needs.version.outputs.release == 'true' || github.event_name == 'workflow_dispatch' strategy: fail-fast: false matrix: @@ -113,7 +137,7 @@ jobs: if-no-files-found: error draft-release: - if: github.ref_type == 'tag' + if: needs.version.outputs.release == 'true' needs: [version, build] runs-on: ubuntu-24.04 permissions: @@ -133,6 +157,11 @@ jobs: TAG: v${{ needs.version.outputs.version }} run: | (cd artifacts && sha256sum *.tar.gz > SHA256SUMS && cat SHA256SUMS) + # a merge to main tags the commit it built; a pushed tag is already there + if [ "$GITHUB_REF_TYPE" = branch ]; then + gh api "repos/$GH_REPO/git/refs" -f ref="refs/tags/$TAG" -f sha="$GITHUB_SHA" >/dev/null + echo "Tagged $GITHUB_SHA as $TAG" + fi if gh release view "$TAG" >/dev/null 2>&1; then test "$(gh release view "$TAG" --json isDraft --jq .isDraft)" = true || { echo 'Refusing to replace the binaries of a published release'; exit 1; diff --git a/AGENTS.md b/AGENTS.md index 5bc5013..62a0d75 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,7 +8,7 @@ are gone from the machines: never run `tasks2`, `tasks2d` or the retired v1 `tas ## Builds and tests go through LoadTable itself Run every cargo build, test and clippy through the installed `loadtable` -(`~/.local/bin/loadtable`; Claude sessions can use the LoadTable MCP tools, server `tasks2`), +(`~/.local/bin/loadtable`; Claude sessions can use the LoadTable MCP tools, server `loadtable`, tools `mcp__loadtable__*`), never bare `cargo` in a shell: ```sh diff --git a/Cargo.lock b/Cargo.lock index 6d84b5e..c0a4fa9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5906,7 +5906,7 @@ dependencies = [ [[package]] name = "tasks-admission" -version = "0.1.3" +version = "0.1.4" dependencies = [ "tasks-core", "tasks-platform", @@ -5915,7 +5915,7 @@ dependencies = [ [[package]] name = "tasks-cli" -version = "0.1.3" +version = "0.1.4" dependencies = [ "anyhow", "clap", @@ -5937,7 +5937,7 @@ dependencies = [ [[package]] name = "tasks-core" -version = "0.1.3" +version = "0.1.4" dependencies = [ "serde", "serde_json", @@ -5946,7 +5946,7 @@ dependencies = [ [[package]] name = "tasks-daemon" -version = "0.1.3" +version = "0.1.4" dependencies = [ "iroh", "rustix 1.1.5", @@ -5971,7 +5971,7 @@ dependencies = [ [[package]] name = "tasks-identity" -version = "0.1.3" +version = "0.1.4" dependencies = [ "bip39", "bs58", @@ -5989,7 +5989,7 @@ dependencies = [ [[package]] name = "tasks-ledger" -version = "0.1.3" +version = "0.1.4" dependencies = [ "rusqlite", "serde_json", @@ -6002,7 +6002,7 @@ dependencies = [ [[package]] name = "tasks-mcp" -version = "0.1.3" +version = "0.1.4" dependencies = [ "serde", "serde_json", @@ -6016,7 +6016,7 @@ dependencies = [ [[package]] name = "tasks-peer" -version = "0.1.3" +version = "0.1.4" dependencies = [ "bao-tree", "base64 0.22.1", @@ -6039,7 +6039,7 @@ dependencies = [ [[package]] name = "tasks-placement" -version = "0.1.3" +version = "0.1.4" dependencies = [ "tasks-core", "tasks-timing", @@ -6048,7 +6048,7 @@ dependencies = [ [[package]] name = "tasks-platform" -version = "0.1.3" +version = "0.1.4" dependencies = [ "landlock", "libc", @@ -6060,7 +6060,7 @@ dependencies = [ [[package]] name = "tasks-policy" -version = "0.1.3" +version = "0.1.4" dependencies = [ "serde", "serde_json", @@ -6072,7 +6072,7 @@ dependencies = [ [[package]] name = "tasks-runner" -version = "0.1.3" +version = "0.1.4" dependencies = [ "tasks-core", "tasks-platform", @@ -6081,7 +6081,7 @@ dependencies = [ [[package]] name = "tasks-timing" -version = "0.1.3" +version = "0.1.4" dependencies = [ "serde", "serde_json", @@ -6091,7 +6091,7 @@ dependencies = [ [[package]] name = "tasks-ui" -version = "0.1.3" +version = "0.1.4" dependencies = [ "eframe", "jiff", diff --git a/Cargo.toml b/Cargo.toml index be76a89..0d649b4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ resolver = "3" members = ["crates/*"] [workspace.package] -version = "0.1.3" +version = "0.1.4" edition = "2024" rust-version = "1.92" license = "MIT" diff --git a/crates/tasks-cli/src/args.rs b/crates/tasks-cli/src/args.rs index 4d901a3..a71a324 100644 --- a/crates/tasks-cli/src/args.rs +++ b/crates/tasks-cli/src/args.rs @@ -617,6 +617,25 @@ pub enum Command { /// Build caches: the compiler wrapper in use, the cache root and its disk against the /// reserve. Reads the machine's files; needs no daemon. Cache, + /// What each checkout's build output takes here and on the peers, and freeing it. Only + /// build output and peers' copies are deleted, never a working tree, and never anything + /// a queued or running job uses. + Gc { + /// Free the checkouts in these states, comma separated: merged, gone, old, idle, + /// active. Without it gc only reports. + #[arg(long, value_name = "STATES", value_delimiter = ',')] + delete: Vec, + /// Free these checkouts too, whatever their state: a path, or the name the report + /// shows (`(unknown checkout …)` included). Repeat it for more. + #[arg(long, value_name = "CHECKOUT")] + free: Vec, + /// Days without a job after which a checkout counts as idle (default 14). + #[arg(long, value_name = "DAYS")] + idle_days: Option, + /// The daemon's reply as JSON. + #[arg(long)] + json: bool, + }, /// This machine's identity. Identity { /// What to do. diff --git a/crates/tasks-cli/src/commands.rs b/crates/tasks-cli/src/commands.rs index 520115f..0a5f67c 100644 --- a/crates/tasks-cli/src/commands.rs +++ b/crates/tasks-cli/src/commands.rs @@ -15,8 +15,8 @@ use std::time::Duration; use tasks_core::plan::Plan; use tasks_core::{DeviceId, JobId, JobSpec, Kind, Owner, Phase, Placement, Priority}; use tasks_daemon::api::{ - ErrorCode, HistoryView, JobView, LogSearchQuery, LogView, MAX_CONTEXT, MAX_TAIL_LINES, Request, - Response, StatusSnapshot, WaiterView, + ErrorCode, GcState, HistoryView, JobView, LogSearchQuery, LogView, MAX_CONTEXT, MAX_TAIL_LINES, + Request, Response, StatusSnapshot, WaiterView, }; use tasks_policy::PolicyFs; use tasks_policy::submit_rules::{self, PlacementClass}; @@ -96,6 +96,12 @@ fn dispatch(ctx: &mut Ctx<'_>, cmd: &Command) -> Result { Command::LogSearch(a) => log_search(ctx, a), Command::Get { id, json } => get(ctx, *id, *json), Command::Metrics { id, json } => metrics(ctx, *id, *json), + Command::Gc { + delete, + free, + idle_days, + json, + } => gc(ctx, delete, free, *idle_days, *json), Command::Cancel { id, who, reason } => cancel(ctx, *id, who.as_deref(), reason.as_deref()), Command::Prioritize { id, @@ -611,6 +617,58 @@ fn metrics(ctx: &mut Ctx<'_>, id: i64, json: bool) -> Result { Ok(EXIT_OK) } +/// `gc`: what each checkout keeps on every machine's build cache, and with `--delete` frees the +/// rows in those states (R-839), with `--free` the checkouts named (R-840). +fn gc( + ctx: &mut Ctx<'_>, + delete: &[String], + free: &[String], + idle_days: Option, + json: bool, +) -> Result { + let mut states = Vec::new(); + for word in delete.iter().filter(|w| !w.trim().is_empty()) { + match GcState::parse(word) { + Some(GcState::InUse) => { + return Err(CliError::Usage( + "gc never deletes a checkout a job uses; leave in_use out of --delete".into(), + )); + } + Some(s) => states.push(s), + None => { + return Err(CliError::Usage(format!( + "--delete takes merged, gone, old, idle or active, not {word:?}" + ))); + } + } + } + // a path on disk goes as its full form; a report name like `(unknown checkout …)` as is + let checkouts = free + .iter() + .map(|c| match std::fs::canonicalize(c) { + Ok(p) if !c.starts_with('(') => p.display().to_string(), + _ => c.clone(), + }) + .collect(); + let resp = ctx.client.call(&Request::Gc { + delete: states, + checkouts, + idle_days, + })?; + let Response::Gc(v) = resp else { + return Err(CliError::Failed(format!( + "unexpected reply to gc: {resp:?}" + ))); + }; + if json { + writeln!(ctx.out, "{}", pretty(&v)?)?; + } else { + ctx.out + .write_all(render::gc(&v, (ctx.now_ms)()).as_bytes())?; + } + Ok(EXIT_OK) +} + /// `wait`: blocks until every job is over, sending only read-only requests (R-720). As each job /// ends it prints the job's last log lines and its result line; the exit code is the first /// failed job's, as `run` would give it (R-721). `--timeout` gives up with 124 and touches diff --git a/crates/tasks-cli/src/commands/tests.rs b/crates/tasks-cli/src/commands/tests.rs index 48e447a..27cda4d 100644 --- a/crates/tasks-cli/src/commands/tests.rs +++ b/crates/tasks-cli/src/commands/tests.rs @@ -2197,3 +2197,185 @@ fn r722_wait_gives_up_after_its_timeout_with_124_and_leaves_the_job_alone() { .count(); assert_eq!(gets, 4, "polls at 0, 1, 2 and 3 s, then gives up"); } + +fn gc_view(deleted: bool) -> tasks_daemon::api::GcView { + use tasks_daemon::api::{GcHeld, GcMachine, GcRow, GcShared, GcState, GcView}; + let held = |m: &str, b: u64| GcHeld { + machine: m.into(), + bytes: b, + }; + GcView { + idle_days: 14, + machines: vec![ + GcMachine { + name: "laptop".into(), + held_bytes: 12 << 30, + reclaimable_bytes: 4 << 30, + error: None, + }, + GcMachine { + name: "nixos".into(), + held_bytes: 3 << 30, + reclaimable_bytes: 1 << 30, + error: None, + }, + GcMachine { + name: "mac".into(), + held_bytes: 0, + reclaimable_bytes: 0, + error: Some( + "mac runs an older loadtabled that cannot report its cache; upgrade it".into(), + ), + }, + ], + rows: vec![ + GcRow { + checkout: "~/w/repo/.worktrees/done".into(), + branch: Some("done".into()), + state: GcState::Merged, + merged_into: Some("dev".into()), + last_used_ms: Some(0), + held: vec![held("laptop", 4 << 30), held("nixos", 1 << 30)], + freed_bytes: if deleted { 5 << 30 } else { 0 }, + kept: None, + }, + GcRow { + checkout: "~/w/repo".into(), + branch: Some("dev".into()), + state: GcState::Idle, + merged_into: None, + last_used_ms: Some(0), + held: vec![held("laptop", 8 << 30)], + freed_bytes: 0, + kept: None, + }, + ], + shared: vec![GcShared { + name: "compiler cache (sccache)".into(), + path: "/c/sccache".into(), + bytes: 30 << 30, + note: "sccache keeps itself under 30G".into(), + }], + notes: vec![], + deleted, + } +} + +// R-839 +#[test] +fn r839_gc_shows_each_machine_and_checkout_and_how_to_free_the_rest() { + let c = Arc::new(FakeClient::from_fn(|req| match req { + Request::Gc { delete, .. } => Ok(Response::Gc(Box::new(gc_view(!delete.is_empty())))), + other => Err(ClientError::BadReply(format!("unexpected {other:?}"))), + })); + let r = Harness::default().run(&c, &["gc"]); + assert_eq!(r.code, 0, "{}", r.err); + assert!( + r.out + .contains("laptop 12.0 GiB kept, 4.0 GiB can be freed"), + "{}", + r.out + ); + assert!( + r.out + .contains("mac could not report: mac runs an older loadtabled"), + "{}", + r.out + ); + let done = r + .out + .lines() + .find(|l| l.contains(".worktrees/done")) + .unwrap(); + assert!( + done.starts_with("merged") && done.contains("4.0 GiB") && done.contains("1.0 GiB"), + "{done}" + ); + assert!( + r.out + .contains("compiler cache (sccache): 30.0 GiB in /c/sccache"), + "{}", + r.out + ); + assert!( + r.out.contains( + "Free 4.0 GiB on laptop, 1.0 GiB on nixos with: loadtable gc --delete merged,gone,old" + ), + "{}", + r.out + ); + assert!( + r.out + .contains("Idle checkouts (no job in 14 days) are kept"), + "{}", + r.out + ); + + let r = Harness::default().run(&c, &["gc", "--delete", "merged,gone", "--idle-days", "30"]); + assert_eq!(r.code, 0, "{}", r.err); + assert!( + r.out.contains("Freed 5.0 GiB from 1 checkout."), + "{}", + r.out + ); + assert!(matches!( + c.requests().last(), + Some(Request::Gc { delete, idle_days: Some(30), .. }) + if *delete == vec![tasks_daemon::api::GcState::Merged, tasks_daemon::api::GcState::Gone] + )); + assert!( + !is_read_only(&c.requests()[1]), + "a gc that deletes is never resent" + ); +} + +// R-839 +#[test] +fn r839_gc_refuses_to_delete_in_use_or_an_unknown_state_before_asking_the_daemon() { + let c = Arc::new(FakeClient::from_fn(|_| { + Err(ClientError::BadReply("asked".into())) + })); + let r = Harness::default().run(&c, &["gc", "--delete", "in-use"]); + assert_ne!(r.code, 0); + assert!( + r.err.contains("never deletes a checkout a job uses"), + "{}", + r.err + ); + let r = Harness::default().run(&c, &["gc", "--delete", "stale"]); + assert!( + r.err + .contains("--delete takes merged, gone, old, idle or active"), + "{}", + r.err + ); + assert!(c.requests().is_empty()); +} + +// R-840 +#[test] +fn r840_gc_free_sends_each_checkout_as_a_full_path_or_the_reports_name() { + let c = Arc::new(FakeClient::from_fn(|req| match req { + Request::Gc { .. } => Ok(Response::Gc(Box::new(gc_view(true)))), + other => Err(ClientError::BadReply(format!("unexpected {other:?}"))), + })); + let dir = std::env::temp_dir(); + let full = std::fs::canonicalize(&dir).unwrap().display().to_string(); + let r = Harness::default().run( + &c, + &[ + "gc", + "--free", + dir.to_str().unwrap(), + "--free", + "(unknown checkout 4ca7e0dc2b4d)", + ], + ); + assert_eq!(r.code, 0, "{}", r.err); + assert!(matches!( + c.requests().last(), + Some(Request::Gc { delete, checkouts, .. }) + if delete.is_empty() + && *checkouts == vec![full.clone(), "(unknown checkout 4ca7e0dc2b4d)".to_string()] + )); +} diff --git a/crates/tasks-cli/src/render.rs b/crates/tasks-cli/src/render.rs index 4793cad..ea204f3 100644 --- a/crates/tasks-cli/src/render.rs +++ b/crates/tasks-cli/src/render.rs @@ -4,7 +4,7 @@ use std::fmt::Write; use std::path::Path; use tasks_core::Phase; use tasks_daemon::api::{ - EstimateView, GrepGroup, JobGrep, JobView, Source, StatusSnapshot, WaiterView, + EstimateView, GcState, GcView, GrepGroup, JobGrep, JobView, Source, StatusSnapshot, WaiterView, }; use tasks_timing::dur::{fmt_eta, fmt_precise}; @@ -644,6 +644,143 @@ pub fn job_metrics(m: &tasks_core::metrics::JobMetricsView) -> String { out } +/// How long ago `ms` was, coarsely: `now`, `5h ago`, `3d ago`. +fn ago(ms: Option, now_ms: i64) -> String { + let Some(t) = ms else { + return "-".to_string(); + }; + let secs = now_ms.saturating_sub(t).max(0) / 1000; + match secs { + s if s < 3600 => "now".to_string(), + s if s < 86_400 => format!("{}h ago", s / 3600), + s => format!("{}d ago", s / 86_400), + } +} + +/// The `gc` report: per machine what checkouts keep and what can go, one row per checkout, +/// the shared caches, and the command that frees the rest (R-839). +pub fn gc(v: &GcView, now_ms: i64) -> String { + let mut out = String::new(); + out.push_str("Build output kept per machine\n"); + let width = v.machines.iter().map(|m| m.name.len()).max().unwrap_or(0); + for m in &v.machines { + match &m.error { + Some(e) => out.push_str(&format!(" {:width$} could not report: {e}\n", m.name)), + None => out.push_str(&format!( + " {:width$} {} kept, {} can be freed\n", + m.name, + fmt_bytes(m.held_bytes), + fmt_bytes(m.reclaimable_bytes) + )), + } + } + let machines: Vec<&str> = v + .machines + .iter() + .filter(|m| m.error.is_none()) + .map(|m| m.name.as_str()) + .collect(); + if v.rows.is_empty() { + out.push_str("\nNo checkout keeps build output.\n"); + } else { + let mut head = format!( + "\n{:<8} {:<44} {:<20} {:<8}", + "STATE", "CHECKOUT", "BRANCH", "LAST JOB" + ); + for m in &machines { + head.push_str(&format!(" {:>10}", truncate_end(m, 10))); + } + if v.deleted { + head.push_str(&format!(" {:>10}", "FREED")); + } + out.push_str(head.trim_end()); + out.push('\n'); + for r in &v.rows { + let mut line = format!( + "{:<8} {:<44} {:<20} {:<8}", + r.state.name(), + truncate_start(&r.checkout, 44), + truncate_end(r.branch.as_deref().unwrap_or("-"), 20), + ago(r.last_used_ms, now_ms) + ); + for m in &machines { + let bytes: u64 = r + .held + .iter() + .filter(|h| h.machine == *m) + .map(|h| h.bytes) + .sum(); + let cell = if r.held.iter().any(|h| h.machine == *m) { + fmt_bytes(bytes) + } else { + "-".to_string() + }; + line.push_str(&format!(" {cell:>10}")); + } + if v.deleted { + let freed = if r.freed_bytes > 0 { + fmt_bytes(r.freed_bytes) + } else { + "-".into() + }; + line.push_str(&format!(" {freed:>10}")); + } + out.push_str(line.trim_end()); + out.push('\n'); + if let Some(k) = &r.kept { + out.push_str(&format!(" kept: {k}\n")); + } + } + } + if !v.shared.is_empty() { + out.push_str("\nShared by every checkout (gc leaves these alone)\n"); + for s in &v.shared { + out.push_str(&format!( + " {}: {} in {} ({})\n", + s.name, + fmt_bytes(s.bytes), + s.path, + s.note + )); + } + } + for n in &v.notes { + out.push_str(&format!("\n{n}\n")); + } + out.push('\n'); + if v.deleted { + let freed: u64 = v.rows.iter().map(|r| r.freed_bytes).sum(); + let n = v.rows.iter().filter(|r| r.freed_bytes > 0).count(); + out.push_str(&format!( + "Freed {} from {n} checkout{}.\n", + fmt_bytes(freed), + if n == 1 { "" } else { "s" } + )); + } else { + let free: Vec = v + .machines + .iter() + .filter(|m| m.reclaimable_bytes > 0) + .map(|m| format!("{} on {}", fmt_bytes(m.reclaimable_bytes), m.name)) + .collect(); + if free.is_empty() { + out.push_str("Nothing merged, gone or old to free.\n"); + } else { + out.push_str(&format!( + "Free {} with: loadtable gc --delete merged,gone,old\n", + free.join(", ") + )); + } + if v.rows.iter().any(|r| r.state == GcState::Idle) { + out.push_str(&format!( + "Idle checkouts (no job in {} days) are kept; add idle to free them too.\n", + v.idle_days + )); + } + } + out +} + /// Bytes as GiB or MiB. pub fn fmt_bytes(b: u64) -> String { const GIB: f64 = (1u64 << 30) as f64; diff --git a/crates/tasks-daemon/src/api.rs b/crates/tasks-daemon/src/api.rs index e9eeac4..5d79323 100644 --- a/crates/tasks-daemon/src/api.rs +++ b/crates/tasks-daemon/src/api.rs @@ -225,6 +225,20 @@ pub enum Request { #[serde(default)] who: Option, }, + /// What each checkout keeps on the build caches here and on the peers; with `delete`, + /// first frees the rows in those states that no job uses (R-839). + Gc { + /// The states to free; empty only reports. + #[serde(default)] + delete: Vec, + /// Checkouts to free as well, named as the report shows them (`~/…` or + /// `(unknown checkout …)`); one a job uses is still kept. + #[serde(default)] + checkouts: Vec, + /// Days without a job after which a checkout is idle (default 14). + #[serde(default)] + idle_days: Option, + }, /// Live samples and process groups of one job on this machine. JobMetrics { /// The job. @@ -594,6 +608,8 @@ pub enum Response { History(HistoryView), /// Live samples, process groups and the usual peak of one job. JobMetrics(tasks_core::metrics::JobMetricsView), + /// Answer to `gc`. + Gc(Box), /// Audit rows, newest first. Audit { /// The rows. @@ -1556,6 +1572,155 @@ impl TripDetails { } } +/// What a checkout's build output is to `gc`, most protected first (R-839). +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum GcState { + /// A job of it is queued, waiting or running: never deleted. + InUse, + /// Used within the idle days. + Active, + /// No job for longer than the idle days. + Idle, + /// Its branch is fully merged into dev, main or master and nothing tracked is changed. + Merged, + /// Its working tree no longer exists. + Gone, + /// An output directory of the older layout no working tree links to. + Old, +} + +impl GcState { + /// Every state, in order. + pub const ALL: [GcState; 6] = [ + GcState::InUse, + GcState::Active, + GcState::Idle, + GcState::Merged, + GcState::Gone, + GcState::Old, + ]; + + /// The word the CLI shows and takes. + pub fn name(self) -> &'static str { + match self { + GcState::InUse => "in_use", + GcState::Active => "active", + GcState::Idle => "idle", + GcState::Merged => "merged", + GcState::Gone => "gone", + GcState::Old => "old", + } + } + + /// The state named `s` (`in-use` also). + pub fn parse(s: &str) -> Option { + let s = s.trim().replace('-', "_"); + GcState::ALL.into_iter().find(|g| g.name() == s) + } + + /// True for the states `gc` suggests freeing: nothing there is still worked on. + pub fn reclaimable(self) -> bool { + matches!(self, GcState::Merged | GcState::Gone | GcState::Old) + } + + /// Sort order of the report: what can go first. + pub fn order(self) -> u8 { + match self { + GcState::Gone => 0, + GcState::Old => 1, + GcState::Merged => 2, + GcState::Idle => 3, + GcState::Active => 4, + GcState::InUse => 5, + } + } +} + +/// The `gc` report (R-839). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct GcView { + /// Days without a job after which a checkout counted as idle. + pub idle_days: u32, + /// This machine first, then each peer. + pub machines: Vec, + /// One row per checkout holding anything, what can go first. + pub rows: Vec, + /// Caches every checkout shares: reported, never deleted. + #[serde(default)] + pub shared: Vec, + /// Anything else worth a line. + #[serde(default)] + pub notes: Vec, + /// True when the request deleted. + #[serde(default)] + pub deleted: bool, +} + +/// One machine's totals in a `gc` report. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct GcMachine { + /// Machine name. + pub name: String, + /// Bytes of checkouts' build output (and, on a peer, synced copies) it keeps for this machine; + /// after a delete, what is left. + pub held_bytes: u64, + /// Of those, the bytes in merged, gone and old rows still there. + pub reclaimable_bytes: u64, + /// Why the machine could not report. + #[serde(default)] + pub error: Option, +} + +/// One checkout in a `gc` report. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct GcRow { + /// The working tree (home-relative), or `(unknown checkout …)`. + pub checkout: String, + /// Its branch; none when detached or gone. + #[serde(default)] + pub branch: Option, + /// What it is to `gc`. + pub state: GcState, + /// The base it is merged into. + #[serde(default)] + pub merged_into: Option, + /// Its last job, ms. + #[serde(default)] + pub last_used_ms: Option, + /// What each machine keeps of it. + #[serde(default)] + pub held: Vec, + /// Bytes this request freed. + #[serde(default)] + pub freed_bytes: u64, + /// Why something of it was kept. + #[serde(default)] + pub kept: Option, +} + +/// What one machine keeps of a checkout. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct GcHeld { + /// Machine name. + pub machine: String, + /// Bytes. + pub bytes: u64, +} + +/// A cache every checkout shares. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct GcShared { + /// What it is. + pub name: String, + /// Where. + pub path: String, + /// Bytes. + pub bytes: u64, + /// How it is kept in bounds. + pub note: String, +} + /// A duration estimate. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct EstimateView { diff --git a/crates/tasks-daemon/src/build_cache/mod.rs b/crates/tasks-daemon/src/build_cache/mod.rs index 5ac4713..1401968 100644 --- a/crates/tasks-daemon/src/build_cache/mod.rs +++ b/crates/tasks-daemon/src/build_cache/mod.rs @@ -92,10 +92,19 @@ const WRAPPER_VARS: &[&str] = &[ const SCCACHE_PREFIX: &str = "SCCACHE_"; /// Where sccache lives besides `PATH`, under `$HOME`. const SCCACHE_FALLBACK_DIRS: &[&str] = &[".cargo/bin"]; -/// The directories under `$HOME` the daemon looks in for tools besides its own `PATH`, as -/// absolute paths, for the job's `PATH` too (R-543). Empty without a `HOME`. +/// Where Homebrew puts tools on Apple silicon and on Intel Macs. A launchd agent's `PATH` has +/// neither, so `pkg-config` and the libraries' tools installed there went unfound (R-841). +const MACOS_TOOL_DIRS: &[&str] = &["/opt/homebrew/bin", "/usr/local/bin"]; +/// The directories the daemon looks in for tools besides its own `PATH`, as absolute paths, +/// for the job's `PATH` too (R-543): `$HOME/.cargo/bin`, then on macOS Homebrew's (R-841). pub fn fallback_tool_dirs(env: &[(String, String)]) -> Vec { - env.iter() + fallback_tool_dirs_on(env, cfg!(target_os = "macos")) +} + +/// [`fallback_tool_dirs`] for a macOS machine or not. +pub(crate) fn fallback_tool_dirs_on(env: &[(String, String)], macos: bool) -> Vec { + let mut dirs: Vec = env + .iter() .find(|(k, _)| k == "HOME") .map(|(_, h)| h.as_str()) .filter(|h| !h.is_empty()) @@ -105,7 +114,11 @@ pub fn fallback_tool_dirs(env: &[(String, String)]) -> Vec { .map(|d| Path::new(home).join(d).to_string_lossy().into_owned()) .collect() }) - .unwrap_or_default() + .unwrap_or_default(); + if macos { + dirs.extend(MACOS_TOOL_DIRS.iter().map(|d| d.to_string())); + } + dirs } /// The sccache directory and the Gradle home under the cache root. diff --git a/crates/tasks-daemon/src/build_cache/tests.rs b/crates/tasks-daemon/src/build_cache/tests.rs index 73bed59..5a9fca0 100644 --- a/crates/tasks-daemon/src/build_cache/tests.rs +++ b/crates/tasks-daemon/src/build_cache/tests.rs @@ -183,3 +183,30 @@ fn r401_the_cache_policy_keys_default_to_auto_and_thirty_gib() { ); assert!(!bad.problems.is_empty()); } + +// R-841 +#[test] +fn r841_on_macos_jobs_and_the_tool_check_also_look_in_homebrews_dirs() { + let env = vec![("HOME".to_string(), "/Users/me".to_string())]; + assert_eq!( + fallback_tool_dirs_on(&env, false), + vec!["/Users/me/.cargo/bin"] + ); + assert_eq!( + fallback_tool_dirs_on(&env, true), + vec![ + "/Users/me/.cargo/bin", + "/opt/homebrew/bin", + "/usr/local/bin" + ] + ); + // without a HOME the Homebrew dirs are still there + assert_eq!( + fallback_tool_dirs_on(&[], true), + vec!["/opt/homebrew/bin", "/usr/local/bin"] + ); + assert_eq!( + fallback_tool_dirs(&env), + fallback_tool_dirs_on(&env, cfg!(target_os = "macos")) + ); +} diff --git a/crates/tasks-daemon/src/confine.rs b/crates/tasks-daemon/src/confine.rs index bd6b203..75da301 100644 --- a/crates/tasks-daemon/src/confine.rs +++ b/crates/tasks-daemon/src/confine.rs @@ -23,6 +23,9 @@ pub const HOME_TOOLS_WRITE: &[&str] = &[ ".npm", ".m2", ".konan", + // the Kotlin compiler daemon keeps its run files here; without it every Gradle build of + // Kotlin falls back to compiling in-process after a failed start (R-843) + ".local/share/kotlin", "Library/Developer", "Library/Android", "Library/Caches/org.swift.swiftpm", @@ -220,6 +223,26 @@ mod tests { assert!(note(&c).contains("cannot open")); } + #[test] + fn r843_a_peer_job_may_write_the_kotlin_daemons_folder_but_not_its_neighbours() { + let (t, cache, env) = rig(); + let home = t.path().join("home"); + std::fs::create_dir_all(home.join(".local/share/kotlin")).unwrap(); + std::fs::create_dir_all(home.join(".local/share/keyrings")).unwrap(); + let helper = vec!["/x/loadtabled".to_string(), "confine".to_string()]; + let c = for_peer_job(&policy(""), &cache, &env, Some(&helper)) + .unwrap() + .unwrap(); + let home = real(&home).unwrap(); + assert!(c.write.contains(&home.join(".local/share/kotlin"))); + let all: Vec<_> = c.write.iter().chain(&c.read).collect(); + assert!( + !all.iter() + .any(|p| p.starts_with(home.join(".local/share/keyrings"))) + ); + assert!(!all.contains(&&home.join(".local/share"))); + } + #[test] fn r821_the_owner_widens_it_or_switches_it_off_in_the_policy() { let (t, cache, env) = rig(); diff --git a/crates/tasks-daemon/src/forwarder.rs b/crates/tasks-daemon/src/forwarder.rs index 220f299..42a582d 100644 --- a/crates/tasks-daemon/src/forwarder.rs +++ b/crates/tasks-daemon/src/forwarder.rs @@ -235,6 +235,33 @@ pub trait Forwarder: Send + Sync { peer.0 ))) } + /// What `peer` keeps for this machine's checkouts (R-839). `Err` says why it cannot tell. + fn peer_cache( + &self, + peer: &MachineName, + ) -> Result, ForwardError> { + Err(ForwardError::Unavailable(format!( + "no peer support in this daemon, cannot reach {}", + peer.0 + ))) + } + /// Asks `peer` to delete its copies and build output of `checkouts` (R-839). + fn peer_cache_drop( + &self, + peer: &MachineName, + checkouts: Vec, + ) -> Result, ForwardError> { + let _ = checkouts; + Err(ForwardError::Unavailable(format!( + "no peer support in this daemon, cannot reach {}", + peer.0 + ))) + } + /// The id a peer keeps the checkout holding `cwd` under; none without peer support. + fn checkout_id(&self, cwd: &Path) -> Option { + let _ = cwd; + None + } /// Drops everything held for the job (its place in the checkout queue). fn forget(&self, job: JobId) { let _ = job; diff --git a/crates/tasks-daemon/src/gc.rs b/crates/tasks-daemon/src/gc.rs new file mode 100644 index 0000000..ac87552 --- /dev/null +++ b/crates/tasks-daemon/src/gc.rs @@ -0,0 +1,650 @@ +//! `loadtable gc`: what each checkout keeps on the build caches of this machine and its peers, +//! and freeing it (R-839). +//! +//! A checkout is a Git working tree that ran jobs here (the ledger) or is a worktree of a +//! repository that did (`git worktree list`). Its build output is every directory under the +//! cache root it owns: `loadtable/checkouts/` (and the older `*/tasks/checkouts/`, +//! the same key), plus wherever its `target` and Android links point. Ownership comes from those +//! links, never from a folder's name, so an older layout a live worktree still builds in is kept +//! with that worktree. +//! +//! Only build output and peers' synced copies are deleted, never a working tree: the worst a +//! deletion costs is a cold build. A checkout with a job queued or running is never touched. + +use crate::api::{ApiError, ErrorCode, GcHeld, GcMachine, GcRow, GcShared, GcState, GcView}; +use crate::build_cache::outputs::LINK_NAMES; +use crate::scheduler::Scheduler; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; +use std::process::Command; +use tasks_core::MachineName; +use tasks_ledger::{JobRecord, ListFilter}; +use tasks_peer::CheckoutId; +use tasks_peer::proto::HeldCheckout; +use tasks_peer::sync::dir_size; + +/// A checkout untouched this long is idle, unless the request says otherwise. +pub const DEFAULT_IDLE_DAYS: u32 = 14; +/// The branches a finished branch is merged into, first found wins. +const BASES: &[&str] = &["dev", "main", "master"]; +const DAY_MS: i64 = 86_400_000; + +/// What the ledger says about one checkout. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct Seen { + /// The newest submit or finish of its jobs, ms. + pub last_ms: Option, + /// A job of it is queued, waiting or running. + pub busy: bool, +} + +/// The checkouts of `records` by root, leaving out the jobs peers sent here (their copies are +/// the peer store's). +pub(crate) fn from_ledger(records: &[JobRecord], peer_store: &Path) -> BTreeMap { + let store = std::fs::canonicalize(peer_store).unwrap_or_else(|_| peer_store.to_path_buf()); + let mut out: BTreeMap = BTreeMap::new(); + for r in records { + if r.origin.is_some() + || r.spec.cwd.starts_with(&store) + || r.spec.cwd.starts_with(peer_store) + { + continue; + } + let root = PathBuf::from(crate::place::checkout_key(&r.spec.cwd)); + let seen = out.entry(root).or_default(); + let t = r.finished_at_ms.unwrap_or(r.submitted_at_ms); + seen.last_ms = seen.last_ms.max(Some(t)); + seen.busy |= !r.phase.is_terminal(); + } + out +} + +/// Runs `git -C dir args`, its trimmed output when it succeeds. +fn git(dir: &Path, args: &[&str]) -> Option { + let out = Command::new("git") + .arg("-C") + .arg(dir) + .args(args) + .output() + .ok()?; + out.status + .success() + .then(|| String::from_utf8_lossy(&out.stdout).trim().to_string()) +} + +/// The worktrees of the repository holding `dir`, `dir` included. +pub(crate) fn worktrees(dir: &Path) -> Vec { + git(dir, &["worktree", "list", "--porcelain"]) + .map(|text| { + text.lines() + .filter_map(|l| l.strip_prefix("worktree ")) + .map(PathBuf::from) + .collect() + }) + .unwrap_or_default() +} + +/// A working tree's branch, and the base it is fully merged into: its commit is in the base, +/// it is not the base itself and no tracked file is changed. +pub(crate) fn branch_of(root: &Path) -> (Option, Option) { + let branch = git(root, &["symbolic-ref", "--short", "-q", "HEAD"]).filter(|b| !b.is_empty()); + let Some(head) = git(root, &["rev-parse", "-q", "--verify", "HEAD"]) else { + return (branch, None); + }; + let clean = + git(root, &["status", "--porcelain", "--untracked-files=no"]).is_some_and(|s| s.is_empty()); + let merged = BASES + .iter() + .find(|b| { + git( + root, + &["rev-parse", "-q", "--verify", &format!("refs/heads/{b}")], + ) + .is_some() + }) + .filter(|b| branch.as_deref() != Some(**b)) + .filter(|b| { + clean + && Command::new("git") + .arg("-C") + .arg(root) + .args(["merge-base", "--is-ancestor", &head, b]) + .output() + .is_ok_and(|o| o.status.success()) + }) + .map(|b| (*b).to_string()); + (branch, merged) +} + +/// 64 lowercase hex characters: the name of a checkout's output directory. +fn is_key(name: &str) -> bool { + name.len() == 64 + && name + .bytes() + .all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase()) +} + +/// The key of a local checkout's output directory, as the placement computes it. +fn key_of(root: &Path) -> String { + use sha2::{Digest, Sha256}; + use std::fmt::Write; + Sha256::digest(root.as_os_str().as_encoded_bytes()) + .iter() + .fold(String::with_capacity(64), |mut s, b| { + let _ = write!(s, "{b:02x}"); + s + }) +} + +/// The directories under the cache root that hold checkouts' outputs, canonical: the current +/// `loadtable/checkouts` and every older `/tasks/checkouts`. +pub(crate) fn output_parents(cache_root: &Path) -> Vec { + let mut out = BTreeSet::new(); + if let Ok(c) = std::fs::canonicalize(cache_root.join(crate::build_cache::CHECKOUTS_SUBDIR)) { + out.insert(c); + } + if let Ok(rd) = std::fs::read_dir(cache_root) { + for e in rd.flatten() { + if let Ok(c) = std::fs::canonicalize(e.path().join("tasks/checkouts")) + && c.is_dir() + { + out.insert(c); + } + } + } + out.into_iter().collect() +} + +/// The output directories `root` owns under `cache` (canonical): its key under each of +/// `parents`, and the directory each of its links points into. +pub(crate) fn owned_dirs(root: &Path, cache: &Path, parents: &[PathBuf]) -> Vec { + let mut out = BTreeSet::new(); + let canon_root = std::fs::canonicalize(root).unwrap_or_else(|_| root.to_path_buf()); + for key in [key_of(&canon_root), key_of(root)] { + for p in parents { + let d = p.join(&key); + if d.is_dir() { + out.insert(d); + } + } + } + for rel in LINK_NAMES { + let link = root.join(rel); + if !std::fs::symlink_metadata(&link).is_ok_and(|m| m.is_symlink()) { + continue; + } + let Ok(target) = std::fs::canonicalize(&link) else { + continue; + }; + // `/target` owns ``; a link elsewhere in the cache owns what it points to + let owner = strip_rel(&target, rel).unwrap_or(target); + if owner.starts_with(cache) && owner != cache { + out.insert(owner); + } + } + out.into_iter().collect() +} + +/// The output key directory `path` is `/` of, if it is. +fn strip_rel(path: &Path, rel: &str) -> Option { + let mut p = path; + for _ in Path::new(rel).components() { + p = p.parent()?; + } + (p.join(rel) == path && p.file_name().is_some_and(|n| is_key(&n.to_string_lossy()))) + .then(|| p.to_path_buf()) +} + +/// The state of a checkout, most important first: a busy one is in use whatever else holds. +pub(crate) fn state_of( + busy: bool, + exists: bool, + merged: bool, + last_ms: Option, + now_ms: i64, + idle_days: u32, +) -> GcState { + if busy { + GcState::InUse + } else if !exists { + GcState::Gone + } else if merged { + GcState::Merged + } else if last_ms.is_some_and(|t| now_ms.saturating_sub(t) > i64::from(idle_days) * DAY_MS) { + GcState::Idle + } else { + GcState::Active + } +} + +/// One row while it is assembled. +struct Draft { + row: GcRow, + dirs: Vec, + /// The id peers keep it under. + id: Option, +} + +/// Sizes of `dirs`, measured in parallel. +fn sizes(dirs: &[PathBuf]) -> BTreeMap { + std::thread::scope(|s| { + let handles: Vec<_> = dirs + .iter() + .map(|d| (d.clone(), s.spawn(move || dir_size(d)))) + .collect(); + handles + .into_iter() + .map(|(d, h)| (d, h.join().unwrap_or(0))) + .collect() + }) +} + +/// The name a row shows for a checkout with no working tree found. +fn unknown(dir: &Path) -> String { + let short: String = dir + .file_name() + .map(|n| n.to_string_lossy().chars().take(12).collect()) + .unwrap_or_default(); + format!("(unknown checkout {short})") +} + +/// Home-relative text of `p`, as the CLI shows paths. +fn shown(p: &Path, home: Option<&Path>) -> String { + match home.and_then(|h| p.strip_prefix(h).ok()) { + Some(rel) => format!("~/{}", rel.display()), + None => p.display().to_string(), + } +} + +impl Scheduler { + /// The `gc` report, deleting first the rows whose state is in `delete` and the rows named + /// in `checkouts` (R-839, R-840). + pub(crate) fn gc( + &self, + delete: &[GcState], + checkouts: &[String], + idle_days: Option, + ) -> Result { + let idle_days = idle_days.unwrap_or(DEFAULT_IDLE_DAYS); + let now = self.clock.now_ms(); + let cache = self.build_cache(); + let cache_root = + std::fs::canonicalize(cache.root()).unwrap_or_else(|_| cache.root().to_path_buf()); + let records = self.ledger.list(&ListFilter::default()).map_err(|e| { + ApiError::new(ErrorCode::Internal, format!("cannot read the ledger: {e}")) + })?; + let mut seen = from_ledger(&records, cache.peer_store()); + let repos: Vec = seen.keys().filter(|r| r.is_dir()).cloned().collect(); + for repo in repos { + for wt in worktrees(&repo) { + seen.entry(wt).or_default(); + } + } + let home = self + .parent_env + .iter() + .find(|(k, _)| k == "HOME") + .map(|(_, v)| PathBuf::from(v)); + let parents = output_parents(&cache_root); + let local = self.cfg.machine.clone(); + + let mut drafts: Vec = Vec::new(); + let mut claimed: BTreeSet = BTreeSet::new(); + for (root, s) in &seen { + let exists = root.is_dir(); + let (branch, merged_into) = if exists { + branch_of(root) + } else { + (None, None) + }; + let dirs: Vec = owned_dirs(root, &cache_root, &parents) + .into_iter() + .filter(|d| claimed.insert(d.clone())) + .collect(); + let last_ms = s.last_ms.or_else(|| newest_mtime(&dirs)); + let state = state_of( + s.busy, + exists, + merged_into.is_some(), + last_ms, + now, + idle_days, + ); + drafts.push(Draft { + row: GcRow { + checkout: shown(root, home.as_deref()), + branch, + state, + merged_into, + last_used_ms: last_ms, + held: Vec::new(), + freed_bytes: 0, + kept: None, + }, + dirs, + id: self.forwarder.checkout_id(root), + }); + } + // output directories no checkout claims: a deleted worktree's, or an older layout's + for parent in &parents { + let old = !parent.ends_with(crate::build_cache::CHECKOUTS_SUBDIR); + let Ok(rd) = std::fs::read_dir(parent) else { + continue; + }; + for e in rd.flatten() { + let d = e.path(); + if !is_key(&e.file_name().to_string_lossy()) + || !d.is_dir() + || !claimed.insert(d.clone()) + { + continue; + } + let last_ms = newest_mtime(std::slice::from_ref(&d)); + drafts.push(Draft { + row: GcRow { + checkout: unknown(&d), + branch: None, + state: if old { GcState::Old } else { GcState::Gone }, + merged_into: None, + last_used_ms: last_ms, + held: Vec::new(), + freed_bytes: 0, + kept: None, + }, + dirs: vec![d], + id: None, + }); + } + } + + let all_dirs: Vec = drafts.iter().flat_map(|d| d.dirs.clone()).collect(); + let (local_sizes, peer_reports) = std::thread::scope(|s| { + let sized = s.spawn(|| sizes(&all_dirs)); + let asked: Vec<_> = self + .forwarder + .peers() + .into_iter() + .map(|p| { + let name = p.machine.name; + let fw = &self.forwarder; + s.spawn(move || { + let r = fw.peer_cache(&name); + (name, r) + }) + }) + .collect(); + let reports: Vec<(MachineName, Result, String>)> = asked + .into_iter() + .filter_map(|h| h.join().ok()) + .map(|(n, r)| (n, r.map_err(|e| e.to_string()))) + .collect(); + (sized.join().unwrap_or_default(), reports) + }); + + for d in &mut drafts { + let bytes: u64 = d.dirs.iter().filter_map(|x| local_sizes.get(x)).sum(); + if bytes > 0 || !d.dirs.is_empty() { + d.row.held.push(GcHeld { + machine: local.clone(), + bytes, + }); + } + } + let mut machines = vec![GcMachine { + name: local.clone(), + held_bytes: local_sizes.values().sum(), + reclaimable_bytes: 0, + error: None, + }]; + for (peer, report) in peer_reports { + let mut m = GcMachine { + name: peer.0.clone(), + held_bytes: 0, + reclaimable_bytes: 0, + error: None, + }; + match report { + Err(e) => m.error = Some(e), + Ok(held) => { + for h in held { + let bytes = h.copy_bytes.saturating_add(h.output_bytes); + m.held_bytes = m.held_bytes.saturating_add(bytes); + let at = drafts + .iter() + .position(|d| d.id.as_ref() == Some(&h.checkout)); + let i = match at { + Some(i) => i, + None => { + drafts.push(Draft { + row: GcRow { + checkout: format!( + "(unknown checkout {})", + h.checkout + .as_str() + .chars() + .take(12) + .collect::() + ), + branch: None, + state: GcState::Gone, + merged_into: None, + last_used_ms: h.last_used_ms, + held: Vec::new(), + freed_bytes: 0, + kept: None, + }, + dirs: Vec::new(), + id: Some(h.checkout.clone()), + }); + drafts.len() - 1 + } + }; + if let Some(d) = drafts.get_mut(i) { + if h.busy { + d.row.state = GcState::InUse; + } + d.row.last_used_ms = d.row.last_used_ms.max(h.last_used_ms); + d.row.held.push(GcHeld { + machine: peer.0.clone(), + bytes, + }); + } + } + } + } + machines.push(m); + } + // rows holding nothing anywhere are not worth a line + drafts.retain(|d| d.row.held.iter().any(|h| h.bytes > 0) || d.row.state == GcState::InUse); + + let deleted = !delete.is_empty() || !checkouts.is_empty(); + let mut notes = Vec::new(); + // a full path names the same row as its `~/…` form + let checkouts: Vec = checkouts + .iter() + .map(|c| match c.starts_with('/') { + true => shown(Path::new(c.trim_end_matches('/')), home.as_deref()), + false => c.clone(), + }) + .collect(); + let checkouts = checkouts.as_slice(); + if deleted { + let freed = self.gc_delete(&mut drafts, delete, checkouts, &cache_root); + // the totals are what each machine keeps now, not what it kept before + for m in &mut machines { + m.held_bytes = m + .held_bytes + .saturating_sub(freed.get(&m.name).copied().unwrap_or(0)); + } + for c in checkouts { + match drafts.iter_mut().find(|d| d.row.checkout == *c) { + Some(d) if d.row.state == GcState::InUse => { + d.row.kept = Some("a job uses it; free it once the job ends".into()); + } + Some(_) => {} + None => notes.push(format!( + "{c} holds no build output here; run loadtable gc to see what does" + )), + } + } + } + for m in &mut machines { + m.reclaimable_bytes = drafts + .iter() + .filter(|d| d.row.state.reclaimable() && d.row.freed_bytes == 0) + .flat_map(|d| &d.row.held) + .filter(|h| h.machine == m.name) + .map(|h| h.bytes) + .sum(); + } + let mut rows: Vec = drafts.into_iter().map(|d| d.row).collect(); + rows.sort_by(|a, b| { + a.state + .order() + .cmp(&b.state.order()) + .then_with(|| total(b).cmp(&total(a))) + }); + Ok(GcView { + idle_days, + machines, + rows, + shared: self.gc_shared(), + notes, + deleted, + }) + } + + /// Deletes the output and peer copies of every row in `delete` or named in `checkouts` + /// that is not in use, and returns the bytes freed on each machine. + fn gc_delete( + &self, + drafts: &mut [Draft], + delete: &[GcState], + checkouts: &[String], + cache_root: &Path, + ) -> BTreeMap { + let mut freed: BTreeMap = BTreeMap::new(); + let mut per_peer: BTreeMap> = BTreeMap::new(); + for d in drafts.iter_mut() { + let chosen = delete.contains(&d.row.state) || checkouts.contains(&d.row.checkout); + if d.row.state == GcState::InUse || !chosen { + continue; + } + let local = d + .row + .held + .iter() + .find(|h| h.machine == self.cfg.machine) + .map_or(0, |h| h.bytes); + let mut failed = Vec::new(); + for dir in &d.dirs { + // a last look: only a directory inside the cache, never the cache itself + if !dir.starts_with(cache_root) || dir == cache_root { + continue; + } + if let Err(e) = std::fs::remove_dir_all(dir) { + failed.push(format!("{}: {e}", dir.display())); + } + } + if failed.is_empty() { + d.row.freed_bytes = d.row.freed_bytes.saturating_add(local); + *freed.entry(self.cfg.machine.clone()).or_default() += local; + } else { + d.row.kept = Some(format!("not all deleted: {}", failed.join("; "))); + } + if let Some(id) = &d.id { + for h in d.row.held.iter().filter(|h| h.machine != self.cfg.machine) { + per_peer + .entry(h.machine.clone()) + .or_default() + .push(id.clone()); + } + } + } + for (peer, ids) in per_peer { + match self + .forwarder + .peer_cache_drop(&MachineName(peer.clone()), ids.clone()) + { + Ok(dropped) => { + for x in dropped { + *freed.entry(peer.clone()).or_default() += x.freed_bytes; + if let Some(d) = drafts + .iter_mut() + .find(|d| d.id.as_ref() == Some(&x.checkout)) + { + d.row.freed_bytes = d.row.freed_bytes.saturating_add(x.freed_bytes); + if let Some(k) = x.kept { + d.row.kept = Some(format!("{peer}: {k}")); + } + } + } + } + Err(e) => { + for d in drafts + .iter_mut() + .filter(|d| d.id.as_ref().is_some_and(|i| ids.contains(i))) + { + d.row.kept = Some(format!("{peer}: {e}")); + } + } + } + } + freed + } + + /// Caches every checkout shares, reported and never deleted by `gc`. + fn gc_shared(&self) -> Vec { + let cache = self.build_cache(); + let policy = self.policy(); + let mut out = Vec::new(); + let sccache = cache.sccache_dir(); + if sccache.is_dir() { + out.push(GcShared { + name: "compiler cache (sccache)".into(), + path: sccache.display().to_string(), + bytes: dir_size(&sccache), + note: format!("sccache keeps itself under {}", policy.sccache_size()), + }); + } + if let Some(g) = cache.gradle_home() { + out.push(GcShared { + name: "Gradle home".into(), + path: g.display().to_string(), + bytes: dir_size(&g), + note: "shared by every Gradle build; not cleaned by gc".into(), + }); + } + let store = cache.peer_store(); + if store.is_dir() { + out.push(GcShared { + name: "copies held for peers".into(), + path: store.display().to_string(), + bytes: dir_size(store), + note: format!( + "removed after {} idle days, with their build output", + policy.sync_retention().idle.as_secs() / 86_400 + ), + }); + } + out + } +} + +/// All bytes a row holds. +fn total(r: &GcRow) -> u64 { + r.held.iter().map(|h| h.bytes).sum() +} + +/// The newest modification time among `dirs`, ms. +fn newest_mtime(dirs: &[PathBuf]) -> Option { + dirs.iter() + .filter_map(|d| std::fs::metadata(d).and_then(|m| m.modified()).ok()) + .filter_map(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + .filter_map(|d| i64::try_from(d.as_millis()).ok()) + .max() +} + +#[cfg(test)] +#[path = "gc_tests.rs"] +mod tests; diff --git a/crates/tasks-daemon/src/gc_tests.rs b/crates/tasks-daemon/src/gc_tests.rs new file mode 100644 index 0000000..8b55035 --- /dev/null +++ b/crates/tasks-daemon/src/gc_tests.rs @@ -0,0 +1,436 @@ +//! `loadtable gc` (R-839): states, ownership through links, and what a deletion touches. + +use super::*; +use crate::api::{Reply, Request, Response}; +use crate::build_cache::BuildCache; +use crate::forwarder::Forwarder; +use crate::testkit::{FakeForwarder, Rig, TempDir}; +use std::fs; +use std::sync::Arc; +use tasks_core::Kind; + +fn git_ok(dir: &Path, args: &[&str]) { + let out = Command::new("git") + .arg("-C") + .arg(dir) + .args([ + "-c", + "user.name=t", + "-c", + "user.email=t@t", + "-c", + "commit.gpgsign=false", + ]) + .args(args) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); +} + +/// A repository on `dev` with one commit, canonical. +fn repo(at: &Path) -> PathBuf { + fs::create_dir_all(at).unwrap(); + let at = fs::canonicalize(at).unwrap(); + git_ok(&at, &["init", "-q", "-b", "dev"]); + fs::write(at.join("Cargo.toml"), "[workspace]\n").unwrap(); + git_ok(&at, &["add", "."]); + git_ok(&at, &["commit", "-q", "-m", "one"]); + at +} + +/// A worktree of `repo` at `name` on a new branch `name`, with one commit of its own when +/// `ahead`. +fn worktree(repo: &Path, name: &str, ahead: bool) -> PathBuf { + let wt = repo.parent().unwrap().join(name); + git_ok( + repo, + &["worktree", "add", "-q", "-b", name, wt.to_str().unwrap()], + ); + if ahead { + fs::write(wt.join("more.txt"), "x").unwrap(); + git_ok(&wt, &["add", "."]); + git_ok(&wt, &["commit", "-q", "-m", "more"]); + } + wt +} + +/// `bytes` of output in `dir/target`. +fn output(dir: &Path, bytes: usize) -> PathBuf { + fs::create_dir_all(dir.join("target")).unwrap(); + fs::write(dir.join("target/blob"), vec![0u8; bytes]).unwrap(); + dir.to_path_buf() +} + +#[test] +fn r839_a_branch_in_dev_with_no_changed_file_is_merged_and_the_base_or_a_newer_one_is_not() { + let t = TempDir::new("gc-merged"); + let main = repo(&t.path().join("main")); + let done = worktree(&main, "done", false); + let wip = worktree(&main, "wip", true); + let dirty = worktree(&main, "dirty", false); + fs::write(dirty.join("Cargo.toml"), "[workspace]\n# edit\n").unwrap(); + + assert_eq!(branch_of(&main), (Some("dev".into()), None)); + assert_eq!(branch_of(&done), (Some("done".into()), Some("dev".into()))); + assert_eq!(branch_of(&wip), (Some("wip".into()), None)); + assert_eq!(branch_of(&dirty), (Some("dirty".into()), None)); + let mut all = worktrees(&main); + all.sort(); + let mut want = vec![main, done, wip, dirty]; + want.sort(); + assert_eq!(all, want); +} + +#[test] +fn r839_output_belongs_to_the_checkout_its_links_point_from_also_in_the_older_layout() { + let t = TempDir::new("gc-links"); + let base = fs::canonicalize(t.path()).unwrap(); + let cache = base.join("cache"); + let new = cache.join(crate::build_cache::CHECKOUTS_SUBDIR); + let old = cache.join("p2p_chat/tasks/checkouts"); + fs::create_dir_all(&new).unwrap(); + fs::create_dir_all(&old).unwrap(); + // the current layout: found by its key alone + let a = base.join("a"); + fs::create_dir_all(a.join(".git")).unwrap(); + let a_out = output(&new.join(key_of(&a)), 10); + // an older layout under another key: found through the link + let b = base.join("b"); + fs::create_dir_all(b.join(".git")).unwrap(); + let b_out = output(&old.join("ab".repeat(32)), 10); + std::os::unix::fs::symlink(b_out.join("target"), b.join("target")).unwrap(); + + let parents = output_parents(&cache); + assert_eq!( + parents, + vec![new.clone(), old.clone()] + .into_iter() + .collect::>() + .into_iter() + .collect::>() + ); + assert_eq!(owned_dirs(&a, &cache, &parents), vec![a_out]); + assert_eq!(owned_dirs(&b, &cache, &parents), vec![b_out]); + // a link out of the cache claims nothing there + let c = base.join("c"); + fs::create_dir_all(c.join(".git")).unwrap(); + fs::create_dir_all(base.join("elsewhere")).unwrap(); + std::os::unix::fs::symlink(base.join("elsewhere"), c.join("target")).unwrap(); + assert!(owned_dirs(&c, &cache, &parents).is_empty()); +} + +#[test] +fn r839_a_busy_checkout_is_in_use_whatever_else_holds() { + let day = DAY_MS; + assert_eq!( + state_of(true, false, true, Some(0), 99 * day, 14), + GcState::InUse + ); + assert_eq!( + state_of(false, false, true, Some(0), 99 * day, 14), + GcState::Gone + ); + assert_eq!( + state_of(false, true, true, Some(0), 99 * day, 14), + GcState::Merged + ); + assert_eq!( + state_of(false, true, false, Some(0), 15 * day, 14), + GcState::Idle + ); + assert_eq!( + state_of(false, true, false, Some(0), 13 * day, 14), + GcState::Active + ); + assert_eq!( + state_of(false, true, false, None, 99 * day, 14), + GcState::Active + ); +} + +fn gc(r: &Rig, delete: Vec) -> GcView { + gc_named(r, delete, vec![]) +} + +fn gc_named(r: &Rig, delete: Vec, checkouts: Vec) -> GcView { + match r.sch.handle(Request::Gc { + delete, + checkouts, + idle_days: None, + }) { + Reply::Ok(Response::Gc(v)) => *v, + other => panic!("unexpected {other:?}"), + } +} + +fn row<'a>(v: &'a GcView, checkout: &str) -> &'a GcRow { + v.rows + .iter() + .find(|r| r.checkout.ends_with(checkout)) + .unwrap_or_else(|| panic!("no row {checkout} in {:?}", v.rows)) +} + +fn on(r: &GcRow, machine: &str) -> u64 { + r.held + .iter() + .filter(|h| h.machine == machine) + .map(|h| h.bytes) + .sum() +} + +#[test] +fn r839_gc_reports_every_machine_and_frees_only_the_chosen_states_never_a_busy_checkout() { + let fw = Arc::new(FakeForwarder::new(vec![crate::tests::peer_view()])); + let r = Rig::with_forwarder(fw.clone()); + let cache_root = r.cfg.cache_dir.clone(); + fs::create_dir_all(&cache_root).unwrap(); + r.sch.set_build_cache(BuildCache::discover( + &cache_root, + &r.cfg.state_dir.join("peer-store"), + &[], + )); + let cache = fs::canonicalize(&cache_root).unwrap(); + let new = cache.join(crate::build_cache::CHECKOUTS_SUBDIR); + let main = repo(&r.dir.path().join("work/main")); + let done = worktree(&main, "done", false); + let busy = worktree(&main, "busy", true); + let mine = worktree(&main, "mine", true); + let done_out = output(&new.join(key_of(&done)), 1000); + let busy_out = output(&new.join(key_of(&busy)), 2000); + let mine_out = output(&new.join(key_of(&mine)), 3000); + let gone_out = output(&new.join("a".repeat(64)), 4000); + let old_out = output( + &cache.join("p2p_chat/tasks/checkouts").join("b".repeat(64)), + 5000, + ); + // a queued job keeps `busy` in use; the repository's other worktrees come from Git + let mut spec = r.spec("o", &["cargo", "build"], Kind::Compile); + spec.cwd = busy.clone(); + r.submit(&spec); + + let id = |p: &Path| fw.checkout_id(p).unwrap(); + let held = |c: CheckoutId, busy: bool| HeldCheckout { + checkout: c, + copy_bytes: 100, + output_bytes: 50, + last_used_ms: Some(1), + busy, + }; + let stranger = CheckoutId::parse(&"c".repeat(32)).unwrap(); + fw.cache.lock().unwrap().insert( + "nixos".into(), + vec![ + held(id(&done), false), + held(id(&busy), true), + held(stranger.clone(), false), + ], + ); + + let v = gc(&r, vec![]); + assert!(!v.deleted); + assert_eq!(row(&v, "/done").state, GcState::Merged); + assert_eq!(row(&v, "/done").merged_into.as_deref(), Some("dev")); + assert!(on(row(&v, "/done"), "testbox") >= 1000); + assert_eq!(on(row(&v, "/done"), "nixos"), 150); + assert_eq!(row(&v, "/busy").state, GcState::InUse); + assert_eq!(row(&v, "/mine").state, GcState::Active); + assert_eq!( + row(&v, "(unknown checkout aaaaaaaaaaaa)").state, + GcState::Gone + ); + assert_eq!( + row(&v, "(unknown checkout bbbbbbbbbbbb)").state, + GcState::Old + ); + assert_eq!( + row(&v, "(unknown checkout cccccccccccc)").state, + GcState::Gone + ); + let nixos = v.machines.iter().find(|m| m.name == "nixos").unwrap(); + assert_eq!( + ( + nixos.held_bytes, + nixos.reclaimable_bytes, + nixos.error.clone() + ), + (450, 300, None) + ); + assert!( + v.machines + .iter() + .any(|m| m.name == "testbox" && m.reclaimable_bytes >= 10_000) + ); + + // asking to free in-use rows too changes nothing for them + let v = gc( + &r, + vec![GcState::Merged, GcState::Gone, GcState::Old, GcState::InUse], + ); + assert!(v.deleted); + assert!(!done_out.exists() && !gone_out.exists() && !old_out.exists()); + assert!(busy_out.exists() && mine_out.exists()); + assert!( + done.join("Cargo.toml").exists(), + "a working tree is never deleted" + ); + assert!(row(&v, "/done").freed_bytes >= 1150); + let dropped = fw.dropped.lock().unwrap().clone(); + assert_eq!(dropped.len(), 1); + let (peer, ids) = &dropped[0]; + assert_eq!(peer, "nixos"); + assert!(ids.contains(&id(&done)) && ids.contains(&stranger) && !ids.contains(&id(&busy))); +} + +#[test] +fn r839_a_peer_that_cannot_answer_is_named_and_the_rest_still_reports() { + let fw = Arc::new(FakeForwarder::new(vec![crate::tests::peer_view()])); + let r = Rig::with_forwarder(fw); + let v = gc(&r, vec![]); + let nixos = v.machines.iter().find(|m| m.name == "nixos").unwrap(); + assert_eq!(nixos.error.as_deref(), Some("nixos unreachable")); + assert!( + v.machines + .iter() + .any(|m| m.name == "testbox" && m.error.is_none()) + ); +} + +#[test] +fn r839_peer_outputs_are_listed_per_checkout_and_dropped_with_it() { + use crate::peer_link::serve::PeerOutputs; + use tasks_peer::{OutputKeeper, PeerId}; + let t = TempDir::new("gc-peer-outputs"); + let peer = PeerId::new("laptop-node").unwrap(); + let a = CheckoutId::parse(&"a".repeat(32)).unwrap(); + let b = CheckoutId::parse(&"b".repeat(32)).unwrap(); + let dir = t.path().join("peer"); + output(&dir.join(peer.as_str()).join(a.as_str()), 300); + output(&dir.join(peer.as_str()).join(b.as_str()), 700); + fs::create_dir_all(dir.join(peer.as_str()).join("not-a-checkout")).unwrap(); + let keep = PeerOutputs::at(dir.clone()); + let mut listed = keep.outputs(&peer); + listed.sort(); + assert_eq!(listed, vec![(a.clone(), 300), (b.clone(), 700)]); + assert_eq!(keep.drop_outputs(&peer, &a), 300); + assert!(!dir.join(peer.as_str()).join(a.as_str()).exists()); + assert_eq!(keep.outputs(&peer), vec![(b, 700)]); + assert!(keep.outputs(&PeerId::new("other-node").unwrap()).is_empty()); +} + +#[test] +fn r840_gc_frees_the_checkouts_named_whatever_their_state_but_never_one_a_job_uses() { + let r = Rig::with_forwarder(Arc::new(FakeForwarder::new(vec![]))); + let cache_root = r.cfg.cache_dir.clone(); + fs::create_dir_all(&cache_root).unwrap(); + r.sch.set_build_cache(BuildCache::discover( + &cache_root, + &r.cfg.state_dir.join("peer-store"), + &[], + )); + let new = fs::canonicalize(&cache_root) + .unwrap() + .join(crate::build_cache::CHECKOUTS_SUBDIR); + let main = repo(&r.dir.path().join("work/main")); + let done = worktree(&main, "done", false); + let busy = worktree(&main, "busy", true); + let mine = worktree(&main, "mine", true); + let done_out = output(&new.join(key_of(&done)), 1000); + let busy_out = output(&new.join(key_of(&busy)), 2000); + let mine_out = output(&new.join(key_of(&mine)), 3000); + let mut spec = r.spec("o", &["cargo", "build"], Kind::Compile); + spec.cwd = busy.clone(); + r.submit(&spec); + + let v = gc(&r, vec![]); + let name = |c: &str| row(&v, c).checkout.clone(); + let v = gc_named( + &r, + vec![], + vec![ + mine.display().to_string(), + name("/busy"), + "~/nowhere".into(), + ], + ); + assert!(v.deleted); + assert!(!mine_out.exists(), "an active checkout named is freed"); + assert!(done_out.exists(), "a merged one not named stays"); + assert!(busy_out.exists(), "one a job uses stays"); + assert!(row(&v, "/mine").freed_bytes >= 3000); + assert_eq!( + row(&v, "/busy").kept.as_deref(), + Some("a job uses it; free it once the job ends") + ); + assert_eq!( + v.notes, + vec!["~/nowhere holds no build output here; run loadtable gc to see what does".to_string()] + ); + assert!(mine.join("Cargo.toml").exists()); +} + +#[test] +fn r844_after_freeing_each_machines_total_is_what_it_keeps_now() { + let fw = Arc::new(FakeForwarder::new(vec![crate::tests::peer_view()])); + let r = Rig::with_forwarder(fw.clone()); + let cache_root = r.cfg.cache_dir.clone(); + fs::create_dir_all(&cache_root).unwrap(); + r.sch.set_build_cache(BuildCache::discover( + &cache_root, + &r.cfg.state_dir.join("peer-store"), + &[], + )); + let new = fs::canonicalize(&cache_root) + .unwrap() + .join(crate::build_cache::CHECKOUTS_SUBDIR); + let main = repo(&r.dir.path().join("work/main")); + let done = worktree(&main, "done", false); + let mine = worktree(&main, "mine", true); + output(&new.join(key_of(&done)), 1000); + output(&new.join(key_of(&mine)), 3000); + // a queued job makes the repository known; it keeps `busy` in use + let busy = worktree(&main, "busy", true); + let mut spec = r.spec("o", &["cargo", "build"], Kind::Compile); + spec.cwd = busy; + r.submit(&spec); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as i64; + let held = |c: CheckoutId, bytes: u64| HeldCheckout { + checkout: c, + copy_bytes: bytes, + output_bytes: 0, + last_used_ms: Some(now), + busy: false, + }; + fw.cache.lock().unwrap().insert( + "nixos".into(), + vec![ + held(fw.checkout_id(&done).unwrap(), 150), + held(fw.checkout_id(&mine).unwrap(), 400), + ], + ); + let machine = |v: &GcView, n: &str| { + let m = v.machines.iter().find(|m| m.name == n).unwrap(); + (m.held_bytes, m.reclaimable_bytes) + }; + + let before = gc(&r, vec![]); + let (here, _) = machine(&before, "testbox"); + assert_eq!(machine(&before, "nixos"), (550, 150)); + + let after = gc(&r, vec![GcState::Merged]); + let freed_here = on(row(&before, "/done"), "testbox"); + assert!(freed_here >= 1000); + assert_eq!(machine(&after, "testbox"), (here - freed_here, 0)); + assert_eq!( + machine(&after, "nixos"), + (400, 0), + "a peer's total drops too" + ); + assert_eq!(row(&after, "/done").freed_bytes, freed_here + 150); +} diff --git a/crates/tasks-daemon/src/handler.rs b/crates/tasks-daemon/src/handler.rs index c765e49..6402e01 100644 --- a/crates/tasks-daemon/src/handler.rs +++ b/crates/tasks-daemon/src/handler.rs @@ -226,6 +226,13 @@ impl Scheduler { reason, } => self.prioritize(id, &who, priority, reason.as_deref()), Request::Status => Ok(Response::Status(Box::new(self.status()))), + Request::Gc { + delete, + checkouts, + idle_days, + } => Ok(Response::Gc(Box::new( + self.gc(&delete, &checkouts, idle_days)?, + ))), Request::JobMetrics { id } => { let rec = self.record(id)?; Ok(Response::JobMetrics(self.api_job_metrics(&rec))) diff --git a/crates/tasks-daemon/src/lib.rs b/crates/tasks-daemon/src/lib.rs index 4c60b62..e7e7c9c 100644 --- a/crates/tasks-daemon/src/lib.rs +++ b/crates/tasks-daemon/src/lib.rs @@ -29,6 +29,7 @@ mod fetch_tests; mod follow_tests; mod forwarded; pub mod forwarder; +mod gc; mod gradle_warmth; mod handler; mod history; diff --git a/crates/tasks-daemon/src/peer_link/cache.rs b/crates/tasks-daemon/src/peer_link/cache.rs new file mode 100644 index 0000000..e4ef053 --- /dev/null +++ b/crates/tasks-daemon/src/peer_link/cache.rs @@ -0,0 +1,97 @@ +//! What a peer keeps for this machine's checkouts, and asking it to drop some (R-839). +//! +//! One request per peer on a thread of its own, so `loadtable gc` waits at most +//! [`CACHE_TIMEOUT`] for a peer measuring tens of gigabytes. An older peer refuses the message +//! as unknown: that is said in words, never a failed report. + +use super::PeerLink; +use crate::forwarder::ForwardError; +use std::sync::{Arc, mpsc}; +use std::time::Duration; +use tasks_core::MachineName; +use tasks_peer::proto::{Body, DroppedCheckout, HeldCheckout}; +use tasks_peer::transport::{PeerError, Transport}; +use tasks_peer::{CheckoutId, Refusal}; + +/// How long a peer may take to measure or delete what it keeps. +pub const CACHE_TIMEOUT: Duration = Duration::from_secs(40); + +fn explain(peer: &str, e: &PeerError) -> String { + match e { + PeerError::Refused(Refusal::UnknownMessage { .. } | Refusal::Malformed { .. }) => { + format!("{peer} runs an older loadtabled that cannot report its cache; upgrade it") + } + PeerError::Offline { .. } | PeerError::Timeout(_) | PeerError::LinkLost(_) => { + format!("{peer} unreachable") + } + other => format!("{peer} could not answer: {other}"), + } +} + +impl PeerLink { + /// Sends `msg` to `peer` and waits at most [`CACHE_TIMEOUT`] for the reply. + fn ask_cache(&self, peer: &MachineName, msg: Body) -> Result { + let Some(spec) = self.spec_of(&peer.0) else { + return Err(ForwardError::Unavailable(format!( + "{} is not a configured peer", + peer.0 + ))); + }; + if self.link_is_down(&peer.0) { + return Err(ForwardError::Unavailable(format!("{} unreachable", peer.0))); + } + let (tx, rx) = mpsc::channel(); + let inner = Arc::clone(&self.inner); + std::thread::Builder::new() + .name("peer-cache".into()) + .spawn(move || { + let r = inner + .transport + .connect(&spec.node_id, inner.opts.metrics_timeout.max(CONNECT)) + .and_then(|mut c| c.request(msg)); + let _ = tx.send(r); + }) + .map_err(|e| ForwardError::Unavailable(format!("{} not asked: {e}", peer.0)))?; + match rx.recv_timeout(CACHE_TIMEOUT) { + Ok(Ok(b)) => Ok(b), + Ok(Err(e)) => Err(ForwardError::Unavailable(explain(&peer.0, &e))), + Err(_) => Err(ForwardError::Unavailable(format!( + "{} did not answer within {}s", + peer.0, + CACHE_TIMEOUT.as_secs() + ))), + } + } + + pub(super) fn peer_cache_of( + &self, + peer: &MachineName, + ) -> Result, ForwardError> { + match self.ask_cache(peer, Body::CacheRequest)? { + Body::CacheHeld { checkouts } => Ok(checkouts), + other => Err(ForwardError::Unavailable(format!( + "{} answered a cache request with {}", + peer.0, + tasks_peer::fake::kind(&other) + ))), + } + } + + pub(super) fn peer_cache_drop_of( + &self, + peer: &MachineName, + checkouts: Vec, + ) -> Result, ForwardError> { + match self.ask_cache(peer, Body::CacheDrop { checkouts })? { + Body::CacheDropped { checkouts } => Ok(checkouts), + other => Err(ForwardError::Unavailable(format!( + "{} answered a cache drop with {}", + peer.0, + tasks_peer::fake::kind(&other) + ))), + } + } +} + +/// How long dialling a peer may take. +const CONNECT: Duration = Duration::from_secs(10); diff --git a/crates/tasks-daemon/src/peer_link/mod.rs b/crates/tasks-daemon/src/peer_link/mod.rs index face0a5..81abbf9 100644 --- a/crates/tasks-daemon/src/peer_link/mod.rs +++ b/crates/tasks-daemon/src/peer_link/mod.rs @@ -15,6 +15,7 @@ //! //! The receiving half (a daemon hosting `tasks_peer::Receiver`) is in [`serve`]. +pub mod cache; mod drive; pub mod fetch; pub mod metrics; @@ -909,6 +910,25 @@ impl Forwarder for PeerLink { self.remote_metrics_of(peer, remote_job) } + fn peer_cache( + &self, + peer: &MachineName, + ) -> Result, ForwardError> { + self.peer_cache_of(peer) + } + + fn peer_cache_drop( + &self, + peer: &MachineName, + checkouts: Vec, + ) -> Result, ForwardError> { + self.peer_cache_drop_of(peer, checkouts) + } + + fn checkout_id(&self, cwd: &Path) -> Option { + Some(checkout_of(&self.inner.opts.local_name, cwd)) + } + fn forget(&self, job: JobId) { lock(&self.inner.jobs).remove(&job); } diff --git a/crates/tasks-daemon/src/peer_link/serve.rs b/crates/tasks-daemon/src/peer_link/serve.rs index a27f98b..21020c0 100644 --- a/crates/tasks-daemon/src/peer_link/serve.rs +++ b/crates/tasks-daemon/src/peer_link/serve.rs @@ -21,7 +21,7 @@ use tasks_peer::receiver::{ Allowlist, FinishedJob, JobRunner, JobView, Receiver, StatusSource, SubmitPolicy, }; use tasks_peer::sync::{GixTransfer, SyncLimits, SyncStore}; -use tasks_peer::{PeerId, RemoteJobId, SubmitKey}; +use tasks_peer::{OutputKeeper, PeerId, RemoteJobId, SubmitKey}; use tasks_platform::CgroupName; use tasks_timing::{Cache, History, Outcome as TimingOutcome, RecipeKey, Sample, SampleKey}; @@ -267,17 +267,75 @@ pub fn sweep_hook( store: Arc, ) -> crate::trust_runtime::MaintenanceHook { let runner = DaemonRunner::new(sch.clone()); + let outputs = PeerOutputs::of(sch); Box::new(move |policy| { let busy = |peer: &PeerId, tree: &Path| runner.checkout_busy(peer, tree); - store.sweep(&policy.sync_retention(), &busy); + // a deleted copy's build output goes with it (R-839) + for e in store.sweep(&policy.sync_retention(), &busy) { + if let tasks_peer::sync::SyncEvent::Deleted { + kind: tasks_peer::sync::Deleted::WorkingCopy, + peer: Some(peer), + target, + .. + } = e + && let Ok(checkout) = tasks_peer::CheckoutId::parse(&target) + { + outputs.drop_outputs(&peer, &checkout); + } + } }) } +/// The build output of peers' checkouts, kept outside their synced copies under +/// `/loadtable/checkouts/peer//` (R-839). +pub struct PeerOutputs { + dir: std::path::PathBuf, +} + +impl PeerOutputs { + /// The peers' output directory of `sch`'s build cache. + pub fn of(sch: &Arc) -> PeerOutputs { + PeerOutputs::at(sch.build_cache().checkouts_dir().join("peer")) + } + + /// The peers' output directory `dir`. + pub fn at(dir: std::path::PathBuf) -> PeerOutputs { + PeerOutputs { dir } + } +} + +impl tasks_peer::OutputKeeper for PeerOutputs { + fn outputs(&self, peer: &PeerId) -> Vec<(tasks_peer::CheckoutId, u64)> { + let Ok(rd) = std::fs::read_dir(self.dir.join(peer.as_str())) else { + return Vec::new(); + }; + rd.flatten() + .filter(|e| e.file_type().is_ok_and(|t| t.is_dir())) + .filter_map(|e| tasks_peer::CheckoutId::parse(&e.file_name().to_string_lossy()).ok()) + .map(|c| { + let bytes = + tasks_peer::sync::dir_size(&self.dir.join(peer.as_str()).join(c.as_str())); + (c, bytes) + }) + .collect() + } + + fn drop_outputs(&self, peer: &PeerId, checkout: &tasks_peer::CheckoutId) -> u64 { + let d = self.dir.join(peer.as_str()).join(checkout.as_str()); + let bytes = tasks_peer::sync::dir_size(&d); + if std::fs::remove_dir_all(&d).is_ok() { + bytes + } else { + 0 + } + } +} + /// The receiver this daemon hosts: jobs into the ledger, synced copies in `store`. Who may /// call it is decided in front of it by the trust gate (`tasks_peer::trust::TrustedHandler`), on /// grants; the receiver itself admits every caller it is handed. pub fn build_receiver_with(sch: &Arc, store: Arc) -> Arc { - Arc::new(Receiver::new( + let r = Arc::new(Receiver::new( &sch.cfg.machine, Allowlist::open(), store, @@ -286,7 +344,9 @@ pub fn build_receiver_with(sch: &Arc, store: Arc) -> Arc Vec { + let mut v: Vec = dirs.iter().map(|d| d.to_string()).collect(); + for d in crate::build_cache::fallback_tool_dirs(&[]) { + if !v.contains(&d) { + v.push(d); + } + } + v + } + // R-732: a daemon under systemd has no ~/.cargo/bin on its PATH, yet jobs get it #[test] fn r732_the_tool_check_looks_in_the_fallback_dirs_every_job_gets() { let daemon = env(&[("PATH", "/usr/bin:/bin"), ("HOME", "/home/u")]); let dirs = job_tool_dirs(&[], &daemon); - assert_eq!(dirs, ["/usr/bin", "/bin", "/home/u/.cargo/bin"]); + assert_eq!(dirs, plus_mac(&["/usr/bin", "/bin", "/home/u/.cargo/bin"])); let has = |p: &Path| p == Path::new("/home/u/.cargo/bin/cargo"); assert!(tool_in_dirs("cargo", &dirs, has)); assert!(!tool_in_dirs("gradle", &dirs, has)); - // without a HOME there is nothing to add + // without a HOME there is nothing to add but Homebrew's on macOS assert_eq!( job_tool_dirs(&[], &env(&[("PATH", "/usr/bin")])), - ["/usr/bin"] + plus_mac(&["/usr/bin"]) ); } @@ -748,10 +759,13 @@ mod tests { let daemon = env(&[("PATH", "/usr/bin"), ("HOME", "/h")]); let spec = env(&[("PATH", "/opt/x/bin::/usr/local/bin")]); let dirs = job_tool_dirs(&spec, &daemon); - assert_eq!(dirs, ["/opt/x/bin", "/usr/local/bin", "/h/.cargo/bin"]); + assert_eq!( + dirs, + plus_mac(&["/opt/x/bin", "/usr/local/bin", "/h/.cargo/bin"]) + ); // a fallback dir already on the path is not repeated let spec = env(&[("PATH", "/h/.cargo/bin")]); - assert_eq!(job_tool_dirs(&spec, &daemon), ["/h/.cargo/bin"]); + assert_eq!(job_tool_dirs(&spec, &daemon), plus_mac(&["/h/.cargo/bin"])); assert!(!tool_in_dirs("cargo", &[], |_| true)); } diff --git a/crates/tasks-daemon/src/testkit.rs b/crates/tasks-daemon/src/testkit.rs index 1c31302..ac427fa 100644 --- a/crates/tasks-daemon/src/testkit.rs +++ b/crates/tasks-daemon/src/testkit.rs @@ -347,6 +347,10 @@ pub struct FakeForwarder { /// What `remote_metrics` answers, and what it was asked. pub metrics: Mutex>, pub metrics_asked: Mutex>, + /// What each peer reports it keeps for `gc`, by peer name; a missing peer cannot be reached. + pub cache: Mutex>>, + /// The drops `gc` asked for, per peer. + pub dropped: Mutex)>>, } impl FakeForwarder { @@ -368,6 +372,8 @@ impl FakeForwarder { classes: Mutex::new(Vec::new()), metrics: Mutex::new(None), metrics_asked: Mutex::new(Vec::new()), + cache: Mutex::new(BTreeMap::new()), + dropped: Mutex::new(Vec::new()), } } pub fn script(&self, id: JobId, p: ForwardPoll) { @@ -454,6 +460,44 @@ impl Forwarder for FakeForwarder { .clone() .ok_or_else(|| ForwardError::Unavailable("no metrics scripted".into())) } + fn peer_cache( + &self, + peer: &MachineName, + ) -> Result, ForwardError> { + lock(&self.cache) + .get(&peer.0) + .cloned() + .ok_or_else(|| ForwardError::Unavailable(format!("{} unreachable", peer.0))) + } + fn peer_cache_drop( + &self, + peer: &MachineName, + checkouts: Vec, + ) -> Result, ForwardError> { + lock(&self.dropped).push((peer.0.clone(), checkouts.clone())); + let mut cache = lock(&self.cache); + let held = cache.entry(peer.0.clone()).or_default(); + let out = checkouts + .iter() + .map(|c| tasks_peer::proto::DroppedCheckout { + checkout: c.clone(), + freed_bytes: held + .iter() + .filter(|h| &h.checkout == c) + .map(|h| h.copy_bytes + h.output_bytes) + .sum(), + kept: None, + }) + .collect(); + held.retain(|h| !checkouts.contains(&h.checkout)); + Ok(out) + } + fn checkout_id(&self, cwd: &std::path::Path) -> Option { + Some(tasks_peer::CheckoutId::derive( + "testbox", + std::path::Path::new(&crate::place::checkout_key(cwd)), + )) + } fn forget(&self, job: JobId) { lock(&self.registered).remove(&job); } diff --git a/crates/tasks-mcp/src/defs.rs b/crates/tasks-mcp/src/defs.rs index f182c5d..4ee4f31 100644 --- a/crates/tasks-mcp/src/defs.rs +++ b/crates/tasks-mcp/src/defs.rs @@ -3,7 +3,7 @@ use serde_json::{Value, json}; /// The tool names, in the order `tools/list` returns them. -pub const TOOL_NAMES: [&str; 12] = [ +pub const TOOL_NAMES: [&str; 13] = [ "submit", "submit_steps", "run_suite", @@ -16,6 +16,7 @@ pub const TOOL_NAMES: [&str; 12] = [ "status", "estimate", "audit", + "gc", ]; fn object(properties: Value, required: &[&str]) -> Value { @@ -244,6 +245,17 @@ pub fn tool_defs() -> Vec { &[], ), }, + ToolDef { + name: "gc", + description: "What each checkout's build output takes on this machine and its peers, and freeing it: a checkout is in_use (a job queued or running), active, idle (no job in idle_days), merged (its branch is in dev/main/master, nothing tracked changed), gone (its working tree is deleted) or old (an older layout's output). Without delete it only reports. delete frees only build output and peers' synced copies, never a working tree and never an in_use checkout; the next build there starts cold.", + input_schema: object( + json!({ + "delete": {"type": "array", "items": {"type": "string", "enum": ["merged", "gone", "old", "idle", "active"]}, "description": "Free the checkouts in these states. Usually [\"merged\", \"gone\", \"old\"]."}, + "idle_days": {"type": "integer", "minimum": 1, "description": "Days without a job after which a checkout is idle (default 14)."} + }), + &[], + ), + }, ] } diff --git a/crates/tasks-mcp/src/tools.rs b/crates/tasks-mcp/src/tools.rs index dd08993..ac9f9f3 100644 --- a/crates/tasks-mcp/src/tools.rs +++ b/crates/tasks-mcp/src/tools.rs @@ -15,8 +15,8 @@ use std::time::{Duration, Instant}; use tasks_core::plan::{Plan, Step}; use tasks_core::{DeviceId, JobId, JobSpec, Kind, MachineName, Owner, Placement}; use tasks_daemon::api::{ - ErrorCode, JobView, LogView, MAX_CONTEXT, MAX_LOG_CHUNK, MAX_TAIL_LINES, Request, Response, - StatusSnapshot, + ErrorCode, GcState, JobView, LogView, MAX_CONTEXT, MAX_LOG_CHUNK, MAX_TAIL_LINES, Request, + Response, StatusSnapshot, }; use tasks_timing::RecipeKey; @@ -335,6 +335,14 @@ struct EstimateArgs { argv: Vec, } +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct GcArgs { + #[serde(default)] + delete: Vec, + idle_days: Option, +} + #[derive(Deserialize)] #[serde(deny_unknown_fields)] struct AuditArgs { @@ -445,6 +453,7 @@ impl Tools { "status" => parse::(args).and_then(|_| self.status()), "estimate" => parse(args).and_then(|a| self.estimate(a)), "audit" => parse(args).and_then(|a| self.audit(a)), + "gc" => parse(args).and_then(|a| self.gc(a)), _ => return None, }; Some(match r { @@ -1222,6 +1231,74 @@ impl Tools { }) } + /// `gc` (R-839): per machine what checkouts keep and what can go; with `delete`, frees it. + fn gc(&self, a: GcArgs) -> ToolOk { + let mut delete = Vec::new(); + for word in &a.delete { + match GcState::parse(word) { + Some(GcState::InUse) | None => { + return Err(ToolError::Args(format!( + "delete takes merged, gone, old, idle or active, not {word:?}; a checkout a job uses is never deleted" + ))); + } + Some(s) => delete.push(s), + } + } + let resp = self.call_daemon(&Request::Gc { + delete, + checkouts: Vec::new(), + idle_days: a.idle_days, + })?; + let Response::Gc(v) = resp else { + return Err(unexpected("gc", &resp)); + }; + let gib = |b: u64| format!("{:.1} GiB", b as f64 / (1u64 << 30) as f64); + let mut text = String::new(); + for m in &v.machines { + match &m.error { + Some(e) => text.push_str(&format!("{}: could not report: {e}\n", m.name)), + None => text.push_str(&format!( + "{}: {} kept, {} can be freed\n", + m.name, + gib(m.held_bytes), + gib(m.reclaimable_bytes) + )), + } + } + for r in &v.rows { + let held: Vec = r + .held + .iter() + .map(|h| format!("{} {}", h.machine, gib(h.bytes))) + .collect(); + text.push_str(&format!( + "{} {} ({}): {}", + r.state.name(), + r.checkout, + r.branch.as_deref().unwrap_or("no branch"), + held.join(", ") + )); + if r.freed_bytes > 0 { + text.push_str(&format!(", freed {}", gib(r.freed_bytes))); + } + if let Some(k) = &r.kept { + text.push_str(&format!(", kept: {k}")); + } + text.push('\n'); + } + if v.deleted { + let freed: u64 = v.rows.iter().map(|r| r.freed_bytes).sum(); + text.push_str(&format!("freed {}\n", gib(freed))); + } else if v.machines.iter().any(|m| m.reclaimable_bytes > 0) { + text.push_str("free it with delete: [\"merged\", \"gone\", \"old\"]\n"); + } + Ok(ToolResult { + text, + structured: serde_json::to_value(&*v).unwrap_or(Value::Null), + is_error: false, + }) + } + fn estimate(&self, a: EstimateArgs) -> ToolOk { if a.argv.is_empty() { return Err(ToolError::Args("argv must name a program".into())); diff --git a/crates/tasks-mcp/tests/session.rs b/crates/tasks-mcp/tests/session.rs index d5d4164..5da9c19 100644 --- a/crates/tasks-mcp/tests/session.rs +++ b/crates/tasks-mcp/tests/session.rs @@ -75,7 +75,8 @@ fn r195_tools_list_matches_the_schemas() { "prioritize", "status", "estimate", - "audit" + "audit", + "gc" ] ); assert_eq!(names, TOOL_NAMES); diff --git a/crates/tasks-mcp/tests/tools.rs b/crates/tasks-mcp/tests/tools.rs index 3be5711..848b276 100644 --- a/crates/tasks-mcp/tests/tools.rs +++ b/crates/tasks-mcp/tests/tools.rs @@ -1664,3 +1664,64 @@ fn r796_logs_and_get_carry_their_text_and_requested_fetches_in_the_structured_re r.text ); } + +// R-839 +#[test] +fn r839_gc_reports_each_machine_and_frees_only_what_was_asked() { + use tasks_daemon::api::{GcHeld, GcMachine, GcRow, GcState, GcView}; + let d = fake(|r| match r { + Request::Gc { delete, .. } => Ok(Response::Gc(Box::new(GcView { + idle_days: 14, + machines: vec![GcMachine { + name: "laptop".into(), + held_bytes: 3 << 30, + reclaimable_bytes: if delete.is_empty() { 2 << 30 } else { 0 }, + error: None, + }], + rows: vec![GcRow { + checkout: "~/w/done".into(), + branch: Some("done".into()), + state: GcState::Merged, + merged_into: Some("dev".into()), + last_used_ms: None, + held: vec![GcHeld { + machine: "laptop".into(), + bytes: 2 << 30, + }], + freed_bytes: if delete.is_empty() { 0 } else { 2 << 30 }, + kept: None, + }], + shared: vec![], + notes: vec![], + deleted: !delete.is_empty(), + }))), + _ => Err(DaemonError::Failed("unexpected".into())), + }); + let r = call(&tools_with(&d), "gc", json!({})); + assert!(!r.is_error, "{}", r.text); + assert!( + r.text + .contains("laptop: 3.0 GiB kept, 2.0 GiB can be freed"), + "{}", + r.text + ); + assert!( + r.text.contains("merged ~/w/done (done): laptop 2.0 GiB"), + "{}", + r.text + ); + assert!(r.text.contains("free it with delete"), "{}", r.text); + let r = call(&tools_with(&d), "gc", json!({"delete": ["merged", "gone"]})); + assert!(r.text.contains("freed 2.0 GiB"), "{}", r.text); + assert!(matches!( + d.requests().last(), + Some(Request::Gc { delete, .. }) if *delete == vec![GcState::Merged, GcState::Gone] + )); + let before = d.requests().len(); + assert!(call(&tools_with(&d), "gc", json!({"delete": ["in_use"]})).is_error); + assert_eq!( + d.requests().len(), + before, + "refused before asking the daemon" + ); +} diff --git a/crates/tasks-peer/src/fake.rs b/crates/tasks-peer/src/fake.rs index 5c94bd1..2d1178f 100644 --- a/crates/tasks-peer/src/fake.rs +++ b/crates/tasks-peer/src/fake.rs @@ -56,6 +56,8 @@ pub fn kind(b: &Body) -> &'static str { Body::FetchList { .. } => "fetch_list", Body::FetchChunk { .. } => "fetch_chunk", Body::JobMetricsRequest { .. } => "job_metrics", + Body::CacheRequest => "cache_request", + Body::CacheDrop { .. } => "cache_drop", _ => "other", } } diff --git a/crates/tasks-peer/src/lib.rs b/crates/tasks-peer/src/lib.rs index 140d982..d4a60b3 100644 --- a/crates/tasks-peer/src/lib.rs +++ b/crates/tasks-peer/src/lib.rs @@ -27,5 +27,5 @@ pub use forward::{ pub use ids::{CheckoutId, GitId, Hash, IdError, PeerId, RemoteJobId, SubmitKey}; pub use locks::{Acquire, CheckoutLocks, CopyTree, LockMode, LockOwner}; pub use proto::{Body, JobEvent, Refusal}; -pub use receiver::{Allowlist, JobRunner, Receiver, StatusSource, SubmitPolicy}; +pub use receiver::{Allowlist, JobRunner, OutputKeeper, Receiver, StatusSource, SubmitPolicy}; pub use transport::{Connection, EventStream, Handler, PeerError, StreamPoll, Transport}; diff --git a/crates/tasks-peer/src/proto.rs b/crates/tasks-peer/src/proto.rs index 8c797dd..54a4da6 100644 --- a/crates/tasks-peer/src/proto.rs +++ b/crates/tasks-peer/src/proto.rs @@ -183,6 +183,8 @@ impl Capability { pub const FETCH: &'static str = "fetch"; /// Log resume from an offset. pub const RESUME: &'static str = "resume"; + /// Reports and drops what it keeps for the asker's checkouts (R-839). + pub const CACHE: &'static str = "cache"; /// The capabilities of this build. pub fn all() -> BTreeSet { [ @@ -191,6 +193,7 @@ impl Capability { Self::SYNC, Self::FETCH, Self::RESUME, + Self::CACHE, ] .iter() .map(|s| Capability((*s).to_string())) @@ -678,6 +681,35 @@ pub enum FetchEntry { }, } +/// A checkout of the asker that the receiver keeps something of (R-839). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct HeldCheckout { + /// The checkout. + pub checkout: CheckoutId, + /// Bytes of the synced working copy; 0 once retention deleted it. + pub copy_bytes: u64, + /// Bytes of its build output in the receiver's cache. + pub output_bytes: u64, + /// When a job last used the copy, ms since the epoch; none without a copy. + #[serde(default)] + pub last_used_ms: Option, + /// A job of the asker runs or waits in it: a drop keeps it. + #[serde(default)] + pub busy: bool, +} + +/// The result of dropping one checkout. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DroppedCheckout { + /// The checkout. + pub checkout: CheckoutId, + /// Bytes freed. + pub freed_bytes: u64, + /// Why it was kept, when it was. + #[serde(default)] + pub kept: Option, +} + /// Every message. Requests and replies share the enum; a reply to any request /// may be [`Body::Refuse`]. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -832,6 +864,24 @@ pub enum Body { }, /// The metrics of the job. JobMetrics(tasks_core::metrics::JobMetricsView), + /// What do you keep for my checkouts? Answered with [`Body::CacheHeld`]; an older peer + /// refuses it as an unknown message (R-839). + CacheRequest, + /// The checkouts the receiver keeps for the asker. + CacheHeld { + /// One per checkout with a copy or build output here. + checkouts: Vec, + }, + /// Delete the synced copy and build output of these checkouts of mine (R-839). + CacheDrop { + /// The checkouts. + checkouts: Vec, + }, + /// What a drop did, one entry per checkout asked. + CacheDropped { + /// The results. + checkouts: Vec, + }, /// A named refusal. Refuse(Refusal), } diff --git a/crates/tasks-peer/src/receiver.rs b/crates/tasks-peer/src/receiver.rs index 63cde9f..ef30045 100644 --- a/crates/tasks-peer/src/receiver.rs +++ b/crates/tasks-peer/src/receiver.rs @@ -4,11 +4,11 @@ //! The daemon wires the traits below to its ledger, admission and runner; this //! module owns the protocol rules and nothing else. -use crate::ids::{PeerId, RemoteJobId, SubmitKey}; +use crate::ids::{CheckoutId, PeerId, RemoteJobId, SubmitKey}; use crate::outputs::{self, BlobShelf, FetchPolicy}; use crate::proto::{ - Body, CancelState, Hello, JobEvent, MAX_CHUNK_BYTES, PeerStatus, Refusal, SubmitJob, Submitted, - SyncApply, WireEnd, WireOutcome, WireSample, + Body, CancelState, DroppedCheckout, HeldCheckout, Hello, JobEvent, MAX_CHUNK_BYTES, PeerStatus, + Refusal, SubmitJob, Submitted, SyncApply, WireEnd, WireOutcome, WireSample, }; use crate::sync::{SyncStore, check_rel_path}; use crate::transport::{EventStream, Handler, PeerError, StreamPoll}; @@ -103,6 +103,16 @@ pub trait JobRunner: Send + Sync { } } +/// The build output this machine keeps for peers' checkouts, outside their synced copies +/// (R-839). +pub trait OutputKeeper: Send + Sync { + /// Bytes of build output kept for each checkout of `peer`, including checkouts whose + /// copy retention already deleted. + fn outputs(&self, peer: &PeerId) -> Vec<(CheckoutId, u64)>; + /// Deletes the build output of `checkout` of `peer`; returns the bytes freed. + fn drop_outputs(&self, peer: &PeerId, checkout: &CheckoutId) -> u64; +} + /// The daemon's status, for the status request. pub trait StatusSource: Send + Sync { /// Current status. @@ -141,6 +151,8 @@ pub struct Receiver { fetch: FetchPolicy, /// Where listed outputs are also offered as blobs, once the transport has one (R-795). blobs: OnceLock>, + /// The build output kept outside the copies, when the daemon has a cache (R-839). + outputs: OnceLock>, } impl Receiver { @@ -163,7 +175,78 @@ impl Receiver { policy, fetch, blobs: OnceLock::new(), + outputs: OnceLock::new(), + } + } + + /// Reports and drops the build output `keeper` holds along with the copies. Set once. + pub fn set_outputs(&self, keeper: Arc) { + let _ = self.outputs.set(keeper); + } + + /// What this machine keeps for `from`: each checkout's copy and build output (R-839). + fn held(&self, from: &PeerId) -> Vec { + let mut by: std::collections::BTreeMap = Default::default(); + for c in self.store.held(from) { + by.insert( + c.checkout.to_string(), + HeldCheckout { + busy: self + .runner + .checkout_busy(from, &self.store.tree_root(from, &c.checkout)), + checkout: c.checkout, + copy_bytes: c.bytes, + output_bytes: 0, + last_used_ms: Some(c.last_used_ms), + }, + ); + } + if let Some(k) = self.outputs.get() { + for (checkout, bytes) in k.outputs(from) { + by.entry(checkout.to_string()) + .or_insert_with(|| HeldCheckout { + busy: self + .runner + .checkout_busy(from, &self.store.tree_root(from, &checkout)), + checkout, + copy_bytes: 0, + output_bytes: 0, + last_used_ms: None, + }) + .output_bytes = bytes; + } } + by.into_values().collect() + } + + /// Deletes the copies and build output of `checkouts` of `from`, keeping any a job of + /// `from` runs or waits in (R-839). + fn drop_checkouts(&self, from: &PeerId, checkouts: Vec) -> Vec { + checkouts + .into_iter() + .map(|checkout| { + if self + .runner + .checkout_busy(from, &self.store.tree_root(from, &checkout)) + { + return DroppedCheckout { + checkout, + freed_bytes: 0, + kept: Some("a job runs or waits in it".to_string()), + }; + } + let copy = self.store.drop_copy(from, &checkout).unwrap_or(0); + let out = self + .outputs + .get() + .map_or(0, |k| k.drop_outputs(from, &checkout)); + DroppedCheckout { + checkout, + freed_bytes: copy + out, + kept: None, + } + }) + .collect() } /// Offers every file a fetch listing finds on `shelf` too. Set once, when serving starts. @@ -250,6 +333,12 @@ impl Receiver { Some(state) => Body::CancelAck { remote, state }, None => Body::Refuse(Refusal::UnknownJob { remote }), }, + Body::CacheRequest => Body::CacheHeld { + checkouts: self.held(from), + }, + Body::CacheDrop { checkouts } => Body::CacheDropped { + checkouts: self.drop_checkouts(from, checkouts), + }, Body::JobMetricsRequest { remote } => match self.runner.metrics(from, remote) { Some(v) => Body::JobMetrics(crate::proto::fit_metrics(v)), None => Body::Refuse(Refusal::UnknownJob { remote }), diff --git a/crates/tasks-peer/src/sync/mod.rs b/crates/tasks-peer/src/sync/mod.rs index 606adf4..c2e142f 100644 --- a/crates/tasks-peer/src/sync/mod.rs +++ b/crates/tasks-peer/src/sync/mod.rs @@ -34,7 +34,7 @@ pub use limits::SyncLimits; pub use origin::{ExtrasProvider, NoExtras, OriginSync, SyncDriver, SyncJob, SyncReport}; pub use paths::{check_rel_path, dir_size, hash_file, join}; pub use refuse::{Refusals, RefuseReason, Refused}; -pub use retention::{OriginRetention, WorkerRetention}; +pub use retention::{HeldCopy, OriginRetention, WorkerRetention}; pub use secrets::{secret_content, secret_name}; pub use snapshot::{ PruneStats, Snapshot, SnapshotError, SnapshotIo, SnapshotRequest, TreeSnapshot, diff --git a/crates/tasks-peer/src/sync/retention.rs b/crates/tasks-peer/src/sync/retention.rs index 5a38dc4..95c82f6 100644 --- a/crates/tasks-peer/src/sync/retention.rs +++ b/crates/tasks-peer/src/sync/retention.rs @@ -75,7 +75,87 @@ fn names(dir: &Path) -> Vec { .unwrap_or_default() } +/// A working copy this machine holds for a peer (R-839). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HeldCopy { + /// The checkout. + pub checkout: CheckoutId, + /// Bytes of the copy (its output links are not followed). + pub bytes: u64, + /// When a job last used it, ms since the epoch; the directory's age for a copy never + /// applied. + pub last_used_ms: i64, +} + impl SyncStore { + /// The working copies held for `peer`. + pub fn held(&self, peer: &PeerId) -> Vec { + names(&self.peer_dir(peer)) + .into_iter() + .filter_map(|c| CheckoutId::parse(&c).ok()) + .map(|checkout| { + let dir = self.checkout_dir(peer, &checkout); + let last_used_ms = self.read_state(&dir).map_or_else( + || { + fs::metadata(&dir) + .and_then(|m| m.modified()) + .ok() + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + .map_or(0, |d| i64::try_from(d.as_millis()).unwrap_or(0)) + }, + |st| st.last_used_ms, + ); + HeldCopy { + bytes: dir_size(&dir), + checkout, + last_used_ms, + } + }) + .collect() + } + + /// Deletes the working copy of `checkout` held for `peer`, as its owner asked (R-839), and + /// reports it to the observer; returns the bytes freed. The caller checks that no job uses + /// it. `None` when there was no copy or it could not be removed. + pub fn drop_copy(&self, peer: &PeerId, checkout: &CheckoutId) -> Option { + let e = self.remove_copy(peer, checkout)?; + self.observer.event(&e); + match e { + SyncEvent::Deleted { bytes, .. } => Some(bytes), + _ => Some(0), + } + } + + /// Deletes the working copy of `checkout` held for `peer` and its refs in the mirror. The + /// mirror itself goes at the next sweep once no copy uses it. + fn remove_copy(&self, peer: &PeerId, checkout: &CheckoutId) -> Option { + let dir = self.checkout_dir(peer, checkout); + if !dir.is_dir() { + return None; + } + let st = self.read_state(&dir); + let bytes = dir_size(&dir); + fs::remove_dir_all(&dir).ok()?; + if let Some(st) = &st + && let Ok(repo) = CheckoutId::parse(&st.repo) + { + let mirror = self.mirror_dir(peer, &repo); + if let Ok(refs) = self + .transfer + .list_refs(&mirror, &format!("refs/tasks/{checkout}/")) + { + let del: Vec = refs.into_iter().map(|(n, _)| n).collect(); + let _ = self.transfer.update_refs(&mirror, &[], &del); + } + } + Some(SyncEvent::Deleted { + kind: Deleted::WorkingCopy, + peer: Some(peer.clone()), + target: checkout.to_string(), + bytes, + }) + } + /// Deletes working copies idle past `policy.idle` (never one `busy` says has a live job), /// mirrors nothing refers to, and garbage collects mirrors that are due. Returns what it /// did; each item was also reported to the observer. @@ -113,28 +193,9 @@ impl SyncStore { if now.saturating_sub(used) < idle_ms { continue; } - let bytes = dir_size(&dir); - if fs::remove_dir_all(&dir).is_err() { - continue; + if let Some(e) = self.remove_copy(&peer, &checkout) { + events.push(e); } - if let Some(st) = &st - && let Ok(repo) = CheckoutId::parse(&st.repo) - { - let mirror = self.mirror_dir(&peer, &repo); - if let Ok(refs) = self - .transfer - .list_refs(&mirror, &format!("refs/tasks/{checkout}/")) - { - let del: Vec = refs.into_iter().map(|(n, _)| n).collect(); - let _ = self.transfer.update_refs(&mirror, &[], &del); - } - } - events.push(SyncEvent::Deleted { - kind: Deleted::WorkingCopy, - peer: Some(peer.clone()), - target: checkout.to_string(), - bytes, - }); } // which repos do the remaining copies use? let mut live_repos: HashSet = HashSet::new(); diff --git a/crates/tasks-peer/src/trust/role.rs b/crates/tasks-peer/src/trust/role.rs index 8b54edb..f3f9f40 100644 --- a/crates/tasks-peer/src/trust/role.rs +++ b/crates/tasks-peer/src/trust/role.rs @@ -67,7 +67,7 @@ impl Role { pub enum Need { /// Any valid grant (hello, status, events of the peer's own jobs). Any, - /// `submit`: jobs, sync and fetch. + /// `submit`: jobs, sync, fetch and dropping cached checkouts. Submit, } @@ -87,7 +87,8 @@ pub fn need_of(body: &Body) -> Need { Body::Hello(_) | Body::StatusRequest | Body::Subscribe { .. } - | Body::JobMetricsRequest { .. } => Need::Any, + | Body::JobMetricsRequest { .. } + | Body::CacheRequest => Need::Any, _ => Need::Submit, } } diff --git a/crates/tasks-peer/tests/retention.rs b/crates/tasks-peer/tests/retention.rs index f5fb200..f472e24 100644 --- a/crates/tasks-peer/tests/retention.rs +++ b/crates/tasks-peer/tests/retention.rs @@ -312,3 +312,108 @@ fn r310_the_origin_keeps_only_the_newest_snapshot_per_checkout_and_reports_what_ let id = CheckoutId::derive("laptop", &w.root); assert!(!sync_dir.join(format!("{id}.index")).exists()); } + +/// Build output kept per checkout, as the daemon's cache would. +#[derive(Default)] +struct Outputs(Mutex>); +impl tasks_peer::OutputKeeper for Outputs { + fn outputs(&self, _peer: &PeerId) -> Vec<(CheckoutId, u64)> { + let m = self.0.lock().unwrap(); + m.iter() + .map(|(k, v)| (CheckoutId::parse(k).unwrap(), *v)) + .collect() + } + fn drop_outputs(&self, _peer: &PeerId, checkout: &CheckoutId) -> u64 { + self.0 + .lock() + .unwrap() + .remove(checkout.as_str()) + .unwrap_or(0) + } +} + +#[test] +fn r839_a_peer_reports_each_copy_with_its_output_and_drops_all_but_one_a_job_uses() { + use tasks_peer::Handler; + use tasks_peer::fake::{FakeRunner, FakeStatus}; + use tasks_peer::outputs::FetchPolicy; + use tasks_peer::proto::Body; + use tasks_peer::receiver::{AllowAll, Allowlist, JobRunner, Receiver}; + let r = rig(); + let w = &r.w; + let other = w.tmp.path().join("origin/second"); + make_repo(&other); + push(w, &w.origin_sync, &w.root, 1); + push(w, &w.origin_sync, &other, 2); + let a = CheckoutId::derive("laptop", &w.root); + let b = CheckoutId::derive("laptop", &other); + // build output left after retention deleted its copy + let c = CheckoutId::parse(&"c".repeat(32)).unwrap(); + let runner = Arc::new(FakeRunner::new()); + let mut spec = w.spec(""); + spec.cwd = r.store.tree_root(&w.origin, &a); + runner + .submit( + &w.origin, + &tasks_peer::SubmitKey::new("laptop:1").unwrap(), + spec, + ) + .unwrap(); + let outputs = Arc::new(Outputs::default()); + for (id, n) in [(&a, 7), (&b, 9), (&c, 11)] { + outputs.0.lock().unwrap().insert(id.to_string(), n); + } + let recv = Receiver::new( + "nixos", + Allowlist::new([w.origin.clone()]), + r.store.clone(), + runner, + Arc::new(FakeStatus::nixos()), + Arc::new(AllowAll), + FetchPolicy::default(), + ); + recv.set_outputs(outputs.clone()); + w.net.serve(&w.nixos, Arc::new(recv) as Arc); + let mut conn = w + .transport + .connect(&w.nixos, Duration::from_secs(5)) + .unwrap(); + + let Body::CacheHeld { checkouts: held } = conn.request(Body::CacheRequest).unwrap() else { + panic!("not a cache report"); + }; + let of = |id: &CheckoutId| held.iter().find(|h| &h.checkout == id).unwrap().clone(); + assert!(of(&a).busy && of(&a).copy_bytes > 0 && of(&a).output_bytes == 7); + assert!(!of(&b).busy && of(&b).copy_bytes > 0 && of(&b).output_bytes == 9); + assert_eq!( + (of(&c).copy_bytes, of(&c).output_bytes, of(&c).last_used_ms), + (0, 11, None) + ); + let b_copy = of(&b).copy_bytes; + + let Body::CacheDropped { checkouts: dropped } = conn + .request(Body::CacheDrop { + checkouts: vec![a.clone(), b.clone(), c.clone()], + }) + .unwrap() + else { + panic!("not a drop answer"); + }; + let of = |id: &CheckoutId| dropped.iter().find(|d| &d.checkout == id).unwrap().clone(); + assert_eq!(of(&a).freed_bytes, 0); + assert_eq!(of(&a).kept.as_deref(), Some("a job runs or waits in it")); + assert_eq!((of(&b).freed_bytes, of(&b).kept), (b_copy + 9, None)); + assert_eq!(of(&c).freed_bytes, 11); + assert!(r.store.tree_root(&w.origin, &a).exists()); + assert!(!r.store.tree_root(&w.origin, &b).exists()); + assert_eq!( + outputs + .0 + .lock() + .unwrap() + .keys() + .cloned() + .collect::>(), + vec![a.to_string()] + ); +} diff --git a/crates/tasks-platform/tests/confine_macos.rs b/crates/tasks-platform/tests/confine_macos.rs index 40876f1..bba2234 100644 --- a/crates/tasks-platform/tests/confine_macos.rs +++ b/crates/tasks-platform/tests/confine_macos.rs @@ -64,8 +64,29 @@ impl Drop for Home { } } +/// False inside a confined job: macOS refuses a sandbox inside a sandbox, so these tests only +/// mean something outside one, as on CI, where a refusal fails instead (R-842). +fn can_confine() -> bool { + let ok = Command::new("/usr/bin/sandbox-exec") + .args(["-p", "(version 1)(allow default)", "/usr/bin/true"]) + .status() + .is_ok_and(|s| s.success()); + // CI runs unconfined: there a failing sandbox is the bug these tests look for + assert!( + ok || std::env::var_os("CI").is_none(), + "sandbox-exec cannot start a plain sandbox" + ); + if !ok { + eprintln!("skipped: already confined (a sandbox cannot start inside another)"); + } + ok +} + #[test] fn r831_a_confined_job_resolves_its_own_folder_and_globs_inside_it() { + if !can_confine() { + return; + } let h = Home::new("resolve"); let tree = h.store().join("tree"); assert!( @@ -88,6 +109,9 @@ fn r831_a_confined_job_resolves_its_own_folder_and_globs_inside_it() { #[test] fn r831_the_folders_above_stay_unlisted_and_the_rest_of_home_closed() { + if !can_confine() { + return; + } let h = Home::new("closed"); let tree = h.store().join("tree"); let root = h.root.display(); diff --git a/crates/tasks-ui/src/app.rs b/crates/tasks-ui/src/app.rs index 2599eda..700c719 100644 --- a/crates/tasks-ui/src/app.rs +++ b/crates/tasks-ui/src/app.rs @@ -5,6 +5,7 @@ use crate::actions::{self, Avail}; use crate::audit::{self, AuditData, AuditFilter}; use crate::columns::{self, MIN_WINDOW_WIDTH}; use crate::detail; +use crate::disk; use crate::feed::{self, ActionOutcome, Dashboard, DashboardResult, Feed, Wake}; use crate::format; use crate::link::DaemonLink; @@ -26,6 +27,7 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use tasks_core::JobId; use tasks_core::metrics::JobMetricsView; +use tasks_daemon::api::GcView; use tasks_daemon::api::HistoryView; const REFRESH: Duration = Duration::from_secs(1); @@ -124,6 +126,13 @@ pub struct App { stalls: Option>, stalls_loading: bool, stalls_asked: bool, + disk_box: Arc>>, + disk: Option>, + disk_loading: bool, + disk_asked: bool, + /// Rows ticked for freeing, by the name the report gives them. + disk_ticked: BTreeSet, + confirm_free: bool, metrics_box: Arc>, metrics: Option, tz: jiff::tz::TimeZone, @@ -182,6 +191,12 @@ impl App { stalls: None, stalls_loading: false, stalls_asked: false, + disk_box: Arc::new(Mailbox::new()), + disk: None, + disk_loading: false, + disk_asked: false, + disk_ticked: BTreeSet::new(), + confirm_free: false, metrics_box: Arc::new(Mailbox::new()), metrics: None, tz: jiff::tz::TimeZone::system(), @@ -268,6 +283,23 @@ impl App { self.audit_asked = true; } + /// Gives the Disk tab its report (for screenshots and tests). + pub fn set_disk(&mut self, view: GcView) { + disk::prune(&view, &mut self.disk_ticked); + self.disk = Some(Ok(view)); + self.disk_asked = true; + } + + /// Ticks a Disk row, as a click on it does. + pub fn tick_disk(&mut self, checkout: &str) { + self.disk_ticked.insert(checkout.to_string()); + } + + /// Opens the question before freeing the ticked rows, as the Free button does. + pub fn ask_free(&mut self) { + self.confirm_free = !self.disk_ticked.is_empty(); + } + /// Gives the Stalls tab its rows (for screenshots). pub fn set_stalls(&mut self, data: StallData) { self.stalls = Some(Ok(data)); @@ -375,6 +407,13 @@ impl App { self.stalls = Some(d); self.stalls_loading = false; } + if let Some(d) = self.disk_box.take() { + if let Ok(v) = &d { + disk::prune(v, &mut self.disk_ticked); + } + self.disk = Some(d); + self.disk_loading = false; + } if let Some(l) = &mut self.log { if let Some(s) = l.updates.take() { l.snap = s; @@ -535,9 +574,11 @@ impl App { Tab::Timings => self.stats_tab(ui, p), Tab::Audit => self.audit_tab(ui, p), Tab::Stalls => self.stalls_tab(ui, p), + Tab::Disk => self.disk_tab(ui, p), }); self.cancel_dialog(ctx); self.drain_dialog(ctx); + self.free_dialog(ctx); self.log_window(ctx); } @@ -547,6 +588,7 @@ impl App { || ctx.wants_keyboard_input() || self.cancel.is_some() || self.confirm_drain + || self.confirm_free { return; } @@ -1579,6 +1621,186 @@ impl App { } } + /// Sends a `gc` request in the background; its report replaces the one shown. + fn ask_disk(&mut self, req: tasks_daemon::api::Request) { + self.disk_asked = true; + self.disk_loading = true; + let link = Arc::clone(&self.link); + feed::spawn_once( + Arc::clone(&self.disk_box), + Arc::clone(&self.wake), + move || disk::fetch(link.as_ref(), &req), + ); + } + + /// The Disk tab (R-840): a card per machine, a row per checkout with a tick box, Free for + /// the ticked rows, and the shared caches, which are only shown. + fn disk_tab(&mut self, ui: &mut egui::Ui, p: &Palette) { + if !self.disk_asked { + self.ask_disk(disk::report_request()); + } + let view = match &self.disk { + Some(Ok(v)) => Some(v.clone()), + _ => None, + }; + ui.horizontal(|ui| { + if ui + .add_enabled(!self.disk_loading, egui::Button::new("Refresh")) + .clicked() + { + self.disk_asked = false; + } + if let Some(v) = &view { + let all = disk::reclaimable(v); + if ui + .add_enabled( + !all.is_empty() && !self.disk_loading, + egui::Button::new("Select merged, gone and old"), + ) + .clicked() + { + self.disk_ticked = all; + } + if ui + .add_enabled(!self.disk_ticked.is_empty(), egui::Button::new("Clear")) + .clicked() + { + self.disk_ticked.clear(); + } + let label = disk::free_label(v, &self.disk_ticked); + let can = !self.disk_ticked.is_empty() && !self.disk_loading; + if paint::button(ui, p, &label, false, true, can).clicked() && can { + self.confirm_free = true; + } + } + if self.disk_loading { + ui.weak("measuring… (a few seconds)"); + } + }); + ui.add_space(8.0); + let v = match (&self.disk, view) { + (None, _) => { + ui.weak("Measuring build output on every machine…"); + return; + } + (Some(Err(e)), _) => { + ui.colored_label(p.bad, e); + return; + } + (_, Some(v)) => v, + (Some(Ok(_)), None) => return, + }; + paint::disk_cards(ui, p, &disk::cards(&v)); + ui.add_space(6.0); + ui.weak(disk::summary(&v)); + for n in &v.notes { + ui.colored_label(p.warn, n); + } + ui.add_space(4.0); + // the table stretches its last column: an empty one, so the sizes keep their width + let mut titles = disk::columns(&v); + let right: Vec = (5..titles.len()).collect(); + let mut widths = vec![22.0, 64.0, 330.0, 200.0, 70.0]; + widths.extend(titles.iter().skip(5).map(|_| 80.0)); + titles.push(String::new()); + widths.push(0.0); + let cols: Vec<(&str, f32)> = titles + .iter() + .map(String::as_str) + .zip(widths.iter().copied()) + .collect(); + let now_ms = self.vm.as_ref().map_or(0, |vm| vm.now_ms); + let rows: Vec> = v + .rows + .iter() + .map(|r| { + let mut c = vec![String::new()]; + c.extend(disk::cells(&v, r, now_ms)); + c + }) + .collect(); + let ticks: Vec> = v + .rows + .iter() + .map(|r| { + (disk::selectable(r) && r.freed_bytes == 0) + .then(|| self.disk_ticked.contains(&r.checkout)) + }) + .collect(); + let tips: Vec = v + .rows + .iter() + .map(|r| match &r.kept { + Some(k) => format!("Kept: {k}"), + None => disk::state_help(r.state).to_string(), + }) + .collect(); + egui::ScrollArea::vertical() + .auto_shrink([false, false]) + .show(ui, |ui| { + let clicked = paint::tick_table( + ui, + p, + egui::Id::new("disk"), + &cols, + &rows, + &[], + &right, + &ticks, + &tips, + ); + for i in clicked { + if let Some(r) = v.rows.get(i) + && !self.disk_ticked.remove(&r.checkout) + { + self.disk_ticked.insert(r.checkout.clone()); + } + } + let shared = disk::shared_lines(&v); + if !shared.is_empty() { + ui.add_space(14.0); + paint::section_head(ui, p, "Shared by every checkout", "never freed here"); + for line in shared { + ui.weak(line); + } + } + }); + } + + fn free_dialog(&mut self, ctx: &egui::Context) { + if !self.confirm_free { + return; + } + let Some(Ok(v)) = &self.disk else { + self.confirm_free = false; + return; + }; + let text = disk::confirm_text(v, &self.disk_ticked); + let mut decision: Option = None; + egui::Window::new("Free build output?") + .collapsible(false) + .resizable(false) + .anchor(egui::Align2::CENTER_CENTER, vec2(0.0, 0.0)) + .show(ctx, |ui| { + ui.set_max_width(460.0); + ui.label(text); + ui.horizontal(|ui| { + if ui.button("Free").clicked() { + decision = Some(true); + } + if ui.button("Keep it").clicked() { + decision = Some(false); + } + }); + }); + if let Some(go) = decision { + self.confirm_free = false; + if go && !self.disk_ticked.is_empty() { + self.ask_disk(disk::free_request(&self.disk_ticked)); + } + } + } + fn stats_tab(&mut self, ui: &mut egui::Ui, p: &Palette) { if !self.stats_asked { self.stats_asked = true; diff --git a/crates/tasks-ui/src/disk.rs b/crates/tasks-ui/src/disk.rs new file mode 100644 index 0000000..5b2fba7 --- /dev/null +++ b/crates/tasks-ui/src/disk.rs @@ -0,0 +1,441 @@ +//! The Disk tab's data: the daemon's `gc` report as machine cards and one row per checkout, +//! and the request that frees the rows the user ticked (R-840). Pure, so every text is tested +//! on a fixed report; the window only draws it and keeps the ticks. + +use crate::link::DaemonLink; +use std::collections::BTreeSet; +use tasks_core::human_bytes; +use tasks_daemon::api::{GcRow, GcState, GcView, Request, Response}; + +/// The report alone: nothing deleted. +pub fn report_request() -> Request { + Request::Gc { + delete: Vec::new(), + checkouts: Vec::new(), + idle_days: None, + } +} + +/// Frees the checkouts named, as the report names them. The daemon still keeps any a job +/// uses. +pub fn free_request(checkouts: &BTreeSet) -> Request { + Request::Gc { + delete: Vec::new(), + checkouts: checkouts.iter().cloned().collect(), + idle_days: None, + } +} + +/// Sends `req` and returns the report. +pub fn fetch(link: &dyn DaemonLink, req: &Request) -> Result { + match link.call(req) { + Ok(Response::Gc(v)) => Ok(*v), + Ok(other) => Err(format!( + "unexpected reply to gc: {}", + crate::feed::kind_of(&other) + )), + Err(e) => Err(e.to_string()), + } +} + +/// One machine's card at the top of the tab. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DiskCard { + /// Machine name. + pub name: String, + /// `build cache 134.5 GiB · 98.5 GiB can be freed`, or why it could not report. + pub line: String, + /// True when the machine could not report. + pub failed: bool, +} + +/// The cards, this machine first as the daemon sends them. +pub fn cards(v: &GcView) -> Vec { + v.machines + .iter() + .map(|m| match &m.error { + Some(e) => DiskCard { + name: m.name.clone(), + line: format!("could not report: {e}"), + failed: true, + }, + None => DiskCard { + name: m.name.clone(), + line: format!( + "build cache {} · {} can be freed", + human_bytes(m.held_bytes), + human_bytes(m.reclaimable_bytes) + ), + failed: false, + }, + }) + .collect() +} + +/// The machines that reported, one size column each. +pub fn machine_columns(v: &GcView) -> Vec { + v.machines + .iter() + .filter(|m| m.error.is_none()) + .map(|m| m.name.clone()) + .collect() +} + +/// The table's column titles: the tick, the fixed columns, one per machine, then what was +/// freed when something was. +pub fn columns(v: &GcView) -> Vec { + let mut c: Vec = ["", "State", "Checkout", "Branch", "Last job"] + .iter() + .map(|s| s.to_string()) + .collect(); + c.extend(machine_columns(v)); + if v.deleted { + c.push("Freed".into()); + } + c +} + +/// How long ago, in the largest whole unit. +pub fn ago(ms: Option, now_ms: i64) -> String { + let Some(ms) = ms else { + return "-".into(); + }; + let s = (now_ms - ms).max(0) / 1000; + match s { + s if s < 60 => "just now".into(), + s if s < 3600 => format!("{}m ago", s / 60), + s if s < 86_400 => format!("{}h ago", s / 3600), + s => format!("{}d ago", s / 86_400), + } +} + +/// One row's cells after the tick, in [`columns`] order. +pub fn cells(v: &GcView, r: &GcRow, now_ms: i64) -> Vec { + let branch = match (&r.branch, &r.merged_into) { + (Some(b), Some(m)) => format!("{b} (in {m})"), + (Some(b), None) => b.clone(), + (None, _) => "-".into(), + }; + let mut c = vec![ + r.state.name().replace('_', " "), + r.checkout.clone(), + branch, + ago(r.last_used_ms, now_ms), + ]; + for m in machine_columns(v) { + c.push(match r.held.iter().find(|h| h.machine == m) { + Some(h) => human_bytes(h.bytes), + None => "-".into(), + }); + } + if v.deleted { + c.push(match r.freed_bytes { + 0 => "-".into(), + b => human_bytes(b), + }); + } + c +} + +/// What a state means, for the State cell's tooltip. +pub fn state_help(s: GcState) -> &'static str { + match s { + GcState::InUse => "A job of it is queued or running: it cannot be freed now.", + GcState::Active => "Used recently. Freeing it means its next build starts cold.", + GcState::Idle => "No job for a while. Freeing it means its next build starts cold.", + GcState::Merged => "Its branch is merged and nothing tracked is changed.", + GcState::Gone => "Its working tree no longer exists.", + GcState::Old => "Output of the older cache layout that no working tree links to.", + } +} + +/// True when the row can be ticked: anything a job does not use. +pub fn selectable(r: &GcRow) -> bool { + r.state != GcState::InUse +} + +/// The rows `Select merged, gone and old` ticks. +pub fn reclaimable(v: &GcView) -> BTreeSet { + v.rows + .iter() + .filter(|r| r.state.reclaimable() && r.freed_bytes == 0) + .map(|r| r.checkout.clone()) + .collect() +} + +/// Drops ticks of rows that are gone from the report or a job now uses. +pub fn prune(v: &GcView, ticked: &mut BTreeSet) { + ticked.retain(|c| { + v.rows + .iter() + .any(|r| &r.checkout == c && selectable(r) && r.freed_bytes == 0) + }); +} + +/// Bytes the ticked rows hold, over every machine. +pub fn ticked_bytes(v: &GcView, ticked: &BTreeSet) -> u64 { + v.rows + .iter() + .filter(|r| ticked.contains(&r.checkout)) + .flat_map(|r| &r.held) + .map(|h| h.bytes) + .sum() +} + +/// The Free button's label: `Free 12.3 GiB from 3 checkouts`, or just `Free` with none ticked. +pub fn free_label(v: &GcView, ticked: &BTreeSet) -> String { + let n = ticked.len(); + if n == 0 { + return "Free".into(); + } + format!( + "Free {} from {n} checkout{}", + human_bytes(ticked_bytes(v, ticked)), + if n == 1 { "" } else { "s" } + ) +} + +/// The confirmation text: what goes, what stays, and that a ticked active or idle row builds +/// cold next time. +pub fn confirm_text(v: &GcView, ticked: &BTreeSet) -> String { + let mut s = format!( + "{}?\n\nOnly build output and the peers' synced copies are deleted. Working trees stay, \ + and a checkout a job starts using meanwhile is kept.", + free_label(v, ticked) + ); + let warm: Vec<&str> = v + .rows + .iter() + .filter(|r| ticked.contains(&r.checkout)) + .filter(|r| matches!(r.state, GcState::Active | GcState::Idle)) + .map(|r| r.checkout.as_str()) + .collect(); + if !warm.is_empty() { + s.push_str(&format!( + "\n\nStill in use, their next build starts cold: {}.", + warm.join(", ") + )); + } + s +} + +/// The line under the cards: after a free what it did, else how much can go. +pub fn summary(v: &GcView) -> String { + if v.deleted { + let freed: u64 = v.rows.iter().map(|r| r.freed_bytes).sum(); + let n = v.rows.iter().filter(|r| r.freed_bytes > 0).count(); + let kept = v.rows.iter().filter(|r| r.kept.is_some()).count(); + let mut s = format!( + "Freed {} from {n} checkout{}.", + human_bytes(freed), + if n == 1 { "" } else { "s" } + ); + if kept > 0 { + s.push_str(&format!(" {kept} kept something; hover its row for why.")); + } + return s; + } + if v.rows.is_empty() { + return "No checkout keeps build output.".into(); + } + let free: u64 = v.machines.iter().map(|m| m.reclaimable_bytes).sum(); + if free == 0 { + format!( + "{} checkout(s). Nothing merged, gone or old to free.", + v.rows.len() + ) + } else { + format!( + "{} checkout(s). {} is merged, gone or old and can be freed.", + v.rows.len(), + human_bytes(free) + ) + } +} + +/// The shared caches, one line each: `compiler cache (sccache): 30 GiB in … (note)`. +pub fn shared_lines(v: &GcView) -> Vec { + v.shared + .iter() + .map(|s| { + format!( + "{}: {} in {} ({})", + s.name, + human_bytes(s.bytes), + s.path, + s.note + ) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::link::{FakeLink, LinkError}; + use std::sync::{Arc, Mutex}; + use tasks_daemon::api::{GcHeld, GcMachine, GcShared}; + + const NOW: i64 = 10 * 86_400_000; + const GIB: u64 = 1 << 30; + + fn held(machine: &str, bytes: u64) -> GcHeld { + GcHeld { + machine: machine.into(), + bytes, + } + } + + fn row(checkout: &str, state: GcState, held: Vec) -> GcRow { + GcRow { + checkout: checkout.into(), + branch: Some("feature".into()), + state, + merged_into: (state == GcState::Merged).then(|| "dev".into()), + last_used_ms: Some(NOW - 2 * 86_400_000), + held, + freed_bytes: 0, + kept: None, + } + } + + fn view() -> GcView { + GcView { + idle_days: 14, + machines: vec![ + GcMachine { + name: "laptop".into(), + held_bytes: 12 * GIB, + reclaimable_bytes: 4 * GIB, + error: None, + }, + GcMachine { + name: "mac".into(), + held_bytes: 0, + reclaimable_bytes: 0, + error: Some("mac runs an older loadtabled".into()), + }, + ], + rows: vec![ + row( + "(unknown checkout 4ca7e0dc2b4d)", + GcState::Gone, + vec![held("laptop", GIB)], + ), + row("~/wt/done", GcState::Merged, vec![held("laptop", 3 * GIB)]), + row("~/main", GcState::Active, vec![held("laptop", 6 * GIB)]), + row("~/wt/busy", GcState::InUse, vec![held("laptop", 2 * GIB)]), + ], + shared: vec![GcShared { + name: "compiler cache (sccache)".into(), + path: "/c/sccache".into(), + bytes: 30 * GIB, + note: "sccache keeps itself under 30G".into(), + }], + notes: vec![], + deleted: false, + } + } + + // R-840 + #[test] + fn r840_the_disk_tab_shows_a_card_per_machine_and_a_row_per_checkout() { + let v = view(); + let c = cards(&v); + assert_eq!(c[0].line, "build cache 12 GiB · 4 GiB can be freed"); + assert!(!c[0].failed); + assert_eq!(c[1].line, "could not report: mac runs an older loadtabled"); + assert!(c[1].failed); + // a machine that did not report gets no column + assert_eq!( + columns(&v), + vec!["", "State", "Checkout", "Branch", "Last job", "laptop"] + ); + assert_eq!( + cells(&v, &v.rows[1], NOW), + vec!["merged", "~/wt/done", "feature (in dev)", "2d ago", "3 GiB"] + ); + assert_eq!(cells(&v, &v.rows[3], NOW)[0], "in use"); + assert_eq!( + summary(&v), + "4 checkout(s). 4 GiB is merged, gone or old and can be freed." + ); + assert_eq!( + shared_lines(&v), + vec!["compiler cache (sccache): 30 GiB in /c/sccache (sccache keeps itself under 30G)"] + ); + } + + // R-840 + #[test] + fn r840_ticking_frees_only_the_named_rows_and_never_offers_one_in_use() { + let v = view(); + assert!(!selectable(&v.rows[3]), "in use cannot be ticked"); + let mut t = reclaimable(&v); + assert_eq!( + t.iter().map(String::as_str).collect::>(), + vec!["(unknown checkout 4ca7e0dc2b4d)", "~/wt/done"] + ); + assert_eq!(free_label(&v, &t), "Free 4 GiB from 2 checkouts"); + assert!(!confirm_text(&v, &t).contains("starts cold")); + t.insert("~/main".into()); + t.insert("~/wt/busy".into()); + t.insert("~/vanished".into()); + prune(&v, &mut t); + assert_eq!(t.len(), 3, "busy and vanished rows are dropped: {t:?}"); + assert!(confirm_text(&v, &t).contains("their next build starts cold: ~/main.")); + + let asked = Arc::new(Mutex::new(None)); + let seen = Arc::clone(&asked); + let link = FakeLink::from_fn(move |req| { + *seen.lock().unwrap() = Some(req.clone()); + let mut done = view(); + done.deleted = true; + done.rows[1].freed_bytes = 3 * GIB; + done.rows[2].kept = Some("nixos: unreachable".into()); + Ok(Response::Gc(Box::new(done))) + }); + let after = fetch(&link, &free_request(&t)).unwrap(); + assert!(matches!( + asked.lock().unwrap().clone(), + Some(Request::Gc { delete, checkouts, .. }) + if delete.is_empty() && checkouts.len() == 3 && checkouts.contains(&"~/main".to_string()) + )); + assert_eq!( + summary(&after), + "Freed 3 GiB from 1 checkout. 1 kept something; hover its row for why." + ); + assert_eq!(columns(&after).last().map(String::as_str), Some("Freed")); + let mut t2 = t.clone(); + prune(&after, &mut t2); + assert!(!t2.contains("~/wt/done"), "a freed row is no longer ticked"); + } + + // R-840 + #[test] + fn r840_a_failed_report_is_said_in_words() { + let down = FakeLink::from_fn(|_| Err(LinkError("socket gone".into()))); + assert_eq!( + fetch(&down, &report_request()), + Err("socket gone".to_string()) + ); + let odd = FakeLink::from_fn(|_| Ok(Response::Ack)); + assert!( + fetch(&odd, &report_request()) + .unwrap_err() + .contains("unexpected reply to gc") + ); + assert_eq!(ago(None, NOW), "-"); + assert_eq!(ago(Some(NOW - 30_000), NOW), "just now"); + assert_eq!(ago(Some(NOW - 3 * 3_600_000), NOW), "3h ago"); + } + + // R-844 + #[test] + fn r844_with_nothing_ticked_the_free_button_names_no_size() { + let v = view(); + assert_eq!(free_label(&v, &BTreeSet::new()), "Free"); + let one: BTreeSet = ["~/wt/done".to_string()].into(); + assert!(free_label(&v, &one).ends_with("from 1 checkout")); + } +} diff --git a/crates/tasks-ui/src/feed.rs b/crates/tasks-ui/src/feed.rs index c6a3279..4f2dd47 100644 --- a/crates/tasks-ui/src/feed.rs +++ b/crates/tasks-ui/src/feed.rs @@ -101,6 +101,7 @@ pub fn kind_of(r: &Response) -> &'static str { Response::Estimate(_) => "estimate", Response::History(_) => "history", Response::JobMetrics(_) => "job_metrics", + Response::Gc(_) => "gc", Response::Audit { .. } => "audit", Response::AuditVerify(_) => "audit_verify", Response::AuditCheckpoint { .. } => "audit_checkpoint", diff --git a/crates/tasks-ui/src/fixtures.rs b/crates/tasks-ui/src/fixtures.rs index d802cc0..5cae803 100644 --- a/crates/tasks-ui/src/fixtures.rs +++ b/crates/tasks-ui/src/fixtures.rs @@ -894,3 +894,109 @@ pub fn offline() -> Dashboard { d.status.peers = vec![peer("offline", Some(420))]; d } + +/// A `gc` report for the Disk tab: three machines, one that cannot report, a row in each +/// state, and the shared caches (R-840). +pub fn disk() -> tasks_daemon::api::GcView { + use tasks_daemon::api::{GcHeld, GcMachine, GcRow, GcShared, GcState, GcView}; + const GIB: u64 = 1 << 30; + const DAY: i64 = 86_400_000; + let held = |pairs: &[(&str, u64)]| { + pairs + .iter() + .map(|(m, b)| GcHeld { + machine: (*m).into(), + bytes: *b, + }) + .collect() + }; + let row = |checkout: &str, branch: Option<&str>, state, days: i64, h: Vec| GcRow { + checkout: checkout.into(), + branch: branch.map(Into::into), + state, + merged_into: (state == GcState::Merged).then(|| "dev".into()), + last_used_ms: Some(NOW_MS - days * DAY), + held: h, + freed_bytes: 0, + kept: None, + }; + let machine = |name: &str, held_bytes, reclaimable_bytes, error: Option<&str>| GcMachine { + name: name.into(), + held_bytes, + reclaimable_bytes, + error: error.map(Into::into), + }; + GcView { + idle_days: 14, + machines: vec![ + machine("laptop", 134 * GIB, 98 * GIB, None), + machine("nixos", 41 * GIB, 22 * GIB, None), + machine( + "mac", + 0, + 0, + Some("mac runs an older loadtabled that cannot report its cache; upgrade it"), + ), + ], + rows: vec![ + row( + "(unknown checkout 4ca7e0dc2b4d)", + None, + GcState::Gone, + 1, + held(&[("laptop", 9 * GIB)]), + ), + row( + "~/app-worktrees/v2", + Some("v2"), + GcState::Merged, + 1, + held(&[("laptop", 32 * GIB), ("nixos", 14 * GIB)]), + ), + row( + "~/app/.claude/worktrees/group-voice", + Some("feature/group-voice"), + GcState::Merged, + 2, + held(&[("laptop", 14 * GIB), ("nixos", 8 * GIB)]), + ), + row( + "~/app/.claude/worktrees/old-spike", + Some("spike/cache"), + GcState::Idle, + 21, + held(&[("laptop", 6 * GIB)]), + ), + row( + "~/app", + Some("dev"), + GcState::Active, + 0, + held(&[("laptop", 18 * GIB), ("nixos", 11 * GIB)]), + ), + row( + "~/task-manager", + Some("dev"), + GcState::InUse, + 0, + held(&[("laptop", 9 * GIB), ("nixos", 8 * GIB)]), + ), + ], + shared: vec![ + GcShared { + name: "compiler cache (sccache)".into(), + path: "/home/demo/build-cache/sccache".into(), + bytes: 30 * GIB, + note: "sccache keeps itself under 30G".into(), + }, + GcShared { + name: "Gradle home".into(), + path: "/home/demo/build-cache/gradle".into(), + bytes: 3 * GIB, + note: "shared by every Gradle build; not cleaned by gc".into(), + }, + ], + notes: vec![], + deleted: false, + } +} diff --git a/crates/tasks-ui/src/lib.rs b/crates/tasks-ui/src/lib.rs index 2d40fcf..dc972f6 100644 --- a/crates/tasks-ui/src/lib.rs +++ b/crates/tasks-ui/src/lib.rs @@ -13,6 +13,7 @@ pub mod actions; pub mod audit; pub mod columns; pub mod detail; +pub mod disk; pub mod feed; pub mod fixtures; pub mod format; diff --git a/crates/tasks-ui/src/model.rs b/crates/tasks-ui/src/model.rs index 700890f..7318bc8 100644 --- a/crates/tasks-ui/src/model.rs +++ b/crates/tasks-ui/src/model.rs @@ -18,6 +18,8 @@ pub enum Tab { History, /// Per-recipe and per-target timing. Timings, + /// What each checkout's build output takes on every machine, and freeing it. + Disk, /// Pressure trips: what caused each stall. Stalls, /// The audit log. @@ -26,10 +28,11 @@ pub enum Tab { impl Tab { /// Tabs with content, in order. - pub const SHOWN: [Tab; 5] = [ + pub const SHOWN: [Tab; 6] = [ Tab::Queue, Tab::History, Tab::Timings, + Tab::Disk, Tab::Stalls, Tab::Audit, ]; @@ -42,6 +45,7 @@ impl Tab { Tab::Queue => "Queue", Tab::History => "History", Tab::Timings => "Timings", + Tab::Disk => "Disk", Tab::Stalls => "Stalls", Tab::Audit => "Audit", } @@ -2327,7 +2331,7 @@ mod tests { let labels: Vec<_> = Tab::SHOWN.iter().map(|t| t.label()).collect(); assert_eq!( labels, - vec!["Queue", "History", "Timings", "Stalls", "Audit"] + vec!["Queue", "History", "Timings", "Disk", "Stalls", "Audit"] ); } diff --git a/crates/tasks-ui/src/paint.rs b/crates/tasks-ui/src/paint.rs index d384ca4..7fb99d5 100644 --- a/crates/tasks-ui/src/paint.rs +++ b/crates/tasks-ui/src/paint.rs @@ -356,6 +356,9 @@ pub fn group_header(ui: &mut Ui, p: &Palette, title: &str, count: usize, note: & } } +/// Narrowest room the left part of a state cell is drawn in; with less it is left out. +const MIN_LEFT: f32 = 40.0; + /// The state cell: a bar or a head line, then the two-part line under it. fn state_cell(painter: &Painter, p: &Palette, rect: Rect, s: &StateCell, section: Section) { let top = rect.min.y + 9.0; @@ -391,21 +394,24 @@ fn state_cell(painter: &Painter, p: &Palette, rect: Rect, s: &StateCell, section put(painter, rect.min.x, line1, g); } } - // right part first: it keeps its room, the left part gets what is left + // right part first (the time left): it is cut to the cell, and the left part gets what is + // left, or is not drawn when too little is, never drawn under it (R-838) let right_w = if s.right.is_empty() { 0.0 } else { - text_right( + let g = fit( painter, - rect.max.x, - line2, &s.right, theme::sans_medium(SECONDARY), p.ink, - ) + rect.width(), + ); + let w = g.size().x; + put(painter, rect.max.x - w, line2, g); + w }; - if !s.left.is_empty() { - let room = (rect.width() - right_w - 10.0).max(20.0); + let room = rect.width() - right_w - 10.0; + if !s.left.is_empty() && room >= MIN_LEFT { put( painter, rect.min.x, @@ -1019,7 +1025,10 @@ pub fn button( enabled: bool, ) -> egui::Response { let font = theme::sans_medium(SECONDARY); - let color = if primary { + // a button that cannot be pressed reads as such: no red or accent text on it + let color = if !enabled { + p.faint + } else if primary { p.panel } else if danger { p.bad @@ -1338,6 +1347,26 @@ pub fn table( mono: &[usize], right: &[usize], ) { + tick_table(ui, p, id, cols, rows, mono, right, &[], &[]); +} + +/// [`table`] with a tick box in the first column of each row whose `ticks` entry is `Some` +/// (R-840). A click anywhere on such a row toggles it; the rows clicked are returned. A row +/// with a `tips` text shows it on hover. +#[allow(clippy::too_many_arguments)] +pub fn tick_table( + ui: &mut Ui, + p: &Palette, + id: egui::Id, + cols: &[(&str, f32)], + rows: &[Vec], + mono: &[usize], + right: &[usize], + ticks: &[Option], + tips: &[String], +) -> Vec { + let mut clicked = Vec::new(); + let ticking = !ticks.is_empty(); let width = ui.available_width(); let gap = 14.0; let fixed: f32 = cols.iter().rev().skip(1).map(|c| c.1 + gap).sum(); @@ -1364,10 +1393,59 @@ pub fn table( put(&hp, gx, hr.center().y, g); } for (r, row) in rows.iter().enumerate() { - let (rect, _) = ui.allocate_exact_size(vec2(width, 28.0), Sense::hover()); + let tick = ticks.get(r).copied().flatten(); + let sense = if tick.is_some() { + Sense::click() + } else { + Sense::hover() + }; + let (rect, resp) = ui.allocate_exact_size(vec2(width, 28.0), sense); let painter = ui.painter_at(rect); + if tick == Some(true) { + painter.rect_filled(rect, 0.0, p.accent.gamma_multiply(0.12)); + } else if tick.is_some() && resp.hovered() { + painter.rect_filled(rect, 0.0, p.raise); + } painter.hline(rect.x_range(), rect.min.y, Stroke::new(1.0_f32, p.line)); + if let Some(on) = tick { + let b = + Rect::from_center_size(pos2(rect.min.x + 8.0, rect.center().y), vec2(13.0, 13.0)); + if on { + painter.rect_filled(b, round(3.0), p.accent); + painter.line_segment( + [ + pos2(b.min.x + 3.0, b.center().y), + pos2(b.center().x - 1.0, b.max.y - 3.5), + ], + Stroke::new(1.6_f32, p.panel), + ); + painter.line_segment( + [ + pos2(b.center().x - 1.0, b.max.y - 3.5), + pos2(b.max.x - 3.0, b.min.y + 3.5), + ], + Stroke::new(1.6_f32, p.panel), + ); + } else { + painter.rect_stroke( + b, + round(3.0), + Stroke::new(1.0_f32, p.dim), + StrokeKind::Inside, + ); + } + } + if resp.clicked() { + clicked.push(r); + } + if let Some(t) = tips.get(r).filter(|t| !t.is_empty()) { + resp.on_hover_text(t); + } for (i, text) in row.iter().enumerate() { + // the first column holds the tick box, drawn above + if ticking && i == 0 { + continue; + } let (x, w) = col_x(i); let font = if mono.contains(&i) { theme::mono(SECONDARY) @@ -1390,4 +1468,33 @@ pub fn table( } } } + clicked +} + +/// The Disk tab's machine cards: the name over what its build cache keeps (R-840). +pub fn disk_cards(ui: &mut Ui, p: &Palette, cards: &[crate::disk::DiskCard]) { + ui.horizontal_wrapped(|ui| { + ui.spacing_mut().item_spacing = vec2(10.0, 10.0); + for c in cards { + egui::Frame::new() + .fill(p.panel) + .stroke(Stroke::new(1.0_f32, p.line)) + .corner_radius(round(6.0)) + .inner_margin(egui::Margin::symmetric(12, 8)) + .show(ui, |ui| { + ui.vertical(|ui| { + ui.label( + RichText::new(&c.name) + .font(theme::sans_semibold(theme::BODY)) + .color(p.ink), + ); + ui.label( + RichText::new(&c.line) + .font(theme::sans(SECONDARY)) + .color(if c.failed { p.warn } else { p.dim }), + ); + }); + }); + } + }); } diff --git a/crates/tasks-ui/src/raster.rs b/crates/tasks-ui/src/raster.rs index acd297e..2325c79 100644 --- a/crates/tasks-ui/src/raster.rs +++ b/crates/tasks-ui/src/raster.rs @@ -382,6 +382,27 @@ mod tests { updated 3s ago Select a job to see its phases, folder and actions. \ PROGRAM CPU IN CORES MEMORY usual peak last hour · the rest is in History", ); + // the Disk tab: its buttons, the report's texts and the question before freeing + { + let v = fixtures::disk(); + let all = crate::disk::reclaimable(&v); + texts.push_str("Select merged, gone and old Clear Free Keep it measuring… (a few seconds) \ + Measuring build output on every machine… Shared by every checkout never freed here \ + Free build output? Kept: "); + texts.push_str(&crate::disk::summary(&v)); + texts.push_str(&crate::disk::confirm_text(&v, &all)); + for c in crate::disk::cards(&v) { + texts.push_str(&c.line); + } + for t in crate::disk::columns(&v) { + texts.push_str(&t); + } + for r in &v.rows { + texts.push_str(&crate::disk::cells(&v, r, fixtures::NOW_MS).join(" ")); + texts.push_str(crate::disk::state_help(r.state)); + } + texts.push_str(&crate::disk::shared_lines(&v).join(" ")); + } // the Audit tab: filters, column titles and the rows' own symbols texts.push_str("Action Peer Job Apply all name or DID id #101 did:key:z6Mk…EGta"); for t in crate::audit::COLUMNS.iter().chain(&crate::audit::FAMILIES) { @@ -706,6 +727,30 @@ mod tests { [1100, 600], true, ), + ( + "disk-dark", + { + let mut a = App::fixed(&fixtures::idle(), dead_link()); + a.show_tab(Tab::Disk); + a.set_disk(fixtures::disk()); + a.tick_disk("~/app-worktrees/v2"); + a.tick_disk("(unknown checkout 4ca7e0dc2b4d)"); + a + }, + [1280, 560], + true, + ), + ( + "disk-light", + { + let mut a = App::fixed(&fixtures::idle(), dead_link()); + a.show_tab(Tab::Disk); + a.set_disk(fixtures::disk()); + a + }, + [1280, 560], + false, + ), ( "stalls-dark", { @@ -787,9 +832,24 @@ mod tests { /// Every text one frame of `app` draws. fn drawn_text(app: &mut App) -> Vec { - fn walk(shape: &egui::Shape, into: &mut Vec) { + drawn(app).into_iter().map(|(t, _)| t).collect() + } + + /// Each text a frame of `app` draws, with where it lands. + fn drawn(app: &mut App) -> Vec<(String, egui::Rect)> { + let ctx = egui::Context::default(); + crate::theme::install(&ctx); + drawn_in(app, &ctx) + } + + /// [`drawn`] in a context earlier frames ran in, so a window opened then is laid out. + fn drawn_in(app: &mut App, ctx: &egui::Context) -> Vec<(String, egui::Rect)> { + fn walk(shape: &egui::Shape, into: &mut Vec<(String, egui::Rect)>) { match shape { - egui::Shape::Text(t) => into.push(t.galley.text().to_string()), + egui::Shape::Text(t) => into.push(( + t.galley.text().to_string(), + t.galley.rect.translate(t.pos.to_vec2()), + )), egui::Shape::Vec(v) => { for s in v { walk(s, into); @@ -798,8 +858,6 @@ mod tests { _ => {} } } - let ctx = egui::Context::default(); - crate::theme::install(&ctx); let input = egui::RawInput { screen_rect: Some(egui::Rect::from_min_size( egui::Pos2::ZERO, @@ -825,6 +883,27 @@ mod tests { assert!(!texts.iter().any(|t| t.starts_with("battery")), "{texts:?}"); } + // R-838 + #[test] + fn r838_a_long_time_left_is_cut_to_its_cell_and_nothing_is_drawn_under_it() { + let mut d = fixtures::busy(); + for r in d.status.running.iter_mut().filter(|r| r.id.0 == 102) { + r.eta_secs = Some(658.0); + r.eta_low_secs = None; + r.eta_rough_from = Some("nixos".into()); + } + let texts = drawn(&mut App::fixed(&d, dead_link())); + let (_, at) = texts + .iter() + .find(|(t, _)| t.starts_with("~10m58s left")) + .unwrap_or_else(|| panic!("the time left is drawn: {texts:?}")); + let under: Vec<_> = texts + .iter() + .filter(|(t, r)| !t.starts_with("~10m58s left") && r.shrink(0.5).intersects(*at)) + .collect(); + assert!(under.is_empty(), "drawn over the time left: {under:?}"); + } + // R-833 #[test] fn r833_the_window_says_loadtable_not_its_old_name() { @@ -937,6 +1016,54 @@ mod tests { assert_eq!(app.selected_id(), None); } + // R-840 + #[test] + fn r840_the_disk_tab_frees_only_after_asking_and_only_the_ticked_rows() { + use tasks_daemon::api::Request; + let ctx = egui::Context::default(); + crate::theme::install(&ctx); + let link = dead_link(); + let mut app = App::fixed(&fixtures::idle(), link.clone()); + app.show_tab(Tab::Disk); + app.set_disk(fixtures::disk()); + let text = drawn_text(&mut app).join("\n"); + for want in [ + "build cache 134 GiB · 98 GiB can be freed", + "could not report: mac runs an older loadtabled", + "~/app-worktrees/v2", + "v2 (in dev)", + "in use", + "Shared by every checkout", + ] { + assert!(text.contains(want), "{want:?} not drawn in:\n{text}"); + } + // R-844: with nothing ticked the button names no size + assert!(text.lines().any(|l| l == "Free"), "a bare Free in:\n{text}"); + assert!(!text.contains("Free 0 B")); + app.tick_disk("~/app-worktrees/v2"); + app.tick_disk("~/task-manager"); + app.set_disk(fixtures::disk()); + app.ask_free(); + frame(&mut app, &ctx, vec![]); + frame(&mut app, &ctx, vec![]); + assert!( + link.requests().iter().all(|r| !matches!( + r, + Request::Gc { checkouts, .. } if !checkouts.is_empty() + )), + "nothing was freed by opening the question" + ); + let text = drawn_in(&mut app, &ctx) + .into_iter() + .map(|(t, _)| t) + .collect::>() + .join("\n"); + assert!( + text.contains("Free 46 GiB from 1 checkout?"), + "the in-use row is never ticked: {text}" + ); + } + fn stall_rows() -> crate::stalls::StallData { use tasks_daemon::api::{TripDetails, TripProc}; let row = |seq: i64, resource: &str, reason: &str| crate::stalls::StallRow { diff --git a/docs/REGRESSIONS.md b/docs/REGRESSIONS.md index 386960a..105f127 100644 --- a/docs/REGRESSIONS.md +++ b/docs/REGRESSIONS.md @@ -829,3 +829,10 @@ Tests of earlier rows whose expectations changed with the redesign, kept and edi | R-835 | A low battery is shown before any job waits on it: the status says `battery low: jobs from peers wait for AC or a charge above 30%`, the machine reports `holds_peer_jobs` with its charge to its peers, their `status` adds `battery low: holds jobs from peers` to its power, and the window draws `battery 22%` in the warning colour in place of the machine's role. | `r835_low_battery_is_said_in_the_status_and_reaches_the_peers` (daemon), `r835_a_worker_on_low_battery_says_so_in_place_of_its_role` (raster) | any | | R-836 | `--anywhere` passes over a peer that holds jobs on low battery whenever another machine can take the job, busy or not, and says `on battery at 25%` in its pressure; when it is the only machine that can (an Apple build and the Mac), the job goes there and waits for the charge. Older peers report no hold. | `r836_a_peer_holding_jobs_on_low_battery_does_not_win_over_a_busy_machine`, `r836_a_peer_holding_jobs_is_still_used_when_nothing_else_can_start_it` (placement) | any | | R-837 | A shown estimate runs from the fastest of the last six passing runs to the slowest (or the twenty-run p90 when that is longer): `~5s–56s (median 16s) from 6 passing runs on laptop`, one number when both ends round alike. A warm cargo run takes seconds with nothing to rebuild and a minute with a changed crate, so one number was wrong both ways. A running job's time left is `~3s–54s left`, `up to ~26s left` once past the fastest, and only past the slowest is it `longer than recent runs (5s–56s) by 30s`; the same in the CLI, MCP and the window. | `r837_an_estimate_is_the_fastest_to_the_slowest_recent_run`, `r837_a_running_job_has_time_left_to_the_fastest_and_slowest_and_is_over_only_past_both` (timing), `r837_time_left_is_a_range_to_the_fastest_and_slowest_recent_run` (cli render), `r671_r672_r675_r837_the_state_cell_bar_and_time_left_rest_on_real_runs` (ui model) | any | +| R-838 | A running row's time left (`~10m58s left (rough, from nixos's runs)`) is cut to its cell, and the part beside it (`estimated from passing runs`) is drawn only in the room left, at least 40 points, never under it. | `r838_a_long_time_left_is_cut_to_its_cell_and_nothing_is_drawn_under_it` (ui raster) | any | +| R-839 | `loadtable gc` (and the MCP tool `gc`) reports, per machine, what each checkout's build output takes (and on a peer its synced copy too), and what can be freed: `laptop 66.0 GiB kept, 30.2 GiB can be freed`. Each checkout is `in_use` (a job queued or running, never deleted), `active`, `idle` (no job in `--idle-days`, 14), `merged` (its branch is in dev, main or master and no tracked file is changed), `gone` (its working tree is deleted) or `old` (an older layout's output no worktree links to). Output belongs to the checkout whose `target` or Android links point into it, also in the older `*/tasks/checkouts` layout. `--delete merged,gone,old` deletes only build output and peers' copies, never a working tree; `in_use` is refused. A peer that cannot answer is named (`mac runs an older loadtabled that cannot report its cache; upgrade it`) and the rest still reports. A peer keeps a copy a job of the asker uses. Retention's hourly sweep deletes a peer copy's build output with the copy. The compiler cache, the Gradle home and the copies held for peers are shown and left alone. | `r839_a_branch_in_dev_with_no_changed_file_is_merged_and_the_base_or_a_newer_one_is_not`, `r839_output_belongs_to_the_checkout_its_links_point_from_also_in_the_older_layout`, `r839_a_busy_checkout_is_in_use_whatever_else_holds`, `r839_gc_reports_every_machine_and_frees_only_the_chosen_states_never_a_busy_checkout`, `r839_a_peer_that_cannot_answer_is_named_and_the_rest_still_reports`, `r839_peer_outputs_are_listed_per_checkout_and_dropped_with_it` (daemon), `r839_a_peer_reports_each_copy_with_its_output_and_drops_all_but_one_a_job_uses` (peer), `r839_gc_shows_each_machine_and_checkout_and_how_to_free_the_rest`, `r839_gc_refuses_to_delete_in_use_or_an_unknown_state_before_asking_the_daemon` (cli), `r839_gc_reports_each_machine_and_frees_only_what_was_asked` (mcp) | any | +| R-840 | The window's Disk tab shows the `gc` report: a card per machine (`build cache 134 GiB · 98 GiB can be freed`, or `could not report: …`), one row per checkout with its state, branch (`v2 (in dev)`), last job and size on each machine that reported, and the shared caches, which are only shown. A row a job uses has no tick box. `Select merged, gone and old` ticks those rows; Free asks first (`Free 46 GiB from 1 checkout?`, naming any active or idle row whose next build starts cold) and only then sends `gc` with the ticked checkouts. The daemon frees the checkouts named, by the report's name or a full path, whatever their state, still keeping one a job uses (`a job uses it; free it once the job ends`) and naming one it does not know; `loadtable gc --free PATH` does the same from the CLI. | `r840_gc_frees_the_checkouts_named_whatever_their_state_but_never_one_a_job_uses` (daemon), `r840_gc_free_sends_each_checkout_as_a_full_path_or_the_reports_name` (cli), `r840_the_disk_tab_shows_a_card_per_machine_and_a_row_per_checkout`, `r840_ticking_frees_only_the_named_rows_and_never_offers_one_in_use`, `r840_a_failed_report_is_said_in_words` (ui disk), `r840_the_disk_tab_frees_only_after_asking_and_only_the_ticked_rows` (ui raster) | any | +| R-841 | On macOS the directories every job's `PATH` gets, and where the tool check looks, are `~/.cargo/bin` and then Homebrew's `/opt/homebrew/bin` and `/usr/local/bin`, so `pkg-config` and other Homebrew tools are found under launchd, whose `PATH` has neither. Found 2026-10-30: a p2p_chat build on the Mac failed in the video crate's build script with pkg-config not found, though Homebrew had it. | `r841_on_macos_jobs_and_the_tool_check_also_look_in_homebrews_dirs` (build_cache), `r732_the_tool_check_looks_in_the_fallback_dirs_every_job_gets`, `r733_the_jobs_own_path_replaces_the_daemons` (place) | macos | +| R-842 | The tests that start a real Seatbelt sandbox skip, saying so, when they run inside a confined job, where macOS refuses a sandbox inside another; on CI (`CI` set) such a refusal fails them instead. Found 2026-10-30: the full suite sent to the Mac failed only these two tests. | `r831_a_confined_job_resolves_its_own_folder_and_globs_inside_it`, `r831_the_folders_above_stay_unlisted_and_the_rest_of_home_closed` (confine_macos) | macos | +| R-843 | A confined peer job may write `~/.local/share/kotlin`, where the Kotlin compiler daemon keeps its run files, and nothing else under `~/.local/share`. Found 2026-10-30: Gradle builds of Kotlin on a peer could not start the Kotlin daemon and compiled in-process after a failed start. | `r843_a_peer_job_may_write_the_kotlin_daemons_folder_but_not_its_neighbours` (confine) | any | +| R-844 | After `gc` frees, each machine's total is what it keeps now: the bytes freed on that machine, here or on a peer, come off its total, so the Disk tab's card and `gc` show the smaller size at once. With nothing ticked, the Disk tab's Free button reads just `Free` and is drawn greyed out, like any button that cannot be pressed, not in red. Found 2026-10-30: after freeing 129.1 GiB on nixos the card still read 195 GiB, and an idle `Free 0 B from 0 checkouts` stood out in red. | `r844_after_freeing_each_machines_total_is_what_it_keeps_now`, `r844_with_nothing_ticked_the_free_button_names_no_size`, `r840_the_disk_tab_frees_only_after_asking_and_only_the_ticked_rows` (window) | any | diff --git a/docs/RELEASING.md b/docs/RELEASING.md index 1fb7c90..1b2e303 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -5,10 +5,12 @@ Three workflows in `.github/workflows`: - `ci.yml`: every push to main, dev or v2 and every pull request. Linux: fmt, clippy with warnings as errors, every test (with the MCP server and the desktop window). macOS: clippy with and without the iroh transport, and the confinement profile run under the real `sandbox-exec`. -- `release.yml`: a pushed `vX.Y.Z` tag. Builds `loadtable` (with the MCP server and the desktop +- `release.yml`: every push to main, and a pushed `vX.Y.Z` tag. On main it stops at once when the + workspace version is already tagged; otherwise it builds `loadtable` (with the MCP server and the desktop window) and `loadtabled` for `x86_64-unknown-linux-gnu`, `aarch64-unknown-linux-gnu` (both on - Ubuntu 22.04, glibc 2.35), `aarch64-apple-darwin` and `x86_64-apple-darwin`, and drafts the GitHub - release with them and the notes in `docs/releases/vX.Y.Z.md`. + Ubuntu 22.04, glibc 2.35), `aarch64-apple-darwin` and `x86_64-apple-darwin`, tags the commit it + built as `vX.Y.Z` (on main), and drafts the GitHub release with them and the notes in + `docs/releases/vX.Y.Z.md`. A manual run on a branch only builds. - `publish-r2.yml`: publishing that release (or a manual run with its tag). Copies the binaries to LoadTable's R2 bucket, served at `https://dl.loadtable.osvauld.com`, and moves `latest` forward. @@ -40,12 +42,14 @@ branches and tags allow only tags matching `v*`. ## Cutting a release -1. Set `[workspace.package] version` in `Cargo.toml` to the new `X.Y.Z` (and the workspace crates - in `Cargo.lock`), write the release notes in `docs/releases/vX.Y.Z.md`, and merge both to main. - The tag must match the version and the notes must exist; the workflow refuses otherwise. -2. `git tag vX.Y.Z && git push origin vX.Y.Z`. -3. Wait for Release builds; it drafts "LoadTable vX.Y.Z" with four archives, their `.sha256` files - and `SHA256SUMS`. +1. On dev, set `[workspace.package] version` in `Cargo.toml` to the new `X.Y.Z` (and the workspace + crates in `Cargo.lock`) and write the release notes in `docs/releases/vX.Y.Z.md`. +2. Merge dev to main by pull request. Nobody tags: Release builds sees a version with no tag, + builds it, tags that commit `vX.Y.Z` and drafts "LoadTable vX.Y.Z" with four archives, their + `.sha256` files and `SHA256SUMS`. Without the notes it fails and says which file to add. + Later merges with the same version find the tag and do nothing, so bump the version to release + again. +3. Wait for Release builds to finish. 4. Download an archive for each machine you have and try it: `loadtable --version`, then `loadtabled install` and a job on a spare machine. 5. Publish the draft. Approve the `release` environment if it asks. The R2 job checks the public @@ -53,6 +57,10 @@ branches and tags allow only tags matching `v*`. 6. Check the installer: `curl -fsSL https://loadtable.osvauld.com/install.sh | sh` on a clean account, then `~/.local/bin/loadtable --version`. +Pushing a tag by hand still works (`git tag vX.Y.Z && git push origin vX.Y.Z`); it must match the +version and have its notes. A wrong tag stops main from releasing that version: delete it +(`git push origin :refs/tags/vX.Y.Z`, and its draft release) and rerun Release builds on main. + Published files are never replaced: rerunning the publish for a tag uploads the same bytes or stops. To fix a bad release, release a new version. `latest` only moves forward, so republishing an old tag does not downgrade new installs. diff --git a/docs/TODO.md b/docs/TODO.md index 5bc7cb9..3020494 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -22,7 +22,8 @@ What stands between `dev` and LoadTable 0.1.0, the first public alpha. - [x] Confine jobs that come from a peer to what LoadTable owns there (R-821): Landlock on Linux, Seatbelt on macOS, `[confine]` in the machine policy. A disk survey sent to the friend's Mac walked their photos and documents. - [ ] Try R-821 on the Mac once its daemon runs a build with it: a job from the laptop builds the iOS core, and `ls ~/Pictures` from the laptop is refused. The macOS profile is type-checked and unit-tested, not yet run. (friend) - [ ] Site security page: say jobs from other machines run confined, and how `[confine]` widens it. -- [ ] `loadtable cache usage` / `cache clean` (CLI, MCP and over the peer link) that only ever walk LoadTable's own folders, so nobody needs a shell `du` on another machine. +- [x] `loadtable gc` (CLI, MCP and over the peer link, R-839) walks only LoadTable's own folders, so nobody needs a shell `du` on another machine. +- [x] The window's Disk view for gc: machine cards and one row per checkout with Free (R-840). ## 3. GitHub diff --git a/docs/releases/v0.1.4.md b/docs/releases/v0.1.4.md new file mode 100644 index 0000000..a305b51 --- /dev/null +++ b/docs/releases/v0.1.4.md @@ -0,0 +1,73 @@ +Binaries for Linux (x86-64, arm64) and macOS (Apple silicon, Intel). Install or upgrade with +`curl -fsSL https://loadtable.osvauld.com/install.sh | sh`, then restart the daemon when it is idle. + +This release also carries everything planned for 0.1.3, which was never published. + +## Freeing disk space + +`loadtable gc` shows what each checkout's build output takes on every machine, and what can go +(R-839): + +``` +Build output kept per machine + laptop 42.4 GiB kept, 0 MiB can be freed + nixos 195.1 GiB kept, 129.2 GiB can be freed +``` + +Each checkout is `in use` (a job queued or running), `active`, `idle` (no job in 14 days), +`merged` (its branch is in dev, main or master), `gone` (its working tree is deleted) or `old` +(output from an older layout). `loadtable gc --delete merged,gone,old` frees those on every +machine at once, and `--free PATH` frees one checkout whatever its state (R-840). Only build output +and the peers' synced copies are deleted: never a working tree, never a checkout a job uses, and +never the shared compiler cache or Gradle home. The MCP server has the same `gc` tool. + +The window has a **Disk** tab: a card per machine, a row per checkout with its size on each +machine, tick boxes, and a Free button that asks before it deletes. After freeing, the cards show +what is left (R-844). + +## Estimates are a range + +An estimate now runs from the fastest recent passing run to the slowest: +`~5s–56s (median 16s) from 6 passing runs on laptop`. A warm cargo build takes seconds when +nothing changed and a minute when a crate did, so one number was wrong both ways. Runs that round +alike still show one number (`~4m`). A running job says `~3s–54s left`, then `up to ~26s left` +once it is past the fastest run, and only past the slowest `longer than recent runs (5s–56s) by +30s`, instead of "over the usual" for a run still inside what recent runs took (R-837). The CLI, +the MCP tools and the window say the same. + +## Battery + +- **Peer jobs run on battery above 30% by default** (R-834). At or below it, jobs from other + machines wait, saying why ("mac is on battery at 25%"); an unknown charge counts as low. Set the + line with `power.peer_jobs_min_battery_pct` in the machine policy (0 turns it off), or hold every + peer job on battery with `power.peer_jobs_on_battery = "false"`. +- **Low battery is shown** (R-835) in that machine's `loadtable status`, beside it in the other + machines' status, and in the window, where `battery 22%` replaces the machine's role. +- **`--anywhere` passes over a peer on low battery** whenever another machine can take the job, + even a busy one; it still uses it when nothing else can run the job, such as an Apple build + (R-836). + +## Fixes + +- **Jobs on a Mac find Homebrew's tools** (R-841). Under launchd a job's `PATH` lacked + `/opt/homebrew/bin` and `/usr/local/bin`, so builds needing `pkg-config` or other Homebrew tools + failed; both are now added after `~/.cargo/bin`. +- **Kotlin builds work on a peer** (R-843). A confined job may now write `~/.local/share/kotlin`, + where the Kotlin compiler daemon keeps its run files, and nothing else under `~/.local/share`. +- **A long time left no longer overlaps its neighbour** in the window (R-838). +- The desktop window's title, top bar and app id say LoadTable (R-833). + +## Releases + +Merging to main now releases: a version with no tag yet is built, tagged and drafted, so nobody +tags by hand. + +## Upgrading + +Upgrade every machine: `gc` reports a peer only when that peer runs 0.1.4, and the battery state +reaches other machines only from 0.1.3 or later. Restart with +`systemctl --user restart loadtabled` (Linux) or `launchctl kickstart -k gui/$(id -u)/loadtabled` +(macOS) once `loadtable list --active` is empty. + +The macOS binaries are not signed or notarized. The installer's binaries run as they are; an +archive downloaded in a browser needs `xattr -d com.apple.quarantine` on each binary.