From f6bb56fd80a3ca9ec44f41fc3b88559761bac99b Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Wed, 14 Oct 2026 07:26:48 -0700 Subject: [PATCH 01/21] fix(cli): wait up to 10 minutes for a search, silently, and name what did not finish MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A plain `uffs` search failed after 60 s whenever it touched a parked drive: paging the drive back in re-reads its MFT, which takes minutes on a large HDD volume (field report 2026-10-14: `\SEC_EDGAR\RAW` on a parked 7.3 TB D: failed twice, then answered after about 3 minutes). A transient wait is not an error, and scripts that just want the result must get it. - uffs-client: the sync client's default per-request deadline goes from 60 s to 600 s. UFFS_CLIENT_TIMEOUT_SECS still overrides it (shorter for callers that want an early error, 0 = no limit). The async client (used by the MCP server) keeps its own 300 s default. New `UffsClientSync::connect_probe(deadline)` connects with an explicit short deadline for diagnostics. - uffs-cli: the readiness wait uses the same limit instead of a fixed 2 minutes. When either limit runs out, the CLI asks the daemon over a 5 s probe connection and names what had not finished: the daemon's index load (n of m drives), the drives still being brought back from parked, a drive still parked with no warm-up running, or a daemon that no longer answers. 8 tests pin each case. - Nothing is printed while waiting. The warm-retry note ("UFFS index is warming up — retrying search") and the auto-start "[uffs] connect attempt" lines are gone from stderr: a waiting command writes only its result, so a script reading stdout or stderr never sees progress lines it cannot tell apart from output. The auto-start lines become tracing events, which reach a log in hosts that install a subscriber (uffsmcp); the CLI installs none, and the daemon logs its own start. `set_quiet_autostart`, which existed only to silence those lines, is removed (no caller outside this repo). - uffs-daemon: a search that needs parked drives now logs the warm-up at info (`shard.transition`, reason promote-on-search) with each drive's elapsed time, so the wait is on record in the daemon log. The promoted-body install moved into its own method to stay under the complexity cap. Docs: the daemon manual explains the wait and the limit; the env-var tables state the 600 s default. --- crates/uffs-cli/src/commands/daemon_mgmt.rs | 8 - crates/uffs-cli/src/commands/search/mod.rs | 2 + crates/uffs-cli/src/commands/search/run.rs | 60 ++++--- .../src/commands/search/unfinished.rs | 165 ++++++++++++++++++ .../src/commands/search/unfinished_tests.rs | 136 +++++++++++++++ crates/uffs-cli/src/search_retry.rs | 13 +- crates/uffs-client/src/connect_sync.rs | 60 +++---- .../uffs-client/src/connect_sync_platform.rs | 73 ++++++-- crates/uffs-client/src/lib.rs | 4 +- crates/uffs-daemon/src/cache/policy.rs | 8 +- crates/uffs-daemon/src/index/dispatch.rs | 90 ++++++---- .../code-quality/build_codegen_policy.md | 2 +- docs/user-manual/daemon.md | 12 ++ 13 files changed, 500 insertions(+), 133 deletions(-) create mode 100644 crates/uffs-cli/src/commands/search/unfinished.rs create mode 100644 crates/uffs-cli/src/commands/search/unfinished_tests.rs diff --git a/crates/uffs-cli/src/commands/daemon_mgmt.rs b/crates/uffs-cli/src/commands/daemon_mgmt.rs index 9292e3da5..92f15b7af 100644 --- a/crates/uffs-cli/src/commands/daemon_mgmt.rs +++ b/crates/uffs-cli/src/commands/daemon_mgmt.rs @@ -48,9 +48,6 @@ struct QuietGuard; impl Drop for QuietGuard { fn drop(&mut self) { QUIET.store(false, core::sync::atomic::Ordering::Relaxed); - // Restore the thin client's auto-start retry chatter (a quiet reload - // may have (re)started the daemon, driving that connect loop). - uffs_client::connect_sync::set_quiet_autostart(false); } } @@ -65,11 +62,6 @@ impl Drop for QuietGuard { /// Exactly [`daemon`]'s errors. pub(crate) fn daemon_quiet(action: &DaemonAction) -> Result<()> { QUIET.store(true, core::sync::atomic::Ordering::Relaxed); - // Also silence the thin client's own auto-start retry chatter, which prints - // straight to stderr from a layer below this flag (the QuietGuard restores - // it). Otherwise a background reload's "[uffs] connect attempt …" bleeds - // onto the caller's spinner line. - uffs_client::connect_sync::set_quiet_autostart(true); let _guard = QuietGuard; daemon(action) } diff --git a/crates/uffs-cli/src/commands/search/mod.rs b/crates/uffs-cli/src/commands/search/mod.rs index d9478d0fc..aa74ce99b 100644 --- a/crates/uffs-cli/src/commands/search/mod.rs +++ b/crates/uffs-cli/src/commands/search/mod.rs @@ -14,5 +14,7 @@ pub(crate) mod args; pub mod dispatch; /// The `search_cli` round trip and payload-to-stdout writer. mod run; +/// The error for a search the client stopped waiting for. +mod unfinished; pub(crate) use run::run_search; diff --git a/crates/uffs-cli/src/commands/search/run.rs b/crates/uffs-cli/src/commands/search/run.rs index 62ff189f4..1f07ebf3a 100644 --- a/crates/uffs-cli/src/commands/search/run.rs +++ b/crates/uffs-cli/src/commands/search/run.rs @@ -16,9 +16,16 @@ use uffs_client::protocol::response::SearchPayload; use super::args::{extract_spawn_args, inject_no_output_for_null_stdout, resolve_out_path}; use super::dispatch::{write_aggregations, write_rows, write_rows_into}; +use super::unfinished; use crate::client_profile::{ClientProfile, OutputCost, PayloadSummary, print_client_profile}; use crate::{args, dispatch, search_retry}; +/// Readiness wait used when `UFFS_CLIENT_TIMEOUT_SECS=0` switches the client +/// limit off: one year, which is no limit in practice while keeping +/// `await_ready`'s `Instant` arithmetic (it adds up to five times this) far +/// from overflow. +const NO_LIMIT: core::time::Duration = core::time::Duration::from_hours(365 * 24); + /// Forward raw search args to the daemon via `search_cli` RPC. /// /// # Errors @@ -67,11 +74,7 @@ pub(crate) fn run_search(args: &[String]) -> Result<()> { let connect_ms = t_connect.elapsed().as_millis(); let t_ready = std::time::Instant::now(); - // 2 minutes — `from_mins` is nightly-only as of 2026-04. - let ready_timeout = core::time::Duration::from_secs(120); - client - .await_ready(ready_timeout) - .with_context(|| "Daemon did not become ready in time")?; + wait_until_ready(&mut client)?; let ready_ms = t_ready.elapsed().as_millis(); let t_search = std::time::Instant::now(); @@ -86,7 +89,11 @@ pub(crate) fn run_search(args: &[String]) -> Result<()> { let args_owned: Vec = inject_no_output_for_null_stdout(resolve_out_path(args)); let raw_response = match search_retry::search_cli_with_warm_retry(&mut client, &args_owned) { Ok(response) => response, - Err(ClientError::Timeout) => anyhow::bail!(client_timeout_message()), + Err(ClientError::Timeout) => anyhow::bail!(unfinished::timeout_message( + unfinished::Stage::Search, + uffs_client::rpc_deadline(), + &unfinished::probe_daemon(), + )), Err(err) => return Err(err).with_context(|| "Daemon search_cli failed"), }; let ipc_ms = t_search.elapsed().as_millis(); @@ -250,6 +257,25 @@ const fn payload_byte_hint(payload: &SearchPayload) -> u64 { } } +/// Wait for the daemon to finish loading its indexes. +/// +/// Uses the same client limit as the search itself (10 minutes unless +/// `UFFS_CLIENT_TIMEOUT_SECS` says otherwise): a daemon still loading is a +/// wait, not an error. `await_ready` treats it as an idle budget, renewed +/// while the load makes progress. +fn wait_until_ready(client: &mut uffs_client::connect_sync::UffsClientSync) -> Result<()> { + let limit = uffs_client::rpc_deadline(); + match client.await_ready(limit.unwrap_or(NO_LIMIT)) { + Ok(()) => Ok(()), + Err(ClientError::Timeout) => anyhow::bail!(unfinished::timeout_message( + unfinished::Stage::Ready, + limit, + &unfinished::probe_daemon(), + )), + Err(err) => Err(err).with_context(|| "Daemon did not become ready"), + } +} + /// Write the aggregation results (if any) to stdout. fn write_aggregation_values( aggregations: &[uffs_client::protocol::AggregateResultWire], @@ -271,28 +297,6 @@ fn write_aggregation_values( write_aggregations(&agg_values, args) } -/// The message for a search the client stopped waiting for. -/// -/// Honest about what happened: the daemon has not failed and is still -/// running the search — in practice it is paging parked drives back in, -/// which re-reads the MFT and can take minutes on a large HDD volume. -/// Before this the client mistook its own deadline for the "index -/// warming" transient, printed a retry note, and re-sent the search up -/// to five times. -fn client_timeout_message() -> String { - let budget = uffs_client::rpc_deadline().map_or_else( - || "the client deadline".to_owned(), - |budget| format!("the client deadline of {}s", budget.as_secs()), - ); - format!( - "the daemon did not answer within {budget} (UFFS_CLIENT_TIMEOUT_SECS).\n\ - The search is still running on the daemon. Most likely it is paging \ - parked drives back in, which re-reads the MFT and can take minutes on a \ - large HDD volume. Re-run once it has finished, or raise \ - UFFS_CLIENT_TIMEOUT_SECS for this query." - ) -} - /// Write the daemon's search payload to stdout, picking the fastest /// transport the daemon selected for this response. /// diff --git a/crates/uffs-cli/src/commands/search/unfinished.rs b/crates/uffs-cli/src/commands/search/unfinished.rs new file mode 100644 index 000000000..d28d9834a --- /dev/null +++ b/crates/uffs-cli/src/commands/search/unfinished.rs @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright (c) 2025-2026 SKY, LLC. + +//! The error for a search the client stopped waiting for. +//! +//! A plain search waits silently, for as long as the client limit allows +//! (`UFFS_CLIENT_TIMEOUT_SECS`, 10 minutes by default): a parked drive being +//! paged back in is a wait, not an error, and a script that just wants the +//! result must get it with nothing else on stdout or stderr. Only when the +//! limit runs out does the command fail, and then the error names what had +//! not finished — the daemon's own index load, a drive still being brought +//! back from parked, or nothing the daemon could report at all — by asking +//! the daemon over a fresh, short-deadline connection. + +use core::time::Duration; + +use uffs_client::connect_sync::UffsClientSync; +use uffs_client::protocol::response::{DaemonStatus, DriveInfo, ShardTier}; + +/// Deadline for the diagnostic status probe. Short on purpose: it must not +/// inherit the long search limit, or a stuck daemon would hang the error too. +const PROBE_DEADLINE: Duration = Duration::from_secs(5); + +/// Which wait ran out. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Stage { + /// Waiting for the daemon to finish loading its indexes. + Ready, + /// Waiting for the search response. + Search, +} + +/// What the daemon reported when asked after the limit ran out. +#[derive(Debug)] +pub(super) struct Probe { + /// The daemon's lifecycle status, or `None` if it did not answer. + status: Option, + /// Per-drive tier and load state, or `None` if it did not answer. + drives: Option>, +} + +/// Ask the daemon what it is still busy with. Never fails: an unanswered +/// probe is itself the finding. +pub(super) fn probe_daemon() -> Probe { + let Ok(mut client) = UffsClientSync::connect_probe(PROBE_DEADLINE) else { + return Probe { + status: None, + drives: None, + }; + }; + let status = client.status().ok().map(|resp| resp.status); + let drives = client.drives().ok().map(|resp| resp.drives); + Probe { status, drives } +} + +/// The full error text for a client timeout at `stage`. +pub(super) fn timeout_message(stage: Stage, limit: Option, probe: &Probe) -> String { + let limit_text = limit.map_or_else(|| "the client limit".to_owned(), describe_limit); + let head = match stage { + Stage::Ready => format!("the UFFS daemon did not become ready within {limit_text}"), + Stage::Search => format!("the search did not finish within {limit_text}"), + }; + let unfinished = describe_unfinished(probe); + format!( + "{head} (UFFS_CLIENT_TIMEOUT_SECS).\n\ + Not finished: {unfinished}.\n\ + Check `uffs --daemon status`, re-run the search once the daemon is \ + idle, or set UFFS_CLIENT_TIMEOUT_SECS to wait longer (0 = no limit)." + ) +} + +/// `limit` as the operator would say it: whole minutes when it is a whole +/// number of minutes, seconds otherwise. +fn describe_limit(limit: Duration) -> String { + let secs = limit.as_secs(); + if secs >= 60 && secs.is_multiple_of(60) { + let mins = secs / 60; + let unit = if mins == 1 { "minute" } else { "minutes" }; + format!("{mins} {unit}") + } else { + format!("{secs} s") + } +} + +/// Everything the daemon reported as still in progress, joined into one +/// clause. +fn describe_unfinished(probe: &Probe) -> String { + let Some(status) = &probe.status else { + return "the daemon did not answer a status check within 5 s either, \ + so it looks stuck rather than busy; its log is the next place \ + to look" + .to_owned(); + }; + + let mut parts: Vec = Vec::new(); + match status { + DaemonStatus::Loading { + drives_loaded, + drives_total, + } => parts.push(format!( + "the daemon was still loading its indexes ({drives_loaded} of {drives_total} drives \ + loaded)" + )), + DaemonStatus::Refreshing { drives } => { + parts.push(format!( + "the daemon was refreshing {}", + letters(drives.iter()) + )); + } + DaemonStatus::Ready => {} + } + + if let Some(drives) = &probe.drives { + let warming: Vec<&DriveInfo> = drives + .iter() + .filter(|drive| drive.loading == Some(true)) + .collect(); + if !warming.is_empty() { + let verb = if warming.len() == 1 { "was" } else { "were" }; + parts.push(format!( + "{} {verb} still being brought back from parked (the daemon re-reads \ + the index from disk, which takes minutes on a large HDD volume)", + letters(warming.iter().map(|drive| &drive.letter)), + )); + } + } + + if parts.is_empty() { + let parked = probe.drives.as_ref().map_or_else(Vec::new, |drives| { + drives + .iter() + .filter(|drive| matches!(drive.tier, Some(ShardTier::Parked | ShardTier::Cold))) + .map(|drive| drive.letter) + .collect::>() + }); + match parked.len() { + 0 => "the daemon answered but reported no index load or drive warm-up \ + in progress, so the search itself was still running or stuck" + .to_owned(), + count => format!( + "the daemon reported no warm-up in progress, but {} {} still parked, \ + so bringing {} back may not have started", + letters(parked.iter()), + if count == 1 { "is" } else { "are" }, + if count == 1 { "it" } else { "them" }, + ), + } + } else { + parts.join("; ") + } +} + +/// `D:` / `D: and E:` / `C:, D: and E:`. +fn letters<'a>(drives: impl Iterator) -> String { + let names: Vec = drives.map(|letter| format!("{letter}:")).collect(); + match names.split_last() { + None => String::new(), + Some((last, [])) => last.clone(), + Some((last, rest)) => format!("{} and {last}", rest.join(", ")), + } +} + +#[cfg(test)] +#[path = "unfinished_tests.rs"] +mod tests; diff --git a/crates/uffs-cli/src/commands/search/unfinished_tests.rs b/crates/uffs-cli/src/commands/search/unfinished_tests.rs new file mode 100644 index 000000000..ac44ebacb --- /dev/null +++ b/crates/uffs-cli/src/commands/search/unfinished_tests.rs @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright (c) 2025-2026 SKY, LLC. + +//! Tests for the client-timeout message: each daemon state the probe can +//! report must be named in the error, and an unanswered probe must say the +//! daemon looks stuck. + +use core::time::Duration; + +use uffs_client::protocol::response::{DaemonStatus, DriveInfo, ShardTier}; +use uffs_mft::platform::DriveLetter; + +use super::{Probe, Stage, describe_limit, letters, timeout_message}; + +fn drive(letter: char, tier: ShardTier, loading: bool) -> DriveInfo { + DriveInfo { + letter: DriveLetter::parse(letter).unwrap(), + records: 0, + source: "live".to_owned(), + tier: Some(tier), + records_when_warm: None, + loading: Some(loading), + } +} + +const TEN_MIN: Option = Some(Duration::from_secs(600)); + +#[test] +fn parked_drive_warming_is_named() { + let probe = Probe { + status: Some(DaemonStatus::Ready), + drives: Some(vec![ + drive('C', ShardTier::Warm, false), + drive('D', ShardTier::Parked, true), + ]), + }; + let msg = timeout_message(Stage::Search, TEN_MIN, &probe); + assert!( + msg.starts_with("the search did not finish within 10 minutes"), + "{msg}" + ); + assert!( + msg.contains("D: was still being brought back from parked"), + "{msg}" + ); + assert!(!msg.contains("C:"), "{msg}"); + assert!(msg.contains("UFFS_CLIENT_TIMEOUT_SECS"), "{msg}"); +} + +#[test] +fn several_warming_drives_are_listed() { + let probe = Probe { + status: Some(DaemonStatus::Ready), + drives: Some(vec![ + drive('D', ShardTier::Parked, true), + drive('M', ShardTier::Parked, true), + drive('S', ShardTier::Cold, true), + ]), + }; + let msg = timeout_message(Stage::Search, TEN_MIN, &probe); + assert!( + msg.contains("D:, M: and S: were still being brought back"), + "{msg}" + ); +} + +#[test] +fn daemon_index_load_is_named() { + let probe = Probe { + status: Some(DaemonStatus::Loading { + drives_loaded: 4, + drives_total: 6, + }), + drives: Some(Vec::new()), + }; + let msg = timeout_message(Stage::Ready, TEN_MIN, &probe); + assert!( + msg.starts_with("the UFFS daemon did not become ready within 10 minutes"), + "{msg}" + ); + assert!( + msg.contains("still loading its indexes (4 of 6 drives loaded)"), + "{msg}" + ); +} + +#[test] +fn unanswered_probe_says_stuck() { + let probe = Probe { + status: None, + drives: None, + }; + let msg = timeout_message(Stage::Search, TEN_MIN, &probe); + assert!(msg.contains("did not answer a status check"), "{msg}"); + assert!(msg.contains("stuck"), "{msg}"); +} + +#[test] +fn idle_daemon_with_parked_drive_says_warm_up_not_started() { + let probe = Probe { + status: Some(DaemonStatus::Ready), + drives: Some(vec![drive('D', ShardTier::Parked, false)]), + }; + let msg = timeout_message(Stage::Search, TEN_MIN, &probe); + assert!(msg.contains("D: is still parked"), "{msg}"); +} + +#[test] +fn idle_daemon_with_nothing_in_progress_says_search_itself() { + let probe = Probe { + status: Some(DaemonStatus::Ready), + drives: Some(vec![drive('C', ShardTier::Warm, false)]), + }; + let msg = timeout_message(Stage::Search, TEN_MIN, &probe); + assert!( + msg.contains("the search itself was still running or stuck"), + "{msg}" + ); +} + +#[test] +fn limit_is_said_in_minutes_or_seconds() { + assert_eq!(describe_limit(Duration::from_secs(600)), "10 minutes"); + assert_eq!(describe_limit(Duration::from_secs(60)), "1 minute"); + assert_eq!(describe_limit(Duration::from_secs(120)), "2 minutes"); + assert_eq!(describe_limit(Duration::from_secs(90)), "90 s"); + assert_eq!(describe_limit(Duration::from_secs(30)), "30 s"); +} + +#[test] +fn letters_join_naturally() { + let parse = |ch| DriveLetter::parse(ch).unwrap(); + assert_eq!(letters([].iter()), ""); + assert_eq!(letters([parse('D')].iter()), "D:"); + assert_eq!(letters([parse('D'), parse('E')].iter()), "D: and E:"); +} diff --git a/crates/uffs-cli/src/search_retry.rs b/crates/uffs-cli/src/search_retry.rs index 8ba1d75e2..167f0d531 100644 --- a/crates/uffs-cli/src/search_retry.rs +++ b/crates/uffs-cli/src/search_retry.rs @@ -10,7 +10,8 @@ //! 995) when the worker thread that issued it is reaped. The daemon's own read //! primitive ([`uffs-mft`'s `read_handle_at`]) already retries that transient; //! this is the client-side belt-and-suspenders so a user never sees a raw I/O -//! error for a recoverable warm-up hiccup — and gets a "warming" note instead. +//! error for a recoverable warm-up hiccup. The retry is silent: the caller +//! waits for a result and gets nothing else; the daemon logs the re-warm. //! //! Only an error the **daemon** reported is a candidate for the retry. The //! client's own per-RPC deadline (`UFFS_CLIENT_TIMEOUT_SECS`, enforced on @@ -51,8 +52,9 @@ pub(crate) fn search_cli_with_warm_retry( match client.search_cli_raw(args) { Ok(response) => return Ok(response), Err(err) if attempt < WARM_RETRY_MAX && is_index_warming_abort(&err) => { + // Silent on purpose: the caller waits for a result and gets + // nothing else; the daemon logs the re-warm it raced. attempt += 1; - report_index_warming(attempt); std::thread::sleep(WARM_RETRY_BACKOFF); } Err(err) => return Err(err), @@ -77,13 +79,6 @@ fn is_index_warming_abort(err: &ClientError) -> bool { message.contains("os error 995") || message.contains("operation has been aborted") } -/// Tell the user (on stderr, so it never pollutes stdout/CSV) that the index is -/// warming and the search is being retried. -#[expect(clippy::print_stderr, reason = "user-facing progress note, off stdout")] -fn report_index_warming(attempt: u32) { - eprintln!("UFFS index is warming up — retrying search ({attempt}/{WARM_RETRY_MAX})…"); -} - #[cfg(test)] mod tests { #![expect( diff --git a/crates/uffs-client/src/connect_sync.rs b/crates/uffs-client/src/connect_sync.rs index fa3740685..be8c54d42 100644 --- a/crates/uffs-client/src/connect_sync.rs +++ b/crates/uffs-client/src/connect_sync.rs @@ -14,7 +14,6 @@ //! | macOS/Linux | `std::os::unix::net::UnixStream` | //! | Windows | Named pipe via `std::fs::OpenOptions` (no Winsock) | -use core::sync::atomic::{AtomicBool, Ordering}; use std::io::{self, BufRead as _, BufReader, Read, Write}; use crate::connect_sync_autostart::auto_start_daemon; @@ -23,26 +22,6 @@ use crate::daemon_spawn::{ElevationPolicy, resolve_elevation_policy}; use crate::error::ClientError; use crate::protocol::response::DaemonStatus; -/// When set, the auto-start connect loop suppresses its user-facing -/// `[uffs] connect attempt …` retry chatter (default off). A caller driving a -/// daemon (re)start *behind its own progress UI* — the uninstall deep-sweep -/// coverage reload, which runs under a spinner on another thread — sets this so -/// the retry lines do not garble that display. Process-wide, best set via a -/// scoped guard on the caller's side so it never sticks. -static QUIET_AUTOSTART: AtomicBool = AtomicBool::new(false); - -/// Suppress (or restore) the auto-start connect retry chatter (the -/// `[uffs] connect attempt …` lines). The caller owns balancing this back to -/// `false` — best via a scoped guard so it never sticks. -pub fn set_quiet_autostart(quiet: bool) { - QUIET_AUTOSTART.store(quiet, Ordering::Relaxed); -} - -/// Whether the auto-start retry chatter is currently suppressed. -fn quiet_autostart() -> bool { - QUIET_AUTOSTART.load(Ordering::Relaxed) -} - /// Synchronous thin client for the UFFS daemon. /// /// One request, one response, no event loop. Phase 3b decisions: @@ -176,6 +155,21 @@ impl UffsClientSync { .map_err(|err| ClientError::ConnectionFailed(format!("No daemon is running: {err}"))) } + /// Connect without auto-starting the daemon, with an explicit + /// per-RPC `deadline` in place of `UFFS_CLIENT_TIMEOUT_SECS`. + /// + /// For a short diagnostic probe — e.g. asking the daemon what it is + /// still busy with after the main request outlived its own deadline — + /// which must not inherit the long default and hang on a stuck daemon. + /// + /// # Errors + /// + /// Returns `ConnectionFailed` if no daemon is listening. + pub fn connect_probe(deadline: core::time::Duration) -> Result { + Self::platform_connect_with_deadline(Some(deadline)) + .map_err(|err| ClientError::ConnectionFailed(format!("No daemon is running: {err}"))) + } + /// Connect to a daemon at an explicit, caller-supplied `endpoint` /// (a Unix socket path, or on Windows a named-pipe path) — /// bypassing the well-known per-user socket/pipe resolution, @@ -347,19 +341,17 @@ impl UffsClientSync { delay_ms = (delay_ms * 2).min(2000); - // Log sparingly — eprintln is intentional user-facing output - // during daemon auto-start retries (no tracing in thin client). - if !quiet_autostart() && (attempt <= 3 || attempt == max_attempts) { - #[expect( - clippy::print_stderr, - reason = "intentional user-facing retry progress" - )] - { - let sock_state = if sock.exists() { "exists" } else { "missing" }; - eprintln!( - "[uffs] connect attempt {attempt}/{max_attempts} (socket: {sock_state})" - ); - } + // Logged, never printed: a waiting command writes nothing but + // its result, so scripts reading stdout or stderr are not handed + // progress lines they have no way to tell apart from output. + if attempt <= 3 || attempt == max_attempts { + let sock_state = if sock.exists() { "exists" } else { "missing" }; + tracing::info!( + attempt, + max_attempts, + socket = sock_state, + "auto-start: daemon not reachable yet, retrying connect" + ); } } diff --git a/crates/uffs-client/src/connect_sync_platform.rs b/crates/uffs-client/src/connect_sync_platform.rs index 6944dc714..c6ed9edd1 100644 --- a/crates/uffs-client/src/connect_sync_platform.rs +++ b/crates/uffs-client/src/connect_sync_platform.rs @@ -16,24 +16,28 @@ use crate::connect_sync::UffsClientSync; use crate::daemon_ctl::socket_path; use crate::error::ClientError; -/// Default per-RPC deadline applied at connect time. +/// Default per-RPC deadline applied at connect time: 10 minutes. /// -/// Every blocking read / write on the IPC transport will fail with -/// `ClientError::Io("...timed out")` after this duration. The cap is -/// deliberately generous — a cold index load can push a single `search` -/// RPC into the tens of seconds on slow disks — but still short enough -/// that a *hung* daemon (stuck kernel-mode I/O, deadlocked worker, etc.) -/// surfaces as an actionable error instead of a CLI that never returns. +/// Every blocking read / write on the IPC transport fails with +/// `ClientError::Timeout` after this duration. A long wait is normal, +/// not an error: a search that touches a parked drive waits while the +/// daemon re-reads that drive's MFT, which takes minutes on a large HDD +/// volume, and a script that just wants the result must get it. The +/// deadline exists only so a daemon that is truly stuck (hung kernel +/// I/O, deadlocked worker) eventually surfaces as an error instead of a +/// command that never returns. It was 60 s until v0.6.46, which failed +/// ordinary parked-drive searches. /// -/// Overridable with the `UFFS_CLIENT_TIMEOUT_SECS` environment variable. -const DEFAULT_RPC_DEADLINE_SECS: u64 = 60; +/// Overridable with the `UFFS_CLIENT_TIMEOUT_SECS` environment variable +/// (a shorter limit for callers that prefer an early error). +const DEFAULT_RPC_DEADLINE_SECS: u64 = 600; /// Resolve the per-RPC deadline from env + default. /// /// * `UFFS_CLIENT_TIMEOUT_SECS=0` → disables the timeout (useful when attaching /// a debugger). /// * `UFFS_CLIENT_TIMEOUT_SECS=N` → `N`-second deadline. -/// * unset or unparseable → `DEFAULT_RPC_DEADLINE_SECS` (60 s). +/// * unset or unparseable → `DEFAULT_RPC_DEADLINE_SECS` (600 s). #[must_use] pub fn rpc_deadline() -> Option { let secs = std::env::var("UFFS_CLIENT_TIMEOUT_SECS") @@ -65,6 +69,22 @@ impl UffsClientSync { /// instance. Shares every other behavior (deadline, buffering) with /// [`Self::platform_connect`]. pub(crate) fn platform_connect_at(sock_path: &std::path::Path) -> Result { + Self::platform_connect_at_with_deadline(sock_path, rpc_deadline()) + } + + /// [`Self::platform_connect`] with an explicit per-RPC `deadline` + /// instead of the env-configured one (`None` disables it). + pub(crate) fn platform_connect_with_deadline( + deadline: Option, + ) -> Result { + Self::platform_connect_at_with_deadline(&socket_path(), deadline) + } + + /// Shared body of the Unix connect paths. + fn platform_connect_at_with_deadline( + sock_path: &std::path::Path, + deadline: Option, + ) -> Result { let stream = std::os::unix::net::UnixStream::connect(sock_path) .map_err(|err| ClientError::ConnectionFailed(err.to_string()))?; @@ -72,7 +92,6 @@ impl UffsClientSync { // either direction — blocked kernel I/O on response, or a full // pipe buffer on request — and we want both cases to surface as // a `ClientError::Io(timed out)` instead of the CLI hanging. - let deadline = rpc_deadline(); stream .set_read_timeout(deadline) .map_err(|err| ClientError::ConnectionFailed(err.to_string()))?; @@ -94,8 +113,8 @@ impl UffsClientSync { #[cfg(windows)] impl UffsClientSync { /// Construct a fresh [`crate::windows_deadline::WindowsDeadlineGuard`] - /// for this client using the env-configured deadline, or return - /// `None` if the deadline is disabled (`UFFS_CLIENT_TIMEOUT_SECS=0`). + /// for this client with `deadline`, or return `None` if the deadline + /// is disabled (`None`, e.g. `UFFS_CLIENT_TIMEOUT_SECS=0`). /// /// Called from `platform_connect` right after the pipe handle /// is opened. The guard captures the *current* thread as its @@ -103,8 +122,10 @@ impl UffsClientSync { /// the thread that will subsequently own the client — which is /// the contract today (CLI, MCP stdio loop both call `connect` /// then immediately use the result on the same thread). - fn build_deadline_guard() -> Option { - let duration = rpc_deadline()?; + fn build_deadline_guard( + deadline: Option, + ) -> Option { + let duration = deadline?; match crate::windows_deadline::WindowsDeadlineGuard::new(duration) { Ok(guard) => Some(guard), Err(err) => { @@ -162,6 +183,24 @@ impl UffsClientSync { /// daemon instance. Shares every other behavior (busy-retry, /// deadline guard) with [`Self::platform_connect`]. pub(crate) fn platform_connect_at(pipe_path: &str) -> Result { + Self::platform_connect_at_with_deadline(pipe_path, rpc_deadline()) + } + + /// [`Self::platform_connect`] with an explicit per-RPC `deadline` + /// instead of the env-configured one (`None` disables it). + pub(crate) fn platform_connect_with_deadline( + deadline: Option, + ) -> Result { + let name = crate::daemon_ctl::pipe_name() + .map_err(|err| ClientError::ConnectionFailed(err.to_string()))?; + Self::platform_connect_at_with_deadline(name.as_str(), deadline) + } + + /// Shared body of the Windows connect paths. + fn platform_connect_at_with_deadline( + pipe_path: &str, + deadline: Option, + ) -> Result { use std::fs::{File, OpenOptions}; /// `ERROR_PIPE_BUSY` — transient, retry with backoff. @@ -207,7 +246,7 @@ impl UffsClientSync { // the watchdog captures the caller's thread (not some future // thread the client might be sent to). A failure to spawn // the watchdog is non-fatal — see `build_deadline_guard`. - let deadline_guard = Self::build_deadline_guard(); + let deadline_guard = Self::build_deadline_guard(deadline); Ok(Self::from_parts_with_deadline_guard( BufReader::new(Box::new(pipe) as Box), @@ -263,7 +302,7 @@ mod rpc_deadline_tests { drop(guard); } - /// Unset env → default deadline applies. Locks the 60 s default + /// Unset env → default deadline applies. Locks the 600 s default /// in place; changing the constant should require a deliberate /// test update so nobody silently shrinks or removes it. #[test] diff --git a/crates/uffs-client/src/lib.rs b/crates/uffs-client/src/lib.rs index 8143ad524..9a5e73eed 100644 --- a/crates/uffs-client/src/lib.rs +++ b/crates/uffs-client/src/lib.rs @@ -16,7 +16,7 @@ //! `mpsc::unbounded_channel()` rate-bounded by the daemon's //! `broadcast::Sender` capacity (Phase 10d C5 verdict). //! * [`connect_sync::UffsClientSync`] (sync) — synchronous I/O on the caller -//! thread. Per-RPC deadline 60 s default, env-overridable via +//! thread. Per-RPC deadline 600 s default, env-overridable via //! `UFFS_CLIENT_TIMEOUT_SECS`; Windows path uses //! `windows_deadline::WindowsDeadlineGuard` (a `CancelSynchronousIo` //! watchdog, `#[cfg(windows)]`-only) to enforce the deadline on blocking pipe @@ -81,7 +81,7 @@ //! |---|---|---|---| //! | `CARGO_PKG_VERSION` | `string` | (set by Cargo) | Read via `env!()` for `ResponseStatus::version`. CARGO semver class. | //! | `UFFS_CLIENT_SKIP_HEALTH_CHECK` | `bool` | `false` | Skips post-connect health probe in [`daemon_ctl`]. Used by `just ship` cross-check. INTERNAL semver class. | -//! | `UFFS_CLIENT_TIMEOUT_SECS` | `int` (seconds) | `5` | Sync connect timeout override in `connect_sync_platform`. INTERNAL semver class. | +//! | `UFFS_CLIENT_TIMEOUT_SECS` | `int` (seconds) | `600` | Per-RPC deadline override (`0` disables it); sync default 600 s in `connect_sync_platform`, async default 300 s in `connect`. INTERNAL semver class. | //! | `UFFS_ELEVATE` | `string` (`auto` / `never` / `always` / `prefer`) | `auto` | Elevation policy for daemon spawn; read in `daemon_spawn::resolve_elevation_policy` (private module). INTERNAL semver class. | //! | `XDG_RUNTIME_DIR` | `path` | (XDG: `/run/user/$UID`) | Linux daemon-socket location in [`daemon_ctl`]. STANDARD semver class. | diff --git a/crates/uffs-daemon/src/cache/policy.rs b/crates/uffs-daemon/src/cache/policy.rs index b1c2cfffb..44c76dd7d 100644 --- a/crates/uffs-daemon/src/cache/policy.rs +++ b/crates/uffs-daemon/src/cache/policy.rs @@ -149,9 +149,11 @@ pub(crate) const USN_REFRESH_INTERVAL_ENV: &str = "UFFS_USN_REFRESH_INTERVAL_SEC /// loop polls `SearchFilters::cancel`) and the client gets a JSON-RPC /// `ERR_SEARCH_TIMEOUT` error — never a success-shaped empty result. /// 30 s covers a full `*.*` over ~45 M records on the reference box -/// with headroom; the budget is deliberately below the client's own -/// 60 s `UFFS_CLIENT_TIMEOUT_SECS` deadline so the error reaches the -/// client instead of the client giving up first and orphaning the scan. +/// with headroom; the budget is deliberately far below the client's own +/// `UFFS_CLIENT_TIMEOUT_SECS` deadline (600 s default) so the error reaches +/// the client instead of the client giving up first and orphaning the scan. +/// Paging parked drives back in happens before the scan and does not count +/// against this budget. pub(crate) const SEARCH_SCAN_BUDGET_SECS: u64 = 30; /// Env var that overrides [`SEARCH_SCAN_BUDGET_SECS`]. diff --git a/crates/uffs-daemon/src/index/dispatch.rs b/crates/uffs-daemon/src/index/dispatch.rs index a1416f630..de2dc7bb8 100644 --- a/crates/uffs-daemon/src/index/dispatch.rs +++ b/crates/uffs-daemon/src/index/dispatch.rs @@ -141,6 +141,15 @@ impl IndexManager { // a single [`futures::future::Shared`] body-load future per // letter, capping transient allocations at one per Parked // drive in flight. + // The client waits silently for this (it prints nothing but the + // result), so the daemon log is where a long warm-up is recorded. + tracing::info!( + target: "shard.transition", + drives = ?needs_promote, + reason = "promote-on-search", + "search needs parked drives back in memory; warming them before dispatch", + ); + let warm_started = std::time::Instant::now(); let mut load_set: futures::stream::FuturesUnordered<_> = futures::stream::FuturesUnordered::new(); for letter in needs_promote { @@ -175,38 +184,57 @@ impl IndexManager { ); continue; }; - let mut guard = self.index.write().await; - if let Some(new_registry) = guard.promote_letter(letter, body) { - // PR-f — refresh the freshly-promoted shard's - // `last_query_at_ms` to "now" so the demote-idle - // controller's first 30 s tick after this promote sees - // a fresh idle clock rather than the stale value the - // shard carried from before parking. Without this - // bump, a shard whose last_query_at_ms is more than - // `WARM_TO_PARKED_IDLE_SECS` (default 300 s) in the - // past — common when re-promoting after a long idle - // window — will be re-demoted on the very next tick, - // before `record_search_dispatch` runs at the end of - // the search to stamp the canonical timestamp. The - // observed promote-then-immediate-demote thrash on - // the v0.5.85 Windows soak (G/F/M demoted within 0.5 - // –5 s of their respective promotes) was caused - // exactly by this gap. See the regression test - // `promote_resets_idle_clock_against_thrash` in - // `crate::index::tests::idle_demote`. - // - // Mirrors the seed in `Self::add_drive` and - // `Self::replace_drive`; uses `mark_loaded_at` - // (single-store, no queries_total bump) so the - // per-drive query count stays accurate. - let now_ms = unix_now_ms(); - if let Some(shard) = new_registry.iter().find(|shard| shard.drive == letter) { - shard.stats.mark_loaded_at(now_ms); - } - *guard = Arc::new(new_registry); - drop(guard); - self.bump_index_version(); + self.install_promoted(letter, body, warm_started).await; + } + } + + /// Install a freshly loaded `body` for `letter` (the write-lock swap + /// half of [`Self::ensure_warm_for_dispatch`]) and log the warm-up's + /// duration, measured from `warm_started`. + async fn install_promoted( + &self, + letter: uffs_mft::platform::DriveLetter, + body: Arc, + warm_started: std::time::Instant, + ) { + let mut guard = self.index.write().await; + if let Some(new_registry) = guard.promote_letter(letter, body) { + // PR-f — refresh the freshly-promoted shard's + // `last_query_at_ms` to "now" so the demote-idle + // controller's first 30 s tick after this promote sees + // a fresh idle clock rather than the stale value the + // shard carried from before parking. Without this + // bump, a shard whose last_query_at_ms is more than + // `WARM_TO_PARKED_IDLE_SECS` (default 300 s) in the + // past — common when re-promoting after a long idle + // window — will be re-demoted on the very next tick, + // before `record_search_dispatch` runs at the end of + // the search to stamp the canonical timestamp. The + // observed promote-then-immediate-demote thrash on + // the v0.5.85 Windows soak (G/F/M demoted within 0.5 + // –5 s of their respective promotes) was caused + // exactly by this gap. See the regression test + // `promote_resets_idle_clock_against_thrash` in + // `crate::index::tests::idle_demote`. + // + // Mirrors the seed in `Self::add_drive` and + // `Self::replace_drive`; uses `mark_loaded_at` + // (single-store, no queries_total bump) so the + // per-drive query count stays accurate. + let now_ms = unix_now_ms(); + if let Some(shard) = new_registry.iter().find(|shard| shard.drive == letter) { + shard.stats.mark_loaded_at(now_ms); } + *guard = Arc::new(new_registry); + drop(guard); + self.bump_index_version(); + tracing::info!( + target: "shard.transition", + drive = %letter, + reason = "promote-on-search", + elapsed_ms = warm_started.elapsed().as_millis(), + "parked drive warmed for search", + ); } } diff --git a/docs/architecture/code-quality/build_codegen_policy.md b/docs/architecture/code-quality/build_codegen_policy.md index a871888b1..152eeb1eb 100644 --- a/docs/architecture/code-quality/build_codegen_policy.md +++ b/docs/architecture/code-quality/build_codegen_policy.md @@ -278,7 +278,7 @@ shapes: `env::var(LITERAL)`, `env::var_os(LITERAL)`, `env!(LITERAL)` / | Name | Type | Default | Set by | Where read | Notes | |---|---|---|---|---|---| | `UFFS_CLIENT_SKIP_HEALTH_CHECK` | `bool` | `false` | `just ship` cross-check | `crates/uffs-client/src/daemon_ctl.rs` | Skips post-connect health probe (used by `just ship` cross-check). INTERNAL class. | -| `UFFS_CLIENT_TIMEOUT_SECS` | `int` (seconds) | `5` | operator | `crates/uffs-client/src/connect_sync_platform.rs` | Sync connect timeout override. INTERNAL class. | +| `UFFS_CLIENT_TIMEOUT_SECS` | `int` (seconds) | `600` | operator | `crates/uffs-client/src/connect_sync_platform.rs` | Per-request client deadline (`0` = no limit; async client default 300). INTERNAL class. | ### 5.6 Build/release knobs (build-time, read by `scripts/ci/`) diff --git a/docs/user-manual/daemon.md b/docs/user-manual/daemon.md index ede9bcbc1..b73104c88 100644 --- a/docs/user-manual/daemon.md +++ b/docs/user-manual/daemon.md @@ -280,6 +280,18 @@ keeps the **process** alive, while the tiering ladder still parks the drives underneath it. A resident daemon left idle overnight still pays the page-in on the next first query unless it was preloaded. +**A search that needs a parked drive waits for it.** Bringing a +`Parked` or `Cold` drive back re-reads its index from disk, which can +take minutes on a large HDD volume. `uffs` waits silently while that +happens: it prints the result and nothing else, so scripts that read its +output are unaffected. The daemon log records each warm-up and how long +it took. The wait is capped by the client limit, `UFFS_CLIENT_TIMEOUT_SECS` +(default 600 s = 10 minutes; `0` means no limit). If the limit runs out, +`uffs` exits with an error that names what had not finished: the +daemon's own index load, the drives still being brought back, or a +daemon that no longer answers. Set the variable lower if you would +rather get that error sooner, for example `UFFS_CLIENT_TIMEOUT_SECS=120`. + --- ## 5 Management Commands From 2be136f841a02b3e91365e4ce628ff104d1a4576 Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Wed, 14 Oct 2026 07:36:52 -0700 Subject: [PATCH 02/21] fix(status): report a re-warming drive truthfully and count its promotions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four defects seen in one field status capture (2026-10-14) while parked D: (30.7 M records) was being paged back in for a search: - Headline read "18,228,236 of 18,228,236 records resident (100% warmed) · loading D". The re-warm denominator was set only as a side effect of a `drives` RPC landing while the body was resident; D: had been loaded at startup and parked without one, so its share was 0. Every body-carrying ShardEntry constructor (new_warm, new_warm_with_stats, new_hot_with_stats) now records the resident count itself; a regression test checks all three and that the count survives the demotion to Parked. - The drive line, the physical-drive line and the `status_drives` TIER column all said "parked" while D: was loading, because the registry keeps the pre-promote tier until the swap lands. `status -v` now shows "loading now · paging N records back in from the on-disk cache" and "loading (search waiting on it)"; `status_drives` shows TIER `loading` from a new `DriveTierStatus::loading` wire field (serde-default false, so older daemons still decode). - PROMOTIONS stayed 0 after the search promoted D:. The counter counted only Cold → Hot preloads. The excuse in the code, that a Parked promote "materialises the body from the parked bloom + trie, not a re-decrypt", is false: both paths re-read the body through DiskBodyLoader (with a USN refresh on Windows), as the CLI's own doc already admitted. It now counts every promote back from Parked or Cold, by a search or a preload, matching the user manual's existing definition ("how often this drive has been promoted back up the ladder"). Warm → Hot preloads still do not count. Tests updated to the new contract plus one for the search path. - `uffs --mcp status` reported "stale PID file, PID 61664" and left it there, so every later status repeated it. It now removes the file of a process that is gone and says it did. --- crates/uffs-cli/src/commands/daemon_status.rs | 42 ++++++++++++++- .../uffs-cli/src/commands/daemon_tiering.rs | 32 +++++------ .../src/protocol/response_tiering.rs | 12 ++++- crates/uffs-client/src/protocol/tests.rs | 2 + crates/uffs-daemon/src/cache/registry.rs | 21 ++++---- crates/uffs-daemon/src/cache/shard.rs | 10 ++-- .../src/cache/shard/drive_stats.rs | 43 ++++++++------- crates/uffs-daemon/src/cache/shard/tests.rs | 32 +++++++++++ crates/uffs-daemon/src/index/status_drives.rs | 31 ++++++----- .../uffs-daemon/src/index/tests/registry.rs | 54 +++++++++++++------ .../src/index/tests/tiering_ops.rs | 22 ++++---- crates/uffs-mcp/src/main.rs | 7 ++- 12 files changed, 215 insertions(+), 93 deletions(-) diff --git a/crates/uffs-cli/src/commands/daemon_status.rs b/crates/uffs-cli/src/commands/daemon_status.rs index f945f1dfa..d58496f7b 100644 --- a/crates/uffs-cli/src/commands/daemon_status.rs +++ b/crates/uffs-cli/src/commands/daemon_status.rs @@ -465,6 +465,12 @@ fn print_drive_line(palette: Palette, dr: &DriveInfo, memory: &[DriveMemoryInfo] } } } + Some(ShardTier::Parked | ShardTier::Cold) if dr.loading == Some(true) => { + println!( + " {glyph} {letter} {}", + palette.dim(&warming_note(dr.records_when_warm)) + ); + } Some(ShardTier::Parked) => { println!( " {glyph} {letter} {}", @@ -530,7 +536,12 @@ fn print_physical_drive_line(palette: Palette, drive: &PhysicalDrive, loaded: &[ info.tier, Some(ShardTier::Parked | ShardTier::Cold | ShardTier::Evicting) ); - let note = if parked { + let note = if parked && info.loading == Some(true) { + format!( + " \u{b7} {}", + palette.dim(&format!("{:<8} (search waiting on it)", "loading")) + ) + } else if parked { // Name the actual tier: a hibernated drive is `cold`, // not `parked`, and calling both "parked" hides which // rung of the ladder a drive is actually on. @@ -645,6 +656,17 @@ pub(crate) fn print_not_running() { let _ignored = render_not_running(false); } +/// The detail line for a demoted drive whose index is being paged back in. +fn warming_note(records_when_warm: Option) -> String { + match records_when_warm { + Some(records) if records > 0 => format!( + "loading now \u{b7} paging {} records back in from the on-disk cache", + uffs_client::format::format_number_commas(records) + ), + _ => "loading now \u{b7} paging its index back in from the on-disk cache".to_owned(), + } +} + /// Format a whole-second duration for display. fn fmt_secs(secs: u64) -> String { uffs_client::format::format_duration(core::time::Duration::from_secs(secs)) @@ -678,3 +700,21 @@ fn hit_rate_percent(hits: u64, lookups: u64) -> f64 { } (hits as f64 / lookups as f64) * 100.0_f64 } + +#[cfg(test)] +mod tests { + use super::warming_note; + + /// A drive mid-promote must read as loading, with its record count + /// when the daemon knows it — never as idle "body released". + #[test] + fn warming_note_names_the_load_and_its_size() { + assert_eq!( + warming_note(Some(30_752_882)), + "loading now \u{b7} paging 30,752,882 records back in from the on-disk cache" + ); + let unknown = warming_note(None); + assert!(unknown.starts_with("loading now"), "{unknown}"); + assert_eq!(warming_note(Some(0)), unknown); + } +} diff --git a/crates/uffs-cli/src/commands/daemon_tiering.rs b/crates/uffs-cli/src/commands/daemon_tiering.rs index c47b15a09..24c1a8df4 100644 --- a/crates/uffs-cli/src/commands/daemon_tiering.rs +++ b/crates/uffs-cli/src/commands/daemon_tiering.rs @@ -239,21 +239,16 @@ pub(crate) fn daemon_forget(drives: &[uffs_mft::platform::DriveLetter], force: b /// F cold 0 B 0.00 - - 0 /// ``` /// -/// The `PROMOTIONS` column surfaces the cumulative `Cold → Hot` -/// promotion count for each drive (Phase 9 — see +/// The `PROMOTIONS` column counts each drive's promotions back from +/// `Parked` or `Cold` since daemon start, by a search or a `preload` +/// (see /// [`uffs_client::protocol::response::DriveTierStatus::promotions_total`]). -/// It bumps once per `preload ` against a fully-evicted -/// (Cold-state) drive, when the daemon has to re-decrypt the -/// encrypted compact cache from disk. Already-Warm preloads (cheap -/// tier-marker flip — no body load) and `Parked → Hot` promotes do -/// **not** bump it. Note that `Parked → Hot` *does* pay a -/// body-decrypt cost (the parked bloom + trie are dropped and the -/// body loader is re-run, see `crates/uffs-daemon/src/index/ -/// tiering_ops.rs` source arms) — the counter still skips because -/// its contract is named for the `Cold → Hot` *tier transition*, -/// not for "promotes that paid a decrypt cost". See -/// `crates/uffs-daemon/src/cache/registry.rs::promote_letter_to_hot` -/// for the bump site (`if from_state == ShardState::Cold`). +/// Each one re-reads the drive's index from the on-disk cache; an +/// already-Warm `preload` (a tier-marker flip) does not count. +/// +/// A drive whose index is being paged back in right now shows +/// `loading` in the `TIER` column instead of its pre-promote tier +/// ([`uffs_client::protocol::response::DriveTierStatus::loading`]). #[expect(clippy::print_stdout, reason = "CLI user-facing output")] pub(crate) fn daemon_status_drives() -> Result<()> { // Read-only commands match `daemon status`'s graceful "daemon @@ -319,10 +314,17 @@ fn print_status_drive_row(drive: &DriveTierStatus) { } else { "-".to_owned() }; + // Mid-promote, the daemon still reports the pre-promote tier; say what + // is actually happening instead of an idle-looking `parked`. + let tier = if drive.loading { + "loading" + } else { + drive.tier.as_str() + }; println!( "{:<6} {:<7} {:<10} {:<7.2} {:<17} {:<16} {:>10}", drive.letter, - drive.tier, + tier, format_bytes(drive.resident_bytes), drive.query_rate_per_min, last_query, diff --git a/crates/uffs-client/src/protocol/response_tiering.rs b/crates/uffs-client/src/protocol/response_tiering.rs index 3d61df074..d9fd0c517 100644 --- a/crates/uffs-client/src/protocol/response_tiering.rs +++ b/crates/uffs-client/src/protocol/response_tiering.rs @@ -230,11 +230,21 @@ pub struct DriveTierStatus { /// queried since daemon start). #[serde(default)] pub last_query_at_ms: i64, - /// Cumulative `Cold → Hot` promotion count since daemon start. + /// Cumulative count of promotions back from `Parked` or `Cold` + /// since daemon start, by a search or a `preload` (each re-reads + /// the drive's index from disk). Daemons before v0.6.46 counted + /// only `Cold → Hot`. #[serde(default)] pub promotions_total: u64, /// Pin expiry as Unix-millis. `0` when the shard is not pinned /// (no `preload` in flight, or pin already elapsed). #[serde(default)] pub pin_until_unix_ms: i64, + /// `true` while this drive's index is being paged back in from a + /// demoted tier (a search or `preload` is waiting on it). `tier` + /// still names the pre-promote tier until the swap lands, so + /// without this a drive mid-load reads as idle `parked`. `false` + /// from daemons before v0.6.46. + #[serde(default)] + pub loading: bool, } diff --git a/crates/uffs-client/src/protocol/tests.rs b/crates/uffs-client/src/protocol/tests.rs index 02f606ed1..e70361a6e 100644 --- a/crates/uffs-client/src/protocol/tests.rs +++ b/crates/uffs-client/src/protocol/tests.rs @@ -2120,6 +2120,7 @@ fn status_drives_response_round_trip() { last_query_at_ms: 1_700_000_000_000_i64, promotions_total: 3_u64, pin_until_unix_ms: 1_700_001_800_000_i64, + loading: false, }, DriveTierStatus { letter: uffs_mft::platform::DriveLetter::D, @@ -2129,6 +2130,7 @@ fn status_drives_response_round_trip() { last_query_at_ms: 0_i64, promotions_total: 0_u64, pin_until_unix_ms: 0_i64, + loading: true, }, ], }; diff --git a/crates/uffs-daemon/src/cache/registry.rs b/crates/uffs-daemon/src/cache/registry.rs index 9425e0d66..9b6bef1b0 100644 --- a/crates/uffs-daemon/src/cache/registry.rs +++ b/crates/uffs-daemon/src/cache/registry.rs @@ -436,6 +436,9 @@ impl ShardRegistry { } let restored_mb = (body.heap_size_bytes().total / 1_048_576) as u64; let stats = Arc::clone(&old_arc.stats); + // Source is Parked or Cold (checked above): the body was re-read + // from the on-disk cache, which is what the counter measures. + stats.record_promote_from_demoted(); let drive = old_arc.drive; let new_arc = Arc::new(ShardEntry::new_warm_with_stats(drive, body, stats)); let shards: Vec> = self @@ -507,18 +510,12 @@ impl ShardRegistry { let restored_mb = (body.heap_size_bytes().total / 1_048_576) as u64; let stats = Arc::clone(&old_arc.stats); let drive = old_arc.drive; - // Phase 9: bump the Cold → Hot promotion counter only when - // the source tier was actually Cold. Already-Warm preload - // calls (where the body is in RAM and only the tier marker - // flips Warm → Hot) are not "Cold → Hot" — we want the - // wire field to count expensive re-decrypts, not cheap - // tier-marker flips. Parked → Hot is also excluded (the - // body is constructed from the existing parked_body bloom - // + trie, NOT from a re-decrypt of the on-disk encrypted - // cache); the wire docstring explicitly scopes - // `promotions_total` to Cold → Hot only. - if from_state == ShardState::Cold { - stats.record_cold_to_hot_promote(); + // Count the expensive promotes: a `Parked` or `Cold` source + // means the caller re-read the body from the on-disk cache. + // An already-Warm preload only flips the tier marker of a body + // in RAM and does not count. + if matches!(from_state, ShardState::Parked | ShardState::Cold) { + stats.record_promote_from_demoted(); } let new_arc = Arc::new(ShardEntry::new_hot_with_stats(drive, body, stats)); let shards: Vec> = self diff --git a/crates/uffs-daemon/src/cache/shard.rs b/crates/uffs-daemon/src/cache/shard.rs index 9752d88d9..c187166d4 100644 --- a/crates/uffs-daemon/src/cache/shard.rs +++ b/crates/uffs-daemon/src/cache/shard.rs @@ -254,10 +254,12 @@ impl ShardEntry { drive: uffs_mft::platform::DriveLetter, body: Arc, ) -> Self { + let stats = Arc::new(DriveStats::new()); + stats.set_last_known_records(body.records.len() as u64); Self { drive, state: AtomicU8::new(ShardState::Warm as u8), - stats: Arc::new(DriveStats::new()), + stats, body: Some(body), parked_body: None, pin_until_ms: AtomicU64::new(0), @@ -270,11 +272,12 @@ impl ShardEntry { /// is lifted into the new `Warm` `ShardEntry` so the per-drive /// query counters survive the round-trip through demote-and-back. #[must_use] - pub(crate) const fn new_warm_with_stats( + pub(crate) fn new_warm_with_stats( drive: uffs_mft::platform::DriveLetter, body: Arc, stats: Arc, ) -> Self { + stats.set_last_known_records(body.records.len() as u64); Self { drive, state: AtomicU8::new(ShardState::Warm as u8), @@ -299,11 +302,12 @@ impl ShardEntry { /// query counters and `last_query_at_ms` survive the round-trip /// through Cold/Parked → Warm → Hot. #[must_use] - pub(crate) const fn new_hot_with_stats( + pub(crate) fn new_hot_with_stats( drive: uffs_mft::platform::DriveLetter, body: Arc, stats: Arc, ) -> Self { + stats.set_last_known_records(body.records.len() as u64); Self { drive, state: AtomicU8::new(ShardState::Hot as u8), diff --git a/crates/uffs-daemon/src/cache/shard/drive_stats.rs b/crates/uffs-daemon/src/cache/shard/drive_stats.rs index e7b1e9273..9c48fa79a 100644 --- a/crates/uffs-daemon/src/cache/shard/drive_stats.rs +++ b/crates/uffs-daemon/src/cache/shard/drive_stats.rs @@ -103,17 +103,20 @@ pub(crate) struct DriveStats { /// in `cache/shard/tests.rs` seeds this directly; production /// code uses [`Self::mark_query_at`] / [`Self::mark_loaded_at`]. pub(super) last_query_at_ms: AtomicU64, - /// Cumulative count of `Cold → Hot` promotions for this drive - /// since daemon start (Phase 9 — `promotions_total` wire field). + /// Cumulative count of promotions back from a demoted tier + /// (`Parked` or `Cold`) for this drive since daemon start (the + /// `promotions_total` wire field). /// - /// Bumped from - /// [`crate::cache::registry::ShardRegistry::promote_letter_to_hot`] - /// only when the pre-promote tier was `Cold` — i.e. the operator - /// ran `uffs --daemon preload ` against a fully-evicted - /// drive and the daemon had to re-decrypt the encrypted compact - /// cache from disk. Already-Warm preload calls (where the body - /// is in RAM and only the tier marker flips Warm → Hot) do - /// **not** bump this counter — they are not "Cold → Hot". + /// Each one re-reads the drive's body from the on-disk compact + /// cache (with a USN refresh on Windows) — the expensive event this + /// counter exists to make visible. Bumped by both promote paths: + /// a search that needs the drive + /// ([`crate::cache::registry::ShardRegistry::promote_letter`]) and + /// an operator `preload` + /// ([`crate::cache::registry::ShardRegistry::promote_letter_to_hot`]). + /// A `Warm → Hot` preload only flips the tier marker of a body + /// already in RAM and does **not** count. Until v0.6.46 only + /// `Cold → Hot` counted, so a drive re-warmed by searches read 0. /// /// Surfaced via the [`StatusDrivesResponse`] wire format's /// `promotions_total` field so operators can count expensive @@ -135,7 +138,10 @@ pub(crate) struct DriveStats { /// `DriveStats` is the right home because the registry preserves /// its `Arc` across every tier transition — the count survives /// demotion for free, with no new plumbing through the - /// `new_parked` / `new_cold` rebuild constructors. Zero means + /// `new_parked` / `new_cold` rebuild constructors. Every + /// body-carrying `ShardEntry` constructor records it, so a drive + /// parked before anyone asked for status still has its denominator + /// (it used to be set only by the `drives` RPC). Zero means /// "never been warm in this daemon's lifetime", which is honest: /// a first-ever load genuinely has no denominator until it /// finishes. @@ -219,24 +225,21 @@ impl DriveStats { self.last_query_at_ms.store(now_ms, Ordering::Relaxed); } - /// Lock-free increment of the `Cold → Hot` promotion counter + /// Lock-free increment of the promotion counter (from `Parked`/`Cold`) /// (Phase 9). /// - /// Called by - /// [`crate::cache::registry::ShardRegistry::promote_letter_to_hot`] - /// after a successful Cold-source promote — i.e. the registry - /// rebuild that landed an `Arc` in `Hot` tier - /// observed `from_state == Cold` before the rebuild. + /// Called by both promote paths after a successful promote whose + /// source tier was `Parked` or `Cold`. /// /// Distinct from [`Self::mark_query_at`] / [`Self::record_query`] /// (the per-search counters) so an operator can decompose the /// shard's lifecycle into "search load" vs "explicit re-promote /// cost" without re-walking the `shard.transition` event log. - pub(crate) fn record_cold_to_hot_promote(&self) { + pub(crate) fn record_promote_from_demoted(&self) { self.promotions_total.fetch_add(1, Ordering::Relaxed); } - /// Read the cumulative `Cold → Hot` promotion count. + /// Read the cumulative count of promotions from `Parked`/`Cold`. /// /// Surfaced via the /// [`uffs_client::protocol::response::StatusDrivesResponse`] wire @@ -429,7 +432,7 @@ pub(crate) struct DriveStatsSnapshot { /// integrating, exactly as a freshly-constructed `DriveStats`. #[serde(default)] pub last_decay_queries_total: u64, - /// Cumulative `Cold → Hot` promotions. + /// Cumulative promotions back from `Parked` or `Cold`. /// See [`DriveStats::promotions_total`]. Defaults to `0` so /// pre-Phase-9 snapshots that don't carry the field deserialise /// as "never promoted from Cold" rather than rejecting — this diff --git a/crates/uffs-daemon/src/cache/shard/tests.rs b/crates/uffs-daemon/src/cache/shard/tests.rs index 34d04009d..30913c701 100644 --- a/crates/uffs-daemon/src/cache/shard/tests.rs +++ b/crates/uffs-daemon/src/cache/shard/tests.rs @@ -679,3 +679,35 @@ fn save_bound_patch_folds_delta_apply_only_keeps_it() { "the created record must survive the fold" ); } + +/// Field 2026-10-14: a drive loaded at startup and parked before any +/// `drives` RPC ran had no warm-progress denominator, so status read +/// "100% warmed" while its 30 M records were still loading. Every +/// body-carrying constructor must record the resident count itself, and +/// the count must survive the demotion to `Parked`. +#[test] +fn body_constructors_record_resident_count_for_rewarm_progress() { + let letter = uffs_mft::platform::DriveLetter::C; + let body = Arc::new(make_test_body(letter)); + let records = body.records.len() as u64; + assert!(records > 0, "fixture must hold records"); + + let warm = ShardEntry::new_warm(letter, Arc::clone(&body)); + assert_eq!(warm.stats.last_known_records(), records); + + let stats = Arc::new(DriveStats::new()); + let promoted = ShardEntry::new_warm_with_stats(letter, Arc::clone(&body), Arc::clone(&stats)); + assert_eq!(promoted.stats.last_known_records(), records); + + let fresh = Arc::new(DriveStats::new()); + let hot = ShardEntry::new_hot_with_stats(letter, Arc::clone(&body), Arc::clone(&fresh)); + assert_eq!(hot.stats.last_known_records(), records); + + let parked = ShardEntry::new_parked( + letter, + Arc::clone(&warm.stats), + Arc::new(make_test_parked_body(letter, 1)), + ); + assert!(parked.body().is_none()); + assert_eq!(parked.stats.last_known_records(), records); +} diff --git a/crates/uffs-daemon/src/index/status_drives.rs b/crates/uffs-daemon/src/index/status_drives.rs index 22c816df1..d67eeca03 100644 --- a/crates/uffs-daemon/src/index/status_drives.rs +++ b/crates/uffs-daemon/src/index/status_drives.rs @@ -49,10 +49,19 @@ impl IndexManager { // guard's lifetime, then `drop(guard)` before sorting so the // lock is released as quickly as possible (mirrors the // pattern used in [`super::drives::IndexManager::drives`]). + // Letters with a promote in flight right now (same source as the + // `drives` RPC's `loading` flag). + let loading: Vec = + self.in_flight_promotes.lock().map_or_else( + |poisoned| poisoned.into_inner().keys().copied().collect(), + |map| map.keys().copied().collect(), + ); let mut drives: Vec = { let guard = self.index.read().await; - let rows: Vec = - guard.iter().map(|shard| build_row(shard, now_ms)).collect(); + let rows: Vec = guard + .iter() + .map(|shard| build_row(shard, now_ms, loading.contains(&shard.drive))) + .collect(); drop(guard); rows }; @@ -68,8 +77,8 @@ impl IndexManager { /// Pure read-only — never mutates the shard. `now_ms` is the wall /// clock anchor for the EMA decay (passed in from the caller so /// every row in one `status_drives` response shares the same -/// timestamp). -fn build_row(shard: &ShardEntry, now_ms: u64) -> DriveTierStatus { +/// timestamp). `loading` says a promote for this drive is in flight. +fn build_row(shard: &ShardEntry, now_ms: u64, loading: bool) -> DriveTierStatus { let state = shard.state(); let tier = shard_state_label(state).to_owned(); let resident_bytes = resident_bytes_for(shard, state); @@ -89,16 +98,14 @@ fn build_row(shard: &ShardEntry, now_ms: u64) -> DriveTierStatus { resident_bytes, query_rate_per_min, last_query_at_ms: i64_from_u64_saturating(last_query_at_ms), - // Phase 9 — `promotions_total` reads the live Cold → Hot - // counter maintained by - // [`crate::cache::shard::DriveStats::record_cold_to_hot_promote`]. - // Bumped from - // [`crate::cache::registry::ShardRegistry::promote_letter_to_hot`] - // when the source tier was `Cold`; remains `0` for drives - // that have only been Warm/Hot since daemon start (no - // explicit re-promote-from-Cold event). + // `promotions_total` reads the counter maintained by + // [`crate::cache::shard::DriveStats::record_promote_from_demoted`]: + // every promote back from `Parked` or `Cold`, by a search or a + // `preload`; `0` for drives that have stayed Warm/Hot since + // daemon start. promotions_total: shard.stats.promotions_total(), pin_until_unix_ms: i64_from_u64_saturating(pin_until_unix_ms), + loading, } } diff --git a/crates/uffs-daemon/src/index/tests/registry.rs b/crates/uffs-daemon/src/index/tests/registry.rs index fb716bf53..dc00fac87 100644 --- a/crates/uffs-daemon/src/index/tests/registry.rs +++ b/crates/uffs-daemon/src/index/tests/registry.rs @@ -406,7 +406,7 @@ fn promote_letter_already_warm_returns_none() { ); } -// ── Phase 9 — Cold → Hot promotion counter on `promote_letter_to_hot` ── +// ── Promotion counter: every promote back from Parked or Cold counts ── /// `promote_letter_to_hot` from a `Cold` source bumps the per-shard /// `promotions_total` counter by 1 — the canonical @@ -451,8 +451,7 @@ fn promote_letter_to_hot_bumps_promotions_total_when_source_is_cold() { /// `promote_letter_to_hot` from a `Warm` source does NOT bump the /// counter — that's an "already in RAM, just flip the tier marker" -/// path, not the expensive Cold-source re-decrypt path the wire -/// docstring scopes the field to. +/// path, not an expensive re-read from the on-disk cache. #[test] fn promote_letter_to_hot_does_not_bump_promotions_total_when_source_is_warm() { use crate::cache::{ShardRegistry, ShardState}; @@ -482,18 +481,16 @@ fn promote_letter_to_hot_does_not_bump_promotions_total_when_source_is_warm() { c.stats.promotions_total(), 0, "Warm → Hot promote must NOT bump promotions_total \ - (only Cold → Hot counts per the wire docstring)", + (the body was already in RAM)", ); } -/// `promote_letter_to_hot` from a `Parked` source does NOT bump the -/// counter — the body is materialised from the existing -/// `parked_body` bloom + trie, NOT from a re-decrypt of the -/// on-disk encrypted compact cache, so it doesn't match the -/// "expensive re-promote" semantics `promotions_total` is meant -/// to measure. +/// `promote_letter_to_hot` from a `Parked` source bumps the counter: +/// the body the caller passes in was re-read from the on-disk cache +/// (`load_or_join_in_flight` → `DiskBodyLoader`), the same expensive +/// re-promote a `Cold` source costs. #[test] -fn promote_letter_to_hot_does_not_bump_promotions_total_when_source_is_parked() { +fn promote_letter_to_hot_bumps_promotions_total_when_source_is_parked() { use crate::cache::{ShardRegistry, ShardState}; let body_c = Arc::new(build_test_drive()); @@ -514,13 +511,40 @@ fn promote_letter_to_hot_does_not_bump_promotions_total_when_source_is_parked() assert_eq!(c.state(), ShardState::Hot); assert_eq!( c.stats.promotions_total(), - 0, - "Parked → Hot promote must NOT bump promotions_total \ - (only Cold → Hot counts; Parked source uses the live \ - parked_body, no re-decrypt cost)", + 1, + "Parked → Hot promote must bump promotions_total by exactly one", ); } +/// Field 2026-10-14: a search that re-warmed parked D: left the +/// PROMOTIONS column at 0. The search path (`promote_letter`, +/// Parked/Cold → Warm) must count like the `preload` path does. +#[test] +fn promote_letter_bumps_promotions_total_from_parked_and_cold() { + use crate::cache::{ShardRegistry, ShardState}; + + let body_c = Arc::new(build_test_drive()); + let mut reg = ShardRegistry::new().add(Arc::clone(&body_c)); + for (round, source) in [(1_u64, ShardState::Parked), (2_u64, ShardState::Cold)] { + reg = reg + .demote_letter(uffs_mft::platform::DriveLetter::C, source) + .expect("demote"); + reg = reg + .promote_letter(uffs_mft::platform::DriveLetter::C, Arc::clone(&body_c)) + .expect("promote_letter from a demoted tier"); + let c = reg + .iter() + .find(|shard| shard.drive == uffs_mft::platform::DriveLetter::C) + .expect("C present post-promote"); + assert_eq!(c.state(), ShardState::Warm); + assert_eq!( + c.stats.promotions_total(), + round, + "after promote from {source}" + ); + } +} + /// Two consecutive Cold → Hot promotes (e.g. operator runs /// `preload C` → `hibernate C` → `preload C` again) bump the /// counter to 2 — the per-drive `Arc` survives the diff --git a/crates/uffs-daemon/src/index/tests/tiering_ops.rs b/crates/uffs-daemon/src/index/tests/tiering_ops.rs index d447d7c08..9886e231e 100644 --- a/crates/uffs-daemon/src/index/tests/tiering_ops.rs +++ b/crates/uffs-daemon/src/index/tests/tiering_ops.rs @@ -573,21 +573,18 @@ async fn preload_warm_to_hot_does_not_bump_promotions_total() { assert_eq!( row.promotions_total, 0, "Warm→Hot is a tier-marker flip; promotions_total counts \ - Cold→Hot transitions only" + only promotions back from Parked or Cold" ); } -/// Phase 9 — the `Parked → Hot` source arm of `promote_letter_to_hot` -/// **does** pay a body-decrypt cost (drops the parked bloom + trie -/// and re-runs the body loader; see +/// The `Parked → Hot` source arm of `promote_letter_to_hot` pays the +/// same body re-load from the on-disk cache as `Cold → Hot` (it drops +/// the parked bloom + trie and re-runs the body loader; see /// `crate::index::tiering_ops::IndexManager::preload_drive` source -/// arms), but the `promotions_total` counter still must **not** bump -/// — the contract is named for the `Cold → Hot` tier transition, -/// not for "transitions that paid a decrypt cost". This guard test -/// pins that distinction so a future refactor that consolidates the -/// Cold/Parked source arms can't silently broaden the bump. +/// arms), so it counts in `promotions_total` exactly once. Until +/// v0.6.46 it did not, and a drive re-warmed from Parked read 0. #[tokio::test] -async fn preload_parked_to_hot_does_not_bump_promotions_total() { +async fn preload_parked_to_hot_bumps_promotions_total_once() { let (tx, _rx) = crate::events::event_channel(); let body = Arc::new(build_test_drive()); let loader = Arc::new(FixedBodyLoader { @@ -616,8 +613,7 @@ async fn preload_parked_to_hot_does_not_bump_promotions_total() { panic!("expected exactly 1 drive; got {}", response.drives.len()); }; assert_eq!( - row.promotions_total, 0, - "Parked→Hot pays a decrypt cost but is NOT a Cold→Hot \ - transition; promotions_total must stay at 0" + row.promotions_total, 1, + "Parked→Hot re-loads the body from disk; promotions_total must be 1" ); } diff --git a/crates/uffs-mcp/src/main.rs b/crates/uffs-mcp/src/main.rs index 2f1ee5f52..b736e179e 100644 --- a/crates/uffs-mcp/src/main.rs +++ b/crates/uffs-mcp/src/main.rs @@ -498,8 +498,13 @@ async fn mcp_status() -> Result<()> { } } } else { + // The process that wrote the file is gone (killed, or its + // host exited without letting it clean up). Remove the + // leftover so it is not reported on every later status, + // and say so. + uffs_client::mcp_pid::remove_mcp_pid_file(); println!( - "MCP server: not running (stale PID file, PID {})", + "MCP server: not running (removed the stale PID file left by PID {})", info.pid ); } From b8cc08d9599b2c6e9f0776fac88f1964677addb8 Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:26:28 -0700 Subject: [PATCH 03/21] fix(broker): keep a client that connected before ConnectNamedPipe `wait_for_client` treated ERROR_PIPE_CONNECTED as success by comparing `windows::core::Error::code().0` against 535. `code()` is an HRESULT (0x80070217), so the check never matched: a client that opened the instance in the gap between CreateNamedPipeW and ConnectNamedPipe was disconnected and read ERROR_PIPE_NOT_CONNECTED (233). That gap is exactly where a second client lands when two arrive together. Field capture 2026-10-15 03:43:42Z: two uffs-content producers asked for a VSS lease 125 us apart; the Broker log shows "ConnectNamedPipe failed: There is a process on other end of the pipe. (0x80070217)" and the second producer failed with os error 233. The fix compares against ERROR_PIPE_CONNECTED.to_hresult(). The check is shared by the handle pipe and the Snapshot Manager pipe, and the VSS helper's control pipe now uses it too instead of its own copy of the same bug (the helper is launched before the Broker waits for it, so a fast helper hit it). The wait moved to broker/pipe.rs, with tests. Same bug class, fixed here too: - uffs-security is_pid_alive compared the HRESULT against the bare ERROR_ACCESS_DENIED (5), so a live process we may not open read as dead and its runtime directory could be swept as an orphan. - the daemon's Broker client opened the pipe once with no retry; a busy pipe (231) for the microseconds between the Broker accepting one client and creating the next instance sent the drive to a direct open a non-elevated daemon cannot make. It now retries only ERROR_PIPE_BUSY, up to 5 opens (10-80 ms backoff), before anything is sent. --- crates/uffs-broker/src/broker.rs | 22 +----- crates/uffs-broker/src/broker/pipe.rs | 68 ++++++++++++++++++- .../src/broker/snapshot_manager/vss_helper.rs | 19 ++---- crates/uffs-daemon/src/broker_client.rs | 38 +++++++++-- crates/uffs-security/src/runtime_dir.rs | 15 ++-- 5 files changed, 113 insertions(+), 49 deletions(-) diff --git a/crates/uffs-broker/src/broker.rs b/crates/uffs-broker/src/broker.rs index bd1145ef0..67d100a14 100644 --- a/crates/uffs-broker/src/broker.rs +++ b/crates/uffs-broker/src/broker.rs @@ -66,7 +66,7 @@ use owned_handle::OwnedHandle; #[path = "broker/pipe.rs"] mod pipe; #[cfg(windows)] -use pipe::create_broker_pipe; +use pipe::{create_broker_pipe, wait_for_client}; // Snapshot Manager: the Coordinator-facing VSS-lease API (a separate pipe // from the daemon's MFT-handle channel above), spawned from @@ -456,26 +456,6 @@ fn audit_log(action: &str, pid: u32, exe: Option<&str>, drive: Option, det // ── D7.3: Named Pipe Operations ───────────────────────────────────────── -/// Wait for a client to connect to the pipe. -#[cfg(windows)] -#[expect(unsafe_code, reason = "ConnectNamedPipe is an FFI call")] -fn wait_for_client(pipe: windows::Win32::Foundation::HANDLE) -> anyhow::Result<()> { - use windows::Win32::System::Pipes::ConnectNamedPipe; - - // SAFETY: `pipe` is an owned valid pipe HANDLE (HANDLE is Copy); the - // second argument is None (synchronous wait). - let result = unsafe { ConnectNamedPipe(pipe, None) }; - - if let Err(win_err) = result { - // ERROR_PIPE_CONNECTED (535) means client connected before we called - // ConnectNamedPipe — that's OK - if win_err.code().0 != 535_i32 { - anyhow::bail!("ConnectNamedPipe failed: {win_err}"); - } - } - Ok(()) -} - /// Disconnect any connected client from a pipe instance (the handle itself is /// closed separately — by [`close_pipe`] or the worker's [`OwnedHandle`] drop). #[cfg(windows)] diff --git a/crates/uffs-broker/src/broker/pipe.rs b/crates/uffs-broker/src/broker/pipe.rs index 617ee2724..82a4a2d97 100644 --- a/crates/uffs-broker/src/broker/pipe.rs +++ b/crates/uffs-broker/src/broker/pipe.rs @@ -1,13 +1,14 @@ // SPDX-License-Identifier: MPL-2.0 // Copyright (c) 2025-2026 SKY, LLC. -//! Broker named-pipe creation with its SDDL security descriptor. +//! Broker named-pipe creation with its SDDL security descriptor, and the +//! accept wait every Broker pipe shares. //! //! Split out of `broker.rs` to keep that file under the 800-LOC ceiling //! (alongside `service.rs`, `process_handle.rs`, `authenticode.rs`, //! `owned_handle.rs`). The accept loop, connection handling, and pipe I/O -//! stay in `broker.rs`; this module owns only *creating* a listening instance -//! with the right ACL + integrity label. +//! stay in `broker.rs`; this module owns *creating* a listening instance +//! with the right ACL + integrity label and *waiting* for its client. #[cfg(windows)] use uffs_broker_protocol::PIPE_NAME; @@ -161,3 +162,64 @@ impl Drop for PipeSecurity { } } } + +/// Wait for a client to connect to the pipe. +/// +/// Shared by every pipe the Broker serves: the daemon-facing handle pipe, +/// the Snapshot Manager pipe and the VSS helper's control pipe. +#[cfg(windows)] +#[expect(unsafe_code, reason = "ConnectNamedPipe is an FFI call")] +pub(super) fn wait_for_client(pipe: windows::Win32::Foundation::HANDLE) -> anyhow::Result<()> { + use windows::Win32::System::Pipes::ConnectNamedPipe; + + // SAFETY: `pipe` is an owned valid pipe HANDLE (HANDLE is Copy); the + // second argument is None (synchronous wait). + let result = unsafe { ConnectNamedPipe(pipe, None) }; + + match result { + Ok(()) => Ok(()), + Err(win_err) if client_connected_first(&win_err) => Ok(()), + Err(win_err) => anyhow::bail!("ConnectNamedPipe failed: {win_err}"), + } +} + +/// Whether a `ConnectNamedPipe` error is `ERROR_PIPE_CONNECTED`: the client +/// opened the instance before the call ran, so the pipe IS connected. +/// +/// `windows::core::Error::code()` is an HRESULT (`0x80070217`), never the +/// bare Win32 code 535. Comparing against `535` never matched, so this +/// success was treated as a failure and the instance was disconnected under +/// a client that had connected fine. It happens whenever a client arrives in +/// the gap between `CreateNamedPipeW` and `ConnectNamedPipe` — two +/// Coordinators asking for a lease in the same instant, or a VSS helper that +/// connects before the Broker starts waiting for it — and the dropped client +/// read `ERROR_PIPE_NOT_CONNECTED` (233). Field capture 2026-10-15 +/// 03:43:42Z, Broker log: `ConnectNamedPipe failed: There is a process on +/// other end of the pipe. (0x80070217)`. +#[cfg(windows)] +fn client_connected_first(err: &windows::core::Error) -> bool { + err.code() == windows::Win32::Foundation::ERROR_PIPE_CONNECTED.to_hresult() +} + +#[cfg(test)] +mod tests { + #[cfg(windows)] + use windows::Win32::Foundation::{ERROR_NO_DATA, ERROR_PIPE_CONNECTED}; + + #[cfg(windows)] + use super::client_connected_first; + + #[cfg(windows)] + #[test] + fn pipe_connected_error_counts_as_a_connected_client() { + let err = windows::core::Error::from(ERROR_PIPE_CONNECTED.to_hresult()); + assert!(client_connected_first(&err), "{err}"); + } + + #[cfg(windows)] + #[test] + fn other_connect_errors_are_failures() { + let err = windows::core::Error::from(ERROR_NO_DATA.to_hresult()); + assert!(!client_connected_first(&err), "{err}"); + } +} diff --git a/crates/uffs-broker/src/broker/snapshot_manager/vss_helper.rs b/crates/uffs-broker/src/broker/snapshot_manager/vss_helper.rs index 8e96c39fe..b6ea4fab1 100644 --- a/crates/uffs-broker/src/broker/snapshot_manager/vss_helper.rs +++ b/crates/uffs-broker/src/broker/snapshot_manager/vss_helper.rs @@ -37,7 +37,7 @@ use windows::Win32::System::JobObjects::{ SetInformationJobObject, }; use windows::Win32::System::Pipes::{ - ConnectNamedPipe, CreateNamedPipeW, PIPE_READMODE_BYTE, PIPE_TYPE_BYTE, PIPE_WAIT, + CreateNamedPipeW, PIPE_READMODE_BYTE, PIPE_TYPE_BYTE, PIPE_WAIT, }; use windows::Win32::System::Threading::{ CREATE_SUSPENDED, CreateProcessW, PROCESS_INFORMATION, ResumeThread, STARTUPINFOW, @@ -535,19 +535,12 @@ fn create_control_pipe(pipe_name: &str) -> anyhow::Result { } /// Block until the helper connects to `pipe_handle`. -#[expect(unsafe_code, reason = "ConnectNamedPipe is an FFI call")] +/// +/// The helper is launched before this wait starts, so it can connect first; +/// the shared Broker accept wait treats that (`ERROR_PIPE_CONNECTED`) as the +/// success it is. fn connect_pipe(pipe_handle: HANDLE) -> anyhow::Result<()> { - // SAFETY: `pipe_handle` is a valid, freshly created pipe instance - // HANDLE; `None` requests a synchronous (blocking) connect wait. - let result = unsafe { ConnectNamedPipe(pipe_handle, None) }; - if let Err(win_err) = result { - // ERROR_PIPE_CONNECTED means the client connected before this - // call ran — not an error. - if win_err.code().0 != 535_i32 { - anyhow::bail!("ConnectNamedPipe failed: {win_err}"); - } - } - Ok(()) + super::super::wait_for_client(pipe_handle) } /// Locate `uffs-vss-requestor.exe`. diff --git a/crates/uffs-daemon/src/broker_client.rs b/crates/uffs-daemon/src/broker_client.rs index 8e5e372ba..7b75bd4e7 100644 --- a/crates/uffs-daemon/src/broker_client.rs +++ b/crates/uffs-daemon/src/broker_client.rs @@ -57,11 +57,7 @@ fn broker_pipe_round_trip( use uffs_broker_protocol::{HandleRequest, PIPE_NAME, RESPONSE_WIRE_LEN}; tracing::debug!(drive = %drive_letter, pipe = PIPE_NAME, "Opening broker pipe"); - let mut pipe = std::fs::OpenOptions::new() - .read(true) - .write(true) - .open(std::path::Path::new(PIPE_NAME)) - .map_err(|err| anyhow::anyhow!("opening broker pipe: {err}"))?; + let mut pipe = open_broker_pipe(PIPE_NAME)?; tracing::debug!(drive = %drive_letter, "Broker pipe opened; sending request"); let request_bytes = HandleRequest { @@ -77,6 +73,38 @@ fn broker_pipe_round_trip( Ok(response) } +/// Open `pipe_name`, retrying only `ERROR_PIPE_BUSY` (231). +/// +/// The Broker keeps one listening instance and creates the next right +/// after a client connects, so for a few microseconds every instance can be +/// taken — by a Coordinator asking for a VSS lease, or by a second daemon +/// overlapping this one across a restart. Retrying is safe because nothing +/// has been sent yet; without it the drive falls back to a direct open, +/// which a non-elevated daemon cannot make. Backoff 10, 20, 40, 80 ms. +#[cfg(windows)] +fn open_broker_pipe(pipe_name: &str) -> anyhow::Result { + /// `ERROR_PIPE_BUSY`: every instance is connected for the moment. + const ERROR_PIPE_BUSY: i32 = 231; + /// Open attempts before the busy pipe counts as a failure. + const OPEN_ATTEMPTS: u32 = 5; + + let mut attempt = 1_u32; + loop { + match std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(std::path::Path::new(pipe_name)) + { + Ok(pipe) => return Ok(pipe), + Err(err) if err.raw_os_error() == Some(ERROR_PIPE_BUSY) && attempt < OPEN_ATTEMPTS => { + std::thread::sleep(core::time::Duration::from_millis(10_u64 << (attempt - 1))); + attempt += 1; + } + Err(err) => anyhow::bail!("opening broker pipe: {err}"), + } + } +} + /// Parse and interpret the broker's 9-byte response into a handle value. /// /// Split out of [`request_volume_handle`] to keep that function under the diff --git a/crates/uffs-security/src/runtime_dir.rs b/crates/uffs-security/src/runtime_dir.rs index a7949c704..ca0d35560 100644 --- a/crates/uffs-security/src/runtime_dir.rs +++ b/crates/uffs-security/src/runtime_dir.rs @@ -504,13 +504,14 @@ mod windows_impl { true } Err(err) => { - // ERROR_ACCESS_DENIED (5) = pid exists but ACL - // forbids us. Treat as alive (don't stomp on a - // protected or other-user process). - err.code().0 - == windows::Win32::Foundation::ERROR_ACCESS_DENIED - .0 - .cast_signed() + // ERROR_ACCESS_DENIED = pid exists but ACL forbids us. + // Treat as alive (don't stomp on a protected or + // other-user process). `code()` is an HRESULT + // (`0x80070005`), so compare against the HRESULT form: + // the bare Win32 value 5 never matched, and a live + // process we may not open read as dead, its runtime + // directory swept as an orphan. + err.code() == windows::Win32::Foundation::ERROR_ACCESS_DENIED.to_hresult() } } } From e5a9c1af0bea601365817812d6643cb0521c1194 Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:27:06 -0700 Subject: [PATCH 04/21] refactor(vss): one shared VSS HRESULT table in uffs-broker-protocol The retryable-HRESULT table lived in uffs-vss-requestor and, separately, in uffs-products' Content Coordinator (RETRYABLE_VSS_HRESULTS), which does not depend on uffs-vss-requestor. The two had to be edited together or drift silently; the 2026-10-13 transposition fix had to be made twice. The table, its named vsserror.h constants, is_retryable, failure_hint and their header-pinning tests move unchanged to uffs_broker_protocol::vss_hresult, the crate both sides already share (the HRESULT crosses the wire in the Snapshot Manager's Error response). uffs-vss-requestor uses it from there; the Coordinator can drop its copy once its pin carries this commit. --- Cargo.lock | 1 + Cargo.toml | 4 +- crates/uffs-broker-protocol/src/lib.rs | 5 ++ .../src/vss_hresult.rs} | 51 ++++++++++++------- crates/uffs-vss-requestor/Cargo.toml | 4 ++ crates/uffs-vss-requestor/src/main.rs | 4 -- crates/uffs-vss-requestor/src/run.rs | 15 +++--- docs/architecture/crate-graph.md | 2 +- 8 files changed, 54 insertions(+), 32 deletions(-) rename crates/{uffs-vss-requestor/src/hresult.rs => uffs-broker-protocol/src/vss_hresult.rs} (81%) diff --git a/Cargo.lock b/Cargo.lock index 9a3fc1d24..9c591b002 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4721,6 +4721,7 @@ dependencies = [ "cc", "serde", "serde_json", + "uffs-broker-protocol", "uffs-version", "windows", ] diff --git a/Cargo.toml b/Cargo.toml index c51a9f29a..96e253c3b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -149,7 +149,9 @@ uffs-client = { path = "crates/uffs-client", version = "0.6.45" } # `uffs-daemon::broker_client` (the handle consumer). Pure-logic # Layer-0 lib — cross-platform tests run on every CI lane. Added in # F5 (issue #205) so neither side duplicates `BROKER_PIPE_NAME` / -# wire-format byte literals. +# wire-format byte literals. Also holds the VSS `HRESULT` retry table +# (`vss_hresult`) shared by `uffs-vss-requestor` and uffs-products' +# Content Coordinator. uffs-broker-protocol = { path = "crates/uffs-broker-protocol", version = "0.6.45" } # `uffs-winsvc` — native Windows service control (SCM query/start/stop) + # the non-connecting broker-pipe readiness probe. Layer-0 leaf: its only diff --git a/crates/uffs-broker-protocol/src/lib.rs b/crates/uffs-broker-protocol/src/lib.rs index 6f01494e0..5ac7d5e77 100644 --- a/crates/uffs-broker-protocol/src/lib.rs +++ b/crates/uffs-broker-protocol/src/lib.rs @@ -4,6 +4,10 @@ //! Wire protocol between `uffs-broker` (elevated handle vendor) and //! `uffs-daemon::broker_client` (handle consumer). //! +//! Also the Snapshot Manager messages ([`snapshot_manager`]) and the VSS +//! `HRESULT` table ([`vss_hresult`]) every component classifies a +//! snapshot failure by. +//! //! This crate is a dedicated cross-platform Layer-0 library — pure //! byte-shuffling, no Windows FFI, no I/O. Both sides of the wire //! (the broker binary on Windows, the daemon-side client also on @@ -49,6 +53,7 @@ use thiserror::Error; pub mod snapshot_manager; +pub mod vss_hresult; /// Named-pipe path the broker listens on. /// diff --git a/crates/uffs-vss-requestor/src/hresult.rs b/crates/uffs-broker-protocol/src/vss_hresult.rs similarity index 81% rename from crates/uffs-vss-requestor/src/hresult.rs rename to crates/uffs-broker-protocol/src/vss_hresult.rs index 33ef14d02..7e48f74c2 100644 --- a/crates/uffs-vss-requestor/src/hresult.rs +++ b/crates/uffs-broker-protocol/src/vss_hresult.rs @@ -1,8 +1,17 @@ // SPDX-License-Identifier: MPL-2.0 // Copyright (c) 2025-2026 SKY, LLC. -//! The VSS `HRESULT`s this requestor treats specially, each pinned to -//! its `vsserror.h` name. +//! The VSS `HRESULT`s the snapshot pipeline treats specially, each +//! pinned to its `vsserror.h` name — the one table every UFFS component +//! classifies a VSS failure by. +//! +//! Lives in the protocol crate because the `HRESULT` crosses the wire: +//! `uffs-vss-requestor` decides here whether to retry inside the Broker, +//! and the Broker's `SnapshotManagerResponse::Error` hands the same code +//! to the Content Coordinator (uffs-products' `uffs-content`), which +//! decides whether its own consumer should retry. The two used to keep +//! separate copies of this table, which had to be edited together or +//! drifted silently; one shared table removes the need. //! //! Platform-independent on purpose: the code ↔ name mapping is plain //! data, so its tests run on every host instead of only on Windows CI. @@ -23,32 +32,36 @@ const fn hr(code: u32) -> i32 { } /// `VSS_E_PROVIDER_VETO` — the provider could not service the request -/// right now; one of the errors Microsoft's requestor guidance says to -/// retry. Observed on real hardware from back-to-back snapshot +/// right now. +/// +/// One of the errors Microsoft's requestor guidance says to retry. +/// Observed on real hardware from back-to-back snapshot /// create/release cycles on the same volume. -pub(crate) const VSS_E_PROVIDER_VETO: i32 = hr(0x8004_2306); +pub const VSS_E_PROVIDER_VETO: i32 = hr(0x8004_2306); /// `VSS_E_FLUSH_WRITES_TIMEOUT` — the flush phase did not finish in /// time; the classic symptom of a busy volume. -pub(crate) const VSS_E_FLUSH_WRITES_TIMEOUT: i32 = hr(0x8004_2313); +pub const VSS_E_FLUSH_WRITES_TIMEOUT: i32 = hr(0x8004_2313); /// `VSS_E_HOLD_WRITES_TIMEOUT` — the write-freeze phase did not finish /// in time; the other half of snapshot contention. -pub(crate) const VSS_E_HOLD_WRITES_TIMEOUT: i32 = hr(0x8004_2314); +pub const VSS_E_HOLD_WRITES_TIMEOUT: i32 = hr(0x8004_2314); /// `VSS_E_SNAPSHOT_SET_IN_PROGRESS` — another shadow-copy operation is /// still in flight on this volume. -pub(crate) const VSS_E_SNAPSHOT_SET_IN_PROGRESS: i32 = hr(0x8004_2316); +pub const VSS_E_SNAPSHOT_SET_IN_PROGRESS: i32 = hr(0x8004_2316); /// `VSS_E_MAXIMUM_NUMBER_OF_SNAPSHOTS_REACHED` — the volume already -/// holds the maximum number of shadow copies. Not transient: retrying +/// holds the maximum number of shadow copies. +/// +/// Not transient: retrying /// cannot help while the limit is reached; deleting an old shadow copy /// does. See [`failure_hint`]. -pub(crate) const VSS_E_MAXIMUM_NUMBER_OF_SNAPSHOTS_REACHED: i32 = hr(0x8004_2317); +pub const VSS_E_MAXIMUM_NUMBER_OF_SNAPSHOTS_REACHED: i32 = hr(0x8004_2317); /// `VSS_E_WRITER_INFRASTRUCTURE` — the VSS writer infrastructure is not /// operating properly. Not a transient a retry can clear. -pub(crate) const VSS_E_WRITER_INFRASTRUCTURE: i32 = hr(0x8004_2318); +pub const VSS_E_WRITER_INFRASTRUCTURE: i32 = hr(0x8004_2318); /// `VSS_E_WRITERERROR_TIMEOUT` — a writer timed out; retryable by /// definition. -pub(crate) const VSS_E_WRITERERROR_TIMEOUT: i32 = hr(0x8004_23F2); +pub const VSS_E_WRITERERROR_TIMEOUT: i32 = hr(0x8004_23F2); /// `VSS_E_WRITERERROR_RETRYABLE` — a writer reported a retryable error. -pub(crate) const VSS_E_WRITERERROR_RETRYABLE: i32 = hr(0x8004_23F3); +pub const VSS_E_WRITERERROR_RETRYABLE: i32 = hr(0x8004_23F3); /// `HRESULT`s a fresh attempt is expected to clear once the underlying /// contention passes. @@ -62,8 +75,7 @@ pub(crate) const VSS_E_WRITERERROR_RETRYABLE: i32 = hr(0x8004_23F3); /// requestor runs `VSS_CTX_FILE_SHARE_BACKUP` with no writer /// coordination (see `native/vss_shim.cpp`), so writers should never be /// in play, but both codes are retryable by definition if they appear. -/// Matches uffs-products' orchestrator table code for code. -pub(crate) const RETRYABLE: [i32; 6] = [ +pub const RETRYABLE: [i32; 6] = [ VSS_E_PROVIDER_VETO, VSS_E_FLUSH_WRITES_TIMEOUT, VSS_E_HOLD_WRITES_TIMEOUT, @@ -73,7 +85,8 @@ pub(crate) const RETRYABLE: [i32; 6] = [ ]; /// `true` when `hresult` is one of the [`RETRYABLE`] transients. -pub(crate) fn is_retryable(hresult: i32) -> bool { +#[must_use] +pub fn is_retryable(hresult: i32) -> bool { RETRYABLE.contains(&hresult) } @@ -82,7 +95,8 @@ pub(crate) fn is_retryable(hresult: i32) -> bool { /// Appended to the failure message the Broker receives, so a caller /// that can act on it is told what to do instead of seeing a generic /// failure. `None` for every code without a specific remedy. -pub(crate) const fn failure_hint(hresult: i32) -> Option<&'static str> { +#[must_use] +pub const fn failure_hint(hresult: i32) -> Option<&'static str> { match hresult { VSS_E_MAXIMUM_NUMBER_OF_SNAPSHOTS_REACHED => Some( "the volume already holds the maximum number of shadow copies; delete an old one \ @@ -163,8 +177,7 @@ mod tests { assert!(!is_retryable(VSS_E_WRITER_INFRASTRUCTURE)); } - /// The retry set is exactly the six transients, matching - /// uffs-products' orchestrator table. + /// The retry set is exactly the six transients. #[test] fn retry_set_is_exactly_the_six_transients() { let mut codes: Vec = RETRYABLE.iter().map(|code| code.cast_unsigned()).collect(); diff --git a/crates/uffs-vss-requestor/Cargo.toml b/crates/uffs-vss-requestor/Cargo.toml index af7dafb66..25f82510d 100644 --- a/crates/uffs-vss-requestor/Cargo.toml +++ b/crates/uffs-vss-requestor/Cargo.toml @@ -54,6 +54,10 @@ uffs-version.workspace = true # for the identical rationale (F5 / issue #205). [target.'cfg(windows)'.dependencies] anyhow.workspace = true +# The VSS HRESULT table (`vss_hresult`): which codes to retry, which carry +# an operator remedy. Shared with the Content Coordinator, which reads the +# same code off the Broker's error response. +uffs-broker-protocol.workspace = true windows.workspace = true serde = { workspace = true, features = ["derive"] } serde_json.workspace = true diff --git a/crates/uffs-vss-requestor/src/main.rs b/crates/uffs-vss-requestor/src/main.rs index 0e7635e02..6d781835a 100644 --- a/crates/uffs-vss-requestor/src/main.rs +++ b/crates/uffs-vss-requestor/src/main.rs @@ -25,10 +25,6 @@ #[cfg(windows)] mod ffi; -// Plain data (codes + names); compiled for tests on every host so the -// code-to-name pins run outside Windows CI too. -#[cfg(any(windows, test))] -mod hresult; #[cfg(windows)] mod pipe; #[cfg(windows)] diff --git a/crates/uffs-vss-requestor/src/run.rs b/crates/uffs-vss-requestor/src/run.rs index c1ed18560..792b14a63 100644 --- a/crates/uffs-vss-requestor/src/run.rs +++ b/crates/uffs-vss-requestor/src/run.rs @@ -83,11 +83,11 @@ const RETRY_BACKOFF_BASE: core::time::Duration = core::time::Duration::from_secs /// Create a `VSS_CTX_FILE_SHARE_BACKUP` snapshot of `volume_path`, /// retrying with backoff on the small set of `HRESULT`s VSS documents as -/// transient ([`crate::hresult::RETRYABLE`]). Each attempt is a fully fresh -/// [`VssSnapshotSession::create`] call — a brand-new `IVssBackupComponents` -/// session — which is exactly what Microsoft's own guidance requires: -/// retrying a transient VSS failure means restarting the whole sequence, -/// never resuming mid-sequence. +/// transient ([`uffs_broker_protocol::vss_hresult::RETRYABLE`]). Each attempt +/// is a fully fresh [`VssSnapshotSession::create`] call — a brand-new +/// `IVssBackupComponents` session — which is exactly what Microsoft's own +/// guidance requires: retrying a transient VSS failure means restarting the +/// whole sequence, never resuming mid-sequence. /// /// # Errors /// Returns the last attempt's [`VssRequestError`] if every attempt @@ -102,7 +102,8 @@ fn create_snapshot_with_retry( match VssSnapshotSession::create(volume_path) { Ok(created) => return Ok(created), Err(err) - if attempt < MAX_SNAPSHOT_ATTEMPTS && crate::hresult::is_retryable(err.hresult) => + if attempt < MAX_SNAPSHOT_ATTEMPTS + && uffs_broker_protocol::vss_hresult::is_retryable(err.hresult) => { debug_log(&format!( "snapshot creation attempt {attempt}/{MAX_SNAPSHOT_ATTEMPTS} failed with \ @@ -214,7 +215,7 @@ pub(crate) fn run() -> anyhow::Result<()> { // A permanent failure with a known remedy (e.g. the volume's // shadow-copy limit) carries it, so the Broker's caller can act // instead of seeing a generic failure. - let message = match crate::hresult::failure_hint(hresult) { + let message = match uffs_broker_protocol::vss_hresult::failure_hint(hresult) { Some(hint) => format!("{} — {hint}", err.message), None => err.message, }; diff --git a/docs/architecture/crate-graph.md b/docs/architecture/crate-graph.md index 7e1d0680d..8d7163f20 100644 --- a/docs/architecture/crate-graph.md +++ b/docs/architecture/crate-graph.md @@ -66,7 +66,7 @@ UFFS uses a strict layered architecture with **5 layers** plus a parallel **tool | `uffs-security` | Crypto, keystore, secure FS ops, FILE_FLAG_RANDOM_ACCESS handling | Windows DPAPI / DACL, libc flock, memmap2 | | `uffs-text` | Unicode/NTFS case folding, trigram keys, i18n primitives | Pure logic; zero unsafe | | `uffs-time` | NTFS FILETIME arithmetic (`const fn`) | Pure logic; zero deps | -| `uffs-broker-protocol` | Cross-platform broker wire-protocol types (`PIPE_NAME`, `SERVICE_NAME`) | Pure logic; zero unsafe | +| `uffs-broker-protocol` | Cross-platform broker wire-protocol types (`PIPE_NAME`, `SERVICE_NAME`, the Snapshot Manager messages, the VSS `HRESULT` retry table in `vss_hresult`) | Pure logic; zero unsafe | | `uffs-winsvc` | Native Windows service control (SCM) + broker-pipe readiness probe; the single home for the `sc`/SCM mechanics shared by uffs-broker, uffs-update, uffs-cli | `windows` (windows-target only); pure stubs off Windows | | `uffs-fetch` | Hardened release-asset transport: GitHub release lookup, streaming any-URL download (retry, connect/inactivity timeouts, caller-chosen byte cap, caller-supplied user-agent), `SHA256SUMS` verification. Extracted from `uffs-update` for reuse by external products; keeps the HTTP/TLS stack out of the lean `uffs` CLI | reqwest (blocking, rustls-tls-native-roots), sha2, hex; cross-platform, zero unsafe | From f8bf91660cf46194f24fadcc2c85b703fb5bea74 Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:31:58 -0700 Subject: [PATCH 05/21] fix(deps): rustls 0.23.46 for RUSTSEC-2026-0346 and RUSTSEC-2026-0347 rustls 0.23.45 leaked TLS session secrets through Debug impls (RUSTSEC-2026-0346) and could send buffered application data to the outer server after an ECH rejection (RUSTSEC-2026-0347); cargo deny blocked the push on both. Reached here through reqwest 0.12 (uffs-fetch, uffs-update, polars-io). The 0.23.45 -> 0.23.46 delta audit is carried over from the docenta ledger, where the same diff was reviewed in full; no exemption. --- Cargo.lock | 6 +++--- supply-chain/audits.toml | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9c591b002..c84e83d12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3441,9 +3441,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "once_cell", "ring", @@ -3972,7 +3972,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", diff --git a/supply-chain/audits.toml b/supply-chain/audits.toml index 886127777..76292f782 100644 --- a/supply-chain/audits.toml +++ b/supply-chain/audits.toml @@ -1499,6 +1499,12 @@ access, and no new dependencies — only the existing `webpki` 0.103.5 -> 0.103.14 and `zeroize`-family minimums move. """ +[[audits.rustls]] +who = "Robert Nio <50460704+githubrobbi@users.noreply.github.com>" +criteria = "safe-to-deploy" +delta = "0.23.45 -> 0.23.46" +notes = "Delta reviewed 2026-10-14 (cargo vet diff, 23 files, 1160/167 lines; the manifest changes only the version). The two advisories: RUSTSEC-2026-0346, hand-written Debug for Tls13ClientSessionValue, ClientSessionCommon and ServerSessionValue that leave the session secret and master secret out (finish_non_exhaustive), with tests asserting the secret bytes are absent; RUSTSEC-2026-0347, the ECH-rejected check now runs BEFORE start_traffic so no buffered application data reaches the outer server, a hello-retry-request decision on ECH is remembered and a server hello that varies it is a fatal illegal_parameter (RFC 9849 6.1.5/6.1.6), the kx hint is stored only after Finished. Beside them: TLS 1.3 key update taken before the record that would use the last sequence number (pre_encrypt_action >= bound, a write crossing the limit is not sent in part, the pending flag cleared after the update); a protected change_cipher_spec record is an unexpected_message alert (RFC 9846 5); write/flush return a prior fatal error; stream flush drains all TLS output and flushes the transport, prepare_read serves buffered plaintext without I/O; zlib and brotli certificate decompressors require full input consumption (trailing data refused, brotli strict mode); HPKE uses LessSafeKey with an explicit nonce and a failed open no longer consumes the sequence number; LimitedCache evicts by an explicit max instead of the VecDeque capacity; early data only when the first PSK was selected; a non-empty certificate_request_context from the client is a decode_error. No new unsafe, no process, filesystem, environment or network access, no build-script change. Tightens negotiation, parsing and caching; no new surface. Here reached through reqwest 0.12 (hyper-rustls, tokio-rustls): uffs-fetch, uffs-update, and polars-io via uffs-polars. Audit carried over from the docenta ledger (docenta fb64b688), same reviewer and the same 0.23.45 -> 0.23.46 tarball diff." + [[audits.rustls-native-certs]] who = "Robert Nio <50460704+githubrobbi@users.noreply.github.com>" criteria = "safe-to-deploy" From 191a339d9e8ad603efa931380af2d72794c8f9ec Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:03:52 +0000 Subject: [PATCH 06/21] chore(security): refresh cargo-vet imports Scheduled weekly refresh of supply-chain/imports.lock from upstream audit sources (Mozilla, Google, Bytecode Alliance, ISRG, Zcash). See .github/workflows/cargo-vet-refresh.yml for rationale and docs/architecture/security/ supply-chain-posture.md for the full supply-chain posture. --- supply-chain/config.toml | 4 ---- supply-chain/imports.lock | 6 ++++++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/supply-chain/config.toml b/supply-chain/config.toml index b8b066283..64a321def 100644 --- a/supply-chain/config.toml +++ b/supply-chain/config.toml @@ -450,10 +450,6 @@ criteria = "safe-to-deploy" version = "0.4.2" criteria = "safe-to-deploy" -[[exemptions.humantime]] -version = "2.3.0" -criteria = "safe-to-deploy" - [[exemptions.iana-time-zone]] version = "0.1.65" criteria = "safe-to-deploy" diff --git a/supply-chain/imports.lock b/supply-chain/imports.lock index 22e7b9e31..841048056 100644 --- a/supply-chain/imports.lock +++ b/supply-chain/imports.lock @@ -1942,6 +1942,12 @@ criteria = "safe-to-deploy" version = "0.4.3" aggregated-from = "https://raw.githubusercontent.com/mozilla-firefox/firefox/main/supply-chain/audits.toml" +[[audits.mozilla.audits.humantime]] +who = "Max Inden " +criteria = "safe-to-deploy" +version = "2.4.0" +aggregated-from = "https://raw.githubusercontent.com/mozilla-firefox/firefox/main/supply-chain/audits.toml" + [[audits.mozilla.audits.idna]] who = "Valentin Gosu " criteria = "safe-to-deploy" From f455a78913ea15728ad88c4de4902a9155c3fdfe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:20:01 +0000 Subject: [PATCH 07/21] ci(deps): bump the github-actions group across 1 directory with 5 updates Bumps the github-actions group with 5 updates in the / directory: | Package | From | To | | --- | --- | --- | | [Swatinem/rust-cache](https://github.com/swatinem/rust-cache) | `2.9.2` | `2.9.3` | | [github/codeql-action/init](https://github.com/github/codeql-action) | `4.38.3` | `4.39.1` | | [github/codeql-action/analyze](https://github.com/github/codeql-action) | `4.38.3` | `4.39.1` | | [release-plz/action](https://github.com/release-plz/action) | `0.5.141` | `0.5.145` | | [rust-lang/crates-io-auth-action](https://github.com/rust-lang/crates-io-auth-action) | `1.0.5` | `1.0.6` | Updates `Swatinem/rust-cache` from 2.9.2 to 2.9.3 - [Release notes](https://github.com/swatinem/rust-cache/releases) - [Changelog](https://github.com/Swatinem/rust-cache/blob/master/CHANGELOG.md) - [Commits](https://github.com/swatinem/rust-cache/compare/6323deb102c322ba6fcbdcafc7e3dddab59af2b6...d1527c8f67ef9af357290475d4af4b0fab8f12e0) Updates `github/codeql-action/init` from 4.38.3 to 4.39.1 - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/24c54180a607b1449ed407dd24f251e4e9147c8d...9b3254085494b55a0f7c4c2b3d2d52cc712ce370) Updates `github/codeql-action/analyze` from 4.38.3 to 4.39.1 - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/24c54180a607b1449ed407dd24f251e4e9147c8d...9b3254085494b55a0f7c4c2b3d2d52cc712ce370) Updates `release-plz/action` from 0.5.141 to 0.5.145 - [Release notes](https://github.com/release-plz/action/releases) - [Commits](https://github.com/release-plz/action/compare/f393a7b38aca9d9b4c58447b291dbac963901df6...7ef5e4d4b0420630cfcaeb182c01e13651d4c577) Updates `rust-lang/crates-io-auth-action` from 1.0.5 to 1.0.6 - [Release notes](https://github.com/rust-lang/crates-io-auth-action/releases) - [Commits](https://github.com/rust-lang/crates-io-auth-action/compare/c6f97d42243bad5fab37ca0427f495c86d5b1a18...3268eb413133e2960fa4ae0e963ca76380c54cd9) --- updated-dependencies: - dependency-name: Swatinem/rust-cache dependency-version: 2.9.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: github-actions - dependency-name: github/codeql-action/init dependency-version: 4.39.1 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: github-actions - dependency-name: github/codeql-action/analyze dependency-version: 4.39.1 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: github-actions - dependency-name: release-plz/action dependency-version: 0.5.145 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: github-actions - dependency-name: rust-lang/crates-io-auth-action dependency-version: 1.0.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: github-actions ... Signed-off-by: dependabot[bot] --- .github/workflows/cargo-vet-refresh.yml | 2 +- .github/workflows/codeql.yml | 4 ++-- .github/workflows/crates-io-dry-run.yml | 2 +- .github/workflows/nightly-canary.yml | 2 +- .github/workflows/pr-fast.yml | 24 ++++++++++++------------ .github/workflows/preview-artifacts.yml | 4 ++-- .github/workflows/release-cache-warm.yml | 2 +- .github/workflows/release-plz.yml | 12 ++++++------ .github/workflows/release.yml | 2 +- .github/workflows/tier-2.yml | 16 ++++++++-------- 10 files changed, 35 insertions(+), 35 deletions(-) diff --git a/.github/workflows/cargo-vet-refresh.yml b/.github/workflows/cargo-vet-refresh.yml index fc209deb3..6ab67c51b 100644 --- a/.github/workflows/cargo-vet-refresh.yml +++ b/.github/workflows/cargo-vet-refresh.yml @@ -113,7 +113,7 @@ jobs: run: rustup show - name: Cache dependencies - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: # Unique suffix so this workflow's cache doesn't evict the # main CI cache (and vice versa). cargo-vet's compile is diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index c853a0489..bef10a1d0 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -138,7 +138,7 @@ jobs: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Initialize CodeQL - uses: github/codeql-action/init@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3 + uses: github/codeql-action/init@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 with: languages: ${{ matrix.language }} # Rust support (public preview) currently only accepts @@ -153,6 +153,6 @@ jobs: # queries: security-and-quality - name: Perform CodeQL analysis - uses: github/codeql-action/analyze@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3 + uses: github/codeql-action/analyze@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 with: category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/crates-io-dry-run.yml b/.github/workflows/crates-io-dry-run.yml index ec038ac1b..7057184b1 100644 --- a/.github/workflows/crates-io-dry-run.yml +++ b/.github/workflows/crates-io-dry-run.yml @@ -158,7 +158,7 @@ jobs: run: rustup show - name: Cache cargo dependencies - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: # Dedicated cache key — dry-run's compile output is small # (per-crate package verifications) and disjoint from the diff --git a/.github/workflows/nightly-canary.yml b/.github/workflows/nightly-canary.yml index 199aed8a5..893fff870 100644 --- a/.github/workflows/nightly-canary.yml +++ b/.github/workflows/nightly-canary.yml @@ -129,7 +129,7 @@ jobs: cargo +nightly clippy --version - name: Cache dependencies - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: # Per-OS, per-canary cache key. Distinct from the pinned- # toolchain caches so a floating-nightly artifact set never diff --git a/.github/workflows/pr-fast.yml b/.github/workflows/pr-fast.yml index 1f4223195..04424a1c0 100644 --- a/.github/workflows/pr-fast.yml +++ b/.github/workflows/pr-fast.yml @@ -230,7 +230,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - run: rustup show - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: shared-key: pr-fast-sanity save-if: 'false' @@ -256,7 +256,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - run: rustup show - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: shared-key: pr-fast-sanity save-if: 'false' @@ -280,7 +280,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - run: rustup show - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: shared-key: pr-fast-sanity save-if: 'false' @@ -309,7 +309,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - run: rustup show - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: shared-key: pr-fast-sanity save-if: 'false' @@ -368,7 +368,7 @@ jobs: df -h / - run: rustup show - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: shared-key: pr-fast-sanity cache-on-failure: 'true' @@ -410,7 +410,7 @@ jobs: df -h / - run: rustup show - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: shared-key: pr-fast-sanity save-if: 'false' @@ -442,7 +442,7 @@ jobs: df -h / - run: rustup show - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: shared-key: pr-fast-clippy-no-default save-if: 'false' @@ -474,7 +474,7 @@ jobs: df -h / - run: rustup show - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: shared-key: pr-fast-sanity save-if: 'false' @@ -511,7 +511,7 @@ jobs: df -h / - run: rustup show - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: shared-key: pr-fast-test-build cache-on-failure: 'true' @@ -548,7 +548,7 @@ jobs: df -h / - run: rustup show - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: shared-key: pr-fast-test-build save-if: 'false' @@ -589,7 +589,7 @@ jobs: fetch-depth: 0 - run: rustup show - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: shared-key: pr-fast-security @@ -672,7 +672,7 @@ jobs: ref: ${{ github.event.pull_request.head.sha || github.sha }} - run: rustup show - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: shared-key: pr-fast-windows diff --git a/.github/workflows/preview-artifacts.yml b/.github/workflows/preview-artifacts.yml index f3d8d9f51..12b3bc933 100644 --- a/.github/workflows/preview-artifacts.yml +++ b/.github/workflows/preview-artifacts.yml @@ -174,7 +174,7 @@ jobs: with: tool: cargo-xwin - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: shared-key: preview-windows @@ -265,7 +265,7 @@ jobs: with: tool: nextest - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: shared-key: preview-test-archive-windows diff --git a/.github/workflows/release-cache-warm.yml b/.github/workflows/release-cache-warm.yml index 3b51d9bfb..7225dacc8 100644 --- a/.github/workflows/release-cache-warm.yml +++ b/.github/workflows/release-cache-warm.yml @@ -183,7 +183,7 @@ jobs: rustc --version - name: Cache dependencies - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: # MUST match `release.yml::build-release-binaries`'s shared-key # exactly so this workflow's saves are reachable from diff --git a/.github/workflows/release-plz.yml b/.github/workflows/release-plz.yml index 18251132c..c73ac4f5e 100644 --- a/.github/workflows/release-plz.yml +++ b/.github/workflows/release-plz.yml @@ -363,7 +363,7 @@ jobs: run: rustup show - name: Cache cargo dependencies - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: # Dedicated cache key — separate from R3 shadow cache so # the cache effects of active mode (which compiles different @@ -375,7 +375,7 @@ jobs: # Pinned to commit SHA of the v0.5.128 tag (resolved 2026-05-08) # for supply-chain hygiene. Bump alongside other action pins # via the standard Dependabot or manual-refresh process. - uses: release-plz/action@f393a7b38aca9d9b4c58447b291dbac963901df6 # v0.5.141 + uses: release-plz/action@7ef5e4d4b0420630cfcaeb182c01e13651d4c577 # v0.5.145 with: command: release-pr config: release-plz.toml @@ -421,7 +421,7 @@ jobs: run: rustup show - name: Cache cargo dependencies - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: shared-key: release-plz-active-release @@ -431,7 +431,7 @@ jobs: # job runs on every push but only does work when the PR # actually merged. id: release-plz - uses: release-plz/action@f393a7b38aca9d9b4c58447b291dbac963901df6 # v0.5.141 + uses: release-plz/action@7ef5e4d4b0420630cfcaeb182c01e13651d4c577 # v0.5.145 with: command: release config: release-plz.toml @@ -526,7 +526,7 @@ jobs: uses: dtolnay/rust-toolchain@nightly - name: Cache cargo dependencies - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: shared-key: crates-io-publish @@ -543,7 +543,7 @@ jobs: # this workflow filename + the `crates.io-publish` environment). - name: Authenticate with crates.io (OIDC) id: auth - uses: rust-lang/crates-io-auth-action@c6f97d42243bad5fab37ca0427f495c86d5b1a18 # v1.0.5 + uses: rust-lang/crates-io-auth-action@3268eb413133e2960fa4ae0e963ca76380c54cd9 # v1.0.6 # Publish the publishable crates, dependency-ordered. The set is # derived from `cargo metadata` (publish != []) so it adapts as diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 57b17e778..6b9f7b122 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -429,7 +429,7 @@ jobs: rustc --version - name: Cache dependencies - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: # `shared-key:` (not `key:`) so the action's built-in fallback # restores partial matches when `Cargo.lock` shifts between diff --git a/.github/workflows/tier-2.yml b/.github/workflows/tier-2.yml index d67e2696c..19d0a2e11 100644 --- a/.github/workflows/tier-2.yml +++ b/.github/workflows/tier-2.yml @@ -81,7 +81,7 @@ jobs: run: rustup show - name: Cache dependencies - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: shared-key: tier-2-coverage cache-on-failure: 'true' @@ -145,7 +145,7 @@ jobs: run: rustup show - name: Cache dependencies - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: shared-key: tier-2-build-validation cache-on-failure: 'true' @@ -186,7 +186,7 @@ jobs: run: rustup show - name: Cache dependencies - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: shared-key: tier-2-udeps cache-on-failure: 'true' @@ -261,7 +261,7 @@ jobs: run: rustup show - name: Cache dependencies - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: shared-key: tier-2-hack cache-on-failure: 'true' @@ -311,7 +311,7 @@ jobs: run: rustup show - name: Cache dependencies - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: shared-key: tier-2-miri cache-on-failure: 'true' @@ -398,7 +398,7 @@ jobs: run: rustup show - name: Cache dependencies - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: # Dedicated cache key — cargo-careful's `-Zbuild-std` # rebuild produces a distinct sysroot / target-dir layout @@ -519,7 +519,7 @@ jobs: run: rustup show - name: Cache dependencies - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: # Dedicated cache key — cargo-mutants does many short # per-mutant rebuilds and benefits from its own cache so @@ -672,7 +672,7 @@ jobs: run: rustup show - name: Cache dependencies - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: # Dedicated cache key — the fuzz crate has its own # `[workspace] members = ["."]` and a separate target-dir From a274ce02bdf66835a0897f1f13afe971462ffd97 Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:24:34 -0700 Subject: [PATCH 08/21] fix(daemon): stop journal saves from taking the machine under heavy churn Issue #639 reported uffsd using about 8 cores and 5.7 GB while cargo builds ran. Measured on the winbox (2026-10-28, 4 M-record C:, 200 k-file burst plus a real cargo build): applies were cheap (25-60 ms), no journal wraps; the cost was the compact-cache save. Each save took 15-35 CPU-seconds on 4-9 cores (rayon's global pool for the delta fold, up to 8 zstd workers) and 1-5 GB of transient memory, and under a burst one was started every ~18 s. The save task was spawned and never awaited, so saves overlapped, each holding its own copy of the index. - One save at a time across all drives (SaveSlot). A save tick that finds the slot taken applies its batch in memory and leaves the write to a later tick. - Events-driven saves wait at least 60 s after the previous save (DEFAULT_SAVE_MIN_INTERVAL). The 5-minute age save is not gated. - The save tick no longer folds the live body. It patches like an apply tick (search is live at once); the background persist folds a private copy on a two-thread pool whose threads run in background-I/O mode, and zstd gets two workers. Two correctness bugs on the same path, fixed here: - The cursor was stored as soon as the save task was spawned, before the file was written (save_compact_cache_background even hands the write to another thread). A crash or failed write left the on-disk cursor ahead of the on-disk body, and the next cold start skipped those changes for good. The cursor is now stored only after the atomic write succeeds, via the blocking save_compact_cache_with_workers. - A save tick whose batch was empty (apply ticks drain the same buffer, so after a quiet spell every change is already applied) returned early as a no-op, yet the cursor advanced with nothing written. An empty save now persists the live body. save_compact_cache now carries the same delta-free guard as the background variant. --- Cargo.lock | 1 + crates/uffs-core/src/compact_cache.rs | 59 ++++-- crates/uffs-daemon/Cargo.toml | 3 + crates/uffs-daemon/src/cache/journal_loop.rs | 16 +- .../src/cache/journal_loop/tests.rs | 1 + .../cache/journal_loop/tests/apply_cadence.rs | 3 + .../cache/journal_loop/tests/integration.rs | 23 ++- .../journal_loop/tests/save_log_message.rs | 1 + .../cache/journal_loop/tests/thresholds.rs | 47 ++++- .../tests/wrap_and_persistence.rs | 1 + .../src/cache/journal_loop/triggers.rs | 32 ++- crates/uffs-daemon/src/cache/journal_sink.rs | 53 +++-- crates/uffs-daemon/src/cache/shard.rs | 16 +- crates/uffs-daemon/src/cache/shard/tests.rs | 63 +++--- crates/uffs-daemon/src/index/journal.rs | 98 +++------- crates/uffs-daemon/src/index/journal_save.rs | 183 ++++++++++++++++++ crates/uffs-daemon/src/index/mod.rs | 1 + crates/uffs-daemon/src/lib.rs | 1 + crates/uffs-mft/src/cache.rs | 4 +- crates/uffs-mft/src/cache_compress.rs | 57 +++++- 20 files changed, 490 insertions(+), 173 deletions(-) create mode 100644 crates/uffs-daemon/src/index/journal_save.rs diff --git a/Cargo.lock b/Cargo.lock index c84e83d12..5f3fac5c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4496,6 +4496,7 @@ dependencies = [ "mimalloc", "proptest", "rand", + "rayon", "serde", "serde_json", "tempfile", diff --git a/crates/uffs-core/src/compact_cache.rs b/crates/uffs-core/src/compact_cache.rs index 5b1c35937..1ef6778f8 100644 --- a/crates/uffs-core/src/compact_cache.rs +++ b/crates/uffs-core/src/compact_cache.rs @@ -1006,6 +1006,26 @@ fn parse_compact_header(data: &[u8]) -> Result<(u64, usize, u16), &'static str> /// # Errors /// Returns an error if compression, encryption, or file writing fails. pub fn save_compact_cache(index: &DriveCompactIndex) -> io::Result<()> { + save_compact_cache_with_workers(index, uffs_mft::cache::default_zstd_workers()) +} + +/// [`save_compact_cache`] with an explicit zstd worker count, blocking until +/// the file is written. +/// +/// The journal loop's background saves use this with a small count: a save +/// that takes every core competes with the build or copy that caused the +/// churn in the first place (issue #639). Returning only after the atomic +/// write is what lets a caller persist the USN cursor strictly after the body +/// it describes. +/// +/// # Errors +/// Returns an error if `index` still carries a delta overlay, or if +/// compression, encryption, or file writing fails. +pub fn save_compact_cache_with_workers( + index: &DriveCompactIndex, + zstd_workers: u32, +) -> io::Result<()> { + refuse_delta_carrying(index)?; let profile = std::env::var_os("UFFS_CACHE_PROFILE").is_some(); let path = compact_cache_path(index.letter); if let Some(dir) = path.parent() { @@ -1016,15 +1036,36 @@ pub fn save_compact_cache(index: &DriveCompactIndex) -> io::Result<()> { // hits an EISDIR rename failure. No-op for cold-boot first saves // and for every save after the first successful one. purge_legacy_compact_cache_dir(&path, index.letter)?; - uffs_mft::cache::compress_encrypt_write_streaming( + uffs_mft::cache::compress_encrypt_write_streaming_with_workers( |encoder| serialize_compact_to_writer(index, encoder), &path, ZSTD_LEVEL, + zstd_workers, profile, "compact", ) } +/// Refuse to persist a delta-carrying index. +/// +/// A delta-carrying index serialises a children CSR sized for the pre-patch +/// record count while the header carries the post-patch count, mis-aligning +/// every later section for the reader (see +/// [`DriveCompactIndex::fold_delta_for_save`] for the full mechanism and the +/// field incidents). Callers fold the delta first; this guard turns any path +/// that forgets into a loud error instead of a poisoned cache that +/// quarantines on the next start. +fn refuse_delta_carrying(index: &DriveCompactIndex) -> io::Result<()> { + if index.delta.is_some() { + return Err(io::Error::other( + "refusing to persist a delta-carrying compact index: fold the delta first \ + (DriveCompactIndex::fold_delta_for_save) — its base sections are stale \ + against the patched records and would serialize misaligned", + )); + } + Ok(()) +} + /// Streams the compact index into a zstd encoder on the calling thread /// (no ~1.1 GB intermediate buffer), then spawns a background thread /// for encryption and disk write. @@ -1039,20 +1080,8 @@ pub fn save_compact_cache(index: &DriveCompactIndex) -> io::Result<()> { /// Returns an error only if compression or directory creation fails. pub fn save_compact_cache_background(index: &DriveCompactIndex) -> io::Result<()> { // Boundary enforcement of the "on-disk cache is always delta-free" - // invariant: a delta-carrying index serialises a children CSR sized - // for the pre-patch record count while the header carries the - // post-patch count, mis-aligning every later section for the reader - // (see `DriveCompactIndex::fold_delta_for_save` for the full - // mechanism + field incidents). Callers fold the delta first; this - // guard turns any future path that forgets into a loud error - // instead of a poisoned cache that quarantines on the next start. - if index.delta.is_some() { - return Err(io::Error::other( - "refusing to persist a delta-carrying compact index: fold the delta first \ - (DriveCompactIndex::fold_delta_for_save) — its base sections are stale \ - against the patched records and would serialize misaligned", - )); - } + // invariant (see `refuse_delta_carrying`). + refuse_delta_carrying(index)?; let profile = std::env::var_os("UFFS_CACHE_PROFILE").is_some(); let t_compress = Instant::now(); diff --git a/crates/uffs-daemon/Cargo.toml b/crates/uffs-daemon/Cargo.toml index 656bd2080..c4f9f3d97 100644 --- a/crates/uffs-daemon/Cargo.toml +++ b/crates/uffs-daemon/Cargo.toml @@ -66,6 +66,9 @@ tokio = { workspace = true, features = ["net"] } # `load_or_join_in_flight` helper (PR-e: thundering-herd promote # fix — see `docs/refactor/promote-thundering-herd-fix.md`). futures = { workspace = true } +# A dedicated two-thread pool for the journal loop's background saves, so a +# save's delta fold does not take every core (`index::journal_save`). +rayon.workspace = true # Serialization (JSON-RPC) serde.workspace = true diff --git a/crates/uffs-daemon/src/cache/journal_loop.rs b/crates/uffs-daemon/src/cache/journal_loop.rs index 295e50cc3..5ab58e3f0 100644 --- a/crates/uffs-daemon/src/cache/journal_loop.rs +++ b/crates/uffs-daemon/src/cache/journal_loop.rs @@ -52,8 +52,8 @@ mod triggers; pub(crate) use poll::{MAX_POLL_BACKOFF, PollBackoff}; pub(crate) use triggers::{ - ApplyTrigger, DEFAULT_APPLY_DEBOUNCE_MS, DEFAULT_APPLY_INTERVAL_MS, DEFAULT_SAVE_THRESHOLD_AGE, - DEFAULT_SAVE_THRESHOLD_EVENTS, SaveReason, SaveTrigger, + ApplyTrigger, DEFAULT_APPLY_DEBOUNCE_MS, DEFAULT_APPLY_INTERVAL_MS, DEFAULT_SAVE_MIN_INTERVAL, + DEFAULT_SAVE_THRESHOLD_AGE, DEFAULT_SAVE_THRESHOLD_EVENTS, SaveReason, SaveTrigger, }; /// Default poll interval for the per-shard journal loop (500 ms). @@ -271,6 +271,9 @@ pub(crate) struct JournalLoopConfig { /// when at least one event is pending. Default /// [`DEFAULT_SAVE_THRESHOLD_AGE`] (5 min). pub(crate) save_threshold_age: Duration, + /// Minimum gap between two events-driven saves. Default + /// [`DEFAULT_SAVE_MIN_INTERVAL`] (60 s). + pub(crate) save_min_interval: Duration, /// Search-freshness apply **max-wait** cap (Phase 5) — the ceiling of the /// debounce model. A burst that never settles is force-applied at least /// this often, bounding both freshness lag and apply CPU under sustained @@ -290,6 +293,7 @@ impl Default for JournalLoopConfig { initial_cursor: 0, save_threshold_events: DEFAULT_SAVE_THRESHOLD_EVENTS, save_threshold_age: DEFAULT_SAVE_THRESHOLD_AGE, + save_min_interval: DEFAULT_SAVE_MIN_INTERVAL, apply_interval: Duration::from_millis(DEFAULT_APPLY_INTERVAL_MS), apply_debounce: Duration::from_millis(DEFAULT_APPLY_DEBOUNCE_MS), } @@ -577,9 +581,11 @@ fn fire_due_cadence( apply_trigger: &mut ApplyTrigger, config: &JournalLoopConfig, ) { - if let Some(reason) = - save_trigger.evaluate(config.save_threshold_events, config.save_threshold_age) - { + if let Some(reason) = save_trigger.evaluate( + config.save_threshold_events, + config.save_threshold_age, + config.save_min_interval, + ) { // A save drains + applies the buffer AND writes the body to // disk + persists the cursor — so it subsumes the apply tick. sink.trigger_save(letter, reason, cursor); diff --git a/crates/uffs-daemon/src/cache/journal_loop/tests.rs b/crates/uffs-daemon/src/cache/journal_loop/tests.rs index f45bd9d11..5bca9942c 100644 --- a/crates/uffs-daemon/src/cache/journal_loop/tests.rs +++ b/crates/uffs-daemon/src/cache/journal_loop/tests.rs @@ -328,6 +328,7 @@ fn fast_config() -> JournalLoopConfig { initial_cursor: 0, save_threshold_events: u64::MAX, save_threshold_age: Duration::from_hours(24), + save_min_interval: Duration::ZERO, // Disabled by default so the generic tick / cancel / cursor // tests don't accidentally fire an apply; the apply-cadence // tests override this with a short interval. diff --git a/crates/uffs-daemon/src/cache/journal_loop/tests/apply_cadence.rs b/crates/uffs-daemon/src/cache/journal_loop/tests/apply_cadence.rs index 381999c64..e6bdb9119 100644 --- a/crates/uffs-daemon/src/cache/journal_loop/tests/apply_cadence.rs +++ b/crates/uffs-daemon/src/cache/journal_loop/tests/apply_cadence.rs @@ -45,6 +45,7 @@ fn apply_only_config() -> JournalLoopConfig { JournalLoopConfig { save_threshold_events: u64::MAX, save_threshold_age: Duration::from_hours(24), + save_min_interval: Duration::ZERO, apply_interval: Duration::ZERO, apply_debounce: Duration::ZERO, ..JournalLoopConfig::default() @@ -94,6 +95,7 @@ fn save_tick_suppresses_redundant_apply() { let config = JournalLoopConfig { save_threshold_events: 1, save_threshold_age: Duration::from_hours(24), + save_min_interval: Duration::ZERO, apply_interval: Duration::ZERO, apply_debounce: Duration::ZERO, ..JournalLoopConfig::default() @@ -158,6 +160,7 @@ async fn loop_applies_body_near_live_without_saving() { poll_interval: Duration::from_millis(5), save_threshold_events: u64::MAX, save_threshold_age: Duration::from_hours(24), + save_min_interval: Duration::ZERO, apply_interval: Duration::ZERO, apply_debounce: Duration::ZERO, ..JournalLoopConfig::default() diff --git a/crates/uffs-daemon/src/cache/journal_loop/tests/integration.rs b/crates/uffs-daemon/src/cache/journal_loop/tests/integration.rs index 919ce5dca..eca3bba32 100644 --- a/crates/uffs-daemon/src/cache/journal_loop/tests/integration.rs +++ b/crates/uffs-daemon/src/cache/journal_loop/tests/integration.rs @@ -72,6 +72,20 @@ const E2E_DEADLINE: Duration = Duration::from_secs(5); /// busy-waiting. const E2E_POLL_INTERVAL: Duration = Duration::from_millis(10); +/// Fast polls, an events-only save threshold and no save gap, so the saves +/// [`ten_thousand_events_end_to_end`] counts come from the event volume alone. +fn end_to_end_config() -> JournalLoopConfig { + JournalLoopConfig { + poll_interval: Duration::from_millis(5), + initial_cursor: 0, + save_threshold_events: SAVE_THRESHOLD_EVENTS, + save_threshold_age: Duration::from_hours(24), + save_min_interval: Duration::ZERO, + apply_interval: Duration::from_hours(24), + apply_debounce: Duration::from_hours(24), + } +} + #[tokio::test] async fn ten_thousand_events_end_to_end() { let source = Arc::new(FakeJournalSource::new()); @@ -91,14 +105,7 @@ async fn ten_thousand_events_end_to_end() { source.enqueue_changes(changes, next_cursor); } - let config = JournalLoopConfig { - poll_interval: Duration::from_millis(5), - initial_cursor: 0, - save_threshold_events: SAVE_THRESHOLD_EVENTS, - save_threshold_age: Duration::from_hours(24), - apply_interval: Duration::from_hours(24), - apply_debounce: Duration::from_hours(24), - }; + let config = end_to_end_config(); let handle = spawn_journal_loop( uffs_mft::platform::DriveLetter::C, Arc::clone(&source) as Arc, diff --git a/crates/uffs-daemon/src/cache/journal_loop/tests/save_log_message.rs b/crates/uffs-daemon/src/cache/journal_loop/tests/save_log_message.rs index 5821d0672..3ed72f680 100644 --- a/crates/uffs-daemon/src/cache/journal_loop/tests/save_log_message.rs +++ b/crates/uffs-daemon/src/cache/journal_loop/tests/save_log_message.rs @@ -85,6 +85,7 @@ fn compact_cache_save_log_message_pins_string_target_and_level() { let config = JournalLoopConfig { save_threshold_events: 1, // tight — crosses on the first evaluate save_threshold_age: Duration::from_hours(1), // generous + save_min_interval: Duration::ZERO, // no gap: the first crossing saves apply_interval: Duration::from_hours(1), // disabled for this test apply_debounce: Duration::from_hours(1), ..JournalLoopConfig::default() diff --git a/crates/uffs-daemon/src/cache/journal_loop/tests/thresholds.rs b/crates/uffs-daemon/src/cache/journal_loop/tests/thresholds.rs index 5333d77b8..4f1ccb0ae 100644 --- a/crates/uffs-daemon/src/cache/journal_loop/tests/thresholds.rs +++ b/crates/uffs-daemon/src/cache/journal_loop/tests/thresholds.rs @@ -14,7 +14,9 @@ use alloc::sync::Arc; use core::time::Duration; -use super::super::{JournalLoopConfig, JournalSource, PatchSink, SaveReason, spawn_journal_loop}; +use super::super::{ + JournalLoopConfig, JournalSource, PatchSink, SaveReason, SaveTrigger, spawn_journal_loop, +}; use super::{ CONVERGENCE_DEADLINE, FakeJournalSource, RecordingSink, null_cursor_store, one_change, wait_for, }; @@ -42,6 +44,7 @@ async fn events_threshold_triggers_save() { initial_cursor: 0, save_threshold_events: 5, save_threshold_age: Duration::from_hours(24), + save_min_interval: Duration::ZERO, apply_interval: Duration::from_hours(24), apply_debounce: Duration::from_hours(24), }; @@ -93,6 +96,7 @@ async fn age_threshold_triggers_save_with_pending_events() { initial_cursor: 0, save_threshold_events: u64::MAX, save_threshold_age: Duration::from_millis(30), + save_min_interval: Duration::ZERO, apply_interval: Duration::from_hours(24), apply_debounce: Duration::from_hours(24), }; @@ -154,6 +158,7 @@ async fn zero_events_drive_does_not_trigger_save() { initial_cursor: 0, save_threshold_events: 1, save_threshold_age: Duration::from_millis(20), + save_min_interval: Duration::ZERO, apply_interval: Duration::from_hours(24), apply_debounce: Duration::from_hours(24), }; @@ -208,6 +213,7 @@ async fn counter_resets_after_save() { initial_cursor: 0, save_threshold_events: 5, save_threshold_age: Duration::from_hours(24), + save_min_interval: Duration::ZERO, apply_interval: Duration::from_hours(24), apply_debounce: Duration::from_hours(24), }; @@ -241,3 +247,42 @@ async fn counter_resets_after_save() { "exactly one save expected (counter reset after first); got {saves:?}" ); } + +/// Issue #639: a sustained burst must not save on every events-threshold +/// crossing. Inside the minimum interval the events threshold is held back; +/// the counter keeps accumulating, so the save fires as soon as the interval +/// has passed. +#[test] +fn events_threshold_waits_for_min_interval() { + let mut trigger = SaveTrigger::new(); + trigger.record(10); + assert_eq!( + trigger.evaluate(5, Duration::from_hours(24), Duration::from_hours(1)), + None, + "events past the threshold must wait for the minimum interval", + ); + trigger.record(10); + assert_eq!( + trigger.evaluate(5, Duration::from_hours(24), Duration::ZERO), + Some(SaveReason::EventsExceeded), + "once the interval has passed, the held-back events save fires", + ); + assert_eq!( + trigger.evaluate(5, Duration::from_hours(24), Duration::ZERO), + None, + "the save reset the counter", + ); +} + +/// The age ceiling is the durability backstop and is never held back by the +/// minimum interval: a drive with a few pending events still saves once the +/// age threshold passes. +#[test] +fn age_threshold_ignores_min_interval() { + let mut trigger = SaveTrigger::new(); + trigger.record(1); + assert_eq!( + trigger.evaluate(1_000, Duration::ZERO, Duration::from_hours(1)), + Some(SaveReason::AgeElapsed), + ); +} diff --git a/crates/uffs-daemon/src/cache/journal_loop/tests/wrap_and_persistence.rs b/crates/uffs-daemon/src/cache/journal_loop/tests/wrap_and_persistence.rs index eeb45b03b..c045aaf8d 100644 --- a/crates/uffs-daemon/src/cache/journal_loop/tests/wrap_and_persistence.rs +++ b/crates/uffs-daemon/src/cache/journal_loop/tests/wrap_and_persistence.rs @@ -90,6 +90,7 @@ async fn cursor_handed_to_sink_on_save_trigger() { initial_cursor: 0, save_threshold_events: 5, save_threshold_age: Duration::from_hours(24), + save_min_interval: Duration::ZERO, apply_interval: Duration::from_hours(24), apply_debounce: Duration::from_hours(24), }; diff --git a/crates/uffs-daemon/src/cache/journal_loop/triggers.rs b/crates/uffs-daemon/src/cache/journal_loop/triggers.rs index 70a845484..c104cd99b 100644 --- a/crates/uffs-daemon/src/cache/journal_loop/triggers.rs +++ b/crates/uffs-daemon/src/cache/journal_loop/triggers.rs @@ -47,6 +47,18 @@ pub(crate) const DEFAULT_SAVE_THRESHOLD_EVENTS: u64 = 50_000; /// path without changing the operator-visible recovery window. pub(crate) const DEFAULT_SAVE_THRESHOLD_AGE: Duration = Duration::from_mins(5); +/// Default minimum gap between two **events-driven** saves — 60 seconds. +/// +/// Measured on the winbox 2026-10-28 (issue #639): one compact-cache save of a +/// 4 M-record C: costs 15-35 CPU-seconds and 1-5 GB of transient memory, and a +/// 200 k-file burst crossed [`DEFAULT_SAVE_THRESHOLD_EVENTS`] every ~18 s, so +/// saves ran back to back for the whole burst. The saves only bound how far a +/// cold start must replay the journal; search freshness comes from the apply +/// tick, which is unaffected. One save a minute under sustained churn keeps +/// that replay window small while giving the CPU back to whatever is causing +/// the churn. The [`DEFAULT_SAVE_THRESHOLD_AGE`] save is not gated by this. +pub(crate) const DEFAULT_SAVE_MIN_INTERVAL: Duration = Duration::from_mins(1); + /// Default apply **max-wait** cap for the per-shard journal loop — 2 seconds /// (Phase 5). /// @@ -143,23 +155,29 @@ impl SaveTrigger { /// Returns `None` when no threshold is crossed *or* when no /// events are pending (zero-churn drives never produce /// no-op saves). + /// + /// The events threshold only fires once `save_min_interval` has passed + /// since the previous save, so a sustained burst saves at most once per + /// interval instead of on every threshold crossing. pub(super) fn evaluate( &mut self, save_threshold_events: u64, save_threshold_age: Duration, + save_min_interval: Duration, ) -> Option { if self.events_since_save == 0 { return None; } let now = Instant::now(); let elapsed = now.saturating_duration_since(self.last_save_at); - let reason = if self.events_since_save >= save_threshold_events { - Some(SaveReason::EventsExceeded) - } else if elapsed >= save_threshold_age { - Some(SaveReason::AgeElapsed) - } else { - None - }; + let reason = + if self.events_since_save >= save_threshold_events && elapsed >= save_min_interval { + Some(SaveReason::EventsExceeded) + } else if elapsed >= save_threshold_age { + Some(SaveReason::AgeElapsed) + } else { + None + }; if reason.is_some() { self.last_save_at = now; self.events_since_save = 0; diff --git a/crates/uffs-daemon/src/cache/journal_sink.rs b/crates/uffs-daemon/src/cache/journal_sink.rs index 7529f34ce..e72cb1a5c 100644 --- a/crates/uffs-daemon/src/cache/journal_sink.rs +++ b/crates/uffs-daemon/src/cache/journal_sink.rs @@ -81,6 +81,7 @@ use uffs_mft::usn::FileChange; use super::journal_loop::{CursorStore, PatchSink, SaveReason}; use crate::index::IndexManager; +use crate::index::journal_save::SaveSlot; /// Cross-task message from a sync sink callback to the async applier. /// @@ -373,6 +374,8 @@ async fn applier_task( // belong to the merged run (different letter, or a Save/Wrap); processed // on the next iteration before blocking on `recv`. let mut carried: Option = None; + // One background save at a time across every drive (issue #639). + let save_slot = Arc::new(SaveSlot::default()); loop { let msg = match carried.take() { Some(held) => held, @@ -406,7 +409,7 @@ async fn applier_task( } else { msg }; - dispatch_msg(&idx_strong, cursor_store.as_ref(), to_dispatch).await; + dispatch_msg(&idx_strong, &cursor_store, &save_slot, to_dispatch).await; } tracing::debug!( target: "shard.journal", @@ -450,7 +453,12 @@ fn coalesce_apply_run( /// also keeps the `Weak::upgrade` lifecycle decision in the parent /// (visible at the recv-loop level) and the per-variant dispatch /// in this helper (focused, easy to extend). -async fn dispatch_msg(idx: &Arc, cursor_store: &dyn CursorStore, msg: ApplyMsg) { +async fn dispatch_msg( + idx: &Arc, + cursor_store: &Arc, + save_slot: &Arc, + msg: ApplyMsg, +) { match msg { ApplyMsg::Save { letter, @@ -459,20 +467,33 @@ async fn dispatch_msg(idx: &Arc, cursor_store: &dyn CursorStore, m cursor, } => { let reason_str = save_reason_str(reason); - // Phase 8 surgical-patch path: hand the drained per-letter - // change buffer to `IndexManager::handle_journal_save`, - // which clones the Warm body, applies the patch, swaps - // the new Arc into the registry, and persists the patched - // body via `save_compact_cache_background`. - let applied = idx.handle_journal_save(letter, reason_str, changes).await; - // Persist the cursor in lockstep with the body save: only - // when the save actually happened (`applied`). A parked - // shard returns `false` here, so its on-disk cursor stays - // pinned to the last real body save and the startup - // warm-load guard can never strand a delta past the - // persisted body. See `cache::guarded_load`. - if applied { - cursor_store.store(letter, cursor); + // One save at a time: while the previous one is still + // writing, apply this batch in memory (search stays live) + // and leave the write to a later save tick. The cursor is + // not stored, so a cold start replays from the last body + // that actually reached disk. + let Some(permit) = save_slot.try_acquire() else { + tracing::debug!( + target: "shard.journal", + drive = %letter, + reason = reason_str, + change_count = changes.len(), + "Save deferred: the previous save is still writing; applied in memory only", + ); + let _applied = idx + .handle_journal_apply(letter, "save-deferred", changes) + .await; + return; + }; + // Patch the batch in (search goes live at once), then hand + // the body to the background persist, which writes it and + // only then stores the cursor — so the on-disk cursor never + // outruns the on-disk body. A parked or missing shard + // yields `None`: nothing is written and the cursor stays + // pinned to the last real body save, which the startup + // warm-load guard relies on (`cache::guarded_load`). + if let Some(body) = idx.handle_journal_save(letter, reason_str, changes).await { + idx.spawn_journal_persist(letter, body, cursor, Arc::clone(cursor_store), permit); } } ApplyMsg::Apply { letter, changes } => { diff --git a/crates/uffs-daemon/src/cache/shard.rs b/crates/uffs-daemon/src/cache/shard.rs index c187166d4..ce98ccbe6 100644 --- a/crates/uffs-daemon/src/cache/shard.rs +++ b/crates/uffs-daemon/src/cache/shard.rs @@ -520,19 +520,14 @@ impl ShardEntry { /// revisits the layout. /// /// [`DriveCompactIndex::frs_to_compact`]: uffs_core::compact::DriveCompactIndex::frs_to_compact - /// `fold_delta` — pass `true` on a SAVE-bound patch: the returned - /// body will be serialised, and a delta-carrying index must never - /// reach the writer (its base CSR sections are stale against the - /// patched records — see - /// [`DriveCompactIndex::fold_delta_for_save`]). The apply-only - /// tick passes `false` and keeps the cheap overlay. - /// - /// [`DriveCompactIndex::fold_delta_for_save`]: uffs_core::compact::DriveCompactIndex::fold_delta_for_save + /// The returned body keeps the cheap delta overlay. A save folds a + /// private copy of it before serialising (see + /// `crate::index::journal_save`), so the live body is never rebuilt + /// O(total records) just because a save is due. #[must_use] pub(crate) fn apply_usn_patch_to_body( &self, changes: &[uffs_mft::usn::FileChange], - fold_delta: bool, ) -> Option<( Arc, uffs_core::compact_loader::PatchStats, @@ -550,9 +545,6 @@ impl ShardEntry { // the small delta, not the hundreds-of-MB inverted indexes. let mut owned: DriveCompactIndex = (**body_arc).clone(); let stats = uffs_core::compact_loader::apply_usn_patch(&mut owned, changes); - if fold_delta { - owned.fold_delta_for_save(); - } Some((Arc::new(owned), stats)) } } diff --git a/crates/uffs-daemon/src/cache/shard/tests.rs b/crates/uffs-daemon/src/cache/shard/tests.rs index 30913c701..24037240c 100644 --- a/crates/uffs-daemon/src/cache/shard/tests.rs +++ b/crates/uffs-daemon/src/cache/shard/tests.rs @@ -523,7 +523,7 @@ fn apply_usn_patch_to_body_returns_new_arc_on_warm() { // Empty change batch — the method must still produce a fresh // Arc so the caller's swap path is exercised even on no-op ticks. - let result = shard.apply_usn_patch_to_body(&[], false); + let result = shard.apply_usn_patch_to_body(&[]); let (new_body, stats) = result.expect("Warm shard must yield Some"); assert!( @@ -554,7 +554,7 @@ fn apply_usn_patch_to_body_returns_none_on_parked() { ..FileChange::default() }]; - let result = shard.apply_usn_patch_to_body(&changes, false); + let result = shard.apply_usn_patch_to_body(&changes); assert!( result.is_none(), "Parked shard has no in-memory body — must return None" @@ -570,7 +570,7 @@ fn apply_usn_patch_to_body_returns_none_on_cold() { let stats = Arc::new(DriveStats::new()); let shard = ShardEntry::new_cold(uffs_mft::platform::DriveLetter::C, stats); - let result = shard.apply_usn_patch_to_body(&[], false); + let result = shard.apply_usn_patch_to_body(&[]); assert!( result.is_none(), "Cold shard has no in-memory body — must return None" @@ -599,7 +599,7 @@ fn apply_usn_patch_to_body_lands_delete_on_new_arc() { }]; let (new_body, stats) = shard - .apply_usn_patch_to_body(&changes, false) + .apply_usn_patch_to_body(&changes) .expect("Warm shard yields Some"); assert_eq!(stats.deleted, 1, "exactly one delete should land"); @@ -631,53 +631,48 @@ fn apply_usn_patch_to_body_lands_delete_on_new_arc() { ); } -/// Save-bound patches fold the delta overlay; apply-only patches keep -/// it. The body handed to the background disk save must be -/// delta-free — a delta-carrying index serialises its base children -/// CSR against a grown record count and poisons the cache for the -/// next start (`bloom k_hashes out of range`; see -/// `DriveCompactIndex::fold_delta_for_save`). A create is the -/// trigger shape: it appends a record, which is exactly what desyncs -/// the Arc-shared bases. +/// A patch keeps the cheap delta overlay on the live body; the save path +/// folds a private copy before it reaches the writer. The body handed to +/// the disk save must be delta-free — a delta-carrying index serialises its +/// base children CSR against a grown record count and poisons the cache for +/// the next start (`bloom k_hashes out of range`; see +/// `DriveCompactIndex::fold_delta_for_save`). A create is the trigger +/// shape: it appends a record, which is exactly what desyncs the Arc-shared +/// bases. #[test] -fn save_bound_patch_folds_delta_apply_only_keeps_it() { - let create = || { - vec![FileChange { - frs: 77_u64.into(), - parent_frs: 5_u64.into(), - filename: "fresh.txt".to_owned(), - created: true, - ..FileChange::default() - }] - }; +fn patch_keeps_delta_and_save_folds_a_copy() { + let create = vec![FileChange { + frs: 77_u64.into(), + parent_frs: 5_u64.into(), + filename: "fresh.txt".to_owned(), + created: true, + ..FileChange::default() + }]; let body = Arc::new(make_test_body(uffs_mft::platform::DriveLetter::C)); - let shard = ShardEntry::new_warm(uffs_mft::platform::DriveLetter::C, Arc::clone(&body)); + let shard = ShardEntry::new_warm(uffs_mft::platform::DriveLetter::C, body); let (applied, stats) = shard - .apply_usn_patch_to_body(&create(), false) + .apply_usn_patch_to_body(&create) .expect("Warm shard yields Some"); assert_eq!(stats.created, 1, "the create must land"); assert!( applied.delta.is_some(), - "apply-only patch must keep the cheap delta overlay" + "the patch must keep the cheap delta overlay" ); - let shard2 = ShardEntry::new_warm(uffs_mft::platform::DriveLetter::C, body); - let (saved, stats2) = shard2 - .apply_usn_patch_to_body(&create(), true) - .expect("Warm shard yields Some"); - assert_eq!( - stats2.created, 1, - "the create must land on the save path too" - ); + let saved = crate::index::journal_save::fold_for_save(&applied); assert!( saved.delta.is_none(), - "save-bound patch must fold the delta before the body reaches the writer" + "the save path must fold the delta before the body reaches the writer" ); assert!( saved.records.len() > 2, "the created record must survive the fold" ); + assert!( + applied.delta.is_some(), + "folding for the save must not touch the live body" + ); } /// Field 2026-10-14: a drive loaded at startup and parked before any diff --git a/crates/uffs-daemon/src/index/journal.rs b/crates/uffs-daemon/src/index/journal.rs index 5177afbaa..055d857f9 100644 --- a/crates/uffs-daemon/src/index/journal.rs +++ b/crates/uffs-daemon/src/index/journal.rs @@ -11,10 +11,9 @@ //! `DriveCompactIndex` body, applies the buffered //! [`uffs_mft::usn::FileChange`] batch via //! [`crate::cache::shard::ShardEntry::apply_usn_patch_to_body`], swaps the -//! new Arc into the registry, and persists via -//! [`uffs_core::compact_cache::save_compact_cache_background`]. Fast path: -//! ~600 ms patch + ~5 s background save on a 7M-record drive, vs ~7 s for the -//! full reload. +//! new Arc into the registry, and returns the body for the background persist +//! in [`super::journal_save`], which writes it and only then advances the +//! on-disk cursor. //! //! * [`IndexManager::handle_journal_refresh`] (Phase 7) — full per-shard body //! refresh via [`uffs_core::compact_loader::load_drive_with_usn_refresh`] @@ -127,32 +126,36 @@ impl IndexManager { self.apply_journal_body(letter, reason, body).await } - /// Phase 8 surgical-patch path: clone the Warm body, apply the - /// drained per-letter [`FileChange`] batch via - /// [`crate::cache::shard::ShardEntry::apply_usn_patch_to_body`], swap - /// the new Arc into the registry, and persist the patched body - /// via [`uffs_core::compact_cache::save_compact_cache_background`]. + /// Phase 8 surgical-patch path for a save tick: patch the drained + /// per-letter [`FileChange`] batch into the Warm body (as an apply tick + /// does, so search goes live at once) and return the body that must be + /// persisted. /// /// Called by the [`crate::cache::journal_sink::RegistryPatchSink`] /// applier task on every `Save` message (events-exceeded / /// age-elapsed) — the *fast* counterpart to /// [`IndexManager::handle_journal_refresh`] (which the applier still /// uses for `Wrap` messages where the cursor reset invalidates - /// the buffered batch). + /// the buffered batch). The heavy part — folding the delta and + /// writing the file — is the caller's background persist + /// ([`super::journal_save`]), off the applier task. + /// + /// An **empty** batch still returns the live body. Apply ticks drain + /// the same buffer, so by the time a save tick fires every change may + /// already be applied in memory but not yet on disk; skipping the write + /// there while the caller advanced the cursor would leave the cursor + /// ahead of the file, and a cold start would never replay those changes. /// /// ## Returns /// - /// * `true` when the body was successfully Arc-swapped into the registry - /// AND the background save task was spawned. An empty `changes` batch - /// also returns `true` (no-op short-circuit). - /// * `false` on every failure mode: - /// - the shard for `letter` is not registered; + /// * `Some(body)` — the body to persist, already serving queries. + /// * `None` on every failure mode: + /// - the shard for `letter` is not registered, or not warm; /// - the patch task aborted (panic, runtime shutdown); /// - the shard demoted to `Parked` / `Cold` between the trigger and the /// patch (`apply_usn_patch_to_body` returns `None`); the next promote - /// re-reads the cached body which a previous patch tick already - /// updated, so the per-shard drift is bounded by the per-shard save - /// cadence; + /// re-reads the cached body which a previous save already wrote, so the + /// per-shard drift is bounded by the per-shard save cadence; /// - the body swap lost a separate demote race (`apply_journal_body` /// returns `false`). pub(crate) async fn handle_journal_save( @@ -160,17 +163,13 @@ impl IndexManager { letter: uffs_mft::platform::DriveLetter, reason: &str, changes: Vec, - ) -> bool { - // Save-bound: fold any delta overlay inside the patch task so - // the body handed to the disk writer is delta-free (the - // serialization invariant `fold_delta_for_save` documents). - match self.apply_to_body(letter, reason, changes, true).await { - BodyApplyOutcome::Applied(new_body) => { - spawn_compact_cache_save_task(letter, new_body); - true - } - BodyApplyOutcome::NoOp => true, - BodyApplyOutcome::Failed => false, + ) -> Option> { + if changes.is_empty() { + return self.snapshot_shard_for_letter(letter).await?.body(); + } + match self.apply_to_body(letter, reason, changes).await { + BodyApplyOutcome::Applied(new_body) => Some(new_body), + BodyApplyOutcome::NoOp | BodyApplyOutcome::Failed => None, } } @@ -204,7 +203,7 @@ impl IndexManager { // serialised on this tick, and folding every ~2 s apply would // be an O(total-records) rebuild per tick. !matches!( - self.apply_to_body(letter, reason, changes, false).await, + self.apply_to_body(letter, reason, changes).await, BodyApplyOutcome::Failed ) } @@ -221,7 +220,6 @@ impl IndexManager { letter: uffs_mft::platform::DriveLetter, reason: &str, changes: Vec, - fold_delta: bool, ) -> BodyApplyOutcome { if changes.is_empty() { log_save_empty_batch(letter, reason); @@ -235,7 +233,7 @@ impl IndexManager { }; let (new_body, stats) = match self - .run_surgical_patch_task(&shard, letter, reason, changes, fold_delta) + .run_surgical_patch_task(&shard, letter, reason, changes) .await { PatchTaskOutcome::Applied(body, stats) => (body, stats), @@ -288,14 +286,13 @@ impl IndexManager { letter: uffs_mft::platform::DriveLetter, reason: &str, changes: Vec, - fold_delta: bool, ) -> PatchTaskOutcome { let background_io = Arc::clone(&self.background_io); let shard_for_patch = Arc::clone(shard); let change_count = changes.len(); let patch_result = tokio::task::spawn_blocking(move || { let _bg_scope = crate::cache::background_io::BackgroundIoScope::enter(background_io); - shard_for_patch.apply_usn_patch_to_body(&changes, fold_delta) + shard_for_patch.apply_usn_patch_to_body(&changes) }) .await; classify_patch_result(patch_result, letter, reason, change_count) @@ -446,39 +443,6 @@ fn log_save_patch_applied( ); } -/// Spawn a blocking task that persists the patched -/// `DriveCompactIndex` to disk via -/// [`uffs_core::compact_cache::save_compact_cache_background`]. -/// -/// **Why blocking?** `save_compact_cache_background` does the heavy -/// serialise + zstd compress synchronously on the calling thread; -/// only the encrypt + write steps run on a child thread that the -/// helper spawns internally. We don't want the serialise + -/// compress on the runtime's async workers — hence -/// `tokio::task::spawn_blocking`. -/// -/// **Best-effort.** A save failure does NOT roll back the -/// in-memory swap (the patched Arc is already serving queries via -/// the `IndexManager` registry); the helper just warn-logs the -/// failure for operator visibility. The next save tick re-attempts -/// from the latest in-memory body, so transient disk errors heal -/// themselves. -fn spawn_compact_cache_save_task( - letter: uffs_mft::platform::DriveLetter, - body: Arc, -) { - let _save_join = tokio::task::spawn_blocking(move || { - if let Err(err) = uffs_core::compact_cache::save_compact_cache_background(&body) { - tracing::warn!( - target: "shard.journal", - drive = %letter, - error = %err, - "Background compact-cache save failed after surgical patch (in-memory body still serving fresh data)", - ); - } - }); -} - /// Outcome of [`IndexManager::apply_to_body`] — the shared body-patch /// core behind the save tick and the apply tick. Carries the patched /// body on success so the save-tick caller can persist it while the diff --git a/crates/uffs-daemon/src/index/journal_save.rs b/crates/uffs-daemon/src/index/journal_save.rs new file mode 100644 index 000000000..0748a2795 --- /dev/null +++ b/crates/uffs-daemon/src/index/journal_save.rs @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright (c) 2025-2026 SKY, LLC. + +//! Background persistence for the journal loop's save tick. +//! +//! A save writes a drive's whole compact cache: fold the delta overlay +//! (rebuild the trigram, extension and children indexes), serialise, +//! compress, encrypt, write. Measured on the winbox 2026-10-28 (issue #639) +//! for a 4 M-record C:, one save cost 15-35 CPU-seconds on 4-9 cores and +//! 1-5 GB of transient memory, and under heavy churn saves were started +//! faster than they finished, each holding its own copy of the index. Three +//! rules keep that in check: +//! +//! * **One save at a time, across all drives** ([`SaveSlot`]). A save tick +//! that finds the slot taken applies its batch in memory and leaves the write +//! to a later tick. +//! * **Small and background.** The fold runs on a two-thread pool whose +//! threads are in background-I/O mode, and zstd gets [`SAVE_ZSTD_WORKERS`] +//! workers, so a save stays near two cores instead of taking the machine away +//! from the build that caused the churn. +//! * **Cursor strictly after the file.** The on-disk USN cursor advances only +//! once the atomic write has succeeded. A cursor ahead of the file would +//! make the next cold start skip the changes in between for good. + +use alloc::sync::Arc; +use core::sync::atomic::{AtomicBool, Ordering}; +use std::sync::OnceLock; + +use uffs_core::compact::DriveCompactIndex; + +use super::IndexManager; +use crate::cache::background_io::{ + BackgroundIoPriority as _, BackgroundIoScope, PlatformBackgroundIoPriority, +}; +use crate::cache::journal_loop::CursorStore; + +/// zstd workers a background save may use. +pub(crate) const SAVE_ZSTD_WORKERS: u32 = 2; + +/// Threads that fold a save's delta overlay. +const SAVE_FOLD_THREADS: usize = 2; + +/// The single background-save slot shared by every drive's save tick. +#[derive(Debug, Default)] +pub(crate) struct SaveSlot { + /// `true` while a save holds the slot. + busy: AtomicBool, +} + +impl SaveSlot { + /// Take the slot, or `None` when another save still holds it. + pub(crate) fn try_acquire(self: &Arc) -> Option { + self.busy + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .ok() + .map(|_was_free| SavePermit { + slot: Arc::clone(self), + }) + } +} + +/// Holds the [`SaveSlot`] until dropped — including on a panic inside the +/// save, so a failed save can never wedge every later one. +#[derive(Debug)] +pub(crate) struct SavePermit { + /// The slot released on drop. + slot: Arc, +} + +impl Drop for SavePermit { + fn drop(&mut self) { + self.slot.busy.store(false, Ordering::Release); + } +} + +impl IndexManager { + /// Persist `body` for `letter` on the blocking pool, then store `cursor`. + /// + /// Fire-and-forget: the body is already serving queries, so a failed + /// write only means the next save tick tries again from the newer body; + /// the cursor stays at the last body that did reach disk. `permit` is + /// held for the whole write. + pub(crate) fn spawn_journal_persist( + &self, + letter: uffs_mft::platform::DriveLetter, + body: Arc, + cursor: u64, + cursor_store: Arc, + permit: SavePermit, + ) { + let background_io = Arc::clone(&self.background_io); + let _persist = tokio::task::spawn_blocking(move || { + let _permit = permit; + let _bg_scope = BackgroundIoScope::enter(background_io); + match persist_body(&body) { + Ok(()) => cursor_store.store(letter, cursor), + Err(err) => tracing::warn!( + target: "shard.journal", + drive = %letter, + error = %err, + "Background compact-cache save failed (in-memory body still serving fresh data; \ + cursor left at the last saved body)", + ), + } + }); + } +} + +/// Write `body`, folded first if it carries a delta overlay. +fn persist_body(body: &Arc) -> std::io::Result<()> { + uffs_core::compact_cache::save_compact_cache_with_workers( + &fold_for_save(body), + SAVE_ZSTD_WORKERS, + ) +} + +/// The delta-free body to serialise: `body` itself when it carries no +/// overlay, else a folded private copy. The live body keeps its overlay, so +/// the O(total-records) rebuild stays off the search path; a delta-carrying +/// index must never reach the writer (see +/// `DriveCompactIndex::fold_delta_for_save`). +pub(crate) fn fold_for_save(body: &Arc) -> Arc { + if body.delta.is_none() { + return Arc::clone(body); + } + let mut folded = DriveCompactIndex::clone(body); + in_save_pool(|| folded.fold_delta_for_save()); + Arc::new(folded) +} + +/// Run `work` on the save pool, or inline when the pool could not be built. +fn in_save_pool(work: impl FnOnce() -> R + Send) -> R { + match save_pool() { + Some(pool) => pool.install(work), + None => work(), + } +} + +/// The two-thread rayon pool background saves fold on. Its threads enter +/// background-I/O mode once, at start, and stay there: they do nothing else. +fn save_pool() -> Option<&'static rayon::ThreadPool> { + /// Built on first use; `None` if the OS refused the threads. + static POOL: OnceLock> = OnceLock::new(); + POOL.get_or_init(|| { + rayon::ThreadPoolBuilder::new() + .num_threads(SAVE_FOLD_THREADS) + .thread_name(|index| format!("uffs-save-{index}")) + .start_handler(|_index| { + if let Err(err) = PlatformBackgroundIoPriority.begin() { + tracing::debug!(error = %err, "save pool thread stays at normal priority"); + } + }) + .build() + .map_err(|err| { + tracing::warn!(error = %err, "save pool unavailable; folding on the calling thread"); + }) + .ok() + }) + .as_ref() +} + +#[cfg(test)] +mod tests { + use alloc::sync::Arc; + + use super::SaveSlot; + + #[test] + fn slot_admits_one_save_until_the_permit_drops() { + let slot = Arc::new(SaveSlot::default()); + let first = slot.try_acquire(); + assert!(first.is_some(), "a free slot must be taken"); + assert!( + slot.try_acquire().is_none(), + "a held slot must refuse a second save" + ); + drop(first); + assert!( + slot.try_acquire().is_some(), + "dropping the permit frees the slot" + ); + } +} diff --git a/crates/uffs-daemon/src/index/mod.rs b/crates/uffs-daemon/src/index/mod.rs index 3ab4fe8b0..0a115e819 100644 --- a/crates/uffs-daemon/src/index/mod.rs +++ b/crates/uffs-daemon/src/index/mod.rs @@ -19,6 +19,7 @@ pub(crate) mod forget_drive; mod hotload; mod info; mod journal; +pub(crate) mod journal_save; mod loading; mod physical_order; mod predicates; diff --git a/crates/uffs-daemon/src/lib.rs b/crates/uffs-daemon/src/lib.rs index 330856293..1a30f6636 100644 --- a/crates/uffs-daemon/src/lib.rs +++ b/crates/uffs-daemon/src/lib.rs @@ -623,6 +623,7 @@ async fn spawn_journal_loops_for_warm_shards( apply_interval_ms = config.apply_interval.as_millis(), save_threshold_events = config.save_threshold_events, save_threshold_age_secs = config.save_threshold_age.as_secs(), + save_min_interval_secs = config.save_min_interval.as_secs(), "Spawning per-shard journal loops", ); let loop_count = letters.len(); diff --git a/crates/uffs-mft/src/cache.rs b/crates/uffs-mft/src/cache.rs index 4ff06fda0..e2056aa29 100644 --- a/crates/uffs-mft/src/cache.rs +++ b/crates/uffs-mft/src/cache.rs @@ -52,7 +52,9 @@ pub use cache_dataframe::load_or_build_dataframe_cached; #[path = "cache_compress.rs"] mod cache_compress; pub use cache_compress::{ - compress_encrypt_write, compress_encrypt_write_streaming, compress_zstd_mt, new_zstd_mt_encoder, + compress_encrypt_write, compress_encrypt_write_streaming, + compress_encrypt_write_streaming_with_workers, compress_zstd_mt, default_zstd_workers, + new_zstd_encoder_with_workers, new_zstd_mt_encoder, }; /// Default TTL for cached indices (4 hours). diff --git a/crates/uffs-mft/src/cache_compress.rs b/crates/uffs-mft/src/cache_compress.rs index 649995301..e448b9351 100644 --- a/crates/uffs-mft/src/cache_compress.rs +++ b/crates/uffs-mft/src/cache_compress.rs @@ -19,10 +19,7 @@ use crate::index::usize_to_f64; /// /// Returns an I/O error if compression fails. pub fn compress_zstd_mt(data: &[u8], level: i32) -> std::io::Result> { - let mut encoder = zstd::Encoder::new(Vec::new(), level)?; - let workers_usize = std::thread::available_parallelism().map_or(4, |n| n.get().min(8)); - let workers = u32::try_from(workers_usize).unwrap_or(8); - let _mt_result: Result<(), std::io::Error> = encoder.multithread(workers); + let mut encoder = new_zstd_mt_encoder(level)?; std::io::Write::write_all(&mut encoder, data)?; encoder.finish() } @@ -36,9 +33,29 @@ pub fn compress_zstd_mt(data: &[u8], level: i32) -> std::io::Result> { /// /// Returns an I/O error if encoder initialisation fails. pub fn new_zstd_mt_encoder(level: i32) -> std::io::Result>> { - let mut encoder = zstd::Encoder::new(Vec::new(), level)?; + new_zstd_encoder_with_workers(level, default_zstd_workers()) +} + +/// The zstd worker count for foreground cache writes: one per core, at most 8. +#[must_use] +pub fn default_zstd_workers() -> u32 { let workers_usize = std::thread::available_parallelism().map_or(4, |n| n.get().min(8)); - let workers = u32::try_from(workers_usize).unwrap_or(8); + u32::try_from(workers_usize).unwrap_or(8) +} + +/// [`new_zstd_mt_encoder`] with an explicit worker count. +/// +/// A background writer passes a small count so a cache save does not take +/// every core away from the user's own work. +/// +/// # Errors +/// +/// Returns an I/O error if encoder initialisation fails. +pub fn new_zstd_encoder_with_workers( + level: i32, + workers: u32, +) -> std::io::Result>> { + let mut encoder = zstd::Encoder::new(Vec::new(), level)?; let _mt_result: Result<(), std::io::Error> = encoder.multithread(workers); Ok(encoder) } @@ -59,13 +76,39 @@ pub fn compress_encrypt_write_streaming( profile: bool, label: &str, ) -> std::io::Result<()> +where + F: FnOnce(&mut zstd::Encoder<'_, Vec>) -> std::io::Result<()>, +{ + compress_encrypt_write_streaming_with_workers( + write_fn, + path, + zstd_level, + default_zstd_workers(), + profile, + label, + ) +} + +/// [`compress_encrypt_write_streaming`] with an explicit zstd worker count. +/// +/// # Errors +/// +/// Returns an I/O error if compression, encryption, or file writing fails. +pub fn compress_encrypt_write_streaming_with_workers( + write_fn: F, + path: &Path, + zstd_level: i32, + zstd_workers: u32, + profile: bool, + label: &str, +) -> std::io::Result<()> where F: FnOnce(&mut zstd::Encoder<'_, Vec>) -> std::io::Result<()>, { let t_total = std::time::Instant::now(); let t_compress = std::time::Instant::now(); - let mut encoder = new_zstd_mt_encoder(zstd_level)?; + let mut encoder = new_zstd_encoder_with_workers(zstd_level, zstd_workers)?; write_fn(&mut encoder)?; let compressed = encoder.finish()?; let compress_ms = t_compress.elapsed().as_millis(); From 4703d66a5abf51fced9f7ab3408d017026ea5c4a Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:34:25 -0700 Subject: [PATCH 09/21] perf(daemon): re-warm a parked drive from its cache plus the journal Searching a parked drive re-loaded it with load_drive_with_usn_refresh: the drive's whole MftIndex, the journal applied to it, then a new compact index built while the MftIndex was still alive, with a cache save on top. For the 30.8 M-record D: on the owner's box (field 2026-10-14) that took about three minutes and pushed the daemon to ~28 GB for a 4.9 GB index. The compact cache already is that index as of its last save, so the re-warm now loads it and replays only the journal since the persisted cursor, synchronously, before the body is served (cache::promote_load). It applies only when the replay is exact; otherwise the full rebuild runs as before: - the persisted cursor is outside the live journal (guarded_load rules), or the journal wrapped or was recreated during the replay; - the compact cache is missing or unreadable; - the replay holds more creates/renames than one targeted MFT read covers (4096), so some files would land without size and timestamps. This relies on the persisted cursor never running ahead of the on-disk body, which a274ce02b made true for the journal loop (store only after the write; a parked drive's save ticks write nothing). Batches for a drive that is not warm are now held rather than dropped. The journal loop keeps polling a parked drive, and its batches were discarded because they could not be applied. The promote's replay covers everything up to its last read, but the changes between that read and the moment the promoted body was installed were lost, and a file created in that window stayed missing from search (minutes wide on the old rebuild path). The applier now puts such a batch back at the front of the drive's pending buffer (newest 100 000 kept), and the loop keeps its apply cadence armed while anything is pending, so the first apply after the install picks it up. Replaying changes the promote already covered is harmless: applying a change is idempotent and the batches keep their order. --- crates/uffs-daemon/src/cache/body_loader.rs | 24 ++- crates/uffs-daemon/src/cache/journal_loop.rs | 13 ++ .../src/cache/journal_loop/sources.rs | 2 +- crates/uffs-daemon/src/cache/journal_sink.rs | 148 +++++++++++++--- .../src/cache/journal_sink/tests.rs | 61 +++++++ crates/uffs-daemon/src/cache/mod.rs | 1 + crates/uffs-daemon/src/cache/promote_load.rs | 167 ++++++++++++++++++ crates/uffs-daemon/src/index/journal.rs | 8 + 8 files changed, 387 insertions(+), 37 deletions(-) create mode 100644 crates/uffs-daemon/src/cache/promote_load.rs diff --git a/crates/uffs-daemon/src/cache/body_loader.rs b/crates/uffs-daemon/src/cache/body_loader.rs index 39b31cd41..2ec8b2b08 100644 --- a/crates/uffs-daemon/src/cache/body_loader.rs +++ b/crates/uffs-daemon/src/cache/body_loader.rs @@ -81,7 +81,16 @@ pub(crate) struct DiskBodyLoader; impl BodyLoader for DiskBodyLoader { fn load(&self, letter: uffs_mft::platform::DriveLetter) -> Option> { - // Phase 5 (#94): primary path — USN-refreshed re-promote. + // Fast path: the compact cache plus the journal changes since its + // last save, replayed synchronously before the body is served. + // Seconds and one index's worth of memory instead of a full + // rebuild; `None` whenever the replay could not be exact. + #[cfg(windows)] + if let Some(body) = super::promote_load::load_parked_drive(letter) { + return Some(Arc::new(body)); + } + + // Phase 5 (#94): full rebuild — USN-refreshed re-promote. // On Windows this applies USN deltas to the cached MftIndex // and rebuilds the compact index, so the body served to the // search hot path reflects the live filesystem state since @@ -91,13 +100,12 @@ impl BodyLoader for DiskBodyLoader { // // NB: the startup warm-load guard (`cache::guarded_load`) is // deliberately NOT used here. Re-promote runs while the - // daemon is live, and the per-shard journal loop keeps - // advancing its persisted cursor while the shard is demoted - // (the apply no-ops with no warm body). Serving the on-disk - // compact cache then deferring to that loop would strand the - // `[demote, now]` delta forever — the loop's cursor is already - // past it. A synchronous refresh is the only correct choice - // on this path. + // daemon is live, and the per-shard journal loop's in-memory + // cursor moved on while the shard was demoted. Serving the + // on-disk compact cache then deferring to that loop would + // strand the `[demote, now]` delta — so the delta is replayed + // synchronously, either by the fast path above or by this + // full rebuild. match uffs_core::compact_loader::load_drive_with_usn_refresh(letter) { Ok((body, _timing)) => return Some(Arc::new(body)), Err(err) => { diff --git a/crates/uffs-daemon/src/cache/journal_loop.rs b/crates/uffs-daemon/src/cache/journal_loop.rs index 5ab58e3f0..1d5d1ecaf 100644 --- a/crates/uffs-daemon/src/cache/journal_loop.rs +++ b/crates/uffs-daemon/src/cache/journal_loop.rs @@ -199,6 +199,16 @@ pub(crate) trait PatchSink: Send + Sync + 'static { /// Fire-and-forget, same as `trigger_save`. fn trigger_apply(&self, letter: uffs_mft::platform::DriveLetter); + /// Whether changes for `letter` are buffered and not yet applied. + /// + /// The production sink puts a batch back when its drive is not warm + /// yet; reporting it here keeps the loop's apply cadence armed, so the + /// held changes are applied as soon as the drive is, even when no new + /// change arrives. Sinks that never hold anything keep the default. + fn has_pending(&self, _letter: uffs_mft::platform::DriveLetter) -> bool { + false + } + /// Notify the sink that the USN journal for `letter` was /// detected to have wrapped (Phase 7 task 7.7). /// @@ -537,6 +547,9 @@ fn process_tick( ) { if changes.is_empty() { tracing::trace!(drive = %letter, "Journal poll: no changes"); + if sink.has_pending(letter) { + apply_trigger.record(); + } } else { let accepted = sink.accept(letter, changes); save_trigger.record(changes.len() as u64); diff --git a/crates/uffs-daemon/src/cache/journal_loop/sources.rs b/crates/uffs-daemon/src/cache/journal_loop/sources.rs index e52d518a7..223a82721 100644 --- a/crates/uffs-daemon/src/cache/journal_loop/sources.rs +++ b/crates/uffs-daemon/src/cache/journal_loop/sources.rs @@ -174,7 +174,7 @@ impl WindowsJournalSource { /// 500 ms window; cap the read so a single poll can't stall the loop. /// Records past the cap keep `meta = None` for this poll and are /// backfilled on a subsequent one (or by the next full re-warm). - const MAX_TARGETED_READS_PER_POLL: usize = 4096; + pub(crate) const MAX_TARGETED_READS_PER_POLL: usize = 4096; /// Issue one batched targeted MFT read for the created/renamed FRSes in /// `changes` and attach the recovered [`uffs_mft::usn::RecordMeta`] to diff --git a/crates/uffs-daemon/src/cache/journal_sink.rs b/crates/uffs-daemon/src/cache/journal_sink.rs index e72cb1a5c..7a393bbe4 100644 --- a/crates/uffs-daemon/src/cache/journal_sink.rs +++ b/crates/uffs-daemon/src/cache/journal_sink.rs @@ -177,9 +177,23 @@ pub(crate) struct RegistryPatchSink { /// tick cadence. Poisoning is recovered via /// [`std::sync::PoisonError::into_inner`] (matching the /// `lock_journal_handles` helper in `index/journal.rs`). - pending: Mutex>>, + pending: PendingBuffers, } +/// The per-letter pending buffers, shared between the sink (which fills +/// them) and the applier task (which puts a not-yet-warm drive's batch +/// back, see [`requeue_front`]). +type PendingBuffers = Arc>>>; + +/// Newest changes a drive that is not warm may hold for its next apply. +/// +/// While a drive is parked its changes cannot be applied; the next promote +/// replays the journal from the persisted cursor itself, so only the changes +/// that arrive after that replay's last read are needed — seconds' worth. +/// Keeping the newest 100 000 covers that with room to spare and bounds the +/// memory of a drive that stays parked for days. +const MAX_HELD_CHANGES: usize = 100_000; + impl RegistryPatchSink { /// Construct a sink + spawn its applier task. /// @@ -232,17 +246,17 @@ impl RegistryPatchSink { ) -> (Arc, JoinHandle<()>) { let (apply_tx, apply_rx) = mpsc::unbounded_channel(); let weak = Arc::downgrade(idx); + let pending: PendingBuffers = Arc::default(); // The cursor store lives on the applier task (it persists the // cursor in lockstep with a successful body save); the sink // itself only forwards the cursor inside `ApplyMsg::Save`. - let handle = tokio::spawn(applier_task(apply_rx, weak, cursor_store)); - ( - Arc::new(Self { - apply_tx, - pending: Mutex::new(HashMap::new()), - }), - handle, - ) + let handle = tokio::spawn(applier_task( + apply_rx, + weak, + cursor_store, + Arc::clone(&pending), + )); + (Arc::new(Self { apply_tx, pending }), handle) } /// Acquire the pending-buffer mutex, recovering from poison. @@ -254,9 +268,7 @@ impl RegistryPatchSink { fn lock_pending( &self, ) -> std::sync::MutexGuard<'_, HashMap>> { - self.pending - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) + lock_buffers(&self.pending) } /// Test-only constructor: build a sink with an explicit @@ -271,7 +283,7 @@ impl RegistryPatchSink { ( Arc::new(Self { apply_tx, - pending: Mutex::new(HashMap::new()), + pending: Arc::default(), }), apply_rx, ) @@ -296,6 +308,12 @@ impl PatchSink for RegistryPatchSink { true } + fn has_pending(&self, letter: uffs_mft::platform::DriveLetter) -> bool { + self.lock_pending() + .get(&letter) + .is_some_and(|buffered| !buffered.is_empty()) + } + fn trigger_save( &self, letter: uffs_mft::platform::DriveLetter, @@ -369,6 +387,7 @@ async fn applier_task( mut rx: mpsc::UnboundedReceiver, idx_weak: Weak, cursor_store: Arc, + pending: PendingBuffers, ) { // A message pulled off the channel during apply-coalescing that did not // belong to the merged run (different letter, or a Save/Wrap); processed @@ -409,7 +428,12 @@ async fn applier_task( } else { msg }; - dispatch_msg(&idx_strong, &cursor_store, &save_slot, to_dispatch).await; + let ctx = DispatchCtx { + cursor_store: &cursor_store, + save_slot: &save_slot, + pending: &pending, + }; + dispatch_msg(&idx_strong, &ctx, to_dispatch).await; } tracing::debug!( target: "shard.journal", @@ -453,12 +477,7 @@ fn coalesce_apply_run( /// also keeps the `Weak::upgrade` lifecycle decision in the parent /// (visible at the recv-loop level) and the per-variant dispatch /// in this helper (focused, easy to extend). -async fn dispatch_msg( - idx: &Arc, - cursor_store: &Arc, - save_slot: &Arc, - msg: ApplyMsg, -) { +async fn dispatch_msg(idx: &Arc, ctx: &DispatchCtx<'_>, msg: ApplyMsg) { match msg { ApplyMsg::Save { letter, @@ -467,21 +486,24 @@ async fn dispatch_msg( cursor, } => { let reason_str = save_reason_str(reason); + let Some(batch) = hold_unless_warm(idx, ctx.pending, letter, changes).await else { + return; + }; // One save at a time: while the previous one is still // writing, apply this batch in memory (search stays live) // and leave the write to a later save tick. The cursor is // not stored, so a cold start replays from the last body // that actually reached disk. - let Some(permit) = save_slot.try_acquire() else { + let Some(permit) = ctx.save_slot.try_acquire() else { tracing::debug!( target: "shard.journal", drive = %letter, reason = reason_str, - change_count = changes.len(), + change_count = batch.len(), "Save deferred: the previous save is still writing; applied in memory only", ); let _applied = idx - .handle_journal_apply(letter, "save-deferred", changes) + .handle_journal_apply(letter, "save-deferred", batch) .await; return; }; @@ -492,20 +514,27 @@ async fn dispatch_msg( // yields `None`: nothing is written and the cursor stays // pinned to the last real body save, which the startup // warm-load guard relies on (`cache::guarded_load`). - if let Some(body) = idx.handle_journal_save(letter, reason_str, changes).await { - idx.spawn_journal_persist(letter, body, cursor, Arc::clone(cursor_store), permit); + if let Some(body) = idx.handle_journal_save(letter, reason_str, batch).await { + idx.spawn_journal_persist( + letter, + body, + cursor, + Arc::clone(ctx.cursor_store), + permit, + ); } } ApplyMsg::Apply { letter, changes } => { + let Some(batch) = hold_unless_warm(idx, ctx.pending, letter, changes).await else { + return; + }; // Apply tick: patch the body + swap it into the registry so // search goes live, but do NOT persist the compact cache or // advance the on-disk cursor. Disk persistence + cursor // advance stay on the rarer `Save` tick; a cold start // re-replays the in-between deltas idempotently from the // last saved cursor. - let _applied = idx - .handle_journal_apply(letter, "apply-tick", changes) - .await; + let _applied = idx.handle_journal_apply(letter, "apply-tick", batch).await; } ApplyMsg::Wrap { letter } => { // Wrap stays on the Phase-7 full-reload path. The @@ -519,6 +548,69 @@ async fn dispatch_msg( } } +/// What [`dispatch_msg`] needs beyond the message: the cursor store, the +/// single background-save slot, and the sink's pending buffers. +struct DispatchCtx<'ctx> { + /// Persists the cursor after a body save reaches disk. + cursor_store: &'ctx Arc, + /// One background save at a time across every drive. + save_slot: &'ctx Arc, + /// Where a not-yet-warm drive's batch goes back to. + pending: &'ctx PendingBuffers, +} + +/// Pass `changes` through when `letter` is warm; otherwise put them back at +/// the front of its pending buffer and return `None`. +/// +/// A batch for a drive that is not warm cannot be applied. Dropping it was +/// safe only for the part the next promote's journal replay covers; the +/// changes that arrive between that replay's last read and the moment the +/// promoted body is installed were lost, and a file created in that window +/// stayed missing from search. Holding the batch (newest +/// [`MAX_HELD_CHANGES`]) lets the first apply after the install pick it up; +/// replaying changes the promote already covered is harmless because +/// applying a change is idempotent and the batches keep their order. +async fn hold_unless_warm( + idx: &Arc, + pending: &PendingBuffers, + letter: uffs_mft::platform::DriveLetter, + changes: Vec, +) -> Option> { + if changes.is_empty() || idx.is_drive_warm(letter).await { + return Some(changes); + } + requeue_front(pending, letter, changes); + None +} + +/// Put `older` back in front of whatever `letter` has buffered since, keeping +/// the newest [`MAX_HELD_CHANGES`]. +fn requeue_front( + pending: &PendingBuffers, + letter: uffs_mft::platform::DriveLetter, + mut older: Vec, +) { + let mut guard = lock_buffers(pending); + let slot = guard.entry(letter).or_default(); + older.append(slot); + let excess = older.len().saturating_sub(MAX_HELD_CHANGES); + if excess > 0 { + drop(older.drain(..excess)); + } + *slot = older; + drop(guard); +} + +/// Lock the pending buffers, recovering from poison (see +/// [`RegistryPatchSink::lock_pending`]). +fn lock_buffers( + pending: &PendingBuffers, +) -> std::sync::MutexGuard<'_, HashMap>> { + pending + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + /// Map a [`SaveReason`] to its stable diagnostic string for tracing /// and the [`IndexManager::handle_journal_refresh`] success/failure /// log. Centralised here so log strings stay consistent across the diff --git a/crates/uffs-daemon/src/cache/journal_sink/tests.rs b/crates/uffs-daemon/src/cache/journal_sink/tests.rs index e3c9e27fe..3586beb5f 100644 --- a/crates/uffs-daemon/src/cache/journal_sink/tests.rs +++ b/crates/uffs-daemon/src/cache/journal_sink/tests.rs @@ -678,3 +678,64 @@ async fn coalesce_does_not_swallow_save() { "Save must be carried out intact, not merged into the apply batch", ); } + +/// A batch that reaches the applier while its drive is not warm goes back +/// to the front of the pending buffer, in order, instead of being dropped — +/// the first apply after a promote installs the body picks it up. +#[tokio::test] +async fn batch_for_drive_that_is_not_warm_is_held_in_order() { + let idx = fresh_index_manager(); // no drives registered: nothing is warm + let (sink, applier) = RegistryPatchSink::spawn_with_applier(&idx, RecordingCursorStore::new()); + let letter = uffs_mft::platform::DriveLetter::C; + + sink.accept(letter, &[make_change(1), make_change(2)]); + sink.trigger_apply(letter); + let deadline = tokio::time::Instant::now() + core::time::Duration::from_secs(5); + while !sink.has_pending(letter) { + assert!( + tokio::time::Instant::now() < deadline, + "the applier must hand the batch back within 5 s", + ); + tokio::time::sleep(core::time::Duration::from_millis(5)).await; + } + + // A change buffered after the hold queues behind the held batch. + sink.accept(letter, &[make_change(3)]); + assert_eq!( + pending_frs_for_letter(&sink, letter), + Some(vec![1, 2, 3]), + "held changes stay in front of newer ones", + ); + + drop(sink); + tokio::time::timeout(core::time::Duration::from_secs(5), applier) + .await + .expect("applier must exit within 5 s") + .expect("applier must not panic"); +} + +/// The hold keeps the newest [`MAX_HELD_CHANGES`]: a drive parked for a long +/// time cannot grow its buffer without bound, and the changes that matter +/// after a promote are the newest ones. +#[test] +fn requeue_keeps_the_newest_changes_up_to_the_cap() { + let pending: PendingBuffers = Arc::default(); + let letter = uffs_mft::platform::DriveLetter::C; + lock_buffers(&pending).insert(letter, vec![make_change(u64::MAX)]); + + let older: Vec = (0..MAX_HELD_CHANGES as u64).map(make_change).collect(); + requeue_front(&pending, letter, older); + + let held = lock_buffers(&pending).remove(&letter).unwrap_or_default(); + assert_eq!(held.len(), MAX_HELD_CHANGES, "the buffer is capped"); + assert_eq!( + held.first().map(|change| change.frs.raw()), + Some(1), + "the oldest change is dropped" + ); + assert_eq!( + held.last().map(|change| change.frs.raw()), + Some(u64::MAX), + "the newest change survives" + ); +} diff --git a/crates/uffs-daemon/src/cache/mod.rs b/crates/uffs-daemon/src/cache/mod.rs index 64939e3d8..a54cb85ec 100644 --- a/crates/uffs-daemon/src/cache/mod.rs +++ b/crates/uffs-daemon/src/cache/mod.rs @@ -39,6 +39,7 @@ pub(crate) mod journal_sink; pub(crate) mod policy; pub(crate) mod prefetch; pub(crate) mod pressure; +pub(crate) mod promote_load; pub(crate) mod registry; pub(crate) mod shard; pub(crate) mod working_set; diff --git a/crates/uffs-daemon/src/cache/promote_load.rs b/crates/uffs-daemon/src/cache/promote_load.rs new file mode 100644 index 000000000..ee691e36d --- /dev/null +++ b/crates/uffs-daemon/src/cache/promote_load.rs @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright (c) 2025-2026 SKY, LLC. + +//! Fast re-warm of a parked drive: the on-disk compact cache plus the +//! journal changes since it was written. +//! +//! The re-promote path used to rebuild from scratch every time +//! (`load_drive_with_usn_refresh`): load the drive's whole `MftIndex`, +//! apply the journal, build a new compact index while the `MftIndex` is +//! still alive, and start a cache save on top. For the owner's 30.8 M-record +//! D: that took about three minutes and pushed the daemon to ~28 GB (field +//! 2026-10-14) for a 4.9 GB index. The compact cache already is that index +//! as of its last save, so this path loads it and replays only the changes +//! since, synchronously, before the body is served. +//! +//! It applies only when the replay is known to be complete and exact; every +//! other case returns `None` and the caller falls back to the full rebuild: +//! +//! * the persisted cursor is outside the live journal's window, or the journal +//! wrapped or was recreated during the replay ([`super::guarded_load`]'s +//! `decide_strategy` rules); +//! * the compact cache is missing or unreadable; +//! * the replay holds more creates/renames than one targeted MFT read covers, +//! so some files would land without size and timestamps. +//! +//! The persisted cursor never runs ahead of the on-disk body: the journal +//! loop stores it only after a save has written the file +//! (`crate::index::journal_save`), and a parked drive's save ticks write +//! nothing. Replaying from it can therefore only overlap the body, never +//! leave a gap, and an overlap is harmless because applying a change is +//! idempotent. + +/// Whether a replay of `changes` can be applied with full metadata: every +/// created or renamed file needs one targeted MFT read, and a single poll +/// reads at most `max_targeted_reads`. +#[cfg_attr( + all(not(windows), not(test)), + expect(dead_code, reason = "used by the Windows re-warm path and its tests") +)] +pub(crate) fn replay_fits( + changes: &[uffs_mft::usn::FileChange], + max_targeted_reads: usize, +) -> bool { + changes + .iter() + .filter(|change| change.created || change.renamed) + .count() + <= max_targeted_reads +} + +/// Re-warm `letter` from its compact cache plus the journal since the +/// persisted cursor, or `None` when that cannot be done exactly (see the +/// module docs); the caller then rebuilds from scratch. +#[cfg(windows)] +pub(crate) fn load_parked_drive( + letter: uffs_mft::platform::DriveLetter, +) -> Option { + use std::time::Instant; + + use super::cursor_store::DiskCursorStore; + use super::guarded_load::{WarmLoadStrategy, decide_strategy}; + use super::journal_loop::sources::WindowsJournalSource; + use super::journal_loop::{CursorStore as _, JournalSource as _}; + + let started = Instant::now(); + let info = uffs_mft::usn::query_usn_journal(letter) + .map_err(|err| fallback(letter, "USN journal unavailable", &err)) + .ok()?; + let cursor = DiskCursorStore::new(uffs_mft::cache::cache_dir()).load(letter); + if decide_strategy(cursor, info.first_usn.raw(), info.next_usn.raw()) + != WarmLoadStrategy::FastFromCompactCache + { + fallback(letter, "persisted cursor outside the live journal", &cursor); + return None; + } + + let mut body = uffs_core::compact_cache::load_compact_cache(letter, u64::MAX, 0, true) + .map_err(|err| fallback(letter, "compact cache unusable", &err)) + .ok()?; + let cache_ms = started.elapsed().as_millis(); + + let replay = WindowsJournalSource::new(letter) + .poll(cursor) + .map_err(|err| fallback(letter, "journal read failed", &err)) + .ok()?; + if replay.next_cursor == 0 || replay.journal_id != info.journal_id { + fallback( + letter, + "journal wrapped or was recreated during the re-warm", + &cursor, + ); + return None; + } + if !replay_fits( + &replay.changes, + WindowsJournalSource::MAX_TARGETED_READS_PER_POLL, + ) { + fallback( + letter, + "too many created/renamed files since the last save to read their metadata", + &replay.changes.len(), + ); + return None; + } + + let stats = uffs_core::compact_loader::apply_usn_patch(&mut body, &replay.changes); + tracing::info!( + target: "shard.transition", + drive = %letter, + cursor, + replayed = replay.changes.len(), + created = stats.created, + deleted = stats.deleted, + renamed = stats.renamed, + cache_ms, + total_ms = started.elapsed().as_millis(), + "parked drive re-warmed from its compact cache plus the journal since the last save", + ); + Some(body) +} + +/// Log why the fast re-warm is not used for `letter`. +#[cfg(windows)] +fn fallback(letter: uffs_mft::platform::DriveLetter, why: &str, detail: &dyn core::fmt::Display) { + tracing::info!( + target: "shard.transition", + drive = %letter, + detail = %detail, + "re-warm falls back to a full rebuild: {why}", + ); +} + +#[cfg(test)] +mod tests { + use uffs_mft::usn::FileChange; + + use super::replay_fits; + + fn change(created: bool, renamed: bool, deleted: bool) -> FileChange { + FileChange { + created, + renamed, + deleted, + ..FileChange::default() + } + } + + #[test] + fn only_creates_and_renames_count_against_the_read_budget() { + let changes = [ + change(true, false, false), + change(false, true, false), + change(false, false, true), + change(false, false, true), + ]; + assert!(replay_fits(&changes, 2), "deletes need no metadata read"); + assert!( + !replay_fits(&changes, 1), + "two creates/renames exceed a budget of one" + ); + } + + #[test] + fn empty_replay_always_fits() { + assert!(replay_fits(&[], 0)); + } +} diff --git a/crates/uffs-daemon/src/index/journal.rs b/crates/uffs-daemon/src/index/journal.rs index 055d857f9..80cd9e4b6 100644 --- a/crates/uffs-daemon/src/index/journal.rs +++ b/crates/uffs-daemon/src/index/journal.rs @@ -256,6 +256,14 @@ impl IndexManager { BodyApplyOutcome::Applied(new_body) } + /// Whether `letter` is registered with a body in memory (Warm or Hot), + /// so a journal batch for it can be applied. + pub(crate) async fn is_drive_warm(&self, letter: uffs_mft::platform::DriveLetter) -> bool { + self.snapshot_shard_for_letter(letter) + .await + .is_some_and(|shard| shard.body().is_some()) + } + /// Snapshot the shard for `letter` from the registry read-lock, /// cloning the `Arc` so subsequent registry /// mutations (demote, promote, `replace_warm_body`) leave this From 57ac06d7104947a99c8cf7e13690f20433b65672 Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:34:47 -0700 Subject: [PATCH 10/21] fix(daemon): seed the startup cursor only after the rebuilt cache is written guarded_load::full_rebuild stored the USN cursor as soon as load_drive returned, while load_drive's compact-cache save was still running in the background. A crash or failed write in between left the cursor ahead of the cache file, so the next warm start served the older file and replayed only from the newer cursor; the changes in between never reached the index. load_drive_then / save_compact_cache_background_then run a callback once the atomic write has succeeded, and full_rebuild seeds the cursor there. load_drive and save_compact_cache_background keep their signatures. --- crates/uffs-core/src/compact.rs | 3 +- crates/uffs-core/src/compact_cache.rs | 22 ++++++++++-- crates/uffs-core/src/compact_loader.rs | 26 +++++++++++++- crates/uffs-daemon/src/cache/guarded_load.rs | 38 +++++++++++--------- 4 files changed, 69 insertions(+), 20 deletions(-) diff --git a/crates/uffs-core/src/compact.rs b/crates/uffs-core/src/compact.rs index 010f8c75a..880b62d2f 100644 --- a/crates/uffs-core/src/compact.rs +++ b/crates/uffs-core/src/compact.rs @@ -23,7 +23,8 @@ pub use crate::compact_loader::apply_usn_patch; // Re-export loader types and functions so callers can still use `compact::*`. #[expect(deprecated, reason = "re-export kept for backward compatibility")] pub use crate::compact_loader::{ - IndexSource, LoadTiming, MftSource, PatchStats, load_drive, load_mft_file, refresh_drive, + IndexSource, LoadTiming, MftSource, PatchStats, load_drive, load_drive_then, load_mft_file, + refresh_drive, }; use crate::compact_storage::ColumnStorage; use crate::path_trie::PathTrie; diff --git a/crates/uffs-core/src/compact_cache.rs b/crates/uffs-core/src/compact_cache.rs index 1ef6778f8..5ca349ab6 100644 --- a/crates/uffs-core/src/compact_cache.rs +++ b/crates/uffs-core/src/compact_cache.rs @@ -1079,6 +1079,21 @@ fn refuse_delta_carrying(index: &DriveCompactIndex) -> io::Result<()> { /// # Errors /// Returns an error only if compression or directory creation fails. pub fn save_compact_cache_background(index: &DriveCompactIndex) -> io::Result<()> { + save_compact_cache_background_then(index, || {}) +} + +/// [`save_compact_cache_background`], running `on_written` on the background +/// thread once the atomic write has succeeded (not when it fails). +/// +/// # Errors +/// Returns an error only if compression or directory creation fails. +pub fn save_compact_cache_background_then( + index: &DriveCompactIndex, + on_written: F, +) -> io::Result<()> +where + F: FnOnce() + Send + 'static, +{ // Boundary enforcement of the "on-disk cache is always delta-free" // invariant (see `refuse_delta_carrying`). refuse_delta_carrying(index)?; @@ -1111,8 +1126,11 @@ pub fn save_compact_cache_background(index: &DriveCompactIndex) -> io::Result<() std::thread::Builder::new() .name(format!("compact-save-{drive}")) .spawn(move || { - if let Err(err) = encrypt_and_write(drive, compressed, &path, profile) { - tracing::warn!(drive = %drive, error = %err, "Background compact cache save failed"); + match encrypt_and_write(drive, compressed, &path, profile) { + Ok(()) => on_written(), + Err(err) => { + tracing::warn!(drive = %drive, error = %err, "Background compact cache save failed"); + } } }) .map_err(|err| io::Error::other(format!("spawn failed: {err}")))?; diff --git a/crates/uffs-core/src/compact_loader.rs b/crates/uffs-core/src/compact_loader.rs index 74a9032b0..c3ff974b7 100644 --- a/crates/uffs-core/src/compact_loader.rs +++ b/crates/uffs-core/src/compact_loader.rs @@ -109,6 +109,28 @@ pub fn load_drive( source: &MftSource, no_cache: bool, ) -> anyhow::Result<(DriveCompactIndex, LoadTiming)> { + load_drive_then(source, no_cache, || {}) +} + +/// [`load_drive`], running `on_cache_written` once the background compact +/// cache save has written the file (never when no save runs or it fails). +/// +/// A caller that persists anything describing the saved body — the daemon's +/// USN cursor — must do it here, not when this function returns: the save +/// is still running then, and a crash before it finishes would leave the +/// description ahead of the file on disk. +/// +/// # Errors +/// +/// Returns an error if the MFT data cannot be read or parsed. +pub fn load_drive_then( + source: &MftSource, + no_cache: bool, + on_cache_written: F, +) -> anyhow::Result<(DriveCompactIndex, LoadTiming)> +where + F: FnOnce() + Send + 'static, +{ let drive_letter = match source { MftSource::File(path, drive_override) => drive_override.unwrap_or_else(|| { let stem = path @@ -173,7 +195,9 @@ pub fn load_drive( // point-in-time capture (see `MftSource::is_ephemeral_device`). if !no_cache && !source.is_ephemeral_device() { let t_compact_save = Instant::now(); - if let Err(err) = crate::compact_cache::save_compact_cache_background(&compact) { + if let Err(err) = + crate::compact_cache::save_compact_cache_background_then(&compact, on_cache_written) + { tracing::warn!(drive = %drive_letter, error = %err, "Failed to start compact cache save"); } let compact_save_ms = t_compact_save.elapsed().as_millis(); diff --git a/crates/uffs-daemon/src/cache/guarded_load.rs b/crates/uffs-daemon/src/cache/guarded_load.rs index aae2efbd8..3c68b8b2a 100644 --- a/crates/uffs-daemon/src/cache/guarded_load.rs +++ b/crates/uffs-daemon/src/cache/guarded_load.rs @@ -196,9 +196,9 @@ pub(crate) fn load_live_drive( } } -/// Run the synchronous full rebuild ([`uffs_core::compact::load_drive`]) -/// and, on success, persist a cursor so a subsequent restart can take -/// the fast path. +/// Run the synchronous full rebuild ([`uffs_core::compact::load_drive_then`]) +/// and, once its cache file is written, persist a cursor so a subsequent +/// restart can take the fast path. /// /// The persisted cursor is the journal's `next_usn` **captured before** /// the rebuild's MFT read. The freshly built body reflects USN state up @@ -214,23 +214,29 @@ fn full_rebuild( use crate::cache::cursor_store::DiskCursorStore; use crate::cache::journal_loop::CursorStore as _; - let result = - uffs_core::compact::load_drive(&uffs_core::compact::MftSource::Live(letter), no_cache)?; - // Best-effort cursor seed so the next warm load is fast. Only when // caching is enabled and we have a live journal reading; never // regress a newer persisted cursor (the journal loop may have - // advanced past this pre-read lower bound already). - if !no_cache && let Some(journal) = info { - let store = DiskCursorStore::new(uffs_mft::cache::cache_dir()); - let pre_read = u64::try_from(journal.next_usn.raw()).unwrap_or(0); - let seed = store.load(letter).max(pre_read); - if seed != 0 { + // advanced past this pre-read lower bound already). Stored only once + // the rebuilt body's cache file is on disk: seeding when `load_drive` + // returns (its save still running) let a crash in between leave the + // cursor ahead of the file, and the next warm load skip the changes + // in between for good. + let pre_read = info.filter(|_journal| !no_cache).map_or(0, |journal| { + u64::try_from(journal.next_usn.raw()).unwrap_or(0) + }); + uffs_core::compact::load_drive_then( + &uffs_core::compact::MftSource::Live(letter), + no_cache, + move || { + if pre_read == 0 { + return; + } + let store = DiskCursorStore::new(uffs_mft::cache::cache_dir()); + let seed = store.load(letter).max(pre_read); store.store(letter, seed); - } - } - - Ok(result) + }, + ) } #[cfg(test)] From 453bad16960e399648f00f75a2b46452b7828d27 Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:39:36 -0700 Subject: [PATCH 11/21] fix(deps): ctutils 0.4.6 for RUSTSEC-2026-0385 ctutils 0.4.3 generated masks and compared u128 values in non-constant time (RUSTSEC-2026-0385). It sits under aes-gcm, which encrypts every on-disk cache through uffs-security; cargo deny blocked the push on it. Vetted by the existing trusted-publisher entry for RustCrypto/utils (0.4.6 published 2026-10-25 by the same publisher); no exemption. --- Cargo.lock | 4 ++-- supply-chain/imports.lock | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5f3fac5c1..3efc9f72c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -920,9 +920,9 @@ dependencies = [ [[package]] name = "ctutils" -version = "0.4.3" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03bb0e1cc970d482d121d9a1744999169b69a07470b3d644a7894e53fcaf4574" +checksum = "fe3e9d21d39e6f39ff46e03d5b621424f88ee23d8a1672c6bfc9ccc7f2da4f6a" dependencies = [ "cmov", ] diff --git a/supply-chain/imports.lock b/supply-chain/imports.lock index 22e7b9e31..86c2f669d 100644 --- a/supply-chain/imports.lock +++ b/supply-chain/imports.lock @@ -138,8 +138,8 @@ when = "2026-05-20" trusted-publisher = "github:RustCrypto/block-modes" [[publisher.ctutils]] -version = "0.4.3" -when = "2026-10-05" +version = "0.4.6" +when = "2026-10-25" trusted-publisher = "github:RustCrypto/utils" [[publisher.digest]] From b96eb7f49f3708c8c137ca650c61527765af5c92 Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:39:38 -0700 Subject: [PATCH 12/21] chore(vet): replace nine exemptions with imported audits cargo vet reported exemptions that existing imports already cover. `cargo vet prune` drops the exemptions for chacha20, getrandom, humantime, rand_core, rand_pcg, sync_wrapper, tinyvec, untrusted and web-time and records the Google, Mozilla and ISRG audits that cover them instead: fully audited 248 -> 256, exempted 144 -> 138. Trust only narrows; nothing new is trusted. --- supply-chain/config.toml | 36 --------- supply-chain/imports.lock | 155 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 155 insertions(+), 36 deletions(-) diff --git a/supply-chain/config.toml b/supply-chain/config.toml index b8b066283..89045ee55 100644 --- a/supply-chain/config.toml +++ b/supply-chain/config.toml @@ -262,10 +262,6 @@ criteria = "safe-to-deploy" version = "0.2.4" criteria = "safe-to-deploy" -[[exemptions.chacha20]] -version = "0.10.1" -criteria = "safe-to-deploy" - [[exemptions.chrono]] version = "0.4.44" criteria = "safe-to-deploy" @@ -426,10 +422,6 @@ criteria = "safe-to-deploy" version = "0.3.4" criteria = "safe-to-deploy" -[[exemptions.getrandom]] -version = "0.4.3" -criteria = "safe-to-deploy" - [[exemptions.half]] version = "2.7.1" criteria = "safe-to-deploy" @@ -450,10 +442,6 @@ criteria = "safe-to-deploy" version = "0.4.2" criteria = "safe-to-deploy" -[[exemptions.humantime]] -version = "2.3.0" -criteria = "safe-to-deploy" - [[exemptions.iana-time-zone]] version = "0.1.65" criteria = "safe-to-deploy" @@ -638,14 +626,6 @@ criteria = "safe-to-deploy" version = "0.10.0" criteria = "safe-to-deploy" -[[exemptions.rand_core]] -version = "0.10.1" -criteria = "safe-to-deploy" - -[[exemptions.rand_pcg]] -version = "0.10.2" -criteria = "safe-to-deploy" - [[exemptions.raw-cpuid]] version = "11.6.0" criteria = "safe-to-deploy" @@ -814,10 +794,6 @@ criteria = "safe-to-deploy" version = "0.1.0" criteria = "safe-to-deploy" -[[exemptions.sync_wrapper]] -version = "1.0.2" -criteria = "safe-to-deploy" - [[exemptions.sysinfo]] version = "0.37.2" criteria = "safe-to-deploy" @@ -842,10 +818,6 @@ criteria = "safe-to-deploy" version = "0.8.3" criteria = "safe-to-deploy" -[[exemptions.tinyvec]] -version = "1.11.0" -criteria = "safe-to-deploy" - [[exemptions.tokio-macros]] version = "2.7.0" criteria = "safe-to-deploy" @@ -894,10 +866,6 @@ criteria = "safe-to-deploy" version = "0.5.2" criteria = "safe-to-deploy" -[[exemptions.untrusted]] -version = "0.9.0" -criteria = "safe-to-deploy" - [[exemptions.unty]] version = "0.0.4" criteria = "safe-to-deploy" @@ -926,10 +894,6 @@ criteria = "safe-to-deploy" version = "0.4.2" criteria = "safe-to-deploy" -[[exemptions.web-time]] -version = "1.1.0" -criteria = "safe-to-deploy" - [[exemptions.which]] version = "8.0.2" criteria = "safe-to-deploy" diff --git a/supply-chain/imports.lock b/supply-chain/imports.lock index 86c2f669d..e2a8e848c 100644 --- a/supply-chain/imports.lock +++ b/supply-chain/imports.lock @@ -956,6 +956,12 @@ who = "Pat Hickey " criteria = "safe-to-deploy" delta = "0.3.28 -> 0.3.31" +[[audits.bytecode-alliance.audits.getrandom]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "0.4.1 -> 0.4.2" +notes = "Nothing awry in this update, standard updates for some platforms and other misc things." + [[audits.bytecode-alliance.audits.heck]] who = "Alex Crichton " criteria = "safe-to-deploy" @@ -1134,6 +1140,16 @@ criteria = "safe-to-deploy" delta = "1.13.2 -> 1.14.0" notes = "Minor new feature, nothing out of the ordinary." +[[audits.bytecode-alliance.audits.tinyvec]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +version = "1.6.0" +notes = """ +This crate, while it implements collections, does so without `std::*` APIs and +without `unsafe`. Skimming the crate everything looks reasonable and what one +would expect from idiomatic safe collections in Rust. +""" + [[audits.bytecode-alliance.audits.tracing-core]] who = "Alex Crichton " criteria = "safe-to-deploy" @@ -1497,6 +1513,15 @@ For more detailed unsafe review notes please see https://crrev.com/c/6362797 """ aggregated-from = "https://chromium.googlesource.com/chromium/src/+/main/third_party/rust/chromium_crates_io/supply-chain/audits.toml?format=TEXT" +[[audits.google.audits.rand_core]] +who = "Lukasz Anforowicz " +criteria = "safe-to-deploy" +version = "0.6.4" +notes = """ +For more detailed unsafe review notes please see https://crrev.com/c/6362797 +""" +aggregated-from = "https://chromium.googlesource.com/chromium/src/+/main/third_party/rust/chromium_crates_io/supply-chain/audits.toml?format=TEXT" + [[audits.google.audits.smallvec]] who = "Manish Goregaokar " criteria = "safe-to-deploy" @@ -1560,6 +1585,42 @@ criteria = "safe-to-run" version = "1.2.1" aggregated-from = "https://chromium.googlesource.com/chromiumos/third_party/rust_crates/+/refs/heads/main/cargo-vet/audits.toml?format=TEXT" +[[audits.google.audits.tinyvec]] +who = "Adrian Taylor " +criteria = "safe-to-deploy" +delta = "1.6.0 -> 1.6.1" +aggregated-from = "https://chromium.googlesource.com/chromium/src/+/main/third_party/rust/chromium_crates_io/supply-chain/audits.toml?format=TEXT" + +[[audits.google.audits.tinyvec]] +who = "Adrian Taylor " +criteria = "safe-to-deploy" +delta = "1.6.1 -> 1.7.0" +aggregated-from = "https://chromium.googlesource.com/chromium/src/+/main/third_party/rust/chromium_crates_io/supply-chain/audits.toml?format=TEXT" + +[[audits.google.audits.tinyvec]] +who = "Dustin J. Mitchell " +criteria = "safe-to-deploy" +delta = "1.7.0 -> 1.8.0" +aggregated-from = "https://chromium.googlesource.com/chromium/src/+/main/third_party/rust/chromium_crates_io/supply-chain/audits.toml?format=TEXT" + +[[audits.google.audits.tinyvec]] +who = "Adrian Taylor " +criteria = "safe-to-deploy" +delta = "1.8.0 -> 1.8.1" +aggregated-from = "https://chromium.googlesource.com/chromium/src/+/main/third_party/rust/chromium_crates_io/supply-chain/audits.toml?format=TEXT" + +[[audits.google.audits.tinyvec]] +who = "Daniel Cheng " +criteria = "safe-to-deploy" +delta = "1.8.1 -> 1.9.0" +notes = """ +Larger delta, but no unsafe code introduced. Deltas for: +- borsh (Binary Object Representation Serializer for Hashing) serialization/deserialization support behind the `borsh` feature. +- trait implementations to interoperate with the generic-array crate +- miscellaneous helper functions and support code, e.g. `into_vec()`. +""" +aggregated-from = "https://chromium.googlesource.com/chromium/src/+/main/third_party/rust/chromium_crates_io/supply-chain/audits.toml?format=TEXT" + [[audits.google.audits.utf8parse]] who = "David Koloski " criteria = "safe-to-deploy" @@ -1578,6 +1639,11 @@ who = "David Cook " criteria = "safe-to-run" version = "0.4.0" +[[audits.isrg.audits.chacha20]] +who = "David Cook " +criteria = "safe-to-deploy" +version = "0.10.0" + [[audits.isrg.audits.criterion-plot]] who = "J.C. Jones " criteria = "safe-to-run" @@ -1588,6 +1654,16 @@ who = "J.C. Jones " criteria = "safe-to-run" delta = "0.8.1 -> 0.8.2" +[[audits.isrg.audits.getrandom]] +who = "David Cook " +criteria = "safe-to-deploy" +delta = "0.3.4 -> 0.4.0" + +[[audits.isrg.audits.getrandom]] +who = "David Cook " +criteria = "safe-to-deploy" +delta = "0.4.0 -> 0.4.1" + [[audits.isrg.audits.inout]] who = "David Cook " criteria = "safe-to-deploy" @@ -1619,6 +1695,21 @@ who = "David Cook " criteria = "safe-to-deploy" delta = "0.9.2 -> 0.10.0" +[[audits.isrg.audits.rand_core]] +who = "David Cook " +criteria = "safe-to-deploy" +delta = "0.6.4 -> 0.9.3" + +[[audits.isrg.audits.rand_core]] +who = "J.C. Jones " +criteria = "safe-to-deploy" +delta = "0.9.3 -> 0.9.5" + +[[audits.isrg.audits.rand_core]] +who = "David Cook " +criteria = "safe-to-deploy" +delta = "0.9.5 -> 0.10.0" + [[audits.isrg.audits.rand_distr]] who = "David Cook " criteria = "safe-to-deploy" @@ -1644,6 +1735,11 @@ who = "David Cook " criteria = "safe-to-deploy" delta = "2.5.0 -> 2.6.1" +[[audits.isrg.audits.untrusted]] +who = "David Cook " +criteria = "safe-to-deploy" +version = "0.7.1" + [[audits.mozilla.wildcard-audits.unicode-normalization]] who = "Manish Goregaokar " criteria = "safe-to-deploy" @@ -1791,6 +1887,12 @@ criteria = "safe-to-deploy" delta = "0.2.1 -> 0.2.2" aggregated-from = "https://raw.githubusercontent.com/mozilla-firefox/firefox/main/supply-chain/audits.toml" +[[audits.mozilla.audits.chacha20]] +who = "Alex Franchuk " +criteria = "safe-to-deploy" +delta = "0.10.0 -> 0.10.2" +aggregated-from = "https://raw.githubusercontent.com/mozilla/cargo-vet/main/supply-chain/audits.toml" + [[audits.mozilla.audits.core_detect]] who = "Henri Sivonen " criteria = "safe-to-deploy" @@ -1918,6 +2020,12 @@ criteria = "safe-to-deploy" delta = "0.3.27 -> 0.3.28" aggregated-from = "https://raw.githubusercontent.com/mozilla-firefox/firefox/main/supply-chain/audits.toml" +[[audits.mozilla.audits.getrandom]] +who = "Alex Franchuk " +criteria = "safe-to-deploy" +delta = "0.4.2 -> 0.4.3" +aggregated-from = "https://raw.githubusercontent.com/mozilla/cargo-vet/main/supply-chain/audits.toml" + [[audits.mozilla.audits.hashbrown]] who = "Erich Gubler " criteria = "safe-to-deploy" @@ -1942,6 +2050,12 @@ criteria = "safe-to-deploy" version = "0.4.3" aggregated-from = "https://raw.githubusercontent.com/mozilla-firefox/firefox/main/supply-chain/audits.toml" +[[audits.mozilla.audits.humantime]] +who = "Max Inden " +criteria = "safe-to-deploy" +version = "2.4.0" +aggregated-from = "https://raw.githubusercontent.com/mozilla-firefox/firefox/main/supply-chain/audits.toml" + [[audits.mozilla.audits.idna]] who = "Valentin Gosu " criteria = "safe-to-deploy" @@ -2095,6 +2209,12 @@ criteria = "safe-to-deploy" delta = "1.0.45 -> 1.0.47" aggregated-from = "https://raw.githubusercontent.com/mozilla/glean/main/supply-chain/audits.toml" +[[audits.mozilla.audits.rand_core]] +who = "Alex Franchuk " +criteria = "safe-to-deploy" +delta = "0.10.0 -> 0.10.1" +aggregated-from = "https://raw.githubusercontent.com/mozilla/cargo-vet/main/supply-chain/audits.toml" + [[audits.mozilla.audits.rand_distr]] who = "Ben Dean-Kawamura " criteria = "safe-to-deploy" @@ -2105,6 +2225,13 @@ As far as I can tell, it does not have any file IO or network access. """ aggregated-from = "https://raw.githubusercontent.com/mozilla-firefox/firefox/main/supply-chain/audits.toml" +[[audits.mozilla.audits.rand_pcg]] +who = "Alex Franchuk " +criteria = "safe-to-deploy" +version = "0.10.2" +notes = "no_std crate with unsafe code forbidden." +aggregated-from = "https://raw.githubusercontent.com/mozilla/cargo-vet/main/supply-chain/audits.toml" + [[audits.mozilla.audits.rmp]] who = "Ben Dean-Kawamura " criteria = "safe-to-deploy" @@ -2199,6 +2326,13 @@ version = "2.5.0" notes = "The goal is to provide some constant-time correctness for cryptographic implementations. The approach is reasonable, it is known to be insufficient but this is pointed out in the documentation." aggregated-from = "https://raw.githubusercontent.com/mozilla-firefox/firefox/main/supply-chain/audits.toml" +[[audits.mozilla.audits.sync_wrapper]] +who = "Alex Franchuk " +criteria = "safe-to-deploy" +version = "1.0.2" +notes = "Simple crate, nothing out of the ordinary." +aggregated-from = "https://raw.githubusercontent.com/mozilla/cargo-vet/main/supply-chain/audits.toml" + [[audits.mozilla.audits.synstructure]] who = "Nika Layzell " criteria = "safe-to-deploy" @@ -2278,6 +2412,13 @@ delta = "0.1.4 -> 0.1.8" notes = "No unsafe code" aggregated-from = "https://raw.githubusercontent.com/mozilla-firefox/firefox/main/supply-chain/audits.toml" +[[audits.mozilla.audits.tinyvec]] +who = "Alex Franchuk " +criteria = "safe-to-deploy" +delta = "1.9.0 -> 1.13.3" +notes = "Unsafe code is forbidden, so there's no new unsafe code. There are 3 new optional dependencies, all reasonable (adding trait impls)." +aggregated-from = "https://raw.githubusercontent.com/mozilla/cargo-vet/main/supply-chain/audits.toml" + [[audits.mozilla.audits.tracing-core]] who = "Alex Franchuk " criteria = "safe-to-deploy" @@ -2301,6 +2442,13 @@ criteria = "safe-to-deploy" delta = "0.1.34 -> 0.1.36" aggregated-from = "https://raw.githubusercontent.com/mozilla/cargo-vet/main/supply-chain/audits.toml" +[[audits.mozilla.audits.untrusted]] +who = "Alex Franchuk " +criteria = "safe-to-deploy" +delta = "0.7.1 -> 0.9.0" +notes = "Mostly a refactor of existing code. No use of std, no unsafe, no dependencies." +aggregated-from = "https://raw.githubusercontent.com/mozilla/cargo-vet/main/supply-chain/audits.toml" + [[audits.mozilla.audits.utf8parse]] who = "Nika Layzell " criteria = "safe-to-deploy" @@ -2313,6 +2461,13 @@ criteria = "safe-to-run" delta = "0.2.0 -> 0.2.1" aggregated-from = "https://raw.githubusercontent.com/mozilla/glean/main/supply-chain/audits.toml" +[[audits.mozilla.audits.web-time]] +who = "Alex Franchuk " +criteria = "safe-to-deploy" +version = "1.1.0" +notes = "Reasonable dependencies, unsafe code denied. The only use of std is to forward std::time when used on non-wasm targets and to reuse a few std::time things that are still applicable." +aggregated-from = "https://raw.githubusercontent.com/mozilla/cargo-vet/main/supply-chain/audits.toml" + [[audits.mozilla.audits.windows-link]] who = "Mark Hammond " criteria = "safe-to-deploy" From b5e3bc2414e26321c161e406369b90bedec3bf84 Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:46:40 -0700 Subject: [PATCH 13/21] fix(deps): rustls-webpki 0.103.16 and the zstd 0.14 line for five advisories - rustls-webpki 0.103.15 -> 0.103.16 (RUSTSEC-2026-0382: certificate verification could accept distrusted intermediates). Covered by the imported Bytecode Alliance delta audit. - zstd 0.14.2 -> 0.14.3 (RUSTSEC-2026-0391), zstd-safe 8.1.0 -> 8.2.0 (RUSTSEC-2026-0398, RUSTSEC-2026-0403), zstd-sys 2.1.1 -> 2.1.2 (RUSTSEC-2026-0410). This is the zstd UFFS itself uses for every cache file. zstd and zstd-safe are audited here as deltas (read in full except zstd-safe's seekable.rs, whose feature is not enabled); zstd-sys is covered by the existing trusted-publisher entry. Still open: polars-arrow 0.55.2, the newest polars, pins zstd 0.13 / zstd-safe 7.3 for its parquet support, so RUSTSEC-2026-0391, -0398 and -0403 remain on that copy until polars moves to zstd 0.14. --- Cargo.lock | 22 +++++++++++----------- supply-chain/audits.toml | 12 ++++++++++++ supply-chain/imports.lock | 16 ++++++++++++++-- 3 files changed, 37 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3efc9f72c..6e965dea3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3477,9 +3477,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.15" +version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ "ring", "rustls-pki-types", @@ -4480,7 +4480,7 @@ dependencies = [ "uffs-security", "uffs-text", "uffs-time", - "zstd 0.14.2", + "zstd 0.14.3", ] [[package]] @@ -4653,7 +4653,7 @@ dependencies = [ "windows", "winresource", "zerocopy", - "zstd 0.14.2", + "zstd 0.14.3", ] [[package]] @@ -5413,11 +5413,11 @@ dependencies = [ [[package]] name = "zstd" -version = "0.14.2" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "057cfd910cfac363a0ada849592624b4c9ff2e10bef504c3433810d78ed96f93" +checksum = "db3ae9bf90af6e40197bef767eeefc46ad6397420fe873830bd85e5fbbfb3c70" dependencies = [ - "zstd-safe 8.1.0", + "zstd-safe 8.2.0", ] [[package]] @@ -5431,18 +5431,18 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "8.1.0" +version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdd44c6a7284e91f3717755b24315a302edd9153a01f753c3cba3d765e8eafac" +checksum = "ded520f16b29e1a9b771c722221b2b0a0398c2316d9bb3c42b458a5414d7434b" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.1.1+zstd.1.5.7" +version = "2.1.2+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aeec9eaf2dffbbd09201e23bd0ffcbaa33bb8e9266a10734fd7ed90a85eca078" +checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3" dependencies = [ "cc", "pkg-config", diff --git a/supply-chain/audits.toml b/supply-chain/audits.toml index 76292f782..9fac6e200 100644 --- a/supply-chain/audits.toml +++ b/supply-chain/audits.toml @@ -2303,6 +2303,12 @@ criteria = "safe-to-deploy" delta = "0.14.0 -> 0.14.2" notes = "Read every src change; no unsafe added. Decompressor sizes its output from the summed frame content sizes (walked with zstd_safe::find_frame_compressed_size/get_frame_content_size, checked_add, slice get) and errors early with both numbers when the caller's capacity is too small; decode_all pre-reserves from the header's claimed size capped at 32 MiB via fallible try_reserve, so a hostile header cannot force a large allocation; dictionaries gain try_copy/try_new returning io errors instead of panicking; new compress_file/compress_sized helpers wrap existing safe APIs. Cargo.toml adds the with-rust-allocator passthrough feature (off for us) and new test targets." +[[audits.zstd]] +who = "Robert M1 <50460704+githubrobbi@users.noreply.github.com>" +criteria = "safe-to-deploy" +delta = "0.14.2 -> 0.14.3" +notes = "Read every src change (11 files, 312+/17-, most of it tests). The RUSTSEC-2026-0391 fix: Encoder/Decoder::with_context reset the reused context (ResetDirective::SessionOnly) so an abandoned stream cannot leak into the next. dict::from_continuous sums sample sizes with checked_add instead of a wrapping sum; the auto-finish/auto-flush wrappers skip finishing over a writer that panicked (zio::writer tracks it); the decoder's finish rejects an incomplete or empty trailing frame (zio::reader `started`). Manifest changes only the version and the zstd-safe 8.2.0 requirement. No unsafe added, no build.rs change, no process, network, filesystem or environment access." + [[audits.zstd-safe]] who = "Robert Nio <50460704+githubrobbi@users.noreply.github.com>" criteria = "safe-to-deploy" @@ -2321,6 +2327,12 @@ criteria = "safe-to-deploy" delta = "8.0.0 -> 8.1.0" notes = "Read lib.rs, rust_allocator.rs and seekable.rs in full. New opt-in with-rust-allocator feature routes zstd's ZSTD_customMem through Rust's global allocator: rust_alloc uses checked_add and a fallible Layout (16-byte header holding the size, 16-byte alignment), returns NULL instead of unwinding across FFI, and rust_free rebuilds the identical layout from that header; sound. With the feature off (our build) every context/dictionary constructor is equivalent to 8.0.0. CCtx/DCtx clone now refuses a poisoned context; new try_create_by_reference returns None instead of panicking; two transmutes in seekable.rs become typed pointer casts; WriteBuf's safety contract is now documented. zstd-sys requirement 2.1.0 -> 2.1.1." +[[audits.zstd-safe]] +who = "Robert M1 <50460704+githubrobbi@users.noreply.github.com>" +criteria = "safe-to-deploy" +delta = "8.1.0 -> 8.2.0" +notes = "Soundness-hardening release (2334-line diff). Read lib.rs in full; seekable.rs (RUSTSEC-2026-0398) skimmed only, as the seekable feature is not enabled in this workspace. The RUSTSEC-2026-0403 fix: In/OutBufferWrapper::check() discards the position zstd reports after a failed call, so the legacy decoders can no longer mark unwritten output as initialized; OutBuffer::around_pos zero-fills initialized..pos within the WriteBuf-promised capacity before handing it to zstd; write_from asserts the reported length is within capacity; InBuffer::wrap panics on pos > src.len() before zstd reads from src + pos; Cursor filled_until handles n == 0 without running past the buffer. CCtx tracks multithreaded frames and refuses dictionary/parameter changes while worker jobs may still read them; one-shot compress calls reset the session; init_using_cdict now ties the dictionary lifetime to the context; DCtx copy carries the window limit and output-buffer mode and starts from a frame boundary; StableInBuffer(true) is refused (a borrowed InBuffer cannot uphold it); train_from_buffer sums sizes with checked_add. Every new unsafe block is an FFI call or the bounded zero-fill above. No build.rs change, no process, network, filesystem or environment access." + [[audits.zstd-sys]] who = "Robert Nio <50460704+githubrobbi@users.noreply.github.com>" criteria = "safe-to-deploy" diff --git a/supply-chain/imports.lock b/supply-chain/imports.lock index e2a8e848c..1fb38b372 100644 --- a/supply-chain/imports.lock +++ b/supply-chain/imports.lock @@ -838,8 +838,8 @@ user-login = "dtolnay" user-name = "David Tolnay" [[publisher.zstd-sys]] -version = "2.1.1+zstd.1.5.7" -when = "2026-10-09" +version = "2.1.2+zstd.1.5.7" +when = "2026-10-16" user-id = 3203 user-login = "gyscos" user-name = "Alexandre Bury" @@ -1122,6 +1122,18 @@ criteria = "safe-to-deploy" delta = "0.3.0 -> 0.4.0" notes = "Minor updates for a new `rand` crate version, nothing awry." +[[audits.bytecode-alliance.audits.rustls-webpki]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "0.103.13 -> 0.103.15" +notes = "Minor updates and feature shufflings." + +[[audits.bytecode-alliance.audits.rustls-webpki]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "0.103.15 -> 0.103.16" +notes = "Updated with new certificates" + [[audits.bytecode-alliance.audits.sharded-slab]] who = "Pat Hickey " criteria = "safe-to-deploy" From 5d32274e46afedc86ccc4a9136047179e93fc869 Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:55:21 -0700 Subject: [PATCH 14/21] chore(deny): document polars' zstd 0.13 advisories until its fix is released polars-arrow 0.55.2, the newest polars, still holds zstd 0.13.3 / zstd-safe 7.3.0 for parquet. RUSTSEC-2026-0391, -0398 and -0403 apply to that copy only; UFFS's own zstd is on the fixed line (b5e3bc241). The fix is merged upstream (pola-rs/polars#30316, zstd 0.14.4) but unreleased. Recorded the same way as the existing polars quick-xml entries: per-ID reachability (0398 not compiled in; 0403 and 0391 only through parquet read/write) and the removal condition (a polars release containing #30316). Owner decision 2026-10-28. --- deny.toml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/deny.toml b/deny.toml index 90fc52e87..41f6976bb 100644 --- a/deny.toml +++ b/deny.toml @@ -43,6 +43,27 @@ ignore = [ # polars-io adopting object_store 0.14. Remove together with the ignore above. # See: https://rustsec.org/advisories/RUSTSEC-2026-0194 "RUSTSEC-2026-0194", + # zstd 0.13.3 / zstd-safe 7.3.0, held by polars-arrow 0.55.2 (the newest polars) + # for its parquet support. UFFS's own zstd (every cache file) is already on the + # fixed 0.14.3 / 8.2.0 line; only polars' private copy is affected. + # * RUSTSEC-2026-0398 (zstd-safe `Seekable::decompress`): not compiled in; the + # `seekable` feature is off for this copy. + # * RUSTSEC-2026-0403 (uninitialized memory on a failed legacy-frame + # decompress): reachable only when UFFS reads a parquet file + # (`uffs-mft inspect`, the parquet query path, the diag tools), and only for + # a crafted file carrying legacy zstd frames. + # * RUSTSEC-2026-0391 (state of an abandoned stream leaking into the next + # stream on a reused context): reachable only through a parquet read or + # write that fails part-way and then reuses the context. + # Upstream fix merged 2026-10-27: pola-rs/polars#30316 bumps zstd to 0.14.4; it is + # not in a release yet (checked 2026-10-28). Remove these three once a polars + # release containing #30316 is adopted. + # See: https://rustsec.org/advisories/RUSTSEC-2026-0391 + "RUSTSEC-2026-0391", + # See: https://rustsec.org/advisories/RUSTSEC-2026-0398 + "RUSTSEC-2026-0398", + # See: https://rustsec.org/advisories/RUSTSEC-2026-0403 + "RUSTSEC-2026-0403", ] [licenses] From 61fc5b0ca35028ff15ce38dec7a44f9315023424 Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Wed, 28 Oct 2026 18:04:44 -0700 Subject: [PATCH 15/21] perf(daemon): write saves without copying the index; skip near-empty age saves Winbox run 2 (2026-10-29, next/v0.6.46 at 5d32274e4) confirmed the save spacing (C: saves 60-61 s apart, 4 instead of 8 over the 200 k burst) but showed two costs: - The background persist folded a private clone of the body before writing. For the 30.8 M-record D: that clone is over 3 GB of records and names on top of the apply's own copy; D:'s age save pushed the daemon to a 17.6 GB peak. The serializer now writes a delta-carrying body directly (serialize_compact_folded_to_writer): it rebuilds only the children and trigram sections from the live records for the write, on the two-thread save pool. A test pins the bytes to those of serializing a folded copy. - The 5-minute age save rewrote D:'s whole cache for a handful of changes (cursor moved a few hundred bytes), ~60 CPU-seconds and several GB each time, in both runs. The age save now needs at least 1 000 pending events (DEFAULT_SAVE_AGE_MIN_EVENTS); fewer wait for a 1-hour ceiling (DEFAULT_SAVE_AGE_CEILING). Fewer changes than that replay from the journal at no cost, and the ceiling keeps a quiet drive's cache well inside what a fast re-warm can replay. SaveTrigger::evaluate takes a SavePolicy built from JournalLoopConfig. --- crates/uffs-core/src/compact_cache.rs | 45 ++++++++-- .../src/compact_cache/tests_cache_poison.rs | 61 ++++++++++--- crates/uffs-daemon/src/cache/journal_loop.rs | 32 +++++-- .../src/cache/journal_loop/tests.rs | 2 + .../cache/journal_loop/tests/apply_cadence.rs | 6 ++ .../cache/journal_loop/tests/integration.rs | 2 + .../journal_loop/tests/save_log_message.rs | 2 + .../cache/journal_loop/tests/thresholds.rs | 90 +++++++++++++++++-- .../tests/wrap_and_persistence.rs | 2 + .../src/cache/journal_loop/triggers.rs | 68 ++++++++++---- crates/uffs-daemon/src/cache/shard/tests.rs | 28 ++---- crates/uffs-daemon/src/index/journal_save.rs | 36 +++----- crates/uffs-daemon/src/lib.rs | 2 + 13 files changed, 279 insertions(+), 97 deletions(-) diff --git a/crates/uffs-core/src/compact_cache.rs b/crates/uffs-core/src/compact_cache.rs index 5ca349ab6..cf5b66027 100644 --- a/crates/uffs-core/src/compact_cache.rs +++ b/crates/uffs-core/src/compact_cache.rs @@ -477,12 +477,43 @@ pub fn serialize_compact(index: &DriveCompactIndex) -> Vec { pub(crate) fn serialize_compact_to_writer( index: &DriveCompactIndex, writer: &mut W, +) -> io::Result<()> { + write_compact_sections(index, &index.children, &index.trigram, writer) +} + +/// [`serialize_compact_to_writer`] for an index that may carry a delta +/// overlay: the children and trigram sections are rebuilt from the records +/// for the write instead of taken from the stale bases. +/// +/// Equivalent to serialising a `fold_delta_for_save`d copy, without copying +/// the records and names first — for a 30 M-record drive that copy alone is +/// over 3 GB. The extension index is not part of the file, so it needs no +/// rebuild here. +pub(crate) fn serialize_compact_folded_to_writer( + index: &DriveCompactIndex, + writer: &mut W, +) -> io::Result<()> { + if index.delta.is_none() { + return serialize_compact_to_writer(index, writer); + } + let children = ChildrenIndex::build(&index.records); + let trigram = TrigramIndex::build(&index.records, &index.names, index.fold); + write_compact_sections(index, &children, &trigram, writer) +} + +/// Write every section of `index`, taking the children and trigram CSRs from +/// `children` / `trigram` (see [`serialize_compact_folded_to_writer`]). +fn write_compact_sections( + index: &DriveCompactIndex, + children: &ChildrenIndex, + trigram: &TrigramIndex, + writer: &mut W, ) -> io::Result<()> { let record_count = index.records.len(); let names_len = index.names.len(); - let (csr_offsets, csr_values) = index.children.as_csr(); - let (tri_keys, tri_offsets, tri_values) = index.trigram.as_csr(); + let (csr_offsets, csr_values) = children.as_csr(); + let (tri_keys, tri_offsets, tri_values) = trigram.as_csr(); // Header (26 bytes for v3+) writer.write_all(COMPACT_MAGIC)?; @@ -1012,6 +1043,10 @@ pub fn save_compact_cache(index: &DriveCompactIndex) -> io::Result<()> { /// [`save_compact_cache`] with an explicit zstd worker count, blocking until /// the file is written. /// +/// A delta-carrying `index` is written folded: the children and trigram +/// sections are rebuilt from its records for the write, so the caller need +/// not copy it to fold it first. +/// /// The journal loop's background saves use this with a small count: a save /// that takes every core competes with the build or copy that caused the /// churn in the first place (issue #639). Returning only after the atomic @@ -1019,13 +1054,11 @@ pub fn save_compact_cache(index: &DriveCompactIndex) -> io::Result<()> { /// it describes. /// /// # Errors -/// Returns an error if `index` still carries a delta overlay, or if -/// compression, encryption, or file writing fails. +/// Returns an error if compression, encryption, or file writing fails. pub fn save_compact_cache_with_workers( index: &DriveCompactIndex, zstd_workers: u32, ) -> io::Result<()> { - refuse_delta_carrying(index)?; let profile = std::env::var_os("UFFS_CACHE_PROFILE").is_some(); let path = compact_cache_path(index.letter); if let Some(dir) = path.parent() { @@ -1037,7 +1070,7 @@ pub fn save_compact_cache_with_workers( // and for every save after the first successful one. purge_legacy_compact_cache_dir(&path, index.letter)?; uffs_mft::cache::compress_encrypt_write_streaming_with_workers( - |encoder| serialize_compact_to_writer(index, encoder), + |encoder| serialize_compact_folded_to_writer(index, encoder), &path, ZSTD_LEVEL, zstd_workers, diff --git a/crates/uffs-core/src/compact_cache/tests_cache_poison.rs b/crates/uffs-core/src/compact_cache/tests_cache_poison.rs index e21a93d30..51b89fab4 100644 --- a/crates/uffs-core/src/compact_cache/tests_cache_poison.rs +++ b/crates/uffs-core/src/compact_cache/tests_cache_poison.rs @@ -79,19 +79,9 @@ fn parked_loader_finds_filters_past_an_empty_trigram() { ); } -/// Regression: a DELTA-CARRYING index must never serialize — its -/// Arc-shared base sections are stale against the patched records. -/// -/// A create appends a record (the header's `rc` grows) while -/// `children` stays the base CSR with `rc_base + 1` offsets; the -/// reader sizes the children section from the header, so every later -/// section is read shifted and the cache quarantines on the next -/// start (field: `bloom k_hashes out of range`, winbox M/C/D/S -/// 2026-08-24 — caches written BY the then-current daemon between two -/// restarts). [`DriveCompactIndex::fold_delta_for_save`] is the -/// repair the save path applies; pin both halves. -#[test] -fn delta_carrying_index_misaligns_and_fold_repairs_it() { +/// The surgical-patch state: one created record appended (name "qux" +/// appended to the blob), bases left Arc-shared/stale, delta overlay armed. +fn index_with_created_record_and_delta() -> DriveCompactIndex { let mut index = make_test_index(); index.bloom = Some(index.build_bloom()); index.path_trie = Some(index.build_path_trie()); @@ -113,6 +103,23 @@ fn delta_carrying_index_misaligns_and_fold_repairs_it() { index.records = ColumnStorage::from_vec(records); index.names = ColumnStorage::from_vec(names); index.delta = Some(crate::compact::IndexDelta::default()); + index +} + +/// Regression: a DELTA-CARRYING index must never serialize — its +/// Arc-shared base sections are stale against the patched records. +/// +/// A create appends a record (the header's `rc` grows) while +/// `children` stays the base CSR with `rc_base + 1` offsets; the +/// reader sizes the children section from the header, so every later +/// section is read shifted and the cache quarantines on the next +/// start (field: `bloom k_hashes out of range`, winbox M/C/D/S +/// 2026-08-24 — caches written BY the then-current daemon between two +/// restarts). [`DriveCompactIndex::fold_delta_for_save`] is the +/// repair the save path applies; pin both halves. +#[test] +fn delta_carrying_index_misaligns_and_fold_repairs_it() { + let mut index = index_with_created_record_and_delta(); // The poisoned bytes can NEVER round-trip faithfully: the children // CSR on disk was written for the 3 base records, so the created @@ -171,3 +178,31 @@ fn background_save_refuses_a_delta_carrying_index() { "the refusal must name the delta invariant: {err}" ); } + +/// The journal loop's save writes a delta-carrying body directly, rebuilding +/// the children and trigram sections for the write instead of folding a copy +/// (a copy of a 30 M-record drive's records and names is over 3 GB). The +/// bytes must be exactly those of the folded index. +#[test] +fn folded_writer_matches_serializing_a_folded_copy() { + let index = index_with_created_record_and_delta(); + + let mut direct = Vec::new(); + serialize_compact_folded_to_writer(&index, &mut direct) + .unwrap_or_else(|err| panic!("folded write failed: {err}")); + + let mut folded = index.clone(); + folded.fold_delta_for_save(); + let mut reference = Vec::new(); + serialize_compact_to_writer(&folded, &mut reference) + .unwrap_or_else(|err| panic!("reference write failed: {err}")); + + assert_eq!( + direct, reference, + "folded write must equal the folded copy byte for byte" + ); + assert!( + index.delta.is_some(), + "writing must not touch the live index" + ); +} diff --git a/crates/uffs-daemon/src/cache/journal_loop.rs b/crates/uffs-daemon/src/cache/journal_loop.rs index 1d5d1ecaf..c8371c9a5 100644 --- a/crates/uffs-daemon/src/cache/journal_loop.rs +++ b/crates/uffs-daemon/src/cache/journal_loop.rs @@ -52,8 +52,9 @@ mod triggers; pub(crate) use poll::{MAX_POLL_BACKOFF, PollBackoff}; pub(crate) use triggers::{ - ApplyTrigger, DEFAULT_APPLY_DEBOUNCE_MS, DEFAULT_APPLY_INTERVAL_MS, DEFAULT_SAVE_MIN_INTERVAL, - DEFAULT_SAVE_THRESHOLD_AGE, DEFAULT_SAVE_THRESHOLD_EVENTS, SaveReason, SaveTrigger, + ApplyTrigger, DEFAULT_APPLY_DEBOUNCE_MS, DEFAULT_APPLY_INTERVAL_MS, DEFAULT_SAVE_AGE_CEILING, + DEFAULT_SAVE_AGE_MIN_EVENTS, DEFAULT_SAVE_MIN_INTERVAL, DEFAULT_SAVE_THRESHOLD_AGE, + DEFAULT_SAVE_THRESHOLD_EVENTS, SavePolicy, SaveReason, SaveTrigger, }; /// Default poll interval for the per-shard journal loop (500 ms). @@ -284,6 +285,12 @@ pub(crate) struct JournalLoopConfig { /// Minimum gap between two events-driven saves. Default /// [`DEFAULT_SAVE_MIN_INTERVAL`] (60 s). pub(crate) save_min_interval: Duration, + /// Fewest pending events for the age save. Default + /// [`DEFAULT_SAVE_AGE_MIN_EVENTS`] (1 000). + pub(crate) save_age_min_events: u64, + /// Age after which any pending event makes a save due. Default + /// [`DEFAULT_SAVE_AGE_CEILING`] (1 h). + pub(crate) save_age_ceiling: Duration, /// Search-freshness apply **max-wait** cap (Phase 5) — the ceiling of the /// debounce model. A burst that never settles is force-applied at least /// this often, bounding both freshness lag and apply CPU under sustained @@ -296,6 +303,19 @@ pub(crate) struct JournalLoopConfig { pub(crate) apply_debounce: Duration, } +impl JournalLoopConfig { + /// The save thresholds as one [`SavePolicy`]. + pub(crate) const fn save_policy(&self) -> SavePolicy { + SavePolicy { + threshold_events: self.save_threshold_events, + min_interval: self.save_min_interval, + threshold_age: self.save_threshold_age, + age_min_events: self.save_age_min_events, + age_ceiling: self.save_age_ceiling, + } + } +} + impl Default for JournalLoopConfig { fn default() -> Self { Self { @@ -304,6 +324,8 @@ impl Default for JournalLoopConfig { save_threshold_events: DEFAULT_SAVE_THRESHOLD_EVENTS, save_threshold_age: DEFAULT_SAVE_THRESHOLD_AGE, save_min_interval: DEFAULT_SAVE_MIN_INTERVAL, + save_age_min_events: DEFAULT_SAVE_AGE_MIN_EVENTS, + save_age_ceiling: DEFAULT_SAVE_AGE_CEILING, apply_interval: Duration::from_millis(DEFAULT_APPLY_INTERVAL_MS), apply_debounce: Duration::from_millis(DEFAULT_APPLY_DEBOUNCE_MS), } @@ -594,11 +616,7 @@ fn fire_due_cadence( apply_trigger: &mut ApplyTrigger, config: &JournalLoopConfig, ) { - if let Some(reason) = save_trigger.evaluate( - config.save_threshold_events, - config.save_threshold_age, - config.save_min_interval, - ) { + if let Some(reason) = save_trigger.evaluate(&config.save_policy()) { // A save drains + applies the buffer AND writes the body to // disk + persists the cursor — so it subsumes the apply tick. sink.trigger_save(letter, reason, cursor); diff --git a/crates/uffs-daemon/src/cache/journal_loop/tests.rs b/crates/uffs-daemon/src/cache/journal_loop/tests.rs index 5bca9942c..86c86c692 100644 --- a/crates/uffs-daemon/src/cache/journal_loop/tests.rs +++ b/crates/uffs-daemon/src/cache/journal_loop/tests.rs @@ -329,6 +329,8 @@ fn fast_config() -> JournalLoopConfig { save_threshold_events: u64::MAX, save_threshold_age: Duration::from_hours(24), save_min_interval: Duration::ZERO, + save_age_min_events: 0, + save_age_ceiling: Duration::MAX, // Disabled by default so the generic tick / cancel / cursor // tests don't accidentally fire an apply; the apply-cadence // tests override this with a short interval. diff --git a/crates/uffs-daemon/src/cache/journal_loop/tests/apply_cadence.rs b/crates/uffs-daemon/src/cache/journal_loop/tests/apply_cadence.rs index e6bdb9119..e33d22dd2 100644 --- a/crates/uffs-daemon/src/cache/journal_loop/tests/apply_cadence.rs +++ b/crates/uffs-daemon/src/cache/journal_loop/tests/apply_cadence.rs @@ -46,6 +46,8 @@ fn apply_only_config() -> JournalLoopConfig { save_threshold_events: u64::MAX, save_threshold_age: Duration::from_hours(24), save_min_interval: Duration::ZERO, + save_age_min_events: 0, + save_age_ceiling: Duration::MAX, apply_interval: Duration::ZERO, apply_debounce: Duration::ZERO, ..JournalLoopConfig::default() @@ -96,6 +98,8 @@ fn save_tick_suppresses_redundant_apply() { save_threshold_events: 1, save_threshold_age: Duration::from_hours(24), save_min_interval: Duration::ZERO, + save_age_min_events: 0, + save_age_ceiling: Duration::MAX, apply_interval: Duration::ZERO, apply_debounce: Duration::ZERO, ..JournalLoopConfig::default() @@ -161,6 +165,8 @@ async fn loop_applies_body_near_live_without_saving() { save_threshold_events: u64::MAX, save_threshold_age: Duration::from_hours(24), save_min_interval: Duration::ZERO, + save_age_min_events: 0, + save_age_ceiling: Duration::MAX, apply_interval: Duration::ZERO, apply_debounce: Duration::ZERO, ..JournalLoopConfig::default() diff --git a/crates/uffs-daemon/src/cache/journal_loop/tests/integration.rs b/crates/uffs-daemon/src/cache/journal_loop/tests/integration.rs index eca3bba32..0479c4ec9 100644 --- a/crates/uffs-daemon/src/cache/journal_loop/tests/integration.rs +++ b/crates/uffs-daemon/src/cache/journal_loop/tests/integration.rs @@ -81,6 +81,8 @@ fn end_to_end_config() -> JournalLoopConfig { save_threshold_events: SAVE_THRESHOLD_EVENTS, save_threshold_age: Duration::from_hours(24), save_min_interval: Duration::ZERO, + save_age_min_events: 0, + save_age_ceiling: Duration::MAX, apply_interval: Duration::from_hours(24), apply_debounce: Duration::from_hours(24), } diff --git a/crates/uffs-daemon/src/cache/journal_loop/tests/save_log_message.rs b/crates/uffs-daemon/src/cache/journal_loop/tests/save_log_message.rs index 3ed72f680..867352553 100644 --- a/crates/uffs-daemon/src/cache/journal_loop/tests/save_log_message.rs +++ b/crates/uffs-daemon/src/cache/journal_loop/tests/save_log_message.rs @@ -86,6 +86,8 @@ fn compact_cache_save_log_message_pins_string_target_and_level() { save_threshold_events: 1, // tight — crosses on the first evaluate save_threshold_age: Duration::from_hours(1), // generous save_min_interval: Duration::ZERO, // no gap: the first crossing saves + save_age_min_events: 0, + save_age_ceiling: Duration::MAX, apply_interval: Duration::from_hours(1), // disabled for this test apply_debounce: Duration::from_hours(1), ..JournalLoopConfig::default() diff --git a/crates/uffs-daemon/src/cache/journal_loop/tests/thresholds.rs b/crates/uffs-daemon/src/cache/journal_loop/tests/thresholds.rs index 4f1ccb0ae..8995a3873 100644 --- a/crates/uffs-daemon/src/cache/journal_loop/tests/thresholds.rs +++ b/crates/uffs-daemon/src/cache/journal_loop/tests/thresholds.rs @@ -15,7 +15,8 @@ use alloc::sync::Arc; use core::time::Duration; use super::super::{ - JournalLoopConfig, JournalSource, PatchSink, SaveReason, SaveTrigger, spawn_journal_loop, + JournalLoopConfig, JournalSource, PatchSink, SavePolicy, SaveReason, SaveTrigger, + spawn_journal_loop, }; use super::{ CONVERGENCE_DEADLINE, FakeJournalSource, RecordingSink, null_cursor_store, one_change, wait_for, @@ -45,6 +46,8 @@ async fn events_threshold_triggers_save() { save_threshold_events: 5, save_threshold_age: Duration::from_hours(24), save_min_interval: Duration::ZERO, + save_age_min_events: 0, + save_age_ceiling: Duration::MAX, apply_interval: Duration::from_hours(24), apply_debounce: Duration::from_hours(24), }; @@ -97,6 +100,8 @@ async fn age_threshold_triggers_save_with_pending_events() { save_threshold_events: u64::MAX, save_threshold_age: Duration::from_millis(30), save_min_interval: Duration::ZERO, + save_age_min_events: 0, + save_age_ceiling: Duration::MAX, apply_interval: Duration::from_hours(24), apply_debounce: Duration::from_hours(24), }; @@ -159,6 +164,8 @@ async fn zero_events_drive_does_not_trigger_save() { save_threshold_events: 1, save_threshold_age: Duration::from_millis(20), save_min_interval: Duration::ZERO, + save_age_min_events: 0, + save_age_ceiling: Duration::MAX, apply_interval: Duration::from_hours(24), apply_debounce: Duration::from_hours(24), }; @@ -214,6 +221,8 @@ async fn counter_resets_after_save() { save_threshold_events: 5, save_threshold_age: Duration::from_hours(24), save_min_interval: Duration::ZERO, + save_age_min_events: 0, + save_age_ceiling: Duration::MAX, apply_interval: Duration::from_hours(24), apply_debounce: Duration::from_hours(24), }; @@ -248,41 +257,106 @@ async fn counter_resets_after_save() { ); } +/// A policy with every threshold given, for driving [`SaveTrigger`] directly. +const fn policy( + threshold_events: u64, + min_interval: Duration, + threshold_age: Duration, + age_min_events: u64, + age_ceiling: Duration, +) -> SavePolicy { + SavePolicy { + threshold_events, + min_interval, + threshold_age, + age_min_events, + age_ceiling, + } +} + /// Issue #639: a sustained burst must not save on every events-threshold /// crossing. Inside the minimum interval the events threshold is held back; /// the counter keeps accumulating, so the save fires as soon as the interval /// has passed. #[test] fn events_threshold_waits_for_min_interval() { + let day = Duration::from_hours(24); let mut trigger = SaveTrigger::new(); trigger.record(10); assert_eq!( - trigger.evaluate(5, Duration::from_hours(24), Duration::from_hours(1)), + trigger.evaluate(&policy(5, Duration::from_hours(1), day, 0, day)), None, "events past the threshold must wait for the minimum interval", ); trigger.record(10); assert_eq!( - trigger.evaluate(5, Duration::from_hours(24), Duration::ZERO), + trigger.evaluate(&policy(5, Duration::ZERO, day, 0, day)), Some(SaveReason::EventsExceeded), "once the interval has passed, the held-back events save fires", ); assert_eq!( - trigger.evaluate(5, Duration::from_hours(24), Duration::ZERO), + trigger.evaluate(&policy(5, Duration::ZERO, day, 0, day)), None, "the save reset the counter", ); } -/// The age ceiling is the durability backstop and is never held back by the -/// minimum interval: a drive with a few pending events still saves once the -/// age threshold passes. +/// The age save is not held back by the minimum interval. #[test] fn age_threshold_ignores_min_interval() { let mut trigger = SaveTrigger::new(); trigger.record(1); assert_eq!( - trigger.evaluate(1_000, Duration::ZERO, Duration::from_hours(1)), + trigger.evaluate(&policy( + 1_000, + Duration::from_hours(1), + Duration::ZERO, + 0, + Duration::MAX + )), + Some(SaveReason::AgeElapsed), + ); +} + +/// Measured on the winbox: the age save rewrote D:'s whole 30 M-record cache +/// every 5 minutes for a handful of changes. Below the event floor the age +/// save waits for the ceiling. +#[test] +fn age_save_needs_enough_events_until_the_ceiling() { + let mut trigger = SaveTrigger::new(); + trigger.record(3); + assert_eq!( + trigger.evaluate(&policy( + 50_000, + Duration::ZERO, + Duration::ZERO, + 1_000, + Duration::from_hours(1) + )), + None, + "three changes do not justify rewriting the cache before the ceiling", + ); + assert_eq!( + trigger.evaluate(&policy( + 50_000, + Duration::ZERO, + Duration::ZERO, + 1_000, + Duration::ZERO + )), + Some(SaveReason::AgeElapsed), + "past the ceiling any pending change is saved", + ); + trigger.record(1_000); + assert_eq!( + trigger.evaluate(&policy( + 50_000, + Duration::ZERO, + Duration::ZERO, + 1_000, + Duration::from_hours(1) + )), Some(SaveReason::AgeElapsed), + "at the event floor the regular age save fires", ); } diff --git a/crates/uffs-daemon/src/cache/journal_loop/tests/wrap_and_persistence.rs b/crates/uffs-daemon/src/cache/journal_loop/tests/wrap_and_persistence.rs index c045aaf8d..ff0c8c873 100644 --- a/crates/uffs-daemon/src/cache/journal_loop/tests/wrap_and_persistence.rs +++ b/crates/uffs-daemon/src/cache/journal_loop/tests/wrap_and_persistence.rs @@ -91,6 +91,8 @@ async fn cursor_handed_to_sink_on_save_trigger() { save_threshold_events: 5, save_threshold_age: Duration::from_hours(24), save_min_interval: Duration::ZERO, + save_age_min_events: 0, + save_age_ceiling: Duration::MAX, apply_interval: Duration::from_hours(24), apply_debounce: Duration::from_hours(24), }; diff --git a/crates/uffs-daemon/src/cache/journal_loop/triggers.rs b/crates/uffs-daemon/src/cache/journal_loop/triggers.rs index c104cd99b..e3e34b8f4 100644 --- a/crates/uffs-daemon/src/cache/journal_loop/triggers.rs +++ b/crates/uffs-daemon/src/cache/journal_loop/triggers.rs @@ -47,6 +47,23 @@ pub(crate) const DEFAULT_SAVE_THRESHOLD_EVENTS: u64 = 50_000; /// path without changing the operator-visible recovery window. pub(crate) const DEFAULT_SAVE_THRESHOLD_AGE: Duration = Duration::from_mins(5); +/// Fewest pending events that make the [`DEFAULT_SAVE_THRESHOLD_AGE`] save +/// worth it — 1 000. +/// +/// A save rewrites the drive's whole cache. Measured on the winbox +/// (2026-10-28/29), the age save of the 30.8 M-record D: cost ~60 CPU-seconds +/// and several GB every 5 minutes for a handful of changes (its cursor moved a +/// few hundred bytes). Fewer changes than this cost nothing to replay from the +/// journal on a cold start or a re-warm, so they wait for +/// [`DEFAULT_SAVE_AGE_CEILING`] instead. +pub(crate) const DEFAULT_SAVE_AGE_MIN_EVENTS: u64 = 1_000; + +/// Longest a drive with any pending events goes without a save — 1 hour. +/// +/// Bounds how far behind the on-disk cache of a quiet drive can fall, so the +/// replay a re-warm needs stays well inside what one targeted MFT read covers. +pub(crate) const DEFAULT_SAVE_AGE_CEILING: Duration = Duration::from_hours(1); + /// Default minimum gap between two **events-driven** saves — 60 seconds. /// /// Measured on the winbox 2026-10-28 (issue #639): one compact-cache save of a @@ -108,6 +125,22 @@ pub(crate) enum SaveReason { AgeElapsed, } +/// The thresholds [`SaveTrigger::evaluate`] checks (see the `DEFAULT_SAVE_*` +/// constants for the defaults and their reasons). +#[derive(Debug, Clone, Copy)] +pub(crate) struct SavePolicy { + /// Pending events that make a save due. + pub(crate) threshold_events: u64, + /// Minimum gap between two events-driven saves. + pub(crate) min_interval: Duration, + /// Age after which a save is due, given `age_min_events`. + pub(crate) threshold_age: Duration, + /// Fewest pending events for the age save. + pub(crate) age_min_events: u64, + /// Age after which any pending event makes a save due. + pub(crate) age_ceiling: Duration, +} + /// Per-shard save-threshold state machine (Phase 7 task 7.4). /// /// Tracks the wall-clock time of the last save trigger and the @@ -156,28 +189,29 @@ impl SaveTrigger { /// events are pending (zero-churn drives never produce /// no-op saves). /// - /// The events threshold only fires once `save_min_interval` has passed - /// since the previous save, so a sustained burst saves at most once per - /// interval instead of on every threshold crossing. - pub(super) fn evaluate( - &mut self, - save_threshold_events: u64, - save_threshold_age: Duration, - save_min_interval: Duration, - ) -> Option { + /// The events threshold only fires once `min_interval` has passed since + /// the previous save, so a sustained burst saves at most once per interval + /// instead of on every threshold crossing. The age threshold only fires + /// with at least `age_min_events` pending, or once `age_ceiling` has + /// passed with any. + pub(super) fn evaluate(&mut self, policy: &SavePolicy) -> Option { if self.events_since_save == 0 { return None; } let now = Instant::now(); let elapsed = now.saturating_duration_since(self.last_save_at); - let reason = - if self.events_since_save >= save_threshold_events && elapsed >= save_min_interval { - Some(SaveReason::EventsExceeded) - } else if elapsed >= save_threshold_age { - Some(SaveReason::AgeElapsed) - } else { - None - }; + let events_due = + self.events_since_save >= policy.threshold_events && elapsed >= policy.min_interval; + let age_due = (elapsed >= policy.threshold_age + && self.events_since_save >= policy.age_min_events) + || elapsed >= policy.age_ceiling; + let reason = if events_due { + Some(SaveReason::EventsExceeded) + } else if age_due { + Some(SaveReason::AgeElapsed) + } else { + None + }; if reason.is_some() { self.last_save_at = now; self.events_since_save = 0; diff --git a/crates/uffs-daemon/src/cache/shard/tests.rs b/crates/uffs-daemon/src/cache/shard/tests.rs index 24037240c..9930f7f0d 100644 --- a/crates/uffs-daemon/src/cache/shard/tests.rs +++ b/crates/uffs-daemon/src/cache/shard/tests.rs @@ -631,16 +631,12 @@ fn apply_usn_patch_to_body_lands_delete_on_new_arc() { ); } -/// A patch keeps the cheap delta overlay on the live body; the save path -/// folds a private copy before it reaches the writer. The body handed to -/// the disk save must be delta-free — a delta-carrying index serialises its -/// base children CSR against a grown record count and poisons the cache for -/// the next start (`bloom k_hashes out of range`; see -/// `DriveCompactIndex::fold_delta_for_save`). A create is the trigger -/// shape: it appends a record, which is exactly what desyncs the Arc-shared -/// bases. +/// A patch keeps the cheap delta overlay on the live body; nothing on the +/// apply or save path folds it in place. The save writes the body folded +/// without copying it (`uffs_core::compact_cache` pins that the bytes equal a +/// folded copy's). #[test] -fn patch_keeps_delta_and_save_folds_a_copy() { +fn patch_keeps_the_delta_overlay() { let create = vec![FileChange { frs: 77_u64.into(), parent_frs: 5_u64.into(), @@ -659,20 +655,6 @@ fn patch_keeps_delta_and_save_folds_a_copy() { applied.delta.is_some(), "the patch must keep the cheap delta overlay" ); - - let saved = crate::index::journal_save::fold_for_save(&applied); - assert!( - saved.delta.is_none(), - "the save path must fold the delta before the body reaches the writer" - ); - assert!( - saved.records.len() > 2, - "the created record must survive the fold" - ); - assert!( - applied.delta.is_some(), - "folding for the save must not touch the live body" - ); } /// Field 2026-10-14: a drive loaded at startup and parked before any diff --git a/crates/uffs-daemon/src/index/journal_save.rs b/crates/uffs-daemon/src/index/journal_save.rs index 0748a2795..95e35e9ae 100644 --- a/crates/uffs-daemon/src/index/journal_save.rs +++ b/crates/uffs-daemon/src/index/journal_save.rs @@ -14,10 +14,11 @@ //! * **One save at a time, across all drives** ([`SaveSlot`]). A save tick //! that finds the slot taken applies its batch in memory and leaves the write //! to a later tick. -//! * **Small and background.** The fold runs on a two-thread pool whose -//! threads are in background-I/O mode, and zstd gets [`SAVE_ZSTD_WORKERS`] -//! workers, so a save stays near two cores instead of taking the machine away -//! from the build that caused the churn. +//! * **Small and background.** The fold (the serializer rebuilding the search +//! sections from the live records, without copying them) runs on a two-thread +//! pool whose threads are in background-I/O mode, and zstd gets +//! [`SAVE_ZSTD_WORKERS`] workers, so a save stays near two cores instead of +//! taking the machine away from the build that caused the churn. //! * **Cursor strictly after the file.** The on-disk USN cursor advances only //! once the atomic write has succeeded. A cursor ahead of the file would //! make the next cold start skip the changes in between for good. @@ -106,26 +107,15 @@ impl IndexManager { } } -/// Write `body`, folded first if it carries a delta overlay. +/// Write `body` as it is. A delta-carrying body is written folded by the +/// serializer, which rebuilds only the children and trigram sections from the +/// live records (on the save pool) — no copy of the records and names, which +/// for a 30 M-record drive is over 3 GB (winbox run 2, 2026-10-29: folding a +/// private copy pushed D:'s age save to a 17.6 GB peak). fn persist_body(body: &Arc) -> std::io::Result<()> { - uffs_core::compact_cache::save_compact_cache_with_workers( - &fold_for_save(body), - SAVE_ZSTD_WORKERS, - ) -} - -/// The delta-free body to serialise: `body` itself when it carries no -/// overlay, else a folded private copy. The live body keeps its overlay, so -/// the O(total-records) rebuild stays off the search path; a delta-carrying -/// index must never reach the writer (see -/// `DriveCompactIndex::fold_delta_for_save`). -pub(crate) fn fold_for_save(body: &Arc) -> Arc { - if body.delta.is_none() { - return Arc::clone(body); - } - let mut folded = DriveCompactIndex::clone(body); - in_save_pool(|| folded.fold_delta_for_save()); - Arc::new(folded) + in_save_pool(|| { + uffs_core::compact_cache::save_compact_cache_with_workers(body, SAVE_ZSTD_WORKERS) + }) } /// Run `work` on the save pool, or inline when the pool could not be built. diff --git a/crates/uffs-daemon/src/lib.rs b/crates/uffs-daemon/src/lib.rs index 1a30f6636..7aebb337f 100644 --- a/crates/uffs-daemon/src/lib.rs +++ b/crates/uffs-daemon/src/lib.rs @@ -624,6 +624,8 @@ async fn spawn_journal_loops_for_warm_shards( save_threshold_events = config.save_threshold_events, save_threshold_age_secs = config.save_threshold_age.as_secs(), save_min_interval_secs = config.save_min_interval.as_secs(), + save_age_min_events = config.save_age_min_events, + save_age_ceiling_secs = config.save_age_ceiling.as_secs(), "Spawning per-shard journal loops", ); let loop_count = letters.len(); From dd08124d9bf8fc431ded8f07ab1f81898c7be9a9 Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Wed, 28 Oct 2026 18:57:32 -0700 Subject: [PATCH 16/21] perf(cache): load caches with one buffer per stage, sized exactly Re-warming the 30.8 M-record D: from its compact cache peaked at 14 GB for a 4.9 GB index (winbox run 3, 2026-10-29). The load held the encrypted file, a decrypted copy, and a decompressed buffer that grew by doubling (old and new buffer alive together at the last step), all at once. - Decrypt in place: uffs_security::crypto::decrypt_cache_in_place authenticates and decrypts inside the buffer the file was read into, then cuts off header and tag. decrypt_cache is now a copying wrapper over it, so both share one validated path. The envelope check also rejects a header whose payload length overflows header+payload+tag; the old sum could wrap. - Exact-size decompression: the streaming zstd encoder never records the decompressed size, so the writers now append one zstd skippable frame holding it (uffs_mft::cache::append_size_trailer). Every zstd decoder skips skippable frames, so older builds still read new caches and decompress_sized reads old caches the growing way. With the trailer it allocates once, at the exact size, and rejects a mismatch. - Applied to all three cache readers (compact cache load, parked filter load, MftIndex cache load) and all three writers. No cache format version change. --- crates/uffs-core/src/compact_cache.rs | 17 +- crates/uffs-core/src/compact_cache/parked.rs | 4 +- .../src/compact_cache/tests_cache_poison.rs | 33 ++++ crates/uffs-mft/src/cache.rs | 6 + crates/uffs-mft/src/cache_compress.rs | 19 +- crates/uffs-mft/src/cache_size_trailer.rs | 186 ++++++++++++++++++ crates/uffs-mft/src/index/storage/file_io.rs | 6 +- crates/uffs-security/src/crypto.rs | 142 ++++++++++--- 8 files changed, 366 insertions(+), 47 deletions(-) create mode 100644 crates/uffs-mft/src/cache_size_trailer.rs diff --git a/crates/uffs-core/src/compact_cache.rs b/crates/uffs-core/src/compact_cache.rs index cf5b66027..4eed65e0c 100644 --- a/crates/uffs-core/src/compact_cache.rs +++ b/crates/uffs-core/src/compact_cache.rs @@ -1133,10 +1133,15 @@ where let profile = std::env::var_os("UFFS_CACHE_PROFILE").is_some(); let t_compress = Instant::now(); - // Serialize directly into the zstd encoder — no 1.1 GB buffer. - let mut encoder = uffs_mft::cache::new_zstd_mt_encoder(ZSTD_LEVEL)?; - serialize_compact_to_writer(index, &mut encoder)?; - let compressed = encoder.finish()?; + // Serialize directly into the zstd encoder — no 1.1 GB buffer — and + // record the decompressed size after it so the loader can allocate it + // exactly (`uffs_mft::cache::decompress_sized`). + let mut writer = + uffs_mft::cache::CountingWriter::new(uffs_mft::cache::new_zstd_mt_encoder(ZSTD_LEVEL)?); + serialize_compact_to_writer(index, &mut writer)?; + let decompressed_len = writer.written(); + let mut compressed = writer.into_inner().finish()?; + uffs_mft::cache::append_size_trailer(&mut compressed, decompressed_len); let compress_ms = t_compress.elapsed().as_millis(); if profile { @@ -1375,14 +1380,14 @@ fn load_compact_cache_at( let key = uffs_security::keystore::get_cache_key() .map_err(|err| LoadCacheError::KeyUnavailable(err.to_string()))?; let t_decrypt = Instant::now(); - let decrypted = uffs_security::crypto::decrypt_cache(&raw, &key) + let decrypted = uffs_security::crypto::decrypt_cache_in_place(raw, &key) .map_err(|err| LoadCacheError::DecryptFailed(err.to_string()))?; let decrypt_ms = t_decrypt.elapsed().as_millis(); let t_decompress = Instant::now(); let is_compressed = decrypted.get(..4).is_some_and(|magic| magic == ZSTD_MAGIC); let plaintext = if is_compressed { - zstd::decode_all(decrypted.as_slice()) + uffs_mft::cache::decompress_sized(&decrypted) .map_err(|err| LoadCacheError::DecompressFailed(err.to_string()))? } else { decrypted diff --git a/crates/uffs-core/src/compact_cache/parked.rs b/crates/uffs-core/src/compact_cache/parked.rs index cafa4f8dc..8eb6bf4c6 100644 --- a/crates/uffs-core/src/compact_cache/parked.rs +++ b/crates/uffs-core/src/compact_cache/parked.rs @@ -318,14 +318,14 @@ fn read_decompressed_plaintext( let key = uffs_security::keystore::get_cache_key() .map_err(|err| LoadCacheError::KeyUnavailable(err.to_string()))?; let t_decrypt = Instant::now(); - let decrypted = uffs_security::crypto::decrypt_cache(&raw, &key) + let decrypted = uffs_security::crypto::decrypt_cache_in_place(raw, &key) .map_err(|err| LoadCacheError::DecryptFailed(err.to_string()))?; let decrypt_ms = t_decrypt.elapsed().as_millis(); let t_decompress = Instant::now(); let is_compressed = decrypted.get(..4).is_some_and(|magic| magic == ZSTD_MAGIC); let plaintext = if is_compressed { - zstd::decode_all(decrypted.as_slice()) + uffs_mft::cache::decompress_sized(&decrypted) .map_err(|err| LoadCacheError::DecompressFailed(err.to_string()))? } else { decrypted diff --git a/crates/uffs-core/src/compact_cache/tests_cache_poison.rs b/crates/uffs-core/src/compact_cache/tests_cache_poison.rs index 51b89fab4..c7d2d8b51 100644 --- a/crates/uffs-core/src/compact_cache/tests_cache_poison.rs +++ b/crates/uffs-core/src/compact_cache/tests_cache_poison.rs @@ -206,3 +206,36 @@ fn folded_writer_matches_serializing_a_folded_copy() { "writing must not touch the live index" ); } + +/// The cache writers append a decompressed-size trailer after the zstd frame +/// so the loader can allocate the plaintext once, at its exact size. The +/// same compressed bytes must deserialize to the same index. +#[test] +fn sized_compression_round_trips_the_index() { + use std::io::Write as _; + + let index = make_test_index(); + let mut writer = uffs_mft::cache::CountingWriter::new( + uffs_mft::cache::new_zstd_mt_encoder(ZSTD_LEVEL).unwrap(), + ); + serialize_compact_to_writer(&index, &mut writer).unwrap(); + writer.flush().unwrap(); + let decompressed_len = writer.written(); + let mut compressed = writer.into_inner().finish().unwrap(); + uffs_mft::cache::append_size_trailer(&mut compressed, decompressed_len); + + let plaintext = uffs_mft::cache::decompress_sized(&compressed).unwrap(); + assert_eq!( + plaintext, + serialize_compact(&index), + "the trailer must not alter the payload" + ); + assert_eq!( + plaintext.capacity(), + plaintext.len(), + "allocated once, at the exact size" + ); + let (loaded, _tri_ms) = deserialize_compact(&plaintext, uffs_mft::platform::DriveLetter::T) + .unwrap_or_else(|err| panic!("sized plaintext must deserialize: {err}")); + assert_eq!(loaded.records.len(), index.records.len()); +} diff --git a/crates/uffs-mft/src/cache.rs b/crates/uffs-mft/src/cache.rs index e2056aa29..bcbe170cd 100644 --- a/crates/uffs-mft/src/cache.rs +++ b/crates/uffs-mft/src/cache.rs @@ -51,11 +51,17 @@ pub use cache_dataframe::load_or_build_dataframe_cached; /// policy. #[path = "cache_compress.rs"] mod cache_compress; +/// Decompressed-size trailer for cache files (exact-size decompression). +#[path = "cache_size_trailer.rs"] +mod cache_size_trailer; pub use cache_compress::{ compress_encrypt_write, compress_encrypt_write_streaming, compress_encrypt_write_streaming_with_workers, compress_zstd_mt, default_zstd_workers, new_zstd_encoder_with_workers, new_zstd_mt_encoder, }; +pub use cache_size_trailer::{ + CountingWriter, append_size_trailer, decompress_sized, size_from_trailer, +}; /// Default TTL for cached indices (4 hours). /// diff --git a/crates/uffs-mft/src/cache_compress.rs b/crates/uffs-mft/src/cache_compress.rs index e448b9351..ac60d74bb 100644 --- a/crates/uffs-mft/src/cache_compress.rs +++ b/crates/uffs-mft/src/cache_compress.rs @@ -7,7 +7,7 @@ use std::path::Path; -use super::atomic_write; +use super::{CountingWriter, append_size_trailer, atomic_write}; use crate::index::usize_to_f64; /// Compresses `data` with zstd using multi-threaded mode. @@ -77,7 +77,7 @@ pub fn compress_encrypt_write_streaming( label: &str, ) -> std::io::Result<()> where - F: FnOnce(&mut zstd::Encoder<'_, Vec>) -> std::io::Result<()>, + F: FnOnce(&mut CountingWriter>>) -> std::io::Result<()>, { compress_encrypt_write_streaming_with_workers( write_fn, @@ -103,14 +103,18 @@ pub fn compress_encrypt_write_streaming_with_workers( label: &str, ) -> std::io::Result<()> where - F: FnOnce(&mut zstd::Encoder<'_, Vec>) -> std::io::Result<()>, + F: FnOnce(&mut CountingWriter>>) -> std::io::Result<()>, { let t_total = std::time::Instant::now(); let t_compress = std::time::Instant::now(); - let mut encoder = new_zstd_encoder_with_workers(zstd_level, zstd_workers)?; - write_fn(&mut encoder)?; - let compressed = encoder.finish()?; + let mut writer = CountingWriter::new(new_zstd_encoder_with_workers(zstd_level, zstd_workers)?); + write_fn(&mut writer)?; + let decompressed_len = writer.written(); + let mut compressed = writer.into_inner().finish()?; + // The streaming encoder does not record the decompressed size; the + // trailer does, so the loader can allocate it exactly. + append_size_trailer(&mut compressed, decompressed_len); let compress_ms = t_compress.elapsed().as_millis(); let compressed_len = compressed.len(); @@ -168,7 +172,8 @@ pub fn compress_encrypt_write( let uncompressed_len = serialized.len(); let t_compress = std::time::Instant::now(); - let compressed = compress_zstd_mt(&serialized, zstd_level)?; + let mut compressed = compress_zstd_mt(&serialized, zstd_level)?; + append_size_trailer(&mut compressed, uncompressed_len as u64); let compress_ms = t_compress.elapsed().as_millis(); let compressed_len = compressed.len(); drop(serialized); diff --git a/crates/uffs-mft/src/cache_size_trailer.rs b/crates/uffs-mft/src/cache_size_trailer.rs new file mode 100644 index 000000000..1bef1d5d9 --- /dev/null +++ b/crates/uffs-mft/src/cache_size_trailer.rs @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright (c) 2025-2026 SKY, LLC. + +//! The decompressed size of a cache, carried next to its compressed bytes. +//! +//! Caches are compressed by a streaming, multi-threaded zstd encoder, which +//! does not record the decompressed size in the frame. Decompressing such a +//! frame into a `Vec` grows it by doubling, so at the last step the old and +//! the new buffer both exist: for a 4.9 GB index, up to ~10 GB at once. +//! +//! After the data frame the writers append one zstd **skippable frame** that +//! holds the decompressed length. Every zstd decoder skips skippable frames +//! (they are part of the zstd format), so a cache written this way still +//! decodes with any reader, including older UFFS builds; a reader that knows +//! the frame allocates the exact size up front. A cache without the frame +//! (written by an older build) decodes as before. + +use std::io; + +/// Skippable-frame magic (`0x184D2A50..=0x184D2A5F` are reserved for +/// skippable frames; this one marks the UFFS size trailer). +const SKIPPABLE_MAGIC: u32 = 0x184D_2A5E; + +/// Payload tag that identifies the trailer among skippable frames. +const TRAILER_TAG: &[u8; 4] = b"ULEN"; + +/// Payload length: the 4-byte tag plus the u64 decompressed length. +const PAYLOAD_LEN: u32 = 12; + +/// Whole trailer: magic, payload length, payload. +const TRAILER_LEN: usize = 8 + 12; + +/// Decompressed sizes beyond this are treated as no trailer at all, so a +/// damaged value can never request an absurd allocation. +const MAX_TRUSTED_LEN: u64 = 1 << 40; + +/// A writer that counts the bytes passed through it. +#[derive(Debug)] +pub struct CountingWriter { + /// The wrapped writer. + inner: W, + /// Bytes written so far. + written: u64, +} + +impl CountingWriter { + /// Wrap `inner`, starting the count at zero. + pub const fn new(inner: W) -> Self { + Self { inner, written: 0 } + } + + /// Bytes written so far. + #[must_use] + pub const fn written(&self) -> u64 { + self.written + } + + /// Unwrap the inner writer. + pub fn into_inner(self) -> W { + self.inner + } +} + +impl io::Write for CountingWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + let written = self.inner.write(buf)?; + self.written = self.written.saturating_add(written as u64); + Ok(written) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} + +/// Append the size trailer for `decompressed_len` bytes to `compressed`. +pub fn append_size_trailer(compressed: &mut Vec, decompressed_len: u64) { + compressed.reserve(TRAILER_LEN); + compressed.extend_from_slice(&SKIPPABLE_MAGIC.to_le_bytes()); + compressed.extend_from_slice(&PAYLOAD_LEN.to_le_bytes()); + compressed.extend_from_slice(TRAILER_TAG); + compressed.extend_from_slice(&decompressed_len.to_le_bytes()); +} + +/// The decompressed length recorded by [`append_size_trailer`], if +/// `compressed` ends with one. +#[must_use] +pub fn size_from_trailer(compressed: &[u8]) -> Option { + let start = compressed.len().checked_sub(TRAILER_LEN)?; + let trailer = compressed.get(start..)?; + let field = |range: core::ops::Range| trailer.get(range); + if field(0..4)? != SKIPPABLE_MAGIC.to_le_bytes() + || field(4..8)? != PAYLOAD_LEN.to_le_bytes() + || field(8..12)? != TRAILER_TAG + { + return None; + } + let len = u64::from_le_bytes(field(12..20)?.try_into().ok()?); + if len > MAX_TRUSTED_LEN { + return None; + } + usize::try_from(len).ok() +} + +/// Decompress `compressed` into a buffer of exactly the size its trailer +/// records, or the growing way when it has none. +/// +/// # Errors +/// +/// Returns an error if the data does not decompress, or if it decompresses +/// to a different length than the trailer recorded. +pub fn decompress_sized(compressed: &[u8]) -> io::Result> { + let Some(len) = size_from_trailer(compressed) else { + return zstd::decode_all(compressed); + }; + let mut decompressed = Vec::with_capacity(len); + zstd::stream::copy_decode(compressed, &mut decompressed)?; + if decompressed.len() != len { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "cache decompressed to {} bytes, its size trailer says {len}", + decompressed.len() + ), + )); + } + Ok(decompressed) +} + +#[cfg(test)] +mod tests { + use std::io::Write as _; + + use super::{CountingWriter, append_size_trailer, decompress_sized, size_from_trailer}; + + fn sample() -> Vec { + (0..200_000_u32).flat_map(u32::to_le_bytes).collect() + } + + fn compress(data: &[u8]) -> Vec { + let mut writer = CountingWriter::new(zstd::Encoder::new(Vec::new(), 3).unwrap()); + writer.write_all(data).unwrap(); + let written = writer.written(); + let mut compressed = writer.into_inner().finish().unwrap(); + append_size_trailer(&mut compressed, written); + compressed + } + + #[test] + fn trailer_records_the_exact_size_and_decodes_sized() { + let data = sample(); + let compressed = compress(&data); + assert_eq!(size_from_trailer(&compressed), Some(data.len())); + let decoded = decompress_sized(&compressed).unwrap(); + assert_eq!(decoded, data); + assert_eq!( + decoded.capacity(), + data.len(), + "allocated once, at the exact size" + ); + } + + #[test] + fn any_zstd_reader_skips_the_trailer() { + let data = sample(); + let compressed = compress(&data); + assert_eq!(zstd::decode_all(compressed.as_slice()).unwrap(), data); + } + + #[test] + fn cache_without_a_trailer_still_decodes() { + let data = sample(); + let plain = zstd::encode_all(data.as_slice(), 3).unwrap(); + assert_eq!(size_from_trailer(&plain), None); + assert_eq!(decompress_sized(&plain).unwrap(), data); + } + + #[test] + fn a_trailer_that_disagrees_with_the_data_is_an_error() { + let data = sample(); + let mut compressed = zstd::encode_all(data.as_slice(), 3).unwrap(); + append_size_trailer(&mut compressed, data.len() as u64 + 1); + let err = decompress_sized(&compressed).unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData, "{err}"); + } +} diff --git a/crates/uffs-mft/src/index/storage/file_io.rs b/crates/uffs-mft/src/index/storage/file_io.rs index d0fab798c..c4fcf9688 100644 --- a/crates/uffs-mft/src/index/storage/file_io.rs +++ b/crates/uffs-mft/src/index/storage/file_io.rs @@ -120,7 +120,7 @@ impl MftIndex { pub fn load_from_file( path: &std::path::Path, ) -> Result<(Self, IndexHeader), Box> { - use uffs_security::crypto::{CacheFormat, decrypt_cache, detect_format}; + use uffs_security::crypto::{CacheFormat, decrypt_cache_in_place, detect_format}; let profile = std::env::var_os("UFFS_CACHE_PROFILE").is_some(); let t_total = std::time::Instant::now(); @@ -137,7 +137,7 @@ impl MftIndex { CacheFormat::Encrypted => { let key = uffs_security::keystore::get_cache_key() .map_err(|err| Box::new(err) as Box)?; - match decrypt_cache(&raw, &key) { + match decrypt_cache_in_place(raw, &key) { Ok(pt) => pt, Err(decrypt_err) => { tracing::warn!( @@ -170,7 +170,7 @@ impl MftIndex { let t_decompress = std::time::Instant::now(); let compressed = is_zstd_compressed(&decrypted); let plaintext = if compressed { - zstd::decode_all(decrypted.as_slice()).map_err(|err| { + crate::cache::decompress_sized(&decrypted).map_err(|err| { Box::new(std::io::Error::new( std::io::ErrorKind::InvalidData, format!("zstd decompression failed: {err}"), diff --git a/crates/uffs-security/src/crypto.rs b/crates/uffs-security/src/crypto.rs index 27ed02431..34be39309 100644 --- a/crates/uffs-security/src/crypto.rs +++ b/crates/uffs-security/src/crypto.rs @@ -178,8 +178,26 @@ fn bad_data(msg: impl Into) -> io::Error { /// Decrypts an UFFSENC-formatted buffer, returning the original plaintext. /// +/// Copies `data` once and decrypts the copy in place; a caller that owns +/// the buffer should use [`decrypt_cache_in_place`] and skip that copy. +/// +/// # Errors +/// +/// See [`decrypt_cache_in_place`]. +pub fn decrypt_cache(data: &[u8], key: &[u8; 32]) -> io::Result> { + decrypt_cache_in_place(data.to_vec(), key) +} + +/// Decrypts an UFFSENC-formatted buffer inside its own allocation and returns +/// that allocation holding only the plaintext. +/// /// Supports both v1 (u32 length, 28-byte header) and v2 (u64 length, 32-byte -/// header) formats. Validates the header, algorithm ID, and GCM auth tag. +/// header) formats. Validates the header, algorithm ID, and GCM auth tag +/// before any plaintext is returned. +/// +/// The ciphertext is decrypted where it lies, then moved to the front of the +/// buffer and the header and tag cut off — no second buffer, so loading a +/// multi-GB cache does not hold the ciphertext and the plaintext at once. /// /// # Errors /// @@ -187,7 +205,48 @@ fn bad_data(msg: impl Into) -> io::Error { /// - The data is too short or has wrong magic /// - The algorithm or version is unsupported /// - GCM authentication fails (tampered data or wrong key) -pub fn decrypt_cache(data: &[u8], key: &[u8; 32]) -> io::Result> { +pub fn decrypt_cache_in_place(mut data: Vec, key: &[u8; 32]) -> io::Result> { + let envelope = parse_envelope(&data)?; + let payload_end = envelope.header_size + envelope.payload_len; + + let tag_bytes: [u8; TAG_SIZE] = data + .get(payload_end..payload_end + TAG_SIZE) + .ok_or_else(|| bad_data("tag OOB"))? + .try_into() + .map_err(|_e| bad_data("tag size mismatch"))?; + let auth_tag: &Tag = (&tag_bytes).into(); + + let (header, rest) = data.split_at_mut(envelope.header_size); + let ciphertext = rest + .get_mut(..envelope.payload_len) + .ok_or_else(|| bad_data("ciphertext OOB"))?; + let cipher = Aes256Gcm::new(key.into()); + cipher + .decrypt_inout_detached( + (&envelope.nonce).into(), + header, + ciphertext.into(), + auth_tag, + ) + .map_err(|_e| bad_data("AES-GCM authentication failed (wrong key or tampered data)"))?; + + data.copy_within(envelope.header_size..payload_end, 0); + data.truncate(envelope.payload_len); + Ok(data) +} + +/// The parts of an UFFSENC header that decryption needs. +struct Envelope { + /// Header length in bytes (also the AAD length): 28 for v1, 32 for v2. + header_size: usize, + /// Ciphertext (= plaintext) length in bytes. + payload_len: usize, + /// The 96-bit GCM nonce. + nonce: [u8; NONCE_SIZE], +} + +/// Validate an UFFSENC header and check `data` holds the whole payload and tag. +fn parse_envelope(data: &[u8]) -> io::Result { // Minimum size: smallest header (v1=28) + tag (16) = 44 bytes if data.len() < HEADER_SIZE_V1 + TAG_SIZE { return Err(bad_data(format!( @@ -246,13 +305,16 @@ pub fn decrypt_cache(data: &[u8], key: &[u8; 32]) -> io::Result> { ver => return Err(bad_data(format!("unsupported format version: {ver}"))), }; - let nonce_bytes: &[u8; NONCE_SIZE] = data + let nonce: [u8; NONCE_SIZE] = data .get(12..24) .ok_or_else(|| bad_data("data too short for nonce"))? .try_into() .map_err(|_e| bad_data("invalid nonce"))?; - let expected = header_size + payload_len + TAG_SIZE; + let expected = header_size + .checked_add(payload_len) + .and_then(|len| len.checked_add(TAG_SIZE)) + .ok_or_else(|| bad_data("payload length overflows"))?; if data.len() < expected { return Err(bad_data(format!( "encrypted cache truncated: have {} bytes, expected {expected}", @@ -260,31 +322,11 @@ pub fn decrypt_cache(data: &[u8], key: &[u8; 32]) -> io::Result> { ))); } - // Extract components — bounds guaranteed by checks above - let aad = data - .get(..header_size) - .ok_or_else(|| bad_data("header OOB"))?; - let ciphertext = data - .get(header_size..header_size + payload_len) - .ok_or_else(|| bad_data("ciphertext OOB"))?; - let tag = data - .get(header_size + payload_len..header_size + payload_len + TAG_SIZE) - .ok_or_else(|| bad_data("tag OOB"))?; - - let cipher = Aes256Gcm::new(key.into()); - let auth_tag: &Tag = tag.try_into().map_err(|_e| bad_data("tag size mismatch"))?; - - let mut plaintext = ciphertext.to_vec(); - cipher - .decrypt_inout_detached( - nonce_bytes.into(), - aad, - plaintext.as_mut_slice().into(), - auth_tag, - ) - .map_err(|_e| bad_data("AES-GCM authentication failed (wrong key or tampered data)"))?; - - Ok(plaintext) + Ok(Envelope { + header_size, + payload_len, + nonce, + }) } // ──────────────────────────────────────────────────────────────────────────── @@ -323,6 +365,48 @@ mod tests { assert_eq!(decrypted, plaintext); } + /// In-place decryption hands back the plaintext in the buffer it was + /// given — no second allocation of the payload's size. + #[test] + fn in_place_round_trip_reuses_the_buffer() { + let key = [0xCD_u8; 32]; + let plaintext: Vec = (0..=255_u8).cycle().take(64 * 1024).collect(); + let encrypted = encrypt_cache(&plaintext, &key).expect("encrypt"); + let capacity = encrypted.capacity(); + let decrypted = decrypt_cache_in_place(encrypted, &key).expect("decrypt"); + assert_eq!(decrypted, plaintext); + assert_eq!( + decrypted.capacity(), + capacity, + "the same allocation must be reused" + ); + } + + /// Tampering is still caught before any plaintext is returned. + #[test] + fn in_place_rejects_tampered_ciphertext() { + let key = [0x11_u8; 32]; + let mut encrypted = encrypt_cache(b"hello world", &key).expect("encrypt"); + if let Some(byte) = encrypted.get_mut(HEADER_SIZE_V2) { + *byte ^= 0xFF; + } + let err = decrypt_cache_in_place(encrypted, &key).unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::InvalidData, "{err}"); + } + + /// A header claiming a length that cannot be added up is rejected, not + /// wrapped around into a small bound. + #[test] + fn overflowing_payload_length_is_rejected() { + let key = [0x22_u8; 32]; + let mut encrypted = encrypt_cache(b"x", &key).expect("encrypt"); + if let Some(len) = encrypted.get_mut(24..32) { + len.copy_from_slice(&u64::MAX.to_le_bytes()); + } + let err = decrypt_cache_in_place(encrypted, &key).unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::InvalidData, "{err}"); + } + /// S2.3.6: tampered ciphertext → decrypt fails. #[test] fn tampered_ciphertext() { From e49187e8268824f1ba5a077515229d608eb08ada Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:02:01 -0700 Subject: [PATCH 17/21] fix(lints): give every #[deprecated] the version that deprecated it The floating-nightly clippy now rejects a `deprecated` attribute without `since`, which failed the nightly canary on 2026-10-19 and 2026-10-26 (#635). The two fragment parsers were deprecated in 0.2.203 and load_mft_file in 0.4.31, per the commits that added the attributes; the legacy path resolver already carried its version. Closes #635. --- crates/uffs-core/src/compact_loader.rs | 5 ++++- crates/uffs-mft/src/io/parser/fragment.rs | 5 ++++- crates/uffs-mft/src/io/parser/fragment_extension.rs | 5 ++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/crates/uffs-core/src/compact_loader.rs b/crates/uffs-core/src/compact_loader.rs index c3ff974b7..1d48dfa9e 100644 --- a/crates/uffs-core/src/compact_loader.rs +++ b/crates/uffs-core/src/compact_loader.rs @@ -537,7 +537,10 @@ pub fn refresh_drive(drive: &DriveCompactIndex) -> anyhow::Result<(DriveCompactI /// # Errors /// /// Returns an error if the MFT file cannot be read or parsed. -#[deprecated(note = "Use load_drive(MftSource::File(...)) instead")] +#[deprecated( + since = "0.4.31", + note = "Use load_drive(MftSource::File(...)) instead" +)] pub fn load_mft_file( mft_path: &std::path::Path, drive: Option, diff --git a/crates/uffs-mft/src/io/parser/fragment.rs b/crates/uffs-mft/src/io/parser/fragment.rs index cb1dd76c9..9ad0cf9db 100644 --- a/crates/uffs-mft/src/io/parser/fragment.rs +++ b/crates/uffs-mft/src/io/parser/fragment.rs @@ -42,7 +42,10 @@ use crate::ntfs::{ /// # Returns /// /// `true` if a record was added to the fragment, `false` if skipped. -#[deprecated(note = "Use parse_record_full() + MftRecordMerger + from_parsed_records() instead")] +#[deprecated( + since = "0.2.203", + note = "Use parse_record_full() + MftRecordMerger + from_parsed_records() instead" +)] #[expect( clippy::too_many_lines, clippy::cognitive_complexity, diff --git a/crates/uffs-mft/src/io/parser/fragment_extension.rs b/crates/uffs-mft/src/io/parser/fragment_extension.rs index 2f9072dfb..605699a53 100644 --- a/crates/uffs-mft/src/io/parser/fragment_extension.rs +++ b/crates/uffs-mft/src/io/parser/fragment_extension.rs @@ -42,7 +42,10 @@ use crate::ntfs::{ /// # Returns /// /// `true` if any names/streams were added, `false` otherwise. -#[deprecated(note = "Use parse_record_full() + MftRecordMerger instead")] +#[deprecated( + since = "0.2.203", + note = "Use parse_record_full() + MftRecordMerger instead" +)] #[expect( clippy::too_many_lines, reason = "monolithic extension parser for performance: nested FileName + Data \ From 4018b21ee8928157e480dec035022c0eb3de26d5 Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:03:18 -0700 Subject: [PATCH 18/21] fix(vet): drop the empty polars policy entries cargo-vet 0.10.3 rejects The weekly import refresh failed on 2026-10-26 because CI installed cargo-vet 0.10.3, which requires a policy for a crates.io crate to name its version ("unversioned policies that must be versioned"). The 23 `[policy.polars-*]` tables were empty, so they set nothing beyond the defaults; removing them leaves the vetting result identical (256 fully audited, 87 partially, 138 exempted) under both 0.10.2 and 0.10.3. --- supply-chain/config.toml | 46 ---------------------------------------- 1 file changed, 46 deletions(-) diff --git a/supply-chain/config.toml b/supply-chain/config.toml index 89045ee55..271b06a6b 100644 --- a/supply-chain/config.toml +++ b/supply-chain/config.toml @@ -31,52 +31,6 @@ url = "https://raw.githubusercontent.com/mozilla/supply-chain/main/audits.toml" [imports.zcash] url = "https://raw.githubusercontent.com/zcash/rust-ecosystem/main/supply-chain/audits.toml" -[policy.polars] - -[policy.polars-arrow] - -[policy.polars-buffer] - -[policy.polars-compute] - -[policy.polars-config] - -[policy.polars-core] - -[policy.polars-dtype] - -[policy.polars-error] - -[policy.polars-expr] - -[policy.polars-io] - -[policy.polars-json] - -[policy.polars-lazy] - -[policy.polars-mem-engine] - -[policy.polars-ooc] - -[policy.polars-ops] - -[policy.polars-parquet] - -[policy.polars-plan] - -[policy.polars-row] - -[policy.polars-schema] - -[policy.polars-sql] - -[policy.polars-stream] - -[policy.polars-time] - -[policy.polars-utils] - [policy.uffs-bench] audit-as-crates-io = false From 1e7770d72217a58d69647ac929b561a66165be55 Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:03:36 -0700 Subject: [PATCH 19/21] chore(deps): bring every dependency to its latest release and vet it `cargo update` plus `cargo upgrade` for the requirements: tokio 1.54.1, rmcp 3.8.0, uuid 1.29.0, libc 0.2.191, memmap2 0.9.12, globset 0.4.21, smallvec 1.16.4, zstd 0.14.3, cc 1.10.0, and the transitive ICU 2.4.0 line among 30 lockfile updates. reqwest stays on 0.12: re-verified today that polars 0.55.2 is still the latest polars and still on the 0.12 line, and 0.13 still pulls the Android JNI stack. Vetting (cargo-vet 0.10.3; 266 fully audited, 136 exempted, down from 138): - Publisher trust only where Mozilla or this project already trusts the publisher: robertbastian for the ICU/zerovec/yoke/tinystr family, djc for openssl-probe, Manishearth for unicode-width, github:RustCrypto/utils for inout, github:rust-lang/libc for libc. - Seven delta audits from a full source diff, recorded in audits.toml: crc32fast, memmap2, rmcp, rmcp-macros, schannel, smallvec, value-trait. - `cargo vet prune` retired the exemptions the new versions replaced. Vet-Reviewed-Diff: crc32fast@1.5.2->1.5.3 Vet-Reviewed-Diff: memmap2@0.9.11->0.9.12 Vet-Reviewed-Diff: rmcp@3.5.1->3.8.0 Vet-Reviewed-Diff: rmcp-macros@3.5.1->3.8.0 Vet-Reviewed-Diff: schannel@0.1.29->0.1.30 Vet-Reviewed-Diff: smallvec@1.16.3->1.16.4 Vet-Reviewed-Diff: value-trait@0.12.2->0.12.3 --- Cargo.lock | 133 +++++++++++----------- Cargo.toml | 18 +-- crates/uffs-vss-requestor/Cargo.toml | 2 +- supply-chain/audits.toml | 132 +++++++++++++++++++++ supply-chain/config.toml | 40 ------- supply-chain/imports.lock | 164 +++++++++++++++++---------- 6 files changed, 315 insertions(+), 174 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6e965dea3..ce9432975 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -527,9 +527,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6651c9ed80effdc7db0ff72512157f901af5e3549e341e24b1dd4887d836d838" +checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0" dependencies = [ "find-msvc-tools", "jobserver", @@ -683,7 +683,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -784,9 +784,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.5.2" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055" dependencies = [ "cfg-if", ] @@ -920,9 +920,9 @@ dependencies = [ [[package]] name = "ctutils" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe3e9d21d39e6f39ff46e03d5b621424f88ee23d8a1672c6bfc9ccc7f2da4f6a" +checksum = "a2376ffd6f96d8a9db59875b4b6b0237d6900b5baf64279714e05493ce64828a" dependencies = [ "cmov", ] @@ -1086,7 +1086,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1135,9 +1135,9 @@ checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "find-msvc-tools" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" [[package]] name = "flate2" @@ -1340,9 +1340,9 @@ checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" [[package]] name = "globset" -version = "0.4.20" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07c34a9410465b45bd9787443bc7370f37735bad04b0f0cd57ff1a3186c98988" +checksum = "f98b0d85522dc0a824cfebbc9594ff4a9b289c568bea0370319dfbdf20ccdd2d" dependencies = [ "aho-corasick", "bstr", @@ -1616,9 +1616,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +checksum = "fe090a42a0d3140fed8c63263ed1fecc02422cb2f4bfe55cd8de2de1b8e105dd" dependencies = [ "displaydoc", "potential_utf", @@ -1630,9 +1630,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +checksum = "25d29b57c17c6528d0dc4c2141cfa2d08200465acb537a3909c8b9b8a4a984c0" dependencies = [ "displaydoc", "litemap", @@ -1643,9 +1643,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +checksum = "404a3b8c486f205c87fe7f23ab6894d84a4ad76d6e4671ae1981ba7def87c599" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -1657,15 +1657,15 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" +checksum = "20f0088bfb7441f7a37fa0c9e20cb1e9cba1cac6b9ce8cd2a6dc44294f8e8cbc" [[package]] name = "icu_properties" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +checksum = "2ea83d991c56158f7cf898bf86436c1dfe7c07ee82fe88951f2695c9f37a379f" dependencies = [ "displaydoc", "icu_collections", @@ -1678,15 +1678,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" +checksum = "b98f29af82d02b163fe50161388350e743bb3107cc8e8c2f7314cb505faddfcf" [[package]] name = "icu_provider" -version = "2.3.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +checksum = "413099315636c588ee8f011190b329b971a09a8fae9d157e01dcae8379468514" dependencies = [ "displaydoc", "icu_locale_core", @@ -1751,9 +1751,9 @@ dependencies = [ [[package]] name = "inout" -version = "0.2.2" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" +checksum = "3b5cf4372d022092674a83b8febf5865f7a573cc0c5baec5e49c879b6a20978f" dependencies = [ "hybrid-array", ] @@ -1834,9 +1834,9 @@ checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" [[package]] name = "libc" -version = "0.2.190" +version = "0.2.191" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" [[package]] name = "libm" @@ -1944,9 +1944,9 @@ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "memmap2" -version = "0.9.11" +version = "0.9.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +checksum = "6b0ca1155a9ecc25aa12e8313a119f32edf0255fea2f44839c4ac0a12b2ac157" dependencies = [ "libc", ] @@ -2136,9 +2136,9 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "openssl-probe" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +checksum = "7e036b411754132fd6b74e94b74e9a13c5cf606fa93d1885c35860d3060820b1" [[package]] name = "page_size" @@ -3102,7 +3102,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3358,9 +3358,9 @@ dependencies = [ [[package]] name = "rmcp" -version = "3.5.1" +version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab5023804cbe0304cbe06197ac16e645a0a0d0ffa2953bf735c15c2fa0213350" +checksum = "f8be15815186873ea063bf005011d16c356804973489e40c39116ae3a2878d1f" dependencies = [ "async-trait", "base64 0.23.1", @@ -3371,6 +3371,7 @@ dependencies = [ "http-body", "http-body-util", "indexmap", + "libc", "pastey", "pin-project-lite", "rand", @@ -3390,9 +3391,9 @@ dependencies = [ [[package]] name = "rmcp-macros" -version = "3.5.1" +version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1da498700030856ba63114bceefa30ba21f5e278cb975e5b9de3529fe842b40" +checksum = "8b0748e054a5f5598b3213cf72099ae6b2745ba4d996c59ccf455150a7dbb839" dependencies = [ "darling", "proc-macro2", @@ -3428,15 +3429,15 @@ checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "rustix" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa" dependencies = [ "bitflags", "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3521,9 +3522,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.29" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +checksum = "9cb23c424d8ee841bb4d813677ca237e2a610067d594fec130f87d6093495ab5" dependencies = [ "windows-sys 0.61.2", ] @@ -3778,9 +3779,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.16.3" +version = "1.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" +checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8" [[package]] name = "snap" @@ -3975,7 +3976,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4075,9 +4076,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +checksum = "abc840c238154d58f9e86b2792261edfde565a17696e1cda1a449de27979801a" dependencies = [ "displaydoc", "zerovec", @@ -4101,9 +4102,9 @@ checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" [[package]] name = "tokio" -version = "1.53.3" +version = "1.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce3335fa71841cda333a58d7615b03901380ecf09d59b3296d21f8bbac0dde4e" +checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" dependencies = [ "bytes", "libc", @@ -4764,9 +4765,9 @@ checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" [[package]] name = "unicode-normalization" -version = "0.1.25" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +checksum = "75cd3fb03d5288b3ba9a1ee103ac7e9cbb59f0b03d64f8ddfcc04dd42758a85d" dependencies = [ "tinyvec", ] @@ -4788,9 +4789,9 @@ checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" [[package]] name = "unicode-width" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +checksum = "ca718471876992059f251e33aafc334346ada08042b7661dd1fe313a9bced557" [[package]] name = "unit-prefix" @@ -4846,9 +4847,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.28.0" +version = "1.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cc1186384beb7dd8eedea376413fd654937285ea6c9cfbb928dc3043ea4b606" +checksum = "4a0e2d11620c50695fe12bbbfde96b3477fe606e7284c22d505c12ce51214514" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -4864,9 +4865,9 @@ checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" [[package]] name = "value-trait" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3f4b4a98dfe54bc9ed3641af7ffcb837240269627dbd5cb047d13daa39736cc" +checksum = "545a3d7089f11f1741d12963c057adf9443fc24325110a6a6daf582e825fd51b" dependencies = [ "float-cmp", "halfbrown", @@ -5049,7 +5050,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -5289,9 +5290,9 @@ checksum = "550a2b930b62486a393c52d5c3b84bff264b28aa437ed64694d31e93b1757af7" [[package]] name = "yoke" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +checksum = "417ef3fcf6c300b7dbd25c85531a746931c02ad536f3cf4786ed1e88223340b3" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -5370,9 +5371,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +checksum = "f58a2799cfe9a1670c7ea8edbd52ef8edb93cb37a7f83bf31ba9e35d9dae6a40" dependencies = [ "yoke", "zerofrom", @@ -5381,9 +5382,9 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.6" +version = "0.11.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +checksum = "c3743ddbf47738d929cd1ae2e199d40d0162979dfe31aef8ae646d2bf00bf97a" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 96e253c3b..5016826e8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -185,7 +185,7 @@ uffs-fetch = { path = "crates/uffs-fetch", version = "0.6.45" } # via `tokio::net`. Crates that actually use `tokio::net` (uffs-daemon, # uffs-mcp, and the `async` feature of uffs-client) re-enable `net` # explicitly in their own Cargo.toml via `features = ["net"]`. -tokio = { version = "1.53.3", default-features = false, features = [ +tokio = { version = "1.54.1", default-features = false, features = [ "io-util", "macros", "rt", @@ -251,7 +251,7 @@ toml = "1.1.8" # ───── Data Structures ───── bitflags = "2.13.2" bytemuck = { version = "1.25.2", features = ["derive"] } -smallvec = "1.16.3" +smallvec = "1.16.4" zerocopy = { version = "0.8.63", features = ["derive"] } # ───── Error Handling ───── @@ -259,7 +259,7 @@ thiserror = "2.0.21" anyhow = "1.0.104" # ───── MCP (Model Context Protocol) ───── -rmcp = { version = "3.5.1", features = ["server", "transport-io", "macros"] } +rmcp = { version = "3.8.0", features = ["server", "transport-io", "macros"] } schemars = "1.2.3" # ───── HTTP / Tower (for MCP Streamable HTTP gateway) ───── @@ -293,7 +293,7 @@ colored = "3.1.1" # ───── Pattern Matching ───── regex = "1.13.1" memchr = "2.8.3" -globset = "0.4.20" +globset = "0.4.21" # ───── Time ───── chrono = { version = "0.4.45", features = ["serde"] } @@ -324,7 +324,7 @@ hex = "0.4.3" # Blocking HTTP with rustls + the system trust store. `rustls-tls-native-roots` # matches the existing transitive reqwest config, so no new crate enters the lock. # -# DELIBERATELY HELD AT 0.12 (re-verified 2026-10-13 against 0.13.5 and polars +# DELIBERATELY HELD AT 0.12 (re-verified 2026-10-28 against 0.13.5 and polars # 0.55.2, the latest of both; everything # else in this workspace is at latest). 0.13 still drops # `rustls-tls-native-roots` and routes system trust through @@ -352,10 +352,10 @@ crossbeam-channel = "0.5.17" # ───── Memory ───── mimalloc = "0.1.52" -memmap2 = "0.9.11" +memmap2 = "0.9.12" # ───── System ───── -libc = "0.2.190" +libc = "0.2.191" hostname = "0.4.2" # CPU-core count detection. `std::thread::available_parallelism` is the # modern alternative, but `num_cpus` handles cgroup / container quotas more @@ -364,7 +364,7 @@ hostname = "0.4.2" num_cpus = "1.17.0" # Random unique identifiers. Only the `v4` (random) feature is enabled # workspace-wide; consumers that need v1/v5/v7 etc. re-add the feature. -uuid = { version = "1.28.0", features = ["v4"] } +uuid = { version = "1.29.0", features = ["v4"] } # ───── Security / Crypto ───── aes-gcm = "0.11.1" @@ -373,7 +373,7 @@ rand_chacha = "0.10.0" security-framework = "3.7.0" # ───── Compression ───── -zstd = { version = "0.14.2", features = ["zstdmt"] } +zstd = { version = "0.14.3", features = ["zstdmt"] } # ───── Testing & Benchmarking ───── criterion = "0.8.2" diff --git a/crates/uffs-vss-requestor/Cargo.toml b/crates/uffs-vss-requestor/Cargo.toml index 25f82510d..322633367 100644 --- a/crates/uffs-vss-requestor/Cargo.toml +++ b/crates/uffs-vss-requestor/Cargo.toml @@ -75,7 +75,7 @@ uffs-version = { workspace = true, features = ["build"] } # for this internal helper — it's spawned by the Broker, never invoked # directly by a user in normal operation, so it skips the `winresource` # step `uffs-broker` uses. -cc = "1.8.0" +cc = "1.10.0" [lints] workspace = true diff --git a/supply-chain/audits.toml b/supply-chain/audits.toml index 9fac6e200..96b1265e4 100644 --- a/supply-chain/audits.toml +++ b/supply-chain/audits.toml @@ -261,6 +261,12 @@ criteria = "safe-to-deploy" delta = "1.5.1 -> 1.5.2" notes = "Delta reviewed for the 2026-09-27 dependency refresh (local diff of the crates.io sources; library code read, generated doc-only lines and tests grepped for fs, env, process, net and include use). Factors the 16-byte fold into a safe bounds-checked helper and applies it to the remaining whole blocks; build.rs unchanged. Mirrored from the docenta repo supply-chain audit performed by the same auditor/owner." +[[audits.crc32fast]] +who = "Robert M1 <50460704+githubrobbi@users.noreply.github.com>" +criteria = "safe-to-deploy" +delta = "1.5.2 -> 1.5.3" +notes = "Refactor of the hasher state. The specialized (SIMD) State becomes a zero-sized token whose existence proves CPU-feature detection succeeded; the running CRC moves to Hasher as a plain u32, and update() returns the new value instead of mutating. The unsafe calls into calculate/calculate_avx2/calculate_avx512 and the aarch64 path are unchanged and still reachable only through that token. Short inputs in the pclmulqdq path now fall back to baseline::update_slow (an existing safe function). Adds total_len() and tests. New dependabot/CI/release workflow files are not part of the build. No build.rs, process, network, filesystem or environment access." + [[audits.crossbeam-channel]] who = "Robert M1 <50460704+githubrobbi@users.noreply.github.com>" criteria = "safe-to-deploy" @@ -830,6 +836,12 @@ criteria = "safe-to-deploy" delta = "0.9.10 -> 0.9.11" notes = "Delta audit (cargo vet diff 0.9.10 -> 0.9.11), full diff reviewed across all 7 files. This is the RUSTSEC-2026-0186 remediation. Core fix in src/unix.rs + src/windows.rs MmapInner: the range methods (flush_range / flush_async_range / advise / advise_range) now bounds-check `offset > self.len || len > self.len - offset` and return InvalidInput BEFORE the pointer arithmetic — closing the unchecked-pointer-offset unsoundness. Pointer math switched from `self.ptr.offset(-(x as isize))` to `self.ptr.sub(x)` / `.add(x)` (equivalent, with SAFETY comments), the mremap success path swaps via std::mem::replace + mem::forget instead of raw ptr::write, and every FFI/pointer op is wrapped in an explicit `unsafe {}` under a new crate-level #![deny(unsafe_op_in_unsafe_fn)]; `advise`/`unchecked_advise` gain explicit `# Safety` docs. src/lib.rs only forwards those unsafe contracts; Cargo.toml/.orig bump MSRV 1.63->1.65; CHANGELOG + one doc typo. No new dependencies, no new capability (extern/network/FS) surface; the only behavioral change is stricter input validation, which can reject previously-UB calls but never newly accepts. Publisher de-vri-es (memmap2 co-maintainer)." +[[audits.memmap2]] +who = "Robert M1 <50460704+githubrobbi@users.noreply.github.com>" +criteria = "safe-to-deploy" +delta = "0.9.11 -> 0.9.12" +notes = "Adds MmapOptions::map_if_safe(): a safe constructor that maps only when the kernel guarantees immutability (memfd sealed with F_SEAL_SHRINK|F_SEAL_WRITE, or fs-verity via FS_IOC_MEASURE_VERITY on Linux/Android) and offset+len (checked_add) fits within the file; unsupported elsewhere, including Windows. New unsafe is limited to fcntl(F_GET_SEALS) and that ioctl into a zeroed MaybeUninit. Adds no_probe_handle() (skips the Windows protection probe; a flag only), caches the Windows allocation granularity in an AtomicUsize, OHOS cfg gating, Result type spelled std::io::Result, and moves tests to test/map.rs. No build.rs, process, network or environment access." + [[audits.mimalloc]] who = "Robert Nio " criteria = "safe-to-deploy" @@ -1347,6 +1359,12 @@ criteria = "safe-to-deploy" delta = "3.5.0 -> 3.5.1" notes = "Read every src change (handler/server.rs, model.rs, service/server.rs, transport/auth.rs, streamable_http_server/tower.rs): protocol 2026-07-28 peers get missing SEP-2549 cache hints filled with the most conservative values (ttlMs 0, cacheScope private); the server answers discover requests and selects the lifecycle before initialize instead of rejecting non-initialize first requests; an OAuth CredentialStore gains an optional on_refresh_token_rejected callback invoked on invalid_grant (default no-op, errors only logged, refresh guard held); INVALID_PARAMS maps to HTTP 400 only for malformed request metadata. No unsafe, no build.rs, no fs/process/env in the delta; network use stays within the existing HTTP transport." +[[audits.rmcp]] +who = "Robert M1 <50460704+githubrobbi@users.noreply.github.com>" +criteria = "safe-to-deploy" +delta = "3.5.1 -> 3.8.0" +notes = "Three minor releases (3.6.0 to 3.8.0). Non-test unsafe: none (the only new unsafe is an fcntl(F_GETFL) in a Linux test). New stdio_pipes transport reopens Linux FIFO stdin/stdout through /proc/self/fd with O_NONBLOCK so the original descriptors keep their flags; falls back to blocking stdio otherwise. AsyncRwTransport bounds incoming lines (16 MiB default). Model deserialization rejects JSON-RPC messages carrying both result and error and reads integral float ids. Client cancels abandoned requests via a drop guard and answers pings during initialize. OAuth (auth feature, not enabled in this workspace): SSRF hardening via a DNS resolver that blocks link-local, metadata and reserved addresses and allows private or loopback only when the base URL names one; reads the standard proxy env vars only to exempt configured proxies; bounded redirects. Streamable-HTTP fixes (feature not enabled here). New Linux-only libc dependency. No build.rs, no process spawning, no new outbound endpoints." + [[audits.rmcp-macros]] who = "Robert Nio <50460704+githubrobbi@users.noreply.github.com>" criteria = "safe-to-deploy" @@ -1441,6 +1459,12 @@ criteria = "safe-to-deploy" delta = "3.5.0 -> 3.5.1" notes = 'Read the full delta: Cargo.toml version bump and removal of the redundant resolver = "2" key. No source change.' +[[audits.rmcp-macros]] +who = "Robert M1 <50460704+githubrobbi@users.noreply.github.com>" +criteria = "safe-to-deploy" +delta = "3.5.1 -> 3.8.0" +notes = "Version bump only, plus the LICENSE file now shipped in the crate. No code change." + [[audits.rustc-hash]] who = "Robert M1 <50460704+githubrobbi@users.noreply.github.com>" criteria = "safe-to-deploy" @@ -1555,6 +1579,12 @@ criteria = "safe-to-deploy" version = "1.0.6" notes = "Full audit (908 LOC). Capability is std::fs File opens by design (the crate's purpose: file-identity comparison via dev/ino on Unix, volume/index via handle info on Windows) — no net/process/env, no build.rs. Unsafe reviewed: File::from_raw_fd(0/1/2) for the std streams is paired with an is_std flag whose Drop impl leaks the descriptor via into_raw_fd, so std fds are never closed (no double-close, no fd hijack); Windows side mirrors this via winapi-util's non-owning HandleRef. Publisher verified on crates.io: BurntSushi." +[[audits.schannel]] +who = "Robert M1 <50460704+githubrobbi@users.noreply.github.com>" +criteria = "safe-to-deploy" +delta = "0.1.29 -> 0.1.30" +notes = "open_current_user/open_local_machine now delegate to a new CertStoreBuilder whose open() makes the same CertOpenStore call as before (same provider, encoding, location shift and NUL-terminated wide name kept alive across the call), with an optional CERT_STORE_READONLY_FLAG. Test-fixture thumbprints updated. No other change." + [[audits.schemars]] who = "Robert M1 <50460704+githubrobbi@users.noreply.github.com>" criteria = "safe-to-deploy" @@ -1686,6 +1716,12 @@ criteria = "safe-to-deploy" delta = "1.16.2 -> 1.16.3" notes = "Delta reviewed with attention to unsafe, since this crate's soundness rests on it: no unsafe line is added or altered in this delta. Mirrored from the uffs-products repo supply-chain audit performed by the same auditor/owner." +[[audits.smallvec]] +who = "Robert M1 <50460704+githubrobbi@users.noreply.github.com>" +criteria = "safe-to-deploy" +delta = "1.16.3 -> 1.16.4" +notes = "Version bump only (plus the crate's own dev Cargo.lock). No source change." + [[audits.snap]] who = "Robert M1 <50460704+githubrobbi@users.noreply.github.com>" criteria = "safe-to-deploy" @@ -2029,6 +2065,12 @@ criteria = "safe-to-deploy" delta = "0.12.1 -> 0.12.2" notes = "Main change is in src/generator.rs: the SSE string writer is renamed sse4.2 -> sse2 (it only ever used SSE2 intrinsics; runtime detection now checks sse2), existing intrinsics bodies are wrapped in explicit `unsafe {}` blocks per edition-2024 style with no logic change, and a real bug is fixed where the scalar fallback `write_string_rust` failed to advance the input slice, which could double-write escaped content; a regression test covers it. Remaining changes are lifetime elaborations on iterator trait methods, `#[derive(Default)]` replacing manual impls, removed redundant `#[must_use]`, and CI workflow reshuffling. No new FFI, no build.rs, no fs/net/process/env surface, dev-only Cargo.lock churn." +[[audits.value-trait]] +who = "Robert M1 <50460704+githubrobbi@users.noreply.github.com>" +criteria = "safe-to-deploy" +delta = "0.12.2 -> 0.12.3" +notes = "BaseGenerator::write_float writes null for NaN and infinities (as serde_json does) instead of a non-JSON token; one test. No unsafe, no other change." + [[audits.want]] who = "Robert M1 <50460704+githubrobbi@users.noreply.github.com>" criteria = "safe-to-deploy" @@ -2585,12 +2627,54 @@ user-id = 359 # Sean McArthur (seanmonstar) start = "2022-01-15" end = "2027-04-23" +[[trusted.icu_collections]] +criteria = "safe-to-deploy" +user-id = 166196 # Robert Bastian (robertbastian) +start = "2023-01-26" +end = "2027-10-29" + +[[trusted.icu_locale_core]] +criteria = "safe-to-deploy" +user-id = 166196 # Robert Bastian (robertbastian) +start = "2025-05-07" +end = "2027-10-29" + +[[trusted.icu_normalizer]] +criteria = "safe-to-deploy" +user-id = 166196 # Robert Bastian (robertbastian) +start = "2023-01-26" +end = "2027-10-29" + [[trusted.icu_normalizer_data]] criteria = "safe-to-deploy" user-id = 1139 # Manish Goregaokar (Manishearth) start = "2023-10-03" end = "2027-08-04" +[[trusted.icu_normalizer_data]] +criteria = "safe-to-deploy" +user-id = 166196 # Robert Bastian (robertbastian) +start = "2023-11-16" +end = "2027-10-29" + +[[trusted.icu_properties]] +criteria = "safe-to-deploy" +user-id = 166196 # Robert Bastian (robertbastian) +start = "2023-01-26" +end = "2027-10-29" + +[[trusted.icu_properties_data]] +criteria = "safe-to-deploy" +user-id = 166196 # Robert Bastian (robertbastian) +start = "2023-11-16" +end = "2027-10-29" + +[[trusted.icu_provider]] +criteria = "safe-to-deploy" +user-id = 166196 # Robert Bastian (robertbastian) +start = "2022-10-08" +end = "2027-10-29" + [[trusted.idna]] criteria = "safe-to-deploy" user-id = 1139 # Manish Goregaokar (Manishearth) @@ -2603,6 +2687,12 @@ user-id = 539 # Josh Stone (cuviper) start = "2020-01-15" end = "2027-08-29" +[[trusted.inout]] +criteria = "safe-to-deploy" +trusted-publisher = "github:RustCrypto/utils" +start = "2026-10-14" +end = "2027-10-29" + [[trusted.js-sys]] criteria = "safe-to-deploy" user-id = 1 # Alex Crichton (alexcrichton) @@ -2627,6 +2717,12 @@ user-id = 55123 # rust-lang-owner start = "2024-08-15" end = "2027-08-04" +[[trusted.libc]] +criteria = "safe-to-deploy" +trusted-publisher = "github:rust-lang/libc" +start = "2026-10-02" +end = "2027-10-29" + [[trusted.litemap]] criteria = "safe-to-deploy" user-id = 1139 # Manish Goregaokar (Manishearth) @@ -2659,6 +2755,12 @@ user-id = 359 # Sean McArthur (seanmonstar) start = "2019-06-10" end = "2027-04-23" +[[trusted.openssl-probe]] +criteria = "safe-to-deploy" +user-id = 4556 # Dirkjan Ochtman (djc) +start = "2025-12-29" +end = "2027-10-29" + [[trusted.polars]] criteria = "safe-to-deploy" user-id = 76923 # Ritchie Vink (ritchie46) @@ -2940,6 +3042,12 @@ start = "2019-10-09" end = "2027-07-20" notes = "dtolnay-published, already-trusted publisher; patch bump 2.0.18->2.0.19." +[[trusted.tinystr]] +criteria = "safe-to-deploy" +user-id = 166196 # Robert Bastian (robertbastian) +start = "2023-01-26" +end = "2027-10-29" + [[trusted.tokio]] criteria = "safe-to-deploy" user-id = 6741 # Alice Ryhl (Darksonn) @@ -3007,6 +3115,12 @@ start = "2021-10-02" end = "2027-09-27" notes = "Mirrored from the docenta repo supply-chain audit performed by the same auditor/owner." +[[trusted.unicode-width]] +criteria = "safe-to-deploy" +user-id = 1139 # Manish Goregaokar (Manishearth) +start = "2019-12-05" +end = "2027-10-29" + [[trusted.universal-hash]] criteria = "safe-to-deploy" trusted-publisher = "github:RustCrypto/traits" @@ -3110,6 +3224,12 @@ user-id = 1139 # Manish Goregaokar (Manishearth) start = "2021-11-01" end = "2027-08-04" +[[trusted.yoke]] +criteria = "safe-to-deploy" +user-id = 166196 # Robert Bastian (robertbastian) +start = "2023-01-26" +end = "2027-10-29" + [[trusted.yoke-derive]] criteria = "safe-to-deploy" user-id = 1139 # Manish Goregaokar (Manishearth) @@ -3151,12 +3271,24 @@ end = "2027-07-20" notes = "Mirrored from the uffs-products repo supply-chain audit performed by the same auditor/owner." aggregated-from = "http://127.0.0.1:8731/uffs-products.toml" +[[trusted.zerovec]] +criteria = "safe-to-deploy" +user-id = 166196 # Robert Bastian (robertbastian) +start = "2023-01-26" +end = "2027-10-29" + [[trusted.zerovec-derive]] criteria = "safe-to-deploy" user-id = 1139 # Manish Goregaokar (Manishearth) start = "2021-12-11" end = "2027-08-04" +[[trusted.zerovec-derive]] +criteria = "safe-to-deploy" +user-id = 166196 # Robert Bastian (robertbastian) +start = "2023-01-26" +end = "2027-10-29" + [[trusted.zlib-rs]] criteria = "safe-to-deploy" user-id = 1303 diff --git a/supply-chain/config.toml b/supply-chain/config.toml index 271b06a6b..5d7c9334e 100644 --- a/supply-chain/config.toml +++ b/supply-chain/config.toml @@ -400,30 +400,6 @@ criteria = "safe-to-deploy" version = "0.1.65" criteria = "safe-to-deploy" -[[exemptions.icu_collections]] -version = "2.2.0" -criteria = "safe-to-deploy" - -[[exemptions.icu_locale_core]] -version = "2.2.0" -criteria = "safe-to-deploy" - -[[exemptions.icu_normalizer]] -version = "2.2.0" -criteria = "safe-to-deploy" - -[[exemptions.icu_properties]] -version = "2.2.0" -criteria = "safe-to-deploy" - -[[exemptions.icu_properties_data]] -version = "2.2.0" -criteria = "safe-to-deploy" - -[[exemptions.icu_provider]] -version = "2.2.0" -criteria = "safe-to-deploy" - [[exemptions.idna_adapter]] version = "1.2.2" criteria = "safe-to-deploy" @@ -512,10 +488,6 @@ criteria = "safe-to-deploy" version = "1.70.2" criteria = "safe-to-deploy" -[[exemptions.openssl-probe]] -version = "0.2.1" -criteria = "safe-to-deploy" - [[exemptions.parking_lot]] version = "0.12.5" criteria = "safe-to-deploy" @@ -768,10 +740,6 @@ criteria = "safe-to-deploy" version = "0.2.32" criteria = "safe-to-deploy" -[[exemptions.tinystr]] -version = "0.8.3" -criteria = "safe-to-deploy" - [[exemptions.tokio-macros]] version = "2.7.0" criteria = "safe-to-deploy" @@ -948,18 +916,10 @@ criteria = "safe-to-deploy" version = "0.52.6" criteria = "safe-to-deploy" -[[exemptions.yoke]] -version = "0.8.2" -criteria = "safe-to-deploy" - [[exemptions.zerotrie]] version = "0.2.4" criteria = "safe-to-deploy" -[[exemptions.zerovec]] -version = "0.11.6" -criteria = "safe-to-deploy" - [[exemptions.zstd]] version = "0.13.3" criteria = "safe-to-deploy" diff --git a/supply-chain/imports.lock b/supply-chain/imports.lock index 1fb38b372..bed2c56f8 100644 --- a/supply-chain/imports.lock +++ b/supply-chain/imports.lock @@ -53,8 +53,8 @@ user-login = "Darksonn" user-name = "Alice Ryhl" [[publisher.cc]] -version = "1.8.0" -when = "2026-10-11" +version = "1.10.0" +when = "2026-10-23" trusted-publisher = "github:rust-lang/cc-rs" [[publisher.cfg-if]] @@ -138,8 +138,8 @@ when = "2026-05-20" trusted-publisher = "github:RustCrypto/block-modes" [[publisher.ctutils]] -version = "0.4.6" -when = "2026-10-25" +version = "0.4.7" +when = "2026-10-29" trusted-publisher = "github:RustCrypto/utils" [[publisher.digest]] @@ -160,8 +160,8 @@ user-login = "cuviper" user-name = "Josh Stone" [[publisher.find-msvc-tools]] -version = "0.1.14" -when = "2026-09-25" +version = "0.1.15" +when = "2026-10-19" trusted-publisher = "github:rust-lang/cc-rs" [[publisher.flate2]] @@ -183,8 +183,8 @@ user-id = 55123 user-login = "rust-lang-owner" [[publisher.globset]] -version = "0.4.20" -when = "2026-08-04" +version = "0.4.21" +when = "2026-10-27" user-id = 189 user-login = "BurntSushi" user-name = "Andrew Gallant" @@ -243,12 +243,54 @@ user-id = 359 user-login = "seanmonstar" user-name = "Sean McArthur" +[[publisher.icu_collections]] +version = "2.4.0" +when = "2026-10-22" +user-id = 166196 +user-login = "robertbastian" +user-name = "Robert Bastian" + +[[publisher.icu_locale_core]] +version = "2.4.0" +when = "2026-10-22" +user-id = 166196 +user-login = "robertbastian" +user-name = "Robert Bastian" + +[[publisher.icu_normalizer]] +version = "2.4.0" +when = "2026-10-22" +user-id = 166196 +user-login = "robertbastian" +user-name = "Robert Bastian" + [[publisher.icu_normalizer_data]] -version = "2.2.0" -when = "2026-04-01" -user-id = 1139 -user-login = "Manishearth" -user-name = "Manish Goregaokar" +version = "2.4.0" +when = "2026-10-22" +user-id = 166196 +user-login = "robertbastian" +user-name = "Robert Bastian" + +[[publisher.icu_properties]] +version = "2.4.0" +when = "2026-10-22" +user-id = 166196 +user-login = "robertbastian" +user-name = "Robert Bastian" + +[[publisher.icu_properties_data]] +version = "2.4.0" +when = "2026-10-22" +user-id = 166196 +user-login = "robertbastian" +user-name = "Robert Bastian" + +[[publisher.icu_provider]] +version = "2.4.0" +when = "2026-10-22" +user-id = 166196 +user-login = "robertbastian" +user-name = "Robert Bastian" [[publisher.idna]] version = "1.0.2" @@ -264,6 +306,11 @@ user-id = 539 user-login = "cuviper" user-name = "Josh Stone" +[[publisher.inout]] +version = "0.2.5" +when = "2026-10-19" +trusted-publisher = "github:RustCrypto/utils" + [[publisher.js-sys]] version = "0.3.106" when = "2026-09-25" @@ -277,10 +324,9 @@ user-login = "ritchie46" user-name = "Ritchie Vink" [[publisher.libc]] -version = "0.2.189" -when = "2026-07-21" -user-id = 55123 -user-login = "rust-lang-owner" +version = "0.2.191" +when = "2026-10-16" +trusted-publisher = "github:rust-lang/libc" [[publisher.litemap]] version = "0.8.2" @@ -317,6 +363,13 @@ user-id = 359 user-login = "seanmonstar" user-name = "Sean McArthur" +[[publisher.openssl-probe]] +version = "0.2.2" +when = "2026-10-26" +user-id = 4556 +user-login = "djc" +user-name = "Dirkjan Ochtman" + [[publisher.polars]] version = "0.55.2" when = "2026-08-06" @@ -540,8 +593,8 @@ user-login = "seanmonstar" user-name = "Sean McArthur" [[publisher.rustix]] -version = "1.1.5" -when = "2026-09-16" +version = "1.1.6" +when = "2026-10-27" user-id = 6825 user-login = "sunfishcode" user-name = "Dan Gohman" @@ -628,9 +681,16 @@ user-id = 3618 user-login = "dtolnay" user-name = "David Tolnay" +[[publisher.tinystr]] +version = "0.8.5" +when = "2026-10-22" +user-id = 166196 +user-login = "robertbastian" +user-name = "Robert Bastian" + [[publisher.tokio]] -version = "1.53.3" -when = "2026-10-13" +version = "1.54.1" +when = "2026-10-21" user-id = 6741 user-login = "Darksonn" user-name = "Alice Ryhl" @@ -699,8 +759,8 @@ user-login = "dtolnay" user-name = "David Tolnay" [[publisher.unicode-normalization]] -version = "0.1.25" -when = "2025-10-30" +version = "0.1.26" +when = "2026-10-16" user-id = 1139 user-login = "Manishearth" user-name = "Manish Goregaokar" @@ -713,8 +773,8 @@ user-login = "Manishearth" user-name = "Manish Goregaokar" [[publisher.unicode-width]] -version = "0.2.2" -when = "2025-10-06" +version = "0.2.3" +when = "2026-10-28" user-id = 1139 user-login = "Manishearth" user-name = "Manish Goregaokar" @@ -739,8 +799,8 @@ user-login = "hsivonen" user-name = "Henri Sivonen" [[publisher.uuid]] -version = "1.28.0" -when = "2026-10-10" +version = "1.29.0" +when = "2026-10-23" user-id = 3204 user-login = "KodrAus" user-name = "Ashley Mannix" @@ -801,6 +861,13 @@ user-id = 1139 user-login = "Manishearth" user-name = "Manish Goregaokar" +[[publisher.yoke]] +version = "0.8.4" +when = "2026-10-22" +user-id = 166196 +user-login = "robertbastian" +user-name = "Robert Bastian" + [[publisher.zerocopy]] version = "0.8.63" when = "2026-10-13" @@ -823,12 +890,19 @@ version = "1.9.1" when = "2026-10-06" trusted-publisher = "github:RustCrypto/utils" +[[publisher.zerovec]] +version = "0.11.9" +when = "2026-10-22" +user-id = 166196 +user-login = "robertbastian" +user-name = "Robert Bastian" + [[publisher.zerovec-derive]] -version = "0.11.5" -when = "2026-08-18" -user-id = 1139 -user-login = "Manishearth" -user-name = "Manish Goregaokar" +version = "0.11.7" +when = "2026-10-22" +user-id = 166196 +user-login = "robertbastian" +user-name = "Robert Bastian" [[publisher.zmij]] version = "1.0.23" @@ -973,12 +1047,6 @@ who = "Dan Gohman " criteria = "safe-to-deploy" version = "0.1.2" -[[audits.bytecode-alliance.audits.inout]] -who = "Andrew Brown " -criteria = "safe-to-deploy" -version = "0.1.3" -notes = "A part of RustCrypto/utils, this crate is designed to handle unsafe buffers and carefully documents the safety concerns throughout. Older versions of this tally up to ~130k daily downloads." - [[audits.bytecode-alliance.audits.itertools]] who = "Nick Fitzgerald " criteria = "safe-to-deploy" @@ -1676,11 +1744,6 @@ who = "David Cook " criteria = "safe-to-deploy" delta = "0.4.0 -> 0.4.1" -[[audits.isrg.audits.inout]] -who = "David Cook " -criteria = "safe-to-deploy" -delta = "0.1.4 -> 0.2.2" - [[audits.isrg.audits.itertools]] who = "Tim Geoghegan " criteria = "safe-to-run" @@ -1770,15 +1833,6 @@ end = "2027-04-23" notes = "All code written or reviewed by Manish" aggregated-from = "https://raw.githubusercontent.com/mozilla-firefox/firefox/main/supply-chain/audits.toml" -[[audits.mozilla.wildcard-audits.unicode-width]] -who = "Manish Goregaokar " -criteria = "safe-to-deploy" -user-id = 1139 # Manish Goregaokar (Manishearth) -start = "2019-12-05" -end = "2026-02-01" -notes = "All code written or reviewed by Manish" -aggregated-from = "https://raw.githubusercontent.com/mozilla-firefox/firefox/main/supply-chain/audits.toml" - [[audits.mozilla.wildcard-audits.utf8_iter]] who = "Makoto Kato " criteria = "safe-to-deploy" @@ -2547,12 +2601,6 @@ criteria = "safe-to-deploy" delta = "0.3.13 -> 0.3.14" aggregated-from = "https://raw.githubusercontent.com/zcash/librustzcash/main/supply-chain/audits.toml" -[[audits.zcash.audits.inout]] -who = "Jack Grigg " -criteria = "safe-to-deploy" -delta = "0.1.3 -> 0.1.4" -aggregated-from = "https://raw.githubusercontent.com/zcash/wallet/main/supply-chain/audits.toml" - [[audits.zcash.audits.proc-macro-crate]] who = "Jack Grigg " criteria = "safe-to-deploy" From 7a8ce7f8229b3793aad413faa2205e5f7c717674 Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:24:18 -0700 Subject: [PATCH 20/21] chore(toolchain): bump the pin to nightly-2026-10-29 and fix what it surfaced BinaryHeap::len is a const fn from nightly-2026-10-25 on, so clippy's missing_const_for_fn now flags SampleHeap::len and is_empty, which only call it. Both are const now. That is the only clippy finding between the 2026-10-14 pin and today's nightly; `just toolchain-sync` validated nightly-2026-10-29 with clippy (all features and none), the Windows xwin clippy and the Linux zigbuild. The new rustfmt wraps comments to the configured width more strictly; 20 files are reformatted, comment wrapping only (a word diff shows no change other than line breaks). --- .../uffs-cli/src/commands/uninstall/effects.rs | 3 ++- crates/uffs-cli/src/commands/update/plan.rs | 18 +++++++++--------- crates/uffs-client/src/protocol/cli_args.rs | 13 +++++++------ crates/uffs-core/src/aggregate/rollup.rs | 3 ++- crates/uffs-core/src/aggregate/sample_heap.rs | 4 ++-- .../src/compact_loader_trigram_oracle_tests.rs | 3 ++- crates/uffs-core/src/compiled_pattern/mod.rs | 6 ++++-- .../src/search/query/numeric_sort_key.rs | 4 ++-- crates/uffs-daemon/src/cache/policy.rs | 8 ++++---- crates/uffs-format/src/writer.rs | 3 ++- crates/uffs-mft/src/commands/windows/save.rs | 5 +++-- crates/uffs-mft/src/io.rs | 3 ++- .../src/io/readers/parallel/sliding_window.rs | 8 +++++--- .../src/io/readers/parallel/to_index.rs | 8 +++++--- .../io/readers/parallel/to_index_parallel.rs | 8 +++++--- crates/uffs-mft/src/lib.rs | 14 ++++++++------ crates/uffs-mft/src/parse/full.rs | 3 ++- crates/uffs-mft/src/platform.rs | 4 ++-- crates/uffs-mft/src/platform/metafile_read.rs | 6 +++--- crates/uffs-mft/src/reader/dataframe_build.rs | 6 ++++-- crates/uffs-update/src/restore.rs | 10 +++++----- rust-toolchain.toml | 2 +- 22 files changed, 81 insertions(+), 61 deletions(-) diff --git a/crates/uffs-cli/src/commands/uninstall/effects.rs b/crates/uffs-cli/src/commands/uninstall/effects.rs index 9dfe6ecdb..c411da93e 100644 --- a/crates/uffs-cli/src/commands/uninstall/effects.rs +++ b/crates/uffs-cli/src/commands/uninstall/effects.rs @@ -481,7 +481,8 @@ fn remove_service_via_uac(service: &str) -> Result<()> { } // Typed so the executor recognises the decline and LEAVES the broker // (service + its locked binary) as a clean outcome, instead of the raw - // Access-denied that deleting the still-running broker's image produces. + // Access-denied that deleting the still-running broker's image + // produces. Some(UAC_NOT_GRANTED_EXIT) => Err(super::remove::ElevationDeclined.into()), other => bail!( "elevated service-removal helper failed (exit {other:?}) — {service} may still \ diff --git a/crates/uffs-cli/src/commands/update/plan.rs b/crates/uffs-cli/src/commands/update/plan.rs index 84ee28acc..b84b2a48d 100644 --- a/crates/uffs-cli/src/commands/update/plan.rs +++ b/crates/uffs-cli/src/commands/update/plan.rs @@ -190,9 +190,9 @@ fn classify_root( let (action, needs_elevation, needs_non_elevated) = match root.channel { Channel::DevBuild => (UpdateAction::SkipDevBuild, false, false), Channel::Unknown => (UpdateAction::SkipUnknown, false, false), - // Unmanaged: the uffs-update helper stops the daemon via the *gated* CLI - // `daemon stop` (no UAC), so an elevated daemon (or a broker) hosted here - // genuinely needs elevation — as does a non-writable dir. + // Unmanaged: the uffs-update helper stops the daemon via the *gated* + // CLI `daemon stop` (no UAC), so an elevated daemon (or a broker) + // hosted here genuinely needs elevation — as does a non-writable dir. Channel::Unmanaged => ( UpdateAction::ReplaceBinaries, !dir_writable(&root.dir) @@ -200,12 +200,12 @@ fn classify_root( false, ), // winget's own `quiesce` stops a daemon inside the package - // **cooperatively** (an IPC shutdown the daemon honors regardless of its - // elevation — not the gated CLI `daemon stop`) and cycles the package - // broker via its own one-UAC path. So a winget root is never gated here; - // it only needs the privilege level winget itself demands: user scope - // must run non-elevated (winget refuses a user package elevated), machine - // scope requires Administrator. + // **cooperatively** (an IPC shutdown the daemon honors regardless of + // its elevation — not the gated CLI `daemon stop`) and cycles the + // package broker via its own one-UAC path. So a winget root is never + // gated here; it only needs the privilege level winget itself demands: + // user scope must run non-elevated (winget refuses a user package + // elevated), machine scope requires Administrator. Channel::WinGet => match root.scope { Scope::Machine => (UpdateAction::WingetUpgrade, true, false), Scope::User | Scope::Unknown => (UpdateAction::WingetUpgrade, false, true), diff --git a/crates/uffs-client/src/protocol/cli_args.rs b/crates/uffs-client/src/protocol/cli_args.rs index 1fa561764..afdeb47f2 100644 --- a/crates/uffs-client/src/protocol/cli_args.rs +++ b/crates/uffs-client/src/protocol/cli_args.rs @@ -52,7 +52,8 @@ impl SearchParams { "--hide-system" => raw.hide_system = true, "--hide-ads" => raw.hide_ads = true, "--normalize-malformed" => raw.normalize_malformed = true, - // Diagnostic only -- uffs-content sets this itself for real jobs. + // Diagnostic only -- uffs-content sets this itself for real + // jobs. "--resolve-lcn-order" => raw.resolve_lcn_order = true, // WI-4.4 forensic filters: find ill-formed (non-UTF-8) names. "--malformed" => raw.malformed = Some(true), @@ -95,9 +96,9 @@ impl SearchParams { raw.agg_page_size = Some(parse_u16("--agg-page-size", &pv)?); } "--attr" => raw.attr = Some(flag_val(&arg, "--attr", &mut iter)?), - // Snapshot delete-visibility diff: search the deleted set of the - // given baseline MFT capture vs the live index. Every other flag - // then filters/shapes that set like a normal search. + // Snapshot delete-visibility diff: search the deleted set of + // the given baseline MFT capture vs the live index. Every other + // flag then filters/shapes that set like a normal search. "--diff" => raw.diff_baseline = Some(flag_val(&arg, "--diff", &mut iter)?), "--newer" => raw.newer = Some(flag_val(&arg, "--newer", &mut iter)?), "--older" => raw.older = Some(flag_val(&arg, "--older", &mut iter)?), @@ -735,8 +736,8 @@ impl RawCliArgs { // Profiling profile: self.profile || self.benchmark, aggregations, - // Row precedence (high → low): --rows (on) > agg (off) > --no-output (off) > default - // (on). + // Row precedence (high → low): --rows (on) > agg (off) > + // --no-output (off) > default (on). include_rows: force_rows || (agg_specs.is_empty() && !self.no_output), resolve_lcn_order: self.resolve_lcn_order, agg_cursor: self.agg_cursor, diff --git a/crates/uffs-core/src/aggregate/rollup.rs b/crates/uffs-core/src/aggregate/rollup.rs index 9232cfb3b..ebf01093c 100644 --- a/crates/uffs-core/src/aggregate/rollup.rs +++ b/crates/uffs-core/src/aggregate/rollup.rs @@ -393,7 +393,8 @@ mod tests { source_epoch: 0, bloom: None, path_trie: None, - // unused by aggregation tests — see compact.rs::frs_to_compact docs. + // unused by aggregation tests — see compact.rs::frs_to_compact + // docs. frs_to_compact: Vec::new(), delta: None, } diff --git a/crates/uffs-core/src/aggregate/sample_heap.rs b/crates/uffs-core/src/aggregate/sample_heap.rs index b547d499a..c7061a397 100644 --- a/crates/uffs-core/src/aggregate/sample_heap.rs +++ b/crates/uffs-core/src/aggregate/sample_heap.rs @@ -243,7 +243,7 @@ impl SampleHeap { /// Number of entries currently in the heap. #[must_use] - pub fn len(&self) -> usize { + pub const fn len(&self) -> usize { if self.sort_desc { self.heap_desc.len() } else { @@ -253,7 +253,7 @@ impl SampleHeap { /// Whether the heap is empty. #[must_use] - pub fn is_empty(&self) -> bool { + pub const fn is_empty(&self) -> bool { self.len() == 0 } diff --git a/crates/uffs-core/src/compact_loader_trigram_oracle_tests.rs b/crates/uffs-core/src/compact_loader_trigram_oracle_tests.rs index 1d12cda62..fb8212787 100644 --- a/crates/uffs-core/src/compact_loader_trigram_oracle_tests.rs +++ b/crates/uffs-core/src/compact_loader_trigram_oracle_tests.rs @@ -278,7 +278,8 @@ fn children_overlay_equals_compacted_rebuild_oracle() { let mut drive = build_nested_dir_fixture(); apply_usn_patch(&mut drive, &[ - // Move moved.txt (frs8) from alpha(6) to beta(7) — same name, new parent. + // Move moved.txt (frs8) from alpha(6) to beta(7) — same name, new + // parent. FileChange { frs: 8_u64.into(), parent_frs: 7_u64.into(), diff --git a/crates/uffs-core/src/compiled_pattern/mod.rs b/crates/uffs-core/src/compiled_pattern/mod.rs index cb570e6d5..030557e2c 100644 --- a/crates/uffs-core/src/compiled_pattern/mod.rs +++ b/crates/uffs-core/src/compiled_pattern/mod.rs @@ -152,7 +152,8 @@ impl CompiledPattern { // Any: always matches Self::Any => lit(true), - // Exact: direct equality check (case-sensitive) or regex (case-insensitive) + // Exact: direct equality check (case-sensitive) or regex + // (case-insensitive) Self::Exact(value) => { if case_sensitive { col_expr.eq(lit(value.clone())) @@ -189,7 +190,8 @@ impl CompiledPattern { } } - // Contains: contains_literal (case-sensitive) or regex (case-insensitive) + // Contains: contains_literal (case-sensitive) or regex + // (case-insensitive) Self::Contains(needle) => { if case_sensitive { col_expr.str().contains_literal(lit(needle.clone())) diff --git a/crates/uffs-core/src/search/query/numeric_sort_key.rs b/crates/uffs-core/src/search/query/numeric_sort_key.rs index 541f91469..716cb7312 100644 --- a/crates/uffs-core/src/search/query/numeric_sort_key.rs +++ b/crates/uffs-core/src/search/query/numeric_sort_key.rs @@ -100,8 +100,8 @@ pub(super) fn extract_sort_key( FieldId::Virtual => i64::from(rec.flags & 0x0001_0000 != 0), // WI-4.4: leaf-name malformity as 0/1, from the lossless bytes (matches // the hot-path filter). `MalformedPath` needs the resolved parent chain - // (unavailable at this sort-key stage) and `NameHex` is not sortable, so - // both fall through to the `Modified` proxy below. + // (unavailable at this sort-key stage) and `NameHex` is not sortable, + // so both fall through to the `Modified` proxy below. FieldId::Malformed => { i64::from(core::str::from_utf8(rec.name_bytes(&drive.names)).is_err()) } diff --git a/crates/uffs-daemon/src/cache/policy.rs b/crates/uffs-daemon/src/cache/policy.rs index 44c76dd7d..0499aa592 100644 --- a/crates/uffs-daemon/src/cache/policy.rs +++ b/crates/uffs-daemon/src/cache/policy.rs @@ -379,10 +379,10 @@ pub(crate) const fn next_state_for_idle_with_thresholds( ShardState::Parked if idle_secs >= thresholds.parked_to_cold_secs => Some(ShardState::Cold), // Two distinct cases collapsed into one arm because they // share the same outcome: - // * Hot / Warm / Parked: idle window not exceeded for the current tier (the guard above - // this arm failed). - // * Cold / Unknown / Evicting: floor + controller-only states; the policy never targets - // these. + // * Hot / Warm / Parked: idle window not exceeded for the current + // tier (the guard above this arm failed). + // * Cold / Unknown / Evicting: floor + controller-only states; the + // policy never targets these. ShardState::Hot | ShardState::Warm | ShardState::Parked diff --git a/crates/uffs-format/src/writer.rs b/crates/uffs-format/src/writer.rs index 71194e621..85ca37f21 100644 --- a/crates/uffs-format/src/writer.rs +++ b/crates/uffs-format/src/writer.rs @@ -463,7 +463,8 @@ mod tests { .with_quote("\""); let cases = [ - // (path, name, expected_ext) + // (path, name, + // expected_ext) ("F:\\Ch\\.bash_history", ".bash_history", ""), ("C:\\Users\\rnio\\.gitignore", ".gitignore", ""), ("C:\\tmp\\README", "README", ""), diff --git a/crates/uffs-mft/src/commands/windows/save.rs b/crates/uffs-mft/src/commands/windows/save.rs index 662c9c41a..1bb724bdb 100644 --- a/crates/uffs-mft/src/commands/windows/save.rs +++ b/crates/uffs-mft/src/commands/windows/save.rs @@ -130,8 +130,9 @@ pub(crate) async fn cmd_save( compression_level, volume_letter: drive, raw_compat, - // `save` doesn't compute the root reserved-cluster adjustment; `capture` - // is the path that bakes it into the header for offline parity. + // `save` doesn't compute the root reserved-cluster adjustment; + // `capture` is the path that bakes it into the header for offline + // parity. reserved_allocated_bytes: 0, }; diff --git a/crates/uffs-mft/src/io.rs b/crates/uffs-mft/src/io.rs index 35d48689d..638d72fe6 100644 --- a/crates/uffs-mft/src/io.rs +++ b/crates/uffs-mft/src/io.rs @@ -16,7 +16,8 @@ pub use crate::ntfs::SECTOR_SIZE; // `SECTOR_SIZE_U64` is the matching u64 alias used by the I/O readers. // Gated to `cfg(windows)` to match its only consumers (the Windows-only -// reader implementations under `io::readers::*` + `reader::persistence_capture`). +// reader implementations under `io::readers::*` + +// `reader::persistence_capture`). #[cfg(windows)] pub(crate) use crate::ntfs::SECTOR_SIZE_U64; diff --git a/crates/uffs-mft/src/io/readers/parallel/sliding_window.rs b/crates/uffs-mft/src/io/readers/parallel/sliding_window.rs index 6ab5e4e20..b5ef50d49 100644 --- a/crates/uffs-mft/src/io/readers/parallel/sliding_window.rs +++ b/crates/uffs-mft/src/io/readers/parallel/sliding_window.rs @@ -220,7 +220,8 @@ impl ParallelMftReader { }; let mut in_flight_op = Box::pin(InFlightOp { // SAFETY: `OVERLAPPED` is a plain Windows FFI struct and an - // all-zero value is the required initial state before offsets are set. + // all-zero value is the required initial state before + // offsets are set. overlapped: unsafe { core::mem::zeroed() }, buffer, op, @@ -407,8 +408,9 @@ impl ParallelMftReader { )); }; let mut new_in_flight = Box::pin(InFlightOp { - // SAFETY: `OVERLAPPED` is a plain Windows FFI struct and an - // all-zero value is the required initial state before offsets are set. + // SAFETY: `OVERLAPPED` is a plain Windows FFI struct + // and an all-zero value is the required initial state + // before offsets are set. overlapped: unsafe { core::mem::zeroed() }, buffer, op: next_op, diff --git a/crates/uffs-mft/src/io/readers/parallel/to_index.rs b/crates/uffs-mft/src/io/readers/parallel/to_index.rs index a91b0d68f..cd0cb5647 100644 --- a/crates/uffs-mft/src/io/readers/parallel/to_index.rs +++ b/crates/uffs-mft/src/io/readers/parallel/to_index.rs @@ -294,7 +294,8 @@ impl ParallelMftReader { }; let mut in_flight_op = Box::pin(InFlightOp { // SAFETY: `OVERLAPPED` is a plain Windows FFI struct and an - // all-zero value is the required initial state before offsets are set. + // all-zero value is the required initial state before + // offsets are set. overlapped: unsafe { core::mem::zeroed() }, buffer, op, @@ -525,8 +526,9 @@ impl ParallelMftReader { )); }; let mut new_in_flight = Box::pin(InFlightOp { - // SAFETY: `OVERLAPPED` is a plain Windows FFI struct and an - // all-zero value is the required initial state before offsets are set. + // SAFETY: `OVERLAPPED` is a plain Windows FFI struct + // and an all-zero value is the required initial state + // before offsets are set. overlapped: unsafe { core::mem::zeroed() }, buffer, op: next_op, diff --git a/crates/uffs-mft/src/io/readers/parallel/to_index_parallel.rs b/crates/uffs-mft/src/io/readers/parallel/to_index_parallel.rs index c79950cfe..f6c678817 100644 --- a/crates/uffs-mft/src/io/readers/parallel/to_index_parallel.rs +++ b/crates/uffs-mft/src/io/readers/parallel/to_index_parallel.rs @@ -376,7 +376,8 @@ impl ParallelMftReader { }; let mut in_flight_op = Box::pin(InFlightOp { // SAFETY: `OVERLAPPED` is a plain Windows FFI struct and an - // all-zero value is the required initial state before offsets are set. + // all-zero value is the required initial state before + // offsets are set. overlapped: unsafe { core::mem::zeroed() }, buffer, op, @@ -557,8 +558,9 @@ impl ParallelMftReader { )); }; let mut new_in_flight = Box::pin(InFlightOp { - // SAFETY: `OVERLAPPED` is a plain Windows FFI struct and an - // all-zero value is the required initial state before offsets are set. + // SAFETY: `OVERLAPPED` is a plain Windows FFI struct + // and an all-zero value is the required initial state + // before offsets are set. overlapped: unsafe { core::mem::zeroed() }, buffer, op: next_op, diff --git a/crates/uffs-mft/src/lib.rs b/crates/uffs-mft/src/lib.rs index d2e108184..2482025fe 100644 --- a/crates/uffs-mft/src/lib.rs +++ b/crates/uffs-mft/src/lib.rs @@ -196,8 +196,9 @@ use rustc_hash as _; // commands (src/commands/windows/info.rs); silence the library's view of it. #[cfg(windows)] use serde_json as _; -// `sha2` + `hex` power the Windows-only `capture` command's manifest / SHA256SUMS -// (src/commands/windows/capture.rs) and the chaos tests; silence the library's view. +// `sha2` + `hex` power the Windows-only `capture` command's manifest / +// SHA256SUMS (src/commands/windows/capture.rs) and the chaos tests; silence the +// library's view. #[cfg(any(test, windows))] use sha2 as _; use smallvec as _; @@ -315,7 +316,8 @@ pub use io::{ pub use lcn_resolve::{RecordOutcome, first_data_lcn}; #[cfg(windows)] pub use lcn_resolve::{for_each_record, resolve_frs_to_lcn}; -// Re-export NTFS constants and types (pure Rust data structures, cross-platform) +// Re-export NTFS constants and types (pure Rust data structures, +// cross-platform) pub use ntfs::SECTOR_SIZE; pub use ntfs::{ AttributeIterator, AttributeListEntry, AttributeRecordHeader, AttributeRef, AttributeType, @@ -334,9 +336,9 @@ pub use platform::current_euid; // uffs-daemon can gate mutating daemon commands on all targets. pub use platform::is_elevated; // Re-export platform types -// Core types (DriveType, MftBitmap, MftExtent) are pure data — available on all platforms -// Windows-specific types and functions (VolumeHandle, detect_ntfs_drives, etc.) only on -// Windows +// Core types (DriveType, MftBitmap, MftExtent) are pure data — available on all +// platforms Windows-specific types and functions (VolumeHandle, +// detect_ntfs_drives, etc.) only on Windows pub use platform::{DriveType, MftBitmap, MftExtent, SystemMemory, query_system_memory}; pub use platform::{Lcn, registered_broker_handle_count}; // External-API anchors with cross-crate consumers. Other Windows-only diff --git a/crates/uffs-mft/src/parse/full.rs b/crates/uffs-mft/src/parse/full.rs index 43b6d9bcc..731ef0a03 100644 --- a/crates/uffs-mft/src/parse/full.rs +++ b/crates/uffs-mft/src/parse/full.rs @@ -494,7 +494,8 @@ pub fn parse_record_full(data: &[u8], frs: u64) -> ParseResult { }); } // Skip known non-stream attributes silently. - // Note: SecurityDescriptor (0x50) IS counted as a stream (catch-all below). + // Note: SecurityDescriptor (0x50) IS counted as a stream (catch-all + // below). Some(AttributeType::StandardInformation | AttributeType::FileName) => {} _ => { // All remaining attribute types are counted as streams. diff --git a/crates/uffs-mft/src/platform.rs b/crates/uffs-mft/src/platform.rs index 49224d56e..e64020d9f 100644 --- a/crates/uffs-mft/src/platform.rs +++ b/crates/uffs-mft/src/platform.rs @@ -61,8 +61,8 @@ pub use drive_letter::{DriveLetter, DriveLetterError}; pub use drive_pattern::split_drive_prefix; pub use extents::MftExtent; pub use lcn::Lcn; -// Export DriveType unconditionally (needed for tests), but Windows-specific functions only on -// Windows +// Export DriveType unconditionally (needed for tests), but Windows-specific +// functions only on Windows pub use physical::{PhysicalDrive, physical_drives}; pub use system::DriveType; // Caller's effective uid — Unix-only; lets daemon-management compare the diff --git a/crates/uffs-mft/src/platform/metafile_read.rs b/crates/uffs-mft/src/platform/metafile_read.rs index 9d6bd6e18..fddff7c65 100644 --- a/crates/uffs-mft/src/platform/metafile_read.rs +++ b/crates/uffs-mft/src/platform/metafile_read.rs @@ -40,9 +40,9 @@ pub fn read_metafile(drive: DriveLetter, kind: MetafileKind) -> Result> MetafileKind::AttrDef => read_data_stream(&handle, vol, 4, None), MetafileKind::MftMirr => read_data_stream(&handle, vol, 1, None), MetafileKind::LogFile => read_data_stream(&handle, vol, 2, None), - // `$Volume` / `$BadClus` keep their useful data in the MFT record itself - // (the $VOLUME_* attributes; the $Bad run list), so capture the fixed-up - // record. + // `$Volume` / `$BadClus` keep their useful data in the MFT record + // itself (the $VOLUME_* attributes; the $Bad run list), so capture the + // fixed-up record. MetafileKind::Volume => read_frs_record(&handle, vol, 3), MetafileKind::BadClus => read_frs_record(&handle, vol, 8), // `$UsnJrnl:$J` lives under `$Extend`; resolve its FRS then read `$J`. diff --git a/crates/uffs-mft/src/reader/dataframe_build.rs b/crates/uffs-mft/src/reader/dataframe_build.rs index ebfecd755..a9dc5b172 100644 --- a/crates/uffs-mft/src/reader/dataframe_build.rs +++ b/crates/uffs-mft/src/reader/dataframe_build.rs @@ -254,7 +254,8 @@ impl MftReader { Series::new("is_pinned".into(), is_pinned_vec).into_column(), Series::new("is_unpinned".into(), is_unpinned_vec).into_column(), Series::new("is_virtual".into(), is_virtual_vec).into_column(), - // Raw attribute flags (combined value for baseline-compatible output) + // Raw attribute flags (combined value for baseline-compatible + // output) Series::new("flags".into(), flags_vec).into_column(), ]; @@ -322,7 +323,8 @@ impl MftReader { Series::new("is_pinned".into(), columns.is_pinned).into_column(), Series::new("is_unpinned".into(), columns.is_unpinned).into_column(), Series::new("is_virtual".into(), columns.is_virtual).into_column(), - // Raw attribute flags (combined value for baseline-compatible output) + // Raw attribute flags (combined value for baseline-compatible + // output) Series::new("flags".into(), columns.flags).into_column(), ]; diff --git a/crates/uffs-update/src/restore.rs b/crates/uffs-update/src/restore.rs index 18c2e9802..c46bbc21e 100644 --- a/crates/uffs-update/src/restore.rs +++ b/crates/uffs-update/src/restore.rs @@ -48,11 +48,11 @@ fn start_component(component: &str, running: &SnapRunning) -> bool { // The stdio MCP server (`uffsmcp`) is spawned and OWNED by its LLM host // (Claude Desktop / Cursor / Windsurf) over stdio — relaunching it // detached has no client to serve. Quiesce stopped it only to free the - // `uffsmcp.exe` file lock for the swap; the host respawns it on the next - // tool call. So restoring mcp is a deliberate no-op (and a success, not - // a "failed to restart"). The HTTP gateway is a separate binary - // (`uffs-mcp-http`), never captured here — detection scans only for - // `uffsmcp`. + // `uffsmcp.exe` file lock for the swap; the host respawns it on the + // next tool call. So restoring mcp is a deliberate no-op (and a + // success, not a "failed to restart"). The HTTP gateway is a separate + // binary (`uffs-mcp-http`), never captured here — detection scans only + // for `uffsmcp`. "mcp" => true, _ => false, } diff --git a/rust-toolchain.toml b/rust-toolchain.toml index f25a29ff7..f8f933b80 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -54,7 +54,7 @@ # Run `just toolchain-sync` to re-attempt a channel bump; the CI # pipeline auto-refreshes on `ship --fresh` unless `--skip-toolchain-sync` # is passed. -channel = "nightly-2026-10-14" +channel = "nightly-2026-10-29" # Specify components that should always be available components = [ From a4378fc063ca5bbbbe56653a6641f526bd99db43 Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:24:59 -0700 Subject: [PATCH 21/21] chore(release): v0.6.46 Workspace version 0.6.45 -> 0.6.46 and the changelog entry for this release. --- CHANGELOG.md | 21 ++++++++++++++- Cargo.lock | 52 +++++++++++++++++++------------------- Cargo.toml | 28 ++++++++++---------- crates/uffs-cli/Cargo.toml | 4 +-- 4 files changed, 62 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d1a7bfba0..075d241ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.6.46] - 2026-10-29 + +### Fixed + +- daemon: stop journal saves from taking the machine under heavy churn +- daemon: seed the startup cursor only after the rebuilt cache is written +- broker: keep a client that connected before ConnectNamedPipe +- status: report a re-warming drive truthfully and count its promotions +- cli: wait up to 10 minutes for a search, silently, and name what did not finish +- deps: rustls 0.23.46, rustls-webpki 0.103.16, ctutils 0.4.6 and the zstd 0.14 line for RUSTSEC advisories +- lints: give every #[deprecated] the version that deprecated it + +### Performance + +- daemon: re-warm a parked drive from its cache plus the journal +- daemon: write saves without copying the index; skip near-empty age saves +- cache: load caches with one buffer per stage, sized exactly + ## [0.6.45] - 2026-10-13 ### Fixed @@ -2903,7 +2921,8 @@ thin clients over a unified `uffsd` process. ### Fixed - Various MFT parsing edge cases -[Unreleased]: https://github.com/skyllc-ai/UltraFastFileSearch/compare/v0.6.45...HEAD +[Unreleased]: https://github.com/skyllc-ai/UltraFastFileSearch/compare/v0.6.46...HEAD +[0.6.46]: https://github.com/skyllc-ai/UltraFastFileSearch/compare/v0.6.45...v0.6.46 [0.6.45]: https://github.com/skyllc-ai/UltraFastFileSearch/compare/v0.6.44...v0.6.45 [0.6.44]: https://github.com/skyllc-ai/UltraFastFileSearch/compare/v0.6.43...v0.6.44 [0.6.43]: https://github.com/skyllc-ai/UltraFastFileSearch/compare/v0.6.42...v0.6.43 diff --git a/Cargo.lock b/Cargo.lock index ce9432975..13d36da14 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4350,7 +4350,7 @@ checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "uffs-bench" -version = "0.6.45" +version = "0.6.46" dependencies = [ "chrono", "clap", @@ -4367,7 +4367,7 @@ dependencies = [ [[package]] name = "uffs-broker" -version = "0.6.45" +version = "0.6.46" dependencies = [ "anyhow", "serde", @@ -4386,14 +4386,14 @@ dependencies = [ [[package]] name = "uffs-broker-protocol" -version = "0.6.45" +version = "0.6.46" dependencies = [ "thiserror 2.0.21", ] [[package]] name = "uffs-ci-pipeline" -version = "0.6.45" +version = "0.6.46" dependencies = [ "anyhow", "chrono", @@ -4412,7 +4412,7 @@ dependencies = [ [[package]] name = "uffs-cli" -version = "0.6.45" +version = "0.6.46" dependencies = [ "anyhow", "assert_cmd", @@ -4435,7 +4435,7 @@ dependencies = [ [[package]] name = "uffs-client" -version = "0.6.45" +version = "0.6.46" dependencies = [ "dirs-next", "libc", @@ -4455,7 +4455,7 @@ dependencies = [ [[package]] name = "uffs-core" -version = "0.6.45" +version = "0.6.46" dependencies = [ "anyhow", "bytemuck", @@ -4486,7 +4486,7 @@ dependencies = [ [[package]] name = "uffs-daemon" -version = "0.6.45" +version = "0.6.46" dependencies = [ "anyhow", "clap", @@ -4520,7 +4520,7 @@ dependencies = [ [[package]] name = "uffs-diag" -version = "0.6.45" +version = "0.6.46" dependencies = [ "anyhow", "chrono", @@ -4535,7 +4535,7 @@ dependencies = [ [[package]] name = "uffs-fetch" -version = "0.6.45" +version = "0.6.46" dependencies = [ "anyhow", "hex", @@ -4546,7 +4546,7 @@ dependencies = [ [[package]] name = "uffs-format" -version = "0.6.45" +version = "0.6.46" dependencies = [ "chrono", "itoa", @@ -4557,7 +4557,7 @@ dependencies = [ [[package]] name = "uffs-gen-hooks" -version = "0.6.45" +version = "0.6.46" dependencies = [ "anyhow", "clap", @@ -4569,7 +4569,7 @@ dependencies = [ [[package]] name = "uffs-gen-workflow" -version = "0.6.45" +version = "0.6.46" dependencies = [ "anyhow", "clap", @@ -4582,7 +4582,7 @@ dependencies = [ [[package]] name = "uffs-manifest-audit" -version = "0.6.45" +version = "0.6.46" dependencies = [ "anyhow", "clap", @@ -4594,7 +4594,7 @@ dependencies = [ [[package]] name = "uffs-mcp" -version = "0.6.45" +version = "0.6.46" dependencies = [ "anyhow", "axum", @@ -4618,7 +4618,7 @@ dependencies = [ [[package]] name = "uffs-mft" -version = "0.6.45" +version = "0.6.46" dependencies = [ "anyhow", "bitflags", @@ -4659,14 +4659,14 @@ dependencies = [ [[package]] name = "uffs-polars" -version = "0.6.45" +version = "0.6.46" dependencies = [ "polars", ] [[package]] name = "uffs-security" -version = "0.6.45" +version = "0.6.46" dependencies = [ "aes-gcm", "dirs-next", @@ -4681,22 +4681,22 @@ dependencies = [ [[package]] name = "uffs-statusfmt" -version = "0.6.45" +version = "0.6.46" [[package]] name = "uffs-text" -version = "0.6.45" +version = "0.6.46" dependencies = [ "bytemuck", ] [[package]] name = "uffs-time" -version = "0.6.45" +version = "0.6.46" [[package]] name = "uffs-update" -version = "0.6.45" +version = "0.6.46" dependencies = [ "anyhow", "dirs-next", @@ -4713,11 +4713,11 @@ dependencies = [ [[package]] name = "uffs-version" -version = "0.6.45" +version = "0.6.46" [[package]] name = "uffs-vss-requestor" -version = "0.6.45" +version = "0.6.46" dependencies = [ "anyhow", "cc", @@ -4730,7 +4730,7 @@ dependencies = [ [[package]] name = "uffs-watchdog" -version = "0.6.45" +version = "0.6.46" dependencies = [ "anyhow", "dirs-next", @@ -4739,7 +4739,7 @@ dependencies = [ [[package]] name = "uffs-winsvc" -version = "0.6.45" +version = "0.6.46" dependencies = [ "anyhow", "windows", diff --git a/Cargo.toml b/Cargo.toml index 5016826e8..8a5837ec3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,7 +68,7 @@ members = [ # Workspace Package Metadata (inherited by all crates) # ───────────────────────────────────────────────────────────────────────────── [workspace.package] -version = "0.6.45" +version = "0.6.46" edition = "2024" # No `rust-version` claim: the workspace is structurally nightly-only. # `crates/uffs-polars` enables `polars/nightly` unconditionally, which @@ -134,16 +134,16 @@ publish = false # proposed-plan output for 12 days because `release-plz update` # failed at `cargo package` with this very error. See # `release-automation-baseline.md` §10 for the diagnostic trail. -uffs-polars = { path = "crates/uffs-polars", version = "0.6.45" } -uffs-security = { path = "crates/uffs-security", version = "0.6.45" } -uffs-text = { path = "crates/uffs-text", version = "0.6.45" } -uffs-time = { path = "crates/uffs-time", version = "0.6.45" } -uffs-version = { path = "crates/uffs-version", version = "0.6.45" } -uffs-statusfmt = { path = "crates/uffs-statusfmt", version = "0.6.45" } -uffs-mft = { path = "crates/uffs-mft", version = "0.6.45" } -uffs-format = { path = "crates/uffs-format", version = "0.6.45" } -uffs-core = { path = "crates/uffs-core", version = "0.6.45" } -uffs-client = { path = "crates/uffs-client", version = "0.6.45" } +uffs-polars = { path = "crates/uffs-polars", version = "0.6.46" } +uffs-security = { path = "crates/uffs-security", version = "0.6.46" } +uffs-text = { path = "crates/uffs-text", version = "0.6.46" } +uffs-time = { path = "crates/uffs-time", version = "0.6.46" } +uffs-version = { path = "crates/uffs-version", version = "0.6.46" } +uffs-statusfmt = { path = "crates/uffs-statusfmt", version = "0.6.46" } +uffs-mft = { path = "crates/uffs-mft", version = "0.6.46" } +uffs-format = { path = "crates/uffs-format", version = "0.6.46" } +uffs-core = { path = "crates/uffs-core", version = "0.6.46" } +uffs-client = { path = "crates/uffs-client", version = "0.6.46" } # `uffs-broker-protocol` carries the wire-protocol types shared between # `uffs-broker` (the elevated handle vendor, Windows-only binary) and # `uffs-daemon::broker_client` (the handle consumer). Pure-logic @@ -152,20 +152,20 @@ uffs-client = { path = "crates/uffs-client", version = "0.6.45" } # wire-format byte literals. Also holds the VSS `HRESULT` retry table # (`vss_hresult`) shared by `uffs-vss-requestor` and uffs-products' # Content Coordinator. -uffs-broker-protocol = { path = "crates/uffs-broker-protocol", version = "0.6.45" } +uffs-broker-protocol = { path = "crates/uffs-broker-protocol", version = "0.6.46" } # `uffs-winsvc` — native Windows service control (SCM query/start/stop) + # the non-connecting broker-pipe readiness probe. Layer-0 leaf: its only # dependency is the `windows` crate (windows-target), with non-Windows # stubs so cross-platform consumers (uffs-update, uffs-cli) compile. # Single source of truth for the `sc`/SCM mechanics previously duplicated # across uffs-broker, uffs-update, and uffs-cli. -uffs-winsvc = { path = "crates/uffs-winsvc", version = "0.6.45" } +uffs-winsvc = { path = "crates/uffs-winsvc", version = "0.6.46" } # `uffs-fetch` — hardened release-asset transport (blocking reqwest + # rustls with retry/timeout/byte-cap, plus `SHA256SUMS` verification), # extracted from `uffs-update` as a small public lib so external products # can reuse it. Cross-platform pure-logic leaf; keeps the HTTP/TLS stack # out of the lean `uffs` CLI exactly as before. -uffs-fetch = { path = "crates/uffs-fetch", version = "0.6.45" } +uffs-fetch = { path = "crates/uffs-fetch", version = "0.6.46" } # NOTE: no `uffs-broker` workspace dependency alias on purpose — # `uffs-broker` is a binary-only crate (the only `[lib]` it carries is # this protocol module's now-extracted sibling); no other workspace diff --git a/crates/uffs-cli/Cargo.toml b/crates/uffs-cli/Cargo.toml index def4bcb4a..37f1125fc 100644 --- a/crates/uffs-cli/Cargo.toml +++ b/crates/uffs-cli/Cargo.toml @@ -70,7 +70,7 @@ path = "src/main.rs" # by the release bump) is required for `cargo package` validation — see # root `Cargo.toml`'s [workspace.dependencies] note for the full # rationale (R6 of `release-automation-plan.md`). -uffs-client = { path = "../uffs-client", version = "0.6.45", default-features = false } +uffs-client = { path = "../uffs-client", version = "0.6.46", default-features = false } # Canonical CSV / parity / legacy-footer writer. Direct dep (not a # re-export chain through `uffs-client`) so the CLI and the daemon @@ -79,7 +79,7 @@ uffs-client = { path = "../uffs-client", version = "0.6.45", default-features = # by the release bump) is required for `cargo package` validation — see # root `Cargo.toml`'s [workspace.dependencies] note for the full # rationale (R6 of `release-automation-plan.md`). -uffs-format = { path = "../uffs-format", version = "0.6.45" } +uffs-format = { path = "../uffs-format", version = "0.6.46" } # Typed drive-letter newtype. Direct dep so the CLI command signatures # (`daemon_load`, `daemon_tiering`, etc.) name `DriveLetter` natively