diff --git a/crates/dua-lib/src/control.rs b/crates/dua-lib/src/control.rs new file mode 100644 index 00000000..c7cde503 --- /dev/null +++ b/crates/dua-lib/src/control.rs @@ -0,0 +1,270 @@ +use std::{ + collections::HashSet, + path::{Path, PathBuf}, + sync::{ + Arc, Condvar, Mutex, + atomic::{AtomicBool, Ordering}, + }, +}; + +/// State of a controlled walk, including whether its current work has stopped. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum Status { + /// Work can proceed. + Running, + /// A pause was requested; in-progress work is still finishing. + Pausing, + /// No work is active and new work cannot start until resumed. + Paused, + /// The iterator finished, or its control was cancelled. + Finished, +} + +#[derive(Default)] +struct State { + paused: bool, + cancelled: bool, + active: usize, + excluded: HashSet, +} + +/// Share pause, resume and removed-path exclusion across related filesystem walks. +/// +/// Pausing is asynchronous: keep consuming output while [`Self::status`] is `Pausing`, +/// since an active worker may be blocked publishing a batch. Once `Paused`, no new +/// filesystem job can start. Buffered output is retained and can still be consumed. +#[derive(Clone, Default)] +pub struct Control { + state: Arc<(Mutex, Condvar)>, +} + +#[allow( + clippy::missing_panics_doc, + reason = "only poisoning of private synchronization state can panic" +)] +impl Control { + /// Stop starting new jobs, allowing in-progress jobs to finish. + pub fn pause(&self) { + self.state.0.lock().unwrap().paused = true; + } + + /// Allow paused jobs to continue. + pub fn resume(&self) { + self.state.0.lock().unwrap().paused = false; + self.state.1.notify_all(); + } + + /// Cancel this control and release threads waiting at a pause checkpoint. + pub fn cancel(&self) { + self.state.0.lock().unwrap().cancelled = true; + self.state.1.notify_all(); + } + + /// Report whether work is running, finishing a pause, paused, or cancelled. + #[must_use] + pub fn status(&self) -> Status { + let state = self.state.0.lock().unwrap(); + if state.cancelled { + Status::Finished + } else if !state.paused { + Status::Running + } else if state.active == 0 { + Status::Paused + } else { + Status::Pausing + } + } + + /// Ignore jobs and buffered entries at or below a path removed while paused. + pub fn exclude_path(&self, path: PathBuf) { + self.state.0.lock().unwrap().excluded.insert(path); + } + + /// Whether a path is below one of the excluded paths. + #[must_use] + pub fn is_excluded(&self, path: &Path) -> bool { + let state = self.state.0.lock().unwrap(); + !state.excluded.is_empty() && path.ancestors().any(|path| state.excluded.contains(path)) + } + + /// Start a related unit of work, waiting if paused and returning `None` if cancelled. + /// Hold the returned guard until the work and its output have finished. + #[must_use] + pub fn activity(&self) -> Option> { + self.enter(None) + } + + pub(crate) fn enter(&self, stopped: Option<&AtomicBool>) -> Option> { + let mut state = self.state.0.lock().unwrap(); + loop { + if state.cancelled || stopped.is_some_and(|stop| stop.load(Ordering::Relaxed)) { + return None; + } + if !state.paused { + state.active += 1; + return Some(Activity(self)); + } + state = self.state.1.wait(state).unwrap(); + } + } + + pub(crate) fn wake(&self) { + let _state = self.state.0.lock().unwrap(); + self.state.1.notify_all(); + } +} + +/// Keeps a unit of work visible to [`Control::status`] until it is dropped. +pub struct Activity<'a>(&'a Control); + +impl Drop for Activity<'_> { + fn drop(&mut self) { + self.0.state.0.lock().unwrap().active -= 1; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{Options, Order, RootEvent, stream_roots_with_control, walk_roots_with_control}; + use crossbeam::channel::bounded; + use std::{ + fs, thread, + time::{Duration, Instant}, + }; + + fn wait_for_pause(control: &Control) { + let deadline = Instant::now() + Duration::from_secs(5); + while control.status() != Status::Paused { + assert!(Instant::now() < deadline, "walk did not finish pausing"); + thread::yield_now(); + } + } + + #[test] + fn pause_drains_active_jobs_and_resume_skips_removed_subtrees() { + for threads in [1, 4] { + let dir = tempfile::tempdir().unwrap(); + fs::create_dir_all(dir.path().join("victim/nested")).unwrap(); + fs::write(dir.path().join("victim/nested/file"), b"removed").unwrap(); + fs::create_dir(dir.path().join("keep")).unwrap(); + fs::write(dir.path().join("keep/file"), b"keep").unwrap(); + let (started_tx, started_rx) = bounded(1); + let (release_tx, release_rx) = bounded(1); + let control = Control::default(); + let walk = walk_roots_with_control( + [(0, dir.path().to_owned())], + threads, + Order::ParentFirst, + Options::default().queued_batches_per_worker(0), + control.clone(), + move |_, entry| { + if entry.file_name == "victim" { + started_tx.send(()).unwrap(); + release_rx.recv().unwrap(); + } + true + }, + ); + let reader = thread::spawn(move || { + walk.filter_map(|(_, event)| match event { + RootEvent::Entry(entry) => Some(entry.map(|entry| entry.path())), + RootEvent::Finished => None, + }) + .collect::>() + }); + started_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + control.pause(); + assert_eq!(control.status(), Status::Pausing); + release_tx.send(()).unwrap(); + wait_for_pause(&control); + let victim = dir.path().join("victim"); + fs::remove_dir_all(&victim).unwrap(); + control.exclude_path(victim.clone()); + control.resume(); + let paths = reader + .join() + .unwrap() + .into_iter() + .collect::, _>>() + .unwrap(); + assert!(paths.contains(&dir.path().join("keep/file"))); + assert!(!paths.contains(&victim.join("nested/file"))); + } + } + + #[test] + fn dropping_a_paused_walk_releases_waiting_workers() { + let dir = tempfile::tempdir().unwrap(); + fs::create_dir(dir.path().join("child")).unwrap(); + let control = Control::default(); + let (started_tx, started_rx) = bounded(1); + let (release_tx, release_rx) = bounded(1); + let walk = walk_roots_with_control( + [(0, dir.path().to_owned())], + 2, + Order::ParentFirst, + Options::default(), + control.clone(), + move |_, entry| { + if entry.file_name == "child" { + started_tx.send(()).unwrap(); + release_rx.recv().unwrap(); + } + true + }, + ); + started_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + control.pause(); + release_tx.send(()).unwrap(); + wait_for_pause(&control); + let (done_tx, done_rx) = bounded(1); + let dropper = thread::spawn(move || { + drop(walk); + done_tx.send(()).unwrap(); + }); + done_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + dropper.join().unwrap(); + } + + #[test] + fn cancellation_wakes_a_reader_waiting_on_paused_workers() { + let control = Control::default(); + let (roots, mut walk) = + stream_roots_with_control(2, Order::ParentFirst, Options::default(), control.clone()); + control.pause(); + assert_eq!(walk.status(), Status::Paused); + let (done_tx, done_rx) = bounded(1); + let reader = thread::spawn(move || { + assert!(walk.next().is_none()); + done_tx.send(()).unwrap(); + }); + control.cancel(); + done_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + reader.join().unwrap(); + drop(roots); + } + + #[test] + fn excluded_buffered_roots_are_filtered_without_a_worker_pool() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("file"); + fs::write(&path, b"file").unwrap(); + let control = Control::default(); + let mut walk = walk_roots_with_control( + [(0, path.clone())], + 1, + Order::ParentFirst, + Options::default(), + control.clone(), + |_, _| true, + ); + control.pause(); + control.exclude_path(path); + assert!(walk.control().is_some()); + assert!(matches!(walk.next(), Some((0, RootEvent::Finished)))); + assert!(walk.next().is_none()); + assert_eq!(walk.status(), Status::Finished); + } +} diff --git a/crates/dua-lib/src/lib.rs b/crates/dua-lib/src/lib.rs index 215e279e..f7d95987 100644 --- a/crates/dua-lib/src/lib.rs +++ b/crates/dua-lib/src/lib.rs @@ -10,6 +10,14 @@ //! Symbolic links are reported but never followed, and filesystem errors are //! returned as iterator items. Dropping the iterator stops and joins its workers. //! +//! # Pausing +//! +//! Pass a shared [`Control`] to [`walk_with_control`], [`walk_roots_with_control`], or +//! [`stream_roots_with_control`]. [`Control::pause`] stops new jobs while active jobs finish; +//! keep draining their output until [`Control::status`] reports [`Status::Paused`]. Use +//! [`Control::exclude_path`] for paths removed during the pause, then [`Control::resume`] to +//! continue queued work. [`Walk::status`] and [`RootWalk::status`] also report iterator completion. +//! //! # Scheduling //! //! The root directory starts in a shared injector queue. On platforms where directory-entry @@ -29,6 +37,8 @@ mod error; pub use error::Error; +mod control; +pub use control::{Activity, Control, Status}; use crossbeam::{ deque::{Injector, Steal, Stealer, Worker}, @@ -345,6 +355,7 @@ struct PoolShared { next_wake: AtomicUsize, /// Allocates dense identifiers to directories as they are discovered. next_directory_id: AtomicUsize, + control: Option, } struct Pool { @@ -360,6 +371,7 @@ pub struct RootWalk { next: Vec<(usize, RootEvent)>, /// See [`Walk::pool`]. pool: Option, + control: Option, } /// Submits roots to a running [`RootWalk`] without starting additional workers. @@ -425,7 +437,27 @@ impl Drop for RootSender { /// allowing callers to retain discovery context without shared mutable lookup tables. #[must_use] pub fn stream_roots(threads: usize, order: Order, options: Options) -> (RootSender, RootWalk) { - let pool = start_pool(threads.max(1), order, options, 0); + stream_roots_inner(threads, order, options, None) +} + +/// Like [`stream_roots`], sharing pause and removed-path control with related walks. +#[must_use] +pub fn stream_roots_with_control( + threads: usize, + order: Order, + options: Options, + control: Control, +) -> (RootSender, RootWalk) { + stream_roots_inner(threads, order, options, Some(control)) +} + +fn stream_roots_inner( + threads: usize, + order: Order, + options: Options, + control: Option, +) -> (RootSender, RootWalk) { + let pool = start_pool_with_control(threads.max(1), order, options, 0, control); pool.shared.active_roots.store(1, AtomicOrdering::Relaxed); let sender = RootSender { shared: Arc::clone(&pool.shared), @@ -435,6 +467,7 @@ pub fn stream_roots(threads: usize, order: Order, options: Options) -> (RootSend sender, RootWalk { next: Vec::new(), + control: pool.shared.control.clone(), pool: Some(pool), }, ) @@ -459,6 +492,7 @@ pub struct Walk { options: Options, /// Whether the worker pool has finished the current traversal. finished: bool, + control: Option, } /// Read a directory using native enumeration, collecting metadata unless @@ -484,6 +518,29 @@ pub fn walk( order: Order, options: Options, descend: impl Fn(&Entry) -> bool + Send + Sync + 'static, +) -> Walk { + walk_inner(root, threads, order, options, None, descend) +} + +/// Like [`walk`], sharing pause and removed-path control with related walks. +pub fn walk_with_control( + root: &Path, + threads: usize, + order: Order, + options: Options, + control: Control, + descend: impl Fn(&Entry) -> bool + Send + Sync + 'static, +) -> Walk { + walk_inner(root, threads, order, options, Some(control), descend) +} + +fn walk_inner( + root: &Path, + threads: usize, + order: Order, + options: Options, + control: Option, + descend: impl Fn(&Entry) -> bool + Send + Sync + 'static, ) -> Walk { let root_path = root.to_owned(); let root_state = Arc::new(Root { @@ -491,6 +548,23 @@ pub fn walk( pending: AtomicUsize::new(0), descend: Arc::new(move |_, entry| descend(entry)), }); + let _activity = if let Some(control) = &control { + let activity = control.activity(); + if activity.is_none() || control.is_excluded(&root_path) { + return Walk { + next: Vec::new(), + pool: None, + root: root_path, + root_state, + options, + finished: true, + control: Some(control.clone()), + }; + } + activity + } else { + None + }; let mut root = Entry::from_path(root, options); if let Ok(entry) = &mut root && entry.file_type.is_dir() @@ -500,7 +574,7 @@ pub fn walk( let pool = match &root { Ok(entry) if entry.file_type.is_dir() && (root_state.descend)(0, entry) => { let path = Arc::from(entry.path()); - let pool = start_pool(threads.max(1), order, options, 1); + let pool = start_pool_with_control(threads.max(1), order, options, 1, control.clone()); start_jobs( &pool, vec![Job::ReadDir { @@ -521,10 +595,28 @@ pub fn walk( root_state, options, finished: false, + control: control.clone(), } } impl Walk { + /// Report traversal completion or the shared pause state. + #[must_use] + pub fn status(&self) -> Status { + if self.finished || (self.pool.is_none() && self.next.is_empty()) { + Status::Finished + } else { + self.control + .as_ref() + .map_or(Status::Running, Control::status) + } + } + + /// The shared control, if this iterator was constructed with one. + #[must_use] + pub fn control(&self) -> Option { + self.control.clone() + } /// Restart an exhausted directory walk while retaining its worker threads. /// /// Returns `false` if the walk is still active or did not start a worker pool. @@ -536,6 +628,18 @@ impl Walk { let Some(pool) = self.pool.as_ref() else { return false; }; + let control = pool.shared.control.clone(); + let _activity = if let Some(control) = &control { + if control.is_excluded(&self.root) { + return false; + } + let Some(activity) = control.activity() else { + return false; + }; + Some(activity) + } else { + None + }; let mut root = Entry::from_path(&self.root, self.options); if let Ok(entry) = &mut root && entry.file_type.is_dir() @@ -578,13 +682,21 @@ impl Walk { cancelled: Option<&AtomicBool>, ) -> Option> { loop { - if cancelled.is_some_and(|cancelled| cancelled.load(AtomicOrdering::Relaxed)) { + let control = self.control.as_ref(); + if cancelled.is_some_and(|cancelled| cancelled.load(AtomicOrdering::Relaxed)) + || control.is_some_and(|control| control.status() == Status::Finished) + { self.next.clear(); self.finished = true; self.pool = None; return None; } if let Some(entry) = self.next.pop() { + if control.is_some_and(|control| { + entry_path(&entry).is_some_and(|path| control.is_excluded(&path)) + }) { + continue; + } return Some(entry); } if self.finished { @@ -592,7 +704,7 @@ impl Walk { } let pool = self.pool.as_ref()?; - let event = if cancelled.is_some() { + let event = if cancelled.is_some() || control.is_some() { match pool .events .recv_timeout(std::time::Duration::from_millis(100)) @@ -619,7 +731,7 @@ impl Walk { } Event::Batch { batch: Err(err), .. - } => return Some(Err(err)), + } => self.next.push(Err(err)), Event::RootFinished { .. } => {} Event::Finished => { self.finished = true; @@ -661,6 +773,37 @@ pub fn walk_roots( descend, |path: PathBuf| Entry::from_path(&path, options), options, + None, + ) +} + +/// Like [`walk_roots`], sharing pause and removed-path control with related walks. +pub fn walk_roots_with_control( + roots: impl IntoIterator, + threads: usize, + order: Order, + options: Options, + control: Control, + descend: impl Fn(usize, &Entry) -> bool + Send + Sync + 'static, +) -> RootWalk { + let preparing = control.clone(); + start_root_walk( + roots.into_iter().collect(), + threads, + order, + descend, + |path: PathBuf| { + if preparing.is_excluded(&path) { + Err(Error::at_path( + io::Error::from(io::ErrorKind::Interrupted), + path, + )) + } else { + Entry::from_path(&path, options) + } + }, + options, + Some(control), ) } @@ -689,6 +832,27 @@ pub fn walk_root_entries( descend, std::convert::identity, options, + None, + ) +} + +/// Like [`walk_root_entries`], sharing pause and removed-path control with related walks. +pub fn walk_root_entries_with_control( + roots: impl IntoIterator)>, + threads: usize, + order: Order, + options: Options, + control: Control, + descend: impl Fn(usize, &Entry) -> bool + Send + Sync + 'static, +) -> RootWalk { + start_root_walk( + roots.into_iter().collect(), + threads, + order, + descend, + |entry| entry, + options, + Some(control), ) } @@ -699,7 +863,20 @@ fn start_root_walk( descend: impl Fn(usize, &Entry) -> bool + Send + Sync + 'static, prepare: impl Fn(Root) -> Result, options: Options, + control: Option, ) -> RootWalk { + let _activity = if let Some(control) = &control { + let Some(activity) = control.activity() else { + return RootWalk { + next: Vec::new(), + pool: None, + control: Some(control.clone()), + }; + }; + Some(activity) + } else { + None + }; let indices = roots .iter() .map(|(index, _)| *index) @@ -709,17 +886,32 @@ fn start_root_walk( let (next, root_jobs, next_directory_id) = begin_walks( roots .into_iter() - .map(|(root_idx, root)| (root_idx, prepare(root))), + .map(|(root_idx, root)| (root_idx, prepare(root))) + .filter(|(_, entry)| { + control.as_ref().is_none_or(|control| { + entry_path(entry).is_none_or(|path| !control.is_excluded(&path)) + }) + }), &descend, ); let pool = if root_jobs.is_empty() { None } else { - let pool = start_pool(threads.max(1), order, options, next_directory_id); + let pool = start_pool_with_control( + threads.max(1), + order, + options, + next_directory_id, + control.clone(), + ); start_jobs(&pool, root_jobs); Some(pool) }; - RootWalk { next, pool } + RootWalk { + next, + pool, + control: control.clone(), + } } impl Iterator for RootWalk { @@ -727,10 +919,39 @@ impl Iterator for RootWalk { fn next(&mut self) -> Option { loop { + if self + .control + .as_ref() + .is_some_and(|control| control.status() == Status::Finished) + { + self.next.clear(); + self.pool = None; + return None; + } if let Some(entry) = self.next.pop() { + if let RootEvent::Entry(result) = &entry.1 + && self.control.as_ref().is_some_and(|control| { + entry_path(result).is_some_and(|path| control.is_excluded(&path)) + }) + { + continue; + } return Some(entry); } - match self.pool.as_ref()?.events.recv() { + let pool = self.pool.as_ref()?; + let event = if pool.shared.control.is_some() { + match pool + .events + .recv_timeout(std::time::Duration::from_millis(100)) + { + Ok(event) => Ok(event), + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => continue, + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => Err(()), + } + } else { + pool.events.recv().map_err(|_| ()) + }; + match event { Ok(Event::Batch { root_idx, batch: Ok(entries), @@ -743,7 +964,7 @@ impl Iterator for RootWalk { Ok(Event::Batch { root_idx, batch: Err(err), - }) => return Some((root_idx, RootEvent::Entry(Err(err)))), + }) => self.next.push((root_idx, RootEvent::Entry(Err(err)))), Ok(Event::RootFinished { root_idx }) => { return Some((root_idx, RootEvent::Finished)); } @@ -751,7 +972,7 @@ impl Iterator for RootWalk { self.pool = None; return None; } - Err(_) => { + Err(()) => { return Some(( 0, RootEvent::Entry(Err(io::Error::other("directory worker stopped").into())), @@ -762,6 +983,33 @@ impl Iterator for RootWalk { } } +impl RootWalk { + /// Report traversal completion or the shared pause state. + #[must_use] + pub fn status(&self) -> Status { + if self.pool.is_none() && self.next.is_empty() { + Status::Finished + } else { + self.control + .as_ref() + .map_or(Status::Running, Control::status) + } + } + + /// The shared control, if this iterator was constructed with one. + #[must_use] + pub fn control(&self) -> Option { + self.control.clone() + } +} + +fn entry_path(entry: &Result) -> Option { + match entry { + Ok(entry) => Some(entry.path()), + Err(error) => error.path().map(Path::to_path_buf), + } +} + impl PoolShared { /// Wake one worker that has announced it is idle. fn wake_worker(&self) { @@ -831,7 +1079,18 @@ impl Entry { } } +#[cfg(test)] fn start_pool(threads: usize, order: Order, options: Options, next_directory_id: usize) -> Pool { + start_pool_with_control(threads, order, options, next_directory_id, None) +} + +fn start_pool_with_control( + threads: usize, + order: Order, + options: Options, + next_directory_id: usize, + control: Option, +) -> Pool { let workers: Vec<_> = (0..threads).map(|_| Worker::new_lifo()).collect(); let parkers: Vec<_> = (0..threads).map(|_| Parker::new()).collect(); let (event_tx, event_rx) = sync_channel( @@ -854,6 +1113,7 @@ fn start_pool(threads: usize, order: Order, options: Options, next_directory_id: idle: (0..threads).map(|_| AtomicBool::new(false)).collect(), next_wake: AtomicUsize::new(0), next_directory_id: AtomicUsize::new(next_directory_id), + control, }); let handles: Vec<_> = workers .into_iter() @@ -969,6 +1229,14 @@ fn worker_loop(idx: usize, worker: Worker, parker: Parker, shared: Arc, parker: Parker, shared: Arc, shared: &PoolShared) -> Option<(Job, bool)> { } fn run_job(job: Job, worker: &Worker, shared: &PoolShared) { + if let Some(control) = &shared.control { + let path = match &job { + Job::StartRoot { path, .. } => Some(path.as_path()), + Job::ReadDir { path, .. } => Some(path.as_ref()), + #[cfg(not(windows))] + Job::Stat { path, .. } => Some(path.as_ref()), + Job::CloseInput => None, + }; + if path.is_some_and(|path| control.is_excluded(path)) { + match job { + Job::StartRoot { root, .. } | Job::ReadDir { root, .. } => { + finish_pending(&root, shared); + } + #[cfg(not(windows))] + Job::Stat { root, .. } => finish_pending(&root, shared), + Job::CloseInput => unreachable!(), + } + return; + } + } match job { Job::StartRoot { root, path } => { let mut entry = Entry::from_path(&path, shared.options); @@ -1336,6 +1627,12 @@ fn stat_entries( let mut jobs = Vec::new(); let entries = entries .into_iter() + .filter(|entry| { + shared + .control + .as_ref() + .is_none_or(|control| !control.is_excluded(&entry.path())) + }) .map(|entry| { #[cfg(target_os = "macos")] let entry = Ok(entry.read_metadata(shared.options)); diff --git a/src/clean.rs b/src/clean.rs index 130ea03a..92fd0ae8 100644 --- a/src/clean.rs +++ b/src/clean.rs @@ -39,13 +39,29 @@ pub(crate) enum DiscoveryEvent { Progress { entries: u64, io_errors: u64 }, } +#[cfg(test)] pub(crate) fn discover( input: Vec, walk_options: crate::WalkOptions, depth: Option, pattern_root: Option, + send: impl FnMut(DiscoveryEvent) -> bool, +) { + discover_with_control(input, walk_options, depth, pattern_root, None, send); +} + +pub(crate) fn discover_with_control( + input: Vec, + walk_options: crate::WalkOptions, + depth: Option, + pattern_root: Option, + control: Option<&crate::walk::Control>, mut send: impl FnMut(DiscoveryEvent) -> bool, ) { + let activity = control.and_then(crate::walk::Control::activity); + if control.is_some() && activity.is_none() { + return; + } let mut progress = Progress::default(); if !progress.send(&mut send, true) { return; @@ -71,6 +87,9 @@ pub(crate) fn discover( }); let mut roots = Vec::new(); for path in input { + if control.is_some_and(|control| control.is_excluded(&path)) { + continue; + } progress.entries += 1; match fs::symlink_metadata(&path) { Ok(metadata) if metadata.is_dir() => match path.canonicalize() { @@ -94,8 +113,16 @@ pub(crate) fn discover( let mut searched = Vec::::new(); let mut candidates = BTreeSet::::new(); let mut git = git::Filter::default(); + drop(activity); // ponytail: one discovery reader; use directory workers if enumeration becomes the bottleneck. for root in roots { + let activity = control.and_then(crate::walk::Control::activity); + if control.is_some() && activity.is_none() { + return; + } + if control.is_some_and(|control| control.is_excluded(&root)) { + continue; + } if root .components() .any(|component| is_git_dir_name(component.as_os_str())) @@ -120,7 +147,15 @@ pub(crate) fn discover( }; // Finish an ancestor input before nested inputs, preserving each input's depth budget. let mut pending = VecDeque::from([(root.clone(), 0, None)]); + drop(activity); while let Some((path, level, parent_has_cargo)) = pending.pop_front() { + let activity = control.and_then(crate::walk::Control::activity); + if control.is_some() && activity.is_none() { + return; + } + if control.is_some_and(|control| control.is_excluded(&path)) { + continue; + } if !progress.send(&mut send, false) { return; } @@ -162,6 +197,7 @@ pub(crate) fn discover( match git.is_candidate(&path) { Ok(true) => { candidates.insert(path.clone()); + drop(activity); if !send(DiscoveryEvent::Candidate { path, search_root: pattern_root.clone(), diff --git a/src/common.rs b/src/common.rs index 66b345ad..740f48d3 100644 --- a/src/common.rs +++ b/src/common.rs @@ -303,6 +303,16 @@ impl WalkOptions { roots: Vec, skip_root: bool, order: walk::Order, + ) -> impl Iterator + use<> { + self.iter_from_paths_with_control(roots, skip_root, order, None) + } + + pub(crate) fn iter_from_paths_with_control( + &self, + roots: Vec, + skip_root: bool, + order: walk::Order, + control: Option, ) -> impl Iterator + use<> { let num_roots = roots .iter() @@ -323,7 +333,15 @@ impl WalkOptions { indexed_roots.push(( root.index, root.entry.map_or_else( - || walk::Entry::from_path(&root.path, self.metadata_options), + || { + let activity = control.as_ref().and_then(walk::Control::activity); + if control.is_some() && activity.is_none() { + return Err( + std::io::Error::from(std::io::ErrorKind::Interrupted).into() + ); + } + walk::Entry::from_path(&root.path, self.metadata_options) + }, Ok, ), )); @@ -376,13 +394,28 @@ impl WalkOptions { && !is_excluded_while_walking(root_idx, entry) }; - let walk = walk_roots( - indexed_roots, - self.threads, - order, - self.metadata_options, - descend, - ); + let walk = if let Some(control) = control { + #[cfg(any(windows, target_os = "macos"))] + use walk::walk_root_entries_with_control as controlled; + #[cfg(not(any(windows, target_os = "macos")))] + use walk::walk_roots_with_control as controlled; + controlled( + indexed_roots, + self.threads, + order, + self.metadata_options, + control, + descend, + ) + } else { + walk_roots( + indexed_roots, + self.threads, + order, + self.metadata_options, + descend, + ) + }; walk.filter(move |(root_idx, event)| match event { walk::RootEvent::Entry(Ok(entry)) => { diff --git a/src/interactive/app/deletion_progress.rs b/src/interactive/app/deletion_progress.rs index 4c0be1ee..e91ba824 100644 --- a/src/interactive/app/deletion_progress.rs +++ b/src/interactive/app/deletion_progress.rs @@ -42,6 +42,7 @@ pub(super) struct FilesystemDeletion { format: ByteFormat, bytes_removed: u128, entries_removed: usize, + totals_complete: bool, } impl FilesystemDeletion { @@ -50,8 +51,12 @@ impl FilesystemDeletion { language.ui_text().cancelling_deletion.into() } else { language.deletion_progress( - self.entries_removed, - &self.format.display(self.bytes_removed).to_string(), + self.totals_complete.then_some(self.entries_removed), + &if self.totals_complete { + self.format.display(self.bytes_removed).to_string() + } else { + "?".into() + }, self.is_trash(), ) } @@ -88,17 +93,16 @@ impl FilesystemDeletion { impl AppState { pub fn is_deleting(&self) -> bool { - self.deletion.is_some() || self.queued_deletion.is_some() + self.deletion.is_some() || self.pending_deletion.is_some() } pub(super) fn block_deletion_changes(&mut self) -> bool { if self.is_deleting() { - let text = self.language.ui_text(); self.message = Some( - if self.queued_deletion.is_some() { - text.deletion_queued + if self.pending_deletion.is_some() { + self.language.ui_text().pausing_scan } else { - text.deletion_running + self.language.ui_text().deletion_running } .into(), ); @@ -121,14 +125,25 @@ impl AppState { if self.read_only { anyhow::bail!(self.language.ui_text().snapshots_read_only); } - let Some(pane) = &window.mark else { - return Ok(()); - }; - if pane.is_empty() { + if self + .scan + .as_ref() + .is_some_and(|scan| scan.snapshot_export.is_some()) + { + anyhow::bail!(self.language.ui_text().traversal_running); + } + if window.mark.as_ref().is_none_or(|pane| pane.is_empty()) { + if let Some(scan) = &self.scan { + scan.active_traversal.resume(); + } return Ok(()); } - if self.scan.is_some() { - self.queued_deletion = Some(mode); + let pane = window.mark.as_ref().expect("nonempty mark pane"); + if let Some(scan) = &self.scan + && !scan.active_traversal.pause_for_deletion() + { + scan.active_traversal.pause(); + self.pending_deletion = Some(mode); self.pending_exit = false; self.reset_message(); return Ok(()); @@ -160,13 +175,24 @@ impl AppState { }); } if roots.is_empty() { + if let Some(scan) = &self.scan { + scan.active_traversal.resume(); + } return Ok(()); } - let task = DeletionTask::start( + let task = match DeletionTask::start( roots.iter().map(|target| target.path.clone()).collect(), self.walk_options.threads, mode, - )?; + ) { + Ok(task) => task, + Err(error) => { + if let Some(scan) = &self.scan { + scan.active_traversal.resume(); + } + return Err(error.into()); + } + }; self.deletion = Some(FilesystemDeletion { task, tick: crossbeam::channel::tick(Duration::from_secs(1)), @@ -177,6 +203,7 @@ impl AppState { format: display.byte_format, bytes_removed: 0, entries_removed: 0, + totals_complete: self.scan.is_none() || self.clean_hub.is_some(), }); self.pending_exit = false; self.reset_message(); @@ -185,7 +212,11 @@ impl AppState { /// Turn an actual exit request into cancellation followed by draining the worker. pub(super) fn exit_after_deletion(&mut self) -> Option { - self.queued_deletion = None; + if self.pending_deletion.take().is_some() + && let Some(scan) = &self.scan + { + scan.active_traversal.resume(); + } if let Some(deletion) = &self.deletion { deletion.task.cancel.store(true, Ordering::Relaxed); self.pending_exit = false; @@ -222,6 +253,9 @@ impl AppState { if let Some(index) = deletion.lookup(target, &path, &tree) { removed.insert(index); } + if let Some(scan) = &self.scan { + scan.active_traversal.forget_removed_path(path); + } } DeletionEvent::TargetFinished { target, errors } => { target_errors.push((target, errors)); @@ -258,6 +292,7 @@ impl AppState { let mut deltas = HashMap::::new(); let mut parents = HashMap::>::new(); + let mut removed_nodes = HashSet::new(); for &index in &roots { let Some(data) = tree.tree().data(index) else { continue; @@ -283,9 +318,20 @@ impl AppState { let mut pending = vec![index]; while let Some(index) = pending.pop() { deletion.children.remove(&index); + if self.scan.is_some() { + removed_nodes.insert(index); + } pending.extend(tree.tree().children(index)); } } + if self + .scan + .as_mut() + .is_some_and(|scan| !scan.active_traversal.forget_removed_nodes(&removed_nodes)) + { + self.scan = None; + tree.traversal.cost = Some(tree.traversal.start_time.elapsed()); + } for target in &mut deletion.targets { if target.index.is_some_and(|index| roots.contains(&index)) { target.index = None; @@ -347,11 +393,11 @@ impl AppState { self.language, action, if deletion.is_trash() { - deletion.entries_removed + deletion.totals_complete.then_some(deletion.entries_removed) } else { - entries + Some(entries) }, - (errors == 0).then_some(deletion.bytes_removed), + (errors == 0 && deletion.totals_complete).then_some(deletion.bytes_removed), deletion.task.started.elapsed(), errors, deletion.format, @@ -364,6 +410,9 @@ impl AppState { log::debug!("Could not emit terminal notification: {err}"); } } + if let Some(scan) = &self.scan { + scan.active_traversal.resume(); + } if should_exit { self.reset_message(); return Ok(Some(WalkResult { @@ -442,6 +491,7 @@ impl FilesystemDeletion { format: ByteFormat::Metric, bytes_removed: 0, entries_removed: 0, + totals_complete: true, } } } diff --git a/src/interactive/app/eventloop.rs b/src/interactive/app/eventloop.rs index 36b28ae8..2e3f00e0 100644 --- a/src/interactive/app/eventloop.rs +++ b/src/interactive/app/eventloop.rs @@ -447,6 +447,18 @@ impl AppState { where B: Backend, { + if self.pending_deletion.is_some() + && self + .scan + .as_ref() + .is_none_or(|scan| scan.active_traversal.pause_for_deletion()) + { + let mode = self.pending_deletion.take().expect("pending deletion"); + let tree = self.tree_view(traversal); + self.start_deletion(window, &tree, *display, mode)?; + self.refresh_screen(window, traversal, display, terminal, config)?; + return Ok(None); + } if let Some(deletion) = &self.deletion { let input = if deletion.input_closed { crossbeam::channel::never() @@ -499,7 +511,7 @@ impl AppState { crossbeam::select! { recv(events) -> event => { let Ok(event) = event else { - self.queued_deletion = None; + self.exit_after_deletion(); return Ok(Some(WalkResult { num_errors: self.stats.io_errors })); }; let res = self.process_terminal_event( @@ -562,10 +574,6 @@ impl AppState { self.navigation_mut().view_root = view; } } - if is_finished && let Some(mode) = self.queued_deletion.take() { - let tree = self.tree_view(traversal); - self.start_deletion(window, &tree, *display, mode)?; - } self.update_state_during_traversal(traversal, previous_selection.as_ref(), is_finished); self.refresh_screen(window, traversal, display, terminal, config)?; if is_finished { @@ -587,7 +595,7 @@ impl AppState { } } }, - default(std::time::Duration::from_secs(1)) => { + default(if self.pending_deletion.is_some() { std::time::Duration::from_millis(20) } else { std::time::Duration::from_secs(1) }) => { // Keep progress visible while discovery or filesystem I/O is waiting. self.stats = active_traversal.stats; let previous_selection = previous_selection.clone(); diff --git a/src/interactive/app/handlers.rs b/src/interactive/app/handlers.rs index 7730e28a..5f73399d 100644 --- a/src/interactive/app/handlers.rs +++ b/src/interactive/app/handlers.rs @@ -221,8 +221,8 @@ impl AppState { pub fn reset_message(&mut self) { if let Some(deletion) = &self.deletion { self.message = Some(deletion.message(self.language)); - } else if self.queued_deletion.is_some() { - self.message = Some(self.language.ui_text().deletion_queued.into()); + } else if self.pending_deletion.is_some() { + self.message = Some(self.language.ui_text().pausing_scan.into()); } else if self.scan.is_some() { self.message = Some(self.language.ui_text().scanning.into()); } else if let Some(hub) = self.clean_hub.as_ref().filter(|hub| hub.root.is_none()) { diff --git a/src/interactive/app/notification.rs b/src/interactive/app/notification.rs index ffdef0b9..443c2dfb 100644 --- a/src/interactive/app/notification.rs +++ b/src/interactive/app/notification.rs @@ -14,7 +14,7 @@ pub fn scan_finished( ) -> String { language.notification_summary( language.ui_text().notification_scan, - entries, + Some(entries), &format.display(bytes).to_string(), &duration(elapsed), errors, @@ -24,7 +24,7 @@ pub fn scan_finished( pub fn deletion_finished( language: Language, action: &str, - entries: usize, + entries: Option, bytes: Option, elapsed: Duration, errors: usize, @@ -32,7 +32,7 @@ pub fn deletion_finished( ) -> String { language.notification_summary( action, - u64::try_from(entries).unwrap_or(u64::MAX), + entries.map(|entries| u64::try_from(entries).unwrap_or(u64::MAX)), &bytes.map_or_else(|| "?".into(), |bytes| format.display(bytes).to_string()), &duration(elapsed), u64::try_from(errors).unwrap_or(u64::MAX), @@ -84,7 +84,7 @@ mod tests { deletion_finished( Language::English, "Deletion", - 2, + Some(2), None, Duration::from_secs(1), 1, @@ -92,6 +92,18 @@ mod tests { ), "Deletion finished: 2 entries, ? in 1.0s, 1 errors" ); + assert_eq!( + deletion_finished( + Language::English, + "Trash", + None, + None, + Duration::from_secs(1), + 0, + ByteFormat::Metric + ), + "Trash finished: ? entries, ? in 1.0s" + ); } #[test] diff --git a/src/interactive/app/state.rs b/src/interactive/app/state.rs index 26ec84ad..4ee5566a 100644 --- a/src/interactive/app/state.rs +++ b/src/interactive/app/state.rs @@ -68,8 +68,8 @@ pub struct AppState { pub scan: Option, /// The frozen deletion batch, kept alive until all workers have stopped. pub(super) deletion: Option, - /// Confirmed action on the frozen marks, waiting for the current scan to finish. - pub queued_deletion: Option, + /// Confirmed action waiting for active scan work and publication to stop. + pub pending_deletion: Option, /// Latest traversal progress and error counters. pub stats: TraversalStats, /// Options used when starting filesystem walks. @@ -115,7 +115,7 @@ impl AppState { received_events: false, scan: None, deletion: None, - queued_deletion: None, + pending_deletion: None, stats: TraversalStats::default(), walk_options, root_paths: input, diff --git a/src/interactive/app/terminal.rs b/src/interactive/app/terminal.rs index c9e5a4f9..158f8e62 100644 --- a/src/interactive/app/terminal.rs +++ b/src/interactive/app/terminal.rs @@ -213,7 +213,8 @@ impl TerminalApp { ); if result.is_err() { self.state.deletion = None; - self.state.queued_deletion = None; + self.state.pending_deletion = None; + self.state.scan = None; } result } @@ -236,7 +237,8 @@ impl TerminalApp { ); if result.is_err() { self.state.deletion = None; - self.state.queued_deletion = None; + self.state.pending_deletion = None; + self.state.scan = None; } result } diff --git a/src/interactive/app/tests/journeys_deletion.rs b/src/interactive/app/tests/journeys_deletion.rs index eedfdbf3..0fef179c 100644 --- a/src/interactive/app/tests/journeys_deletion.rs +++ b/src/interactive/app/tests/journeys_deletion.rs @@ -1,4 +1,9 @@ -use std::{fs, sync::atomic::Ordering, time::Instant}; +use std::{ + fs, + path::Path, + sync::atomic::Ordering, + time::{Duration, Instant}, +}; use anyhow::Result; use crossbeam::channel::{Receiver, Sender, never, unbounded}; @@ -9,11 +14,13 @@ use tui::{Terminal, backend::TestBackend}; use super::utils::{ index_by_name, initialized_app_and_terminal_from_paths, into_codes, into_events, + untraversed_app_and_terminal_with_closure, }; use crate::interactive::app::{ deletion::DeletionEvent, deletion_progress::FilesystemDeletion, state::FocussedPane, terminal::TerminalApp, }; +use crate::interactive::widgets::{MarkMode, MarkPane}; fn prepared() -> Result<(TempDir, Terminal, TerminalApp)> { let fixture = TempDir::new()?; @@ -70,54 +77,306 @@ fn key(app: &mut TerminalApp, terminal: &mut Terminal, key: KeyEven Ok(()) } +fn drive_until( + app: &mut TerminalApp, + terminal: &mut Terminal, + done: impl Fn(&TerminalApp) -> bool, +) -> Result<()> { + let deadline = Instant::now() + Duration::from_secs(10); + while !done(app) { + assert!( + Instant::now() < deadline, + "scan/deletion stopped making progress" + ); + assert!(step(app, terminal, &never())?.is_none()); + } + Ok(()) +} + +fn request_deletion(app: &mut TerminalApp, target: TreeIndex, mode: MarkMode) { + let mut tree = app.state.tree_view(&mut app.traversal); + app.window.mark = MarkPane::default().toggle_index(target, &tree, true, false); + let key = match mode { + MarkMode::Delete => 'r', + #[cfg(feature = "trash-move")] + MarkMode::Trash => 't', + }; + app.state.dispatch_to_mark_pane( + KeyEvent::new(KeyCode::Char(key), KeyModifiers::CONTROL), + &mut app.window, + &mut tree, + app.display, + &app.config, + ); + assert!(app.state.is_deleting()); +} + +#[test] +fn deletion_pauses_a_scan_and_resumes_its_remaining_entries() -> Result<()> { + for threads in [1, 4] { + let fixture = TempDir::new()?; + let root = fixture.path().canonicalize()?; + let victim = root.join("delete"); + let keep = root.join("keep"); + for index in 0..128 { + let path = keep.join(format!("branch-{index}/nested")); + fs::create_dir_all(&path)?; + fs::write(path.join("payload"), b"keep")?; + } + for index in 0..8 { + let path = victim.join(format!("branch-{index}")); + fs::create_dir_all(&path)?; + fs::write(path.join("payload"), b"remove")?; + } + let kept_entries = dua_core::walk( + &keep, + 1, + dua_core::Order::ParentFirst, + dua_core::Options::default(), + |_| true, + ) + .collect::, _>>()?; + let expected_size: u128 = kept_entries + .iter() + .map(|entry| u128::from(entry.metadata.as_ref().unwrap().as_ref().unwrap().len())) + .sum(); + let (mut terminal, mut app) = untraversed_app_and_terminal_with_closure( + &[victim.clone(), keep.clone()], + Path::to_path_buf, + )?; + app.state.walk_options.threads = threads; + app.traverse()?; + drive_until(&mut app, &mut terminal, |app| { + app.traversal + .tree + .children(app.traversal.root_index) + .any(|index| app.traversal.tree.name(index).as_deref() == Some(victim.as_path())) + })?; + let target = index_by_name(&app, &victim); + request_deletion(&mut app, target, MarkMode::Delete); + assert!(app.state.pending_deletion.is_some()); + assert!(victim.exists(), "deletion waits for the pause"); + drive_until(&mut app, &mut terminal, |app| app.state.deletion.is_some())?; + assert!(app.state.scan.is_some()); + assert_eq!(app.state.message.as_deref(), Some("Deleted ? items (?)...")); + drive_until(&mut app, &mut terminal, |app| !app.state.is_deleting())?; + assert!(!victim.exists()); + assert!( + app.state.scan.is_some(), + "remaining work resumes after deletion" + ); + drive_until(&mut app, &mut terminal, |app| app.state.scan.is_none())?; + assert_eq!(app.state.stats.io_errors, 0); + assert_eq!(app.traversal.tree.len(), kept_entries.len() + 1); + assert_eq!(app.state.stats.total_bytes, Some(expected_size)); + assert!(app.window.mark.is_none()); + assert!( + app.traversal + .tree + .indices() + .all(|index| { !app.traversal.tree.path_of(index).starts_with(&victim) }) + ); + } + Ok(()) +} + +#[test] +fn clean_deletion_resumes_discovery_and_sizing() -> Result<()> { + for threads in [1, 4] { + let fixture = TempDir::new()?; + let root = fixture.path().canonicalize()?; + for index in 0..80 { + let path = root.join(format!("project-{index}/node_modules/nested")); + fs::create_dir_all(&path)?; + fs::write(path.join("payload"), b"cache")?; + } + let (mut terminal, mut app) = untraversed_app_and_terminal_with_closure( + std::slice::from_ref(&root), + Path::to_path_buf, + )?; + app.state.walk_options.threads = threads; + app.traverse_clean(None)?; + drive_until(&mut app, &mut terminal, |app| { + app.traversal + .tree + .children(app.traversal.root_index) + .next() + .is_some() + })?; + let target = app + .traversal + .tree + .children(app.traversal.root_index) + .next() + .unwrap(); + let victim = app.traversal.tree.path_of(target); + request_deletion(&mut app, target, MarkMode::Delete); + drive_until(&mut app, &mut terminal, |app| app.state.deletion.is_some())?; + assert!(app.state.scan.is_some()); + assert_eq!( + app.state.message.as_deref(), + Some("Deleted 0 items (0 B)...") + ); + drive_until(&mut app, &mut terminal, |app| !app.state.is_deleting())?; + assert!(!victim.exists()); + assert!( + app.state.scan.is_some(), + "cleanup continues beyond the deleted candidate" + ); + drive_until(&mut app, &mut terminal, |app| app.state.scan.is_none())?; + assert_eq!(app.state.stats.io_errors, 0); + let candidates: Vec<_> = app + .traversal + .tree + .children(app.traversal.root_index) + .collect(); + assert_eq!(candidates.len(), 79); + for index in candidates { + let path = app.traversal.tree.path_of(index); + assert_ne!(path, victim); + assert_eq!(fs::read(path.join("nested/payload"))?, b"cache"); + } + assert!(app.window.mark.is_none()); + assert_eq!(app.state.clean_hub.as_ref().unwrap().rows.len(), 79); + } + Ok(()) +} + #[test] -fn quitting_cancels_queued_delete_and_trash_without_starting_a_worker() -> Result<()> { - use crate::interactive::{app::state::FilesystemScan, widgets::MarkMode}; +fn cancelling_before_the_scan_pauses_preserves_marks_and_files() -> Result<()> { + use crate::interactive::app::state::FilesystemScan; use dua::traverse::BackgroundTraversal; + let (fixture, mut terminal, mut app) = prepared()?; + let target = index_by_name(&app, "delete"); + // Keep the Finished event buffered so a pause handoff is needed even after workers finish. + app.state.scan = Some(FilesystemScan { + active_traversal: BackgroundTraversal::start( + app.traversal.root_index, + &app.state.walk_options, + Vec::new(), + None, + false, + false, + )?, + previous_selection: None, + previous_cleanup_view: None, + snapshot_export: None, + }); for mode in [ MarkMode::Delete, #[cfg(feature = "trash-move")] MarkMode::Trash, ] { - let (fixture, mut terminal, mut app) = prepared()?; - app.state.scan = Some(FilesystemScan { - active_traversal: BackgroundTraversal::start( - app.traversal.root_index, - &app.state.walk_options, - Vec::new(), - None, - false, - false, - )?, - previous_selection: None, - previous_cleanup_view: None, - snapshot_export: None, - }); - let scan_events = std::mem::replace( - &mut app.state.scan.as_mut().unwrap().active_traversal.event_rx, - never(), + request_deletion(&mut app, target, mode); + assert_eq!( + app.state.message.as_deref(), + Some("Pausing the scan before deletion...") ); - let tree = app.state.tree_view(&mut app.traversal); - app.state - .start_deletion(&app.window, &tree, app.display, mode)?; - assert!(app.state.queued_deletion.is_some()); + assert!(app.state.pending_deletion.is_some()); assert!(app.state.deletion.is_none()); - - let result = step( - &mut app, - &mut terminal, - &into_events([Event::Key(KeyEvent::new( - KeyCode::Char('c'), - KeyModifiers::CONTROL, - ))]), - )?; - assert_eq!(result.unwrap().num_errors, 0); + assert!(app.state.exit_after_deletion().is_some()); assert!(!app.state.is_deleting()); - app.state.scan.as_mut().unwrap().active_traversal.event_rx = scan_events; - app.process_events_once(&mut terminal, into_events([]))?; assert!(fixture.path().join("delete/remove.bin").exists()); + assert!( + app.window + .mark + .as_ref() + .unwrap() + .marked() + .contains_key(&target) + ); + } + drive_until(&mut app, &mut terminal, |app| app.state.scan.is_none())?; + Ok(()) +} + +#[test] +fn partial_deletion_excludes_a_child_not_yet_in_the_tree() -> Result<()> { + use crate::interactive::app::state::FilesystemScan; + use dua::traverse::{BackgroundTraversal, TraversalEvent}; + use std::sync::Arc; + + let (fixture, mut terminal, mut app) = prepared()?; + let root = fixture.path().canonicalize()?; + let path = root.join("delete/late.bin"); + fs::write(&path, b"not integrated yet")?; + let late = BackgroundTraversal::start( + app.traversal.root_index, + &app.state.walk_options, + vec![path.clone()], + None, + false, + true, + )?; + let queued = late.event_rx.recv_timeout(Duration::from_secs(5))?; + assert!(matches!(queued, TraversalEvent::Entry(Ok(_), _, _, _))); + drop(late); + let scan = BackgroundTraversal::start( + app.traversal.root_index, + &app.state.walk_options, + vec![root.clone()], + None, + false, + true, + )?; + scan.pause(); + let deadline = Instant::now() + Duration::from_secs(5); + while !scan.pause_for_deletion() { + assert!(Instant::now() < deadline, "scan did not pause"); + // Leave this child's discovered metadata outside the UI tree. + let _ = scan.event_rx.recv_timeout(Duration::from_millis(10)); } + app.state.scan = Some(FilesystemScan { + active_traversal: scan, + previous_selection: None, + previous_cleanup_view: None, + snapshot_export: None, + }); + let target = index_by_name(&app, "delete"); + let node_count = app.traversal.tree.len(); + let (send, _tick) = inject(&mut app, target); + fs::remove_file(&path)?; + send.send(DeletionEvent::Removed { + target: 0, + path: path.clone(), + })?; + send.send(DeletionEvent::TargetFinished { + target: 0, + errors: 1, + })?; + send.send(DeletionEvent::Finished { + entries: 1, + errors: 1, + cancelled: false, + })?; + step(&mut app, &mut terminal, &never())?; + assert!(!app.state.is_deleting()); + assert!(app.traversal.tree.contains(target)); + assert_eq!( + app.window.mark.as_ref().unwrap().marked()[&target].num_errors_during_deletion, + 1 + ); + + let scan = &mut app.state.scan.as_mut().unwrap().active_traversal; + let error = dua_core::Entry::from_path(&path, dua_core::Options::default()) + .err() + .expect("file was removed"); + for event in [ + queued, + TraversalEvent::Entry(Err(error), Arc::new(root), 0, 0), + ] { + assert_eq!( + scan.integrate_traversal_event(&mut app.traversal, event), + None + ); + } + assert_eq!( + scan.stats.io_errors, 0, + "late errors under the removed child are ignored" + ); + assert_eq!(scan.stats.entries_traversed, 0); + assert_eq!(app.traversal.tree.len(), node_count); Ok(()) } diff --git a/src/interactive/app/tests/journeys_with_writes.rs b/src/interactive/app/tests/journeys_with_writes.rs index 9af8916f..44a65707 100644 --- a/src/interactive/app/tests/journeys_with_writes.rs +++ b/src/interactive/app/tests/journeys_with_writes.rs @@ -148,94 +148,6 @@ fn refresh_discards_marks_before_reusing_tree_indices() -> Result<()> { Ok(()) } -#[test] -fn deletion_is_queued_and_runs_automatically_after_the_scan() -> Result<()> { - use crate::interactive::app::{state::FilesystemScan, tree_view::TreeView}; - use dua::traverse::BackgroundTraversal; - - let dir = TempDir::new()?; - let root = dir.path().join("to-delete"); - fs::create_dir(&root)?; - fs::write(root.join("file"), b"keep until the scan finishes")?; - let (mut terminal, mut app) = - initialized_app_and_terminal_from_paths(std::slice::from_ref(&root))?; - app.process_events( - &mut terminal, - into_events([ - Event::Key(KeyCode::Char('d').into()), - Event::Key(KeyCode::Tab.into()), - ]), - )?; - let marked = marked_file_names(&app, "directory is marked"); - let node_count = app.traversal.tree.len(); - - // Keep a scan active until its Finished event is processed. - app.state.scan = Some(FilesystemScan { - active_traversal: BackgroundTraversal::start( - app.traversal.root_index, - &app.state.walk_options, - Vec::new(), - None, - false, - false, - )?, - previous_selection: None, - previous_cleanup_view: None, - snapshot_export: None, - }); - let delete_key = KeyEvent::new(KeyCode::Char('r'), KeyModifiers::CONTROL); - for key in [ - delete_key, - KeyCode::Char('x').into(), - #[cfg(feature = "trash-move")] - KeyEvent::new(KeyCode::Char('t'), KeyModifiers::CONTROL), - ] { - app.state.dispatch_to_mark_pane( - key, - &mut app.window, - &mut TreeView { - traversal: &mut app.traversal, - scope: None, - glob_tree_root: None, - glob_matches: None, - }, - app.display, - &app.config, - ); - assert_eq!( - app.state.message.as_deref(), - Some("Deletion queued; waiting for the scan to finish") - ); - assert!( - app.state.deletion.is_none(), - "no worker starts during the scan" - ); - assert!(matches!( - app.state.queued_deletion, - Some(crate::interactive::widgets::MarkMode::Delete) - )); - assert!(root.join("file").exists()); - assert_eq!(app.traversal.tree.len(), node_count); - assert_eq!( - marked_file_names(&app, "queued marks cannot be changed"), - marked - ); - assert!(app.state.scan.is_some()); - } - - let result = app.process_events_once(&mut terminal, into_events([]))?; - assert_eq!(result.num_errors, 0); - assert!( - !root.exists(), - "the queued deletion runs without another confirmation" - ); - assert!(app.state.scan.is_none()); - assert!(!app.state.is_deleting()); - assert!(app.window.mark.is_none()); - assert_eq!(app.traversal.tree.len(), 1); - Ok(()) -} - #[test] #[cfg(not(target_os = "windows"))] // it stopped working here, don't know if it's truly broken or if it's the test. Let's wait for windows users to report. fn basic_user_journey_with_deletion() -> Result<()> { diff --git a/src/interactive/widgets/i18n.rs b/src/interactive/widgets/i18n.rs index 3a273ee0..b6ef1b6f 100644 --- a/src/interactive/widgets/i18n.rs +++ b/src/interactive/widgets/i18n.rs @@ -543,7 +543,7 @@ pub struct UiText { pub snapshots_read_only: &'static str, pub traversal_running: &'static str, pub deletion_running: &'static str, - pub deletion_queued: &'static str, + pub pausing_scan: &'static str, pub cancelling_deletion: &'static str, pub no_cleanup_candidates: &'static str, pub cleanup_candidates_already_marked: &'static str, @@ -667,7 +667,9 @@ impl Language { } } - pub fn deletion_progress(self, count: usize, removed: &str, trash: bool) -> String { + pub fn deletion_progress(self, count: Option, removed: &str, trash: bool) -> String { + let singular = count == Some(1); + let count = count.map_or_else(|| "?".into(), |count| count.to_string()); match (self, trash) { (Language::English, false) => { format!("Deleted {count} items ({removed})...") @@ -689,10 +691,10 @@ impl Language { } (Language::German, false) => format!( "{count} {} gelöscht ({removed})...", - if count == 1 { "Eintrag" } else { "Einträge" } + if singular { "Eintrag" } else { "Einträge" } ), (Language::German, true) => { - let label = if count == 1 { "Eintrag" } else { "Einträge" }; + let label = if singular { "Eintrag" } else { "Einträge" }; format!("{count} {label} in den Papierkorb verschoben ({removed})...") } } @@ -773,11 +775,13 @@ impl Language { pub fn notification_summary( self, action: &str, - entries: u64, + entries: Option, bytes: &str, elapsed: &str, errors: u64, ) -> String { + let singular = entries == Some(1); + let entries = entries.map_or_else(|| "?".into(), |entries| entries.to_string()); let errors = match (self, errors) { (_, 0) => String::new(), (Language::English, count) => format!(", {count} errors"), @@ -807,7 +811,7 @@ impl Language { t.notification_finished ), Language::German => { - let label = if entries == 1 { "Eintrag" } else { "Einträge" }; + let label = if singular { "Eintrag" } else { "Einträge" }; format!( "{action} {}: {entries} {label}, {bytes} in {elapsed}{errors}", t.notification_finished @@ -865,7 +869,7 @@ const EN_UI: UiText = UiText { snapshots_read_only: "Snapshots are read-only", traversal_running: "Traversal already running", deletion_running: "Deletion in progress; changes are disabled", - deletion_queued: "Deletion queued; waiting for the scan to finish", + pausing_scan: "Pausing the scan before deletion...", cancelling_deletion: "Cancelling deletion...", no_cleanup_candidates: "No cleanup candidates in view", cleanup_candidates_already_marked: "Cleanup candidates are already marked", @@ -929,7 +933,7 @@ const JA_UI: UiText = UiText { snapshots_read_only: "スナップショットは読み取り専用です", traversal_running: "スキャンはすでに実行中です", deletion_running: "削除中のため変更できません", - deletion_queued: "削除を予約しました。スキャン完了を待っています", + pausing_scan: "削除の前にスキャンを一時停止しています...", cancelling_deletion: "削除を中止しています...", no_cleanup_candidates: "現在の表示にクリーンアップ候補はありません", cleanup_candidates_already_marked: "クリーンアップ候補はすでにマーク済みです", @@ -993,7 +997,7 @@ const KO_UI: UiText = UiText { snapshots_read_only: "스냅샷은 읽기 전용입니다", traversal_running: "스캔이 이미 실행 중입니다", deletion_running: "삭제 중에는 변경할 수 없습니다", - deletion_queued: "삭제 예약됨; 스캔이 끝날 때까지 기다리는 중", + pausing_scan: "삭제 전에 스캔을 일시 중지하는 중...", cancelling_deletion: "삭제 취소 중...", no_cleanup_candidates: "현재 보기에 정리 후보가 없습니다", cleanup_candidates_already_marked: "정리 후보가 이미 표시되어 있습니다", @@ -1057,7 +1061,7 @@ const ZH_UI: UiText = UiText { snapshots_read_only: "快照为只读", traversal_running: "扫描已在进行", deletion_running: "正在删除,暂时无法更改", - deletion_queued: "删除已排队,等待扫描完成", + pausing_scan: "正在暂停扫描以进行删除...", cancelling_deletion: "正在取消删除...", no_cleanup_candidates: "当前视图中没有清理候选项", cleanup_candidates_already_marked: "清理候选项已标记", @@ -1121,7 +1125,7 @@ const DE_UI: UiText = UiText { snapshots_read_only: "Snapshots sind schreibgeschützt", traversal_running: "Scan läuft bereits", deletion_running: "Löschung läuft; Änderungen sind gesperrt", - deletion_queued: "Löschung vorgemerkt; wartet auf das Scan-Ende", + pausing_scan: "Scan wird vor der Löschung angehalten...", cancelling_deletion: "Löschung wird abgebrochen...", no_cleanup_candidates: "Keine Bereinigungskandidaten in der Ansicht", cleanup_candidates_already_marked: "Bereinigungskandidaten sind bereits markiert", @@ -1245,7 +1249,7 @@ mod tests { Language::German, ] { for trash in [false, true] { - let progress = language.deletion_progress(42, "123 MB", trash); + let progress = language.deletion_progress(Some(42), "123 MB", trash); assert!(progress.contains("42")); assert!(progress.contains("123 MB")); } @@ -1375,7 +1379,7 @@ mod tests { assert_eq!( language.notification_summary( language.ui_text().notification_scan, - 42, + Some(42), "2.00 MB", "1.2s", 2, diff --git a/src/interactive/widgets/main.rs b/src/interactive/widgets/main.rs index 6db4093c..2cfaa4d3 100644 --- a/src/interactive/widgets/main.rs +++ b/src/interactive/widgets/main.rs @@ -73,8 +73,8 @@ impl MainWindow { let (entries_style, help_style, mark_style, glob_style) = pane_border_style(state.focussed); let (header_area, content_area, footer_area) = main_window_layout(area); - let safety_notice = if state.queued_deletion.is_some() { - Some(language.ui_text().deletion_queued) + let safety_notice = if state.pending_deletion.is_some() { + Some(language.ui_text().pausing_scan) } else { mark_safety_notice(state.read_only, state.is_deleting(), &config.keys, language) }; diff --git a/src/traverse.rs b/src/traverse.rs index 89c66040..8b9667b5 100644 --- a/src/traverse.rs +++ b/src/traverse.rs @@ -10,7 +10,7 @@ use std::{ num::NonZeroU32, path::{Path, PathBuf}, sync::{ - Arc, + Arc, Condvar, Mutex, MutexGuard, TryLockError, atomic::{AtomicBool, Ordering}, }, time::{Duration, Instant, SystemTime, UNIX_EPOCH}, @@ -833,7 +833,73 @@ pub struct BackgroundTraversal { /// Receiver used to obtain traversal events from the worker thread. pub event_rx: Receiver, clean_stages: HashMap, - _cancelled: Option, + _cancelled: CancelOnDrop, + control: Arc, +} + +#[derive(Default)] +struct ScanControl { + walk: crate::walk::Control, + output_paused: AtomicBool, + output: Mutex<()>, + resumed: Condvar, +} + +impl ScanControl { + fn publishing(&self) -> Option> { + let mut output = self.output.lock().unwrap(); + while self.output_paused.load(Ordering::Acquire) + && self.walk.status() != crate::walk::Status::Finished + { + output = self + .resumed + .wait_timeout(output, Duration::from_millis(50)) + .unwrap() + .0; + } + (self.walk.status() != crate::walk::Status::Finished).then_some(output) + } + + fn send(&self, tx: &crossbeam::channel::Sender, event: TraversalEvent) -> bool { + let Some(_publishing) = self.publishing() else { + return false; + }; + tx.send(event).is_ok() + } + + fn pause_output(&self) -> bool { + if self.walk.status() != crate::walk::Status::Paused { + return false; + } + match self.output.try_lock() { + Ok(_output) => { + self.output_paused.store(true, Ordering::Release); + true + } + Err(TryLockError::WouldBlock) => false, + Err(TryLockError::Poisoned(error)) => panic!("scan output lock poisoned: {error}"), + } + } + + fn resume(&self) { + // Never wait for a publisher blocked on a full output channel to release its lock. + self.output_paused.store(false, Ordering::Release); + self.walk.resume(); + self.resumed.notify_all(); + } + + fn cancel(&self) { + self.output_paused.store(false, Ordering::Release); + self.walk.cancel(); + self.resumed.notify_all(); + } + + fn removed(&self, entry: &Result) -> bool { + match entry { + Ok(entry) => self.walk.is_excluded(&entry.path()), + Err(error) => error.path().is_some_and(|path| self.walk.is_excluded(path)), + } + } } struct CleanStage { @@ -844,11 +910,17 @@ struct CleanStage { valid: bool, } -struct CancelOnDrop(Arc); +struct CancelOnDrop { + cancelled: Option>, + control: Arc, +} impl Drop for CancelOnDrop { fn drop(&mut self) { - self.0.store(true, Ordering::Relaxed); + if let Some(cancelled) = &self.cancelled { + cancelled.store(true, Ordering::Relaxed); + } + self.control.cancel(); } } @@ -865,6 +937,7 @@ impl CleanInput { fn discover( self, options: WalkOptions, + control: &crate::walk::Control, mut send: impl FnMut(crate::clean::DiscoveryEvent) -> bool, ) { match self { @@ -873,7 +946,14 @@ impl CleanInput { depth, pattern_root, } => { - crate::clean::discover(paths, options, depth, pattern_root, send); + crate::clean::discover_with_control( + paths, + options, + depth, + pattern_root, + Some(control), + send, + ); } Self::Refresh(candidates) => { let mut keep_going = true; @@ -881,11 +961,12 @@ impl CleanInput { if !keep_going { break; } - crate::clean::discover( + crate::clean::discover_with_control( vec![path], options.clone(), Some(0), Some(search_root), + Some(control), |event| { keep_going = send(event); keep_going @@ -933,16 +1014,18 @@ impl BackgroundTraversal { ) -> anyhow::Result { let (entry_tx, event_rx) = crossbeam::channel::bounded(100); let cancelled = Arc::new(AtomicBool::new(false)); + let control = Arc::new(ScanControl::default()); std::thread::Builder::new() .name("dua-clean-dispatcher".into()) .spawn({ let walk_options = walk_options.clone(); let cancelled = Arc::clone(&cancelled); + let control = Arc::clone(&control); move || { use crate::clean::DiscoveryEvent; if walk_options.threads <= 1 { let mut index = 0; - input.discover(walk_options.clone(), |event| { + input.discover(walk_options.clone(), &control.walk, |event| { if cancelled.load(Ordering::Relaxed) { return false; } @@ -955,19 +1038,27 @@ impl BackgroundTraversal { index, &entry_tx, &cancelled, + &control, ); index += 1; keep_going } - DiscoveryEvent::Progress { entries, io_errors } => entry_tx - .send(TraversalEvent::DiscoveryProgress { entries, io_errors }) - .is_ok(), + DiscoveryEvent::Progress { entries, io_errors } => control.send( + &entry_tx, + TraversalEvent::DiscoveryProgress { entries, io_errors }, + ), } }); } else { - walk_clean_candidates(input, &walk_options, &entry_tx, &cancelled); + walk_clean_candidates( + input, + &walk_options, + &entry_tx, + &cancelled, + &control, + ); } - let _ = entry_tx.send(TraversalEvent::Finished); + control.send(&entry_tx, TraversalEvent::Finished); } })?; Ok(Self { @@ -984,7 +1075,11 @@ impl BackgroundTraversal { preexisting_nodes: HashMap::new(), event_rx, clean_stages: HashMap::new(), - _cancelled: Some(CancelOnDrop(cancelled)), + _cancelled: CancelOnDrop { + cancelled: Some(cancelled), + control: Arc::clone(&control), + }, + control, }) } @@ -1057,10 +1152,12 @@ impl BackgroundTraversal { let num_roots = input.len(); let (entry_tx, entry_rx) = crossbeam::channel::bounded(100); let pattern_roots = pattern_roots.map(<[PathBuf]>::to_owned); + let control = Arc::new(ScanControl::default()); std::thread::Builder::new() .name("dua-fs-walk-dispatcher".to_string()) .spawn({ let walk_options = walk_options.clone(); + let control = Arc::clone(&control); move || { let (mut root_paths, mut root_indices, mut device_ids, mut walk_roots) = ( Vec::with_capacity(input.len()), @@ -1069,15 +1166,18 @@ impl BackgroundTraversal { Vec::with_capacity(input.len()), ); for (root_idx, root_path) in input.into_iter().enumerate() { + let Some(_activity) = control.walk.activity() else { + return; + }; log::info!("Walking {}", root_path.display()); let device_id = if walk_options.cross_filesystems { 0 } else { let Ok(device_id) = crossdev::init(&root_path) else { - if entry_tx - .send(TraversalEvent::RootError(Arc::new(root_path), root_idx)) - .is_err() - { + if !control.send( + &entry_tx, + TraversalEvent::RootError(Arc::new(root_path), root_idx), + ) { return; } continue; @@ -1105,14 +1205,21 @@ impl BackgroundTraversal { root_paths.push(Arc::new(root_path)); } - for (root, event) in walk_options.iter_from_paths( + for (root, event) in walk_options.iter_from_paths_with_control( walk_roots, skip_root, crate::walk::Order::ParentFirst, + Some(control.walk.clone()), ) { let crate::walk::RootEvent::Entry(entry) = event else { continue; }; + let Some(_publishing) = control.publishing() else { + return; + }; + if control.removed(&entry) { + continue; + } if entry_tx .send(TraversalEvent::Entry( entry.map(TraversalEntry), @@ -1127,7 +1234,7 @@ impl BackgroundTraversal { return; } } - if entry_tx.send(TraversalEvent::Finished).is_err() { + if !control.send(&entry_tx, TraversalEvent::Finished) { log::error!("Failed to send TraversalEvents::Finished event"); } } @@ -1147,7 +1254,11 @@ impl BackgroundTraversal { preexisting_nodes, event_rx: entry_rx, clean_stages: HashMap::new(), - _cancelled: None, + _cancelled: CancelOnDrop { + cancelled: None, + control: Arc::clone(&control), + }, + control, }) } @@ -1157,6 +1268,47 @@ impl BackgroundTraversal { self.root_nodes.iter().copied().collect() } + /// Request a pause before the next filesystem jobs start. + pub fn pause(&self) { + self.control.walk.pause(); + } + + /// Stop publication once workers are quiet; keep consuming events until this returns true. + #[must_use] + pub fn pause_for_deletion(&self) -> bool { + self.control.pause_output() && self.event_rx.is_empty() + } + + /// Resume filesystem work and publication after deletion has completed. + pub fn resume(&self) { + self.control.resume(); + } + + /// Retain a confirmed removal path so resumed work cannot rediscover it. + pub fn forget_removed_path(&self, path: PathBuf) { + self.control.walk.exclude_path(path); + } + + /// Clear side-table indices before their removed tree slots can be reused. + #[must_use] + pub fn forget_removed_nodes(&mut self, removed: &HashSet) -> bool { + if removed.contains(&self.root_idx) { + return false; + } + for node in self + .nodes_by_directory + .iter_mut() + .chain(self.root_nodes.iter_mut()) + { + if node.is_some_and(|node| removed.contains(&node)) { + *node = None; + } + } + self.preexisting_nodes + .retain(|_, (node, _)| !removed.contains(node)); + true + } + /// Keep tree nodes through `depth`, while still aggregating all sizes, or retain all nodes when /// it is `None`. For example, 0 retains roots only, 1 also retains their immediate children, /// and 2 also retains grandchildren. @@ -1174,7 +1326,7 @@ impl BackgroundTraversal { if self.skip_root { return; } - // Entry errors carry no descendant path, so report them on the corresponding root. + // Keep surviving traversal errors on the corresponding input root. if let Some(root) = self.root_nodes[root_idx] { traversal .tree @@ -1345,6 +1497,15 @@ impl BackgroundTraversal { return Some(false); } TraversalEvent::Entry(entry, root_path, device_id, root_idx) => { + let removed = match &entry { + Ok(TraversalEntry(entry)) => self.control.walk.is_excluded(&entry.path()), + Err(error) => error + .path() + .is_some_and(|path| self.control.walk.is_excluded(path)), + }; + if removed { + return None; + } self.stats.entries_traversed += 1; let is_clean = self.clean_stages.contains_key(&root_idx); let mut data = EntryData::default(); @@ -1515,6 +1676,9 @@ impl BackgroundTraversal { } } TraversalEvent::RootError(root_path, root_idx) => { + if self.control.walk.is_excluded(&root_path) { + return None; + } if let Some(stage) = self.clean_stages.get_mut(&root_idx) { stage.valid = false; } @@ -1635,13 +1799,15 @@ fn walk_clean_candidates( options: &WalkOptions, tx: &crossbeam::channel::Sender, cancelled: &Arc, + control: &Arc, ) { std::thread::scope(|scope| { // One discovery reader and a shared sizing pool, within the existing I/O thread budget. - let (mut roots, walk) = crate::walk::stream_roots( + let (mut roots, walk) = crate::walk::stream_roots_with_control( options.threads - 1, crate::walk::Order::ParentFirst, options.metadata_options, + control.walk.clone(), ); let (candidate_tx, candidate_rx) = crossbeam::channel::bounded(32); // Bound active candidate staging, not just the queue waiting to reach the walker. @@ -1653,15 +1819,20 @@ fn walk_clean_candidates( .name("dua-clean-discovery".into()) .spawn_scoped(scope, move || { let mut index = 0; - input.discover(options.clone(), |event| { + input.discover(options.clone(), &control.walk, |event| { if cancelled.load(Ordering::Relaxed) { return false; } match event { - crate::clean::DiscoveryEvent::Progress { entries, io_errors } => tx - .send(TraversalEvent::DiscoveryProgress { entries, io_errors }) - .is_ok(), + crate::clean::DiscoveryEvent::Progress { entries, io_errors } => control + .send(tx, TraversalEvent::DiscoveryProgress { entries, io_errors }), crate::clean::DiscoveryEvent::Candidate { path, search_root } => { + let Some(activity) = control.walk.activity() else { + return false; + }; + if control.walk.is_excluded(&path) { + return true; + } let candidate = match CleanCandidate::new(options, path.clone(), search_root) { Ok(candidate) => candidate, @@ -1670,14 +1841,16 @@ fn walk_clean_candidates( "Could not inspect cleanup root {}: {err}", path.display() ); - return tx - .send(TraversalEvent::DiscoveryProgress { + return control.send( + tx, + TraversalEvent::DiscoveryProgress { entries: 0, io_errors: 1, - }) - .is_ok(); + }, + ); } }; + drop(activity); if slot_rx.recv().is_err() { return false; } @@ -1710,10 +1883,13 @@ fn walk_clean_candidates( }); if let Err(err) = discovery { log::error!("Could not start cleanup discovery: {err}"); - let _ = tx.send(TraversalEvent::DiscoveryProgress { - entries: 0, - io_errors: 1, - }); + control.send( + tx, + TraversalEvent::DiscoveryProgress { + entries: 0, + io_errors: 1, + }, + ); return; } let cwd = std::env::current_dir().unwrap_or_default(); @@ -1722,6 +1898,9 @@ fn walk_clean_candidates( if cancelled.load(Ordering::Relaxed) { break; } + let Some(_publishing) = control.publishing() else { + break; + }; // Discovery queues context before submitting a root, so it is available even when // another root's metadata finishes first. while !candidates.contains_key(&index) { @@ -1744,7 +1923,8 @@ fn walk_clean_candidates( } let keep_going = match event { crate::walk::RootEvent::Entry(entry) => { - candidates[&index].forward_entry(options, &cwd, index, entry, tx) + control.removed(&entry) + || candidates[&index].forward_entry(options, &cwd, index, entry, tx) } crate::walk::RootEvent::Finished => { let candidate = candidates.remove(&index).expect("root has started"); @@ -1776,29 +1956,35 @@ fn walk_clean_candidate( index: usize, tx: &crossbeam::channel::Sender, cancelled: &AtomicBool, + control: &ScanControl, ) -> bool { if cancelled.load(Ordering::Relaxed) { return false; } + let Some(activity) = control.walk.activity() else { + return false; + }; + if control.walk.is_excluded(&path) { + return true; + } let candidate = match CleanCandidate::new(options, path, search_root) { Ok(candidate) => candidate, Err(err) => { log::debug!("Could not inspect cleanup root: {err}"); - return tx - .send(TraversalEvent::DiscoveryProgress { + return control.send( + tx, + TraversalEvent::DiscoveryProgress { entries: 0, io_errors: 1, - }) - .is_ok(); + }, + ); } }; - if tx - .send(TraversalEvent::CandidateStarted( - index, - candidate.search_root.clone(), - )) - .is_err() - { + drop(activity); + if !control.send( + tx, + TraversalEvent::CandidateStarted(index, candidate.search_root.clone()), + ) { return false; } // Inspect exclusions as well: deleting their containing candidate would delete them too. @@ -1814,11 +2000,22 @@ fn walk_clean_candidate( device_id: candidate.device, }]; let cwd = std::env::current_dir().unwrap_or_default(); - for (_, event) in unfiltered.iter_from_paths(roots, false, crate::walk::Order::ParentFirst) { + for (_, event) in unfiltered.iter_from_paths_with_control( + roots, + false, + crate::walk::Order::ParentFirst, + Some(control.walk.clone()), + ) { if cancelled.load(Ordering::Relaxed) { return false; } + let Some(_publishing) = control.publishing() else { + return false; + }; if let crate::walk::RootEvent::Entry(entry) = event { + if control.removed(&entry) { + continue; + } if !candidate.forward_entry(options, &cwd, index, entry, tx) { return false; } @@ -1827,11 +2024,10 @@ fn walk_clean_candidate( } } } - tx.send(TraversalEvent::CandidateFinished( - index, - candidate.accepted.load(Ordering::Relaxed), - )) - .is_ok() + control.send( + tx, + TraversalEvent::CandidateFinished(index, candidate.accepted.load(Ordering::Relaxed)), + ) } #[cfg(not(any(windows, target_os = "macos")))] @@ -1897,6 +2093,132 @@ mod tests { } } + #[test] + fn pause_handoff_and_resume_do_not_wait_for_a_full_output_channel() { + let control = Arc::new(ScanControl::default()); + let (tx, rx) = crossbeam::channel::bounded(0); + let (started_tx, started_rx) = crossbeam::channel::bounded(0); + let publisher = std::thread::spawn({ + let control = Arc::clone(&control); + move || { + let _publishing = control.publishing().unwrap(); + started_tx.send(()).unwrap(); + tx.send(()).unwrap(); + } + }); + started_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + control.walk.pause(); + assert!( + !control.pause_output(), + "drain the publisher before completing the pause" + ); + let (done_tx, done_rx) = crossbeam::channel::bounded(1); + let resumer = std::thread::spawn({ + let control = Arc::clone(&control); + move || { + control.resume(); + done_tx.send(()).unwrap(); + } + }); + done_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + rx.recv().unwrap(); + publisher.join().unwrap(); + resumer.join().unwrap(); + } + + #[test] + fn dropping_a_paused_clean_scan_releases_discovery_and_sizing() { + let directory = tempfile::tempdir().unwrap(); + for index in 0..80 { + let path = directory + .path() + .join(format!("project-{index}/node_modules")); + std::fs::create_dir_all(&path).unwrap(); + std::fs::write(path.join("payload"), b"cache").unwrap(); + } + for threads in [1, 4] { + let mut options = clean_walk_options(); + options.threads = threads; + let scan = BackgroundTraversal::start_clean( + TreeIndex::default(), + &options, + vec![directory.path().to_owned()], + None, + None, + ) + .unwrap(); + let events = scan.event_rx.clone(); + let deadline = Instant::now() + Duration::from_secs(5); + while events.len() < 100 { + assert!( + Instant::now() < deadline, + "cleanup did not fill its output queue" + ); + std::thread::sleep(Duration::from_millis(1)); + } + scan.pause(); + let deadline = Instant::now() + Duration::from_secs(5); + while !scan.pause_for_deletion() { + assert!(Instant::now() < deadline, "cleanup did not pause"); + let _ = events.recv_timeout(Duration::from_millis(10)); + } + drop(scan); + assert!( + matches!( + events.recv_timeout(Duration::from_secs(5)), + Err(crossbeam::channel::RecvTimeoutError::Disconnected) + ), + "cancelled cleanup threads release their output channel" + ); + } + } + + #[test] + fn removed_path_errors_are_ignored_while_unrelated_errors_are_recorded() { + let directory = tempfile::tempdir().unwrap(); + let root_path = Arc::new(directory.path().to_owned()); + let mut traversal = Traversal::new(); + let mut scan = BackgroundTraversal::start( + traversal.root_index, + &clean_walk_options(), + vec![root_path.as_ref().clone()], + None, + false, + true, + ) + .unwrap(); + loop { + let event = scan.event_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + if scan.integrate_traversal_event(&mut traversal, event) == Some(true) { + break; + } + } + let root = scan.root_nodes().unwrap()[0]; + let removed = root_path.join("removed"); + scan.forget_removed_path(removed.clone()); + for path in [removed.clone(), removed.join("child")] { + let error = crate::walk::Entry::from_path(&path, crate::TraversalOptions::default()); + scan.integrate_traversal_event( + &mut traversal, + TraversalEvent::Entry(error.map(TraversalEntry), Arc::clone(&root_path), 0, 0), + ); + scan.integrate_traversal_event( + &mut traversal, + TraversalEvent::RootError(Arc::new(path), 0), + ); + } + assert_eq!(scan.stats.io_errors, 0); + assert!(!traversal.tree.data(root).unwrap().metadata_io_error); + let unrelated = root_path.join("unrelated"); + let error = crate::walk::Entry::from_path(&unrelated, crate::TraversalOptions::default()); + scan.integrate_traversal_event( + &mut traversal, + TraversalEvent::Entry(error.map(TraversalEntry), root_path, 0, 0), + ); + assert_eq!(scan.stats.io_errors, 1); + assert!(traversal.tree.data(root).unwrap().metadata_io_error); + } + #[test] fn clean_candidates_can_finish_out_of_order_without_sharing_unaccepted_inodes() { let directory = tempfile::tempdir().unwrap();