From 5efb6334918b9a5ee9117ca6d43a68989c89b7cf Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 7 Oct 2026 15:10:13 +0530 Subject: [PATCH 1/6] Add a networkquiet wait that counts quiet from the start waitforloadstate networkidle starts its 500 ms of quiet only after a first 600 ms receive window passes empty, so even an already idle page waits about 1.1 s. networkquiet counts the 500 ms from the start of the wait and polls every 100 ms: an idle page resolves after 500 ms (never instantly, as #846 requires), and a page with requests in flight still waits for them. networkidle is unchanged. --- cli/src/native/browser.rs | 98 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 95 insertions(+), 3 deletions(-) diff --git a/cli/src/native/browser.rs b/cli/src/native/browser.rs index 5952b192b1..354c0945d3 100644 --- a/cli/src/native/browser.rs +++ b/cli/src/native/browser.rs @@ -348,6 +348,10 @@ pub enum WaitUntil { Load, DomContentLoaded, NetworkIdle, + /// Like `NetworkIdle`, but quiet is counted from the start of the wait, + /// so a page that is already idle resolves after 500 ms of silence + /// rather than after a first 600 ms receive timeout and then 500 ms more. + NetworkQuiet, None, } @@ -356,6 +360,7 @@ impl WaitUntil { match s { "domcontentloaded" => Self::DomContentLoaded, "networkidle" => Self::NetworkIdle, + "networkquiet" => Self::NetworkQuiet, "none" => Self::None, _ => Self::Load, } @@ -1172,6 +1177,10 @@ impl BrowserManager { WaitUntil::Load => "Page.loadEventFired", WaitUntil::DomContentLoaded => "Page.domContentEventFired", WaitUntil::NetworkIdle => return self.wait_for_network_idle(session_id, rx).await, + WaitUntil::NetworkQuiet => { + let timeout = tokio::time::Duration::from_millis(self.default_timeout_ms); + return poll_network(session_id, rx, timeout, Quiet::FromStart).await; + } WaitUntil::None => return Ok(()), }; @@ -1276,7 +1285,7 @@ impl BrowserManager { WaitUntil::DomContentLoaded => Some("document.readyState !== 'loading'"), // Network idle is tracked from live network events; its poller // already treats a quiet stream as idle. - WaitUntil::NetworkIdle | WaitUntil::None => None, + WaitUntil::NetworkIdle | WaitUntil::NetworkQuiet | WaitUntil::None => None, }; if let Some(expression) = already_reached { let probe: Result = self @@ -2230,15 +2239,42 @@ async fn poll_network_idle( session_id: &str, rx: &mut broadcast::Receiver, overall_timeout: tokio::time::Duration, +) -> Result<(), String> { + poll_network(session_id, rx, overall_timeout, Quiet::AfterSilence).await +} + +/// When a network wait starts counting quiet time. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Quiet { + /// After a first receive window passes with nothing in flight: an idle + /// page resolves after about 1.1 s (`networkidle`). + AfterSilence, + /// From the start of the wait, polling every 100 ms: an idle page + /// resolves after 500 ms of silence (`networkquiet`). + FromStart, +} + +/// The network-idle loop: `Ok(())` once nothing has been in flight for +/// 500 ms, counted as `quiet` says, or `Err` after `overall_timeout`. +async fn poll_network( + session_id: &str, + rx: &mut broadcast::Receiver, + overall_timeout: tokio::time::Duration, + quiet: Quiet, ) -> Result<(), String> { let pending = Arc::new(Mutex::new(HashSet::::new())); + let window = match quiet { + Quiet::AfterSilence => 600, + Quiet::FromStart => 100, + }; tokio::time::timeout(overall_timeout, async { - let mut idle_start: Option = None; + let mut idle_start: Option = + (quiet == Quiet::FromStart).then(tokio::time::Instant::now); loop { let recv_result = - tokio::time::timeout(tokio::time::Duration::from_millis(600), rx.recv()).await; + tokio::time::timeout(tokio::time::Duration::from_millis(window), rx.recv()).await; match recv_result { Ok(Ok(event)) if event.session_id.as_deref() == Some(session_id) => { @@ -2925,6 +2961,62 @@ mod tests { drop(tx); } + /// `networkquiet` counts quiet from the start: a silent stream resolves + /// after the 500 ms window (never instantly, as #846 requires), well + /// before `networkidle`'s first 600 ms receive timeout plus 500 ms. + #[tokio::test] + async fn test_network_quiet_counts_from_the_start() { + let (tx, mut rx) = broadcast::channel::(16); + let start = tokio::time::Instant::now(); + let result = poll_network("s1", &mut rx, Duration::from_secs(5), Quiet::FromStart).await; + assert!(result.is_ok()); + let elapsed = start.elapsed(); + assert!( + elapsed >= Duration::from_millis(500) && elapsed < Duration::from_millis(800), + "network quiet returned in {:?}, expected 500-800ms", + elapsed + ); + drop(tx); + } + + /// `networkquiet` still waits out a request in flight, then 500 ms more. + #[tokio::test] + async fn test_network_quiet_waits_for_requests_in_flight() { + let (tx, mut rx) = broadcast::channel::(16); + let session = "s1"; + let _keep_alive = tx.clone(); + tokio::spawn(async move { + sleep(Duration::from_millis(50)).await; + let _ = tx.send(cdp_event( + "Network.requestWillBeSent", + session, + json!({ "requestId": "r1" }), + )); + sleep(Duration::from_millis(600)).await; + let _ = tx.send(cdp_event( + "Network.loadingFinished", + session, + json!({ "requestId": "r1" }), + )); + }); + let start = tokio::time::Instant::now(); + let result = poll_network(session, &mut rx, Duration::from_secs(5), Quiet::FromStart).await; + assert!(result.is_ok()); + let elapsed = start.elapsed(); + assert!( + elapsed >= Duration::from_millis(1150), + "network quiet returned in {:?}, before the request finished and 500ms passed", + elapsed + ); + } + + /// `WaitUntil` reads `networkquiet` as its own state. + #[test] + fn test_wait_until_reads_network_quiet() { + assert!(WaitUntil::from_str("networkquiet") == WaitUntil::NetworkQuiet); + assert!(WaitUntil::from_str("networkidle") == WaitUntil::NetworkIdle); + } + /// Normal flow: requests start and finish, idle is detected after the last /// request completes and 500 ms of silence passes. #[tokio::test] From ce670649419d3fa7f28b2334f9e4b8159e5d0e12 Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 7 Oct 2026 17:57:48 +0530 Subject: [PATCH 2/6] Count only page-changing requests in the networkquiet wait On Amazon every action that opened a page waited the whole 2 s cap: of the 100+ requests such a page sends, analytics pings and the documents of ad and helper frames in other processes never report finishing to the page's session, so nothing ever counted as quiet. And the end of any request the wait never saw start reset its quiet timer. networkquiet now counts only requests that can change what the page shows: the page's own document (a page target's main frame shares its target id), scripts, stylesheets, and XHR or fetch data. Pings, pictures, fonts, media, prefetches, streams, and other frames' documents are not counted, and only a counted request's end restarts the quiet window. networkidle is unchanged. --- cli/src/native/browser.rs | 160 ++++++++++++++++++++++++++++++++++---- 1 file changed, 143 insertions(+), 17 deletions(-) diff --git a/cli/src/native/browser.rs b/cli/src/native/browser.rs index 354c0945d3..6b05af31dc 100644 --- a/cli/src/native/browser.rs +++ b/cli/src/native/browser.rs @@ -1179,7 +1179,21 @@ impl BrowserManager { WaitUntil::NetworkIdle => return self.wait_for_network_idle(session_id, rx).await, WaitUntil::NetworkQuiet => { let timeout = tokio::time::Duration::from_millis(self.default_timeout_ms); - return poll_network(session_id, rx, timeout, Quiet::FromStart).await; + // A page target's main frame shares its target's id. + let main_frame = self + .pages + .iter() + .find(|page| page.session_id == session_id) + .map(|page| page.target_id.clone()); + return poll_network( + session_id, + rx, + timeout, + Quiet::FromStart { + main_frame: main_frame.as_deref(), + }, + ) + .await; } WaitUntil::None => return Ok(()), }; @@ -2243,15 +2257,35 @@ async fn poll_network_idle( poll_network(session_id, rx, overall_timeout, Quiet::AfterSilence).await } -/// When a network wait starts counting quiet time. +/// When a network wait starts counting quiet time, and what it counts. #[derive(Clone, Copy, PartialEq, Eq, Debug)] -enum Quiet { +enum Quiet<'a> { /// After a first receive window passes with nothing in flight: an idle /// page resolves after about 1.1 s (`networkidle`). AfterSilence, - /// From the start of the wait, polling every 100 ms: an idle page - /// resolves after 500 ms of silence (`networkquiet`). - FromStart, + /// From the start of the wait, polling every 100 ms, and counting only + /// the requests that can change what the page shows ([`changes_page`]), + /// `main_frame`'s documents among them: an idle page resolves after + /// 500 ms of silence (`networkquiet`). + FromStart { main_frame: Option<&'a str> }, +} + +/// Whether a request can change what the page shows: the page's own +/// document, a script, a stylesheet, or data it fetches. An analytics ping, +/// a picture, a font, media, a prefetch, a stream, and a document of +/// another frame cannot; some of them (a ping, the document of a frame in +/// another process) never even report finishing to the page's session, so +/// a page that sent one would never go quiet. A request of no known type +/// counts. +fn changes_page(params: &Value, main_frame: Option<&str>) -> bool { + match params.get("type").and_then(Value::as_str) { + Some("XHR" | "Fetch" | "Script" | "Stylesheet") | None => true, + Some("Document") => match (main_frame, params.get("frameId").and_then(Value::as_str)) { + (Some(main), Some(frame)) => main == frame, + _ => true, + }, + Some(_) => false, + } } /// The network-idle loop: `Ok(())` once nothing has been in flight for @@ -2260,17 +2294,17 @@ async fn poll_network( session_id: &str, rx: &mut broadcast::Receiver, overall_timeout: tokio::time::Duration, - quiet: Quiet, + quiet: Quiet<'_>, ) -> Result<(), String> { let pending = Arc::new(Mutex::new(HashSet::::new())); - let window = match quiet { - Quiet::AfterSilence => 600, - Quiet::FromStart => 100, + let (window, counted): (u64, Option>) = match quiet { + Quiet::AfterSilence => (600, None), + Quiet::FromStart { main_frame } => (100, Some(main_frame)), }; tokio::time::timeout(overall_timeout, async { let mut idle_start: Option = - (quiet == Quiet::FromStart).then(tokio::time::Instant::now); + counted.is_some().then(tokio::time::Instant::now); loop { let recv_result = @@ -2281,17 +2315,24 @@ async fn poll_network( let mut p = pending.lock().await; match event.method.as_str() { "Network.requestWillBeSent" => { + let counts = counted + .is_none_or(|main_frame| changes_page(&event.params, main_frame)); if let Some(id) = event.params.get("requestId").and_then(|v| v.as_str()) { - p.insert(id.to_string()); - idle_start = None; + if counts { + p.insert(id.to_string()); + idle_start = None; + } } } "Network.loadingFinished" | "Network.loadingFailed" => { if let Some(id) = event.params.get("requestId").and_then(|v| v.as_str()) { - p.remove(id); - if p.is_empty() { + let was_pending = p.remove(id); + // Counting only what changes the page, a request + // not counted, or sent before the wait, ending + // says nothing about the page going quiet. + if p.is_empty() && (was_pending || counted.is_none()) { idle_start = Some(tokio::time::Instant::now()); } } @@ -2968,7 +3009,13 @@ mod tests { async fn test_network_quiet_counts_from_the_start() { let (tx, mut rx) = broadcast::channel::(16); let start = tokio::time::Instant::now(); - let result = poll_network("s1", &mut rx, Duration::from_secs(5), Quiet::FromStart).await; + let result = poll_network( + "s1", + &mut rx, + Duration::from_secs(5), + Quiet::FromStart { main_frame: None }, + ) + .await; assert!(result.is_ok()); let elapsed = start.elapsed(); assert!( @@ -3000,7 +3047,13 @@ mod tests { )); }); let start = tokio::time::Instant::now(); - let result = poll_network(session, &mut rx, Duration::from_secs(5), Quiet::FromStart).await; + let result = poll_network( + session, + &mut rx, + Duration::from_secs(5), + Quiet::FromStart { main_frame: None }, + ) + .await; assert!(result.is_ok()); let elapsed = start.elapsed(); assert!( @@ -3010,6 +3063,79 @@ mod tests { ); } + /// `networkquiet` counts only what can change the page: a ping, a + /// picture, and another frame's document that never report finishing + /// leave it quiet, and so does the end of a request it never saw start. + #[tokio::test] + async fn test_network_quiet_counts_only_what_changes_the_page() { + let (tx, mut rx) = broadcast::channel::(16); + let session = "s1"; + let _keep_alive = tx.clone(); + tokio::spawn(async move { + sleep(Duration::from_millis(50)).await; + for params in [ + json!({ "requestId": "ping", "type": "Ping" }), + json!({ "requestId": "image", "type": "Image" }), + json!({ "requestId": "frame", "type": "Document", "frameId": "ads" }), + ] { + let _ = tx.send(cdp_event("Network.requestWillBeSent", session, params)); + } + for later in 0..4 { + sleep(Duration::from_millis(150)).await; + let _ = tx.send(cdp_event( + "Network.loadingFinished", + session, + json!({ "requestId": format!("before-{later}") }), + )); + } + }); + let start = tokio::time::Instant::now(); + let result = poll_network( + session, + &mut rx, + Duration::from_secs(5), + Quiet::FromStart { + main_frame: Some("main"), + }, + ) + .await; + assert!(result.is_ok()); + let elapsed = start.elapsed(); + assert!( + elapsed >= Duration::from_millis(500) && elapsed < Duration::from_millis(800), + "network quiet returned in {:?}, expected 500-800ms", + elapsed + ); + } + + /// The page's own document, a script, a stylesheet, and fetched data + /// change the page; everything else does not. + #[test] + fn test_changes_page_reads_the_request_type_and_frame() { + let request = |kind: &str, frame: &str| json!({ "type": kind, "frameId": frame }); + for kind in ["XHR", "Fetch", "Script", "Stylesheet"] { + assert!(changes_page(&request(kind, "ads"), Some("main")), "{kind}"); + } + assert!(changes_page(&request("Document", "main"), Some("main"))); + assert!(!changes_page(&request("Document", "ads"), Some("main"))); + assert!(changes_page(&request("Document", "ads"), None)); + for kind in [ + "Ping", + "Image", + "Font", + "Media", + "Other", + "Prefetch", + "EventSource", + ] { + assert!( + !changes_page(&request(kind, "main"), Some("main")), + "{kind}" + ); + } + assert!(changes_page(&json!({}), Some("main")), "no type: counted"); + } + /// `WaitUntil` reads `networkquiet` as its own state. #[test] fn test_wait_until_reads_network_quiet() { From d8e93d8861d11bac957cabf41ce5a65fe8553559 Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 7 Oct 2026 18:45:36 +0530 Subject: [PATCH 3/6] Describe a selector snapshot's subtree through shadow roots A snapshot rooted at a selector collects the backendNodeIds under the matched element with DOM.describeNode, then keeps the accessibility nodes among them. Without `pierce`, describeNode stops at a shadow host, so a web component's controls were never in the subtree. On Lenskart, a consent banner's host (display: contents) yielded two DOM nodes and no accessibility node at all, and the snapshot failed with "No accessibility node found for selector"; with `pierce`, the same subtree holds 36 accessibility nodes, its "Allow Selection" and "Allow all" buttons among them. --- cli/src/native/snapshot.rs | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/cli/src/native/snapshot.rs b/cli/src/native/snapshot.rs index 80c5de0ab1..fb022e5b7d 100644 --- a/cli/src/native/snapshot.rs +++ b/cli/src/native/snapshot.rs @@ -374,12 +374,10 @@ pub async fn take_snapshot( .object_id .ok_or_else(|| format!("Selector '{}' did not match any element", selector))?; - // Request the full DOM subtree (depth: -1) so we can collect all - // backendNodeIds that live under the matched element. let describe: Value = client .send_command( "DOM.describeNode", - Some(serde_json::json!({ "objectId": object_id, "depth": -1 })), + Some(describe_subtree_params(&object_id)), Some(session_id), ) .await?; @@ -1479,6 +1477,15 @@ fn build_dedup_set(ref_map: &RefMap) -> std::collections::HashSet { /// Recursively collect all `backendNodeId` values from a CDP DOM node tree /// (as returned by `DOM.describeNode` with `depth: -1`). +/// `DOM.describeNode` parameters for the whole DOM subtree (depth -1) of +/// `object_id`, so every backendNodeId under the matched element can be +/// collected, through shadow roots too: a web component's controls live in +/// its shadow root, and without `pierce` the subtree stops at the host (on +/// a live page, a consent banner's host gave no node with its buttons). +fn describe_subtree_params(object_id: &str) -> Value { + serde_json::json!({ "objectId": object_id, "depth": -1, "pierce": true }) +} + fn collect_backend_node_ids(node: &Value, ids: &mut std::collections::HashSet) { if let Some(id) = node.get("backendNodeId").and_then(|v| v.as_i64()) { ids.insert(id); @@ -1501,6 +1508,18 @@ fn collect_backend_node_ids(node: &Value, ids: &mut std::collections::HashSet Date: Wed, 7 Oct 2026 20:02:53 +0530 Subject: [PATCH 4/6] Give collect_backend_node_ids its doc comment back describe_subtree_params was inserted between the comment and the function it describes, so its rustdoc opened with the wrong sentence. --- cli/src/native/snapshot.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/src/native/snapshot.rs b/cli/src/native/snapshot.rs index fb022e5b7d..80a39dbad6 100644 --- a/cli/src/native/snapshot.rs +++ b/cli/src/native/snapshot.rs @@ -1475,8 +1475,6 @@ fn build_dedup_set(ref_map: &RefMap) -> std::collections::HashSet { .collect() } -/// Recursively collect all `backendNodeId` values from a CDP DOM node tree -/// (as returned by `DOM.describeNode` with `depth: -1`). /// `DOM.describeNode` parameters for the whole DOM subtree (depth -1) of /// `object_id`, so every backendNodeId under the matched element can be /// collected, through shadow roots too: a web component's controls live in @@ -1486,6 +1484,8 @@ fn describe_subtree_params(object_id: &str) -> Value { serde_json::json!({ "objectId": object_id, "depth": -1, "pierce": true }) } +/// Recursively collect all `backendNodeId` values from a CDP DOM node tree +/// (as returned by `DOM.describeNode` with `depth: -1`). fn collect_backend_node_ids(node: &Value, ids: &mut std::collections::HashSet) { if let Some(id) = node.get("backendNodeId").and_then(|v| v.as_i64()) { ids.insert(id); From d7e87605b645c1dc8717bf67b4f0894c7cf49a3d Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 7 Oct 2026 20:02:53 +0530 Subject: [PATCH 5/6] Wait in networkquiet for what the action before it sent A click's requests start while the click runs, before a separate waitforloadstate can subscribe, so networkquiet never saw them: a navigation or a fetch slower than its 500 ms window resolved the wait on the page as it was before the action. The daemon now keeps each page session's page-changing requests that have not finished, from the events it drains. A networkquiet wait subscribes, drains what came before, and starts with those sent in the last 3 s (CDP's wallTime) as pending; a long poll older than that is not waited for, and past 256 entries the old ones are forgotten. networkidle is unchanged. A networkquiet timeout now says networkquiet, and its tests allow up to 1,050 ms, still under networkidle's 1.1 s floor, so a slow machine does not fail them. --- cli/src/native/actions.rs | 55 ++++++- cli/src/native/browser.rs | 304 ++++++++++++++++++++++++++++++++++---- 2 files changed, 328 insertions(+), 31 deletions(-) diff --git a/cli/src/native/actions.rs b/cli/src/native/actions.rs index 4768d86114..16106f6cd2 100644 --- a/cli/src/native/actions.rs +++ b/cli/src/native/actions.rs @@ -15,7 +15,7 @@ use crate::validation::{is_valid_session_name, session_name_error}; use super::a11y; use super::auth; -use super::browser::{should_track_target, BrowserManager, WaitUntil}; +use super::browser::{should_track_target, wall_now, BrowserManager, InFlight, WaitUntil}; use super::cdp::chrome::{prepare_nss_home, LaunchOptions}; use super::cdp::client::CdpClient; use super::cdp::types::{ @@ -597,6 +597,9 @@ pub struct DaemonState { pub routes: Arc>>, pub tracked_requests: Vec, pub request_tracking: bool, + /// The active page's page-changing requests not yet finished, for a + /// `networkquiet` wait that starts after the action that sent them. + pub(crate) in_flight: InFlight, pub active_frame_id: Option, /// Cross-origin iframe frame_id → dedicated CDP session_id. /// Populated by Target.attachedToTarget events from Target.setAutoAttach. @@ -848,6 +851,7 @@ impl DaemonState { routes: Arc::new(RwLock::new(Vec::new())), tracked_requests: Vec::new(), request_tracking: false, + in_flight: InFlight::default(), active_frame_id: None, iframe_sessions: HashMap::new(), active_iframe_sessions: HashSet::new(), @@ -1875,6 +1879,20 @@ impl DaemonState { false }; + if let (true, Some(browser), Some(session_id)) = ( + session_matches, + self.browser.as_ref(), + event.session_id.as_deref(), + ) { + self.in_flight.note( + session_id, + &event.method, + &event.params, + browser.page_target_id(session_id), + wall_now(), + ); + } + // Allow Network events from cross-origin iframe sessions // when HAR recording or request tracking is active. let iframe_network_event = !session_matches @@ -6751,8 +6769,7 @@ async fn handle_wait(cmd: &Value, state: &mut DaemonState) -> Result Result Result { +async fn handle_waitforloadstate(cmd: &Value, state: &mut DaemonState) -> Result { let mgr = state.browser.as_ref().ok_or("Browser not launched")?; let session_id = mgr.active_session_id()?.to_string(); let load_state = cmd.get("state").and_then(|v| v.as_str()).unwrap_or("load"); @@ -9840,7 +9857,7 @@ async fn handle_waitforloadstate(cmd: &Value, state: &DaemonState) -> Result Result Result<(), String> { + if wait_until != WaitUntil::NetworkQuiet { + let mgr = state.browser.as_ref().ok_or("Browser not launched")?; + return mgr + .wait_for_lifecycle_external(wait_until, session_id) + .await; + } + let mut rx = state + .browser + .as_ref() + .ok_or("Browser not launched")? + .client + .subscribe(); + state.drain_cdp_events_background().await?; + let in_flight = state.in_flight.recent(session_id, wall_now()); + let mgr = state.browser.as_ref().ok_or("Browser not launched")?; + mgr.wait_for_network_quiet(session_id, &mut rx, in_flight) + .await +} + async fn handle_waitforfunction(cmd: &Value, state: &DaemonState) -> Result { let mgr = state.browser.as_ref().ok_or("Browser not launched")?; let session_id = mgr.active_session_id()?.to_string(); diff --git a/cli/src/native/browser.rs b/cli/src/native/browser.rs index 6b05af31dc..b9198d39b3 100644 --- a/cli/src/native/browser.rs +++ b/cli/src/native/browser.rs @@ -351,6 +351,9 @@ pub enum WaitUntil { /// Like `NetworkIdle`, but quiet is counted from the start of the wait, /// so a page that is already idle resolves after 500 ms of silence /// rather than after a first 600 ms receive timeout and then 500 ms more. + /// It counts only the requests that can change what the page shows + /// (see `changes_page`), and a `wait --load networkquiet` also waits for + /// those an action sent moments before the wait began. NetworkQuiet, None, } @@ -1178,22 +1181,9 @@ impl BrowserManager { WaitUntil::DomContentLoaded => "Page.domContentEventFired", WaitUntil::NetworkIdle => return self.wait_for_network_idle(session_id, rx).await, WaitUntil::NetworkQuiet => { - let timeout = tokio::time::Duration::from_millis(self.default_timeout_ms); - // A page target's main frame shares its target's id. - let main_frame = self - .pages - .iter() - .find(|page| page.session_id == session_id) - .map(|page| page.target_id.clone()); - return poll_network( - session_id, - rx, - timeout, - Quiet::FromStart { - main_frame: main_frame.as_deref(), - }, - ) - .await; + return self + .wait_for_network_quiet(session_id, rx, HashSet::new()) + .await; } WaitUntil::None => return Ok(()), }; @@ -1229,6 +1219,29 @@ impl BrowserManager { poll_network_idle(session_id, rx, timeout).await } + /// Waits for `networkquiet` on `session_id`, reading events from `rx`. + /// `in_flight` holds the page-changing requests sent before `rx` + /// subscribed and not finished since: they are waited for like any + /// request `rx` sees start. + pub async fn wait_for_network_quiet( + &self, + session_id: &str, + rx: &mut broadcast::Receiver, + in_flight: HashSet, + ) -> Result<(), String> { + let timeout = tokio::time::Duration::from_millis(self.default_timeout_ms); + poll_network( + session_id, + rx, + timeout, + Quiet::FromStart { + main_frame: self.page_target_id(session_id), + }, + in_flight, + ) + .await + } + pub async fn get_url(&self) -> Result { let result = self.evaluate_simple("location.href").await?; Ok(result.as_str().unwrap_or("").to_string()) @@ -1421,6 +1434,15 @@ impl BrowserManager { self.pages.iter().any(|page| page.session_id == session_id) } + /// The target id of the page whose session is `session_id`, which is + /// also the id of its main frame. + pub fn page_target_id(&self, session_id: &str) -> Option<&str> { + self.pages + .iter() + .find(|page| page.session_id == session_id) + .map(|page| page.target_id.as_str()) + } + pub fn session_id_for_target(&self, target_id: &str) -> Option<&str> { self.pages .iter() @@ -2254,7 +2276,14 @@ async fn poll_network_idle( rx: &mut broadcast::Receiver, overall_timeout: tokio::time::Duration, ) -> Result<(), String> { - poll_network(session_id, rx, overall_timeout, Quiet::AfterSilence).await + poll_network( + session_id, + rx, + overall_timeout, + Quiet::AfterSilence, + HashSet::new(), + ) + .await } /// When a network wait starts counting quiet time, and what it counts. @@ -2288,15 +2317,96 @@ fn changes_page(params: &Value, main_frame: Option<&str>) -> bool { } } +/// How long ago, in seconds, a request still in flight may have started for +/// a `networkquiet` wait that begins after it to wait for it: the action +/// that sent it came moments before the wait. A request older than that, +/// such as a long poll, is not waited for. +const IN_FLIGHT_SECS: f64 = 3.0; + +/// More requests in flight than this, and those older than +/// [`IN_FLIGHT_SECS`] are forgotten: some never report finishing. +const IN_FLIGHT_LIMIT: usize = 256; + +/// The page-changing requests ([`changes_page`]) each page session has sent +/// and not finished, kept from the events the daemon drains, so that a +/// `networkquiet` wait that starts after an action still waits for what the +/// action sent before the wait could subscribe. +#[derive(Debug, Default)] +pub(crate) struct InFlight { + /// Request id to the session that sent it and the wall time, in seconds + /// since the epoch, it was sent at. + requests: HashMap, +} + +impl InFlight { + /// Notes `method` with `params`, an event of `session`, whose main frame + /// is `main_frame`, at wall time `now`. + pub(crate) fn note( + &mut self, + session: &str, + method: &str, + params: &Value, + main_frame: Option<&str>, + now: f64, + ) { + let Some(id) = params.get("requestId").and_then(Value::as_str) else { + return; + }; + match method { + "Network.requestWillBeSent" if changes_page(params, main_frame) => { + let sent = params + .get("wallTime") + .and_then(Value::as_f64) + .unwrap_or(now); + self.requests + .insert(id.to_string(), (session.to_string(), sent)); + if self.requests.len() > IN_FLIGHT_LIMIT { + self.requests + .retain(|_, (_, sent)| now - *sent <= IN_FLIGHT_SECS); + } + } + "Network.loadingFinished" | "Network.loadingFailed" => { + self.requests.remove(id); + } + _ => {} + } + } + + /// The requests `session` sent at most [`IN_FLIGHT_SECS`] before `now` + /// and has not finished. + pub(crate) fn recent(&self, session: &str, now: f64) -> HashSet { + self.requests + .iter() + .filter(|(_, (owner, sent))| owner == session && now - sent <= IN_FLIGHT_SECS) + .map(|(id, _)| id.clone()) + .collect() + } +} + +/// The wall clock, in seconds since the epoch, as CDP's `wallTime` counts. +pub(crate) fn wall_now() -> f64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0.0, |since| since.as_secs_f64()) +} + /// The network-idle loop: `Ok(())` once nothing has been in flight for /// 500 ms, counted as `quiet` says, or `Err` after `overall_timeout`. +/// `in_flight` are requests already in flight when the wait starts, which +/// count as pending until they finish. async fn poll_network( session_id: &str, rx: &mut broadcast::Receiver, overall_timeout: tokio::time::Duration, quiet: Quiet<'_>, + in_flight: HashSet, ) -> Result<(), String> { - let pending = Arc::new(Mutex::new(HashSet::::new())); + let state = match quiet { + Quiet::AfterSilence => "networkidle", + Quiet::FromStart { .. } => "networkquiet", + }; + let quiet_from_start = in_flight.is_empty(); + let pending = Arc::new(Mutex::new(in_flight)); let (window, counted): (u64, Option>) = match quiet { Quiet::AfterSilence => (600, None), Quiet::FromStart { main_frame } => (100, Some(main_frame)), @@ -2304,7 +2414,7 @@ async fn poll_network( tokio::time::timeout(overall_timeout, async { let mut idle_start: Option = - counted.is_some().then(tokio::time::Instant::now); + (counted.is_some() && quiet_from_start).then(tokio::time::Instant::now); loop { let recv_result = @@ -2330,8 +2440,9 @@ async fn poll_network( { let was_pending = p.remove(id); // Counting only what changes the page, a request - // not counted, or sent before the wait, ending - // says nothing about the page going quiet. + // not counted, or sent too long before the wait + // to be in flight for it, ending says nothing + // about the page going quiet. if p.is_empty() && (was_pending || counted.is_none()) { idle_start = Some(tokio::time::Instant::now()); } @@ -2369,7 +2480,7 @@ async fn poll_network( Ok(()) }) .await - .map_err(|_| "Timeout waiting for networkidle".to_string())? + .map_err(|_| format!("Timeout waiting for {state}"))? } async fn connect_cdp_with_retry( @@ -3014,13 +3125,15 @@ mod tests { &mut rx, Duration::from_secs(5), Quiet::FromStart { main_frame: None }, + HashSet::new(), ) .await; assert!(result.is_ok()); let elapsed = start.elapsed(); + // Under networkidle's 1.1 s floor, with room for a slow machine. assert!( - elapsed >= Duration::from_millis(500) && elapsed < Duration::from_millis(800), - "network quiet returned in {:?}, expected 500-800ms", + elapsed >= Duration::from_millis(500) && elapsed < Duration::from_millis(1_050), + "network quiet returned in {:?}, expected 500-1050ms", elapsed ); drop(tx); @@ -3052,6 +3165,7 @@ mod tests { &mut rx, Duration::from_secs(5), Quiet::FromStart { main_frame: None }, + HashSet::new(), ) .await; assert!(result.is_ok()); @@ -3097,17 +3211,155 @@ mod tests { Quiet::FromStart { main_frame: Some("main"), }, + HashSet::new(), + ) + .await; + assert!(result.is_ok()); + let elapsed = start.elapsed(); + // Under networkidle's 1.1 s floor, with room for a slow machine. + assert!( + elapsed >= Duration::from_millis(500) && elapsed < Duration::from_millis(1_050), + "network quiet returned in {:?}, expected 500-1050ms", + elapsed + ); + } + + /// A request sent before the wait began, and still in flight, is waited + /// for: the wait resolves 500 ms after it finishes, not 500 ms after it + /// starts. + #[tokio::test] + async fn test_network_quiet_waits_for_what_was_in_flight_before_it() { + let (tx, mut rx) = broadcast::channel::(16); + let session = "s1"; + let _keep_alive = tx.clone(); + tokio::spawn(async move { + sleep(Duration::from_millis(600)).await; + let _ = tx.send(cdp_event( + "Network.loadingFinished", + session, + json!({ "requestId": "click" }), + )); + }); + let start = tokio::time::Instant::now(); + let result = poll_network( + session, + &mut rx, + Duration::from_secs(5), + Quiet::FromStart { main_frame: None }, + HashSet::from(["click".to_string()]), ) .await; assert!(result.is_ok()); let elapsed = start.elapsed(); assert!( - elapsed >= Duration::from_millis(500) && elapsed < Duration::from_millis(800), - "network quiet returned in {:?}, expected 500-800ms", + elapsed >= Duration::from_millis(1_100), + "network quiet returned in {:?}, before the earlier request finished and 500ms passed", elapsed ); } + /// A `networkquiet` wait that runs out says so by its own name. + #[tokio::test] + async fn test_network_quiet_timeout_names_the_state() { + let (_tx, mut rx) = broadcast::channel::(16); + let result = poll_network( + "s1", + &mut rx, + Duration::from_millis(200), + Quiet::FromStart { main_frame: None }, + HashSet::from(["never".to_string()]), + ) + .await; + assert_eq!(result, Err("Timeout waiting for networkquiet".to_string())); + } + + /// What is kept in flight: a session's recent page-changing requests, + /// until they finish or fail; not a picture, another frame's document, + /// another session's request, or one sent too long ago. + #[test] + fn test_in_flight_keeps_recent_page_changing_requests() { + let now = 1_000.0; + let sent = |id: &str, kind: &str, frame: &str, ago: f64| json!({ "requestId": id, "type": kind, "frameId": frame, "wallTime": now - ago }); + let mut in_flight = InFlight::default(); + for params in [ + sent("xhr", "XHR", "main", 0.2), + sent("page", "Document", "main", 0.1), + sent("picture", "Image", "main", 0.1), + sent("ad", "Document", "ads", 0.1), + sent("poll", "Fetch", "main", 30.0), + sent("done", "Script", "main", 0.5), + sent("failed", "Fetch", "main", 0.5), + ] { + in_flight.note( + "s1", + "Network.requestWillBeSent", + ¶ms, + Some("main"), + now, + ); + } + in_flight.note( + "s2", + "Network.requestWillBeSent", + &sent("other", "XHR", "tab", 0.1), + Some("tab"), + now, + ); + in_flight.note( + "s1", + "Network.loadingFinished", + &json!({ "requestId": "done" }), + Some("main"), + now, + ); + in_flight.note( + "s1", + "Network.loadingFailed", + &json!({ "requestId": "failed" }), + Some("main"), + now, + ); + let mut recent: Vec<_> = in_flight.recent("s1", now).into_iter().collect(); + recent.sort(); + assert_eq!(recent, vec!["page".to_string(), "xhr".to_string()]); + assert_eq!( + in_flight.recent("s2", now), + HashSet::from(["other".to_string()]) + ); + // An event without a request id is not a request. + in_flight.note("s1", "Network.requestWillBeSent", &json!({}), None, now); + assert_eq!(in_flight.recent("s1", now).len(), 2); + } + + /// Requests that never report finishing do not pile up: past + /// `IN_FLIGHT_LIMIT`, those sent too long ago are forgotten. + #[test] + fn test_in_flight_forgets_old_requests_past_its_limit() { + let mut in_flight = InFlight::default(); + for id in 0..IN_FLIGHT_LIMIT { + in_flight.note( + "s1", + "Network.requestWillBeSent", + &json!({ "requestId": format!("poll-{id}"), "type": "Fetch", "wallTime": 1.0 }), + None, + 1.0, + ); + } + assert_eq!(in_flight.requests.len(), IN_FLIGHT_LIMIT); + in_flight.note( + "s1", + "Network.requestWillBeSent", + &json!({ "requestId": "fresh", "type": "XHR" }), + None, + 100.0, + ); + assert_eq!( + in_flight.requests.keys().collect::>(), + vec!["fresh"], + "a request with no wall time is sent now" + ); + } + /// The page's own document, a script, a stylesheet, and fetched data /// change the page; everything else does not. #[test] From ebf10fb3685d3b8ec3728e681d7d53eb7186a5a7 Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 7 Oct 2026 20:02:53 +0530 Subject: [PATCH 6/6] Document networkquiet in the help, the MCP tool, and the docs wait --load networkquiet and diff url --wait-until networkquiet already reached it through the CLI parser; the MCP wait_for_load enum refused it and no help or docs page named it. A test now checks that every state the MCP tool offers parses as itself. --- README.md | 7 +++++- cli/src/commands.rs | 2 +- cli/src/mcp.rs | 30 +++++++++++++++++++++++++- cli/src/output.rs | 6 ++++-- docs/src/app/commands/page.mdx | 3 ++- skill-data/core/references/commands.md | 3 ++- 6 files changed, 44 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 72fb08628f..3b03c1db2c 100644 --- a/README.md +++ b/README.md @@ -272,15 +272,20 @@ agent-browser wait --fn "window.ready === true" # Wait for JS condition # Use networkidle only when the page is known to become quiet agent-browser wait --load networkidle +# Wait until the requests that change the page are done, ignoring pings, images, and other frames +agent-browser wait --load networkquiet + # Wait for text/element to disappear agent-browser wait --fn "!document.body.innerText.includes('Loading...')" agent-browser wait "#spinner" --state hidden ``` -**Load states:** `load`, `domcontentloaded`, `networkidle` +**Load states:** `load`, `domcontentloaded`, `networkidle`, `networkquiet` After a page change, prefer a selector, text, URL, or JavaScript condition that represents the state you need. Use `load` or `domcontentloaded` when the lifecycle event is the milestone. `networkidle` is supported for pages known to become quiet, but SSE, WebSockets, polling, and long-polling can keep it from resolving. +`networkquiet` resolves once the requests that can change what the page shows have been done for 500 ms: the page's own document, scripts, stylesheets, and XHR or fetch data. Analytics pings, images, fonts, media, prefetches, and other frames' documents are not counted, so a page that keeps sending them still goes quiet. Quiet is counted from the start of the wait, and requests an action sent in the 3 seconds before the wait are waited for too. A fetch that never ends, such as a long poll, can still keep it from resolving. + ### Batch Execution Execute multiple commands in a single invocation. Commands can be passed as quoted arguments or piped as JSON via stdin. This avoids per-command process startup overhead when running multi-step workflows. diff --git a/cli/src/commands.rs b/cli/src/commands.rs index db5154a71d..5eacc46d2b 100644 --- a/cli/src/commands.rs +++ b/cli/src/commands.rs @@ -2807,7 +2807,7 @@ fn parse_diff(rest: &[&str], id: &str) -> Result { } else { return Err(ParseError::MissingArguments { context: "diff url --wait-until".to_string(), - usage: "diff url --wait-until ", + usage: "diff url --wait-until ", }); } } diff --git a/cli/src/mcp.rs b/cli/src/mcp.rs index 88e7931c6b..e81f4c22a9 100644 --- a/cli/src/mcp.rs +++ b/cli/src/mcp.rs @@ -925,7 +925,7 @@ fn tools() -> Vec { wait_tool(TOOL_WAIT_FOR_SELECTOR, "Wait for selector", "Wait for an element to appear.", json!({ "selector": selector_schema() }), &["selector"]), wait_tool(TOOL_WAIT_FOR_TEXT, "Wait for text", "Wait for text to appear.", json!({ "text": { "type": "string" } }), &["text"]), wait_tool(TOOL_WAIT_FOR_URL, "Wait for URL", "Wait for the current URL to match a pattern.", json!({ "url": { "type": "string", "description": "URL glob or pattern." } }), &["url"]), - wait_tool(TOOL_WAIT_FOR_LOAD, "Wait for load state", "Wait for a page load state.", json!({ "state": { "type": "string", "enum": ["load", "domcontentloaded", "networkidle"] } }), &["state"]), + wait_tool(TOOL_WAIT_FOR_LOAD, "Wait for load state", "Wait for a page load state.", json!({ "state": { "type": "string", "enum": ["load", "domcontentloaded", "networkidle", "networkquiet"] } }), &["state"]), wait_tool(TOOL_WAIT_FOR_FUNCTION, "Wait for function", "Wait for a JavaScript expression to become truthy.", json!({ "expression": { "type": "string" } }), &["expression"]), tool( TOOL_SCREENSHOT, @@ -4850,6 +4850,34 @@ mod tests { assert_eq!(modes, &vec![json!("all"), json!("text"), json!("none")]); } + #[test] + fn tool_schema_wait_for_load_states_match_cli() { + use crate::native::browser::WaitUntil; + let tools = tools(); + let wait_for_load = tools + .iter() + .find(|t| t["name"].as_str() == Some(TOOL_WAIT_FOR_LOAD)) + .unwrap(); + let states = wait_for_load["inputSchema"]["properties"]["state"]["enum"] + .as_array() + .unwrap(); + // Every state the CLI's `wait --load` reads, and each read as itself + // rather than falling back to `load`. + let read: Vec<_> = states + .iter() + .map(|state| WaitUntil::from_str(state.as_str().unwrap())) + .collect(); + assert_eq!( + read, + vec![ + WaitUntil::Load, + WaitUntil::DomContentLoaded, + WaitUntil::NetworkIdle, + WaitUntil::NetworkQuiet, + ] + ); + } + #[test] fn record_urls_preserve_navigation_schemes() { for url in ["data:text/html,hello", "about:blank", "https://example.com"] { diff --git a/cli/src/output.rs b/cli/src/output.rs index a9bd0da005..897bf1de80 100644 --- a/cli/src/output.rs +++ b/cli/src/output.rs @@ -2127,7 +2127,7 @@ Modes: Wait for element to appear Wait for specified milliseconds --url Wait for URL to match pattern - --load Wait for load state (load, domcontentloaded, networkidle) + --load Wait for load state (load, domcontentloaded, networkidle, networkquiet) --fn Wait for JavaScript expression to be truthy --text Wait for text to appear on page (substring match) --download [path] Wait for a download to complete (optionally save to path) @@ -2151,6 +2151,8 @@ Examples: agent-browser wait --url "**/dashboard" # Use networkidle only for pages known to become quiet: agent-browser wait --load networkidle + # Wait for the requests that change the page (not pings or images): + agent-browser wait --load networkquiet agent-browser wait --fn "window.appReady === true" agent-browser wait --text "Welcome back" agent-browser wait --download ./file.pdf @@ -3471,7 +3473,7 @@ URL Diff: Options: --screenshot Also compare screenshots (default: snapshot only) --full Full page screenshots - --wait-until Navigation wait strategy: load, domcontentloaded, networkidle (default: load) + --wait-until Navigation wait strategy: load, domcontentloaded, networkidle, networkquiet (default: load) -s, --selector Scope snapshots to a CSS selector or @ref -c, --compact Use compact snapshot format -d, --depth Limit snapshot tree depth diff --git a/docs/src/app/commands/page.mdx b/docs/src/app/commands/page.mdx index 618a402bcb..743108e2c3 100644 --- a/docs/src/app/commands/page.mdx +++ b/docs/src/app/commands/page.mdx @@ -134,13 +134,14 @@ agent-browser wait --url "**/dash" # Wait for URL pattern agent-browser wait --load domcontentloaded # Wait for DOMContentLoaded agent-browser wait --load load # Wait for the load event agent-browser wait --load networkidle # Use only for pages known to become quiet +agent-browser wait --load networkquiet # Wait for the requests that change the page agent-browser wait --fn "condition" # Wait for JS condition agent-browser wait --download [path] # Wait for download agent-browser wait --fn "!document.body.innerText.includes('Loading...')" # Wait for text to disappear agent-browser wait "#spinner" --state hidden # Wait for element to disappear ``` -After a page change, prefer a selector, text, URL, or JavaScript condition that represents the state you need. Use `networkidle` only when the page is known to become quiet, because SSE, WebSockets, polling, and long-polling can keep it from resolving. +After a page change, prefer a selector, text, URL, or JavaScript condition that represents the state you need. Use `networkidle` only when the page is known to become quiet, because SSE, WebSockets, polling, and long-polling can keep it from resolving. `networkquiet` counts only the requests that can change what the page shows (the page's document, scripts, stylesheets, XHR and fetch), so analytics pings, images, and other frames' documents do not hold it up. ## Downloads diff --git a/skill-data/core/references/commands.md b/skill-data/core/references/commands.md index 613dfd3b02..360c14acc8 100644 --- a/skill-data/core/references/commands.md +++ b/skill-data/core/references/commands.md @@ -148,10 +148,11 @@ agent-browser wait --url "**/dashboard" # Wait for URL pattern (or -u) agent-browser wait --load domcontentloaded # Wait for DOMContentLoaded (or -l) agent-browser wait --load load # Wait for the load event agent-browser wait --load networkidle # Wait for network idle on known-quiet pages +agent-browser wait --load networkquiet # Wait for the requests that change the page (not pings or images) agent-browser wait --fn "window.ready" # Wait for JS condition (or -f) ``` -After a page-changing action, prefer the selector, text, URL, or JavaScript condition that represents the result you need. Use a lifecycle wait when the lifecycle event is the milestone. `networkidle` is also supported, but use it only for pages known to become quiet because SSE, WebSockets, polling, and long-polling can keep it from resolving. +After a page-changing action, prefer the selector, text, URL, or JavaScript condition that represents the result you need. Use a lifecycle wait when the lifecycle event is the milestone. `networkidle` is also supported, but use it only for pages known to become quiet because SSE, WebSockets, polling, and long-polling can keep it from resolving. `networkquiet` counts only the requests that can change what the page shows (the page's document, scripts, stylesheets, XHR and fetch), so analytics pings and images do not hold it up. ## Mouse Control