diff --git a/README.md b/README.md index 72fb08628f..3b03c1db2c 100644 --- a/README.md +++ b/README.md @@ -272,15 +272,20 @@ agent-browser wait --fn "window.ready === true" # Wait for JS condition # Use networkidle only when the page is known to become quiet agent-browser wait --load networkidle +# Wait until the requests that change the page are done, ignoring pings, images, and other frames +agent-browser wait --load networkquiet + # Wait for text/element to disappear agent-browser wait --fn "!document.body.innerText.includes('Loading...')" agent-browser wait "#spinner" --state hidden ``` -**Load states:** `load`, `domcontentloaded`, `networkidle` +**Load states:** `load`, `domcontentloaded`, `networkidle`, `networkquiet` After a page change, prefer a selector, text, URL, or JavaScript condition that represents the state you need. Use `load` or `domcontentloaded` when the lifecycle event is the milestone. `networkidle` is supported for pages known to become quiet, but SSE, WebSockets, polling, and long-polling can keep it from resolving. +`networkquiet` resolves once the requests that can change what the page shows have been done for 500 ms: the page's own document, scripts, stylesheets, and XHR or fetch data. Analytics pings, images, fonts, media, prefetches, and other frames' documents are not counted, so a page that keeps sending them still goes quiet. Quiet is counted from the start of the wait, and requests an action sent in the 3 seconds before the wait are waited for too. A fetch that never ends, such as a long poll, can still keep it from resolving. + ### Batch Execution Execute multiple commands in a single invocation. Commands can be passed as quoted arguments or piped as JSON via stdin. This avoids per-command process startup overhead when running multi-step workflows. diff --git a/cli/src/commands.rs b/cli/src/commands.rs index db5154a71d..5eacc46d2b 100644 --- a/cli/src/commands.rs +++ b/cli/src/commands.rs @@ -2807,7 +2807,7 @@ fn parse_diff(rest: &[&str], id: &str) -> Result { } else { return Err(ParseError::MissingArguments { context: "diff url --wait-until".to_string(), - usage: "diff url --wait-until ", + usage: "diff url --wait-until ", }); } } diff --git a/cli/src/mcp.rs b/cli/src/mcp.rs index 88e7931c6b..e81f4c22a9 100644 --- a/cli/src/mcp.rs +++ b/cli/src/mcp.rs @@ -925,7 +925,7 @@ fn tools() -> Vec { wait_tool(TOOL_WAIT_FOR_SELECTOR, "Wait for selector", "Wait for an element to appear.", json!({ "selector": selector_schema() }), &["selector"]), wait_tool(TOOL_WAIT_FOR_TEXT, "Wait for text", "Wait for text to appear.", json!({ "text": { "type": "string" } }), &["text"]), wait_tool(TOOL_WAIT_FOR_URL, "Wait for URL", "Wait for the current URL to match a pattern.", json!({ "url": { "type": "string", "description": "URL glob or pattern." } }), &["url"]), - wait_tool(TOOL_WAIT_FOR_LOAD, "Wait for load state", "Wait for a page load state.", json!({ "state": { "type": "string", "enum": ["load", "domcontentloaded", "networkidle"] } }), &["state"]), + wait_tool(TOOL_WAIT_FOR_LOAD, "Wait for load state", "Wait for a page load state.", json!({ "state": { "type": "string", "enum": ["load", "domcontentloaded", "networkidle", "networkquiet"] } }), &["state"]), wait_tool(TOOL_WAIT_FOR_FUNCTION, "Wait for function", "Wait for a JavaScript expression to become truthy.", json!({ "expression": { "type": "string" } }), &["expression"]), tool( TOOL_SCREENSHOT, @@ -4850,6 +4850,34 @@ mod tests { assert_eq!(modes, &vec![json!("all"), json!("text"), json!("none")]); } + #[test] + fn tool_schema_wait_for_load_states_match_cli() { + use crate::native::browser::WaitUntil; + let tools = tools(); + let wait_for_load = tools + .iter() + .find(|t| t["name"].as_str() == Some(TOOL_WAIT_FOR_LOAD)) + .unwrap(); + let states = wait_for_load["inputSchema"]["properties"]["state"]["enum"] + .as_array() + .unwrap(); + // Every state the CLI's `wait --load` reads, and each read as itself + // rather than falling back to `load`. + let read: Vec<_> = states + .iter() + .map(|state| WaitUntil::from_str(state.as_str().unwrap())) + .collect(); + assert_eq!( + read, + vec![ + WaitUntil::Load, + WaitUntil::DomContentLoaded, + WaitUntil::NetworkIdle, + WaitUntil::NetworkQuiet, + ] + ); + } + #[test] fn record_urls_preserve_navigation_schemes() { for url in ["data:text/html,hello", "about:blank", "https://example.com"] { diff --git a/cli/src/native/actions.rs b/cli/src/native/actions.rs index 4768d86114..16106f6cd2 100644 --- a/cli/src/native/actions.rs +++ b/cli/src/native/actions.rs @@ -15,7 +15,7 @@ use crate::validation::{is_valid_session_name, session_name_error}; use super::a11y; use super::auth; -use super::browser::{should_track_target, BrowserManager, WaitUntil}; +use super::browser::{should_track_target, wall_now, BrowserManager, InFlight, WaitUntil}; use super::cdp::chrome::{prepare_nss_home, LaunchOptions}; use super::cdp::client::CdpClient; use super::cdp::types::{ @@ -597,6 +597,9 @@ pub struct DaemonState { pub routes: Arc>>, pub tracked_requests: Vec, pub request_tracking: bool, + /// The active page's page-changing requests not yet finished, for a + /// `networkquiet` wait that starts after the action that sent them. + pub(crate) in_flight: InFlight, pub active_frame_id: Option, /// Cross-origin iframe frame_id → dedicated CDP session_id. /// Populated by Target.attachedToTarget events from Target.setAutoAttach. @@ -848,6 +851,7 @@ impl DaemonState { routes: Arc::new(RwLock::new(Vec::new())), tracked_requests: Vec::new(), request_tracking: false, + in_flight: InFlight::default(), active_frame_id: None, iframe_sessions: HashMap::new(), active_iframe_sessions: HashSet::new(), @@ -1875,6 +1879,20 @@ impl DaemonState { false }; + if let (true, Some(browser), Some(session_id)) = ( + session_matches, + self.browser.as_ref(), + event.session_id.as_deref(), + ) { + self.in_flight.note( + session_id, + &event.method, + &event.params, + browser.page_target_id(session_id), + wall_now(), + ); + } + // Allow Network events from cross-origin iframe sessions // when HAR recording or request tracking is active. let iframe_network_event = !session_matches @@ -6751,8 +6769,7 @@ async fn handle_wait(cmd: &Value, state: &mut DaemonState) -> Result Result Result { +async fn handle_waitforloadstate(cmd: &Value, state: &mut DaemonState) -> Result { let mgr = state.browser.as_ref().ok_or("Browser not launched")?; let session_id = mgr.active_session_id()?.to_string(); let load_state = cmd.get("state").and_then(|v| v.as_str()).unwrap_or("load"); @@ -9840,7 +9857,7 @@ async fn handle_waitforloadstate(cmd: &Value, state: &DaemonState) -> Result Result Result<(), String> { + if wait_until != WaitUntil::NetworkQuiet { + let mgr = state.browser.as_ref().ok_or("Browser not launched")?; + return mgr + .wait_for_lifecycle_external(wait_until, session_id) + .await; + } + let mut rx = state + .browser + .as_ref() + .ok_or("Browser not launched")? + .client + .subscribe(); + state.drain_cdp_events_background().await?; + let in_flight = state.in_flight.recent(session_id, wall_now()); + let mgr = state.browser.as_ref().ok_or("Browser not launched")?; + mgr.wait_for_network_quiet(session_id, &mut rx, in_flight) + .await +} + async fn handle_waitforfunction(cmd: &Value, state: &DaemonState) -> Result { let mgr = state.browser.as_ref().ok_or("Browser not launched")?; let session_id = mgr.active_session_id()?.to_string(); diff --git a/cli/src/native/browser.rs b/cli/src/native/browser.rs index 5952b192b1..b9198d39b3 100644 --- a/cli/src/native/browser.rs +++ b/cli/src/native/browser.rs @@ -348,6 +348,13 @@ pub enum WaitUntil { Load, DomContentLoaded, NetworkIdle, + /// Like `NetworkIdle`, but quiet is counted from the start of the wait, + /// so a page that is already idle resolves after 500 ms of silence + /// rather than after a first 600 ms receive timeout and then 500 ms more. + /// It counts only the requests that can change what the page shows + /// (see `changes_page`), and a `wait --load networkquiet` also waits for + /// those an action sent moments before the wait began. + NetworkQuiet, None, } @@ -356,6 +363,7 @@ impl WaitUntil { match s { "domcontentloaded" => Self::DomContentLoaded, "networkidle" => Self::NetworkIdle, + "networkquiet" => Self::NetworkQuiet, "none" => Self::None, _ => Self::Load, } @@ -1172,6 +1180,11 @@ impl BrowserManager { WaitUntil::Load => "Page.loadEventFired", WaitUntil::DomContentLoaded => "Page.domContentEventFired", WaitUntil::NetworkIdle => return self.wait_for_network_idle(session_id, rx).await, + WaitUntil::NetworkQuiet => { + return self + .wait_for_network_quiet(session_id, rx, HashSet::new()) + .await; + } WaitUntil::None => return Ok(()), }; @@ -1206,6 +1219,29 @@ impl BrowserManager { poll_network_idle(session_id, rx, timeout).await } + /// Waits for `networkquiet` on `session_id`, reading events from `rx`. + /// `in_flight` holds the page-changing requests sent before `rx` + /// subscribed and not finished since: they are waited for like any + /// request `rx` sees start. + pub async fn wait_for_network_quiet( + &self, + session_id: &str, + rx: &mut broadcast::Receiver, + in_flight: HashSet, + ) -> Result<(), String> { + let timeout = tokio::time::Duration::from_millis(self.default_timeout_ms); + poll_network( + session_id, + rx, + timeout, + Quiet::FromStart { + main_frame: self.page_target_id(session_id), + }, + in_flight, + ) + .await + } + pub async fn get_url(&self) -> Result { let result = self.evaluate_simple("location.href").await?; Ok(result.as_str().unwrap_or("").to_string()) @@ -1276,7 +1312,7 @@ impl BrowserManager { WaitUntil::DomContentLoaded => Some("document.readyState !== 'loading'"), // Network idle is tracked from live network events; its poller // already treats a quiet stream as idle. - WaitUntil::NetworkIdle | WaitUntil::None => None, + WaitUntil::NetworkIdle | WaitUntil::NetworkQuiet | WaitUntil::None => None, }; if let Some(expression) = already_reached { let probe: Result = self @@ -1398,6 +1434,15 @@ impl BrowserManager { self.pages.iter().any(|page| page.session_id == session_id) } + /// The target id of the page whose session is `session_id`, which is + /// also the id of its main frame. + pub fn page_target_id(&self, session_id: &str) -> Option<&str> { + self.pages + .iter() + .find(|page| page.session_id == session_id) + .map(|page| page.target_id.as_str()) + } + pub fn session_id_for_target(&self, target_id: &str) -> Option<&str> { self.pages .iter() @@ -2231,31 +2276,174 @@ async fn poll_network_idle( rx: &mut broadcast::Receiver, overall_timeout: tokio::time::Duration, ) -> Result<(), String> { - let pending = Arc::new(Mutex::new(HashSet::::new())); + poll_network( + session_id, + rx, + overall_timeout, + Quiet::AfterSilence, + HashSet::new(), + ) + .await +} + +/// When a network wait starts counting quiet time, and what it counts. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Quiet<'a> { + /// After a first receive window passes with nothing in flight: an idle + /// page resolves after about 1.1 s (`networkidle`). + AfterSilence, + /// From the start of the wait, polling every 100 ms, and counting only + /// the requests that can change what the page shows ([`changes_page`]), + /// `main_frame`'s documents among them: an idle page resolves after + /// 500 ms of silence (`networkquiet`). + FromStart { main_frame: Option<&'a str> }, +} + +/// Whether a request can change what the page shows: the page's own +/// document, a script, a stylesheet, or data it fetches. An analytics ping, +/// a picture, a font, media, a prefetch, a stream, and a document of +/// another frame cannot; some of them (a ping, the document of a frame in +/// another process) never even report finishing to the page's session, so +/// a page that sent one would never go quiet. A request of no known type +/// counts. +fn changes_page(params: &Value, main_frame: Option<&str>) -> bool { + match params.get("type").and_then(Value::as_str) { + Some("XHR" | "Fetch" | "Script" | "Stylesheet") | None => true, + Some("Document") => match (main_frame, params.get("frameId").and_then(Value::as_str)) { + (Some(main), Some(frame)) => main == frame, + _ => true, + }, + Some(_) => false, + } +} + +/// How long ago, in seconds, a request still in flight may have started for +/// a `networkquiet` wait that begins after it to wait for it: the action +/// that sent it came moments before the wait. A request older than that, +/// such as a long poll, is not waited for. +const IN_FLIGHT_SECS: f64 = 3.0; + +/// More requests in flight than this, and those older than +/// [`IN_FLIGHT_SECS`] are forgotten: some never report finishing. +const IN_FLIGHT_LIMIT: usize = 256; + +/// The page-changing requests ([`changes_page`]) each page session has sent +/// and not finished, kept from the events the daemon drains, so that a +/// `networkquiet` wait that starts after an action still waits for what the +/// action sent before the wait could subscribe. +#[derive(Debug, Default)] +pub(crate) struct InFlight { + /// Request id to the session that sent it and the wall time, in seconds + /// since the epoch, it was sent at. + requests: HashMap, +} + +impl InFlight { + /// Notes `method` with `params`, an event of `session`, whose main frame + /// is `main_frame`, at wall time `now`. + pub(crate) fn note( + &mut self, + session: &str, + method: &str, + params: &Value, + main_frame: Option<&str>, + now: f64, + ) { + let Some(id) = params.get("requestId").and_then(Value::as_str) else { + return; + }; + match method { + "Network.requestWillBeSent" if changes_page(params, main_frame) => { + let sent = params + .get("wallTime") + .and_then(Value::as_f64) + .unwrap_or(now); + self.requests + .insert(id.to_string(), (session.to_string(), sent)); + if self.requests.len() > IN_FLIGHT_LIMIT { + self.requests + .retain(|_, (_, sent)| now - *sent <= IN_FLIGHT_SECS); + } + } + "Network.loadingFinished" | "Network.loadingFailed" => { + self.requests.remove(id); + } + _ => {} + } + } + + /// The requests `session` sent at most [`IN_FLIGHT_SECS`] before `now` + /// and has not finished. + pub(crate) fn recent(&self, session: &str, now: f64) -> HashSet { + self.requests + .iter() + .filter(|(_, (owner, sent))| owner == session && now - sent <= IN_FLIGHT_SECS) + .map(|(id, _)| id.clone()) + .collect() + } +} + +/// The wall clock, in seconds since the epoch, as CDP's `wallTime` counts. +pub(crate) fn wall_now() -> f64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0.0, |since| since.as_secs_f64()) +} + +/// The network-idle loop: `Ok(())` once nothing has been in flight for +/// 500 ms, counted as `quiet` says, or `Err` after `overall_timeout`. +/// `in_flight` are requests already in flight when the wait starts, which +/// count as pending until they finish. +async fn poll_network( + session_id: &str, + rx: &mut broadcast::Receiver, + overall_timeout: tokio::time::Duration, + quiet: Quiet<'_>, + in_flight: HashSet, +) -> Result<(), String> { + let state = match quiet { + Quiet::AfterSilence => "networkidle", + Quiet::FromStart { .. } => "networkquiet", + }; + let quiet_from_start = in_flight.is_empty(); + let pending = Arc::new(Mutex::new(in_flight)); + let (window, counted): (u64, Option>) = match quiet { + Quiet::AfterSilence => (600, None), + Quiet::FromStart { main_frame } => (100, Some(main_frame)), + }; tokio::time::timeout(overall_timeout, async { - let mut idle_start: Option = None; + let mut idle_start: Option = + (counted.is_some() && quiet_from_start).then(tokio::time::Instant::now); loop { let recv_result = - tokio::time::timeout(tokio::time::Duration::from_millis(600), rx.recv()).await; + tokio::time::timeout(tokio::time::Duration::from_millis(window), rx.recv()).await; match recv_result { Ok(Ok(event)) if event.session_id.as_deref() == Some(session_id) => { let mut p = pending.lock().await; match event.method.as_str() { "Network.requestWillBeSent" => { + let counts = counted + .is_none_or(|main_frame| changes_page(&event.params, main_frame)); if let Some(id) = event.params.get("requestId").and_then(|v| v.as_str()) { - p.insert(id.to_string()); - idle_start = None; + if counts { + p.insert(id.to_string()); + idle_start = None; + } } } "Network.loadingFinished" | "Network.loadingFailed" => { if let Some(id) = event.params.get("requestId").and_then(|v| v.as_str()) { - p.remove(id); - if p.is_empty() { + let was_pending = p.remove(id); + // Counting only what changes the page, a request + // not counted, or sent too long before the wait + // to be in flight for it, ending says nothing + // about the page going quiet. + if p.is_empty() && (was_pending || counted.is_none()) { idle_start = Some(tokio::time::Instant::now()); } } @@ -2292,7 +2480,7 @@ async fn poll_network_idle( Ok(()) }) .await - .map_err(|_| "Timeout waiting for networkidle".to_string())? + .map_err(|_| format!("Timeout waiting for {state}"))? } async fn connect_cdp_with_retry( @@ -2925,6 +3113,288 @@ mod tests { drop(tx); } + /// `networkquiet` counts quiet from the start: a silent stream resolves + /// after the 500 ms window (never instantly, as #846 requires), well + /// before `networkidle`'s first 600 ms receive timeout plus 500 ms. + #[tokio::test] + async fn test_network_quiet_counts_from_the_start() { + let (tx, mut rx) = broadcast::channel::(16); + let start = tokio::time::Instant::now(); + let result = poll_network( + "s1", + &mut rx, + Duration::from_secs(5), + Quiet::FromStart { main_frame: None }, + HashSet::new(), + ) + .await; + assert!(result.is_ok()); + let elapsed = start.elapsed(); + // Under networkidle's 1.1 s floor, with room for a slow machine. + assert!( + elapsed >= Duration::from_millis(500) && elapsed < Duration::from_millis(1_050), + "network quiet returned in {:?}, expected 500-1050ms", + elapsed + ); + drop(tx); + } + + /// `networkquiet` still waits out a request in flight, then 500 ms more. + #[tokio::test] + async fn test_network_quiet_waits_for_requests_in_flight() { + let (tx, mut rx) = broadcast::channel::(16); + let session = "s1"; + let _keep_alive = tx.clone(); + tokio::spawn(async move { + sleep(Duration::from_millis(50)).await; + let _ = tx.send(cdp_event( + "Network.requestWillBeSent", + session, + json!({ "requestId": "r1" }), + )); + sleep(Duration::from_millis(600)).await; + let _ = tx.send(cdp_event( + "Network.loadingFinished", + session, + json!({ "requestId": "r1" }), + )); + }); + let start = tokio::time::Instant::now(); + let result = poll_network( + session, + &mut rx, + Duration::from_secs(5), + Quiet::FromStart { main_frame: None }, + HashSet::new(), + ) + .await; + assert!(result.is_ok()); + let elapsed = start.elapsed(); + assert!( + elapsed >= Duration::from_millis(1150), + "network quiet returned in {:?}, before the request finished and 500ms passed", + elapsed + ); + } + + /// `networkquiet` counts only what can change the page: a ping, a + /// picture, and another frame's document that never report finishing + /// leave it quiet, and so does the end of a request it never saw start. + #[tokio::test] + async fn test_network_quiet_counts_only_what_changes_the_page() { + let (tx, mut rx) = broadcast::channel::(16); + let session = "s1"; + let _keep_alive = tx.clone(); + tokio::spawn(async move { + sleep(Duration::from_millis(50)).await; + for params in [ + json!({ "requestId": "ping", "type": "Ping" }), + json!({ "requestId": "image", "type": "Image" }), + json!({ "requestId": "frame", "type": "Document", "frameId": "ads" }), + ] { + let _ = tx.send(cdp_event("Network.requestWillBeSent", session, params)); + } + for later in 0..4 { + sleep(Duration::from_millis(150)).await; + let _ = tx.send(cdp_event( + "Network.loadingFinished", + session, + json!({ "requestId": format!("before-{later}") }), + )); + } + }); + let start = tokio::time::Instant::now(); + let result = poll_network( + session, + &mut rx, + Duration::from_secs(5), + Quiet::FromStart { + main_frame: Some("main"), + }, + HashSet::new(), + ) + .await; + assert!(result.is_ok()); + let elapsed = start.elapsed(); + // Under networkidle's 1.1 s floor, with room for a slow machine. + assert!( + elapsed >= Duration::from_millis(500) && elapsed < Duration::from_millis(1_050), + "network quiet returned in {:?}, expected 500-1050ms", + elapsed + ); + } + + /// A request sent before the wait began, and still in flight, is waited + /// for: the wait resolves 500 ms after it finishes, not 500 ms after it + /// starts. + #[tokio::test] + async fn test_network_quiet_waits_for_what_was_in_flight_before_it() { + let (tx, mut rx) = broadcast::channel::(16); + let session = "s1"; + let _keep_alive = tx.clone(); + tokio::spawn(async move { + sleep(Duration::from_millis(600)).await; + let _ = tx.send(cdp_event( + "Network.loadingFinished", + session, + json!({ "requestId": "click" }), + )); + }); + let start = tokio::time::Instant::now(); + let result = poll_network( + session, + &mut rx, + Duration::from_secs(5), + Quiet::FromStart { main_frame: None }, + HashSet::from(["click".to_string()]), + ) + .await; + assert!(result.is_ok()); + let elapsed = start.elapsed(); + assert!( + elapsed >= Duration::from_millis(1_100), + "network quiet returned in {:?}, before the earlier request finished and 500ms passed", + elapsed + ); + } + + /// A `networkquiet` wait that runs out says so by its own name. + #[tokio::test] + async fn test_network_quiet_timeout_names_the_state() { + let (_tx, mut rx) = broadcast::channel::(16); + let result = poll_network( + "s1", + &mut rx, + Duration::from_millis(200), + Quiet::FromStart { main_frame: None }, + HashSet::from(["never".to_string()]), + ) + .await; + assert_eq!(result, Err("Timeout waiting for networkquiet".to_string())); + } + + /// What is kept in flight: a session's recent page-changing requests, + /// until they finish or fail; not a picture, another frame's document, + /// another session's request, or one sent too long ago. + #[test] + fn test_in_flight_keeps_recent_page_changing_requests() { + let now = 1_000.0; + let sent = |id: &str, kind: &str, frame: &str, ago: f64| json!({ "requestId": id, "type": kind, "frameId": frame, "wallTime": now - ago }); + let mut in_flight = InFlight::default(); + for params in [ + sent("xhr", "XHR", "main", 0.2), + sent("page", "Document", "main", 0.1), + sent("picture", "Image", "main", 0.1), + sent("ad", "Document", "ads", 0.1), + sent("poll", "Fetch", "main", 30.0), + sent("done", "Script", "main", 0.5), + sent("failed", "Fetch", "main", 0.5), + ] { + in_flight.note( + "s1", + "Network.requestWillBeSent", + ¶ms, + Some("main"), + now, + ); + } + in_flight.note( + "s2", + "Network.requestWillBeSent", + &sent("other", "XHR", "tab", 0.1), + Some("tab"), + now, + ); + in_flight.note( + "s1", + "Network.loadingFinished", + &json!({ "requestId": "done" }), + Some("main"), + now, + ); + in_flight.note( + "s1", + "Network.loadingFailed", + &json!({ "requestId": "failed" }), + Some("main"), + now, + ); + let mut recent: Vec<_> = in_flight.recent("s1", now).into_iter().collect(); + recent.sort(); + assert_eq!(recent, vec!["page".to_string(), "xhr".to_string()]); + assert_eq!( + in_flight.recent("s2", now), + HashSet::from(["other".to_string()]) + ); + // An event without a request id is not a request. + in_flight.note("s1", "Network.requestWillBeSent", &json!({}), None, now); + assert_eq!(in_flight.recent("s1", now).len(), 2); + } + + /// Requests that never report finishing do not pile up: past + /// `IN_FLIGHT_LIMIT`, those sent too long ago are forgotten. + #[test] + fn test_in_flight_forgets_old_requests_past_its_limit() { + let mut in_flight = InFlight::default(); + for id in 0..IN_FLIGHT_LIMIT { + in_flight.note( + "s1", + "Network.requestWillBeSent", + &json!({ "requestId": format!("poll-{id}"), "type": "Fetch", "wallTime": 1.0 }), + None, + 1.0, + ); + } + assert_eq!(in_flight.requests.len(), IN_FLIGHT_LIMIT); + in_flight.note( + "s1", + "Network.requestWillBeSent", + &json!({ "requestId": "fresh", "type": "XHR" }), + None, + 100.0, + ); + assert_eq!( + in_flight.requests.keys().collect::>(), + vec!["fresh"], + "a request with no wall time is sent now" + ); + } + + /// The page's own document, a script, a stylesheet, and fetched data + /// change the page; everything else does not. + #[test] + fn test_changes_page_reads_the_request_type_and_frame() { + let request = |kind: &str, frame: &str| json!({ "type": kind, "frameId": frame }); + for kind in ["XHR", "Fetch", "Script", "Stylesheet"] { + assert!(changes_page(&request(kind, "ads"), Some("main")), "{kind}"); + } + assert!(changes_page(&request("Document", "main"), Some("main"))); + assert!(!changes_page(&request("Document", "ads"), Some("main"))); + assert!(changes_page(&request("Document", "ads"), None)); + for kind in [ + "Ping", + "Image", + "Font", + "Media", + "Other", + "Prefetch", + "EventSource", + ] { + assert!( + !changes_page(&request(kind, "main"), Some("main")), + "{kind}" + ); + } + assert!(changes_page(&json!({}), Some("main")), "no type: counted"); + } + + /// `WaitUntil` reads `networkquiet` as its own state. + #[test] + fn test_wait_until_reads_network_quiet() { + assert!(WaitUntil::from_str("networkquiet") == WaitUntil::NetworkQuiet); + assert!(WaitUntil::from_str("networkidle") == WaitUntil::NetworkIdle); + } + /// Normal flow: requests start and finish, idle is detected after the last /// request completes and 500 ms of silence passes. #[tokio::test] diff --git a/cli/src/native/snapshot.rs b/cli/src/native/snapshot.rs index 80c5de0ab1..80a39dbad6 100644 --- a/cli/src/native/snapshot.rs +++ b/cli/src/native/snapshot.rs @@ -374,12 +374,10 @@ pub async fn take_snapshot( .object_id .ok_or_else(|| format!("Selector '{}' did not match any element", selector))?; - // Request the full DOM subtree (depth: -1) so we can collect all - // backendNodeIds that live under the matched element. let describe: Value = client .send_command( "DOM.describeNode", - Some(serde_json::json!({ "objectId": object_id, "depth": -1 })), + Some(describe_subtree_params(&object_id)), Some(session_id), ) .await?; @@ -1477,6 +1475,15 @@ fn build_dedup_set(ref_map: &RefMap) -> std::collections::HashSet { .collect() } +/// `DOM.describeNode` parameters for the whole DOM subtree (depth -1) of +/// `object_id`, so every backendNodeId under the matched element can be +/// collected, through shadow roots too: a web component's controls live in +/// its shadow root, and without `pierce` the subtree stops at the host (on +/// a live page, a consent banner's host gave no node with its buttons). +fn describe_subtree_params(object_id: &str) -> Value { + serde_json::json!({ "objectId": object_id, "depth": -1, "pierce": true }) +} + /// Recursively collect all `backendNodeId` values from a CDP DOM node tree /// (as returned by `DOM.describeNode` with `depth: -1`). fn collect_backend_node_ids(node: &Value, ids: &mut std::collections::HashSet) { @@ -1501,6 +1508,18 @@ fn collect_backend_node_ids(node: &Value, ids: &mut std::collections::HashSet Wait for element to appear Wait for specified milliseconds --url Wait for URL to match pattern - --load Wait for load state (load, domcontentloaded, networkidle) + --load Wait for load state (load, domcontentloaded, networkidle, networkquiet) --fn Wait for JavaScript expression to be truthy --text Wait for text to appear on page (substring match) --download [path] Wait for a download to complete (optionally save to path) @@ -2151,6 +2151,8 @@ Examples: agent-browser wait --url "**/dashboard" # Use networkidle only for pages known to become quiet: agent-browser wait --load networkidle + # Wait for the requests that change the page (not pings or images): + agent-browser wait --load networkquiet agent-browser wait --fn "window.appReady === true" agent-browser wait --text "Welcome back" agent-browser wait --download ./file.pdf @@ -3471,7 +3473,7 @@ URL Diff: Options: --screenshot Also compare screenshots (default: snapshot only) --full Full page screenshots - --wait-until Navigation wait strategy: load, domcontentloaded, networkidle (default: load) + --wait-until Navigation wait strategy: load, domcontentloaded, networkidle, networkquiet (default: load) -s, --selector Scope snapshots to a CSS selector or @ref -c, --compact Use compact snapshot format -d, --depth Limit snapshot tree depth diff --git a/docs/src/app/commands/page.mdx b/docs/src/app/commands/page.mdx index 618a402bcb..743108e2c3 100644 --- a/docs/src/app/commands/page.mdx +++ b/docs/src/app/commands/page.mdx @@ -134,13 +134,14 @@ agent-browser wait --url "**/dash" # Wait for URL pattern agent-browser wait --load domcontentloaded # Wait for DOMContentLoaded agent-browser wait --load load # Wait for the load event agent-browser wait --load networkidle # Use only for pages known to become quiet +agent-browser wait --load networkquiet # Wait for the requests that change the page agent-browser wait --fn "condition" # Wait for JS condition agent-browser wait --download [path] # Wait for download agent-browser wait --fn "!document.body.innerText.includes('Loading...')" # Wait for text to disappear agent-browser wait "#spinner" --state hidden # Wait for element to disappear ``` -After a page change, prefer a selector, text, URL, or JavaScript condition that represents the state you need. Use `networkidle` only when the page is known to become quiet, because SSE, WebSockets, polling, and long-polling can keep it from resolving. +After a page change, prefer a selector, text, URL, or JavaScript condition that represents the state you need. Use `networkidle` only when the page is known to become quiet, because SSE, WebSockets, polling, and long-polling can keep it from resolving. `networkquiet` counts only the requests that can change what the page shows (the page's document, scripts, stylesheets, XHR and fetch), so analytics pings, images, and other frames' documents do not hold it up. ## Downloads diff --git a/skill-data/core/references/commands.md b/skill-data/core/references/commands.md index 613dfd3b02..360c14acc8 100644 --- a/skill-data/core/references/commands.md +++ b/skill-data/core/references/commands.md @@ -148,10 +148,11 @@ agent-browser wait --url "**/dashboard" # Wait for URL pattern (or -u) agent-browser wait --load domcontentloaded # Wait for DOMContentLoaded (or -l) agent-browser wait --load load # Wait for the load event agent-browser wait --load networkidle # Wait for network idle on known-quiet pages +agent-browser wait --load networkquiet # Wait for the requests that change the page (not pings or images) agent-browser wait --fn "window.ready" # Wait for JS condition (or -f) ``` -After a page-changing action, prefer the selector, text, URL, or JavaScript condition that represents the result you need. Use a lifecycle wait when the lifecycle event is the milestone. `networkidle` is also supported, but use it only for pages known to become quiet because SSE, WebSockets, polling, and long-polling can keep it from resolving. +After a page-changing action, prefer the selector, text, URL, or JavaScript condition that represents the result you need. Use a lifecycle wait when the lifecycle event is the milestone. `networkidle` is also supported, but use it only for pages known to become quiet because SSE, WebSockets, polling, and long-polling can keep it from resolving. `networkquiet` counts only the requests that can change what the page shows (the page's document, scripts, stylesheets, XHR and fetch), so analytics pings and images do not hold it up. ## Mouse Control