feat: browser agent fixes from live tests on ten utility sites

.env.example

@@ -61,21 +61,27 @@ TINYCOMPUTER_LAB_SELF_EMAIL=
 # task_live: 1 shows the browser the task launches instead of running it
 # headless (a headed run needs a display, so it runs on the host).
 # TASK_HEADED=1
+# task_live: 1 waits for the person at the terminal where only a person can
+# go on (approve, log in or solve a captcha, type a missing detail, pay)
+# instead of ending the run there.
+# TASK_INTERACTIVE=1
 # The travel fixture's address for browser_fixture.
 # TINYCOMPUTER_FIXTURE_URL=http://127.0.0.1:8000
 # task_live: a Tiny Humans bearer (a session token, or an API key with the
 # `inference` scope) in place of OPENROUTER_API_KEY; Jev and the planner then
-# go through Tiny Humans' routes, and the gateway's agentic-v1 plans, rescues,
-# and shapes unless the model variables below name another.
+# go through Tiny Humans' routes, and openrouter/deepseek/deepseek-v4-flash
+# plans, rescues, and shapes unless the model variables below name another.
 # TINYHUMANS_TOKEN=
 # The planner model task_live asks for.
 # TINYCOMPUTER_PLANNER_MODEL=
 # The reasoning model task_live rescues a failed step with (default
-# openai/gpt-6-luna), and how many rescues a task gets (0 to 5, default 5).
+# openai/gpt-6-luna on OpenRouter), and how many rescues a task gets (0 to 5,
+# default 5).
 # TINYCOMPUTER_RESCUE_MODEL=
 # TASK_RESCUES=5
 # The reasoning model task_live shapes a finished task's answer with
-# (default openai/gpt-6-luna), and a JSON TaskOutput file asking for it.
+# (default openai/gpt-6-luna on OpenRouter), and a JSON TaskOutput file
+# asking for it.
 # TINYCOMPUTER_OUTPUT_MODEL=
 # OUTPUT_FILE=crates/tinycomputer-examples/tasks/whatsapp/output.json
 # Which decision model task_live's module asks: TINYCOMPUTER_DECISIONS=sage

crates/tinycomputer-browser/src/fake/mod.rs

@@ -48,6 +48,31 @@ impl Fake {
             .collect()
     }
 
+    /// Every command sent, in order.
+    pub(crate) fn sent(&self) -> Vec<Value> {
+        self.sent.lock().unwrap().clone()
+    }
+
+    /// Whether a page script ran besides the one that keeps a press in the
+    /// tab, which runs before every click.
+    pub(crate) fn evaluated_besides_keeping_the_tab(&self) -> bool {
+        self.sent().iter().any(|command| {
+            command["action"] == "evaluate"
+                && !command["script"]
+                    .as_str()
+                    .unwrap_or_default()
+                    .contains("__tcOpen")
+        })
+    }
+
+    /// Whether the pointer pressed anywhere by position, rather than only
+    /// moving.
+    pub(crate) fn pressed_by_position(&self) -> bool {
+        self.sent()
+            .iter()
+            .any(|command| command["action"] == "mouse" && command["eventType"] != "mouseMoved")
+    }
+
     pub(crate) fn last(&self, action: &str) -> Value {
         self.sent
             .lock()

crates/tinycomputer-browser/src/surface/location.rs

@@ -0,0 +1,56 @@
+//! Letting a page read where the person is, when the agent presses the
+//! page's own "use my current location" button.
+
+use serde_json::json;
+use tinycomputer_core::surface::Candidate;
+
+use super::BrowserSurface;
+
+/// Words of a control that asks the page to find where the person is.
+const ASKS_WHERE: &[&str] = &[
+    "current location",
+    "my location",
+    "detect location",
+    "detect my location",
+    "locate me",
+    "use location",
+    "use my current",
+];
+
+/// Whether pressing `target` asks the page for the person's location.
+pub(super) fn asks_where(target: &Candidate) -> bool {
+    let said = format!(
+        "{} {}",
+        target.name.as_deref().unwrap_or_default(),
+        target.description.as_deref().unwrap_or_default()
+    )
+    .to_lowercase()
+    .replace('-', " ");
+    ASKS_WHERE.iter().any(|words| said.contains(words))
+}
+
+impl BrowserSurface {
+    /// Grants the session's pages the location permission before a press
+    /// that asks for it (`asks_where`). The browser asks a person in a
+    /// bubble outside the page, which the agent can neither see nor press,
+    /// so a store's "use my current location" waited on it forever; the
+    /// press stands in for that person's "Allow". Best effort: a browser
+    /// that refuses is pressed as it is.
+    ///
+    /// Only in a browser the module launched on a throwaway profile: the
+    /// grant covers every page of the browser for the session, and a
+    /// person's own browser (an `endpoint`) or profile (a `user_data_dir`)
+    /// keeps its own say, in the bubble they can answer.
+    pub(super) fn allow_location(&self) {
+        if self.options.endpoint.is_some() || self.options.user_data_dir.is_some() {
+            return;
+        }
+        let Ok(id) = self.ensure_session() else {
+            return;
+        };
+        let _granted = self.block(self.browser.command(
+            &id,
+            json!({"action": "permissions", "permissions": ["geolocation"]}),
+        ));
+    }
+}

crates/tinycomputer-browser/src/surface/mod.rs

@@ -15,10 +15,13 @@ mod card;
 mod cursor;
 mod envelope;
 mod fields;
+mod location;
 mod native_select;
 mod operations;
 mod sight;
+mod tabs;
 mod tree;
+mod uncover;
 
 pub use sight::Denoised;
 
@@ -48,6 +51,12 @@ const SETTLE_MS: u64 = 400;
 /// that polls forever is never idle, so this is a cap, not an expectation.
 const NETWORK_IDLE_MS: u64 = 2_000;
 
+/// The longest one reading of the page may take, by sight or as a tree. A
+/// reading sent while a page was being replaced waited out the browser's
+/// own deadline live, 30 s for sight and again for the tree, so one look
+/// took a minute; a reading this late is retried on the next look instead.
+const READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
+
 /// How a [`BrowserSurface`] reads a page.
 #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
 pub enum Perception {
@@ -185,11 +194,14 @@ impl BrowserSurface {
     fn see(&self, root: Option<&str>) -> Option<Screen> {
         self.keep_denoised(Denoised::default());
         let id = self.ensure_session().ok()?;
+        // The timer is made inside the runtime `block` enters, not before.
+        let reading = self.browser.command(
+            &id,
+            json!({"action": "evaluate", "script": sight::script(root)}),
+        );
         let reply = self
-            .block(self.browser.command(
-                &id,
-                json!({"action": "evaluate", "script": sight::script(root)}),
-            ))
+            .block(async { tokio::time::timeout(READ_TIMEOUT, reading).await })
+            .ok()?
             .ok()?;
         let result = reply.get("result")?;
         let screen = sight::screen(result)?;

crates/tinycomputer-browser/src/surface/operations.rs

@@ -9,11 +9,12 @@ use tinycomputer_bus::{DesktopError, DesktopResponse, JevOperation};
 use tinycomputer_core::surface::{Candidate, Depth, Screen, Surface, uses_pointer};
 use tinycomputer_core::{Key, Platform};
 
-use super::card::selects_on_click;
-use super::envelope::{covered, failure, not_a_text_field, reply};
+use crate::error::Error;
+
+use super::envelope::{failure, not_a_text_field, reply};
 use super::sight;
 use super::{BrowserSurface, Perception};
-use super::{NETWORK_IDLE_MS, SETTLE_MS, SKELETON_DEPTH, tree};
+use super::{NETWORK_IDLE_MS, READ_TIMEOUT, SETTLE_MS, SKELETON_DEPTH, tree};
 
 impl Surface for BrowserSurface {
     fn observe(
@@ -37,7 +38,16 @@ impl Surface for BrowserSurface {
         };
         let snapshot = self
             .ensure_session()
-            .and_then(|id| self.block(self.browser.snapshot(&id, request)))
+            .and_then(|id| {
+                let reading = self.browser.snapshot(&id, request);
+                self.block(async { tokio::time::timeout(READ_TIMEOUT, reading).await })
+                    .unwrap_or_else(|_| {
+                        Err(Error::timeout(
+                            "snapshot",
+                            u64::try_from(READ_TIMEOUT.as_millis()).unwrap_or(u64::MAX),
+                        ))
+                    })
+            })
             .map_err(|error| Box::new(failure("snapshot", &error)))?;
         let mut screen = tree::screen(&snapshot.tree, &snapshot.title);
         if !app.is_empty() {
@@ -78,29 +88,11 @@ impl Surface for BrowserSurface {
             return reply;
         }
         match operation {
-            JevOperation::Click | JevOperation::Expand | JevOperation::Collapse => {
-                let reply = targeted("click", |target, _| Action::Click {
-                    target,
-                    new_tab: false,
-                });
-                let name = target.as_ref().and_then(|node| node.name.as_deref());
-                let reply = match (&reference, name) {
-                    (Some(reference), name)
-                        if covered(&reply) && (name.is_some() || sight::is_seen(reference)) =>
-                    {
-                        self.click_through_own_card(reference, name.unwrap_or_default())
-                            .unwrap_or(reply)
-                    }
-                    _ => reply,
-                };
-                if reply.ok
-                    && let (Some(reference), Some(node)) = (&reference, &target)
-                    && selects_on_click(node)
-                    && sight::is_seen(reference)
-                {
-                    self.select_if_ignored(reference);
-                }
-                reply
+            JevOperation::Click => self.press_element(target.as_ref(), reference.as_deref(), true),
+            // Opening or closing in place goes nowhere by design: never
+            // followed as a link that ignored its press.
+            JevOperation::Expand | JevOperation::Collapse => {
+                self.press_element(target.as_ref(), reference.as_deref(), false)
             }
             // Without a target the text goes where the focus is, as into an
             // autocomplete's unnamed input once it has been opened — but
@@ -256,10 +248,16 @@ impl Surface for BrowserSurface {
     fn navigate(&self, url: &str) -> DesktopResponse {
         let page = self
             .ensure_session()
-            .and_then(|id| self.block(self.browser.navigate(&id, NavigateRequest::new(url))));
+            .and_then(|id| self.block(self.browser.navigate(&id, NavigateRequest::new(url))))
+            .map(|page| (page.url, page.title));
+        // A heavy page can be read long before its `load` event fires.
+        let page = match page {
+            Err(error @ Error::Timeout { .. }) => self.drawn_page(url).ok_or(error),
+            other => other,
+        };
         reply(
             "navigate",
-            page.map(|page| json!({"url": page.url, "title": page.title})),
+            page.map(|(url, title)| json!({"url": url, "title": title})),
         )
     }