From f63acdb3f432dbf457d93f5dd32e1d858819a149 Mon Sep 17 00:00:00 2001 From: Shanu Date: Thu, 8 Oct 2026 22:42:37 +0530 Subject: [PATCH 01/15] Bring a control sight found into the window before pressing it A press on a sight mark goes through a selector, and a selector's element that shows at all is not scrolled first: the press lands on its middle whether the window shows that point or not, and reports success. Live, a store's "Add to cart" sat at the window's foot with its middle below it, and every press went nowhere. Before pressing a sight mark, the surface now runs INTO_VIEW_JS: an element whose middle lies outside the window is scrolled to the window's middle, and one in view is left where it is. A ref of the tree is still brought into view by agent-browser itself. The fake's check for stray page scripts (evaluated_besides_every_press) leaves this script out, as it does the one that keeps a press in the tab. --- crates/tinycomputer-browser/src/fake/mod.rs | 14 +++---- .../tinycomputer-browser/src/surface/mod.rs | 3 ++ .../src/surface/surface_tests/card_tests.rs | 40 +++++++++++++++---- .../surface_tests/native_select_tests.rs | 2 +- .../src/surface/uncover.rs | 36 +++++++++++++++++ .../tinycomputer-browser/interacting.md | 9 +++++ 6 files changed, 89 insertions(+), 15 deletions(-) diff --git a/crates/tinycomputer-browser/src/fake/mod.rs b/crates/tinycomputer-browser/src/fake/mod.rs index dd4f3389..6acbe541 100644 --- a/crates/tinycomputer-browser/src/fake/mod.rs +++ b/crates/tinycomputer-browser/src/fake/mod.rs @@ -66,15 +66,15 @@ impl Fake { self.sent.lock().unwrap().clone() } - /// Whether a page script ran besides the one that keeps a press in the - /// tab, which runs before every click. - pub(crate) fn evaluated_besides_keeping_the_tab(&self) -> bool { + /// Whether a page script ran besides those every press runs: the one + /// that keeps a press in the tab, and the one that brings a control + /// sight found into the window. + pub(crate) fn evaluated_besides_every_press(&self) -> bool { self.sent().iter().any(|command| { + let script = command["script"].as_str().unwrap_or_default(); command["action"] == "evaluate" - && !command["script"] - .as_str() - .unwrap_or_default() - .contains("__tcOpen") + && !script.contains("__tcOpen") + && !script.starts_with(crate::surface::INTO_VIEW_JS) }) } diff --git a/crates/tinycomputer-browser/src/surface/mod.rs b/crates/tinycomputer-browser/src/surface/mod.rs index 8839b603..215081a1 100644 --- a/crates/tinycomputer-browser/src/surface/mod.rs +++ b/crates/tinycomputer-browser/src/surface/mod.rs @@ -24,6 +24,9 @@ mod tree; mod uncover; mod watch; +#[cfg(test)] +pub(crate) use uncover::INTO_VIEW_JS; + pub use sight::Denoised; use std::sync::atomic::{AtomicBool, Ordering}; diff --git a/crates/tinycomputer-browser/src/surface/surface_tests/card_tests.rs b/crates/tinycomputer-browser/src/surface/surface_tests/card_tests.rs index 3d40f350..fff5c740 100644 --- a/crates/tinycomputer-browser/src/surface/surface_tests/card_tests.rs +++ b/crates/tinycomputer-browser/src/surface/surface_tests/card_tests.rs @@ -1,5 +1,6 @@ -//! Tests for clicking through a result card's own cover, and pressing a -//! selection again through the DOM. +//! Tests for bringing a control into the window before it is pressed, +//! clicking through a result card's own cover, and pressing a selection +//! again through the DOM. use serde_json::json; use tinycomputer_bus::JevOperation; @@ -72,7 +73,7 @@ fn a_click_covered_by_anything_else_stays_refused() { .execute(JevOperation::Click, Some(node("e5", &["Click"])), None) .ok ); - assert!(!fake.evaluated_besides_keeping_the_tab()); + assert!(!fake.evaluated_besides_every_press()); } /// A page that takes every click, and says through `evaluate` whether the @@ -113,9 +114,34 @@ fn a_tab_click_the_page_ignored_is_pressed_again_through_the_dom() { ..node(reference, &["Click"]) }; assert!(surface.execute(JevOperation::Click, Some(node), None).ok); - assert!( - !fake.evaluated_besides_keeping_the_tab(), - "{reference} {role}" - ); + assert!(!fake.evaluated_besides_every_press(), "{reference} {role}"); } } + +#[test] +fn a_control_sight_found_is_brought_into_the_window_before_it_is_pressed() { + // Live, a store's "Add to cart" sat at the window's foot with its middle + // below it, and every press went nowhere while reporting success. + let into_view = |command: &serde_json::Value| { + command["action"] == "evaluate" + && command["script"] + .as_str() + .unwrap_or_default() + .starts_with(crate::surface::INTO_VIEW_JS) + }; + let Harness { fake, surface, .. } = harness("into-view", Fake::new()); + let reply = surface.execute(JevOperation::Click, Some(node("seen:21", &["Click"])), None); + assert!(reply.ok, "{:?}", reply.error); + let sent = fake.sent(); + let brought = sent.iter().position(into_view).expect("brought into view"); + let pressed = sent + .iter() + .position(|command| command["action"] == "click") + .expect("pressed"); + assert!(brought < pressed, "{:?}", fake.actions()); + + // A ref of the tree is brought into view by the browser itself. + let Harness { fake, surface, .. } = harness("tree-into-view", Fake::new()); + surface.execute(JevOperation::Click, Some(node("e5", &["Click"])), None); + assert!(!fake.sent().iter().any(into_view)); +} diff --git a/crates/tinycomputer-browser/src/surface/surface_tests/native_select_tests.rs b/crates/tinycomputer-browser/src/surface/surface_tests/native_select_tests.rs index 2585d199..a08e9fc3 100644 --- a/crates/tinycomputer-browser/src/surface/surface_tests/native_select_tests.rs +++ b/crates/tinycomputer-browser/src/surface/surface_tests/native_select_tests.rs @@ -72,5 +72,5 @@ fn an_option_that_is_not_native_is_pressed_as_a_control() { .ok ); assert!(fake.actions().iter().any(|action| action == "click")); - assert!(!fake.evaluated_besides_keeping_the_tab()); + assert!(!fake.evaluated_besides_every_press()); } diff --git a/crates/tinycomputer-browser/src/surface/uncover.rs b/crates/tinycomputer-browser/src/surface/uncover.rs index 1afe34de..3315882b 100644 --- a/crates/tinycomputer-browser/src/surface/uncover.rs +++ b/crates/tinycomputer-browser/src/surface/uncover.rs @@ -35,6 +35,22 @@ const CENTRE_JS: &str = r"(element => { return true; })"; +/// Brings an element whose middle lies outside the window to the window's +/// middle, and leaves one whose middle shows where it is; `true` when it +/// moved the element. A press lands on the element's middle, and the browser +/// presses there whether the window shows that point or not: live, a store's +/// "Add to cart" sat at the window's foot with its middle below it, and every +/// press went nowhere while reporting success. +pub(crate) const INTO_VIEW_JS: &str = r"(element => { + if (!element) return false; + const box = element.getBoundingClientRect(); + const x = box.left + box.width / 2; + const y = box.top + box.height / 2; + if (x >= 0 && y >= 0 && x < window.innerWidth && y < window.innerHeight) return false; + element.scrollIntoView({ block: 'center', inline: 'center', behavior: 'instant' }); + return true; +})"; + /// How long a link's press is given to start leaving the page. const LEAVE_MS: u64 = 400; @@ -91,6 +107,9 @@ impl BrowserSurface { self.allow_location(); } let seen = sight::is_seen(reference); + if seen { + self.bring_into_view(reference); + } // Where a link's press starts from, to tell whether it went. let before = (follows && seen && node.is_some_and(|node| node.role == "link")) .then(|| self.page_url()) @@ -130,6 +149,23 @@ impl BrowserSurface { reply } + /// Brings the element `reference` names into the window when its middle + /// lies outside it ([`INTO_VIEW_JS`]), before it is pressed. A ref of the + /// tree is brought into view by the browser itself. + fn bring_into_view(&self, reference: &str) { + let (Ok(id), Ok(selector)) = ( + self.ensure_session(), + serde_json::to_string(&sight::selector(reference)), + ) else { + return; + }; + let script = format!("{INTO_VIEW_JS}(document.querySelector({selector}))"); + let _moved = self.block( + self.browser + .command(&id, json!({"action": "evaluate", "script": script})), + ); + } + /// Clicks `reference` once more after the browser refused because /// something covered its middle, with the element brought to the middle /// of the window and the pointer moved off to the window's corner first. diff --git a/docs/crates/tinycomputer-browser/interacting.md b/docs/crates/tinycomputer-browser/interacting.md index 1f2d8991..72ae369c 100644 --- a/docs/crates/tinycomputer-browser/interacting.md +++ b/docs/crates/tinycomputer-browser/interacting.md @@ -54,6 +54,15 @@ than clicking through it and reporting success on a click that landed nowhere useful. That refusal message is what `reply::classify` turns into `Error::NotActionable`. +A selector's element that shows at all is not scrolled, though, and its +press lands on its middle whether the window shows that point or not, with +success reported. So before pressing a control sight found, the surface +brings it to the middle of the window when its middle lies outside the +window (`INTO_VIEW_JS` in `surface/uncover.rs`), and leaves a control in +view where it is. Live, a store's "Add to cart" sat at the window's foot +with its middle below it, and was pressed again and again with nothing +happening. A ref of the tree is brought into view by agent-browser itself. + ### Click-through on result cards A refusal because something is "covered" is not always the end of the From a4faf848f59319435a6cffa9153de56b704bd527 Mon Sep 17 00:00:00 2001 From: Shanu Date: Thu, 8 Oct 2026 22:43:25 +0530 Subject: [PATCH 02/15] Read Preact presses, variant classes, sprite closers and grid calendars A click listener Preact keeps on an element (`l` once minified, or `_listeners`, as "click" or "clickfalse") makes it a control, as a React press handler does. Live, a hotel site's place suggestions were rows with only such a listener, read as page text, and the place typed was never chosen. Inside another control, an element with a press handler of its own is a control too: a sort menu's options sat inside their pointer-cursor trigger and read as one button naming them all. A class behind a variant prefix (`placeholder:text-disabled`, `disabled:opacity-50`) no longer reads as disabled or selected: a flight site's place box carried the first and was never seen. Words run together in camel case in an icon's class ("icClose") are read as words, so a sign-up pop-up whose only way out was such a sprite can be closed. A calendar drawn without a table, a grid of 28 to 49 cells numbered from 1 below the month and year it shows, is read as one, its days as dates; a title naming two months names two grids in order. A picker showing two months side by side is paged by the block that holds both, and a table's header is read as shown, so a hidden month list no longer hides the month it shows. A row scrolled out of its own list's view (a popover's airport rows below its fold) reads offscreen rather than covered, so it is ranked and offered rather than dropped. Each has a live sight test against a page in Chrome, in the new live_controls_tests.rs and live_calendar_tests.rs topics, along with one that a control below the window's foot is brought into it and pressed. --- .../src/surface/sight/sight.js | 156 ++++++++++++-- .../src/surface/sight/sight_tests.rs | 2 + .../sight/sight_tests/live_calendar_tests.rs | 128 ++++++++++++ .../sight/sight_tests/live_controls_tests.rs | 192 ++++++++++++++++++ .../surface/sight/sight_tests/live_tests.rs | 54 ++++- docs/crates/tinycomputer-browser/sight.md | 8 +- 6 files changed, 517 insertions(+), 23 deletions(-) create mode 100644 crates/tinycomputer-browser/src/surface/sight/sight_tests/live_calendar_tests.rs create mode 100644 crates/tinycomputer-browser/src/surface/sight/sight_tests/live_controls_tests.rs diff --git a/crates/tinycomputer-browser/src/surface/sight/sight.js b/crates/tinycomputer-browser/src/surface/sight/sight.js index 97776376..a74dbe48 100644 --- a/crates/tinycomputer-browser/src/surface/sight/sight.js +++ b/crates/tinycomputer-browser/src/surface/sight/sight.js @@ -58,11 +58,17 @@ // step choosing it never saw it chosen. Never `unselected`. const SELECTED_CLASS = /(?:^|[-_])(?:selected|checked)$/i; const UNSELECTED_CLASS = /(?:^|[-_])(?:un|not[-_]?)(?:selected|checked)$/i; - const classChosen = (element) => [...element.classList] + // A class behind a variant (`placeholder:text-disabled`, which colours + // only the placeholder; `disabled:opacity-50`, which applies only once + // disabled) says nothing of the element's own state: live, a flight + // site's place box carried `placeholder:text-disabled`, was read as + // disabled, and its box was never seen. + const stateClasses = (element) => [...element.classList].filter((name) => !name.includes(':')); + const classChosen = (element) => stateClasses(element) .some((name) => SELECTED_CLASS.test(name) && !UNSELECTED_CLASS.test(name)); const disabled = (element) => element.disabled === true || element.getAttribute('aria-disabled') === 'true' - || [...element.classList].some((name) => DISABLED_CLASS.test(name)); + || stateClasses(element).some((name) => DISABLED_CLASS.test(name)); const insideText = (element) => { for (let parent = element; parent; parent = parent.parentElement) { @@ -101,22 +107,33 @@ const pointer = (element) => style(element).cursor === 'pointer'; // A click handler a script framework keeps on the element itself (React - // stores each element's props on it), on a box smaller than a quarter of - // the window: a page can wire a plain `div` to a click with neither a - // cursor nor a tab stop. Live, a store's "Add to cart" and "Buy now" read - // as plain words, and the step to add to the cart had nothing to press. - // Only a press handler: a carousel's track or a select's menu listens for - // the mouse going down and is no button to press. + // stores each element's props on it, Preact its listeners), on a box + // smaller than a quarter of the window: a page can wire a plain `div` to a + // click with neither a cursor nor a tab stop. Live, a store's "Add to + // cart" and "Buy now" read as plain words, and the step to add to the cart + // had nothing to press. Only a press handler: a carousel's track or a + // select's menu listens for the mouse going down and is no button to press. const HANDLERS = ['onClick', 'onPress']; + // Preact keeps them by event name (`_listeners`, `l` once minified), a + // capture flag after it in newer releases ("clickfalse"). Live, a hotel + // site's place suggestions were rows with only such a listener, read as + // page text, and the place typed was never chosen. + const PREACT_PRESS = /^click(true|false)?$/; + const pressHandler = (element) => { + const key = Object.keys(element).find((name) => name.startsWith('__reactProps$')); + const props = key && element[key]; + if (props && HANDLERS.some((handler) => typeof props[handler] === 'function')) return true; + const own = ['_listeners', 'l'].find((name) => Object.prototype.hasOwnProperty.call(element, name)); + const listeners = own && element[own]; + return !!listeners && typeof listeners === 'object' && Object.keys(listeners) + .some((name) => PREACT_PRESS.test(name) && typeof listeners[name] === 'function'); + }; // What is a control only by such a handler: it hides nothing pressable // inside it, since a page can wire a whole card to a click around its own // "Add" button. const scriptedOnly = new Set(); const scripted = (element) => { - if (!element || element === document.body) return false; - const key = Object.keys(element).find((name) => name.startsWith('__reactProps$')); - const props = key && element[key]; - if (!props || !HANDLERS.some((handler) => typeof props[handler] === 'function')) return false; + if (!element || element === document.body || !pressHandler(element)) return false; const rect = element.getBoundingClientRect(); return rect.width * rect.height < window.innerWidth * window.innerHeight * 0.25; }; @@ -131,6 +148,25 @@ const MONTH_AND_YEAR = new RegExp(`\\b(${MONTHS.join('|')})\\s+(\\d{4})\\b`, 'i'); const calendarDays = new Map(); const calendars = []; + // The month and year a grid of days shows: the nearest short text before + // it, or before one of its four nearest ancestors, that names one, as + // `{ element, months: [[month, year], …] }`. A longer block (another + // month's whole grid) ends the search at its level. + const MONTHS_AND_YEARS = new RegExp(MONTH_AND_YEAR.source, 'gi'); + const gridTitle = (grid) => { + let node = grid; + for (let depth = 0; node && node !== base && depth < 4; depth += 1, node = node.parentElement) { + let sibling = node.previousElementSibling; + for (let step = 0; sibling && step < 3; step += 1, sibling = sibling.previousElementSibling) { + const said = shownWords(sibling); + if (said.length > 120) break; + const months = [...said.matchAll(MONTHS_AND_YEARS)] + .map((found) => [MONTHS.indexOf(found[1].toLowerCase()), Number(found[2])]); + if (months.length) return { element: sibling, months }; + } + } + return null; + }; const findCalendars = () => { for (const table of base.querySelectorAll('table')) { // A week-number column is numbers too, but no day. @@ -139,12 +175,14 @@ if (cells.length < 28 || !shown(table)) continue; // The month is named in the table's own heading, or in a short header // drawn just before it; never by words elsewhere on the page, nor by a - // calendar that happens to come before it. + // calendar that happens to come before it. A header is read as shown: + // live, one held a hidden month list for the picker's own menu, whose + // twelve names hid the month it showed. const before = table.previousElementSibling; - const header = before && !before.querySelector('table') && squash(before.innerText).length <= 80 + const header = before && !before.querySelector('table') && shownWords(before).length <= 80 ? before : null; const titled = MONTH_AND_YEAR.exec(squash([table.caption, table.tHead, header] - .filter(Boolean).map((part) => part.innerText).join(' '))); + .filter(Boolean).map((part) => shownWords(part)).join(' '))); if (!titled) continue; const holder = table.parentElement; calendars.push(holder && holder !== document.body && holder !== document.documentElement ? holder : table); @@ -163,6 +201,51 @@ calendarDays.set(cell, `${day} ${spelled[0].toUpperCase()}${spelled.slice(1)} ${date.getUTCFullYear()}`); } } + // A calendar drawn without a table: a grid whose cells each begin with + // their day number (a fare may follow, "22 6529"), numbered from 1 to the + // month's last day, below the month and year it shows. A title naming + // two months, one header over two grids, names them in order. Live, a + // flight site drew its days as buttons in such a grid, no day read as a + // date, and the departure was never picked. + const titleUses = new Map(); + for (const grid of base.querySelectorAll('div, ul, ol, tbody')) { + const kids = grid.children; + if (kids.length < 28 || kids.length > 49 + || calendars.some((calendar) => calendar.contains(grid))) continue; + const days = [...kids].map((kid) => { + const leading = /^(\d{1,2})(?:\s|$)/.exec(squash(kid.innerText)); + return leading && shown(kid) ? { cell: kid, day: Number(leading[1]) } : null; + }); + const start = days.findIndex((entry) => entry && entry.day === 1); + if (start < 0) continue; + const run = []; + for (const entry of days.slice(start)) { + if (!entry || entry.day !== run.length + 1) break; + run.push(entry); + } + const title = run.length >= 28 && gridTitle(grid); + if (!title) continue; + const used = titleUses.get(title.element) || 0; + titleUses.set(title.element, used + 1); + const [month, year] = title.months[Math.min(used, title.months.length - 1)]; + const spelled = MONTHS[month]; + const holder = grid.parentElement; + calendars.push(holder && holder !== document.body ? holder : grid); + for (const { cell, day } of run) { + const pressed = cell.matches(NESTED) ? cell : cell.querySelector(NESTED) || cell; + calendarDays.set(pressed, `${day} ${spelled[0].toUpperCase()}${spelled.slice(1)} ${year}`); + } + } + // A picker showing two months beside each other pages both with one + // pair of arrows, drawn beside the months rather than inside either: + // the block that holds more than one month pages them too. + for (const holder of [...calendars]) { + const picker = holder.parentElement; + if (picker && picker !== document.body && !calendars.includes(picker) + && calendars.filter((other) => other !== picker && picker.contains(other)).length > 1) { + calendars.push(picker); + } + } }; // A calendar's paging arrow, read as what it does: an arrow glyph, or a // bare "Next", inside a calendar turns its month. @@ -212,7 +295,11 @@ // is still a button. if ((GROUP_ROLES.includes(claimed) || claimed === 'dialog' || claimed === 'alertdialog') && !element.hasAttribute('onclick') && !pointer(element)) return null; - if (insideControl) return null; + // Inside another control, what a press handler of its own wires is a + // control too, as a native button inside a pressable card is: live, a + // sort menu's options sat inside its pointer-cursor trigger, each wired + // to a click by itself, and the menu read as one button naming them all. + if (insideControl && !pressHandler(element)) return null; const tabindex = element.getAttribute('tabindex'); const byPage = element.hasAttribute('onclick') || (tabindex !== null && tabindex !== '-1') @@ -241,6 +328,10 @@ ]; // A picture-only control's meaning, from the words in its own or its // icon's class, id, or test id: all a person would see is the picture. + // Words run together in camel case are words too ("icClose", + // "closeIcon"): live, a sign-up pop-up's only way out was a sprite with + // the class "icClose", dropped as a blank box, and nothing closed it. A + // generated class ("fCarEc", "cUpXyz") has no two such words in a row. const iconWords = (element) => { const sources = [element, ...element.querySelectorAll('svg, use, i, img, span')].slice(0, 8); const words = new Set(); @@ -257,7 +348,7 @@ source.getAttribute('data-icon') || '', tag(source) === 'use' ? used : '', file, - ].join(' ').toLowerCase(); + ].join(' ').replace(/([a-z]{2,})(?=[A-Z][a-z]{2,})/g, '$1 ').toLowerCase(); for (const word of text.split(/[^a-z]+/)) { if (ICON_WORDS.includes(word)) words.add(word); } @@ -807,6 +898,35 @@ return rect.bottom <= 0 || rect.top >= height || rect.right <= 0 || rect.left >= width; }; + // The nearest container that scrolls its content (a popover's list), or + // null. Each container is looked at once. + const scrollers = new Map(); + const scrollerOf = (element) => { + const parent = element.parentElement; + if (!parent || parent === document.body || parent === document.documentElement) return null; + if (scrollers.has(parent)) return scrollers.get(parent); + const style = getComputedStyle(parent); + const scrolls = (/(auto|scroll)/.test(style.overflowY) && parent.scrollHeight > parent.clientHeight) + || (/(auto|scroll)/.test(style.overflowX) && parent.scrollWidth > parent.clientWidth); + const found = scrolls ? parent : scrollerOf(parent); + scrollers.set(parent, found); + return found; + }; + // Whether the element's middle is scrolled out of its container's view: + // what shows at that point is the container's neighbour or the page, + // neither of which covers the element, and a press scrolls it back. Live, + // a popover's airport rows below its list's fold read as covered, ranked + // last, and were never offered. + const scrolledAway = (element) => { + const scroller = scrollerOf(element); + if (!scroller) return false; + const rect = box(element); + const view = scroller.getBoundingClientRect(); + const x = (rect.left + rect.right) / 2; + const y = (rect.top + rect.bottom) / 2; + return x < view.left || x > view.right || y < view.top || y > view.bottom; + }; + const statesOf = (element, what) => { const states = []; const input = standIn(element) || element; @@ -818,7 +938,7 @@ if (aria('selected') === 'true' || (aria('current') && aria('current') !== 'false') || (!states.includes('checked') && classChosen(element))) states.push('selected'); if (input.required === true || aria('required') === 'true') states.push('required'); - if (offscreen(element)) states.push('offscreen'); + if (offscreen(element) || scrolledAway(element)) states.push('offscreen'); else if (covered(element)) states.push('covered'); return states; }; diff --git a/crates/tinycomputer-browser/src/surface/sight/sight_tests.rs b/crates/tinycomputer-browser/src/surface/sight/sight_tests.rs index 98bac9d8..209e5467 100644 --- a/crates/tinycomputer-browser/src/surface/sight/sight_tests.rs +++ b/crates/tinycomputer-browser/src/surface/sight/sight_tests.rs @@ -3,5 +3,7 @@ #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod live_calendar_tests; +mod live_controls_tests; mod live_tests; mod reading_tests; diff --git a/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_calendar_tests.rs b/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_calendar_tests.rs new file mode 100644 index 00000000..562f4398 --- /dev/null +++ b/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_calendar_tests.rs @@ -0,0 +1,128 @@ +//! Live tests of calendars sight reads as dates, gated on +//! `TINYCOMPUTER_LIVE_BROWSER=1`: two months that share their arrows, and +//! months drawn as grids of buttons rather than tables. + +#[cfg(feature = "agent-browser")] +use super::live_tests::{live_reading, shown_names}; + +/// Two months drawn as grids of buttons, not tables, under one header that +/// names both: each day shows its fare after its number ("22 6529"). +#[cfg(feature = "agent-browser")] +fn grid_calendar_page() -> String { + let grid = |blanks: u32, days: u32, fares: u32| { + let cells = (0..blanks) + .map(|_| "".to_owned()) + .chain((1..=days).map(|day| format!("", fares + day * 7))) + .collect::(); + format!( + "
Su Mo Tu We Th Fr Sa
\ +
{cells}
" + ) + }; + format!( + "
October 2026 November 2026
\ +
{}{}
", + grid(4, 31, 6_000), + grid(0, 30, 7_000) + ) +} + +#[cfg(feature = "agent-browser")] +#[tokio::test] +async fn live_a_calendar_drawn_as_grids_of_buttons_reads_its_days_as_dates() { + let Some(reading) = live_reading(&grid_calendar_page()).await else { + return; + }; + let described = |name: &str| { + reading["nodes"] + .as_array() + .unwrap() + .iter() + .find(|node| node["name"] == name) + .map_or_else( + || panic!("{name} not offered: {:?}", shown_names(&reading)), + |node| node["description"].as_str().unwrap_or_default().to_owned(), + ) + }; + assert_eq!(described("22 6154"), "22 October 2026"); + assert_eq!(described("5 7035"), "5 November 2026"); +} + +/// A picker showing two months, drawn as a hotel site's was: each month a +/// table under a header that holds a hidden month menu, and one pair of icon +/// arrows beside both months, inside neither. +#[cfg(feature = "agent-browser")] +const TWO_MONTH_PICKER: &str = r#" +
+
+
+
October 2026
+ +
MonTueWedThuFriSatSun
+
+
+
November 2026
+ +
MonTueWedThuFriSatSun
+
+
+
+"#; + +#[cfg(feature = "agent-browser")] +#[tokio::test] +async fn live_a_two_month_pickers_days_and_shared_arrows_are_read_as_dates() { + let Some(reading) = live_reading(TWO_MONTH_PICKER).await else { + return; + }; + let nodes = reading["nodes"].as_array().unwrap(); + let days = nodes + .iter() + .filter(|node| node["role"] == "gridcell") + .map(|node| { + format!( + "{} ({})", + node["name"].as_str().unwrap(), + node["description"].as_str().unwrap() + ) + }) + .collect::>(); + // Each month by its header as shown, not by its hidden menu's names. + assert_eq!(days.len(), 61, "{days:?}"); + for day in [ + "22 (22 October 2026)", + "31 (31 October 2026)", + "22 (22 November 2026)", + ] { + assert!(days.iter().any(|seen| seen == day), "{day} in {days:?}"); + } + let names = shown_names(&reading); + assert_eq!( + names + .iter() + .filter(|name| name.ends_with(" month")) + .collect::>(), + ["previous month", "next month"], + "the arrows beside both months page them" + ); +} diff --git a/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_controls_tests.rs b/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_controls_tests.rs new file mode 100644 index 00000000..e71caa28 --- /dev/null +++ b/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_controls_tests.rs @@ -0,0 +1,192 @@ +//! Live tests of what sight reads as a control, gated on +//! `TINYCOMPUTER_LIVE_BROWSER=1`: rows a script framework wires to a click, +//! options inside their trigger, classes behind a variant, icons named in +//! camel case, and a control brought into the window before its press. + +#[cfg(feature = "agent-browser")] +use serde_json::json; + +#[cfg(feature = "agent-browser")] +use super::live_tests::{live_page, live_reading, shown_names}; + +/// Place suggestions drawn as plain rows that only a Preact click listener +/// makes pressable (`l` once minified, `_listeners` with its capture flag), +/// and one whose listener waits for the mouse going down. +#[cfg(feature = "agent-browser")] +const PREACT_ROWS_PAGE: &str = r#"
+ +
Goa, India
+
Baga Beach North Goa, Goa
+
Panjim, Panaji, Goa
+
+"#; + +#[cfg(feature = "agent-browser")] +#[tokio::test] +async fn live_rows_a_preact_click_listener_wires_are_buttons() { + let Some(reading) = live_reading(PREACT_ROWS_PAGE).await else { + return; + }; + let role = |name: &str| { + reading["nodes"] + .as_array() + .unwrap() + .iter() + .find(|node| node["name"] == name) + .and_then(|node| node["role"].as_str()) + .map(str::to_owned) + }; + assert_eq!(role("Goa, India").as_deref(), Some("button")); + assert_eq!(role("Baga Beach North Goa, Goa").as_deref(), Some("button")); + assert_eq!( + role("Panjim, Panaji, Goa"), + None, + "a mouse-down listener is no press" + ); +} + +/// A sort menu whose list sits inside its pointer-cursor trigger, each +/// option wired to a click by its own Preact listener. +#[cfg(feature = "agent-browser")] +const MENU_IN_TRIGGER_PAGE: &str = r#"
+ + Popularity +
    +
  • Popularity
    • +
  • Guest Ratings
    • +
  • Price Low to High
    • +
+ +
+"#; + +#[cfg(feature = "agent-browser")] +#[tokio::test] +async fn live_options_wired_by_their_own_handler_inside_a_trigger_are_buttons() { + let Some(reading) = live_reading(MENU_IN_TRIGGER_PAGE).await else { + return; + }; + for option in ["Guest Ratings", "Price Low to High"] { + let found = reading["nodes"] + .as_array() + .unwrap() + .iter() + .find(|node| node["name"] == option) + .unwrap_or_else(|| panic!("{option} not offered: {:?}", shown_names(&reading))); + assert_eq!(found["role"], "button", "{option}"); + } +} + +/// A place box styled by a variant class that ends in "disabled", beside +/// a day a calendar greys out by class, and a button whose variant class +/// styles it only once disabled. +#[cfg(feature = "agent-browser")] +const VARIANT_CLASSES_PAGE: &str = r#" + +21"#; + +#[cfg(feature = "agent-browser")] +#[tokio::test] +async fn live_a_variant_class_says_nothing_of_a_controls_own_state() { + let Some(reading) = live_reading(VARIANT_CLASSES_PAGE).await else { + return; + }; + let role = |name: &str| { + reading["nodes"] + .as_array() + .unwrap() + .iter() + .find(|node| node["name"] == name) + .and_then(|node| node["role"].as_str()) + .map(str::to_owned) + }; + assert_eq!(role("From").as_deref(), Some("textbox")); + assert_eq!(role("Search").as_deref(), Some("button")); + assert_eq!(role("21"), None, "a day greyed out by class is no control"); +} + +/// A button at the window's foot whose middle lies below the window, and +/// one in full view; each records its press. +#[cfg(feature = "agent-browser")] +const FOOT_BUTTON_PAGE: &str = r#" + +
+"#; + +#[cfg(feature = "agent-browser")] +#[tokio::test] +async fn live_a_control_whose_middle_is_below_the_window_is_brought_into_it_and_pressed() { + let Some((browser, info)) = live_page(FOOT_BUTTON_PAGE).await else { + return; + }; + let evaluate = + |script: String| browser.command(&info.id, json!({"action": "evaluate", "script": script})); + let into_view = |selector: &str| { + format!( + "{}(document.querySelector({}))", + crate::surface::INTO_VIEW_JS, + serde_json::Value::String(selector.to_owned()) + ) + }; + let shown = evaluate(into_view("#shown")).await.unwrap(); + assert_eq!( + shown["result"], false, + "a control in view stays where it is" + ); + let moved = evaluate(into_view("#foot")).await.unwrap(); + assert_eq!(moved["result"], true); + let middle = evaluate( + "(() => { const box = document.getElementById('foot').getBoundingClientRect(); \ + const y = box.top + box.height / 2; return y >= 0 && y < window.innerHeight; })()" + .to_owned(), + ) + .await + .unwrap(); + assert_eq!(middle["result"], true, "its middle now shows"); + browser + .command(&info.id, json!({"action": "click", "selector": "#foot"})) + .await + .unwrap(); + let pressed = evaluate("window.pressed".to_owned()).await.unwrap(); + browser.close_session(&info.id).await.unwrap(); + assert_eq!(pressed["result"], json!(["foot"])); +} + +/// A sign-up pop-up whose only way out is a sprite its class names in camel +/// case ("icClose"): no words, no picture, a pointer cursor. +#[cfg(feature = "agent-browser")] +const SPRITE_CLOSE_PAGE: &str = r#"
+ +

Login/Signup

+ +
"#; + +#[cfg(feature = "agent-browser")] +#[tokio::test] +async fn live_a_sprite_its_class_names_in_camel_case_is_read_as_that_icon() { + let Some(reading) = live_reading(SPRITE_CLOSE_PAGE).await else { + return; + }; + let close = reading["nodes"] + .as_array() + .unwrap() + .iter() + .find(|node| node["name"] == "close") + .unwrap_or_else(|| panic!("no close control: {:?}", shown_names(&reading))); + assert_eq!(close["role"], "button"); + assert_eq!(close["description"], "an icon"); +} diff --git a/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_tests.rs b/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_tests.rs index 0d89257b..09481dd3 100644 --- a/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_tests.rs +++ b/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_tests.rs @@ -12,7 +12,7 @@ use crate::surface::sight::script; /// page is written into a blank tab, so nothing is fetched but what the /// fixture itself asks for. #[cfg(feature = "agent-browser")] -async fn live_reading(html: &str) -> Option { +pub(super) async fn live_reading(html: &str) -> Option { live_results(html, &[script(None)]) .await .map(|mut results| results.remove(0)) @@ -21,7 +21,7 @@ async fn live_reading(html: &str) -> Option { /// A blank tab of a real browser with `html` written into it: `None` /// unless `TINYCOMPUTER_LIVE_BROWSER=1`, since CI has no browser to launch. #[cfg(feature = "agent-browser")] -async fn live_page( +pub(super) async fn live_page( html: &str, ) -> Option<( crate::sessions::Browser, @@ -79,7 +79,7 @@ async fn live_results(html: &str, scripts: &[String]) -> Option Vec { +pub(super) fn shown_names(reading: &serde_json::Value) -> Vec { reading["nodes"] .as_array() .unwrap() @@ -518,6 +518,54 @@ const CHOICES_PAGE: &str = r#"
document.addEventListener('change', (event) => window.changes.push(event.target.id)); "#; +/// A popover's list scrolled so its last rows are out of its view, beside +/// a button a modal covers. +#[cfg(feature = "agent-browser")] +const SCROLLED_LIST_PAGE: &str = r#"
+
    +
  • DEL Delhi
  • BLR Bengaluru
  • MAA Chennai
    • +
  • HYD Hyderabad
  • CCU Kolkata
  • BOM Mumbai
    • +
  • GOI Goa
  • PNQ Pune
    • +
+
Planning a holiday?
+
+ +
+ Sign up for offers
+"#; + +#[cfg(feature = "agent-browser")] +#[tokio::test] +async fn live_a_row_scrolled_out_of_its_list_is_offscreen_not_covered() { + let Some(reading) = live_reading(SCROLLED_LIST_PAGE).await else { + return; + }; + let states = |name: &str| { + reading["nodes"] + .as_array() + .unwrap() + .iter() + .find(|node| node["name"] == name) + .map_or_else( + || panic!("{name} not offered"), + |node| node["states"].to_string(), + ) + }; + assert!( + !states("DEL Delhi").contains("offscreen"), + "{}", + states("DEL Delhi") + ); + for row in ["BOM Mumbai", "PNQ Pune"] { + let read = states(row); + assert!( + read.contains("offscreen") && !read.contains("covered"), + "{row}: {read}" + ); + } + assert!(states("Search").contains("covered"), "a modal still covers"); +} + /// The options a reading offers, each as " in ". #[cfg(feature = "agent-browser")] fn offered(reading: &serde_json::Value) -> Vec { diff --git a/docs/crates/tinycomputer-browser/sight.md b/docs/crates/tinycomputer-browser/sight.md index a5f813a5..2171ec03 100644 --- a/docs/crates/tinycomputer-browser/sight.md +++ b/docs/crates/tinycomputer-browser/sight.md @@ -64,7 +64,8 @@ Sight decides what is a control by behavior, not by role: those (so a `` inside a button does not become a second control just because it inherits the button's pointer cursor). - A small element (under a quarter of the window) whose React props hold a - press handler (`onClick`, `onPress`) is a control too: a page can wire a + press handler (`onClick`, `onPress`), or whose Preact listeners (`l` once + minified, `_listeners`) hold a click one, is a control too: a page can wire a plain `div` to a click with neither a cursor nor a tab stop. A handler for the mouse going down alone (a carousel's track) makes no control, and a control made only by such a handler hides nothing pressable inside it. @@ -73,7 +74,10 @@ Sight decides what is a control by behavior, not by role: `aria-disabled="true"` and a class name that *ends* in `disabled`. That last rule exists because plenty of date pickers grey out a past day purely through a CSS class, `rdrDay rdrDayDisabled` on a real calendar widget, - with no ARIA attribute at all. + with no ARIA attribute at all. A class behind a variant prefix + (`placeholder:text-disabled`, `disabled:opacity-50`) is never read this + way, nor for `selected` below: it styles a part or a state the element may + not be in. - Chosen controls read `selected` by the same reasoning: besides `aria-selected`, `aria-current`, `aria-checked`, and `aria-pressed`, a class name ending in `-selected` or `-checked` (never `unselected` or From 4d0464b303481cfdc9d0dc7d67246b02661e2bdf Mon Sep 17 00:00:00 2001 From: Shanu Date: Thu, 8 Oct 2026 22:43:45 +0530 Subject: [PATCH 03/15] Work through pop-ups, calendars and place boxes the page draws Live runs on travel sites stalled on a few patterns any site can draw; each is now handled by what the page shows, not by where it is. What Jev is shown. Of the 120 elements a state holds, those in view come first and the rest keep a quarter of the room in page order (seen_first): a sign-up pop-up drawn at the end of a long page was cut off, and a calendar in front crowded out the guests button it covers. A knockout that must drop chunks drops them first from regions with nothing in view, so the list in front and a dialog's rows are offered whole. A place box's suggestion is a row in view and in front, never a footer link of the same name. Dialogs. Front keeps whose the dialog in front is (Dialog: the page's, the task's still asking, answered, or served by a step before), in place of separate flags. A dialog at a run's first look is the task's only when the task's run before left its own dialog in front, which contract 2.9 carries: RunFlowRequest and FlowRunResult gain `dialog_left_open`, kept by the task store for a rescue. A sign-up or a menu the page opened at a rescue's first look no longer refuses every move past it. A calendar the task opened and has pressed in since no longer blocks a press behind it, and on a page that draws its pop-up without a dialog's role, a closer that went with what it was pressed on is the pop-up dismissed. Calendars and place boxes. An enter step picks each date from a calendar already showing before it looks for a box, rather than pressing the calendar's own button closed. A form that draws its place boxes as buttons ("From DEL", "To BLR") has each slot's opener pressed once, one slot at a time (enter/open.rs). A date the field's button shows ("Departure Thu, 22 Oct") counts as made. An enter value with no box is picked only from what the screen shows, a list's option is one option however long its label, and "to" or "from" alone names only a container that begins with it. A pick opens only lists whose items hold something to press, never a strip of bare fares. RunFlowRequest's new flag sits beside three others that callers set by name; its struct_excessive_bools allow says why folding them into one enum is not worth a major bump. --- crates/tinycomputer-bus/src/flow/guide.md | 3 + .../src/flow/types/request.rs | 12 + .../tinycomputer-bus/src/flow/types/result.rs | 5 + crates/tinycomputer-bus/src/version/mod.rs | 6 +- .../src/version/version_tests.rs | 7 +- .../src/agentic/flow/act/mod.rs | 44 +++- .../src/agentic/flow/act/moves.rs | 8 +- .../src/agentic/flow/act/recover.rs | 4 +- .../src/agentic/flow/act/turns.rs | 4 +- .../src/agentic/flow/act/uncover.rs | 12 +- .../src/agentic/flow/ask/screen_state.rs | 35 ++- .../src/agentic/flow/attention/clear.rs | 4 +- .../src/agentic/flow/enter/fill.rs | 218 ++++++------------ .../src/agentic/flow/enter/mod.rs | 8 +- .../src/agentic/flow/enter/open.rs | 193 ++++++++++++++++ .../src/agentic/flow/flow_tests.rs | 1 + .../agentic/flow/flow_tests/brief_tests.rs | 67 ++++++ .../agentic/flow/flow_tests/choose_tests.rs | 64 +++++ .../agentic/flow/flow_tests/do_loop_tests.rs | 58 +++-- .../agentic/flow/flow_tests/enter_tests.rs | 48 ++++ .../flow/flow_tests/grounding_tests.rs | 69 ++++++ .../src/agentic/flow/flow_tests/pick_tests.rs | 23 ++ .../agentic/flow/flow_tests/picker_tests.rs | 193 ++++++++++++++++ .../src/agentic/flow/flow_tests/places.rs | 36 +++ .../src/agentic/flow/flow_tests/screens.rs | 27 ++- .../src/agentic/flow/flow_tests/simulator.rs | 40 +++- .../flow/flow_tests/suggestion_tests.rs | 99 ++++++++ .../src/agentic/flow/front.rs | 153 ++++++++---- .../src/agentic/flow/ground/mod.rs | 3 + .../src/agentic/flow/ground/narrow.rs | 79 +++++-- .../src/agentic/flow/look.rs | 3 +- .../src/agentic/flow/mod.rs | 3 +- .../src/agentic/flow/run.rs | 3 +- .../src/agentic/flow/steps/choose.rs | 70 ++++-- .../src/agentic/flow/steps/condition.rs | 2 +- .../src/agentic/flow/steps/date.rs | 9 + .../src/agentic/flow/steps/list.rs | 23 +- .../src/agentic/flow/steps/matching.rs | 70 +++++- .../src/agentic/flow/steps/mod.rs | 6 +- .../src/agentic/flow/steps/suggestion.rs | 13 +- crates/tinycomputer-engine/src/task/budget.rs | 1 + crates/tinycomputer-engine/src/task/drive.rs | 3 + crates/tinycomputer-engine/src/task/store.rs | 6 + .../src/task/task_tests.rs | 1 + .../src/task/task_tests/rescue_tests.rs | 28 +++ crates/tinycomputer/tests/public_api_tests.rs | 2 +- docs/technical/decision-thresholds.md | 1 + docs/technical/jev-harness.md | 4 +- .../specs/desktop-module-contract.md | 5 +- 49 files changed, 1476 insertions(+), 300 deletions(-) create mode 100644 crates/tinycomputer-engine/src/agentic/flow/enter/open.rs create mode 100644 crates/tinycomputer-engine/src/agentic/flow/flow_tests/picker_tests.rs diff --git a/crates/tinycomputer-bus/src/flow/guide.md b/crates/tinycomputer-bus/src/flow/guide.md index 6b3791e2..5b489564 100644 --- a/crates/tinycomputer-bus/src/flow/guide.md +++ b/crates/tinycomputer-bus/src/flow/guide.md @@ -129,6 +129,9 @@ do. on which result to take ("skip Sponsored items", "rated 4 stars or more") belongs in that `pick`'s `from` or `by`, never in a step of its own: there is nothing on screen to do for it, so such a step fails. + For the same reason, a pop-up that may or may not show (a login, + cookie, or offer pop-up) is closed in an `if` on it being visible, + never in a plain step. Keep the task's own words in a `pick`: its `from` names the item the task asks for ("the results for ", not just "the search results"), and its `by` is the task's criterion, "first" when the task diff --git a/crates/tinycomputer-bus/src/flow/types/request.rs b/crates/tinycomputer-bus/src/flow/types/request.rs index 48ce5ac5..4856e257 100644 --- a/crates/tinycomputer-bus/src/flow/types/request.rs +++ b/crates/tinycomputer-bus/src/flow/types/request.rs @@ -146,6 +146,10 @@ pub struct GroundingHint { /// /// Requires confidential delivery, like `RunGoal`: the texts a flow enters /// travel with it. +// Each flag is an independent switch a caller sets by name; folding them +// into one enum would rename `allow_destructive`, `include_values` and +// `trace` on the wire, a major bump, for a tidiness no caller gains from. +#[allow(clippy::struct_excessive_bools)] #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct RunFlowRequest { @@ -207,6 +211,13 @@ pub struct RunFlowRequest { /// knows what it has done. #[serde(skip_serializing_if = "BTreeMap::is_empty")] pub collected: BTreeMap, + /// Whether the run before this one, of the same task, left the task's + /// own dialog in front ([`FlowRunResult::dialog_left_open`]): a dialog + /// in front at this run's first look is then the task's current stage, + /// to work within. Otherwise such a dialog is the page's (a promotion or + /// a sign-up the page opened itself, a menu), never the task's. + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub dialog_left_open: bool, } impl Default for RunFlowRequest { @@ -227,6 +238,7 @@ impl Default for RunFlowRequest { strategy: FlowStrategy::Narrow, deliberation: Deliberation::Deep, collected: BTreeMap::new(), + dialog_left_open: false, } } } diff --git a/crates/tinycomputer-bus/src/flow/types/result.rs b/crates/tinycomputer-bus/src/flow/types/result.rs index 281d1de1..75d36c40 100644 --- a/crates/tinycomputer-bus/src/flow/types/result.rs +++ b/crates/tinycomputer-bus/src/flow/types/result.rs @@ -106,6 +106,11 @@ pub struct FlowRunResult { /// Every Jev exchange, when [`RunFlowRequest::trace`] asked for them. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub trace: Vec, + /// Whether the run ended with the task's own dialog in front: one a + /// press of the run opened, still asking its question. The task hands + /// it to its next run as [`RunFlowRequest::dialog_left_open`]. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub dialog_left_open: bool, } /// One Jev request and its answers, as recorded by a traced run. diff --git a/crates/tinycomputer-bus/src/version/mod.rs b/crates/tinycomputer-bus/src/version/mod.rs index 646d2d62..ea4ae418 100644 --- a/crates/tinycomputer-bus/src/version/mod.rs +++ b/crates/tinycomputer-bus/src/version/mod.rs @@ -8,7 +8,7 @@ //! release workflow bumps, which tracks the shipped artifact. /// The wire contract version this crate defines. -pub const CONTRACT_VERSION: (u32, u32) = (2, 8); +pub const CONTRACT_VERSION: (u32, u32) = (2, 9); /// Returns whether a host holding [`CONTRACT_VERSION`] can bind to a module /// reporting `module`. @@ -22,8 +22,8 @@ pub const CONTRACT_VERSION: (u32, u32) = (2, 8); /// ``` /// # use tinycomputer_bus::{is_compatible, CONTRACT_VERSION}; /// assert!(is_compatible(CONTRACT_VERSION)); -/// assert!(is_compatible((2, 8))); -/// assert!(!is_compatible((2, 7))); +/// assert!(is_compatible((2, 9))); +/// assert!(!is_compatible((2, 8))); /// assert!(!is_compatible((1, 8))); /// ``` #[must_use] diff --git a/crates/tinycomputer-bus/src/version/version_tests.rs b/crates/tinycomputer-bus/src/version/version_tests.rs index 52abe8ec..9573f136 100644 --- a/crates/tinycomputer-bus/src/version/version_tests.rs +++ b/crates/tinycomputer-bus/src/version/version_tests.rs @@ -6,7 +6,7 @@ use super::{CONTRACT_VERSION, binds, is_compatible}; #[test] fn the_shipped_contract_version_is_pinned() { - assert_eq!(CONTRACT_VERSION, (2, 8)); + assert_eq!(CONTRACT_VERSION, (2, 9)); } #[test] @@ -16,8 +16,11 @@ fn the_contract_binds_to_itself() { #[test] fn a_newer_minor_on_the_module_side_binds() { - assert!(is_compatible((2, 8))); + assert!(is_compatible((2, 9))); assert!(is_compatible((2, 97))); + // 2.9 gives `RunFlow` and its result `dialog_left_open`: a 2.9 host may + // send it, which a 2.8 module would ignore. + assert!(!is_compatible((2, 8))); // 2.8 accepts the `open_jev` and `sage` decision providers and the // planner's `tiny_humans` route: a 2.8 host may configure them, which a // 2.7 module refuses. diff --git a/crates/tinycomputer-engine/src/agentic/flow/act/mod.rs b/crates/tinycomputer-engine/src/agentic/flow/act/mod.rs index 600ca55d..40bfef2b 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/act/mod.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/act/mod.rs @@ -297,12 +297,15 @@ fn plural(word: &str) -> bool { /// Words of a step that ask for an overlay to go away. const DISMISS_VERBS: &[&str] = &["dismiss", "close", "accept", "decline", "reject", "skip"]; -/// What such a step asks to go away. +/// What such a step asks to go away ("pop" of "pop-up"). const OVERLAYS: &[&str] = &[ - "banner", "dialog", "popup", "cookie", "cookies", "consent", "modal", "overlay", "prompt", - "notice", + "banner", "dialog", "popup", "popups", "pop", "cookie", "cookies", "consent", "modal", + "overlay", "prompt", "notice", ]; +/// Words of a control that closes what it sits on. +const CLOSERS: &[&str] = &["close", "dismiss", "skip", "later", "decline", "reject"]; + fn words(text: &str) -> Vec { text.split(|character: char| !character.is_alphanumeric()) .filter(|word| !word.is_empty()) @@ -318,21 +321,44 @@ fn words(text: &str) -> Vec { /// A completion judge sees only the screen after the fact, where a closed /// cookie banner leaves no trace of which button closed it; this is the /// evidence it cannot see. +/// +/// A pop-up the page draws without a dialog's role leaves the surface the +/// window throughout, so for a dismissal step a closer ("Close") that went +/// with what it was pressed on is that evidence instead. Live, an offer +/// pop-up closed at the first press, the judge could not tell whether +/// "declining optional cookies" was done with no cookie banner shown, and +/// the step stalled into a rescue. fn closed_the_overlay(last: &LastAction, screen: &Screen, intent: &str) -> Option { - let name = last.target.as_ref()?.name.as_deref()?; - if last.before.surface == "window" || screen.surface != "window" { + let target = last.target.as_ref()?; + let name = target.name.as_deref()?; + if screen.surface != "window" { return None; } let intent = words(intent); + let dismissal = intent + .iter() + .any(|word| DISMISS_VERBS.contains(&word.as_str())) + && intent.iter().any(|word| OVERLAYS.contains(&word.as_str())); + if last.before.surface == "window" { + let closer = words(name) + .iter() + .any(|word| CLOSERS.contains(&word.as_str())); + let gone = !screen + .candidates + .iter() + .any(|candidate| candidate.role == target.role && candidate.name == target.name); + return (dismissal && closer && gone).then(|| { + Ended::new( + StepOutcome::Done, + format!("pressed {name:?} and it closed with what it was on"), + ) + }); + } let named = words(name) .iter() .filter(|word| word.len() > 2) .all(|word| intent.iter().any(|said| said.starts_with(word.as_str()))) && words(name).iter().any(|word| word.len() > 2); - let dismissal = intent - .iter() - .any(|word| DISMISS_VERBS.contains(&word.as_str())) - && intent.iter().any(|word| OVERLAYS.contains(&word.as_str())); (named || dismissal).then(|| { Ended::new( StepOutcome::Done, diff --git a/crates/tinycomputer-engine/src/agentic/flow/act/moves.rs b/crates/tinycomputer-engine/src/agentic/flow/act/moves.rs index 0e6ce49f..8867fcac 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/act/moves.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/act/moves.rs @@ -151,10 +151,12 @@ impl FlowRun<'_, B> { // a press scrolls it out from under the bar (live, a seat table's // lower rows sat under its "Pay" bar). && !in_dialog(candidate); - if covered && self.front.surface != "window" { + // A calendar the task has picked in is closed for such a press + // (`press_uncovering`), so what it covers can be pressed. + if covered && self.front.surface != "window" && !self.front.served_calendar() { return false; } - !(self.front.opened_dialog && closes(candidate) && !asks_to_close(intent)) + !(self.front.opened_dialog() && closes(candidate) && !asks_to_close(intent)) } /// Grounds and performs an `activate`, `expand`, or `scroll` move; the @@ -201,7 +203,7 @@ impl FlowRun<'_, B> { // front: the dialog asks something first, and what answers it is // pressed instead, though not remembered as the step's control. let (grounded, answers_dialog) = match grounded { - None if operation == "activate" && self.front.opened_dialog => { + None if operation == "activate" && self.front.opened_dialog() => { (self.answer_dialog(log, screen, intent, banned).await?, true) } grounded => (grounded, false), diff --git a/crates/tinycomputer-engine/src/agentic/flow/act/recover.rs b/crates/tinycomputer-engine/src/agentic/flow/act/recover.rs index 5b69a134..172b6dcf 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/act/recover.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/act/recover.rs @@ -34,7 +34,7 @@ impl FlowRun<'_, B> { ) -> Result { if judged.blocked.unwrap_or_default() >= BLOCKED && state.obstacles < MAX_OBSTACLES - && !self.front.opened_dialog + && !self.front.opened_dialog() { state.obstacles += 1; log.used(FlowLoop::Obstacles); @@ -99,7 +99,7 @@ impl FlowRun<'_, B> { // A press that opened a dialog of the task's (the seat count after // a showtime) moved the flow on, whatever the judge made of it: // undoing it with Escape closed the dialog live. - if !self.enabled(FlowLoop::Undo) || state.undos >= MAX_UNDOS || self.front.opened_dialog { + if !self.enabled(FlowLoop::Undo) || state.undos >= MAX_UNDOS || self.front.opened_dialog() { return Ok(false); } state.undos += 1; diff --git a/crates/tinycomputer-engine/src/agentic/flow/act/turns.rs b/crates/tinycomputer-engine/src/agentic/flow/act/turns.rs index 896a42b6..93a9e79e 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/act/turns.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/act/turns.rs @@ -36,7 +36,7 @@ impl FlowRun<'_, B> { // choose a heading in it: live, rescues kept choosing a language // heading above a format dialog's buttons. match ended { - Err(Halt::Failed(note)) if self.front.opened_dialog => { + Err(Halt::Failed(note)) if self.front.opened_dialog() => { let offers = match self.look().await { Ok(screen) => front_controls(&screen), Err(_) => Vec::new(), @@ -96,7 +96,7 @@ impl FlowRun<'_, B> { // The root of the turn's tree: what needs attention first. A // distraction cleared means a fresh look before judging. A dialog // the step's own press just opened is the step's to work in. - if !self.front.opened_dialog + if !self.front.opened_dialog() && self .attend(log, &screen, intent, &mut state.cleared) .await? diff --git a/crates/tinycomputer-engine/src/agentic/flow/act/uncover.rs b/crates/tinycomputer-engine/src/agentic/flow/act/uncover.rs index 5032fb1d..055ed048 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/act/uncover.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/act/uncover.rs @@ -28,6 +28,8 @@ impl FlowRun<'_, B> { operation: JevOperation, intent: &str, ) -> Result { + // Read before this press, which would count as one in the dialog. + let served = self.front.served_calendar(); let chosen = target.clone(); let reply = self .act(log, verb, Some(target), move |backend| { @@ -42,8 +44,14 @@ impl FlowRun<'_, B> { // lies behind it leaves the flow it began (live, a movie's language // link behind its booking dialog led to a listing of other films). // A layer drawn over the window is such a question only when the - // task's own press opened it; a calendar left open is in the way. - if self.front.opened_dialog || !matches!(self.front.surface.as_str(), "window" | "layer") { + // task's own press opened it; a calendar left open is in the way, + // and so is one the task opened and has pressed in since: live, a + // calendar stayed in front of the guests and Search buttons once + // both dates were picked, and every press behind it was refused. + if (self.front.opened_dialog() + || !matches!(self.front.surface.as_str(), "window" | "layer")) + && !served + { self.history.push(format!( "{} lies behind the dialog in front; act within the dialog instead", label(target) diff --git a/crates/tinycomputer-engine/src/agentic/flow/ask/screen_state.rs b/crates/tinycomputer-engine/src/agentic/flow/ask/screen_state.rs index 80cbd443..31857a86 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/ask/screen_state.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/ask/screen_state.rs @@ -3,7 +3,10 @@ use serde_json::{Value, json}; -use crate::agentic::flow::view::{Candidate, Screen, element_line, label, untrusted_context}; +use crate::agentic::flow::{ + denoise::{Tier, tier}, + view::{Candidate, Screen, element_line, label, untrusted_context}, +}; use super::{MAX_FIELDS, MAX_HISTORY, MAX_STATE_ELEMENTS}; @@ -14,10 +17,8 @@ pub(in crate::agentic::flow) fn state( history: &[String], include_values: bool, ) -> Value { - let elements = screen - .candidates - .iter() - .take(MAX_STATE_ELEMENTS) + let elements = seen_first(&screen.candidates, MAX_STATE_ELEMENTS) + .into_iter() .map(|node| element_line(node, include_values)) .collect::>(); let mut state = json!({ @@ -35,6 +36,30 @@ pub(in crate::agentic::flow) fn state( state } +/// The `most` of `candidates` Jev is shown, in screen order: those in view +/// first, then the rest as the page orders them, which keep a quarter of +/// the room. Live, a sign-up pop-up a long page drew at the end of its +/// document fell outside the first 120 elements, behind the covered page, +/// and Jev never saw it; and a calendar open in front would fill the room +/// and hide the guests button it covers, which the next step reads. +fn seen_first(candidates: &[Candidate], most: usize) -> Vec<&Candidate> { + if candidates.len() <= most { + return candidates.iter().collect(); + } + let (in_view, rest): (Vec<_>, Vec<_>) = candidates + .iter() + .enumerate() + .partition(|(_, candidate)| tier(candidate) == Tier::InView); + let front = in_view.len().min(most - rest.len().min(most / 4)); + let mut kept = in_view + .into_iter() + .take(front) + .chain(rest.into_iter().take(most - front)) + .collect::>(); + kept.sort_by_key(|(at, _)| *at); + kept.into_iter().map(|(_, candidate)| candidate).collect() +} + /// What each text-holding element shows, at more length than the element /// list allows: whether a draft "shows the body" is decided here. /// diff --git a/crates/tinycomputer-engine/src/agentic/flow/attention/clear.rs b/crates/tinycomputer-engine/src/agentic/flow/attention/clear.rs index fd1f0442..23e1226c 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/attention/clear.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/attention/clear.rs @@ -30,10 +30,10 @@ impl FlowRun<'_, B> { cleared: &mut Cleared, ) -> Result { // A dialog the run's own press opened is its next stage, not a - // distraction (`FlowRun::opened_dialog`). + // distraction (`Front::opened_dialog`). if !self.deliberates(FlowLoop::Attention) || cleared.count >= MAX_CLEARED - || self.front.opened_dialog + || self.front.opened_dialog() { return Ok(false); } diff --git a/crates/tinycomputer-engine/src/agentic/flow/enter/fill.rs b/crates/tinycomputer-engine/src/agentic/flow/enter/fill.rs index a1868ba4..c7d88a2d 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/enter/fill.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/enter/fill.rs @@ -4,100 +4,56 @@ use std::collections::BTreeSet; use serde_json::Value; -use tinycomputer_bus::{JevOperation, Slot}; +use tinycomputer_bus::Slot; use tinycomputer_core::reformat_date; use crate::agentic::flow::{ AgentBackend, FlowRun, Halt, StepLog, - ask::{self, Questions, corroborate, probability}, backend::deliver_text, memory::{learn, remember}, - view::{Candidate, Screen, element_kind, is_destructive, label}, + steps::looks_like_date, + view::{Candidate, Screen, element_kind, label}, }; -use super::{BLIND_PICK_MISSES, OPENER_FLOOR, REVEAL_TURNS, editable, names}; +use super::{BLIND_PICK_MISSES, editable, names}; impl FlowRun<'_, B> { - /// Presses the control on `screen` whose label holds a pending slot's - /// own word (`named_opener`); `true` when it did. A control named by - /// the slot, a search link for the "search box", is how the field - /// shows: live, the turns that look for a way in hesitated over it, and - /// the search was never typed. - async fn open_by_name( + /// Picks each pending date of `untried` from the calendar showing, + /// before any box is looked for, and says whether one arrived: the box + /// looked for is the calendar's own button, and pressing it closes the + /// calendar. Live, a check-out date's step found the calendar open from + /// the check-in and pressed its button four times, and a departure's + /// calendar opened by the step's own press was closed again the same way. + async fn pick_shown_dates( &mut self, log: &mut StepLog, - screen: &Screen, slots: &[Slot], - pending: &BTreeSet, + private: &[bool], + pending: &mut BTreeSet, + untried: &mut BTreeSet, ) -> Result { - let Some(opener) = named_opener(screen, slots, pending) - .filter(|opener| !is_destructive(opener, screen, &self.stop_before)) - else { - return Ok(false); - }; - // A shared word is a hint, not a reason to press: a link named - // "Email us" shares the slot "email", and pressing it left the form. - let purpose = format!("click to show the box for: {}", names(slots, pending)); - let answers = self - .ask( - log, - ask::request( - self.model(), - self.state(screen, &purpose), - Questions::default().with( - "confirm", - corroborate(&purpose, &opener, self.include_values), - ), - ), - ) - .await?; - if probability(&answers, "confirm").is_none_or(|yes| yes < OPENER_FLOOR) { - return Ok(false); - } - let pressed = opener.clone(); - let reply = self - .act( - log, - "click (show the field)", - Some(&opener), - move |backend| backend.execute(JevOperation::Click, Some(pressed), None), - ) - .await?; - if reply.ok { - self.history.push(format!( - "pressed {} to show the field for {}", - label(&opener), - names(slots, pending) - )); - } - Ok(reply.ok) - } - - /// Runs a short `do` loop that shows the fields for the `pending` slots, - /// any field at all when `no_fields` showed. A field that cannot be revealed - /// is looked for another way, or found not to be asked for; it is not a - /// failure. - async fn reveal_fields( - &mut self, - log: &mut StepLog, - slots: &[Slot], - pending: &BTreeSet, - no_fields: bool, - ) -> Result<(), Halt> { - let reveal = if no_fields { - format!("show the editable fields for: {}", names(slots, pending)) - } else { - format!("show the fields for: {}", names(slots, pending)) - }; - match self.accomplish(log, &reveal, REVEAL_TURNS).await { - Err(Halt::Failed(note)) => self - .history - .push(format!("could not reveal the fields ({note})")), - other => { - other?; + let dates = pending + .iter() + .copied() + .filter(|index| untried.contains(index) && looks_like_date(&slots[*index].text)) + .collect::>(); + let mut picked = false; + for index in dates { + untried.remove(&index); + let slot = &slots[index]; + match self + .pick_option(log, &slot.slot, &slot.text, private[index], false) + .await + { + Ok(_) => { + pending.remove(&index); + picked = true; + } + Err(Halt::Failed(_)) => {} + Err(halt) => return Err(halt), } } - Ok(()) + Ok(picked) } /// Fills every slot in `pending` it can find a field or an option for, @@ -110,7 +66,14 @@ impl FlowRun<'_, B> { pending: &mut BTreeSet, ) -> Result<(), Halt> { let mut revealed = false; - let mut opened = false; + // The dates not yet picked from a calendar showing, each tried once. + let mut dates = pending.clone(); + // The slots whose own opener was looked for, each once: a form that + // draws its place boxes as buttons ("From DEL", "To BLR") opens one + // box at a time. Live, the "to" box was never opened once "from" + // had been, and the place was pressed in a link at the foot of the + // page instead. + let mut opened: BTreeSet = BTreeSet::new(); // The fields this step already filled: one slot's box is never // another's. Live, a pickup box that had not yet become the place // chosen was the only box on screen in the next round, and the drop @@ -125,11 +88,19 @@ impl FlowRun<'_, B> { // by the city it holds, are struck together rather than one at a // time (`a_row_that_refused_the_text_is_never_pressed_while_revealing_a_field`). let mut struck: BTreeSet = BTreeSet::new(); - for _ in 0..3 { + // Each slot may take a round to open its box and one to fill it. + for _ in 0..3.max(2 * pending.len() + 1) { if pending.is_empty() { break; } let mut screen = self.look().await?; + if self.front.calendar + && self + .pick_shown_dates(log, slots, private, pending, &mut dates) + .await? + { + continue; + } if editable(&screen).len() < pending.len() && !screen.unexplored.is_empty() { self.explore(&mut screen).await; } @@ -141,11 +112,11 @@ impl FlowRun<'_, B> { self.assign(log, &screen, slots, pending, &fields).await? }; if assignments.is_empty() { - if !opened { - opened = true; - if self.open_by_name(log, &screen, slots, pending).await? { - continue; - } + if self + .open_next_box(log, &screen, slots, pending, &mut opened) + .await? + { + continue; } if revealed { break; @@ -189,12 +160,24 @@ impl FlowRun<'_, B> { } } } - // A value with no field to type into is picked instead, as a date - // from a calendar or a city from a list of suggestions. Only a "the - // value was not found" failure is safe to shrug off and move to the - // next slot; a budget stop or a backend error means acting further - // is unsafe or pointless, and must end the step instead of being - // read as "this slot has no picker". + self.pick_unboxed(log, slots, private, pending, saw_fields) + .await + } + + /// Picks each slot still `pending` instead of typing it, as a date from a + /// calendar or a city from a list of suggestions, when no field took it. + /// Only a "the value was not found" failure is safe to shrug off and move + /// to the next slot; a budget stop or a backend error means acting + /// further is unsafe or pointless, and must end the step instead of being + /// read as "this slot has no picker". + async fn pick_unboxed( + &mut self, + log: &mut StepLog, + slots: &[Slot], + private: &[bool], + pending: &mut BTreeSet, + saw_fields: bool, + ) -> Result<(), Halt> { let mut missed = 0; for index in pending.clone() { if !saw_fields && missed >= BLIND_PICK_MISSES { @@ -271,59 +254,6 @@ impl FlowRun<'_, B> { } } -/// Words of a slot's name that say only that it is a box, or name a kind -/// of control rather than what the slot is. -const BOX_WORDS: &[&str] = &[ - "field", "box", "input", "bar", "the", "your", "text", "here", "link", "button", "menu", "icon", -]; - -/// A control on `screen` that is no field itself and whose label holds a -/// word of a pending slot's name (four letters or more, not a box word): -/// the link or button that shows the slot's field, such as a store's -/// search link for the slot "search box". -fn named_opener(screen: &Screen, slots: &[Slot], pending: &BTreeSet) -> Option { - let wanted = pending - .iter() - .filter_map(|index| slots.get(*index)) - .flat_map(|slot| words(&slot.slot)) - .filter(|word| word.chars().count() > 3 && !BOX_WORDS.contains(&word.as_str())) - .collect::>(); - if wanted.is_empty() { - return None; - } - let fields = editable(screen); - screen - .candidates - .iter() - .filter(|candidate| { - matches!(candidate.role.as_str(), "link" | "button") - && candidate - .available_actions - .iter() - .any(|action| action == "Click") - && !fields.iter().any(|field| field.ref_id == candidate.ref_id) - && !candidate - .states - .iter() - .any(|state| state.eq_ignore_ascii_case("covered")) - }) - .find(|candidate| { - words(candidate.name.as_deref().unwrap_or_default()) - .iter() - .take(3) - .any(|word| wanted.contains(word)) - }) - .cloned() -} - -/// The lower-case words of `text`. -fn words(text: &str) -> Vec { - text.split(|character: char| !character.is_alphanumeric()) - .filter(|word| !word.is_empty()) - .map(str::to_lowercase) - .collect() -} - /// The `fields` a slot may still take: of no kind struck this step, and /// neither filled this step (`filled`, by ref) nor holding a text this step /// delivered (`delivered`). diff --git a/crates/tinycomputer-engine/src/agentic/flow/enter/mod.rs b/crates/tinycomputer-engine/src/agentic/flow/enter/mod.rs index c383fec8..0876609b 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/enter/mod.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/enter/mod.rs @@ -6,11 +6,15 @@ //! (`deliver_text`), top to bottom in screen order. A slot with no visible //! field first runs a short `do` loop to reveal one. //! -//! `assign` matches slots to fields and asks which the form wants; `fill` -//! delivers each text and verifies it. +//! `assign` matches slots to fields and asks which the form wants; `open` +//! shows a slot's box when the screen has none for it; `fill` delivers each +//! text and verifies it. mod assign; mod fill; +mod open; +#[cfg(test)] +pub(super) use open::named_opener; use std::collections::BTreeSet; diff --git a/crates/tinycomputer-engine/src/agentic/flow/enter/open.rs b/crates/tinycomputer-engine/src/agentic/flow/enter/open.rs new file mode 100644 index 00000000..55dca52d --- /dev/null +++ b/crates/tinycomputer-engine/src/agentic/flow/enter/open.rs @@ -0,0 +1,193 @@ +//! Showing a slot's box when the screen has none for it: pressing the +//! control a slot's own name names ("To BLR" for the slot "to"), one slot at +//! a time, or else a short `do` loop that reveals the fields. + +use std::collections::BTreeSet; + +use tinycomputer_bus::{JevOperation, Slot}; + +use crate::agentic::flow::{ + AgentBackend, FlowRun, Halt, StepLog, + ask::{self, Questions, corroborate, probability}, + view::{Candidate, Screen, is_destructive, label}, +}; + +use super::{OPENER_FLOOR, REVEAL_TURNS, editable, names}; + +impl FlowRun<'_, B> { + /// Presses the control on `screen` whose label holds a pending slot's + /// own word (`named_opener`); `true` when it did. A control named by + /// the slot, a search link for the "search box", is how the field + /// shows: live, the turns that look for a way in hesitated over it, and + /// the search was never typed. + async fn open_by_name( + &mut self, + log: &mut StepLog, + screen: &Screen, + slots: &[Slot], + pending: &BTreeSet, + ) -> Result { + let Some(opener) = named_opener(screen, slots, pending) + .filter(|opener| !is_destructive(opener, screen, &self.stop_before)) + else { + return Ok(false); + }; + // A shared word is a hint, not a reason to press: a link named + // "Email us" shares the slot "email", and pressing it left the form. + let purpose = format!("click to show the box for: {}", names(slots, pending)); + let answers = self + .ask( + log, + ask::request( + self.model(), + self.state(screen, &purpose), + Questions::default().with( + "confirm", + corroborate(&purpose, &opener, self.include_values), + ), + ), + ) + .await?; + if probability(&answers, "confirm").is_none_or(|yes| yes < OPENER_FLOOR) { + return Ok(false); + } + let pressed = opener.clone(); + let reply = self + .act( + log, + "click (show the field)", + Some(&opener), + move |backend| backend.execute(JevOperation::Click, Some(pressed), None), + ) + .await?; + if reply.ok { + self.history.push(format!( + "pressed {} to show the field for {}", + label(&opener), + names(slots, pending) + )); + } + Ok(reply.ok) + } + + /// Runs a short `do` loop that shows the fields for the `pending` slots, + /// any field at all when `no_fields` showed. A field that cannot be revealed + /// is looked for another way, or found not to be asked for; it is not a + /// failure. + pub(super) async fn reveal_fields( + &mut self, + log: &mut StepLog, + slots: &[Slot], + pending: &BTreeSet, + no_fields: bool, + ) -> Result<(), Halt> { + let reveal = if no_fields { + format!("show the editable fields for: {}", names(slots, pending)) + } else { + format!("show the fields for: {}", names(slots, pending)) + }; + match self.accomplish(log, &reveal, REVEAL_TURNS).await { + Err(Halt::Failed(note)) => self + .history + .push(format!("could not reveal the fields ({note})")), + other => { + other?; + } + } + Ok(()) + } + + /// Presses the opener of the first slot of `pending` whose own opener was + /// not looked for yet (`opened`), noting each slot it looks for; `true` + /// when it pressed one. + pub(super) async fn open_next_box( + &mut self, + log: &mut StepLog, + screen: &Screen, + slots: &[Slot], + pending: &BTreeSet, + opened: &mut BTreeSet, + ) -> Result { + for index in pending.difference(opened).copied().collect::>() { + opened.insert(index); + if self + .open_by_name(log, screen, slots, &BTreeSet::from([index])) + .await? + { + return Ok(true); + } + } + Ok(false) + } +} + +/// Words of a slot's name that say only that it is a box, or name a kind +/// of control rather than what the slot is. +const BOX_WORDS: &[&str] = &[ + "field", "box", "input", "bar", "the", "your", "text", "here", "link", "button", "menu", "icon", +]; + +/// Short words of a place slot ("to") that name its box only as the first +/// word of a control's label ("To BLR, Bengaluru"): anywhere else in a +/// label, "to" is any sentence's ("Tap to add a return date"). +const PLACE_LEADS: &[&str] = &["to", "via"]; + +/// A control on `screen` that is no field itself and whose label holds a +/// word of a pending slot's name (four letters or more, not a box word), +/// or begins with its short place word ([`PLACE_LEADS`]): the link or +/// button that shows the slot's field, such as a store's search link for +/// the slot "search box", or a flight form's "To BLR" for the slot "to". +pub(in crate::agentic::flow) fn named_opener( + screen: &Screen, + slots: &[Slot], + pending: &BTreeSet, +) -> Option { + let named = pending + .iter() + .filter_map(|index| slots.get(*index)) + .flat_map(|slot| words(&slot.slot)) + .collect::>(); + let wanted = named + .iter() + .filter(|word| word.chars().count() > 3 && !BOX_WORDS.contains(&word.as_str())) + .cloned() + .collect::>(); + let leads = named + .iter() + .filter(|word| PLACE_LEADS.contains(&word.as_str())) + .cloned() + .collect::>(); + if wanted.is_empty() && leads.is_empty() { + return None; + } + let fields = editable(screen); + screen + .candidates + .iter() + .filter(|candidate| { + matches!(candidate.role.as_str(), "link" | "button") + && candidate + .available_actions + .iter() + .any(|action| action == "Click") + && !fields.iter().any(|field| field.ref_id == candidate.ref_id) + && !candidate + .states + .iter() + .any(|state| state.eq_ignore_ascii_case("covered")) + }) + .find(|candidate| { + let label = words(candidate.name.as_deref().unwrap_or_default()); + label.iter().take(3).any(|word| wanted.contains(word)) + || label.first().is_some_and(|word| leads.contains(word)) + }) + .cloned() +} + +/// The lower-case words of `text`. +fn words(text: &str) -> Vec { + text.split(|character: char| !character.is_alphanumeric()) + .filter(|word| !word.is_empty()) + .map(str::to_lowercase) + .collect() +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests.rs index e39d5cd5..b09e4131 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests.rs @@ -31,6 +31,7 @@ mod hedge_tests; mod helpers_tests; mod journal_tests; mod pick_tests; +mod picker_tests; mod quorum_tests; mod reflection_tests; mod split_tests; diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/brief_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/brief_tests.rs index b7646ba1..3cc919db 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/brief_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/brief_tests.rs @@ -363,3 +363,70 @@ async fn a_long_goal_is_clipped_in_the_brief() { assert_eq!(goal.chars().count(), 601); assert!(goal.ends_with('…')); } + +/// A browser screen of `candidates`, in the order given. +fn long_screen(candidates: Vec) -> Screen { + Screen { + app: "browser".to_owned(), + window: None, + surface: "sheet".to_owned(), + candidates, + context: Vec::new(), + unexplored: Vec::new(), + text_nodes: Vec::new(), + } +} + +/// `count` buttons named `prefix` and their number, each `covered` or not. +fn buttons(prefix: &str, count: u32, covered: bool) -> Vec { + (0..count) + .map(|index| { + let mut control = node( + &format!("{prefix} {index}"), + "button", + &["Click"], + &["main"], + f64::from(index), + ); + if covered { + control.states = vec!["covered".to_owned()]; + } + control + }) + .collect() +} + +/// The element lines Jev is shown for `screen`. +fn element_lines(screen: &Screen) -> Vec { + ask::state(screen, "x", &[], false)["elements"]["untrusted_accessibility_data"] + .as_array() + .unwrap() + .iter() + .map(|line| line.as_str().unwrap().to_owned()) + .collect() +} + +#[test] +fn a_long_screen_shows_jev_what_is_in_view_and_keeps_room_for_the_rest() { + // Live, a sign-up pop-up a long page drew at the end of its document + // fell outside the first 120 elements, behind the covered page. + let mut candidates = buttons("Page", 130, true); + for name in ["close", "Enter your Mobile Number"] { + candidates.push(node(name, "button", &["Click"], &["dialog"], 200.0)); + } + let lines = element_lines(&long_screen(candidates)); + assert_eq!(lines.len(), 120); + assert!(lines[0].contains("Page 0"), "kept in screen order"); + assert!(lines[118].contains("close"), "{:?}", &lines[115..]); + assert!(lines[119].contains("Enter your Mobile Number")); + + // A calendar open in front fills no more than three quarters of the + // room: the guests button it covers, early on the page, stays. + let mut candidates = buttons("Search form", 3, true); + candidates.extend(buttons("Day", 140, false)); + candidates.extend(buttons("Footer", 50, true)); + let lines = element_lines(&long_screen(candidates)); + assert_eq!(lines.len(), 120); + assert!(lines[0].contains("Search form 0"), "{:?}", &lines[..4]); + assert_eq!(lines.iter().filter(|line| line.contains("Day")).count(), 90); +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/choose_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/choose_tests.rs index 5f383d6b..57626d17 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/choose_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/choose_tests.rs @@ -237,6 +237,18 @@ fn a_date_is_told_from_other_options_and_containers_give_way() { "a lone match stays" ); assert!(closest(Vec::new()).is_empty()); + + // A list's option stays however long its label: one airport row says + // much more than a footer link that only names the city. + let link = node("Mumbai", "link", &["Click"], &[], 0.0); + let row = node( + "BOM Mumbai, India Chhatrapati Shivaji International Airport 3 Nearby Airports found", + "option", + &["Click"], + &[], + 0.0, + ); + assert_eq!(names(closest(vec![link.clone(), row.clone()])).len(), 2); } #[tokio::test] @@ -455,6 +467,27 @@ fn in_region_prefers_the_ancestor_named_region_but_keeps_every_match_when_none_i // `pick_option` falls back to the unnarrowed pool when nothing on the // page names the region at all. assert!(in_region(&unrelated, "")); + + // A placing word alone is in too many labels to place an option: only a + // container whose name begins with it holds one. + let route = node( + "Delhi to Mumbai flights", + "link", + &["Click"], + &["root", "list \"Popular routes to Mumbai\""], + 30.0, + ); + let airport = node( + "BOM Mumbai, India", + "option", + &["Click"], + &["root", "dialog \"To\"", "listbox \"Airports\""], + 40.0, + ); + assert!(!in_region(&route, "to")); + assert!(in_region(&airport, "to")); + // A longer region name still counts on the option's own label. + assert!(in_region(&route, "popular routes")); } #[test] @@ -487,3 +520,34 @@ fn a_day_in_a_strip_of_dates_is_found_by_its_short_label() { ); assert!(shows_date("7 Sept", &date_words("7 September"))); } + +#[test] +fn a_date_shown_under_the_fields_own_name_is_already_chosen() { + // Live, the day was pressed and the departure button showed it, but the + // step looked for a box to type the date into and failed. A day of the + // calendar names no field, so it never counts. + use super::steps::date_shown_in; + let screen = |names: &[&str]| Screen { + app: "browser".to_owned(), + window: None, + surface: "window".to_owned(), + candidates: names + .iter() + .map(|name| node(name, "button", &["Click"], &["form"], 0.0)) + .collect(), + context: Vec::new(), + unexplored: Vec::new(), + text_nodes: Vec::new(), + }; + let set = screen(&["Departure Thu, 22 Oct", "22 6845"]); + assert_eq!( + date_shown_in(&set, "departure date", "22 October 2026").and_then(|holder| holder.name), + Some("Departure Thu, 22 Oct".to_owned()) + ); + let unset = screen(&["Departure Fri, 09 Oct", "Thursday, October 22, 2026"]); + assert!(date_shown_in(&unset, "departure date", "22 October 2026").is_none()); + assert!( + date_shown_in(&set, "date", "22 October 2026").is_none(), + "no word names the field" + ); +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs index cd69140c..773237f0 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs @@ -653,7 +653,7 @@ async fn a_step_finding_nothing_to_press_answers_the_dialog_the_task_opened() { let run = run_with( App::with(|sim| sim.obstacle = true), json!({"app": "browser", "steps": ["choose Wednesday 7 October 2026 in the date picker"]}), - |_| {}, + |request| request.dialog_left_open = true, |id, question, sim| { let answering = serde_json::to_string(question) .unwrap() @@ -689,7 +689,8 @@ async fn a_step_finding_nothing_to_press_answers_the_dialog_the_task_opened() { async fn a_control_the_dialogs_own_bar_covers_is_pressed_and_one_behind_it_is_not() { // Live, a seat table's lower rows sat under its "Pay" bar and were never // offered; a press scrolls such a control out from under the bar. A - // browser run takes a dialog at its first look as the task's own. + // browser run takes a dialog at its first look as the task's own when + // the run before it left that dialog open. let pressing = |wanted: &'static str| { move |id: &str, question: &Question, sim: &Sim| match id { "done" => Some(noul(if sim.obstacle { 0.05 } else { 0.95 })), @@ -707,7 +708,7 @@ async fn a_control_the_dialogs_own_bar_covers_is_pressed_and_one_behind_it_is_no sim.quirks.insert(Quirk::BarOverSheet); }), json!({"app": "browser", "steps": ["keep editing the draft"]}), - |_| {}, + |request| request.dialog_left_open = true, pressing("Keep Editing"), ) .await; @@ -734,7 +735,7 @@ async fn a_control_the_dialogs_own_bar_covers_is_pressed_and_one_behind_it_is_no sim.quirks.insert(Quirk::Covered); }), json!({"app": "browser", "steps": ["start a new email message"]}), - |_| {}, + |request| request.dialog_left_open = true, pressing("New Message"), ) .await; @@ -782,25 +783,22 @@ fn a_dialog_the_task_worked_in_is_in_the_way_of_the_next_step() { let (window, sheet) = (page_at("window", 0), page_at("sheet", 3)); let mut front = Front::default(); front.act("browse https://flights.test/", false); - assert!(front.look(&window, at, true).is_none()); + assert!(front.look(&window, at).is_none()); front.act("click", true); - assert!( - front.look(&sheet, at, true).is_some(), - "the press opened it" - ); + assert!(front.look(&sheet, at).is_some(), "the press opened it"); front.next_step(); - assert!(front.opened_dialog, "the next step answers what it asks"); + assert!(front.opened_dialog(), "the next step answers what it asks"); front.act("click", true); - front.look(&sheet, at, true); + front.look(&sheet, at); assert!( - front.opened_dialog, + front.opened_dialog(), "still its own within the step that works in it" ); front.next_step(); - assert!(!front.opened_dialog, "a step later, it is in the way"); - front.look(&sheet, at, true); + assert!(!front.opened_dialog(), "a step later, it is in the way"); + front.look(&sheet, at); assert!( - !front.opened_dialog, + !front.opened_dialog(), "and it does not become the task's again" ); @@ -808,20 +806,38 @@ fn a_dialog_the_task_worked_in_is_in_the_way_of_the_next_step() { for action in ["scroll", "click (clear distraction)", "click (dismiss)"] { let mut front = Front::default(); front.act("browse https://flights.test/", false); - front.look(&window, at, true); + front.look(&window, at); front.act(action, true); - assert!(front.look(&sheet, at, true).is_none(), "{action}"); - assert!(!front.opened_dialog, "{action}"); + assert!(front.look(&sheet, at).is_none(), "{action}"); + assert!(!front.opened_dialog(), "{action}"); } // Opening an address leaves what was in front behind. let mut front = Front::default(); front.act("browse https://flights.test/", false); - front.look(&window, at, true); + front.look(&window, at); front.act("click", true); - front.look(&sheet, at, true); + front.look(&sheet, at); front.act("browse https://flights.test/next", false); - assert!(!front.opened_dialog); + assert!(!front.opened_dialog()); +} + +#[test] +fn a_dialog_at_a_runs_first_look_is_the_tasks_only_when_the_run_before_left_it() { + use crate::agentic::flow::front::Front; + let at = Some("https://flights.test/"); + let sheet = page_at("sheet", 3); + // A pop-up the page opened itself, at a rescue's first look. + let mut front = Front::new(false); + assert!(front.look(&sheet, at).is_none()); + assert!( + !front.opened_dialog(), + "the page's own, cleared like any other" + ); + // The dialog the task's run before left open. + let mut front = Front::new(true); + assert!(front.look(&sheet, at).is_some()); + assert!(front.opened_dialog(), "the task's current stage"); } #[tokio::test] diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/enter_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/enter_tests.rs index f05aced7..35ae1c5a 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/enter_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/enter_tests.rs @@ -426,3 +426,51 @@ async fn a_control_named_by_the_slot_opens_its_box_only_when_jev_agrees() { refused.result.steps[0].actions ); } + +#[test] +fn a_short_place_word_names_its_box_only_as_a_labels_first_word() { + // Live, a flight form drew its place boxes as buttons ("From DEL", + // "To BLR"), and the "to" box was never opened: "to" is too short a + // word to look for anywhere in a label ("Tap to add a return date"). + use super::enter::named_opener; + let slots = vec![tinycomputer_bus::Slot { + slot: "to".to_owned(), + text: "Mumbai".to_owned(), + }]; + let screen = Screen { + app: "browser".to_owned(), + window: None, + surface: "window".to_owned(), + candidates: vec![ + node( + "From DEL, Delhi Airport India", + "button", + &["Click"], + &["form"], + 0.0, + ), + node( + "Return Tap to add a return date", + "button", + &["Click"], + &["form"], + 1.0, + ), + node( + "To BLR, Bengaluru Airport India", + "button", + &["Click"], + &["form"], + 2.0, + ), + ], + context: Vec::new(), + unexplored: Vec::new(), + text_nodes: Vec::new(), + }; + let opener = named_opener(&screen, &slots, &BTreeSet::from([0])).unwrap(); + assert_eq!( + opener.name.as_deref(), + Some("To BLR, Bengaluru Airport India") + ); +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/grounding_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/grounding_tests.rs index fbd86b1a..65c80b12 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/grounding_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/grounding_tests.rs @@ -58,6 +58,75 @@ async fn a_large_screen_is_narrowed_by_region_before_choosing() { ); } +#[test] +fn a_small_region_is_never_cut_from_a_crowded_knockout() { + use crate::agentic::flow::ground::{Regions, knockout_groups}; + + let member = |name: String, region: &str| { + node( + &name, + "button", + &["Click"], + &["window \"Flights\"", region], + 10.0, + ) + }; + let routes = (0..450) + .map(|index| member(format!("Route {index}"), "list \"Popular routes\"")) + .collect::>(); + let airports = (0..5) + .map(|index| member(format!("Airport {index}"), "listbox \"Airports\"")) + .collect::>(); + let pool = [routes.clone(), airports.clone()].concat(); + let regions: Regions = vec![ + ("list \"Popular routes\"".to_owned(), routes), + ("listbox \"Airports\"".to_owned(), airports), + ]; + let groups = knockout_groups(&pool, Some(®ions)); + assert!(groups.len() <= ask::CAP, "{}", groups.len()); + let offered = groups + .iter() + .flat_map(|(_, group)| group) + .filter_map(|candidate| candidate.name.clone()) + .collect::>(); + for index in 0..5 { + assert!( + offered.contains(&format!("Airport {index}")), + "the small region, last on the page, is offered whole" + ); + } + // With no regions, the knockout takes the first groups in page order. + assert_eq!(knockout_groups(&pool, None).len(), ask::CAP); + + // Many small regions out of view give up their chunks before a region in + // view: live, a travellers pop-up drawn after twenty regions of links out + // of view lost its "Done". + let mut regions: Regions = (0..30) + .map(|index| { + let mut link = member(format!("Link {index}"), "contentinfo"); + link.states = vec!["offscreen".to_owned()]; + (format!("list {index}"), vec![link]) + }) + .collect(); + regions.push(( + "dialog \"Travellers\"".to_owned(), + vec![member("Done".to_owned(), "dialog \"Travellers\"")], + )); + let pool = regions + .iter() + .flat_map(|(_, members)| members.clone()) + .collect::>(); + let groups = knockout_groups(&pool, Some(®ions)); + assert_eq!(groups.len(), ask::CAP); + assert!( + groups + .iter() + .flat_map(|(_, group)| group) + .any(|candidate| candidate.name.as_deref() == Some("Done")), + "the region in view keeps its chunk" + ); +} + #[tokio::test] async fn one_crowded_region_falls_back_to_a_knockout() { let run = run_with( diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/pick_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/pick_tests.rs index 7cb730f3..2203c57f 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/pick_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/pick_tests.rs @@ -120,6 +120,29 @@ async fn pick_ranks_the_list_that_has_prices_not_the_longest_one() { assert!(run.result.steps[0].note.contains("ranked")); } +#[tokio::test] +async fn pick_ranks_a_list_it_can_open_not_bare_fares() { + // Live, "the cheapest flight" ranked a list of bare fares and took + // "₹ 6,054", which had nothing to open. + let app = flights(); + app.sim().fare_chips = 8; + let run = run( + app, + json!({"app": "Mail", "steps": [ + {"pick": {"from": "the flight results", "by": "lowest price", "into": "flight"}} + ]}), + ) + .await; + assert_eq!( + run.result.stop, + FlowStopReason::Completed, + "{:?}", + run.result.steps + ); + assert_eq!(run.app.sim().picked, ["@s:select-1"]); + assert!(run.result.vars["flight"].starts_with("IndiGo")); +} + #[tokio::test] async fn pick_asks_jev_when_the_criterion_needs_judgement() { let run = run_with( diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/picker_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/picker_tests.rs new file mode 100644 index 00000000..61e9b6fb --- /dev/null +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/picker_tests.rs @@ -0,0 +1,193 @@ +//! Pop-ups and calendars a step works with: a closer that goes with its +//! pop-up, a calendar the task picked in closed for a press behind it, in +//! its own step or a later one, and a date picked from a calendar already +//! open. + +use super::*; + +/// Answers that press the toast's "Close" and never judge the step done. +fn press_close(id: &str, question: &Question, _: &Sim) -> Option { + match id { + "done" | "blocked" => Some(noul(0.05)), + "move" => Some(pick(question, "activate", 0.9)), + _ if id == "target" || id == "region" || id.starts_with("group_") => { + Some(pick(question, "Close", 0.9)) + } + _ => None, + } +} + +#[tokio::test] +async fn a_closer_that_goes_with_its_pop_up_ends_a_step_closing_it() { + // Live, an offer pop-up drawn without a dialog's role closed at the + // first press, and the step stalled: the judge could not tell whether + // "declining optional cookies" was done with no cookie banner shown. + let toast = || { + App::with(|sim| { + sim.quirks.insert(Quirk::PromoToast); + }) + }; + let run = run_with( + toast(), + json!({"app": "Mail", "steps": ["close any login or offer pop-up, declining optional cookies"]}), + |request| request.disabled_loops.push(FlowLoop::Attention), + press_close, + ) + .await; + assert_eq!(run.result.stop, FlowStopReason::Completed); + assert_eq!(run.app.sim().clicks, ["Close"]); + assert!( + run.result.steps[0] + .note + .contains("closed with what it was on"), + "{}", + run.result.steps[0].note + ); + let unrelated = run_with( + toast(), + json!({"app": "Mail", "steps": ["archive the message"]}), + |request| { + request.disabled_loops.push(FlowLoop::Attention); + request.max_actions = 1; + }, + press_close, + ) + .await; + assert_ne!( + unrelated.result.stop, + FlowStopReason::Completed, + "a step that closes nothing is not finished by a closer" + ); +} + +/// Answers that open the calendar, pick a day, then press "Find flights", +/// judging the step done once that press went through. +fn pick_then_find(id: &str, question: &Question, sim: &Sim) -> Option { + let found = sim.clicks.iter().any(|click| click == "Find flights"); + let next = if sim.fields.contains_key("Departure") { + "Find flights" + } else if sim + .booking + .as_ref() + .is_some_and(|booking| booking.calendar.is_some()) + { + "12 September 2026" + } else { + "Departure" + }; + match id { + "done" => Some(noul(if found { 0.95 } else { 0.05 })), + "blocked" => Some(noul(0.05)), + "move" => Some(pick(question, "activate", 0.9)), + _ if id == "target" || id == "region" || id.starts_with("group_") => { + Some(pick(question, next, 0.9)) + } + _ => None, + } +} + +#[tokio::test] +async fn a_calendar_the_task_picked_in_is_closed_for_a_press_behind_it() { + // Live, a calendar the task opened stayed in front of the guests and + // Search buttons once both dates were picked, and every press behind + // it was refused as lying behind the task's own dialog. + let run = run_with( + App::with(|sim| { + sim.booking = Some(Booking::default()); + sim.quirks.insert(Quirk::CalendarStaysOpen); + }), + json!({"app": "Mail", "steps": ["pick 12 September 2026 as the departure date, then press Find flights"]}), + |request| request.disabled_loops.push(FlowLoop::Attention), + pick_then_find, + ) + .await; + assert_eq!( + run.result.stop, + FlowStopReason::Completed, + "{:?}", + run.result.steps + ); + let sim = run.app.sim(); + assert_eq!(sim.presses, ["escape"]); + assert_eq!( + sim.clicks, + ["Departure", "12 September 2026", "Find flights"] + ); + assert!( + run.result.steps[0] + .actions + .iter() + .any(|action| action.action == "press escape (uncover)") + ); +} + +#[tokio::test] +async fn a_calendar_a_step_before_picked_in_is_closed_for_a_press_behind_it() { + // Live, the dates were picked a step each, and the calendar left open + // in front of the Search button was handed back to the page with the + // next step: every press behind it was refused, and no step closed it. + let run = run_with( + App::with(|sim| { + sim.booking = Some(Booking::default()); + sim.quirks.insert(Quirk::CalendarStaysOpen); + }), + json!({"app": "Mail", "steps": [ + "pick 12 September 2026 as the departure date", + "press Find flights" + ]}), + |request| request.disabled_loops.push(FlowLoop::Attention), + |id, question, sim| { + if id != "done" { + return pick_then_find(id, question, sim); + } + let finished = if text_of(question, "step").contains("find flights") { + sim.clicks.iter().any(|click| click == "Find flights") + } else { + sim.fields.contains_key("Departure") + }; + Some(noul(if finished { 0.95 } else { 0.05 })) + }, + ) + .await; + assert_eq!( + run.result.stop, + FlowStopReason::Completed, + "{:?}", + run.result.steps + ); + let sim = run.app.sim(); + assert_eq!(sim.presses, ["escape"]); + assert_eq!( + sim.clicks, + ["Departure", "12 September 2026", "Find flights"] + ); +} + +#[tokio::test] +async fn a_date_whose_calendar_is_open_is_picked_without_pressing_its_button() { + // Live, a check-out date's step found the calendar open from the + // check-in, pressed its button, which closed it, four times, and never + // picked the day. + let run = run_with( + App::with(|sim| { + sim.booking = Some(Booking { + calendar: Some(8), + ..Booking::default() + }); + sim.quirks.insert(Quirk::CalendarStaysOpen); + }), + json!({"app": "Mail", "steps": [{"enter": {"departure date": "12 September 2026"}}]}), + |request| request.disabled_loops.push(FlowLoop::Attention), + |_, _, _| None, + ) + .await; + assert_eq!( + run.result.stop, + FlowStopReason::Completed, + "{:?}", + run.result.steps + ); + let sim = run.app.sim(); + assert_eq!(sim.clicks, ["12 September 2026"]); + assert_eq!(sim.fields["Departure"], "12 September 2026"); +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/places.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/places.rs index 29da5249..c89a095d 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/places.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/places.rs @@ -47,8 +47,16 @@ pub(super) struct Places { /// still to come, as a ride app's did live ("Allow location access", /// "Search in a different city"). pub(super) starters: bool, + /// Whether each box shows only once its own button is pressed ("From + /// …", "To …"), one at a time, as a flight form draws them. + pub(super) behind_buttons: bool, + /// The box whose button was pressed last (`behind_buttons`). + pub(super) door: Option, } +/// The buttons that show the boxes of a form drawn `behind_buttons`. +const DOORS: [&str; 2] = ["From", "To"]; + /// The places suggested for `typed`: each one that holds every typed word. fn suggested(typed: &str) -> Vec<&'static str> { let words = typed @@ -77,6 +85,25 @@ pub(super) fn places_widget( ) { let form = [root, "group \"Get a ride\""]; for (index, name) in PLACE_BOXES.iter().enumerate() { + if places.behind_buttons { + // The button shows a place once one is picked, not what is typed. + let shown = sim + .fields + .get(*name) + .filter(|_| places.picked.contains(*name)) + .cloned() + .unwrap_or_else(|| "Select a place".to_owned()); + candidates.push(node( + &format!("{} {shown}", DOORS[index]), + "button", + &["Click"], + &form, + 190.0 + 40.0 * f64::from(u8::try_from(index).unwrap()), + )); + if places.door.as_deref() != Some(*name) { + continue; + } + } let mut field = node( name, "textbox", @@ -165,6 +192,14 @@ pub(super) fn drop_unpicked(sim: &mut Sim) { /// whichever that is; a press anywhere else but the box moves the focus on, /// so the list closes and drops the box's unpicked text, as a page does. pub(super) fn press_place(sim: &mut Sim, name: &str) -> bool { + if let Some(places) = sim.places.as_mut().filter(|places| places.behind_buttons) + && let Some(index) = DOORS + .iter() + .position(|door| name.starts_with(&format!("{door} "))) + { + places.door = Some(PLACE_BOXES[index].to_owned()); + return true; + } let Some(open) = sim.places.as_ref().and_then(|places| places.open.clone()) else { return false; }; @@ -183,6 +218,7 @@ pub(super) fn press_place(sim: &mut Sim, name: &str) -> bool { if let Some(places) = sim.places.as_mut() { places.picked.insert(open.clone()); places.open = None; + places.door = None; } sim.fields.insert(open, place.to_owned()); true diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/screens.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/screens.rs index 5779d16c..c7706d00 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/screens.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/screens.rs @@ -131,6 +131,7 @@ pub(super) struct Booking { /// What pressing `name` does to the booking form. pub(super) fn press_booking(sim: &mut Sim, name: &str) { + let stays_open = sim.has(Quirk::CalendarStaysOpen); let Some(booking) = sim.booking.as_mut() else { return; }; @@ -142,6 +143,8 @@ pub(super) fn press_booking(sim: &mut Sim, name: &str) { sim.fields .insert("Destination".to_owned(), "Srinagar, SXR".to_owned()); } + // A calendar that stays open is closed by its own button again. + "Departure" if stays_open && booking.calendar.is_some() => booking.calendar = None, "Departure" => booking.calendar = Some(8), "Next Month" => booking.calendar = booking.calendar.map(|month| (month + 1) % 12), // The calendar's own aggregated-label container also ends with @@ -150,7 +153,9 @@ pub(super) fn press_booking(sim: &mut Sim, name: &str) { // a bug that let production code ground and press that container // instead of a day must not be able to pass this simulated test. day if booking.calendar.is_some() && is_single_day_label(day) => { - booking.calendar = None; + if !stays_open { + booking.calendar = None; + } sim.fields.insert("Departure".to_owned(), day.to_owned()); } _ => {} @@ -257,6 +262,13 @@ pub(super) fn booking_widget( } } candidates.push(node("Departure", "button", &["Click"], &widget, 120.0)); + if sim.has(Quirk::CalendarStaysOpen) { + let mut find = node("Find flights", "button", &["Click"], &widget, 200.0); + if booking.calendar.is_some() { + find.states = vec!["covered".to_owned()]; + } + candidates.push(find); + } if let Some(month) = booking.calendar { // The date field's own label lists the whole open calendar. let listing = (1..=28) @@ -380,6 +392,19 @@ pub(super) fn result_cards( ..Candidate::default() }); } + for chip in 0..sim.fare_chips { + text_nodes.push(Candidate { + role: "text".to_owned(), + value: Some(json!(format!("₹ {}", 1_000 + chip))), + path: vec![ + root.to_owned(), + "list \"Fares\"".to_owned(), + format!("listitem #{}", chip + 1), + ], + order: 300 + chip * 10, + ..Candidate::default() + }); + } for day in 0..sim.date_strip { let path = vec![ root.to_owned(), diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs index 645678c5..4d6ab0e4 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs @@ -56,6 +56,9 @@ pub(super) enum Quirk { SearchBehindLink, /// The search behind "Search mail" shows. SearchOpen, + /// The booking calendar is a dialog that stays open once a day is + /// picked, covering a "Find flights" button, until Escape closes it. + CalendarStaysOpen, } #[derive(Debug, Default)] @@ -76,6 +79,9 @@ pub(super) struct Sim { pub(super) selected_result: Option, /// Days in a date strip above the results, a longer list than they are. pub(super) date_strip: usize, + /// Bare fares in a list above the results, cheaper than every flight, + /// with nothing to open. + pub(super) fare_chips: usize, pub(super) extra_buttons: usize, /// A booking form with an autocomplete destination and a calendar. pub(super) booking: Option, @@ -240,9 +246,8 @@ impl App { } overlays(&sim, &root, &mut candidates); let text_nodes = result_cards(&sim, &root, &mut candidates); - let mut surface = "window".to_owned(); + let surface = surface_of(&sim); if sim.obstacle { - surface = "sheet".to_owned(); obstacle_sheet(&sim, &mut candidates); } Screen { @@ -259,6 +264,23 @@ impl App { } } +/// What the simulated page shows in front: the obstacle's sheet, a +/// calendar that stays open as a dialog, or the window. +fn surface_of(sim: &Sim) -> String { + if sim.obstacle { + "sheet".to_owned() + } else if sim.has(Quirk::CalendarStaysOpen) + && sim + .booking + .as_ref() + .is_some_and(|booking| booking.calendar.is_some()) + { + "dialog".to_owned() + } else { + "window".to_owned() + } +} + /// The reply to a click on `name` that something on the simulated page /// refuses or takes over, or `None` when the click goes through: an /// unclickable page, a consent banner whose own buttons close it, or a @@ -290,6 +312,15 @@ fn refused_click(sim: &mut Sim, name: &str) -> Option { } return Some(covered("consent")); } + if name == "Find flights" + && sim.has(Quirk::CalendarStaysOpen) + && sim + .booking + .as_ref() + .is_some_and(|booking| booking.calendar.is_some()) + { + return Some(covered("calendar")); + } sim.has(Quirk::Drawer).then(|| covered("drawer")) } @@ -459,6 +490,11 @@ impl AgentBackend for App { sim.obstacle = false; sim.quirks.remove(&Quirk::Drawer); drop_unpicked(&mut sim); + if sim.has(Quirk::CalendarStaysOpen) + && let Some(booking) = sim.booking.as_mut() + { + booking.calendar = None; + } } _ => {} } diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/suggestion_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/suggestion_tests.rs index 93a0c9b4..9e33338f 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/suggestion_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/suggestion_tests.rs @@ -456,3 +456,102 @@ fn a_box_that_appears_beside_the_suggestions_is_never_taken_for_one() { [row] ); } + +#[test] +fn a_matching_row_already_shown_counts_only_in_view_and_in_front() { + // A place box's own list is in view and in front: live, a footer link + // "hotels in Goa" at the foot of the page was pressed as the place typed. + // A new row still counts out of view, as one its list scrolled away. + use super::steps::fresh_rows; + let place_box = node("Where to?", "textbox", &["Click", "SetValue"], &[], 0.0); + let row = |name: &str, role: &str, states: &[&str], y: f64| { + let mut row = node(name, role, &["Click"], &[], y); + row.states = states.iter().map(|state| (*state).to_owned()).collect(); + row + }; + let before = vec![ + place_box.clone(), + row("Goa beaches", "button", &[], 1.0), + row("hotels in Goa", "link", &["offscreen"], 2.0), + row("Goa villas", "link", &["covered"], 3.0), + ]; + let shown = before + .iter() + .map(|candidate| (candidate.role.as_str(), candidate.name.as_deref())) + .collect::>(); + let mut candidates = before.clone(); + candidates.push(row( + "Goa International Airport", + "option", + &["offscreen"], + 4.0, + )); + let screen = Screen { + app: "browser".to_owned(), + window: None, + surface: "window".to_owned(), + candidates, + context: Vec::new(), + unexplored: Vec::new(), + text_nodes: Vec::new(), + }; + let fresh = fresh_rows(&screen, &shown, &place_box, "Goa", &[], true); + assert_eq!( + fresh + .iter() + .filter_map(|candidate| candidate.name.as_deref()) + .collect::>(), + ["Goa beaches", "Goa International Airport"] + ); +} + +#[tokio::test] +async fn enter_opens_each_place_box_behind_its_own_button_in_turn() { + // Live, a flight form drew its place boxes as buttons ("From DEL", "To + // BLR"), each box showing only once its button is pressed. The "to" box + // was never opened, and the place was pressed in a link at the foot of + // the page instead. + let run = run_with( + App::with(|sim| { + sim.places = Some(Places { + behind_buttons: true, + ..Places::default() + }); + }), + json!({"app": "Mail", "steps": [ + {"enter": {"from": "Connaught Place", "to": "Indira Gandhi International Airport"}} + ]}), + |_| {}, + |id, question, sim| { + if id.starts_with("slot_") { + let purpose = text_of(question, "purpose"); + let field = if purpose.contains("the from ") { + "Pickup location" + } else { + "Dropoff location" + }; + return Some(pick(question, field, 0.9)); + } + suggesting(id, question, sim, 0.9) + }, + ) + .await; + assert_eq!( + run.result.stop, + FlowStopReason::Completed, + "{:?}", + run.result.steps + ); + let sim = run.app.sim(); + assert_eq!( + sim.fields["Pickup location"], + "Connaught Place New Delhi, Delhi, India" + ); + assert_eq!( + sim.fields["Dropoff location"], + "Indira Gandhi International Airport New Delhi, Delhi, India", + "{:?} {:?}", + run.result.steps[0].actions, + sim.clicks + ); +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/front.rs b/crates/tinycomputer-engine/src/agentic/flow/front.rs index 7e4676c4..5cc92255 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/front.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/front.rs @@ -2,12 +2,15 @@ //! there: a dialog the task opened is the flow's next stage, never cleared //! as a distraction or an obstacle, in this step or the next. -use super::view::Screen; +use super::{steps::names_a_month, view::Screen}; /// Controls a layer must cover, beyond what was covered before the press /// that opened it, before it counts as in front of the page. pub(super) const LAYER_COVERS: usize = 3; +/// Day cells nothing covers that make what is in front a calendar: a week. +const CALENDAR_DAYS: usize = 7; + /// The history note for a dialog left open by the run before a rescue. const LEFT_OPEN: &str = "a dialog the task opened before is still in front: it is the task's current stage, so work within it"; @@ -29,6 +32,26 @@ pub(super) enum Acted { Typed, } +/// Whose the dialog in front is, and how far the task has worked in it. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(super) enum Dialog { + /// None of the task's: the window, or what the page itself put in + /// front. + #[default] + Page, + /// The task's own, opened by its press (or left open by its run before) + /// and not pressed in since: the question it asks first, such as a + /// format or a quantity. Live, such a dialog was closed at the start of + /// the following step. + Asking, + /// The task's own, pressed in since it opened. + Answered, + /// The task's own, worked in by a step before this one: no longer its + /// question, and in the way of the next step, but in front until the + /// window is (a calendar left open after its day was chosen). + Served, +} + /// What was in front on the last look, and how the run's own actions /// brought it there. #[derive(Debug, Clone)] @@ -38,16 +61,13 @@ pub(super) struct Front { pub(super) surface: String, /// What the run did since the last look. pub(super) acted: Acted, - /// Whether the dialog in front was opened by the run's own press (a - /// question a booking or purchase button asks first, such as a format - /// or a quantity). Live, such a dialog was closed at the start of the - /// following step. - pub(super) opened_dialog: bool, - /// Whether the run has neither looked nor browsed yet: a dialog in - /// front at a run's first look, before any browsing, was left there by - /// the task's run before it (a rescue continues where that run - /// stopped), and counts as opened by the task. - pub(super) fresh: bool, + /// Whose the dialog in front is ([`Dialog`]). + pub(super) dialog: Dialog, + /// What the run's first look takes a dialog in front for, until it has + /// looked or browsed: the task's own when the task's run before this one + /// left its own dialog there (`true`; a rescue continues where that run + /// stopped), else the page's. + first_look: Option, /// How many controls something covered on the last look with nothing /// in front: what a sticky header always covers, which a layer a press /// opens must add to before it counts. @@ -56,11 +76,8 @@ pub(super) struct Front { /// either changed went to another page, and what covers that page is /// the page's own, not an answer the press asked for. pub(super) looked_at: (Option, Option), - /// Whether the run has pressed or typed inside the task's dialog since - /// it opened: such a dialog has served the step that worked in it, and - /// a later step finds it in the way rather than asking (a calendar left - /// open after its day was chosen). - pub(super) answered: bool, + /// Whether the last look showed a calendar ([`holds_calendar`]). + pub(super) calendar: bool, } /// What the run's own housekeeping does, never a press of the task's: a @@ -76,19 +93,32 @@ const HOUSEKEEPING: &[&str] = &[ impl Default for Front { fn default() -> Self { + Self::new(false) + } +} + +impl Front { + /// A run's front before its first look: `inherited` when the task's run + /// before this one left its own dialog in front. + pub(super) fn new(inherited: bool) -> Self { Self { surface: "window".to_owned(), acted: Acted::Nothing, - opened_dialog: false, - fresh: true, + dialog: Dialog::Page, + first_look: Some(inherited), covered_base: 0, looked_at: (None, None), - answered: false, + calendar: false, } } -} -impl Front { + /// Whether the dialog in front is the task's own and still its stage: + /// opened by its press, or left open by its run before, and not yet + /// handed back by a step that worked in it. + pub(super) fn opened_dialog(&self) -> bool { + matches!(self.dialog, Dialog::Asking | Dialog::Answered) + } + /// Notes one action, `action` as the run logs it (`click`, `fill …`, /// `browse …`, `click (dismiss)`), on an element when `targeted`. /// @@ -99,14 +129,14 @@ impl Front { pub(super) fn act(&mut self, action: &str, targeted: bool) { if action.starts_with("browse ") { *self = Self { - fresh: false, + first_look: None, ..Self::default() }; return; } if action.starts_with("launch ") { *self = Self { - fresh: self.fresh, + first_look: self.first_look, ..Self::default() }; return; @@ -117,48 +147,50 @@ impl Front { let typing = action.starts_with("fill") || action.starts_with("type"); self.acted = if typing { Acted::Typed } else { Acted::Pressed }; } - if pressing && self.opened_dialog { - self.answered = true; + if pressing && self.dialog == Dialog::Asking { + self.dialog = Dialog::Answered; } } + /// Whether the dialog in front is a calendar the task opened and has + /// pressed in since, in this step or one before: a picker that has + /// served its field, in the way of a press behind it rather than asking + /// anything. Live, a calendar stayed in front of the guests and Search + /// buttons once both dates were picked, a step each, and every press + /// behind it was refused. + pub(super) fn served_calendar(&self) -> bool { + matches!(self.dialog, Dialog::Answered | Dialog::Served) && self.calendar + } + /// Begins a step: a dialog the task opened and then worked in has /// served the step before, and is no longer the task's own; one the /// last step's last press opened still asks its question (a format /// dialog a booking button raised). pub(super) fn next_step(&mut self) { - if self.answered { - self.opened_dialog = false; - self.answered = false; + if self.dialog == Dialog::Answered { + self.dialog = Dialog::Served; } } /// Takes in a look at `screen`, at the address `location`: what is in /// front, and whether the task opened it. The note for the history - /// when the dialog in front became the task's. Only a `browsing` run - /// takes a dialog at its first look as the task's: a browser task's - /// first run always browses first, so a dialog then is a rescue's - /// inheritance, while an application can open with its own alert. - pub(super) fn look( - &mut self, - screen: &Screen, - location: Option<&str>, - browsing: bool, - ) -> Option<&'static str> { - let left_open = self.fresh && browsing; - self.fresh = false; + /// when the dialog in front became the task's. A dialog at a run's first + /// look is the task's only when the task's run before this one left its + /// own dialog in front ([`Front::new`]): live, a sign-up the page opened + /// on load, and a site's menu drawer, were taken for the task's at a + /// rescue's first look, and every move past them was refused. + pub(super) fn look(&mut self, screen: &Screen, location: Option<&str>) -> Option<&'static str> { + let left_open = self.first_look.take() == Some(true); + self.calendar = holds_calendar(screen); let front = self.front_of(screen, location); let note = if front == "window" { - self.opened_dialog = false; - self.answered = false; + self.dialog = Dialog::Page; None - } else if left_open && !self.opened_dialog { - self.opened_dialog = true; - self.answered = false; + } else if left_open && !self.opened_dialog() { + self.dialog = Dialog::Asking; Some(LEFT_OPEN) } else if self.acted != Acted::Nothing && self.surface == "window" { - self.opened_dialog = true; - self.answered = false; + self.dialog = Dialog::Asking; Some(OPENED) } else { None @@ -209,3 +241,30 @@ fn covered_count(screen: &Screen) -> usize { }) .count() } + +/// Whether `screen` shows a calendar: [`CALENDAR_DAYS`] or more day numbers +/// nothing covers, each a calendar's grid cell or beside a month's name ("1 +/// September 2026", or "1" described "Thu Oct 01 2026"). +fn holds_calendar(screen: &Screen) -> bool { + screen + .candidates + .iter() + .filter(|candidate| { + let name = candidate.name.as_deref().unwrap_or_default(); + let day = name + .split_whitespace() + .next() + .and_then(|word| word.parse::().ok()) + .is_some_and(|day| (1..=31).contains(&day)); + let dated = candidate.role.eq_ignore_ascii_case("gridcell") + || names_a_month(name) + || candidate.description.as_deref().is_some_and(names_a_month); + let covered = candidate + .states + .iter() + .any(|state| state.eq_ignore_ascii_case("covered")); + day && dated && !covered + }) + .count() + >= CALENDAR_DAYS +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/ground/mod.rs b/crates/tinycomputer-engine/src/agentic/flow/ground/mod.rs index 8afcb48e..e959d12c 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/ground/mod.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/ground/mod.rs @@ -32,6 +32,9 @@ mod decide; mod narrow; +#[cfg(test)] +pub(super) use narrow::knockout_groups; + use std::collections::BTreeMap; use tinyinference_decisions::{Answer, EvaluationRequest}; diff --git a/crates/tinycomputer-engine/src/agentic/flow/ground/narrow.rs b/crates/tinycomputer-engine/src/agentic/flow/ground/narrow.rs index f19d8a6e..f44d7e50 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/ground/narrow.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/ground/narrow.rs @@ -16,7 +16,7 @@ use crate::agentic::flow::{ view::{Candidate, Screen, distinct, label, named_first}, }; -use super::{AGREED, BRANCH_MARGIN, First, Grounded, Opening, split, winners}; +use super::{AGREED, BRANCH_MARGIN, First, Grounded, Opening, Regions, split, winners}; impl FlowRun<'_, B> { /// Picks the element of `pool` that serves `purpose`, or `None` when no @@ -127,25 +127,7 @@ impl FlowRun<'_, B> { /// round trip of its own. fn narrowing(&self, screen: &Screen, purpose: &str, pool: &[Candidate]) -> First { let regions = split(pool, 0).map(|(_, regions)| regions); - let aligned = regions.as_ref().map(|regions| { - regions - .iter() - .enumerate() - .flat_map(|(index, (_, members))| { - members - .chunks(CAP) - .map(move |chunk| (Some(index), chunk.to_vec())) - }) - .collect::>() - }); - let groups = match aligned { - Some(groups) if groups.len() <= CAP => groups, - _ => pool - .chunks(CAP) - .take(CAP) - .map(|chunk| (None, chunk.to_vec())) - .collect(), - }; + let groups = knockout_groups(pool, regions.as_ref()); let mut questions = Questions::default(); for (index, (_, group)) in groups.iter().enumerate() { questions = questions.with( @@ -328,3 +310,60 @@ impl FlowRun<'_, B> { kept } } + +/// The groups of one knockout over `pool`: along `regions` when the screen +/// has them, each region in [`CAP`]-sized chunks, at most [`CAP`] chunks in +/// all; otherwise the pool's first [`CAP`] chunks. When the regions need more +/// chunks than that, the regions with nothing in view give up their last +/// chunks first, the largest first, then the largest of the rest, so a small +/// region (the list in front, a dialog's rows) is offered whole: live, the +/// rows of an airport list that came after a page's 400 route links were cut +/// off unseen, and so was a travellers pop-up's "Done" drawn after twenty +/// regions of links out of view. +pub(in crate::agentic::flow) fn knockout_groups( + pool: &[Candidate], + regions: Option<&Regions>, +) -> Vec<(Option, Vec)> { + let Some(regions) = regions else { + return pool + .chunks(CAP) + .take(CAP) + .map(|chunk| (None, chunk.to_vec())) + .collect(); + }; + let mut kept = regions + .iter() + .map(|(_, members)| members.len().div_ceil(CAP)) + .collect::>(); + let unseen = regions + .iter() + .map(|(_, members)| { + !members + .iter() + .any(|member| denoise::tier(member) == denoise::Tier::InView) + }) + .collect::>(); + while kept.iter().sum::() > CAP { + let Some(largest) = kept + .iter() + .enumerate() + .filter(|(_, chunks)| **chunks > 0) + .max_by_key(|(index, chunks)| (unseen[*index], **chunks)) + .map(|(index, _)| index) + else { + break; + }; + kept[largest] -= 1; + } + regions + .iter() + .zip(kept) + .enumerate() + .flat_map(|(index, ((_, members), chunks))| { + members + .chunks(CAP) + .take(chunks) + .map(move |chunk| (Some(index), chunk.to_vec())) + }) + .collect() +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/look.rs b/crates/tinycomputer-engine/src/agentic/flow/look.rs index ddeef4b6..528e2d10 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/look.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/look.rs @@ -37,8 +37,7 @@ impl FlowRun<'_, B> { match observed { Ok(screen) => { self.blind_looks = 0; - let browsing = self.app.eq_ignore_ascii_case("browser"); - if let Some(note) = self.front.look(&screen, self.location.as_deref(), browsing) { + if let Some(note) = self.front.look(&screen, self.location.as_deref()) { self.history.push(note.to_owned()); } Ok(screen) diff --git a/crates/tinycomputer-engine/src/agentic/flow/mod.rs b/crates/tinycomputer-engine/src/agentic/flow/mod.rs index f0b4c6c5..70889571 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/mod.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/mod.rs @@ -314,7 +314,8 @@ pub(super) struct FlowRun<'r, B> { /// Escape or a close that did not clear it once will not the next time. pub(super) step_cleared: BTreeSet, /// What is in front, and whether the run's own press put it there - /// (`front.rs`). + /// (`front.rs`); built knowing whether the task's run before this one + /// left its own dialog in front ([`RunFlowRequest::dialog_left_open`]). pub(in crate::agentic::flow) front: Front, } diff --git a/crates/tinycomputer-engine/src/agentic/flow/run.rs b/crates/tinycomputer-engine/src/agentic/flow/run.rs index a592a7f3..8f791649 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/run.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/run.rs @@ -140,7 +140,7 @@ impl<'r, B: AgentBackend + Sync> FlowRun<'r, B> { expecting: None, step_location: None, step_cleared: BTreeSet::new(), - front: Front::default(), + front: Front::new(request.dialog_left_open), } } @@ -346,6 +346,7 @@ impl<'r, B: AgentBackend + Sync> FlowRun<'r, B> { actions: self.actions, metrics: self.metrics, trace: self.trace, + dialog_left_open: self.front.opened_dialog(), }, ) } diff --git a/crates/tinycomputer-engine/src/agentic/flow/steps/choose.rs b/crates/tinycomputer-engine/src/agentic/flow/steps/choose.rs index 3fcb3c54..001b87a1 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/steps/choose.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/steps/choose.rs @@ -15,9 +15,11 @@ use crate::agentic::flow::{ use super::{ LOCATE_FLOOR, + date::looks_like_date, matching::{ - already_chosen, already_holds, clickable, closest, editable, held_text, is_checked, - is_one_option, lists_more_than, mentions, one_option, plainest, redacted, within, + already_chosen, already_holds, clickable, closest, date_shown_in, editable, held_text, + is_checked, is_one_option, lists_more_than, mentions, one_option, plainest, redacted, + within, }, }; @@ -166,22 +168,7 @@ impl FlowRun<'_, B> { if !private && let Some(ended) = self.made_already(&screen, what, option, attempt) { return Ok(ended); } - let pool = clickable(&screen.candidates) - .into_iter() - .filter(|candidate| !is_destructive(candidate, &screen, &self.stop_before)) - .collect::>(); - // A field that holds the typed option is where it was typed, - // not one of the options it offers. - let pool = closest( - pool.into_iter() - .filter(|candidate| { - mentions(candidate, option) - && !editable(candidate) - && !lists_more_than(candidate, option) - }) - .collect(), - ); - let pool = within(pool, what); + let pool = option_pool(&screen, what, option, into_focus, &self.stop_before); // Matches that all name one option leave nothing to judge; a // private option is never judged, since Jev is not told it. An // option no control names is a description ("the lowest fare"), @@ -275,6 +262,7 @@ impl FlowRun<'_, B> { (attempt == 0) .then(|| already_holds(screen, option, &self.typed)) .flatten() + .or_else(|| date_shown_in(screen, what, option)) .map(|holder| format!("{} already shows {option:?}", label(&holder))) })?; self.history.push(shown); @@ -310,3 +298,49 @@ impl FlowRun<'_, B> { .filter(|grounded| grounded.confidence >= LOCATE_FLOOR)) } } + +/// The controls on `screen` that could be `option` in `what`: pressable and +/// not irreversible, naming the option, no box it was typed into, the +/// closest matches, within `what` when the page says where; for an `enter` +/// value with no box (`into_focus` unset), only those the screen shows. +fn option_pool( + screen: &Screen, + what: &str, + option: &str, + into_focus: bool, + stop_before: &[String], +) -> Vec { + let pool = clickable(&screen.candidates) + .into_iter() + .filter(|candidate| !is_destructive(candidate, screen, stop_before)) + .collect::>(); + // A field that holds the typed option is where it was typed, + // not one of the options it offers. + let pool = closest( + pool.into_iter() + .filter(|candidate| { + mentions(candidate, option) + && !editable(candidate) + && !lists_more_than(candidate, option) + }) + .collect(), + ); + let pool = within(pool, what); + // An `enter` value with no box to type it into is picked from + // what the screen shows, a date aside (a calendar can scroll its + // days out of its own view): live, the only "Mumbai" was a link + // out of view at the foot of the page, pressed as the place to + // fly to, and the search went from Mumbai instead. + if into_focus || looks_like_date(option) { + pool + } else { + pool.into_iter() + .filter(|candidate| { + !candidate + .states + .iter() + .any(|state| state.eq_ignore_ascii_case("offscreen")) + }) + .collect() + } +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/steps/condition.rs b/crates/tinycomputer-engine/src/agentic/flow/steps/condition.rs index b8e242a6..2377ca94 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/steps/condition.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/steps/condition.rs @@ -180,7 +180,7 @@ impl FlowRun<'_, B> { } // A dialog the task opened asks its question first (a format, a // quantity): what lies past it will not show while it waits. - if self.front.opened_dialog && check >= 1 { + if self.front.opened_dialog() && check >= 1 { return Err(Halt::Failed( "a dialog the task opened is waiting for an answer, so nothing past it shows: choose what it asks, then continue" .to_owned(), diff --git a/crates/tinycomputer-engine/src/agentic/flow/steps/date.rs b/crates/tinycomputer-engine/src/agentic/flow/steps/date.rs index 793a9474..7d3ed3f6 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/steps/date.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/steps/date.rs @@ -22,6 +22,15 @@ const MONTHS: &[&str] = &[ "december", ]; +/// Whether `text` names a month, in full or cut to three letters or more +/// ("Oct", "Sept"), as a calendar's day cells and headings do. +pub(in crate::agentic::flow) fn names_a_month(text: &str) -> bool { + text.split(|character: char| !character.is_alphabetic()) + .filter(|word| word.chars().count() >= 3) + .map(str::to_lowercase) + .any(|word| MONTHS.iter().any(|month| month.starts_with(&word))) +} + /// Whether `option` names a calendar day: a month name and a day number that /// is a real day of that month (a year, when given, decides February's 28th /// against its 29th). `February 31` or `April 31` names no such day, and is diff --git a/crates/tinycomputer-engine/src/agentic/flow/steps/list.rs b/crates/tinycomputer-engine/src/agentic/flow/steps/list.rs index 75697c51..f16f9a32 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/steps/list.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/steps/list.rs @@ -34,7 +34,7 @@ impl FlowRun<'_, B> { let by = substitute_safe(&pick.by, &self.vars, &self.facts); let mut screen = self.look().await?; self.explore(&mut screen).await; - let families = result_families(&screen); + let families = openable(result_families(&screen)); if families.is_empty() { return Err(Halt::Failed(format!("no list of {from} is showing"))); } @@ -341,6 +341,27 @@ impl FlowRun<'_, B> { } } +/// The lists of `families` a pick can open an item of: those whose items +/// mostly hold something to press, when any list's do. A pick opens what +/// it takes, and a list of bare fares (a strip above the flights, or each +/// card's price read apart from it) has nothing to open: live, "the +/// cheapest flight" ranked such a list and took "₹ 6,054". +fn openable(families: Vec>) -> Vec> { + let opens = |groups: &Vec| { + groups + .iter() + .filter(|group| group.primary.is_some()) + .count() + * 2 + > groups.len() + }; + if families.iter().any(opens) { + families.into_iter().filter(opens).collect() + } else { + families + } +} + /// Each card's text as a record, its fields numbered in reading order. fn records_of(groups: &[Group]) -> Vec { groups diff --git a/crates/tinycomputer-engine/src/agentic/flow/steps/matching.rs b/crates/tinycomputer-engine/src/agentic/flow/steps/matching.rs index ec09e337..c4228000 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/steps/matching.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/steps/matching.rs @@ -29,6 +29,9 @@ pub(super) fn held_text(screen: &Screen) -> Vec<(Candidate, String)> { /// The matches whose labels say little besides the option: a container /// whose label strings together everything inside it (a calendar button /// named with every day of the month) is dropped when a plainer match exists. +/// A list's option is one option however much its label says: live, an +/// airport row ("BOM Mumbai, India … 3 Nearby Airports found") was dropped +/// for a footer link that only said "Mumbai". pub(in crate::agentic::flow) fn closest(matches: Vec) -> Vec { let length = |candidate: &Candidate| candidate.name.as_deref().map_or(0, str::len); let Some(shortest) = matches.iter().map(length).min() else { @@ -36,7 +39,10 @@ pub(in crate::agentic::flow) fn closest(matches: Vec) -> Vec Option { + if !looks_like_date(option) { + return None; + } + let wanted = date_words(option); + let named = plain(what) + .split(' ') + .filter(|word| word.chars().count() > 3 && *word != "date") + .map(str::to_owned) + .collect::>(); + if named.is_empty() { + return None; + } + screen + .candidates + .iter() + .find(|candidate| { + let shown = [ + candidate.name.as_deref(), + candidate.value.as_ref().and_then(serde_json::Value::as_str), + ] + .into_iter() + .flatten() + .collect::>() + .join(" "); + let words = plain(&shown); + named + .iter() + .any(|word| words.split(' ').any(|said| said == word)) + && shows_date(&shown, &wanted) + }) + .cloned() +} + /// Roles a page marks as the one chosen among its siblings. const SELECTABLE_ROLES: &[&str] = &["tab", "radio", "radiobutton", "option", "menuitemradio"]; @@ -309,15 +358,34 @@ pub(super) fn within(pool: Vec, what: &str) -> Vec { if regional.is_empty() { pool } else { regional } } +/// Words that place one thing against another ("from Delhi to Mumbai"), +/// too common in labels to say which region an option is in. +const PLACING_WORDS: &[&str] = &[ + "to", "from", "via", "at", "in", "on", "for", "of", "by", "with", +]; + /// Whether `candidate` sits inside — or itself names — the region `what` /// describes. A page rarely echoes a description such as "the outbound /// flight list" on an option's own label, so this also checks the option's /// ancestor labels (`path`), which the snapshot records outermost first. +/// +/// A region named by a placing word alone ("to", "from") holds only what +/// sits under a container whose name begins with it: live, "to" kept a +/// page's "Delhi to Mumbai flights" links and dropped the airport list. pub(in crate::agentic::flow) fn in_region(candidate: &Candidate, what: &str) -> bool { let wanted = plain(what); if wanted.is_empty() { return true; } + if PLACING_WORDS.contains(&wanted.as_str()) { + return candidate.path.iter().any(|ancestor| { + let name = ancestor + .split_once('"') + .map_or("", |(_, quoted)| quoted.trim_end_matches('"')); + let name = plain(name); + name == wanted || name.starts_with(&format!("{wanted} ")) + }); + } let names = |text: &str| format!(" {} ", plain(text)).contains(&format!(" {wanted} ")); [candidate.name.as_deref(), candidate.description.as_deref()] .into_iter() diff --git a/crates/tinycomputer-engine/src/agentic/flow/steps/mod.rs b/crates/tinycomputer-engine/src/agentic/flow/steps/mod.rs index 1cd28bd7..1b226d85 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/steps/mod.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/steps/mod.rs @@ -21,13 +21,15 @@ mod stop; mod suggestion; mod typing; +pub(super) use date::{looks_like_date, names_a_month}; pub(super) use matching::left_unchosen; #[cfg(test)] pub(super) use { - date::{date_words, looks_like_date, shows_date}, + date::{date_words, shows_date}, list::{first_meeting, leaning}, matching::{ - already_chosen, already_holds, closest, in_region, lists_more_than, redacted, search_text, + already_chosen, already_holds, closest, date_shown_in, in_region, lists_more_than, + redacted, search_text, }, read::chosen_together, read::readable, diff --git a/crates/tinycomputer-engine/src/agentic/flow/steps/suggestion.rs b/crates/tinycomputer-engine/src/agentic/flow/steps/suggestion.rs index d4d91bf0..116e04d3 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/steps/suggestion.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/steps/suggestion.rs @@ -334,7 +334,10 @@ pub(in crate::agentic::flow) fn suggests(slot: &str, _field: &Candidate) -> bool /// press. For a `place` box, a row that was already showing counts too when /// it matches the text: a ride app lists popular places as soon as its box /// has the focus, and live, the place typed was among them, so nothing new -/// appeared and nothing was picked. +/// appeared and nothing was picked. Such a row is in view and in front, as +/// a list under the box is: live, a footer link "hotels in Goa", off screen +/// at the foot of the page, was pressed as the place typed, and the box's +/// own rows were never waited for. pub(in crate::agentic::flow) fn fresh_rows( screen: &Screen, shown: &BTreeSet<(&str, Option<&str>)>, @@ -347,8 +350,12 @@ pub(in crate::agentic::flow) fn fresh_rows( .into_iter() .filter(|candidate| { let new = !shown.contains(&(candidate.role.as_str(), candidate.name.as_deref())); - let matches = - place && (mentions(candidate, text) || shares_most_words(candidate, text)); + let in_view = !candidate.states.iter().any(|state| { + state.eq_ignore_ascii_case("offscreen") || state.eq_ignore_ascii_case("covered") + }); + let matches = place + && in_view + && (mentions(candidate, text) || shares_most_words(candidate, text)); let a_box = candidate .available_actions .iter() diff --git a/crates/tinycomputer-engine/src/task/budget.rs b/crates/tinycomputer-engine/src/task/budget.rs index cdb6039c..3a821266 100644 --- a/crates/tinycomputer-engine/src/task/budget.rs +++ b/crates/tinycomputer-engine/src/task/budget.rs @@ -65,6 +65,7 @@ pub(super) fn run_request( trace: state.trace, // What earlier runs saved, so a resumed run remembers it. collected: state.reads.clone(), + dialog_left_open: state.dialog_left_open, ..RunFlowRequest::default() }, state.constraints.clone(), diff --git a/crates/tinycomputer-engine/src/task/drive.rs b/crates/tinycomputer-engine/src/task/drive.rs index ea9ec2c0..b4f03708 100644 --- a/crates/tinycomputer-engine/src/task/drive.rs +++ b/crates/tinycomputer-engine/src/task/drive.rs @@ -178,6 +178,9 @@ pub(super) async fn drive(cell: Arc, runner: Arc, runs: Ve if let Some(rescue) = run.rescue.and_then(|index| state.rescues.get_mut(index)) { rescue.outcome = rescue_outcome(&reached, rescue.steps.len()); } + state.dialog_left_open = result + .as_ref() + .is_some_and(|result| result.dialog_left_open); if let Some(result) = result { state.spent.actions = state.spent.actions.saturating_add(result.actions); state.spent.model_calls = diff --git a/crates/tinycomputer-engine/src/task/store.rs b/crates/tinycomputer-engine/src/task/store.rs index 72a83412..926bc49b 100644 --- a/crates/tinycomputer-engine/src/task/store.rs +++ b/crates/tinycomputer-engine/src/task/store.rs @@ -47,6 +47,11 @@ pub(super) struct State { pub(super) exchanges: Vec, pub(super) learned: Vec, pub(super) reads: BTreeMap, + /// Whether the task's last run left its own dialog in front, for the + /// next run to work within ([`RunFlowRequest::dialog_left_open`]). + /// + /// [`RunFlowRequest::dialog_left_open`]: tinycomputer_bus::RunFlowRequest::dialog_left_open + pub(super) dialog_left_open: bool, pub(super) finished: usize, pub(super) resume: Option, /// What every run of this task has spent so far, so an approval or a @@ -112,6 +117,7 @@ impl Tasks { exchanges: Vec::new(), learned: Vec::new(), reads: BTreeMap::new(), + dialog_left_open: false, finished: 0, resume: None, spent: Spent::default(), diff --git a/crates/tinycomputer-engine/src/task/task_tests.rs b/crates/tinycomputer-engine/src/task/task_tests.rs index 85849548..faee5b3d 100644 --- a/crates/tinycomputer-engine/src/task/task_tests.rs +++ b/crates/tinycomputer-engine/src/task/task_tests.rs @@ -188,6 +188,7 @@ fn finished_run( actions: 3, metrics: JevMetrics::default(), trace: Vec::new(), + dialog_left_open: false, }; DesktopResponse::ok("run-flow", serde_json::to_value(result).unwrap()) } diff --git a/crates/tinycomputer-engine/src/task/task_tests/rescue_tests.rs b/crates/tinycomputer-engine/src/task/task_tests/rescue_tests.rs index ae6e91b1..c08b9dbd 100644 --- a/crates/tinycomputer-engine/src/task/task_tests/rescue_tests.rs +++ b/crates/tinycomputer-engine/src/task/task_tests/rescue_tests.rs @@ -378,3 +378,31 @@ async fn a_skip_resumes_at_the_next_step_with_the_guard_kept() { assert_eq!(rescue.covers, 1); assert_eq!(rescue.outcome, RescueOutcome::Recovered); } + +#[tokio::test] +async fn a_rescue_takes_the_dialog_in_front_as_the_tasks_only_when_the_run_before_left_it() { + let rest = || finished_run(FlowStopReason::Completed, vec![], &[], None); + // The failed run left the task's own dialog in front: its rescue works + // within it. + let mut left_open = failed_at_step_two(); + left_open.data.as_mut().unwrap()["dialog_left_open"] = json!(true); + let (tasks, script, _model) = rescued(vec![left_open, rest()], &[Ok(ONE_STEP)]); + let view = begin(&tasks, TaskBudget::default()); + settle(&tasks, &view.id).await; + let requests = script.requests.lock().unwrap().clone(); + assert_eq!(requests.len(), 2); + assert!( + !requests[0].dialog_left_open, + "a first run inherits nothing" + ); + assert!(requests[1].dialog_left_open); + + // One that left none: a dialog in front at the rescue's first look is + // the page's own (a sign-up it opened, a menu), not the task's. + let (tasks, script, _model) = rescued(vec![failed_at_step_two(), rest()], &[Ok(ONE_STEP)]); + let view = begin(&tasks, TaskBudget::default()); + settle(&tasks, &view.id).await; + let requests = script.requests.lock().unwrap().clone(); + assert_eq!(requests.len(), 2); + assert!(!requests[1].dialog_left_open); +} diff --git a/crates/tinycomputer/tests/public_api_tests.rs b/crates/tinycomputer/tests/public_api_tests.rs index 3b1cf339..8f8047ac 100644 --- a/crates/tinycomputer/tests/public_api_tests.rs +++ b/crates/tinycomputer/tests/public_api_tests.rs @@ -34,7 +34,7 @@ fn the_crate_re_exports_the_contract_rather_than_redefining_it() { #[test] fn the_shipped_contract_binds_to_itself() { assert!(is_compatible(CONTRACT_VERSION)); - assert_eq!(CONTRACT_VERSION, (2, 8)); + assert_eq!(CONTRACT_VERSION, (2, 9)); } #[test] diff --git a/docs/technical/decision-thresholds.md b/docs/technical/decision-thresholds.md index 3f47667d..73e60847 100644 --- a/docs/technical/decision-thresholds.md +++ b/docs/technical/decision-thresholds.md @@ -49,6 +49,7 @@ Change a constant and its row together. | `MAX_REPEAT_PRESSES` | 3 | `act/mod.rs` | presses of one control (by label and place, so a toggle's two looks count as one) or one key in one `do` step after which it is struck off for the step; a scroll is no press. Pressing a named control also strikes off its copies on the other items of its list (same label, another card of the same list), once the next look shows the press changed the screen, unless the step says all, every, each, or both, or chooses several items (a choosing verb in its first two words and a count of 2 to 20, or a number word, within three words before a plural: "choose 2 adjacent seats"); undoing that press lifts its own copies again | | `MAX_IDLE_SCROLLS` | 1 | `act/mod.rs` | scrolls that showed nothing new after which a step's "scroll" move is taken as "activate": the screen already lists what lies below the fold | | `LAYER_COVERS` | 3 | `front.rs` | controls something drawn over the window must cover, beyond what was covered before the press that opened it, on the same page, before it counts as a dialog the task opened (`surface` `layer`); a step that pressed inside such a dialog hands it back at the next step, and opening an address forgets it | +| `CALENDAR_DAYS` | 7 | `front.rs` | day cells nothing covers (a grid cell, or a day number beside a month's name) that make what is in front a calendar: a press refused as covered behind a calendar the task opened and has pressed in since, in that step or one before, closes it and presses again, where any other dialog of the task's is worked within | | `FRONT_CONTROLS` | 8 | `act/turns.rs` | most controls of the task's dialog in front a failed step's note names, so a rescue answers with one of them | | `STEADY_HOLD` / `STEADY_CHECKS` | 0.65 / 3 | `steps/mod.rs` | belief a `wait_for` condition must keep, on checks in a row of one unchanged screen, to be taken as held under `DONE` | | `HEDGE_AFTER` / `HEDGE_AFTER_LARGE` | 4 s / 5 s | `hedge.rs` | how long a framing runs before a copy of it is sent and the first answer of the two taken; the longer wait is for a request of `HEDGE_LARGE_BYTES` (32 KB) or more. Sage gets no copy | diff --git a/docs/technical/jev-harness.md b/docs/technical/jev-harness.md index c0e051d3..7d7d8e3d 100644 --- a/docs/technical/jev-harness.md +++ b/docs/technical/jev-harness.md @@ -76,7 +76,9 @@ framing of every request together: one round trip. In order: 1. **Build.** A step's code builds the questions (`ask/questions.rs`) and the shared state (`ask::state`, in `ask/screen_state.rs`: app, window, surface, current step, visible text, - up to 120 elements, the last eight history lines, and field contents when + up to 120 elements (on a longer screen, those in view first, up to three + quarters of them, then the rest as the page orders them, all listed in + screen order), the last eight history lines, and field contents when `include_values` is set). Screen text is always wrapped as `untrusted_accessibility_data`. 2. **Budget.** The run's call budget is checked first; a spent budget stops diff --git a/docs/technical/specs/desktop-module-contract.md b/docs/technical/specs/desktop-module-contract.md index 4114c46f..f973f11f 100644 --- a/docs/technical/specs/desktop-module-contract.md +++ b/docs/technical/specs/desktop-module-contract.md @@ -41,7 +41,10 @@ engine's argument types, the permission preflight, and the bus surface. `open_jev` and `sage` decision providers, `JevConfig.fast`, `JevConfiguration.fast`, the planner's `LanguageModelProvider` routes, and `Capabilities.decision_model`, `planner_model`, `rescue_model`, and - `output_model`, all optional) return an + `output_model`; 2.9 added `RunFlowRequest.dialog_left_open` and + `FlowRunResult.dialog_left_open`, so a run takes a dialog at its first + look as the task's only when the task's run before it left that dialog + open, all optional) return an `AgentResponse` instead — see [`unified-agent.md`](unified-agent.md). They share this interface because a TinyBus module exports one interface. - The thirteen browser members (2.6) close the list, each prefixed `Browser` From b54e3cadb08a7c074cfb8d14d762f95f2d4061ac Mon Sep 17 00:00:00 2001 From: Shanu Date: Thu, 8 Oct 2026 23:29:36 +0530 Subject: [PATCH 04/15] Tell a turned month, a seat map, and an unlooked dialog apart Front took any press in a dialog the task opened for its answer, so a calendar whose month was turned counted as served, and a covered press could close it before its day was picked. Pressing a control that only turns the month ("next month", "Previous month") now leaves the calendar asking; a day pressed in it still serves its field. A grid of bare numbers counted as a calendar's days even with no month on screen, so a seat map pressed in could be closed for a press behind it. Grid cells named by a number now count only when the screen names a month somewhere; cells that name or are described with a month count as before. A run that ended before its first look reported no dialog left open, dropping the one its run before had left. Front::left_open now hands that dialog on until the run has looked. --- .../src/agentic/flow/action.rs | 2 +- .../agentic/flow/flow_tests/do_loop_tests.rs | 100 ++++++++++++++++-- .../src/agentic/flow/front.rs | 56 ++++++++-- .../src/agentic/flow/run.rs | 2 +- 4 files changed, 141 insertions(+), 19 deletions(-) diff --git a/crates/tinycomputer-engine/src/agentic/flow/action.rs b/crates/tinycomputer-engine/src/agentic/flow/action.rs index b70f9eeb..a6ff02ab 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/action.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/action.rs @@ -28,7 +28,7 @@ impl FlowRun<'_, B> { return Err(Halt::Stop(FlowStopReason::ActionBudget)); } self.actions = self.actions.saturating_add(1); - self.front.act(action, target.is_some()); + self.front.act(action, target); let started = Instant::now(); let reply = self.backend_call(call).await; let acted_ms = millis(started.elapsed()); diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs index 773237f0..ebd91c62 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs @@ -781,14 +781,15 @@ fn a_dialog_the_task_worked_in_is_in_the_way_of_the_next_step() { use crate::agentic::flow::front::Front; let at = Some("https://flights.test/"); let (window, sheet) = (page_at("window", 0), page_at("sheet", 3)); + let press = node("Control 4", "button", &["Click"], &["main"], 4.0); let mut front = Front::default(); - front.act("browse https://flights.test/", false); + front.act("browse https://flights.test/", None); assert!(front.look(&window, at).is_none()); - front.act("click", true); + front.act("click", Some(&press)); assert!(front.look(&sheet, at).is_some(), "the press opened it"); front.next_step(); assert!(front.opened_dialog(), "the next step answers what it asks"); - front.act("click", true); + front.act("click", Some(&press)); front.look(&sheet, at); assert!( front.opened_dialog(), @@ -805,20 +806,20 @@ fn a_dialog_the_task_worked_in_is_in_the_way_of_the_next_step() { // A scroll or the run's own housekeeping opens no dialog of the task's. for action in ["scroll", "click (clear distraction)", "click (dismiss)"] { let mut front = Front::default(); - front.act("browse https://flights.test/", false); + front.act("browse https://flights.test/", None); front.look(&window, at); - front.act(action, true); + front.act(action, Some(&press)); assert!(front.look(&sheet, at).is_none(), "{action}"); assert!(!front.opened_dialog(), "{action}"); } // Opening an address leaves what was in front behind. let mut front = Front::default(); - front.act("browse https://flights.test/", false); + front.act("browse https://flights.test/", None); front.look(&window, at); - front.act("click", true); + front.act("click", Some(&press)); front.look(&sheet, at); - front.act("browse https://flights.test/next", false); + front.act("browse https://flights.test/next", None); assert!(!front.opened_dialog()); } @@ -840,6 +841,89 @@ fn a_dialog_at_a_runs_first_look_is_the_tasks_only_when_the_run_before_left_it() assert!(front.opened_dialog(), "the task's current stage"); } +/// A sheet of ten grid cells named by bare numbers, under `month` when +/// given. +fn days_at(month: Option<&str>) -> Screen { + let mut screen = page_at("sheet", 3); + screen.candidates.extend((1..=10).map(|day: u32| { + node( + &day.to_string(), + "gridcell", + &["Click"], + &["main"], + f64::from(100 + day), + ) + })); + screen.context.extend(month.map(str::to_owned)); + screen +} + +#[test] +fn turning_a_calendars_month_answers_nothing_it_asks() { + // An arrow pressed is no day chosen: the calendar still asks for one, + // and nothing it covers may be pressed through it yet. + use crate::agentic::flow::front::Front; + let at = Some("https://flights.test/"); + let calendar = days_at(Some("October 2026")); + let mut front = Front::default(); + front.act("browse https://flights.test/", None); + front.look(&page_at("window", 0), at); + front.act( + "click", + Some(&node("Departure", "button", &["Click"], &["main"], 0.0)), + ); + assert!(front.look(&calendar, at).is_some(), "the press opened it"); + front.act( + "click", + Some(&node("Next month", "button", &["Click"], &["main"], 90.0)), + ); + front.look(&calendar, at); + assert!(!front.served_calendar(), "a month turned is no day chosen"); + assert!(front.opened_dialog()); + front.act( + "click", + Some(&node("5", "gridcell", &["Click"], &["main"], 105.0)), + ); + front.look(&calendar, at); + assert!(front.served_calendar(), "a day chosen serves its field"); +} + +#[test] +fn a_grid_of_bare_numbers_is_a_calendar_only_beside_a_month() { + // A seat map's cells are numbers in a grid too: pressed in, it is no + // calendar that has served its field and may be closed for a press + // behind it. + use crate::agentic::flow::front::Front; + let at = Some("https://cinema.test/"); + for (month, calendar) in [(None, false), (Some("October 2026"), true)] { + let grid = days_at(month); + let mut front = Front::default(); + front.act("browse https://cinema.test/", None); + front.look(&page_at("window", 0), at); + front.act( + "click", + Some(&node("Select seats", "button", &["Click"], &["main"], 0.0)), + ); + front.look(&grid, at); + front.act( + "click", + Some(&node("5", "gridcell", &["Click"], &["main"], 105.0)), + ); + front.look(&grid, at); + assert_eq!(front.served_calendar(), calendar, "{month:?}"); + } +} + +#[test] +fn a_run_that_never_looked_hands_on_the_dialog_it_was_left() { + use crate::agentic::flow::front::Front; + assert!(Front::new(true).left_open(), "nothing changed in front"); + assert!(!Front::new(false).left_open()); + let mut front = Front::new(true); + front.look(&page_at("window", 0), Some("https://flights.test/")); + assert!(!front.left_open(), "the window is in front once it looked"); +} + #[tokio::test] async fn answering_the_task_dialog_never_presses_what_commits() { // The fallback presses what serves no step's words, so it never diff --git a/crates/tinycomputer-engine/src/agentic/flow/front.rs b/crates/tinycomputer-engine/src/agentic/flow/front.rs index 5cc92255..36206739 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/front.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/front.rs @@ -2,7 +2,10 @@ //! there: a dialog the task opened is the flow's next stage, never cleared //! as a distraction or an obstacle, in this step or the next. -use super::{steps::names_a_month, view::Screen}; +use super::{ + steps::names_a_month, + view::{Candidate, Screen}, +}; /// Controls a layer must cover, beyond what was covered before the press /// that opened it, before it counts as in front of the page. @@ -119,14 +122,25 @@ impl Front { matches!(self.dialog, Dialog::Asking | Dialog::Answered) } + /// Whether the run leaves the task's own dialog in front, for the task's + /// next run ([`tinycomputer_bus::RunFlowRequest::dialog_left_open`]): one + /// it opened, or the one its run before left there while it has not + /// looked since. A run that ends before its first look changed nothing + /// in front. + pub(super) fn left_open(&self) -> bool { + self.opened_dialog() || self.first_look == Some(true) + } + /// Notes one action, `action` as the run logs it (`click`, `fill …`, - /// `browse …`, `click (dismiss)`), on an element when `targeted`. + /// `browse …`, `click (dismiss)`), on `target` when it had one. /// /// Only the task's own presses and typing count: a scroll moves no /// question into view, and the run's housekeeping (a distraction - /// cleared, an undo) answers none. Opening an address or an application - /// leaves whatever was in front behind. - pub(super) fn act(&mut self, action: &str, targeted: bool) { + /// cleared, an undo) answers none. Nor does turning a calendar's month + /// (`turns_the_month`): the calendar still asks for its day. Opening an + /// address or an application leaves whatever was in front behind. + pub(super) fn act(&mut self, action: &str, target: Option<&Candidate>) { + let targeted = target.is_some(); if action.starts_with("browse ") { *self = Self { first_look: None, @@ -147,7 +161,7 @@ impl Front { let typing = action.starts_with("fill") || action.starts_with("type"); self.acted = if typing { Acted::Typed } else { Acted::Pressed }; } - if pressing && self.dialog == Dialog::Asking { + if pressing && self.dialog == Dialog::Asking && !target.is_some_and(turns_the_month) { self.dialog = Dialog::Answered; } } @@ -242,10 +256,34 @@ fn covered_count(screen: &Screen) -> usize { .count() } +/// Whether `target` only turns a calendar's month ("next month", "Previous +/// month"), which answers nothing the calendar asks. +fn turns_the_month(target: &Candidate) -> bool { + let words = target + .name + .as_deref() + .unwrap_or_default() + .split(|character: char| !character.is_alphanumeric()) + .filter(|word| !word.is_empty()) + .map(str::to_lowercase) + .collect::>(); + let said = words.join(" "); + words.len() <= 4 && (said.contains("next month") || said.contains("previous month")) +} + /// Whether `screen` shows a calendar: [`CALENDAR_DAYS`] or more day numbers -/// nothing covers, each a calendar's grid cell or beside a month's name ("1 -/// September 2026", or "1" described "Thu Oct 01 2026"). +/// nothing covers, each beside a month's name ("1 September 2026", or "1" +/// described "Thu Oct 01 2026"), or a grid cell on a screen that names a +/// month somewhere: a seat map's or a table's cells are bare numbers too. fn holds_calendar(screen: &Screen) -> bool { + let month_shown = screen + .candidates + .iter() + .chain(&screen.text_nodes) + .flat_map(|node| [node.name.as_deref(), node.description.as_deref()]) + .flatten() + .chain(screen.context.iter().map(String::as_str)) + .any(names_a_month); screen .candidates .iter() @@ -256,7 +294,7 @@ fn holds_calendar(screen: &Screen) -> bool { .next() .and_then(|word| word.parse::().ok()) .is_some_and(|day| (1..=31).contains(&day)); - let dated = candidate.role.eq_ignore_ascii_case("gridcell") + let dated = (month_shown && candidate.role.eq_ignore_ascii_case("gridcell")) || names_a_month(name) || candidate.description.as_deref().is_some_and(names_a_month); let covered = candidate diff --git a/crates/tinycomputer-engine/src/agentic/flow/run.rs b/crates/tinycomputer-engine/src/agentic/flow/run.rs index 8f791649..091fb958 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/run.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/run.rs @@ -346,7 +346,7 @@ impl<'r, B: AgentBackend + Sync> FlowRun<'r, B> { actions: self.actions, metrics: self.metrics, trace: self.trace, - dialog_left_open: self.front.opened_dialog(), + dialog_left_open: self.front.left_open(), }, ) } From 3fd6e4093606a069ebb7f7604cb4a0431efe3d29 Mon Sep 17 00:00:00 2001 From: Shanu Date: Thu, 8 Oct 2026 23:30:07 +0530 Subject: [PATCH 05/15] Count a consent bar's Accept and I agree as closers A dismissal step on a pop-up drawn without a dialog's role ends when the control pressed names a closer and went with what it sat on. "Accept all" and "I agree" close a consent bar as surely as "Reject all" does, but were not closer words, so a step accepting cookies on such a bar fell back to a judge that cannot see which button closed it. A simulated cookie bar now checks that accepting it ends the step. --- .../src/agentic/flow/act/mod.rs | 7 +++- .../agentic/flow/flow_tests/picker_tests.rs | 40 ++++++++++++++++++- .../src/agentic/flow/flow_tests/screens.rs | 5 +++ .../src/agentic/flow/flow_tests/simulator.rs | 7 ++++ 4 files changed, 56 insertions(+), 3 deletions(-) diff --git a/crates/tinycomputer-engine/src/agentic/flow/act/mod.rs b/crates/tinycomputer-engine/src/agentic/flow/act/mod.rs index 40bfef2b..d455528e 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/act/mod.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/act/mod.rs @@ -303,8 +303,11 @@ const OVERLAYS: &[&str] = &[ "overlay", "prompt", "notice", ]; -/// Words of a control that closes what it sits on. -const CLOSERS: &[&str] = &["close", "dismiss", "skip", "later", "decline", "reject"]; +/// Words of a control that closes what it sits on: a consent bar's "Accept +/// all" or "I agree" closes it as surely as its "Reject all" does. +const CLOSERS: &[&str] = &[ + "close", "dismiss", "skip", "later", "decline", "reject", "accept", "agree", +]; fn words(text: &str) -> Vec { text.split(|character: char| !character.is_alphanumeric()) diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/picker_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/picker_tests.rs index 61e9b6fb..218cb7d4 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/picker_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/picker_tests.rs @@ -1,5 +1,5 @@ //! Pop-ups and calendars a step works with: a closer that goes with its -//! pop-up, a calendar the task picked in closed for a press behind it, in +//! pop-up (a "Close" or a consent bar's "Accept all"), a calendar the task picked in closed for a press behind it, in //! its own step or a later one, and a date picked from a calendar already //! open. @@ -60,6 +60,44 @@ async fn a_closer_that_goes_with_its_pop_up_ends_a_step_closing_it() { ); } +/// Answers that press the cookie bar's "Accept all" and never judge the +/// step done. +fn press_accept(id: &str, question: &Question, _: &Sim) -> Option { + match id { + "done" | "blocked" => Some(noul(0.05)), + "move" => Some(pick(question, "activate", 0.9)), + _ if id == "target" || id == "region" || id.starts_with("group_") => { + Some(pick(question, "Accept all", 0.9)) + } + _ => None, + } +} + +#[tokio::test] +async fn accepting_a_cookie_bar_ends_the_step_that_accepts_it() { + // A consent bar drawn without a dialog's role closes on "Accept all" + // as surely as a pop-up on "Close": the press going with the bar is + // the evidence the judge cannot see. + let run = run_with( + App::with(|sim| { + sim.quirks.insert(Quirk::CookieBar); + }), + json!({"app": "Mail", "steps": ["accept the cookie banner"]}), + |request| request.disabled_loops.push(FlowLoop::Attention), + press_accept, + ) + .await; + assert_eq!(run.result.stop, FlowStopReason::Completed); + assert_eq!(run.app.sim().clicks, ["Accept all"]); + assert!( + run.result.steps[0] + .note + .contains("closed with what it was on"), + "{}", + run.result.steps[0].note + ); +} + /// Answers that open the calendar, pick a day, then press "Find flights", /// judging the step done once that press went through. fn pick_then_find(id: &str, question: &Question, sim: &Sim) -> Option { diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/screens.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/screens.rs index c7706d00..fe1476a4 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/screens.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/screens.rs @@ -446,6 +446,11 @@ pub(super) fn overlays(sim: &Sim, root: &str, candidates: &mut Vec) { candidates.push(node("Close", "button", &["Click"], &toast, 700.0)); candidates.push(node("Learn more", "link", &["Click"], &toast, 720.0)); } + if sim.has(Quirk::CookieBar) { + let bar = [root, "region \"We use cookies\""]; + candidates.push(node("Accept all", "button", &["Click"], &bar, 700.0)); + candidates.push(node("Cookie policy", "link", &["Click"], &bar, 720.0)); + } if sim.has(Quirk::ConsentBanner) { for candidate in candidates.iter_mut() { candidate.states = vec!["covered".to_owned()]; diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs index 4d6ab0e4..39ab6f3f 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs @@ -35,6 +35,9 @@ pub(super) enum Quirk { DisabledArchive, /// A promo toast with a Close button sits over the page until closed. PromoToast, + /// A cookie bar drawn without a dialog's role, its "Accept all" button + /// the way out, sits over the page until accepted. + CookieBar, /// A consent banner lies over the page as a popover: every other click /// is refused as covered, Escape leaves it, and its "Allow Selection" /// or "Allow all" closes it. @@ -386,6 +389,10 @@ impl AgentBackend for App { sim.quirks.remove(&Quirk::PromoToast); return DesktopResponse::ok("click", json!({})); } + if name == "Accept all" && sim.has(Quirk::CookieBar) { + sim.quirks.remove(&Quirk::CookieBar); + return DesktopResponse::ok("click", json!({})); + } if sim.page().is_some() { press_shop(&mut sim, &name); return sim.located("click"); From d258d57f59ab2a1a15fe23a621d6da3e7fc86d08 Mon Sep 17 00:00:00 2001 From: Shanu Date: Thu, 8 Oct 2026 23:30:07 +0530 Subject: [PATCH 06/15] Read a list option's role in any case closest kept a long option row by an exact role match, while is_one_option, beside it, compares roles in any case. It now uses that helper, so an "Option" row is kept like an "option" one. The in_region rustdoc now says a placing word's region never holds a control that only names it, which is what the code does on purpose. --- .../src/agentic/flow/flow_tests/choose_tests.rs | 6 ++++++ .../src/agentic/flow/steps/matching.rs | 8 +++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/choose_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/choose_tests.rs index 57626d17..b96aa8b0 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/choose_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/choose_tests.rs @@ -249,6 +249,12 @@ fn a_date_is_told_from_other_options_and_containers_give_way() { 0.0, ); assert_eq!(names(closest(vec![link.clone(), row.clone()])).len(), 2); + // However the surface spells its role. + let row = Candidate { + role: "Option".to_owned(), + ..row + }; + assert_eq!(names(closest(vec![link, row])).len(), 2); } #[tokio::test] diff --git a/crates/tinycomputer-engine/src/agentic/flow/steps/matching.rs b/crates/tinycomputer-engine/src/agentic/flow/steps/matching.rs index c4228000..b3d08c2f 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/steps/matching.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/steps/matching.rs @@ -40,7 +40,7 @@ pub(in crate::agentic::flow) fn closest(matches: Vec) -> Vec bool { let wanted = plain(what); if wanted.is_empty() { From 579ec49445010bf019fa5d5b67e777144e983845 Mon Sep 17 00:00:00 2001 From: Shanu Date: Thu, 8 Oct 2026 23:30:07 +0530 Subject: [PATCH 07/15] Close the simulated place list when another box's button is pressed The simulated ride form drops a box's unpicked text when a press lands anywhere else, as a page does, except when the press was the other box's "From"/"To" button: that switched boxes and kept the old list on show. It now closes the open list first, so a flow cannot pass by picking from a list a real page would have closed. --- .../src/agentic/flow/flow_tests/places.rs | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/places.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/places.rs index c89a095d..666e6b5a 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/places.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/places.rs @@ -192,12 +192,25 @@ pub(super) fn drop_unpicked(sim: &mut Sim) { /// whichever that is; a press anywhere else but the box moves the focus on, /// so the list closes and drops the box's unpicked text, as a page does. pub(super) fn press_place(sim: &mut Sim, name: &str) -> bool { - if let Some(places) = sim.places.as_mut().filter(|places| places.behind_buttons) - && let Some(index) = DOORS - .iter() - .position(|door| name.starts_with(&format!("{door} "))) - { - places.door = Some(PLACE_BOXES[index].to_owned()); + let door = sim + .places + .as_ref() + .filter(|places| places.behind_buttons) + .and_then(|_| { + DOORS + .iter() + .position(|door| name.starts_with(&format!("{door} "))) + }); + if let Some(index) = door { + // Another box's button moves the focus on, as a press anywhere else + // does: the open list closes and drops its box's unpicked text. + let open = sim.places.as_ref().and_then(|places| places.open.clone()); + if open.as_deref() != Some(PLACE_BOXES[index]) { + drop_unpicked(sim); + } + if let Some(places) = sim.places.as_mut() { + places.door = Some(PLACE_BOXES[index].to_owned()); + } return true; } let Some(open) = sim.places.as_ref().and_then(|places| places.open.clone()) else { From 86b3b87e7de8abcbe2bc919a2479ce9a14365591 Mon Sep 17 00:00:00 2001 From: Shanu Date: Thu, 8 Oct 2026 23:30:07 +0530 Subject: [PATCH 08/15] Read side-by-side month grids and controls fixed in scrolling lists Sight took each grid calendar's parent as its month while still looking for grids, so when two months' grids sat side by side in one block, the first claimed the block and the second was skipped: none of its days read as dates. Grids are now all found first, and a block holding two of them is no one month; the picker pass still adds it so its arrows page both. A row's middle outside its scrolling container reads offscreen, but a control fixed to the window moves with no container and none clips it. A fixed button drawn from inside a scrolling list read offscreen wherever it showed. scrollerOf now stops at a fixed element. Live tests in Chrome cover both, and each fails without its fix. --- .../src/surface/sight/sight.js | 26 ++++++--- .../sight/sight_tests/live_calendar_tests.rs | 54 +++++++++++-------- .../sight/sight_tests/live_controls_tests.rs | 39 +++++++++++++- 3 files changed, 89 insertions(+), 30 deletions(-) diff --git a/crates/tinycomputer-browser/src/surface/sight/sight.js b/crates/tinycomputer-browser/src/surface/sight/sight.js index a74dbe48..6474a1be 100644 --- a/crates/tinycomputer-browser/src/surface/sight/sight.js +++ b/crates/tinycomputer-browser/src/surface/sight/sight.js @@ -208,10 +208,11 @@ // flight site drew its days as buttons in such a grid, no day read as a // date, and the departure was never picked. const titleUses = new Map(); + const grids = []; for (const grid of base.querySelectorAll('div, ul, ol, tbody')) { const kids = grid.children; if (kids.length < 28 || kids.length > 49 - || calendars.some((calendar) => calendar.contains(grid))) continue; + || [...calendars, ...grids].some((calendar) => calendar.contains(grid))) continue; const days = [...kids].map((kid) => { const leading = /^(\d{1,2})(?:\s|$)/.exec(squash(kid.innerText)); return leading && shown(kid) ? { cell: kid, day: Number(leading[1]) } : null; @@ -229,13 +230,21 @@ titleUses.set(title.element, used + 1); const [month, year] = title.months[Math.min(used, title.months.length - 1)]; const spelled = MONTHS[month]; - const holder = grid.parentElement; - calendars.push(holder && holder !== document.body ? holder : grid); + grids.push(grid); for (const { cell, day } of run) { const pressed = cell.matches(NESTED) ? cell : cell.querySelector(NESTED) || cell; calendarDays.set(pressed, `${day} ${spelled[0].toUpperCase()}${spelled.slice(1)} ${year}`); } } + // Each month is the block holding its grid, with its heading and + // arrows, once every grid is found: a block holding two months' grids + // side by side, with no box of each month's own, is no one month, and + // is added as the picker below. + for (const grid of grids) { + const holder = grid.parentElement; + const shared = holder && grids.some((other) => other !== grid && holder.contains(other)); + calendars.push(holder && holder !== document.body && !shared ? holder : grid); + } // A picker showing two months beside each other pages both with one // pair of arrows, drawn beside the months rather than inside either: // the block that holds more than one month pages them too. @@ -899,15 +908,18 @@ }; // The nearest container that scrolls its content (a popover's list), or - // null. Each container is looked at once. + // null. Each container is looked at once. An element fixed to the window + // moves with no container, and none of them clips it: a fixed pop-up + // drawn from inside a scrolling list shows wherever it is placed. const scrollers = new Map(); const scrollerOf = (element) => { + if (style(element).position === 'fixed') return null; const parent = element.parentElement; if (!parent || parent === document.body || parent === document.documentElement) return null; if (scrollers.has(parent)) return scrollers.get(parent); - const style = getComputedStyle(parent); - const scrolls = (/(auto|scroll)/.test(style.overflowY) && parent.scrollHeight > parent.clientHeight) - || (/(auto|scroll)/.test(style.overflowX) && parent.scrollWidth > parent.clientWidth); + const overflow = style(parent); + const scrolls = (/(auto|scroll)/.test(overflow.overflowY) && parent.scrollHeight > parent.clientHeight) + || (/(auto|scroll)/.test(overflow.overflowX) && parent.scrollWidth > parent.clientWidth); const found = scrolls ? parent : scrollerOf(parent); scrollers.set(parent, found); return found; diff --git a/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_calendar_tests.rs b/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_calendar_tests.rs index 562f4398..e626753c 100644 --- a/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_calendar_tests.rs +++ b/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_calendar_tests.rs @@ -6,18 +6,24 @@ use super::live_tests::{live_reading, shown_names}; /// Two months drawn as grids of buttons, not tables, under one header that -/// names both: each day shows its fare after its number ("22 6529"). +/// names both: each day shows its fare after its number ("22 6529"). Each +/// grid sits in a box of its month's own when `boxed`, or both side by +/// side in one. #[cfg(feature = "agent-browser")] -fn grid_calendar_page() -> String { +fn grid_calendar_page(boxed: bool) -> String { let grid = |blanks: u32, days: u32, fares: u32| { let cells = (0..blanks) .map(|_| "".to_owned()) .chain((1..=days).map(|day| format!("", fares + day * 7))) .collect::(); - format!( - "
Su Mo Tu We Th Fr Sa
\ -
{cells}
" - ) + let grid = format!( + "
{cells}
" + ); + if boxed { + format!("
Su Mo Tu We Th Fr Sa
{grid}
") + } else { + grid + } }; format!( "
October 2026 November 2026
\ @@ -30,22 +36,26 @@ fn grid_calendar_page() -> String { #[cfg(feature = "agent-browser")] #[tokio::test] async fn live_a_calendar_drawn_as_grids_of_buttons_reads_its_days_as_dates() { - let Some(reading) = live_reading(&grid_calendar_page()).await else { - return; - }; - let described = |name: &str| { - reading["nodes"] - .as_array() - .unwrap() - .iter() - .find(|node| node["name"] == name) - .map_or_else( - || panic!("{name} not offered: {:?}", shown_names(&reading)), - |node| node["description"].as_str().unwrap_or_default().to_owned(), - ) - }; - assert_eq!(described("22 6154"), "22 October 2026"); - assert_eq!(described("5 7035"), "5 November 2026"); + // Two grids side by side in one block are two months too, not one + // calendar that hides the second. + for boxed in [true, false] { + let Some(reading) = live_reading(&grid_calendar_page(boxed)).await else { + return; + }; + let described = |name: &str| { + reading["nodes"] + .as_array() + .unwrap() + .iter() + .find(|node| node["name"] == name) + .map_or_else( + || panic!("{name} not offered: {:?}", shown_names(&reading)), + |node| node["description"].as_str().unwrap_or_default().to_owned(), + ) + }; + assert_eq!(described("22 6154"), "22 October 2026", "boxed: {boxed}"); + assert_eq!(described("5 7035"), "5 November 2026", "boxed: {boxed}"); + } } /// A picker showing two months, drawn as a hotel site's was: each month a diff --git a/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_controls_tests.rs b/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_controls_tests.rs index e71caa28..7a4df457 100644 --- a/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_controls_tests.rs +++ b/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_controls_tests.rs @@ -1,7 +1,8 @@ //! Live tests of what sight reads as a control, gated on //! `TINYCOMPUTER_LIVE_BROWSER=1`: rows a script framework wires to a click, //! options inside their trigger, classes behind a variant, icons named in -//! camel case, and a control brought into the window before its press. +//! camel case, a control brought into the window before its press, and a +//! control fixed to the window inside a scrolling list. #[cfg(feature = "agent-browser")] use serde_json::json; @@ -190,3 +191,39 @@ async fn live_a_sprite_its_class_names_in_camel_case_is_read_as_that_icon() { assert_eq!(close["role"], "button"); assert_eq!(close["description"], "an icon"); } + +/// A scrolling list of airports whose last row holds a "Done" button fixed +/// to the window below the list, outside the list's view. +#[cfg(feature = "agent-browser")] +const FIXED_IN_LIST_PAGE: &str = r#"
    +
  • DEL Delhi
  • BLR Bengaluru
  • MAA Chennai
    • +
  • HYD Hyderabad
  • CCU Kolkata
  • BOM Mumbai
    • +
  • GOI Goa
  • PNQ Pune
    • +
    • +
+"#; + +#[cfg(feature = "agent-browser")] +#[tokio::test] +async fn live_a_control_fixed_to_the_window_inside_a_scrolling_list_is_not_scrolled_away() { + let Some(reading) = live_reading(FIXED_IN_LIST_PAGE).await else { + return; + }; + let states = |name: &str| { + reading["nodes"] + .as_array() + .unwrap() + .iter() + .find(|node| node["name"] == name) + .map_or_else( + || panic!("{name} not offered: {:?}", shown_names(&reading)), + |node| node["states"].to_string(), + ) + }; + assert!(!states("Done").contains("offscreen"), "{}", states("Done")); + assert!( + states("PNQ Pune").contains("offscreen"), + "a row below the list's fold still is: {}", + states("PNQ Pune") + ); +} From 27c4886697cf2edb71a5da0b60e0a47dfcca74c1 Mon Sep 17 00:00:00 2001 From: Shanu Date: Thu, 8 Oct 2026 23:30:07 +0530 Subject: [PATCH 09/15] Say what the opener and element-order rules and dialog_left_open mean named_opener's rustdoc said a label holding a slot's word anywhere, while it looks only at the first three words; it now says so and why. seen_first said "those in view first" where it keeps them first but lists every kept element in screen order, as jev-harness.md says. The contract spec's 2.9 note broke the sentence it sat in; it now closes the version list and explains dialog_left_open apart. --- .../src/agentic/flow/ask/screen_state.rs | 13 +++++++------ .../src/agentic/flow/enter/open.rs | 11 +++++++---- docs/technical/specs/desktop-module-contract.md | 9 +++++---- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/crates/tinycomputer-engine/src/agentic/flow/ask/screen_state.rs b/crates/tinycomputer-engine/src/agentic/flow/ask/screen_state.rs index 31857a86..22ead64a 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/ask/screen_state.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/ask/screen_state.rs @@ -36,12 +36,13 @@ pub(in crate::agentic::flow) fn state( state } -/// The `most` of `candidates` Jev is shown, in screen order: those in view -/// first, then the rest as the page orders them, which keep a quarter of -/// the room. Live, a sign-up pop-up a long page drew at the end of its -/// document fell outside the first 120 elements, behind the covered page, -/// and Jev never saw it; and a calendar open in front would fill the room -/// and hide the guests button it covers, which the next step reads. +/// The `most` of `candidates` Jev is shown, all listed in screen order. +/// On a longer screen, those in view are kept first, and the rest, as the +/// page orders them, keep a quarter of the room. Live, a sign-up pop-up a +/// long page drew at the end of its document fell outside the first 120 +/// elements, behind the covered page, and Jev never saw it; and a calendar +/// open in front would fill the room and hide the guests button it covers, +/// which the next step reads. fn seen_first(candidates: &[Candidate], most: usize) -> Vec<&Candidate> { if candidates.len() <= most { return candidates.iter().collect(); diff --git a/crates/tinycomputer-engine/src/agentic/flow/enter/open.rs b/crates/tinycomputer-engine/src/agentic/flow/enter/open.rs index 55dca52d..0b7d7076 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/enter/open.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/enter/open.rs @@ -133,10 +133,13 @@ const BOX_WORDS: &[&str] = &[ const PLACE_LEADS: &[&str] = &["to", "via"]; /// A control on `screen` that is no field itself and whose label holds a -/// word of a pending slot's name (four letters or more, not a box word), -/// or begins with its short place word ([`PLACE_LEADS`]): the link or -/// button that shows the slot's field, such as a store's search link for -/// the slot "search box", or a flight form's "To BLR" for the slot "to". +/// word of a pending slot's name (four letters or more, not a box word) +/// among its first three words, or begins with its short place word +/// ([`PLACE_LEADS`]): the link or button that shows the slot's field, such +/// as a store's search link for the slot "search box", or a flight form's +/// "To BLR" for the slot "to". Further into a label, the slot's word is a +/// sentence's ("Read our tips to search faster"), and a press there leaves +/// the form. pub(in crate::agentic::flow) fn named_opener( screen: &Screen, slots: &[Slot], diff --git a/docs/technical/specs/desktop-module-contract.md b/docs/technical/specs/desktop-module-contract.md index f973f11f..8d1a4013 100644 --- a/docs/technical/specs/desktop-module-contract.md +++ b/docs/technical/specs/desktop-module-contract.md @@ -42,11 +42,12 @@ engine's argument types, the permission preflight, and the bus surface. `JevConfiguration.fast`, the planner's `LanguageModelProvider` routes, and `Capabilities.decision_model`, `planner_model`, `rescue_model`, and `output_model`; 2.9 added `RunFlowRequest.dialog_left_open` and - `FlowRunResult.dialog_left_open`, so a run takes a dialog at its first - look as the task's only when the task's run before it left that dialog - open, all optional) return an + `FlowRunResult.dialog_left_open`; all optional) return an `AgentResponse` instead — see [`unified-agent.md`](unified-agent.md). They - share this interface because a TinyBus module exports one interface. + share this interface because a TinyBus module exports one interface. A + task hands `dialog_left_open` from one run's result to its next run's + request: a run takes a dialog in front at its first look as the task's + own stage only when the run before it left that dialog open. - The thirteen browser members (2.6) close the list, each prefixed `Browser` (`tinycomputer_bus::browser::names`): sessions, navigate, snapshot, perform, read, evaluate, screenshot, held outputs, and downloads. A member From 73d1a3995241790d3fd09a612138615ba883a17f Mon Sep 17 00:00:00 2001 From: Shanu Date: Fri, 9 Oct 2026 00:34:57 +0530 Subject: [PATCH 10/15] Rank a pick of what comes closest to a number by distance A pick "closest to 9" was judged item by item, and live, with a store's sizes 9 and 10 sold out, Jev found none of 6, 7, and 8 clearly closest, so the step failed and every rescue went back to the sold-out 9. tinycomputer-core gains closest_to, which reads the number a criterion such as "closest to 9" or "nearest to size 42" names, and rank_closest, which ranks records by the distance of the first number each shows. pick ranks such a criterion within the list from names (asking which when several show, as for "first", through the new named_list), and takes the first ranked item that belongs, as for prices. Both are additive. --- crates/tinycomputer-bus/src/flow/guide.md | 2 +- crates/tinycomputer-core/src/lib.rs | 3 +- crates/tinycomputer-core/src/records/mod.rs | 2 +- crates/tinycomputer-core/src/records/rank.rs | 68 +++++++++++++++++++ .../src/records/records_tests.rs | 28 +++++++- .../src/agentic/flow/flow_tests/pick_tests.rs | 31 +++++++++ .../src/agentic/flow/steps/list.rs | 62 ++++++++++------- .../tinycomputer-engine/flow/step-kinds.md | 6 +- docs/technical/specs/unified-agent.md | 3 +- 9 files changed, 176 insertions(+), 29 deletions(-) diff --git a/crates/tinycomputer-bus/src/flow/guide.md b/crates/tinycomputer-bus/src/flow/guide.md index 5b489564..b031d347 100644 --- a/crates/tinycomputer-bus/src/flow/guide.md +++ b/crates/tinycomputer-bus/src/flow/guide.md @@ -41,7 +41,7 @@ do. | `choose` | `{"choose": {"what": "the font list", "option": "Helvetica"}}` | Pick an option in a list, menu, or popup, or in a group of option buttons (a size, a colour, a quantity, a day in a strip of dates). | | `read` | `{"read": {"what": "the newest message's subject", "into": "subject"}}` | Store visible text in a variable. Read the price of one item before any step raises its count: after that, its line and the cart show the total for all of them. | | `extract` | `{"extract": {"what": "the flight results", "into": "flights"}}` | Store every item of a list, as JSON rows of their text, in a variable. | -| `pick` | `{"pick": {"from": "the flight results", "by": "lowest price", "into": "flight"}}` | Choose the best of a list of results (cards or rows, each an item to open) and open it; `into` stores its text. Prices, times, durations, and stops are compared exactly. | +| `pick` | `{"pick": {"from": "the flight results", "by": "lowest price", "into": "flight"}}` | Choose the best of a list of results (cards or rows, each an item to open) and open it; `into` stores its text. Prices, times, durations, stops, and nearness to a number ("closest to 9", for a size when 9 may be sold out) are compared exactly. | | `verify` | `{"verify": "the draft shows a recipient"}` | Fail the flow unless this holds. | | `wait_for` | `{"wait_for": "the search results are showing"}` | Wait until this holds. | | `stop_before` | `{"stop_before": "sending the email"}` | Find an irreversible action and stop in front of it. | diff --git a/crates/tinycomputer-core/src/lib.rs b/crates/tinycomputer-core/src/lib.rs index da2e18a3..3ec751cd 100644 --- a/crates/tinycomputer-core/src/lib.rs +++ b/crates/tinycomputer-core/src/lib.rs @@ -52,7 +52,8 @@ pub use error::{Error, Result}; pub use facts::{Facts, is_sensitive_name}; pub use keymap::{Key, Platform}; pub use records::{ - Criterion, Price, Record, parse_clock, parse_duration, parse_price, parse_stops, rank, + Criterion, Price, Record, closest_to, parse_clock, parse_duration, parse_price, parse_stops, + rank, rank_closest, }; pub use safety::{ Consequence, FieldHint, PaymentEvidence, adjusts_a_count, consequence, human_needed, diff --git a/crates/tinycomputer-core/src/records/mod.rs b/crates/tinycomputer-core/src/records/mod.rs index 4845a045..cb81e470 100644 --- a/crates/tinycomputer-core/src/records/mod.rs +++ b/crates/tinycomputer-core/src/records/mod.rs @@ -13,7 +13,7 @@ mod rank; mod schedule; pub use price::{Price, parse_price}; -pub use rank::{Criterion, rank}; +pub use rank::{Criterion, closest_to, rank, rank_closest}; pub use schedule::{parse_clock, parse_duration, parse_stops}; /// One extracted item: field name to the text shown for it. diff --git a/crates/tinycomputer-core/src/records/rank.rs b/crates/tinycomputer-core/src/records/rank.rs index 0d5325fe..e25ffb1b 100644 --- a/crates/tinycomputer-core/src/records/rank.rs +++ b/crates/tinycomputer-core/src/records/rank.rs @@ -155,3 +155,71 @@ pub fn rank(records: &[Record], criterion: Criterion) -> Option> { }); Some(order) } + +/// The number a criterion such as "closest to 9" or "nearest to size 42" +/// asks items to come nearest to; `None` for any other criterion. +/// +/// ``` +/// use tinycomputer_core::closest_to; +/// +/// assert_eq!(closest_to("closest to 9"), Some(9.0)); +/// assert_eq!(closest_to("the size nearest to UK 8.5"), Some(8.5)); +/// assert_eq!(closest_to("lowest price"), None); +/// ``` +#[must_use] +pub fn closest_to(text: &str) -> Option { + let lower = text.to_ascii_lowercase(); + let after = ["closest to", "nearest to"] + .iter() + .find_map(|lead| lower.find(lead).map(|at| &lower[at + lead.len()..]))?; + first_number(after) +} + +/// Record indexes, nearest to `target` first, by the first number each +/// record shows: a size list of 6, 7, and 8 with 9 sold out ranks 8, 7, 6. +/// +/// Ties keep the records' own order, and records that show no number go +/// last. `None` when no record shows one. +/// +/// ``` +/// use tinycomputer_core::{Record, rank_closest}; +/// +/// let sizes = [ +/// Record::from_pairs([("size", "6")]), +/// Record::from_pairs([("size", "7"), ("stock", "3 left")]), +/// Record::from_pairs([("size", "8"), ("stock", "2 left")]), +/// ]; +/// assert_eq!(rank_closest(&sizes, 9.0), Some(vec![2, 1, 0])); +/// ``` +#[must_use] +pub fn rank_closest(records: &[Record], target: f64) -> Option> { + let keyed = records + .iter() + .map(|record| { + record + .fields + .values() + .find_map(|text| first_number(text)) + .map(|number| (number - target).abs()) + }) + .collect::>(); + if keyed.iter().all(Option::is_none) { + return None; + } + let mut order = (0..records.len()).collect::>(); + order.sort_by(|&left, &right| match (keyed[left], keyed[right]) { + (Some(a), Some(b)) => a.total_cmp(&b), + (Some(_), None) => std::cmp::Ordering::Less, + (None, Some(_)) => std::cmp::Ordering::Greater, + (None, None) => std::cmp::Ordering::Equal, + }); + Some(order) +} + +/// The first number `text` shows ("7" of "7 3 left", "8.5" of "UK 8.5"). +fn first_number(text: &str) -> Option { + text.split(|character: char| !(character.is_ascii_digit() || character == '.')) + .map(|word| word.trim_matches('.')) + .find(|word| !word.is_empty()) + .and_then(|word| word.parse().ok()) +} diff --git a/crates/tinycomputer-core/src/records/records_tests.rs b/crates/tinycomputer-core/src/records/records_tests.rs index b9ca5fbf..6b916e2a 100644 --- a/crates/tinycomputer-core/src/records/records_tests.rs +++ b/crates/tinycomputer-core/src/records/records_tests.rs @@ -3,7 +3,8 @@ #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] use super::{ - Criterion, Price, Record, parse_clock, parse_duration, parse_price, parse_stops, rank, + Criterion, Price, Record, closest_to, parse_clock, parse_duration, parse_price, parse_stops, + rank, rank_closest, }; fn price(text: &str) -> (f64, Option<&'static str>) { @@ -238,3 +239,28 @@ fn first_and_last_alone_are_the_lists_own_order() { ); assert_eq!(rank(&[], Criterion::First), None, "nothing to rank"); } + +#[test] +fn closest_to_reads_the_number_a_criterion_comes_nearest_to() { + assert_eq!(closest_to("closest to 9"), Some(9.0)); + assert_eq!(closest_to("Nearest to size 42"), Some(42.0)); + assert_eq!(closest_to("the size nearest to UK 8.5"), Some(8.5)); + assert_eq!(closest_to("closest to the station"), None); + assert_eq!(closest_to("cheapest"), None); +} + +#[test] +fn nearness_ranks_by_distance_and_numberless_records_go_last() { + let sizes = [ + Record::from_pairs([("size", "6")]), + Record::from_pairs([("label", "Size chart")]), + Record::from_pairs([("size", "10"), ("stock", "1 left")]), + Record::from_pairs([("size", "8"), ("stock", "2 left")]), + ]; + // 10 and 8 are as near to 9: the list's own order breaks the tie. + assert_eq!(rank_closest(&sizes, 9.0), Some(vec![2, 3, 0, 1])); + assert_eq!( + rank_closest(&[Record::from_pairs([("label", "Size chart")])], 9.0), + None + ); +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/pick_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/pick_tests.rs index 2203c57f..d32efdda 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/pick_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/pick_tests.rs @@ -32,6 +32,37 @@ async fn pick_ranks_a_measurable_criterion_exactly_and_opens_the_winner() { } } +#[tokio::test] +async fn pick_ranks_nearness_to_a_number_by_distance() { + // Live, a store's sizes 9 and 10 were sold out, and "closest to 9", + // judged item by item, took none of 6, 7, and 8. + let run = run( + App::with(|sim| { + sim.results = vec![ + ("6", "₹255", "in stock"), + ("7", "₹255", "3 left"), + ("8", "₹255", "2 left"), + ]; + }), + json!({"app": "Mail", "steps": [ + {"pick": {"from": "the size options", "by": "closest to 9", "into": "size"}} + ]}), + ) + .await; + assert_eq!(run.app.sim().picked, ["@s:select-3"]); + assert!( + run.result.steps[0].note.contains("ranked"), + "{}", + run.result.steps[0].note + ); + assert!( + !run.requests + .iter() + .any(|request| request.questions.contains_key("record")), + "nearness to a number needs no judgement" + ); +} + /// Says an item belongs to the list picked from only when it shows `brand`. fn belongs_when(brand: &'static str) -> impl Fn(&str, &Question, &Sim) -> Option { move |id, question, _| { diff --git a/crates/tinycomputer-engine/src/agentic/flow/steps/list.rs b/crates/tinycomputer-engine/src/agentic/flow/steps/list.rs index f16f9a32..00394cd2 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/steps/list.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/steps/list.rs @@ -6,7 +6,7 @@ use std::collections::BTreeMap; use serde_json::json; use tinycomputer_bus::{FlowLoop, JevOperation, PickStep, ReadStep, StepOutcome}; use tinycomputer_core::surface::{Group, result_families}; -use tinycomputer_core::{Criterion, Record, rank}; +use tinycomputer_core::{Criterion, Record, closest_to, rank, rank_closest}; use tinyinference_decisions::Answer; use crate::agentic::flow::{ @@ -23,12 +23,13 @@ use super::{ impl FlowRun<'_, B> { /// Picks the best of a list of results by `pick.by`, stores its text, - /// and opens it. A criterion over prices, times, durations, or stops is - /// ranked exactly, and the first ranked item that Jev confirms belongs - /// to `pick.from` is taken: the ranking reads only its measure, and - /// live, "the cheapest of the results rated 4 stars or more" took a - /// 3.1-star item of another brand. Anything else, or a ranking none of - /// whose leaders belongs, is judged by Jev among the records. + /// and opens it. A criterion over prices, times, durations, or stops, + /// or nearness to a number ("closest to 9"), is ranked exactly, and the + /// first ranked item that Jev confirms belongs to `pick.from` is taken: + /// the ranking reads only its measure, and live, "the cheapest of the + /// results rated 4 stars or more" took a 3.1-star item of another brand. + /// Anything else, or a ranking none of whose leaders belongs, is judged + /// by Jev among the records. pub(super) async fn pick(&mut self, log: &mut StepLog, pick: &PickStep) -> Result { let from = substitute_safe(&pick.from, &self.vars, &self.facts); let by = substitute_safe(&pick.by, &self.vars, &self.facts); @@ -47,22 +48,25 @@ impl FlowRun<'_, B> { let condition = first_meeting(&by); let criterion = Criterion::parse(&by).or_else(|| condition.as_ref().map(|_| Criterion::First)); - let ranked = match criterion { + let ranked = match (closest_to(&by), criterion) { + // Nearness to a number ranks by distance, within the list `from` + // names, since numbers show in every list. Live, a store's sizes + // 9 and 10 were sold out, and "closest to 9", judged item by + // item, took none of 6, 7, and 8. + (Some(target), _) => { + let groups = self.named_list(log, &screen, &from, &families).await?; + rank_closest(&records_of(groups), target).map(|order| (groups, order)) + } // The list's own order fits every list on the page, so the one // `from` names is asked for first, as a judged pick does. - Some(order @ (Criterion::First | Criterion::Last)) => { - let list = if families.len() > 1 { - self.judge_list(log, &screen, &from, &families).await? - } else { - 0 - }; - let groups = &families[list]; + (None, Some(order @ (Criterion::First | Criterion::Last))) => { + let groups = self.named_list(log, &screen, &from, &families).await?; rank(&records_of(groups), order).map(|ranking| (groups, ranking)) } - Some(criterion) => families.iter().find_map(|groups| { + (None, Some(criterion)) => families.iter().find_map(|groups| { rank(&records_of(groups), criterion).map(|order| (groups, order)) }), - None => None, + (None, None) => None, }; let meets = condition.map_or_else(|| from.clone(), |condition| format!("{from}, {condition}")); @@ -78,12 +82,7 @@ impl FlowRun<'_, B> { } else { // Several lists show (a chat list beside the open chat's // messages): judge within the one `from` names. - let list = if families.len() > 1 { - self.judge_list(log, &screen, &from, &families).await? - } else { - 0 - }; - let groups = &families[list]; + let groups = self.named_list(log, &screen, &from, &families).await?; let best = self.judge_pick(log, &screen, &from, &by, groups).await?; (groups, best, "judged") }; @@ -197,6 +196,23 @@ impl FlowRun<'_, B> { )) } + /// The list of `families` that `from` names: the only one, or the one + /// Jev chooses when several show. + async fn named_list<'f>( + &mut self, + log: &mut StepLog, + screen: &crate::agentic::flow::view::Screen, + from: &str, + families: &'f [Vec], + ) -> Result<&'f Vec, Halt> { + let list = if families.len() > 1 { + self.judge_list(log, screen, from, families).await? + } else { + 0 + }; + Ok(&families[list]) + } + /// Asks Jev which of the lists showing is `what`, each shown by its /// first [`LIST_PREVIEW`] items, among the first [`MAX_LISTS`]. A list /// not clearly chosen falls back to the one Jev leaned to diff --git a/docs/crates/tinycomputer-engine/flow/step-kinds.md b/docs/crates/tinycomputer-engine/flow/step-kinds.md index 4d17c6fc..a25d7b9f 100644 --- a/docs/crates/tinycomputer-engine/flow/step-kinds.md +++ b/docs/crates/tinycomputer-engine/flow/step-kinds.md @@ -121,7 +121,11 @@ The exact rankings are: - lowest or highest price, - earliest or latest time, - fewest stops, -- shortest duration. +- shortest duration, +- nearest to a number ("closest to 9", "nearest to size 42"): each card's + first number, by its distance, within the list `from` names (asked for + first when several show, as for "first"), since numbers show in every + list. When it parses ("lowest price"), the parsers in `tinycomputer-core/src/records/` read prices with currency symbols, clock diff --git a/docs/technical/specs/unified-agent.md b/docs/technical/specs/unified-agent.md index c2703bd3..7e37cba6 100644 --- a/docs/technical/specs/unified-agent.md +++ b/docs/technical/specs/unified-agent.md @@ -104,7 +104,8 @@ These are additive to the grammar in [`jev-intent-flows.md`](jev-intent-flows.md - `in "" {steps}` - `extract {what, fields, into}` - `pick {from, by, into}` (implemented): chooses the best result card and - opens it; prices, times, durations, and stops are ranked exactly + opens it; prices, times, durations, stops, and nearness to a number + ("closest to 9") are ranked exactly - `ask [slots]` - `checkpoint ""` From 4441225dfcc27317696862297599c51c3e673a77 Mon Sep 17 00:00:00 2001 From: Shanu Date: Fri, 9 Oct 2026 00:34:57 +0530 Subject: [PATCH 11/15] Let Return run the search typed into a box in front of a sheet Return is refused while a sheet or alert shows, since there it presses the dialog's default button. Live, a store's search opened as a full-window sheet, and the step to press Enter in the box just typed into was refused 154 times; a rescue had to pick a suggestion instead. FlowRun now keeps the field its last action typed into (typed_last, kept through waits), and Return goes through in front of a sheet when that field is a search box: a searchbox, or a box that takes text and names itself for searching. A simulated search that opens as a sheet checks both that Return runs it and that Return stays refused with no search box typed into. --- .../src/agentic/flow/act/moves.rs | 23 +++++++- .../src/agentic/flow/action.rs | 6 +++ .../agentic/flow/flow_tests/do_loop_tests.rs | 53 +++++++++++++++++++ .../src/agentic/flow/flow_tests/simulator.rs | 4 +- .../src/agentic/flow/mod.rs | 3 ++ .../src/agentic/flow/run.rs | 1 + .../tinycomputer-engine/flow/the-do-loop.md | 5 +- docs/technical/decision-loops.md | 3 +- 8 files changed, 94 insertions(+), 4 deletions(-) diff --git a/crates/tinycomputer-engine/src/agentic/flow/act/moves.rs b/crates/tinycomputer-engine/src/agentic/flow/act/moves.rs index 8867fcac..feca51b4 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/act/moves.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/act/moves.rs @@ -70,7 +70,12 @@ impl FlowRun<'_, B> { )); return Ok(Move::Skipped); } - if combo == "return" && screen.surface != "window" { + // Return in a search box runs its search wherever the box + // sits: live, a store's search opened as a full-window + // sheet, and its step to press Enter in the box just typed + // into was refused 154 times. + let searching = self.typed_last.as_ref().is_some_and(is_search_box); + if combo == "return" && screen.surface != "window" && !searching { self.history.push(format!( "refused return while a {} is showing: it would press its default button", screen.surface @@ -343,3 +348,19 @@ fn asks_to_close(intent: &str) -> bool { ) }) } + +/// Whether `field` is a search box: a `searchbox`, or a box that takes text +/// and names itself for searching ("Search Lenskart", "Search for atta dal +/// and more"). Return there runs the search, never a dialog's default +/// button. +pub(in crate::agentic::flow) fn is_search_box(field: &Candidate) -> bool { + let takes_text = field + .available_actions + .iter() + .any(|action| action == "SetValue" || action == "TypeText"); + let named = [field.name.as_deref(), field.description.as_deref()] + .into_iter() + .flatten() + .any(|text| text.to_lowercase().contains("search")); + field.role.eq_ignore_ascii_case("searchbox") || (takes_text && named) +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/action.rs b/crates/tinycomputer-engine/src/agentic/flow/action.rs index a6ff02ab..5c49a9ad 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/action.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/action.rs @@ -29,6 +29,12 @@ impl FlowRun<'_, B> { } self.actions = self.actions.saturating_add(1); self.front.act(action, target); + if action != "wait" { + let typing = ["fill", "type", "paste"] + .iter() + .any(|verb| action.starts_with(verb)); + self.typed_last = target.filter(|_| typing).cloned(); + } let started = Instant::now(); let reply = self.backend_call(call).await; let acted_ms = millis(started.elapsed()); diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs index ebd91c62..3181e245 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs @@ -746,6 +746,59 @@ async fn a_control_the_dialogs_own_bar_covers_is_pressed_and_one_behind_it_is_no ); } +#[tokio::test] +async fn return_in_a_search_box_runs_the_search_with_a_sheet_in_front() { + // Live, a store's search opened as a full-window sheet, and the step to + // press Enter in the box just typed into was refused 154 times, as if + // Return would press the sheet's default button. + let pressing_return = |id: &str, question: &Question, sim: &Sim| match id { + "done" => Some(noul(if sim.presses.iter().any(|key| key == "return") { + 0.95 + } else { + 0.05 + })), + "move" => Some(pick(question, "shortcut", 0.9)), + "shortcut" => Some(pick(question, "confirm", 0.95)), + _ => None, + }; + let run = run_with( + App::with(|sim| { + sim.quirks.insert(Quirk::SearchBehindLink); + sim.quirks.insert(Quirk::SearchOpen); + sim.quirks.insert(Quirk::SearchSheet); + }), + json!({"app": "Mail", "steps": [ + {"enter": {"search": "invoices"}}, + "press Enter in the search box" + ]}), + |request| request.disabled_loops.push(FlowLoop::Attention), + pressing_return, + ) + .await; + let presses = run.app.sim().presses.clone(); + assert!(presses.iter().any(|key| key == "return"), "{presses:?}"); + assert_eq!(run.result.stop, FlowStopReason::Completed); + + // Without a search box typed into, Return in front of a sheet stays + // refused: it would press the sheet's default button. + let run = run_with( + App::with(|sim| { + sim.quirks.insert(Quirk::SearchBehindLink); + sim.quirks.insert(Quirk::SearchOpen); + sim.quirks.insert(Quirk::SearchSheet); + }), + json!({"app": "Mail", "steps": ["press Enter"]}), + |request| { + request.disabled_loops.push(FlowLoop::Attention); + request.max_actions = 3; + }, + pressing_return, + ) + .await; + let presses = run.app.sim().presses.clone(); + assert!(!presses.iter().any(|key| key == "return"), "{presses:?}"); +} + /// A page at `surface`, its `covered` controls drawn under something. fn page_at(surface: &str, covered: usize) -> Screen { Screen { diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs index 39ab6f3f..3a65cb98 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs @@ -38,6 +38,8 @@ pub(super) enum Quirk { /// A cookie bar drawn without a dialog's role, its "Accept all" button /// the way out, sits over the page until accepted. CookieBar, + /// The inbox's search, once open, fills the window as a sheet. + SearchSheet, /// A consent banner lies over the page as a popover: every other click /// is refused as covered, Escape leaves it, and its "Allow Selection" /// or "Allow all" closes it. @@ -270,7 +272,7 @@ impl App { /// What the simulated page shows in front: the obstacle's sheet, a /// calendar that stays open as a dialog, or the window. fn surface_of(sim: &Sim) -> String { - if sim.obstacle { + if sim.obstacle || (sim.has(Quirk::SearchSheet) && sim.has(Quirk::SearchOpen)) { "sheet".to_owned() } else if sim.has(Quirk::CalendarStaysOpen) && sim diff --git a/crates/tinycomputer-engine/src/agentic/flow/mod.rs b/crates/tinycomputer-engine/src/agentic/flow/mod.rs index 70889571..4ab977ea 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/mod.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/mod.rs @@ -286,6 +286,9 @@ pub(super) struct FlowRun<'r, B> { /// field holding text the flow typed shows no choice the page made /// (`steps::already_holds`). pub(super) typed: BTreeSet, + /// The field the run's last action typed into, while nothing else has + /// acted since but waits: the focus is still in it. + pub(super) typed_last: Option, /// How much the run deliberates before acting on a decision. deliberation: Deliberation, /// Every framing's own answer to each question, under the original diff --git a/crates/tinycomputer-engine/src/agentic/flow/run.rs b/crates/tinycomputer-engine/src/agentic/flow/run.rs index 091fb958..889b6a2a 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/run.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/run.rs @@ -132,6 +132,7 @@ impl<'r, B: AgentBackend + Sync> FlowRun<'r, B> { read: request.collected.keys().cloned().collect(), refused: BTreeSet::new(), typed: BTreeSet::new(), + typed_last: None, deliberation: request.deliberation, ballots: BTreeMap::new(), asked: BTreeMap::new(), diff --git a/docs/crates/tinycomputer-engine/flow/the-do-loop.md b/docs/crates/tinycomputer-engine/flow/the-do-loop.md index 10f1c811..1adc4406 100644 --- a/docs/crates/tinycomputer-engine/flow/the-do-loop.md +++ b/docs/crates/tinycomputer-engine/flow/the-do-loop.md @@ -152,7 +152,10 @@ They are written in macOS spelling (`cmd+n`) and each surface translates them: the desktop sends `ctrl+n` on Windows and Linux, the browser sends `Meta+n` or `Control+n`. Return is refused while a sheet or alert is showing, because there it presses that dialog's default button, which -might not be the one the step wants. +might not be the one the step wants, unless the run's last action typed +into a search box (`is_search_box`): Return there runs the search. Live, a +store's search opened as a full-window sheet, and pressing Enter in it +was refused 154 times. ### The destructive check diff --git a/docs/technical/decision-loops.md b/docs/technical/decision-loops.md index 3b049df3..bc238dab 100644 --- a/docs/technical/decision-loops.md +++ b/docs/technical/decision-loops.md @@ -272,7 +272,8 @@ folder, find, reply, settings, back, next field, confirm (Return), and dismiss spelling (`cmd+n`); each surface translates them, so the desktop sends `ctrl+n` on Windows and Linux and the browser sends `Meta+n` or `Control+n`. Return is refused while a sheet or alert is showing, because there it presses -the default button. +the default button, unless the run's last action typed into a search box, +where Return runs the search. Before any click, `is_destructive` runs. It refuses the click, and fails the step, when: From eb42bbf11ddea7f607777cc25d8492eaa31b405a Mon Sep 17 00:00:00 2001 From: Shanu Date: Fri, 9 Oct 2026 00:34:57 +0530 Subject: [PATCH 12/15] Describe a control each card repeats by the card it sits in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A product grid repeats one "ADD" on every card, and by its name alone every copy is the same. Live, "add the first Maggi product" pressed the first card's ADD, on a ramen above the Maggi, and two packets of the wrong product went into the cart. Sight now describes a control whose role and name repeat, with no description of its own, by the name of the card holding it when that card is a control itself ("in Maggi Double Masala 95 g ₹20"). A live test in Chrome checks it, and that a control no other card repeats is left alone. --- .../src/surface/sight/sight.js | 20 ++++++++++ .../sight/sight_tests/live_controls_tests.rs | 39 ++++++++++++++++++- docs/crates/tinycomputer-browser/sight.md | 5 +++ 3 files changed, 62 insertions(+), 2 deletions(-) diff --git a/crates/tinycomputer-browser/src/surface/sight/sight.js b/crates/tinycomputer-browser/src/surface/sight/sight.js index 6474a1be..49609b9e 100644 --- a/crates/tinycomputer-browser/src/surface/sight/sight.js +++ b/crates/tinycomputer-browser/src/surface/sight/sight.js @@ -1257,6 +1257,26 @@ seen.push({ element, record }); offerChoices(element, record); } + // A control repeated on every card ("ADD" on each product), kept inside + // the card that is a control itself, says nothing of which card it acts + // on: it is described by that card's name. Live, "add the first Maggi" + // pressed the first card's "ADD", on a ramen above the Maggi. + const recordOf = new Map(seen.map(({ element, record }) => [element, record])); + const copies = new Map(); + for (const { record } of seen) { + const key = `${record.role}\u0000${record.name}`; + copies.set(key, (copies.get(key) || 0) + 1); + } + for (const { element, record } of seen) { + if (record.description || !record.name + || copies.get(`${record.role}\u0000${record.name}`) < 2) continue; + for (let parent = element.parentElement; parent; parent = parent.parentElement) { + const card = recordOf.get(parent); + if (!card) continue; + if (card.name && card.name !== record.name) record.description = clip(`in ${card.name}`, limits.name); + break; + } + } window.__tinycomputerSeen = next; const middle = document.elementFromPoint(width / 2, height / 2); diff --git a/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_controls_tests.rs b/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_controls_tests.rs index 7a4df457..0b256817 100644 --- a/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_controls_tests.rs +++ b/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_controls_tests.rs @@ -1,8 +1,9 @@ //! Live tests of what sight reads as a control, gated on //! `TINYCOMPUTER_LIVE_BROWSER=1`: rows a script framework wires to a click, //! options inside their trigger, classes behind a variant, icons named in -//! camel case, a control brought into the window before its press, and a -//! control fixed to the window inside a scrolling list. +//! camel case, a control brought into the window before its press, a +//! control fixed to the window inside a scrolling list, and a control each +//! card repeats, told apart by its card. #[cfg(feature = "agent-browser")] use serde_json::json; @@ -227,3 +228,37 @@ async fn live_a_control_fixed_to_the_window_inside_a_scrolling_list_is_not_scrol states("PNQ Pune") ); } + +/// Product cards a press opens, each with its own "ADD" button, and one card +/// with a button no other card has. +#[cfg(feature = "agent-browser")] +const CARD_BUTTONS_PAGE: &str = r#"
+
Too Yumm Korean Ramen 79 g ₹49
+
Maggi Double Masala 95 g ₹20
+
Maggi Masala 280 g ₹56
+
"#; + +#[cfg(feature = "agent-browser")] +#[tokio::test] +async fn live_a_control_each_card_repeats_is_described_by_its_card() { + let Some(reading) = live_reading(CARD_BUTTONS_PAGE).await else { + return; + }; + let nodes = reading["nodes"].as_array().unwrap(); + let described = |name: &str| { + nodes + .iter() + .filter(|node| node["name"] == name) + .map(|node| node["description"].as_str().unwrap_or_default().to_owned()) + .collect::>() + }; + let adds = described("ADD"); + assert_eq!(adds.len(), 2, "{:?}", shown_names(&reading)); + assert!(adds[0].starts_with("in Too Yumm Korean Ramen"), "{adds:?}"); + assert!(adds[1].starts_with("in Maggi Double Masala"), "{adds:?}"); + assert_eq!( + described("Notify me"), + [""], + "a control no other card repeats needs no card to tell it apart" + ); +} diff --git a/docs/crates/tinycomputer-browser/sight.md b/docs/crates/tinycomputer-browser/sight.md index 2171ec03..9d897916 100644 --- a/docs/crates/tinycomputer-browser/sight.md +++ b/docs/crates/tinycomputer-browser/sight.md @@ -83,6 +83,11 @@ Sight decides what is a control by behavior, not by role: name ending in `-selected` or `-checked` (never `unselected` or `not-selected`) marks one, as a store's picked size (`size-buttons-size-button-selected`) carries no ARIA state at all. +- A control repeated on every card ("ADD" on each product), inside a card + that is a control itself, is described by that card's name (`in Maggi + Double Masala 95 g ₹20`) when it has no description of its own: by its + name alone, every copy is the same, and live, "add the first Maggi" + pressed the first card's "ADD", on a ramen above the Maggi. The IndiGo case above is the sharpest illustration of the next rule: **a claimed text box that takes no text is a button, not a text box.** Sight From fa2e6a69a45fa730305c16859685a43efdb8b958 Mon Sep 17 00:00:00 2001 From: Shanu Date: Fri, 9 Oct 2026 12:40:56 +0530 Subject: [PATCH 13/15] Read calendar labels in full and keep Return to a box still shown From the second review of this branch: holds_calendar read a day only as a label's first word, so cells named whole dates ("Thu Oct 01 2026", "Choose Thursday, October 22nd, 2026") were no calendar's days. A short label that names a month now holds its day anywhere, with a leading zero or an ordinal ending (day_of). turns_the_month reads the description too: a "Next" described "next month" turns the month. Return in front of a sheet went through on the field typed into last even after a dialog came up over it. The field must still show, uncovered (still_shows): a dialog over it takes the keys. named_opener took a link or button only in lower case; any case now. records_of numbers its fields with three digits, so their keys sort in reading order past ten fields, as rank_closest and the price parsers read them. --- .../src/agentic/flow/act/mod.rs | 2 + .../src/agentic/flow/act/moves.rs | 19 +++++- .../src/agentic/flow/enter/open.rs | 4 +- .../agentic/flow/flow_tests/do_loop_tests.rs | 62 +++++++++++++++++++ .../agentic/flow/flow_tests/enter_tests.rs | 10 +++ .../src/agentic/flow/front.rs | 55 +++++++++++----- .../src/agentic/flow/steps/list.rs | 5 +- 7 files changed, 137 insertions(+), 20 deletions(-) diff --git a/crates/tinycomputer-engine/src/agentic/flow/act/mod.rs b/crates/tinycomputer-engine/src/agentic/flow/act/mod.rs index d455528e..66e3db00 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/act/mod.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/act/mod.rs @@ -31,6 +31,8 @@ mod dialog; pub(in crate::agentic::flow) use copies::copies_of; mod judge; mod moves; +#[cfg(test)] +pub(in crate::agentic::flow) use moves::still_shows; mod recover; mod turns; mod uncover; diff --git a/crates/tinycomputer-engine/src/agentic/flow/act/moves.rs b/crates/tinycomputer-engine/src/agentic/flow/act/moves.rs index feca51b4..0ec2f604 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/act/moves.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/act/moves.rs @@ -74,7 +74,10 @@ impl FlowRun<'_, B> { // sits: live, a store's search opened as a full-window // sheet, and its step to press Enter in the box just typed // into was refused 154 times. - let searching = self.typed_last.as_ref().is_some_and(is_search_box); + let searching = self + .typed_last + .as_ref() + .is_some_and(|field| is_search_box(field) && still_shows(screen, field)); if combo == "return" && screen.surface != "window" && !searching { self.history.push(format!( "refused return while a {} is showing: it would press its default button", @@ -364,3 +367,17 @@ pub(in crate::agentic::flow) fn is_search_box(field: &Candidate) -> bool { .any(|text| text.to_lowercase().contains("search")); field.role.eq_ignore_ascii_case("searchbox") || (takes_text && named) } + +/// Whether `field` still shows on `screen` with nothing covering it: a +/// dialog that has come up over it since it was typed into takes the keys +/// itself. +pub(in crate::agentic::flow) fn still_shows(screen: &Screen, field: &Candidate) -> bool { + screen.candidates.iter().any(|candidate| { + candidate.role == field.role + && candidate.name == field.name + && !candidate + .states + .iter() + .any(|state| state.eq_ignore_ascii_case("covered")) + }) +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/enter/open.rs b/crates/tinycomputer-engine/src/agentic/flow/enter/open.rs index 0b7d7076..19d25e17 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/enter/open.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/enter/open.rs @@ -168,7 +168,9 @@ pub(in crate::agentic::flow) fn named_opener( .candidates .iter() .filter(|candidate| { - matches!(candidate.role.as_str(), "link" | "button") + ["link", "button"] + .iter() + .any(|role| candidate.role.eq_ignore_ascii_case(role)) && candidate .available_actions .iter() diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs index 3181e245..94d6f2c4 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs @@ -967,6 +967,68 @@ fn a_grid_of_bare_numbers_is_a_calendar_only_beside_a_month() { } } +#[test] +fn a_calendar_reads_its_days_from_labels_that_name_their_month() { + // Day cells named whole dates, the day not first ("Thu Oct 01 2026", + // "Choose Thursday, October 22nd, 2026"), are a calendar's days too. + use crate::agentic::flow::front::Front; + let at = Some("https://stays.test/"); + let mut calendar = page_at("sheet", 3); + calendar.candidates.extend((1..=7).map(|day: u32| { + node( + &format!("Choose Thursday, October {day}th, 2026"), + "button", + &["Click"], + &["main"], + f64::from(100 + day), + ) + })); + let mut front = Front::default(); + front.act("browse https://stays.test/", None); + front.look(&page_at("window", 0), at); + front.act( + "click", + Some(&node("Check-in", "button", &["Click"], &["main"], 0.0)), + ); + front.look(&calendar, at); + // An arrow named "Next" and described "next month" turns the month. + let next = Candidate { + description: Some("next month".to_owned()), + ..node("Next", "button", &["Click"], &["main"], 90.0) + }; + front.act("click", Some(&next)); + front.look(&calendar, at); + assert!(!front.served_calendar(), "a month turned is no day chosen"); + front.act( + "click", + Some(&node( + "Choose Thursday, October 2th, 2026", + "button", + &["Click"], + &["main"], + 102.0, + )), + ); + front.look(&calendar, at); + assert!(front.served_calendar(), "its days are read as a calendar's"); +} + +#[test] +fn a_search_box_typed_into_counts_only_while_it_shows_uncovered() { + use crate::agentic::flow::act::still_shows; + let field = node("Search Lenskart", "textbox", &["SetValue"], &["main"], 0.0); + let mut screen = page_at("sheet", 0); + screen.candidates.push(field.clone()); + assert!(still_shows(&screen, &field)); + screen.candidates.last_mut().unwrap().states = vec!["covered".to_owned()]; + assert!( + !still_shows(&screen, &field), + "a dialog over it takes the keys" + ); + screen.candidates.pop(); + assert!(!still_shows(&screen, &field), "nor once it is gone"); +} + #[test] fn a_run_that_never_looked_hands_on_the_dialog_it_was_left() { use crate::agentic::flow::front::Front; diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/enter_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/enter_tests.rs index 35ae1c5a..58dcf204 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/enter_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/enter_tests.rs @@ -473,4 +473,14 @@ fn a_short_place_word_names_its_box_only_as_a_labels_first_word() { opener.name.as_deref(), Some("To BLR, Bengaluru Airport India") ); + // However the surface spells the role. + let mut screen = screen; + for candidate in &mut screen.candidates { + candidate.role = "Button".to_owned(); + } + let opener = named_opener(&screen, &slots, &BTreeSet::from([0])).unwrap(); + assert_eq!( + opener.name.as_deref(), + Some("To BLR, Bengaluru Airport India") + ); } diff --git a/crates/tinycomputer-engine/src/agentic/flow/front.rs b/crates/tinycomputer-engine/src/agentic/flow/front.rs index 36206739..c7103f53 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/front.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/front.rs @@ -257,24 +257,46 @@ fn covered_count(screen: &Screen) -> usize { } /// Whether `target` only turns a calendar's month ("next month", "Previous -/// month"), which answers nothing the calendar asks. +/// month", or a "Next" described "next month"), which answers nothing the +/// calendar asks. fn turns_the_month(target: &Candidate) -> bool { - let words = target - .name - .as_deref() - .unwrap_or_default() - .split(|character: char| !character.is_alphanumeric()) - .filter(|word| !word.is_empty()) - .map(str::to_lowercase) - .collect::>(); - let said = words.join(" "); - words.len() <= 4 && (said.contains("next month") || said.contains("previous month")) + [target.name.as_deref(), target.description.as_deref()] + .into_iter() + .flatten() + .any(|text| { + let words = text + .split(|character: char| !character.is_alphanumeric()) + .filter(|word| !word.is_empty()) + .map(str::to_lowercase) + .collect::>(); + let said = words.join(" "); + words.len() <= 4 && (said.contains("next month") || said.contains("previous month")) + }) +} + +/// The day of the month `word` names: 1 to 31, with a leading zero or an +/// ordinal ending ("01", "22nd"). +fn day_of(word: &str) -> Option { + let digits = ["st", "nd", "rd", "th"] + .iter() + .find_map(|ending| { + word.to_ascii_lowercase() + .strip_suffix(ending) + .map(str::to_owned) + }) + .unwrap_or_else(|| word.to_owned()); + digits + .parse::() + .ok() + .filter(|day| (1..=31).contains(day)) } /// Whether `screen` shows a calendar: [`CALENDAR_DAYS`] or more day numbers /// nothing covers, each beside a month's name ("1 September 2026", or "1" /// described "Thu Oct 01 2026"), or a grid cell on a screen that names a /// month somewhere: a seat map's or a table's cells are bare numbers too. +/// A short label that names a month holds its day anywhere ("Thu Oct 01 +/// 2026", "Choose Thursday, October 22nd, 2026"). fn holds_calendar(screen: &Screen) -> bool { let month_shown = screen .candidates @@ -289,11 +311,12 @@ fn holds_calendar(screen: &Screen) -> bool { .iter() .filter(|candidate| { let name = candidate.name.as_deref().unwrap_or_default(); - let day = name - .split_whitespace() - .next() - .and_then(|word| word.parse::().ok()) - .is_some_and(|day| (1..=31).contains(&day)); + let day = name.split_whitespace().next().and_then(day_of).is_some() + || (names_a_month(name) + && name.split_whitespace().count() <= 6 + && name + .split(|character: char| !character.is_alphanumeric()) + .any(|word| day_of(word).is_some())); let dated = (month_shown && candidate.role.eq_ignore_ascii_case("gridcell")) || names_a_month(name) || candidate.description.as_deref().is_some_and(names_a_month); diff --git a/crates/tinycomputer-engine/src/agentic/flow/steps/list.rs b/crates/tinycomputer-engine/src/agentic/flow/steps/list.rs index 00394cd2..dd04fa52 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/steps/list.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/steps/list.rs @@ -378,7 +378,8 @@ fn openable(families: Vec>) -> Vec> { } } -/// Each card's text as a record, its fields numbered in reading order. +/// Each card's text as a record, its fields numbered in reading order, and +/// numbered so that their keys sort in that order too. fn records_of(groups: &[Group]) -> Vec { groups .iter() @@ -387,7 +388,7 @@ fn records_of(groups: &[Group]) -> Vec { .fields .iter() .enumerate() - .map(|(index, text)| (format!("field {index}"), text.clone())) + .map(|(index, text)| (format!("field {index:03}"), text.clone())) .collect(), }) .collect() From f8685d460e1a0a81c2320966e4819b52553bc484 Mon Sep 17 00:00:00 2001 From: Shanu Date: Fri, 9 Oct 2026 12:40:56 +0530 Subject: [PATCH 14/15] Close a simulated place box with its list The simulated ride form hid a box behind its button again only once a suggestion was picked: pressed elsewhere or escaped, its list closed but its box stayed on show. drop_unpicked now closes the box with its list. The cookie bar's doc says it covers no control, as the promo toast does, since nothing behind it is refused. --- .../tinycomputer-engine/src/agentic/flow/flow_tests/places.rs | 3 ++- .../src/agentic/flow/flow_tests/simulator.rs | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/places.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/places.rs index 666e6b5a..41753e44 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/places.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/places.rs @@ -174,11 +174,12 @@ pub(super) fn await_place_rows(sim: &mut Sim) -> bool { } /// Closes the open list, dropping its box's text unless a suggestion was -/// picked for it. +/// picked for it; a box shown behind its button closes with it. pub(super) fn drop_unpicked(sim: &mut Sim) { let Some(places) = sim.places.as_mut() else { return; }; + places.door = None; if let Some(open) = places.open.take() && !places.picked.contains(&open) { diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs index 3a65cb98..f22ef657 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs @@ -36,7 +36,8 @@ pub(super) enum Quirk { /// A promo toast with a Close button sits over the page until closed. PromoToast, /// A cookie bar drawn without a dialog's role, its "Accept all" button - /// the way out, sits over the page until accepted. + /// the way out, shows until accepted; like the promo toast, it covers + /// no control. CookieBar, /// The inbox's search, once open, fills the window as a sheet. SearchSheet, From b301aaa23324906d72b270d1d3efd3d9144e4329 Mon Sep 17 00:00:00 2001 From: Shanu Date: Fri, 9 Oct 2026 12:40:56 +0530 Subject: [PATCH 15/15] Title a grid calendar only by text that shows A grid of days took its month from a sibling before it, read through shownWords, which falls back to innerText, and a hidden element's innerText still holds its text: a template or a month menu the page never shows could title a grid of numbered buttons. A sibling must now show. A live test checks that a grid titled only by hidden text is no calendar. --- .../src/surface/sight/sight.js | 3 ++ .../sight/sight_tests/live_calendar_tests.rs | 39 ++++++++++++++++++- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/crates/tinycomputer-browser/src/surface/sight/sight.js b/crates/tinycomputer-browser/src/surface/sight/sight.js index 49609b9e..e9162746 100644 --- a/crates/tinycomputer-browser/src/surface/sight/sight.js +++ b/crates/tinycomputer-browser/src/surface/sight/sight.js @@ -158,6 +158,9 @@ for (let depth = 0; node && node !== base && depth < 4; depth += 1, node = node.parentElement) { let sibling = node.previousElementSibling; for (let step = 0; sibling && step < 3; step += 1, sibling = sibling.previousElementSibling) { + // A hidden element's text still reads out (a template, a month + // menu): only what shows titles a grid. + if (!shown(sibling)) continue; const said = shownWords(sibling); if (said.length > 120) break; const months = [...said.matchAll(MONTHS_AND_YEARS)] diff --git a/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_calendar_tests.rs b/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_calendar_tests.rs index e626753c..45b18492 100644 --- a/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_calendar_tests.rs +++ b/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_calendar_tests.rs @@ -1,6 +1,7 @@ //! Live tests of calendars sight reads as dates, gated on -//! `TINYCOMPUTER_LIVE_BROWSER=1`: two months that share their arrows, and -//! months drawn as grids of buttons rather than tables. +//! `TINYCOMPUTER_LIVE_BROWSER=1`: two months that share their arrows, +//! months drawn as grids of buttons rather than tables, and a grid only a +//! hidden element titles, which is none. #[cfg(feature = "agent-browser")] use super::live_tests::{live_reading, shown_names}; @@ -136,3 +137,37 @@ async fn live_a_two_month_pickers_days_and_shared_arrows_are_read_as_dates() { "the arrows beside both months page them" ); } + +/// A grid of 31 numbered buttons whose only month and year sit in a hidden +/// element before it, such as a template the page never shows. +#[cfg(feature = "agent-browser")] +fn hidden_title_grid_page() -> String { + use std::fmt::Write as _; + let cells = (1..=31).fold(String::new(), |mut cells, day| { + let _ = write!(cells, ""); + cells + }); + format!( + "
October 2026
\ +
{cells}
" + ) +} + +#[cfg(feature = "agent-browser")] +#[tokio::test] +async fn live_a_grid_titled_only_by_hidden_text_is_no_calendar() { + let Some(reading) = live_reading(&hidden_title_grid_page()).await else { + return; + }; + let nodes = reading["nodes"].as_array().unwrap(); + assert!( + !nodes.iter().any(|node| node["role"] == "gridcell"), + "{:?}", + shown_names(&reading) + ); + let day = nodes + .iter() + .find(|node| node["name"] == "22") + .unwrap_or_else(|| panic!("22 not offered: {:?}", shown_names(&reading))); + assert_eq!(day["description"].as_str().unwrap_or_default(), ""); +}