diff --git a/crates/tinycomputer-browser/src/fake/mod.rs b/crates/tinycomputer-browser/src/fake/mod.rs index dd4f3389..6acbe541 100644 --- a/crates/tinycomputer-browser/src/fake/mod.rs +++ b/crates/tinycomputer-browser/src/fake/mod.rs @@ -66,15 +66,15 @@ impl Fake { self.sent.lock().unwrap().clone() } - /// Whether a page script ran besides the one that keeps a press in the - /// tab, which runs before every click. - pub(crate) fn evaluated_besides_keeping_the_tab(&self) -> bool { + /// Whether a page script ran besides those every press runs: the one + /// that keeps a press in the tab, and the one that brings a control + /// sight found into the window. + pub(crate) fn evaluated_besides_every_press(&self) -> bool { self.sent().iter().any(|command| { + let script = command["script"].as_str().unwrap_or_default(); command["action"] == "evaluate" - && !command["script"] - .as_str() - .unwrap_or_default() - .contains("__tcOpen") + && !script.contains("__tcOpen") + && !script.starts_with(crate::surface::INTO_VIEW_JS) }) } diff --git a/crates/tinycomputer-browser/src/surface/mod.rs b/crates/tinycomputer-browser/src/surface/mod.rs index 8839b603..215081a1 100644 --- a/crates/tinycomputer-browser/src/surface/mod.rs +++ b/crates/tinycomputer-browser/src/surface/mod.rs @@ -24,6 +24,9 @@ mod tree; mod uncover; mod watch; +#[cfg(test)] +pub(crate) use uncover::INTO_VIEW_JS; + pub use sight::Denoised; use std::sync::atomic::{AtomicBool, Ordering}; diff --git a/crates/tinycomputer-browser/src/surface/sight/sight.js b/crates/tinycomputer-browser/src/surface/sight/sight.js index 97776376..e9162746 100644 --- a/crates/tinycomputer-browser/src/surface/sight/sight.js +++ b/crates/tinycomputer-browser/src/surface/sight/sight.js @@ -58,11 +58,17 @@ // step choosing it never saw it chosen. Never `unselected`. const SELECTED_CLASS = /(?:^|[-_])(?:selected|checked)$/i; const UNSELECTED_CLASS = /(?:^|[-_])(?:un|not[-_]?)(?:selected|checked)$/i; - const classChosen = (element) => [...element.classList] + // A class behind a variant (`placeholder:text-disabled`, which colours + // only the placeholder; `disabled:opacity-50`, which applies only once + // disabled) says nothing of the element's own state: live, a flight + // site's place box carried `placeholder:text-disabled`, was read as + // disabled, and its box was never seen. + const stateClasses = (element) => [...element.classList].filter((name) => !name.includes(':')); + const classChosen = (element) => stateClasses(element) .some((name) => SELECTED_CLASS.test(name) && !UNSELECTED_CLASS.test(name)); const disabled = (element) => element.disabled === true || element.getAttribute('aria-disabled') === 'true' - || [...element.classList].some((name) => DISABLED_CLASS.test(name)); + || stateClasses(element).some((name) => DISABLED_CLASS.test(name)); const insideText = (element) => { for (let parent = element; parent; parent = parent.parentElement) { @@ -101,22 +107,33 @@ const pointer = (element) => style(element).cursor === 'pointer'; // A click handler a script framework keeps on the element itself (React - // stores each element's props on it), on a box smaller than a quarter of - // the window: a page can wire a plain `div` to a click with neither a - // cursor nor a tab stop. Live, a store's "Add to cart" and "Buy now" read - // as plain words, and the step to add to the cart had nothing to press. - // Only a press handler: a carousel's track or a select's menu listens for - // the mouse going down and is no button to press. + // stores each element's props on it, Preact its listeners), on a box + // smaller than a quarter of the window: a page can wire a plain `div` to a + // click with neither a cursor nor a tab stop. Live, a store's "Add to + // cart" and "Buy now" read as plain words, and the step to add to the cart + // had nothing to press. Only a press handler: a carousel's track or a + // select's menu listens for the mouse going down and is no button to press. const HANDLERS = ['onClick', 'onPress']; + // Preact keeps them by event name (`_listeners`, `l` once minified), a + // capture flag after it in newer releases ("clickfalse"). Live, a hotel + // site's place suggestions were rows with only such a listener, read as + // page text, and the place typed was never chosen. + const PREACT_PRESS = /^click(true|false)?$/; + const pressHandler = (element) => { + const key = Object.keys(element).find((name) => name.startsWith('__reactProps$')); + const props = key && element[key]; + if (props && HANDLERS.some((handler) => typeof props[handler] === 'function')) return true; + const own = ['_listeners', 'l'].find((name) => Object.prototype.hasOwnProperty.call(element, name)); + const listeners = own && element[own]; + return !!listeners && typeof listeners === 'object' && Object.keys(listeners) + .some((name) => PREACT_PRESS.test(name) && typeof listeners[name] === 'function'); + }; // What is a control only by such a handler: it hides nothing pressable // inside it, since a page can wire a whole card to a click around its own // "Add" button. const scriptedOnly = new Set(); const scripted = (element) => { - if (!element || element === document.body) return false; - const key = Object.keys(element).find((name) => name.startsWith('__reactProps$')); - const props = key && element[key]; - if (!props || !HANDLERS.some((handler) => typeof props[handler] === 'function')) return false; + if (!element || element === document.body || !pressHandler(element)) return false; const rect = element.getBoundingClientRect(); return rect.width * rect.height < window.innerWidth * window.innerHeight * 0.25; }; @@ -131,6 +148,28 @@ const MONTH_AND_YEAR = new RegExp(`\\b(${MONTHS.join('|')})\\s+(\\d{4})\\b`, 'i'); const calendarDays = new Map(); const calendars = []; + // The month and year a grid of days shows: the nearest short text before + // it, or before one of its four nearest ancestors, that names one, as + // `{ element, months: [[month, year], …] }`. A longer block (another + // month's whole grid) ends the search at its level. + const MONTHS_AND_YEARS = new RegExp(MONTH_AND_YEAR.source, 'gi'); + const gridTitle = (grid) => { + let node = grid; + for (let depth = 0; node && node !== base && depth < 4; depth += 1, node = node.parentElement) { + let sibling = node.previousElementSibling; + for (let step = 0; sibling && step < 3; step += 1, sibling = sibling.previousElementSibling) { + // A hidden element's text still reads out (a template, a month + // menu): only what shows titles a grid. + if (!shown(sibling)) continue; + const said = shownWords(sibling); + if (said.length > 120) break; + const months = [...said.matchAll(MONTHS_AND_YEARS)] + .map((found) => [MONTHS.indexOf(found[1].toLowerCase()), Number(found[2])]); + if (months.length) return { element: sibling, months }; + } + } + return null; + }; const findCalendars = () => { for (const table of base.querySelectorAll('table')) { // A week-number column is numbers too, but no day. @@ -139,12 +178,14 @@ if (cells.length < 28 || !shown(table)) continue; // The month is named in the table's own heading, or in a short header // drawn just before it; never by words elsewhere on the page, nor by a - // calendar that happens to come before it. + // calendar that happens to come before it. A header is read as shown: + // live, one held a hidden month list for the picker's own menu, whose + // twelve names hid the month it showed. const before = table.previousElementSibling; - const header = before && !before.querySelector('table') && squash(before.innerText).length <= 80 + const header = before && !before.querySelector('table') && shownWords(before).length <= 80 ? before : null; const titled = MONTH_AND_YEAR.exec(squash([table.caption, table.tHead, header] - .filter(Boolean).map((part) => part.innerText).join(' '))); + .filter(Boolean).map((part) => shownWords(part)).join(' '))); if (!titled) continue; const holder = table.parentElement; calendars.push(holder && holder !== document.body && holder !== document.documentElement ? holder : table); @@ -163,6 +204,60 @@ calendarDays.set(cell, `${day} ${spelled[0].toUpperCase()}${spelled.slice(1)} ${date.getUTCFullYear()}`); } } + // A calendar drawn without a table: a grid whose cells each begin with + // their day number (a fare may follow, "22 6529"), numbered from 1 to the + // month's last day, below the month and year it shows. A title naming + // two months, one header over two grids, names them in order. Live, a + // flight site drew its days as buttons in such a grid, no day read as a + // date, and the departure was never picked. + const titleUses = new Map(); + const grids = []; + for (const grid of base.querySelectorAll('div, ul, ol, tbody')) { + const kids = grid.children; + if (kids.length < 28 || kids.length > 49 + || [...calendars, ...grids].some((calendar) => calendar.contains(grid))) continue; + const days = [...kids].map((kid) => { + const leading = /^(\d{1,2})(?:\s|$)/.exec(squash(kid.innerText)); + return leading && shown(kid) ? { cell: kid, day: Number(leading[1]) } : null; + }); + const start = days.findIndex((entry) => entry && entry.day === 1); + if (start < 0) continue; + const run = []; + for (const entry of days.slice(start)) { + if (!entry || entry.day !== run.length + 1) break; + run.push(entry); + } + const title = run.length >= 28 && gridTitle(grid); + if (!title) continue; + const used = titleUses.get(title.element) || 0; + titleUses.set(title.element, used + 1); + const [month, year] = title.months[Math.min(used, title.months.length - 1)]; + const spelled = MONTHS[month]; + grids.push(grid); + for (const { cell, day } of run) { + const pressed = cell.matches(NESTED) ? cell : cell.querySelector(NESTED) || cell; + calendarDays.set(pressed, `${day} ${spelled[0].toUpperCase()}${spelled.slice(1)} ${year}`); + } + } + // Each month is the block holding its grid, with its heading and + // arrows, once every grid is found: a block holding two months' grids + // side by side, with no box of each month's own, is no one month, and + // is added as the picker below. + for (const grid of grids) { + const holder = grid.parentElement; + const shared = holder && grids.some((other) => other !== grid && holder.contains(other)); + calendars.push(holder && holder !== document.body && !shared ? holder : grid); + } + // A picker showing two months beside each other pages both with one + // pair of arrows, drawn beside the months rather than inside either: + // the block that holds more than one month pages them too. + for (const holder of [...calendars]) { + const picker = holder.parentElement; + if (picker && picker !== document.body && !calendars.includes(picker) + && calendars.filter((other) => other !== picker && picker.contains(other)).length > 1) { + calendars.push(picker); + } + } }; // A calendar's paging arrow, read as what it does: an arrow glyph, or a // bare "Next", inside a calendar turns its month. @@ -212,7 +307,11 @@ // is still a button. if ((GROUP_ROLES.includes(claimed) || claimed === 'dialog' || claimed === 'alertdialog') && !element.hasAttribute('onclick') && !pointer(element)) return null; - if (insideControl) return null; + // Inside another control, what a press handler of its own wires is a + // control too, as a native button inside a pressable card is: live, a + // sort menu's options sat inside its pointer-cursor trigger, each wired + // to a click by itself, and the menu read as one button naming them all. + if (insideControl && !pressHandler(element)) return null; const tabindex = element.getAttribute('tabindex'); const byPage = element.hasAttribute('onclick') || (tabindex !== null && tabindex !== '-1') @@ -241,6 +340,10 @@ ]; // A picture-only control's meaning, from the words in its own or its // icon's class, id, or test id: all a person would see is the picture. + // Words run together in camel case are words too ("icClose", + // "closeIcon"): live, a sign-up pop-up's only way out was a sprite with + // the class "icClose", dropped as a blank box, and nothing closed it. A + // generated class ("fCarEc", "cUpXyz") has no two such words in a row. const iconWords = (element) => { const sources = [element, ...element.querySelectorAll('svg, use, i, img, span')].slice(0, 8); const words = new Set(); @@ -257,7 +360,7 @@ source.getAttribute('data-icon') || '', tag(source) === 'use' ? used : '', file, - ].join(' ').toLowerCase(); + ].join(' ').replace(/([a-z]{2,})(?=[A-Z][a-z]{2,})/g, '$1 ').toLowerCase(); for (const word of text.split(/[^a-z]+/)) { if (ICON_WORDS.includes(word)) words.add(word); } @@ -807,6 +910,38 @@ return rect.bottom <= 0 || rect.top >= height || rect.right <= 0 || rect.left >= width; }; + // The nearest container that scrolls its content (a popover's list), or + // null. Each container is looked at once. An element fixed to the window + // moves with no container, and none of them clips it: a fixed pop-up + // drawn from inside a scrolling list shows wherever it is placed. + const scrollers = new Map(); + const scrollerOf = (element) => { + if (style(element).position === 'fixed') return null; + const parent = element.parentElement; + if (!parent || parent === document.body || parent === document.documentElement) return null; + if (scrollers.has(parent)) return scrollers.get(parent); + const overflow = style(parent); + const scrolls = (/(auto|scroll)/.test(overflow.overflowY) && parent.scrollHeight > parent.clientHeight) + || (/(auto|scroll)/.test(overflow.overflowX) && parent.scrollWidth > parent.clientWidth); + const found = scrolls ? parent : scrollerOf(parent); + scrollers.set(parent, found); + return found; + }; + // Whether the element's middle is scrolled out of its container's view: + // what shows at that point is the container's neighbour or the page, + // neither of which covers the element, and a press scrolls it back. Live, + // a popover's airport rows below its list's fold read as covered, ranked + // last, and were never offered. + const scrolledAway = (element) => { + const scroller = scrollerOf(element); + if (!scroller) return false; + const rect = box(element); + const view = scroller.getBoundingClientRect(); + const x = (rect.left + rect.right) / 2; + const y = (rect.top + rect.bottom) / 2; + return x < view.left || x > view.right || y < view.top || y > view.bottom; + }; + const statesOf = (element, what) => { const states = []; const input = standIn(element) || element; @@ -818,7 +953,7 @@ if (aria('selected') === 'true' || (aria('current') && aria('current') !== 'false') || (!states.includes('checked') && classChosen(element))) states.push('selected'); if (input.required === true || aria('required') === 'true') states.push('required'); - if (offscreen(element)) states.push('offscreen'); + if (offscreen(element) || scrolledAway(element)) states.push('offscreen'); else if (covered(element)) states.push('covered'); return states; }; @@ -1125,6 +1260,26 @@ seen.push({ element, record }); offerChoices(element, record); } + // A control repeated on every card ("ADD" on each product), kept inside + // the card that is a control itself, says nothing of which card it acts + // on: it is described by that card's name. Live, "add the first Maggi" + // pressed the first card's "ADD", on a ramen above the Maggi. + const recordOf = new Map(seen.map(({ element, record }) => [element, record])); + const copies = new Map(); + for (const { record } of seen) { + const key = `${record.role}\u0000${record.name}`; + copies.set(key, (copies.get(key) || 0) + 1); + } + for (const { element, record } of seen) { + if (record.description || !record.name + || copies.get(`${record.role}\u0000${record.name}`) < 2) continue; + for (let parent = element.parentElement; parent; parent = parent.parentElement) { + const card = recordOf.get(parent); + if (!card) continue; + if (card.name && card.name !== record.name) record.description = clip(`in ${card.name}`, limits.name); + break; + } + } window.__tinycomputerSeen = next; const middle = document.elementFromPoint(width / 2, height / 2); diff --git a/crates/tinycomputer-browser/src/surface/sight/sight_tests.rs b/crates/tinycomputer-browser/src/surface/sight/sight_tests.rs index 98bac9d8..209e5467 100644 --- a/crates/tinycomputer-browser/src/surface/sight/sight_tests.rs +++ b/crates/tinycomputer-browser/src/surface/sight/sight_tests.rs @@ -3,5 +3,7 @@ #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod live_calendar_tests; +mod live_controls_tests; mod live_tests; mod reading_tests; diff --git a/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_calendar_tests.rs b/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_calendar_tests.rs new file mode 100644 index 00000000..45b18492 --- /dev/null +++ b/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_calendar_tests.rs @@ -0,0 +1,173 @@ +//! Live tests of calendars sight reads as dates, gated on +//! `TINYCOMPUTER_LIVE_BROWSER=1`: two months that share their arrows, +//! months drawn as grids of buttons rather than tables, and a grid only a +//! hidden element titles, which is none. + +#[cfg(feature = "agent-browser")] +use super::live_tests::{live_reading, shown_names}; + +/// Two months drawn as grids of buttons, not tables, under one header that +/// names both: each day shows its fare after its number ("22 6529"). Each +/// grid sits in a box of its month's own when `boxed`, or both side by +/// side in one. +#[cfg(feature = "agent-browser")] +fn grid_calendar_page(boxed: bool) -> String { + let grid = |blanks: u32, days: u32, fares: u32| { + let cells = (0..blanks) + .map(|_| "".to_owned()) + .chain((1..=days).map(|day| format!("", fares + day * 7))) + .collect::(); + let grid = format!( + "
{cells}
" + ); + if boxed { + format!("
Su Mo Tu We Th Fr Sa
{grid}
") + } else { + grid + } + }; + format!( + "
October 2026 November 2026
\ +
{}{}
", + grid(4, 31, 6_000), + grid(0, 30, 7_000) + ) +} + +#[cfg(feature = "agent-browser")] +#[tokio::test] +async fn live_a_calendar_drawn_as_grids_of_buttons_reads_its_days_as_dates() { + // Two grids side by side in one block are two months too, not one + // calendar that hides the second. + for boxed in [true, false] { + let Some(reading) = live_reading(&grid_calendar_page(boxed)).await else { + return; + }; + let described = |name: &str| { + reading["nodes"] + .as_array() + .unwrap() + .iter() + .find(|node| node["name"] == name) + .map_or_else( + || panic!("{name} not offered: {:?}", shown_names(&reading)), + |node| node["description"].as_str().unwrap_or_default().to_owned(), + ) + }; + assert_eq!(described("22 6154"), "22 October 2026", "boxed: {boxed}"); + assert_eq!(described("5 7035"), "5 November 2026", "boxed: {boxed}"); + } +} + +/// A picker showing two months, drawn as a hotel site's was: each month a +/// table under a header that holds a hidden month menu, and one pair of icon +/// arrows beside both months, inside neither. +#[cfg(feature = "agent-browser")] +const TWO_MONTH_PICKER: &str = r#" +
+
+
+
October 2026
+ +
MonTueWedThuFriSatSun
+
+
+
November 2026
+ +
MonTueWedThuFriSatSun
+
+
+
+"#; + +#[cfg(feature = "agent-browser")] +#[tokio::test] +async fn live_a_two_month_pickers_days_and_shared_arrows_are_read_as_dates() { + let Some(reading) = live_reading(TWO_MONTH_PICKER).await else { + return; + }; + let nodes = reading["nodes"].as_array().unwrap(); + let days = nodes + .iter() + .filter(|node| node["role"] == "gridcell") + .map(|node| { + format!( + "{} ({})", + node["name"].as_str().unwrap(), + node["description"].as_str().unwrap() + ) + }) + .collect::>(); + // Each month by its header as shown, not by its hidden menu's names. + assert_eq!(days.len(), 61, "{days:?}"); + for day in [ + "22 (22 October 2026)", + "31 (31 October 2026)", + "22 (22 November 2026)", + ] { + assert!(days.iter().any(|seen| seen == day), "{day} in {days:?}"); + } + let names = shown_names(&reading); + assert_eq!( + names + .iter() + .filter(|name| name.ends_with(" month")) + .collect::>(), + ["previous month", "next month"], + "the arrows beside both months page them" + ); +} + +/// A grid of 31 numbered buttons whose only month and year sit in a hidden +/// element before it, such as a template the page never shows. +#[cfg(feature = "agent-browser")] +fn hidden_title_grid_page() -> String { + use std::fmt::Write as _; + let cells = (1..=31).fold(String::new(), |mut cells, day| { + let _ = write!(cells, ""); + cells + }); + format!( + "
October 2026
\ +
{cells}
" + ) +} + +#[cfg(feature = "agent-browser")] +#[tokio::test] +async fn live_a_grid_titled_only_by_hidden_text_is_no_calendar() { + let Some(reading) = live_reading(&hidden_title_grid_page()).await else { + return; + }; + let nodes = reading["nodes"].as_array().unwrap(); + assert!( + !nodes.iter().any(|node| node["role"] == "gridcell"), + "{:?}", + shown_names(&reading) + ); + let day = nodes + .iter() + .find(|node| node["name"] == "22") + .unwrap_or_else(|| panic!("22 not offered: {:?}", shown_names(&reading))); + assert_eq!(day["description"].as_str().unwrap_or_default(), ""); +} diff --git a/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_controls_tests.rs b/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_controls_tests.rs new file mode 100644 index 00000000..0b256817 --- /dev/null +++ b/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_controls_tests.rs @@ -0,0 +1,264 @@ +//! Live tests of what sight reads as a control, gated on +//! `TINYCOMPUTER_LIVE_BROWSER=1`: rows a script framework wires to a click, +//! options inside their trigger, classes behind a variant, icons named in +//! camel case, a control brought into the window before its press, a +//! control fixed to the window inside a scrolling list, and a control each +//! card repeats, told apart by its card. + +#[cfg(feature = "agent-browser")] +use serde_json::json; + +#[cfg(feature = "agent-browser")] +use super::live_tests::{live_page, live_reading, shown_names}; + +/// Place suggestions drawn as plain rows that only a Preact click listener +/// makes pressable (`l` once minified, `_listeners` with its capture flag), +/// and one whose listener waits for the mouse going down. +#[cfg(feature = "agent-browser")] +const PREACT_ROWS_PAGE: &str = r#"
+ +
Goa, India
+
Baga Beach North Goa, Goa
+
Panjim, Panaji, Goa
+
+"#; + +#[cfg(feature = "agent-browser")] +#[tokio::test] +async fn live_rows_a_preact_click_listener_wires_are_buttons() { + let Some(reading) = live_reading(PREACT_ROWS_PAGE).await else { + return; + }; + let role = |name: &str| { + reading["nodes"] + .as_array() + .unwrap() + .iter() + .find(|node| node["name"] == name) + .and_then(|node| node["role"].as_str()) + .map(str::to_owned) + }; + assert_eq!(role("Goa, India").as_deref(), Some("button")); + assert_eq!(role("Baga Beach North Goa, Goa").as_deref(), Some("button")); + assert_eq!( + role("Panjim, Panaji, Goa"), + None, + "a mouse-down listener is no press" + ); +} + +/// A sort menu whose list sits inside its pointer-cursor trigger, each +/// option wired to a click by its own Preact listener. +#[cfg(feature = "agent-browser")] +const MENU_IN_TRIGGER_PAGE: &str = r#"
+ + Popularity +
    +
  • Popularity
  • +
  • Guest Ratings
  • +
  • Price Low to High
  • +
+
+
+"#; + +#[cfg(feature = "agent-browser")] +#[tokio::test] +async fn live_options_wired_by_their_own_handler_inside_a_trigger_are_buttons() { + let Some(reading) = live_reading(MENU_IN_TRIGGER_PAGE).await else { + return; + }; + for option in ["Guest Ratings", "Price Low to High"] { + let found = reading["nodes"] + .as_array() + .unwrap() + .iter() + .find(|node| node["name"] == option) + .unwrap_or_else(|| panic!("{option} not offered: {:?}", shown_names(&reading))); + assert_eq!(found["role"], "button", "{option}"); + } +} + +/// A place box styled by a variant class that ends in "disabled", beside +/// a day a calendar greys out by class, and a button whose variant class +/// styles it only once disabled. +#[cfg(feature = "agent-browser")] +const VARIANT_CLASSES_PAGE: &str = r#" + +21"#; + +#[cfg(feature = "agent-browser")] +#[tokio::test] +async fn live_a_variant_class_says_nothing_of_a_controls_own_state() { + let Some(reading) = live_reading(VARIANT_CLASSES_PAGE).await else { + return; + }; + let role = |name: &str| { + reading["nodes"] + .as_array() + .unwrap() + .iter() + .find(|node| node["name"] == name) + .and_then(|node| node["role"].as_str()) + .map(str::to_owned) + }; + assert_eq!(role("From").as_deref(), Some("textbox")); + assert_eq!(role("Search").as_deref(), Some("button")); + assert_eq!(role("21"), None, "a day greyed out by class is no control"); +} + +/// A button at the window's foot whose middle lies below the window, and +/// one in full view; each records its press. +#[cfg(feature = "agent-browser")] +const FOOT_BUTTON_PAGE: &str = r#" + +
+"#; + +#[cfg(feature = "agent-browser")] +#[tokio::test] +async fn live_a_control_whose_middle_is_below_the_window_is_brought_into_it_and_pressed() { + let Some((browser, info)) = live_page(FOOT_BUTTON_PAGE).await else { + return; + }; + let evaluate = + |script: String| browser.command(&info.id, json!({"action": "evaluate", "script": script})); + let into_view = |selector: &str| { + format!( + "{}(document.querySelector({}))", + crate::surface::INTO_VIEW_JS, + serde_json::Value::String(selector.to_owned()) + ) + }; + let shown = evaluate(into_view("#shown")).await.unwrap(); + assert_eq!( + shown["result"], false, + "a control in view stays where it is" + ); + let moved = evaluate(into_view("#foot")).await.unwrap(); + assert_eq!(moved["result"], true); + let middle = evaluate( + "(() => { const box = document.getElementById('foot').getBoundingClientRect(); \ + const y = box.top + box.height / 2; return y >= 0 && y < window.innerHeight; })()" + .to_owned(), + ) + .await + .unwrap(); + assert_eq!(middle["result"], true, "its middle now shows"); + browser + .command(&info.id, json!({"action": "click", "selector": "#foot"})) + .await + .unwrap(); + let pressed = evaluate("window.pressed".to_owned()).await.unwrap(); + browser.close_session(&info.id).await.unwrap(); + assert_eq!(pressed["result"], json!(["foot"])); +} + +/// A sign-up pop-up whose only way out is a sprite its class names in camel +/// case ("icClose"): no words, no picture, a pointer cursor. +#[cfg(feature = "agent-browser")] +const SPRITE_CLOSE_PAGE: &str = r#"
+ +

Login/Signup

+ +
"#; + +#[cfg(feature = "agent-browser")] +#[tokio::test] +async fn live_a_sprite_its_class_names_in_camel_case_is_read_as_that_icon() { + let Some(reading) = live_reading(SPRITE_CLOSE_PAGE).await else { + return; + }; + let close = reading["nodes"] + .as_array() + .unwrap() + .iter() + .find(|node| node["name"] == "close") + .unwrap_or_else(|| panic!("no close control: {:?}", shown_names(&reading))); + assert_eq!(close["role"], "button"); + assert_eq!(close["description"], "an icon"); +} + +/// A scrolling list of airports whose last row holds a "Done" button fixed +/// to the window below the list, outside the list's view. +#[cfg(feature = "agent-browser")] +const FIXED_IN_LIST_PAGE: &str = r#"
    +
  • DEL Delhi
  • BLR Bengaluru
  • MAA Chennai
  • +
  • HYD Hyderabad
  • CCU Kolkata
  • BOM Mumbai
  • +
  • GOI Goa
  • PNQ Pune
  • +
  • +
+"#; + +#[cfg(feature = "agent-browser")] +#[tokio::test] +async fn live_a_control_fixed_to_the_window_inside_a_scrolling_list_is_not_scrolled_away() { + let Some(reading) = live_reading(FIXED_IN_LIST_PAGE).await else { + return; + }; + let states = |name: &str| { + reading["nodes"] + .as_array() + .unwrap() + .iter() + .find(|node| node["name"] == name) + .map_or_else( + || panic!("{name} not offered: {:?}", shown_names(&reading)), + |node| node["states"].to_string(), + ) + }; + assert!(!states("Done").contains("offscreen"), "{}", states("Done")); + assert!( + states("PNQ Pune").contains("offscreen"), + "a row below the list's fold still is: {}", + states("PNQ Pune") + ); +} + +/// Product cards a press opens, each with its own "ADD" button, and one card +/// with a button no other card has. +#[cfg(feature = "agent-browser")] +const CARD_BUTTONS_PAGE: &str = r#"
+
Too Yumm Korean Ramen 79 g ₹49
+
Maggi Double Masala 95 g ₹20
+
Maggi Masala 280 g ₹56
+
"#; + +#[cfg(feature = "agent-browser")] +#[tokio::test] +async fn live_a_control_each_card_repeats_is_described_by_its_card() { + let Some(reading) = live_reading(CARD_BUTTONS_PAGE).await else { + return; + }; + let nodes = reading["nodes"].as_array().unwrap(); + let described = |name: &str| { + nodes + .iter() + .filter(|node| node["name"] == name) + .map(|node| node["description"].as_str().unwrap_or_default().to_owned()) + .collect::>() + }; + let adds = described("ADD"); + assert_eq!(adds.len(), 2, "{:?}", shown_names(&reading)); + assert!(adds[0].starts_with("in Too Yumm Korean Ramen"), "{adds:?}"); + assert!(adds[1].starts_with("in Maggi Double Masala"), "{adds:?}"); + assert_eq!( + described("Notify me"), + [""], + "a control no other card repeats needs no card to tell it apart" + ); +} diff --git a/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_tests.rs b/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_tests.rs index 0d89257b..09481dd3 100644 --- a/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_tests.rs +++ b/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_tests.rs @@ -12,7 +12,7 @@ use crate::surface::sight::script; /// page is written into a blank tab, so nothing is fetched but what the /// fixture itself asks for. #[cfg(feature = "agent-browser")] -async fn live_reading(html: &str) -> Option { +pub(super) async fn live_reading(html: &str) -> Option { live_results(html, &[script(None)]) .await .map(|mut results| results.remove(0)) @@ -21,7 +21,7 @@ async fn live_reading(html: &str) -> Option { /// A blank tab of a real browser with `html` written into it: `None` /// unless `TINYCOMPUTER_LIVE_BROWSER=1`, since CI has no browser to launch. #[cfg(feature = "agent-browser")] -async fn live_page( +pub(super) async fn live_page( html: &str, ) -> Option<( crate::sessions::Browser, @@ -79,7 +79,7 @@ async fn live_results(html: &str, scripts: &[String]) -> Option Vec { +pub(super) fn shown_names(reading: &serde_json::Value) -> Vec { reading["nodes"] .as_array() .unwrap() @@ -518,6 +518,54 @@ const CHOICES_PAGE: &str = r#"
document.addEventListener('change', (event) => window.changes.push(event.target.id)); "#; +/// A popover's list scrolled so its last rows are out of its view, beside +/// a button a modal covers. +#[cfg(feature = "agent-browser")] +const SCROLLED_LIST_PAGE: &str = r#"
+
    +
  • DEL Delhi
  • BLR Bengaluru
  • MAA Chennai
  • +
  • HYD Hyderabad
  • CCU Kolkata
  • BOM Mumbai
  • +
  • GOI Goa
  • PNQ Pune
  • +
+
Planning a holiday?
+
+ +
+ Sign up for offers
+"#; + +#[cfg(feature = "agent-browser")] +#[tokio::test] +async fn live_a_row_scrolled_out_of_its_list_is_offscreen_not_covered() { + let Some(reading) = live_reading(SCROLLED_LIST_PAGE).await else { + return; + }; + let states = |name: &str| { + reading["nodes"] + .as_array() + .unwrap() + .iter() + .find(|node| node["name"] == name) + .map_or_else( + || panic!("{name} not offered"), + |node| node["states"].to_string(), + ) + }; + assert!( + !states("DEL Delhi").contains("offscreen"), + "{}", + states("DEL Delhi") + ); + for row in ["BOM Mumbai", "PNQ Pune"] { + let read = states(row); + assert!( + read.contains("offscreen") && !read.contains("covered"), + "{row}: {read}" + ); + } + assert!(states("Search").contains("covered"), "a modal still covers"); +} + /// The options a reading offers, each as " in ". #[cfg(feature = "agent-browser")] fn offered(reading: &serde_json::Value) -> Vec { diff --git a/crates/tinycomputer-browser/src/surface/surface_tests/card_tests.rs b/crates/tinycomputer-browser/src/surface/surface_tests/card_tests.rs index 3d40f350..fff5c740 100644 --- a/crates/tinycomputer-browser/src/surface/surface_tests/card_tests.rs +++ b/crates/tinycomputer-browser/src/surface/surface_tests/card_tests.rs @@ -1,5 +1,6 @@ -//! Tests for clicking through a result card's own cover, and pressing a -//! selection again through the DOM. +//! Tests for bringing a control into the window before it is pressed, +//! clicking through a result card's own cover, and pressing a selection +//! again through the DOM. use serde_json::json; use tinycomputer_bus::JevOperation; @@ -72,7 +73,7 @@ fn a_click_covered_by_anything_else_stays_refused() { .execute(JevOperation::Click, Some(node("e5", &["Click"])), None) .ok ); - assert!(!fake.evaluated_besides_keeping_the_tab()); + assert!(!fake.evaluated_besides_every_press()); } /// A page that takes every click, and says through `evaluate` whether the @@ -113,9 +114,34 @@ fn a_tab_click_the_page_ignored_is_pressed_again_through_the_dom() { ..node(reference, &["Click"]) }; assert!(surface.execute(JevOperation::Click, Some(node), None).ok); - assert!( - !fake.evaluated_besides_keeping_the_tab(), - "{reference} {role}" - ); + assert!(!fake.evaluated_besides_every_press(), "{reference} {role}"); } } + +#[test] +fn a_control_sight_found_is_brought_into_the_window_before_it_is_pressed() { + // Live, a store's "Add to cart" sat at the window's foot with its middle + // below it, and every press went nowhere while reporting success. + let into_view = |command: &serde_json::Value| { + command["action"] == "evaluate" + && command["script"] + .as_str() + .unwrap_or_default() + .starts_with(crate::surface::INTO_VIEW_JS) + }; + let Harness { fake, surface, .. } = harness("into-view", Fake::new()); + let reply = surface.execute(JevOperation::Click, Some(node("seen:21", &["Click"])), None); + assert!(reply.ok, "{:?}", reply.error); + let sent = fake.sent(); + let brought = sent.iter().position(into_view).expect("brought into view"); + let pressed = sent + .iter() + .position(|command| command["action"] == "click") + .expect("pressed"); + assert!(brought < pressed, "{:?}", fake.actions()); + + // A ref of the tree is brought into view by the browser itself. + let Harness { fake, surface, .. } = harness("tree-into-view", Fake::new()); + surface.execute(JevOperation::Click, Some(node("e5", &["Click"])), None); + assert!(!fake.sent().iter().any(into_view)); +} diff --git a/crates/tinycomputer-browser/src/surface/surface_tests/native_select_tests.rs b/crates/tinycomputer-browser/src/surface/surface_tests/native_select_tests.rs index 2585d199..a08e9fc3 100644 --- a/crates/tinycomputer-browser/src/surface/surface_tests/native_select_tests.rs +++ b/crates/tinycomputer-browser/src/surface/surface_tests/native_select_tests.rs @@ -72,5 +72,5 @@ fn an_option_that_is_not_native_is_pressed_as_a_control() { .ok ); assert!(fake.actions().iter().any(|action| action == "click")); - assert!(!fake.evaluated_besides_keeping_the_tab()); + assert!(!fake.evaluated_besides_every_press()); } diff --git a/crates/tinycomputer-browser/src/surface/uncover.rs b/crates/tinycomputer-browser/src/surface/uncover.rs index 1afe34de..3315882b 100644 --- a/crates/tinycomputer-browser/src/surface/uncover.rs +++ b/crates/tinycomputer-browser/src/surface/uncover.rs @@ -35,6 +35,22 @@ const CENTRE_JS: &str = r"(element => { return true; })"; +/// Brings an element whose middle lies outside the window to the window's +/// middle, and leaves one whose middle shows where it is; `true` when it +/// moved the element. A press lands on the element's middle, and the browser +/// presses there whether the window shows that point or not: live, a store's +/// "Add to cart" sat at the window's foot with its middle below it, and every +/// press went nowhere while reporting success. +pub(crate) const INTO_VIEW_JS: &str = r"(element => { + if (!element) return false; + const box = element.getBoundingClientRect(); + const x = box.left + box.width / 2; + const y = box.top + box.height / 2; + if (x >= 0 && y >= 0 && x < window.innerWidth && y < window.innerHeight) return false; + element.scrollIntoView({ block: 'center', inline: 'center', behavior: 'instant' }); + return true; +})"; + /// How long a link's press is given to start leaving the page. const LEAVE_MS: u64 = 400; @@ -91,6 +107,9 @@ impl BrowserSurface { self.allow_location(); } let seen = sight::is_seen(reference); + if seen { + self.bring_into_view(reference); + } // Where a link's press starts from, to tell whether it went. let before = (follows && seen && node.is_some_and(|node| node.role == "link")) .then(|| self.page_url()) @@ -130,6 +149,23 @@ impl BrowserSurface { reply } + /// Brings the element `reference` names into the window when its middle + /// lies outside it ([`INTO_VIEW_JS`]), before it is pressed. A ref of the + /// tree is brought into view by the browser itself. + fn bring_into_view(&self, reference: &str) { + let (Ok(id), Ok(selector)) = ( + self.ensure_session(), + serde_json::to_string(&sight::selector(reference)), + ) else { + return; + }; + let script = format!("{INTO_VIEW_JS}(document.querySelector({selector}))"); + let _moved = self.block( + self.browser + .command(&id, json!({"action": "evaluate", "script": script})), + ); + } + /// Clicks `reference` once more after the browser refused because /// something covered its middle, with the element brought to the middle /// of the window and the pointer moved off to the window's corner first. diff --git a/crates/tinycomputer-bus/src/flow/guide.md b/crates/tinycomputer-bus/src/flow/guide.md index 6b3791e2..b031d347 100644 --- a/crates/tinycomputer-bus/src/flow/guide.md +++ b/crates/tinycomputer-bus/src/flow/guide.md @@ -41,7 +41,7 @@ do. | `choose` | `{"choose": {"what": "the font list", "option": "Helvetica"}}` | Pick an option in a list, menu, or popup, or in a group of option buttons (a size, a colour, a quantity, a day in a strip of dates). | | `read` | `{"read": {"what": "the newest message's subject", "into": "subject"}}` | Store visible text in a variable. Read the price of one item before any step raises its count: after that, its line and the cart show the total for all of them. | | `extract` | `{"extract": {"what": "the flight results", "into": "flights"}}` | Store every item of a list, as JSON rows of their text, in a variable. | -| `pick` | `{"pick": {"from": "the flight results", "by": "lowest price", "into": "flight"}}` | Choose the best of a list of results (cards or rows, each an item to open) and open it; `into` stores its text. Prices, times, durations, and stops are compared exactly. | +| `pick` | `{"pick": {"from": "the flight results", "by": "lowest price", "into": "flight"}}` | Choose the best of a list of results (cards or rows, each an item to open) and open it; `into` stores its text. Prices, times, durations, stops, and nearness to a number ("closest to 9", for a size when 9 may be sold out) are compared exactly. | | `verify` | `{"verify": "the draft shows a recipient"}` | Fail the flow unless this holds. | | `wait_for` | `{"wait_for": "the search results are showing"}` | Wait until this holds. | | `stop_before` | `{"stop_before": "sending the email"}` | Find an irreversible action and stop in front of it. | @@ -129,6 +129,9 @@ do. on which result to take ("skip Sponsored items", "rated 4 stars or more") belongs in that `pick`'s `from` or `by`, never in a step of its own: there is nothing on screen to do for it, so such a step fails. + For the same reason, a pop-up that may or may not show (a login, + cookie, or offer pop-up) is closed in an `if` on it being visible, + never in a plain step. Keep the task's own words in a `pick`: its `from` names the item the task asks for ("the results for ", not just "the search results"), and its `by` is the task's criterion, "first" when the task diff --git a/crates/tinycomputer-bus/src/flow/types/request.rs b/crates/tinycomputer-bus/src/flow/types/request.rs index 48ce5ac5..4856e257 100644 --- a/crates/tinycomputer-bus/src/flow/types/request.rs +++ b/crates/tinycomputer-bus/src/flow/types/request.rs @@ -146,6 +146,10 @@ pub struct GroundingHint { /// /// Requires confidential delivery, like `RunGoal`: the texts a flow enters /// travel with it. +// Each flag is an independent switch a caller sets by name; folding them +// into one enum would rename `allow_destructive`, `include_values` and +// `trace` on the wire, a major bump, for a tidiness no caller gains from. +#[allow(clippy::struct_excessive_bools)] #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct RunFlowRequest { @@ -207,6 +211,13 @@ pub struct RunFlowRequest { /// knows what it has done. #[serde(skip_serializing_if = "BTreeMap::is_empty")] pub collected: BTreeMap, + /// Whether the run before this one, of the same task, left the task's + /// own dialog in front ([`FlowRunResult::dialog_left_open`]): a dialog + /// in front at this run's first look is then the task's current stage, + /// to work within. Otherwise such a dialog is the page's (a promotion or + /// a sign-up the page opened itself, a menu), never the task's. + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub dialog_left_open: bool, } impl Default for RunFlowRequest { @@ -227,6 +238,7 @@ impl Default for RunFlowRequest { strategy: FlowStrategy::Narrow, deliberation: Deliberation::Deep, collected: BTreeMap::new(), + dialog_left_open: false, } } } diff --git a/crates/tinycomputer-bus/src/flow/types/result.rs b/crates/tinycomputer-bus/src/flow/types/result.rs index 281d1de1..75d36c40 100644 --- a/crates/tinycomputer-bus/src/flow/types/result.rs +++ b/crates/tinycomputer-bus/src/flow/types/result.rs @@ -106,6 +106,11 @@ pub struct FlowRunResult { /// Every Jev exchange, when [`RunFlowRequest::trace`] asked for them. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub trace: Vec, + /// Whether the run ended with the task's own dialog in front: one a + /// press of the run opened, still asking its question. The task hands + /// it to its next run as [`RunFlowRequest::dialog_left_open`]. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub dialog_left_open: bool, } /// One Jev request and its answers, as recorded by a traced run. diff --git a/crates/tinycomputer-bus/src/version/mod.rs b/crates/tinycomputer-bus/src/version/mod.rs index 646d2d62..ea4ae418 100644 --- a/crates/tinycomputer-bus/src/version/mod.rs +++ b/crates/tinycomputer-bus/src/version/mod.rs @@ -8,7 +8,7 @@ //! release workflow bumps, which tracks the shipped artifact. /// The wire contract version this crate defines. -pub const CONTRACT_VERSION: (u32, u32) = (2, 8); +pub const CONTRACT_VERSION: (u32, u32) = (2, 9); /// Returns whether a host holding [`CONTRACT_VERSION`] can bind to a module /// reporting `module`. @@ -22,8 +22,8 @@ pub const CONTRACT_VERSION: (u32, u32) = (2, 8); /// ``` /// # use tinycomputer_bus::{is_compatible, CONTRACT_VERSION}; /// assert!(is_compatible(CONTRACT_VERSION)); -/// assert!(is_compatible((2, 8))); -/// assert!(!is_compatible((2, 7))); +/// assert!(is_compatible((2, 9))); +/// assert!(!is_compatible((2, 8))); /// assert!(!is_compatible((1, 8))); /// ``` #[must_use] diff --git a/crates/tinycomputer-bus/src/version/version_tests.rs b/crates/tinycomputer-bus/src/version/version_tests.rs index 52abe8ec..9573f136 100644 --- a/crates/tinycomputer-bus/src/version/version_tests.rs +++ b/crates/tinycomputer-bus/src/version/version_tests.rs @@ -6,7 +6,7 @@ use super::{CONTRACT_VERSION, binds, is_compatible}; #[test] fn the_shipped_contract_version_is_pinned() { - assert_eq!(CONTRACT_VERSION, (2, 8)); + assert_eq!(CONTRACT_VERSION, (2, 9)); } #[test] @@ -16,8 +16,11 @@ fn the_contract_binds_to_itself() { #[test] fn a_newer_minor_on_the_module_side_binds() { - assert!(is_compatible((2, 8))); + assert!(is_compatible((2, 9))); assert!(is_compatible((2, 97))); + // 2.9 gives `RunFlow` and its result `dialog_left_open`: a 2.9 host may + // send it, which a 2.8 module would ignore. + assert!(!is_compatible((2, 8))); // 2.8 accepts the `open_jev` and `sage` decision providers and the // planner's `tiny_humans` route: a 2.8 host may configure them, which a // 2.7 module refuses. diff --git a/crates/tinycomputer-core/src/lib.rs b/crates/tinycomputer-core/src/lib.rs index da2e18a3..3ec751cd 100644 --- a/crates/tinycomputer-core/src/lib.rs +++ b/crates/tinycomputer-core/src/lib.rs @@ -52,7 +52,8 @@ pub use error::{Error, Result}; pub use facts::{Facts, is_sensitive_name}; pub use keymap::{Key, Platform}; pub use records::{ - Criterion, Price, Record, parse_clock, parse_duration, parse_price, parse_stops, rank, + Criterion, Price, Record, closest_to, parse_clock, parse_duration, parse_price, parse_stops, + rank, rank_closest, }; pub use safety::{ Consequence, FieldHint, PaymentEvidence, adjusts_a_count, consequence, human_needed, diff --git a/crates/tinycomputer-core/src/records/mod.rs b/crates/tinycomputer-core/src/records/mod.rs index 4845a045..cb81e470 100644 --- a/crates/tinycomputer-core/src/records/mod.rs +++ b/crates/tinycomputer-core/src/records/mod.rs @@ -13,7 +13,7 @@ mod rank; mod schedule; pub use price::{Price, parse_price}; -pub use rank::{Criterion, rank}; +pub use rank::{Criterion, closest_to, rank, rank_closest}; pub use schedule::{parse_clock, parse_duration, parse_stops}; /// One extracted item: field name to the text shown for it. diff --git a/crates/tinycomputer-core/src/records/rank.rs b/crates/tinycomputer-core/src/records/rank.rs index 0d5325fe..e25ffb1b 100644 --- a/crates/tinycomputer-core/src/records/rank.rs +++ b/crates/tinycomputer-core/src/records/rank.rs @@ -155,3 +155,71 @@ pub fn rank(records: &[Record], criterion: Criterion) -> Option> { }); Some(order) } + +/// The number a criterion such as "closest to 9" or "nearest to size 42" +/// asks items to come nearest to; `None` for any other criterion. +/// +/// ``` +/// use tinycomputer_core::closest_to; +/// +/// assert_eq!(closest_to("closest to 9"), Some(9.0)); +/// assert_eq!(closest_to("the size nearest to UK 8.5"), Some(8.5)); +/// assert_eq!(closest_to("lowest price"), None); +/// ``` +#[must_use] +pub fn closest_to(text: &str) -> Option { + let lower = text.to_ascii_lowercase(); + let after = ["closest to", "nearest to"] + .iter() + .find_map(|lead| lower.find(lead).map(|at| &lower[at + lead.len()..]))?; + first_number(after) +} + +/// Record indexes, nearest to `target` first, by the first number each +/// record shows: a size list of 6, 7, and 8 with 9 sold out ranks 8, 7, 6. +/// +/// Ties keep the records' own order, and records that show no number go +/// last. `None` when no record shows one. +/// +/// ``` +/// use tinycomputer_core::{Record, rank_closest}; +/// +/// let sizes = [ +/// Record::from_pairs([("size", "6")]), +/// Record::from_pairs([("size", "7"), ("stock", "3 left")]), +/// Record::from_pairs([("size", "8"), ("stock", "2 left")]), +/// ]; +/// assert_eq!(rank_closest(&sizes, 9.0), Some(vec![2, 1, 0])); +/// ``` +#[must_use] +pub fn rank_closest(records: &[Record], target: f64) -> Option> { + let keyed = records + .iter() + .map(|record| { + record + .fields + .values() + .find_map(|text| first_number(text)) + .map(|number| (number - target).abs()) + }) + .collect::>(); + if keyed.iter().all(Option::is_none) { + return None; + } + let mut order = (0..records.len()).collect::>(); + order.sort_by(|&left, &right| match (keyed[left], keyed[right]) { + (Some(a), Some(b)) => a.total_cmp(&b), + (Some(_), None) => std::cmp::Ordering::Less, + (None, Some(_)) => std::cmp::Ordering::Greater, + (None, None) => std::cmp::Ordering::Equal, + }); + Some(order) +} + +/// The first number `text` shows ("7" of "7 3 left", "8.5" of "UK 8.5"). +fn first_number(text: &str) -> Option { + text.split(|character: char| !(character.is_ascii_digit() || character == '.')) + .map(|word| word.trim_matches('.')) + .find(|word| !word.is_empty()) + .and_then(|word| word.parse().ok()) +} diff --git a/crates/tinycomputer-core/src/records/records_tests.rs b/crates/tinycomputer-core/src/records/records_tests.rs index b9ca5fbf..6b916e2a 100644 --- a/crates/tinycomputer-core/src/records/records_tests.rs +++ b/crates/tinycomputer-core/src/records/records_tests.rs @@ -3,7 +3,8 @@ #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] use super::{ - Criterion, Price, Record, parse_clock, parse_duration, parse_price, parse_stops, rank, + Criterion, Price, Record, closest_to, parse_clock, parse_duration, parse_price, parse_stops, + rank, rank_closest, }; fn price(text: &str) -> (f64, Option<&'static str>) { @@ -238,3 +239,28 @@ fn first_and_last_alone_are_the_lists_own_order() { ); assert_eq!(rank(&[], Criterion::First), None, "nothing to rank"); } + +#[test] +fn closest_to_reads_the_number_a_criterion_comes_nearest_to() { + assert_eq!(closest_to("closest to 9"), Some(9.0)); + assert_eq!(closest_to("Nearest to size 42"), Some(42.0)); + assert_eq!(closest_to("the size nearest to UK 8.5"), Some(8.5)); + assert_eq!(closest_to("closest to the station"), None); + assert_eq!(closest_to("cheapest"), None); +} + +#[test] +fn nearness_ranks_by_distance_and_numberless_records_go_last() { + let sizes = [ + Record::from_pairs([("size", "6")]), + Record::from_pairs([("label", "Size chart")]), + Record::from_pairs([("size", "10"), ("stock", "1 left")]), + Record::from_pairs([("size", "8"), ("stock", "2 left")]), + ]; + // 10 and 8 are as near to 9: the list's own order breaks the tie. + assert_eq!(rank_closest(&sizes, 9.0), Some(vec![2, 3, 0, 1])); + assert_eq!( + rank_closest(&[Record::from_pairs([("label", "Size chart")])], 9.0), + None + ); +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/act/mod.rs b/crates/tinycomputer-engine/src/agentic/flow/act/mod.rs index 600ca55d..66e3db00 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/act/mod.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/act/mod.rs @@ -31,6 +31,8 @@ mod dialog; pub(in crate::agentic::flow) use copies::copies_of; mod judge; mod moves; +#[cfg(test)] +pub(in crate::agentic::flow) use moves::still_shows; mod recover; mod turns; mod uncover; @@ -297,10 +299,16 @@ fn plural(word: &str) -> bool { /// Words of a step that ask for an overlay to go away. const DISMISS_VERBS: &[&str] = &["dismiss", "close", "accept", "decline", "reject", "skip"]; -/// What such a step asks to go away. +/// What such a step asks to go away ("pop" of "pop-up"). const OVERLAYS: &[&str] = &[ - "banner", "dialog", "popup", "cookie", "cookies", "consent", "modal", "overlay", "prompt", - "notice", + "banner", "dialog", "popup", "popups", "pop", "cookie", "cookies", "consent", "modal", + "overlay", "prompt", "notice", +]; + +/// Words of a control that closes what it sits on: a consent bar's "Accept +/// all" or "I agree" closes it as surely as its "Reject all" does. +const CLOSERS: &[&str] = &[ + "close", "dismiss", "skip", "later", "decline", "reject", "accept", "agree", ]; fn words(text: &str) -> Vec { @@ -318,21 +326,44 @@ fn words(text: &str) -> Vec { /// A completion judge sees only the screen after the fact, where a closed /// cookie banner leaves no trace of which button closed it; this is the /// evidence it cannot see. +/// +/// A pop-up the page draws without a dialog's role leaves the surface the +/// window throughout, so for a dismissal step a closer ("Close") that went +/// with what it was pressed on is that evidence instead. Live, an offer +/// pop-up closed at the first press, the judge could not tell whether +/// "declining optional cookies" was done with no cookie banner shown, and +/// the step stalled into a rescue. fn closed_the_overlay(last: &LastAction, screen: &Screen, intent: &str) -> Option { - let name = last.target.as_ref()?.name.as_deref()?; - if last.before.surface == "window" || screen.surface != "window" { + let target = last.target.as_ref()?; + let name = target.name.as_deref()?; + if screen.surface != "window" { return None; } let intent = words(intent); + let dismissal = intent + .iter() + .any(|word| DISMISS_VERBS.contains(&word.as_str())) + && intent.iter().any(|word| OVERLAYS.contains(&word.as_str())); + if last.before.surface == "window" { + let closer = words(name) + .iter() + .any(|word| CLOSERS.contains(&word.as_str())); + let gone = !screen + .candidates + .iter() + .any(|candidate| candidate.role == target.role && candidate.name == target.name); + return (dismissal && closer && gone).then(|| { + Ended::new( + StepOutcome::Done, + format!("pressed {name:?} and it closed with what it was on"), + ) + }); + } let named = words(name) .iter() .filter(|word| word.len() > 2) .all(|word| intent.iter().any(|said| said.starts_with(word.as_str()))) && words(name).iter().any(|word| word.len() > 2); - let dismissal = intent - .iter() - .any(|word| DISMISS_VERBS.contains(&word.as_str())) - && intent.iter().any(|word| OVERLAYS.contains(&word.as_str())); (named || dismissal).then(|| { Ended::new( StepOutcome::Done, diff --git a/crates/tinycomputer-engine/src/agentic/flow/act/moves.rs b/crates/tinycomputer-engine/src/agentic/flow/act/moves.rs index 0e6ce49f..0ec2f604 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/act/moves.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/act/moves.rs @@ -70,7 +70,15 @@ impl FlowRun<'_, B> { )); return Ok(Move::Skipped); } - if combo == "return" && screen.surface != "window" { + // Return in a search box runs its search wherever the box + // sits: live, a store's search opened as a full-window + // sheet, and its step to press Enter in the box just typed + // into was refused 154 times. + let searching = self + .typed_last + .as_ref() + .is_some_and(|field| is_search_box(field) && still_shows(screen, field)); + if combo == "return" && screen.surface != "window" && !searching { self.history.push(format!( "refused return while a {} is showing: it would press its default button", screen.surface @@ -151,10 +159,12 @@ impl FlowRun<'_, B> { // a press scrolls it out from under the bar (live, a seat table's // lower rows sat under its "Pay" bar). && !in_dialog(candidate); - if covered && self.front.surface != "window" { + // A calendar the task has picked in is closed for such a press + // (`press_uncovering`), so what it covers can be pressed. + if covered && self.front.surface != "window" && !self.front.served_calendar() { return false; } - !(self.front.opened_dialog && closes(candidate) && !asks_to_close(intent)) + !(self.front.opened_dialog() && closes(candidate) && !asks_to_close(intent)) } /// Grounds and performs an `activate`, `expand`, or `scroll` move; the @@ -201,7 +211,7 @@ impl FlowRun<'_, B> { // front: the dialog asks something first, and what answers it is // pressed instead, though not remembered as the step's control. let (grounded, answers_dialog) = match grounded { - None if operation == "activate" && self.front.opened_dialog => { + None if operation == "activate" && self.front.opened_dialog() => { (self.answer_dialog(log, screen, intent, banned).await?, true) } grounded => (grounded, false), @@ -341,3 +351,33 @@ fn asks_to_close(intent: &str) -> bool { ) }) } + +/// Whether `field` is a search box: a `searchbox`, or a box that takes text +/// and names itself for searching ("Search Lenskart", "Search for atta dal +/// and more"). Return there runs the search, never a dialog's default +/// button. +pub(in crate::agentic::flow) fn is_search_box(field: &Candidate) -> bool { + let takes_text = field + .available_actions + .iter() + .any(|action| action == "SetValue" || action == "TypeText"); + let named = [field.name.as_deref(), field.description.as_deref()] + .into_iter() + .flatten() + .any(|text| text.to_lowercase().contains("search")); + field.role.eq_ignore_ascii_case("searchbox") || (takes_text && named) +} + +/// Whether `field` still shows on `screen` with nothing covering it: a +/// dialog that has come up over it since it was typed into takes the keys +/// itself. +pub(in crate::agentic::flow) fn still_shows(screen: &Screen, field: &Candidate) -> bool { + screen.candidates.iter().any(|candidate| { + candidate.role == field.role + && candidate.name == field.name + && !candidate + .states + .iter() + .any(|state| state.eq_ignore_ascii_case("covered")) + }) +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/act/recover.rs b/crates/tinycomputer-engine/src/agentic/flow/act/recover.rs index 5b69a134..172b6dcf 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/act/recover.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/act/recover.rs @@ -34,7 +34,7 @@ impl FlowRun<'_, B> { ) -> Result { if judged.blocked.unwrap_or_default() >= BLOCKED && state.obstacles < MAX_OBSTACLES - && !self.front.opened_dialog + && !self.front.opened_dialog() { state.obstacles += 1; log.used(FlowLoop::Obstacles); @@ -99,7 +99,7 @@ impl FlowRun<'_, B> { // A press that opened a dialog of the task's (the seat count after // a showtime) moved the flow on, whatever the judge made of it: // undoing it with Escape closed the dialog live. - if !self.enabled(FlowLoop::Undo) || state.undos >= MAX_UNDOS || self.front.opened_dialog { + if !self.enabled(FlowLoop::Undo) || state.undos >= MAX_UNDOS || self.front.opened_dialog() { return Ok(false); } state.undos += 1; diff --git a/crates/tinycomputer-engine/src/agentic/flow/act/turns.rs b/crates/tinycomputer-engine/src/agentic/flow/act/turns.rs index 896a42b6..93a9e79e 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/act/turns.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/act/turns.rs @@ -36,7 +36,7 @@ impl FlowRun<'_, B> { // choose a heading in it: live, rescues kept choosing a language // heading above a format dialog's buttons. match ended { - Err(Halt::Failed(note)) if self.front.opened_dialog => { + Err(Halt::Failed(note)) if self.front.opened_dialog() => { let offers = match self.look().await { Ok(screen) => front_controls(&screen), Err(_) => Vec::new(), @@ -96,7 +96,7 @@ impl FlowRun<'_, B> { // The root of the turn's tree: what needs attention first. A // distraction cleared means a fresh look before judging. A dialog // the step's own press just opened is the step's to work in. - if !self.front.opened_dialog + if !self.front.opened_dialog() && self .attend(log, &screen, intent, &mut state.cleared) .await? diff --git a/crates/tinycomputer-engine/src/agentic/flow/act/uncover.rs b/crates/tinycomputer-engine/src/agentic/flow/act/uncover.rs index 5032fb1d..055ed048 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/act/uncover.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/act/uncover.rs @@ -28,6 +28,8 @@ impl FlowRun<'_, B> { operation: JevOperation, intent: &str, ) -> Result { + // Read before this press, which would count as one in the dialog. + let served = self.front.served_calendar(); let chosen = target.clone(); let reply = self .act(log, verb, Some(target), move |backend| { @@ -42,8 +44,14 @@ impl FlowRun<'_, B> { // lies behind it leaves the flow it began (live, a movie's language // link behind its booking dialog led to a listing of other films). // A layer drawn over the window is such a question only when the - // task's own press opened it; a calendar left open is in the way. - if self.front.opened_dialog || !matches!(self.front.surface.as_str(), "window" | "layer") { + // task's own press opened it; a calendar left open is in the way, + // and so is one the task opened and has pressed in since: live, a + // calendar stayed in front of the guests and Search buttons once + // both dates were picked, and every press behind it was refused. + if (self.front.opened_dialog() + || !matches!(self.front.surface.as_str(), "window" | "layer")) + && !served + { self.history.push(format!( "{} lies behind the dialog in front; act within the dialog instead", label(target) diff --git a/crates/tinycomputer-engine/src/agentic/flow/action.rs b/crates/tinycomputer-engine/src/agentic/flow/action.rs index b70f9eeb..5c49a9ad 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/action.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/action.rs @@ -28,7 +28,13 @@ impl FlowRun<'_, B> { return Err(Halt::Stop(FlowStopReason::ActionBudget)); } self.actions = self.actions.saturating_add(1); - self.front.act(action, target.is_some()); + self.front.act(action, target); + if action != "wait" { + let typing = ["fill", "type", "paste"] + .iter() + .any(|verb| action.starts_with(verb)); + self.typed_last = target.filter(|_| typing).cloned(); + } let started = Instant::now(); let reply = self.backend_call(call).await; let acted_ms = millis(started.elapsed()); diff --git a/crates/tinycomputer-engine/src/agentic/flow/ask/screen_state.rs b/crates/tinycomputer-engine/src/agentic/flow/ask/screen_state.rs index 80cbd443..22ead64a 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/ask/screen_state.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/ask/screen_state.rs @@ -3,7 +3,10 @@ use serde_json::{Value, json}; -use crate::agentic::flow::view::{Candidate, Screen, element_line, label, untrusted_context}; +use crate::agentic::flow::{ + denoise::{Tier, tier}, + view::{Candidate, Screen, element_line, label, untrusted_context}, +}; use super::{MAX_FIELDS, MAX_HISTORY, MAX_STATE_ELEMENTS}; @@ -14,10 +17,8 @@ pub(in crate::agentic::flow) fn state( history: &[String], include_values: bool, ) -> Value { - let elements = screen - .candidates - .iter() - .take(MAX_STATE_ELEMENTS) + let elements = seen_first(&screen.candidates, MAX_STATE_ELEMENTS) + .into_iter() .map(|node| element_line(node, include_values)) .collect::>(); let mut state = json!({ @@ -35,6 +36,31 @@ pub(in crate::agentic::flow) fn state( state } +/// The `most` of `candidates` Jev is shown, all listed in screen order. +/// On a longer screen, those in view are kept first, and the rest, as the +/// page orders them, keep a quarter of the room. Live, a sign-up pop-up a +/// long page drew at the end of its document fell outside the first 120 +/// elements, behind the covered page, and Jev never saw it; and a calendar +/// open in front would fill the room and hide the guests button it covers, +/// which the next step reads. +fn seen_first(candidates: &[Candidate], most: usize) -> Vec<&Candidate> { + if candidates.len() <= most { + return candidates.iter().collect(); + } + let (in_view, rest): (Vec<_>, Vec<_>) = candidates + .iter() + .enumerate() + .partition(|(_, candidate)| tier(candidate) == Tier::InView); + let front = in_view.len().min(most - rest.len().min(most / 4)); + let mut kept = in_view + .into_iter() + .take(front) + .chain(rest.into_iter().take(most - front)) + .collect::>(); + kept.sort_by_key(|(at, _)| *at); + kept.into_iter().map(|(_, candidate)| candidate).collect() +} + /// What each text-holding element shows, at more length than the element /// list allows: whether a draft "shows the body" is decided here. /// diff --git a/crates/tinycomputer-engine/src/agentic/flow/attention/clear.rs b/crates/tinycomputer-engine/src/agentic/flow/attention/clear.rs index fd1f0442..23e1226c 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/attention/clear.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/attention/clear.rs @@ -30,10 +30,10 @@ impl FlowRun<'_, B> { cleared: &mut Cleared, ) -> Result { // A dialog the run's own press opened is its next stage, not a - // distraction (`FlowRun::opened_dialog`). + // distraction (`Front::opened_dialog`). if !self.deliberates(FlowLoop::Attention) || cleared.count >= MAX_CLEARED - || self.front.opened_dialog + || self.front.opened_dialog() { return Ok(false); } diff --git a/crates/tinycomputer-engine/src/agentic/flow/enter/fill.rs b/crates/tinycomputer-engine/src/agentic/flow/enter/fill.rs index a1868ba4..c7d88a2d 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/enter/fill.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/enter/fill.rs @@ -4,100 +4,56 @@ use std::collections::BTreeSet; use serde_json::Value; -use tinycomputer_bus::{JevOperation, Slot}; +use tinycomputer_bus::Slot; use tinycomputer_core::reformat_date; use crate::agentic::flow::{ AgentBackend, FlowRun, Halt, StepLog, - ask::{self, Questions, corroborate, probability}, backend::deliver_text, memory::{learn, remember}, - view::{Candidate, Screen, element_kind, is_destructive, label}, + steps::looks_like_date, + view::{Candidate, Screen, element_kind, label}, }; -use super::{BLIND_PICK_MISSES, OPENER_FLOOR, REVEAL_TURNS, editable, names}; +use super::{BLIND_PICK_MISSES, editable, names}; impl FlowRun<'_, B> { - /// Presses the control on `screen` whose label holds a pending slot's - /// own word (`named_opener`); `true` when it did. A control named by - /// the slot, a search link for the "search box", is how the field - /// shows: live, the turns that look for a way in hesitated over it, and - /// the search was never typed. - async fn open_by_name( + /// Picks each pending date of `untried` from the calendar showing, + /// before any box is looked for, and says whether one arrived: the box + /// looked for is the calendar's own button, and pressing it closes the + /// calendar. Live, a check-out date's step found the calendar open from + /// the check-in and pressed its button four times, and a departure's + /// calendar opened by the step's own press was closed again the same way. + async fn pick_shown_dates( &mut self, log: &mut StepLog, - screen: &Screen, slots: &[Slot], - pending: &BTreeSet, + private: &[bool], + pending: &mut BTreeSet, + untried: &mut BTreeSet, ) -> Result { - let Some(opener) = named_opener(screen, slots, pending) - .filter(|opener| !is_destructive(opener, screen, &self.stop_before)) - else { - return Ok(false); - }; - // A shared word is a hint, not a reason to press: a link named - // "Email us" shares the slot "email", and pressing it left the form. - let purpose = format!("click to show the box for: {}", names(slots, pending)); - let answers = self - .ask( - log, - ask::request( - self.model(), - self.state(screen, &purpose), - Questions::default().with( - "confirm", - corroborate(&purpose, &opener, self.include_values), - ), - ), - ) - .await?; - if probability(&answers, "confirm").is_none_or(|yes| yes < OPENER_FLOOR) { - return Ok(false); - } - let pressed = opener.clone(); - let reply = self - .act( - log, - "click (show the field)", - Some(&opener), - move |backend| backend.execute(JevOperation::Click, Some(pressed), None), - ) - .await?; - if reply.ok { - self.history.push(format!( - "pressed {} to show the field for {}", - label(&opener), - names(slots, pending) - )); - } - Ok(reply.ok) - } - - /// Runs a short `do` loop that shows the fields for the `pending` slots, - /// any field at all when `no_fields` showed. A field that cannot be revealed - /// is looked for another way, or found not to be asked for; it is not a - /// failure. - async fn reveal_fields( - &mut self, - log: &mut StepLog, - slots: &[Slot], - pending: &BTreeSet, - no_fields: bool, - ) -> Result<(), Halt> { - let reveal = if no_fields { - format!("show the editable fields for: {}", names(slots, pending)) - } else { - format!("show the fields for: {}", names(slots, pending)) - }; - match self.accomplish(log, &reveal, REVEAL_TURNS).await { - Err(Halt::Failed(note)) => self - .history - .push(format!("could not reveal the fields ({note})")), - other => { - other?; + let dates = pending + .iter() + .copied() + .filter(|index| untried.contains(index) && looks_like_date(&slots[*index].text)) + .collect::>(); + let mut picked = false; + for index in dates { + untried.remove(&index); + let slot = &slots[index]; + match self + .pick_option(log, &slot.slot, &slot.text, private[index], false) + .await + { + Ok(_) => { + pending.remove(&index); + picked = true; + } + Err(Halt::Failed(_)) => {} + Err(halt) => return Err(halt), } } - Ok(()) + Ok(picked) } /// Fills every slot in `pending` it can find a field or an option for, @@ -110,7 +66,14 @@ impl FlowRun<'_, B> { pending: &mut BTreeSet, ) -> Result<(), Halt> { let mut revealed = false; - let mut opened = false; + // The dates not yet picked from a calendar showing, each tried once. + let mut dates = pending.clone(); + // The slots whose own opener was looked for, each once: a form that + // draws its place boxes as buttons ("From DEL", "To BLR") opens one + // box at a time. Live, the "to" box was never opened once "from" + // had been, and the place was pressed in a link at the foot of the + // page instead. + let mut opened: BTreeSet = BTreeSet::new(); // The fields this step already filled: one slot's box is never // another's. Live, a pickup box that had not yet become the place // chosen was the only box on screen in the next round, and the drop @@ -125,11 +88,19 @@ impl FlowRun<'_, B> { // by the city it holds, are struck together rather than one at a // time (`a_row_that_refused_the_text_is_never_pressed_while_revealing_a_field`). let mut struck: BTreeSet = BTreeSet::new(); - for _ in 0..3 { + // Each slot may take a round to open its box and one to fill it. + for _ in 0..3.max(2 * pending.len() + 1) { if pending.is_empty() { break; } let mut screen = self.look().await?; + if self.front.calendar + && self + .pick_shown_dates(log, slots, private, pending, &mut dates) + .await? + { + continue; + } if editable(&screen).len() < pending.len() && !screen.unexplored.is_empty() { self.explore(&mut screen).await; } @@ -141,11 +112,11 @@ impl FlowRun<'_, B> { self.assign(log, &screen, slots, pending, &fields).await? }; if assignments.is_empty() { - if !opened { - opened = true; - if self.open_by_name(log, &screen, slots, pending).await? { - continue; - } + if self + .open_next_box(log, &screen, slots, pending, &mut opened) + .await? + { + continue; } if revealed { break; @@ -189,12 +160,24 @@ impl FlowRun<'_, B> { } } } - // A value with no field to type into is picked instead, as a date - // from a calendar or a city from a list of suggestions. Only a "the - // value was not found" failure is safe to shrug off and move to the - // next slot; a budget stop or a backend error means acting further - // is unsafe or pointless, and must end the step instead of being - // read as "this slot has no picker". + self.pick_unboxed(log, slots, private, pending, saw_fields) + .await + } + + /// Picks each slot still `pending` instead of typing it, as a date from a + /// calendar or a city from a list of suggestions, when no field took it. + /// Only a "the value was not found" failure is safe to shrug off and move + /// to the next slot; a budget stop or a backend error means acting + /// further is unsafe or pointless, and must end the step instead of being + /// read as "this slot has no picker". + async fn pick_unboxed( + &mut self, + log: &mut StepLog, + slots: &[Slot], + private: &[bool], + pending: &mut BTreeSet, + saw_fields: bool, + ) -> Result<(), Halt> { let mut missed = 0; for index in pending.clone() { if !saw_fields && missed >= BLIND_PICK_MISSES { @@ -271,59 +254,6 @@ impl FlowRun<'_, B> { } } -/// Words of a slot's name that say only that it is a box, or name a kind -/// of control rather than what the slot is. -const BOX_WORDS: &[&str] = &[ - "field", "box", "input", "bar", "the", "your", "text", "here", "link", "button", "menu", "icon", -]; - -/// A control on `screen` that is no field itself and whose label holds a -/// word of a pending slot's name (four letters or more, not a box word): -/// the link or button that shows the slot's field, such as a store's -/// search link for the slot "search box". -fn named_opener(screen: &Screen, slots: &[Slot], pending: &BTreeSet) -> Option { - let wanted = pending - .iter() - .filter_map(|index| slots.get(*index)) - .flat_map(|slot| words(&slot.slot)) - .filter(|word| word.chars().count() > 3 && !BOX_WORDS.contains(&word.as_str())) - .collect::>(); - if wanted.is_empty() { - return None; - } - let fields = editable(screen); - screen - .candidates - .iter() - .filter(|candidate| { - matches!(candidate.role.as_str(), "link" | "button") - && candidate - .available_actions - .iter() - .any(|action| action == "Click") - && !fields.iter().any(|field| field.ref_id == candidate.ref_id) - && !candidate - .states - .iter() - .any(|state| state.eq_ignore_ascii_case("covered")) - }) - .find(|candidate| { - words(candidate.name.as_deref().unwrap_or_default()) - .iter() - .take(3) - .any(|word| wanted.contains(word)) - }) - .cloned() -} - -/// The lower-case words of `text`. -fn words(text: &str) -> Vec { - text.split(|character: char| !character.is_alphanumeric()) - .filter(|word| !word.is_empty()) - .map(str::to_lowercase) - .collect() -} - /// The `fields` a slot may still take: of no kind struck this step, and /// neither filled this step (`filled`, by ref) nor holding a text this step /// delivered (`delivered`). diff --git a/crates/tinycomputer-engine/src/agentic/flow/enter/mod.rs b/crates/tinycomputer-engine/src/agentic/flow/enter/mod.rs index c383fec8..0876609b 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/enter/mod.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/enter/mod.rs @@ -6,11 +6,15 @@ //! (`deliver_text`), top to bottom in screen order. A slot with no visible //! field first runs a short `do` loop to reveal one. //! -//! `assign` matches slots to fields and asks which the form wants; `fill` -//! delivers each text and verifies it. +//! `assign` matches slots to fields and asks which the form wants; `open` +//! shows a slot's box when the screen has none for it; `fill` delivers each +//! text and verifies it. mod assign; mod fill; +mod open; +#[cfg(test)] +pub(super) use open::named_opener; use std::collections::BTreeSet; diff --git a/crates/tinycomputer-engine/src/agentic/flow/enter/open.rs b/crates/tinycomputer-engine/src/agentic/flow/enter/open.rs new file mode 100644 index 00000000..19d25e17 --- /dev/null +++ b/crates/tinycomputer-engine/src/agentic/flow/enter/open.rs @@ -0,0 +1,198 @@ +//! Showing a slot's box when the screen has none for it: pressing the +//! control a slot's own name names ("To BLR" for the slot "to"), one slot at +//! a time, or else a short `do` loop that reveals the fields. + +use std::collections::BTreeSet; + +use tinycomputer_bus::{JevOperation, Slot}; + +use crate::agentic::flow::{ + AgentBackend, FlowRun, Halt, StepLog, + ask::{self, Questions, corroborate, probability}, + view::{Candidate, Screen, is_destructive, label}, +}; + +use super::{OPENER_FLOOR, REVEAL_TURNS, editable, names}; + +impl FlowRun<'_, B> { + /// Presses the control on `screen` whose label holds a pending slot's + /// own word (`named_opener`); `true` when it did. A control named by + /// the slot, a search link for the "search box", is how the field + /// shows: live, the turns that look for a way in hesitated over it, and + /// the search was never typed. + async fn open_by_name( + &mut self, + log: &mut StepLog, + screen: &Screen, + slots: &[Slot], + pending: &BTreeSet, + ) -> Result { + let Some(opener) = named_opener(screen, slots, pending) + .filter(|opener| !is_destructive(opener, screen, &self.stop_before)) + else { + return Ok(false); + }; + // A shared word is a hint, not a reason to press: a link named + // "Email us" shares the slot "email", and pressing it left the form. + let purpose = format!("click to show the box for: {}", names(slots, pending)); + let answers = self + .ask( + log, + ask::request( + self.model(), + self.state(screen, &purpose), + Questions::default().with( + "confirm", + corroborate(&purpose, &opener, self.include_values), + ), + ), + ) + .await?; + if probability(&answers, "confirm").is_none_or(|yes| yes < OPENER_FLOOR) { + return Ok(false); + } + let pressed = opener.clone(); + let reply = self + .act( + log, + "click (show the field)", + Some(&opener), + move |backend| backend.execute(JevOperation::Click, Some(pressed), None), + ) + .await?; + if reply.ok { + self.history.push(format!( + "pressed {} to show the field for {}", + label(&opener), + names(slots, pending) + )); + } + Ok(reply.ok) + } + + /// Runs a short `do` loop that shows the fields for the `pending` slots, + /// any field at all when `no_fields` showed. A field that cannot be revealed + /// is looked for another way, or found not to be asked for; it is not a + /// failure. + pub(super) async fn reveal_fields( + &mut self, + log: &mut StepLog, + slots: &[Slot], + pending: &BTreeSet, + no_fields: bool, + ) -> Result<(), Halt> { + let reveal = if no_fields { + format!("show the editable fields for: {}", names(slots, pending)) + } else { + format!("show the fields for: {}", names(slots, pending)) + }; + match self.accomplish(log, &reveal, REVEAL_TURNS).await { + Err(Halt::Failed(note)) => self + .history + .push(format!("could not reveal the fields ({note})")), + other => { + other?; + } + } + Ok(()) + } + + /// Presses the opener of the first slot of `pending` whose own opener was + /// not looked for yet (`opened`), noting each slot it looks for; `true` + /// when it pressed one. + pub(super) async fn open_next_box( + &mut self, + log: &mut StepLog, + screen: &Screen, + slots: &[Slot], + pending: &BTreeSet, + opened: &mut BTreeSet, + ) -> Result { + for index in pending.difference(opened).copied().collect::>() { + opened.insert(index); + if self + .open_by_name(log, screen, slots, &BTreeSet::from([index])) + .await? + { + return Ok(true); + } + } + Ok(false) + } +} + +/// Words of a slot's name that say only that it is a box, or name a kind +/// of control rather than what the slot is. +const BOX_WORDS: &[&str] = &[ + "field", "box", "input", "bar", "the", "your", "text", "here", "link", "button", "menu", "icon", +]; + +/// Short words of a place slot ("to") that name its box only as the first +/// word of a control's label ("To BLR, Bengaluru"): anywhere else in a +/// label, "to" is any sentence's ("Tap to add a return date"). +const PLACE_LEADS: &[&str] = &["to", "via"]; + +/// A control on `screen` that is no field itself and whose label holds a +/// word of a pending slot's name (four letters or more, not a box word) +/// among its first three words, or begins with its short place word +/// ([`PLACE_LEADS`]): the link or button that shows the slot's field, such +/// as a store's search link for the slot "search box", or a flight form's +/// "To BLR" for the slot "to". Further into a label, the slot's word is a +/// sentence's ("Read our tips to search faster"), and a press there leaves +/// the form. +pub(in crate::agentic::flow) fn named_opener( + screen: &Screen, + slots: &[Slot], + pending: &BTreeSet, +) -> Option { + let named = pending + .iter() + .filter_map(|index| slots.get(*index)) + .flat_map(|slot| words(&slot.slot)) + .collect::>(); + let wanted = named + .iter() + .filter(|word| word.chars().count() > 3 && !BOX_WORDS.contains(&word.as_str())) + .cloned() + .collect::>(); + let leads = named + .iter() + .filter(|word| PLACE_LEADS.contains(&word.as_str())) + .cloned() + .collect::>(); + if wanted.is_empty() && leads.is_empty() { + return None; + } + let fields = editable(screen); + screen + .candidates + .iter() + .filter(|candidate| { + ["link", "button"] + .iter() + .any(|role| candidate.role.eq_ignore_ascii_case(role)) + && candidate + .available_actions + .iter() + .any(|action| action == "Click") + && !fields.iter().any(|field| field.ref_id == candidate.ref_id) + && !candidate + .states + .iter() + .any(|state| state.eq_ignore_ascii_case("covered")) + }) + .find(|candidate| { + let label = words(candidate.name.as_deref().unwrap_or_default()); + label.iter().take(3).any(|word| wanted.contains(word)) + || label.first().is_some_and(|word| leads.contains(word)) + }) + .cloned() +} + +/// The lower-case words of `text`. +fn words(text: &str) -> Vec { + text.split(|character: char| !character.is_alphanumeric()) + .filter(|word| !word.is_empty()) + .map(str::to_lowercase) + .collect() +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests.rs index e39d5cd5..b09e4131 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests.rs @@ -31,6 +31,7 @@ mod hedge_tests; mod helpers_tests; mod journal_tests; mod pick_tests; +mod picker_tests; mod quorum_tests; mod reflection_tests; mod split_tests; diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/brief_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/brief_tests.rs index b7646ba1..3cc919db 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/brief_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/brief_tests.rs @@ -363,3 +363,70 @@ async fn a_long_goal_is_clipped_in_the_brief() { assert_eq!(goal.chars().count(), 601); assert!(goal.ends_with('…')); } + +/// A browser screen of `candidates`, in the order given. +fn long_screen(candidates: Vec) -> Screen { + Screen { + app: "browser".to_owned(), + window: None, + surface: "sheet".to_owned(), + candidates, + context: Vec::new(), + unexplored: Vec::new(), + text_nodes: Vec::new(), + } +} + +/// `count` buttons named `prefix` and their number, each `covered` or not. +fn buttons(prefix: &str, count: u32, covered: bool) -> Vec { + (0..count) + .map(|index| { + let mut control = node( + &format!("{prefix} {index}"), + "button", + &["Click"], + &["main"], + f64::from(index), + ); + if covered { + control.states = vec!["covered".to_owned()]; + } + control + }) + .collect() +} + +/// The element lines Jev is shown for `screen`. +fn element_lines(screen: &Screen) -> Vec { + ask::state(screen, "x", &[], false)["elements"]["untrusted_accessibility_data"] + .as_array() + .unwrap() + .iter() + .map(|line| line.as_str().unwrap().to_owned()) + .collect() +} + +#[test] +fn a_long_screen_shows_jev_what_is_in_view_and_keeps_room_for_the_rest() { + // Live, a sign-up pop-up a long page drew at the end of its document + // fell outside the first 120 elements, behind the covered page. + let mut candidates = buttons("Page", 130, true); + for name in ["close", "Enter your Mobile Number"] { + candidates.push(node(name, "button", &["Click"], &["dialog"], 200.0)); + } + let lines = element_lines(&long_screen(candidates)); + assert_eq!(lines.len(), 120); + assert!(lines[0].contains("Page 0"), "kept in screen order"); + assert!(lines[118].contains("close"), "{:?}", &lines[115..]); + assert!(lines[119].contains("Enter your Mobile Number")); + + // A calendar open in front fills no more than three quarters of the + // room: the guests button it covers, early on the page, stays. + let mut candidates = buttons("Search form", 3, true); + candidates.extend(buttons("Day", 140, false)); + candidates.extend(buttons("Footer", 50, true)); + let lines = element_lines(&long_screen(candidates)); + assert_eq!(lines.len(), 120); + assert!(lines[0].contains("Search form 0"), "{:?}", &lines[..4]); + assert_eq!(lines.iter().filter(|line| line.contains("Day")).count(), 90); +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/choose_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/choose_tests.rs index 5f383d6b..b96aa8b0 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/choose_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/choose_tests.rs @@ -237,6 +237,24 @@ fn a_date_is_told_from_other_options_and_containers_give_way() { "a lone match stays" ); assert!(closest(Vec::new()).is_empty()); + + // A list's option stays however long its label: one airport row says + // much more than a footer link that only names the city. + let link = node("Mumbai", "link", &["Click"], &[], 0.0); + let row = node( + "BOM Mumbai, India Chhatrapati Shivaji International Airport 3 Nearby Airports found", + "option", + &["Click"], + &[], + 0.0, + ); + assert_eq!(names(closest(vec![link.clone(), row.clone()])).len(), 2); + // However the surface spells its role. + let row = Candidate { + role: "Option".to_owned(), + ..row + }; + assert_eq!(names(closest(vec![link, row])).len(), 2); } #[tokio::test] @@ -455,6 +473,27 @@ fn in_region_prefers_the_ancestor_named_region_but_keeps_every_match_when_none_i // `pick_option` falls back to the unnarrowed pool when nothing on the // page names the region at all. assert!(in_region(&unrelated, "")); + + // A placing word alone is in too many labels to place an option: only a + // container whose name begins with it holds one. + let route = node( + "Delhi to Mumbai flights", + "link", + &["Click"], + &["root", "list \"Popular routes to Mumbai\""], + 30.0, + ); + let airport = node( + "BOM Mumbai, India", + "option", + &["Click"], + &["root", "dialog \"To\"", "listbox \"Airports\""], + 40.0, + ); + assert!(!in_region(&route, "to")); + assert!(in_region(&airport, "to")); + // A longer region name still counts on the option's own label. + assert!(in_region(&route, "popular routes")); } #[test] @@ -487,3 +526,34 @@ fn a_day_in_a_strip_of_dates_is_found_by_its_short_label() { ); assert!(shows_date("7 Sept", &date_words("7 September"))); } + +#[test] +fn a_date_shown_under_the_fields_own_name_is_already_chosen() { + // Live, the day was pressed and the departure button showed it, but the + // step looked for a box to type the date into and failed. A day of the + // calendar names no field, so it never counts. + use super::steps::date_shown_in; + let screen = |names: &[&str]| Screen { + app: "browser".to_owned(), + window: None, + surface: "window".to_owned(), + candidates: names + .iter() + .map(|name| node(name, "button", &["Click"], &["form"], 0.0)) + .collect(), + context: Vec::new(), + unexplored: Vec::new(), + text_nodes: Vec::new(), + }; + let set = screen(&["Departure Thu, 22 Oct", "22 6845"]); + assert_eq!( + date_shown_in(&set, "departure date", "22 October 2026").and_then(|holder| holder.name), + Some("Departure Thu, 22 Oct".to_owned()) + ); + let unset = screen(&["Departure Fri, 09 Oct", "Thursday, October 22, 2026"]); + assert!(date_shown_in(&unset, "departure date", "22 October 2026").is_none()); + assert!( + date_shown_in(&set, "date", "22 October 2026").is_none(), + "no word names the field" + ); +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs index cd69140c..94d6f2c4 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs @@ -653,7 +653,7 @@ async fn a_step_finding_nothing_to_press_answers_the_dialog_the_task_opened() { let run = run_with( App::with(|sim| sim.obstacle = true), json!({"app": "browser", "steps": ["choose Wednesday 7 October 2026 in the date picker"]}), - |_| {}, + |request| request.dialog_left_open = true, |id, question, sim| { let answering = serde_json::to_string(question) .unwrap() @@ -689,7 +689,8 @@ async fn a_step_finding_nothing_to_press_answers_the_dialog_the_task_opened() { async fn a_control_the_dialogs_own_bar_covers_is_pressed_and_one_behind_it_is_not() { // Live, a seat table's lower rows sat under its "Pay" bar and were never // offered; a press scrolls such a control out from under the bar. A - // browser run takes a dialog at its first look as the task's own. + // browser run takes a dialog at its first look as the task's own when + // the run before it left that dialog open. let pressing = |wanted: &'static str| { move |id: &str, question: &Question, sim: &Sim| match id { "done" => Some(noul(if sim.obstacle { 0.05 } else { 0.95 })), @@ -707,7 +708,7 @@ async fn a_control_the_dialogs_own_bar_covers_is_pressed_and_one_behind_it_is_no sim.quirks.insert(Quirk::BarOverSheet); }), json!({"app": "browser", "steps": ["keep editing the draft"]}), - |_| {}, + |request| request.dialog_left_open = true, pressing("Keep Editing"), ) .await; @@ -734,7 +735,7 @@ async fn a_control_the_dialogs_own_bar_covers_is_pressed_and_one_behind_it_is_no sim.quirks.insert(Quirk::Covered); }), json!({"app": "browser", "steps": ["start a new email message"]}), - |_| {}, + |request| request.dialog_left_open = true, pressing("New Message"), ) .await; @@ -745,6 +746,59 @@ async fn a_control_the_dialogs_own_bar_covers_is_pressed_and_one_behind_it_is_no ); } +#[tokio::test] +async fn return_in_a_search_box_runs_the_search_with_a_sheet_in_front() { + // Live, a store's search opened as a full-window sheet, and the step to + // press Enter in the box just typed into was refused 154 times, as if + // Return would press the sheet's default button. + let pressing_return = |id: &str, question: &Question, sim: &Sim| match id { + "done" => Some(noul(if sim.presses.iter().any(|key| key == "return") { + 0.95 + } else { + 0.05 + })), + "move" => Some(pick(question, "shortcut", 0.9)), + "shortcut" => Some(pick(question, "confirm", 0.95)), + _ => None, + }; + let run = run_with( + App::with(|sim| { + sim.quirks.insert(Quirk::SearchBehindLink); + sim.quirks.insert(Quirk::SearchOpen); + sim.quirks.insert(Quirk::SearchSheet); + }), + json!({"app": "Mail", "steps": [ + {"enter": {"search": "invoices"}}, + "press Enter in the search box" + ]}), + |request| request.disabled_loops.push(FlowLoop::Attention), + pressing_return, + ) + .await; + let presses = run.app.sim().presses.clone(); + assert!(presses.iter().any(|key| key == "return"), "{presses:?}"); + assert_eq!(run.result.stop, FlowStopReason::Completed); + + // Without a search box typed into, Return in front of a sheet stays + // refused: it would press the sheet's default button. + let run = run_with( + App::with(|sim| { + sim.quirks.insert(Quirk::SearchBehindLink); + sim.quirks.insert(Quirk::SearchOpen); + sim.quirks.insert(Quirk::SearchSheet); + }), + json!({"app": "Mail", "steps": ["press Enter"]}), + |request| { + request.disabled_loops.push(FlowLoop::Attention); + request.max_actions = 3; + }, + pressing_return, + ) + .await; + let presses = run.app.sim().presses.clone(); + assert!(!presses.iter().any(|key| key == "return"), "{presses:?}"); +} + /// A page at `surface`, its `covered` controls drawn under something. fn page_at(surface: &str, covered: usize) -> Screen { Screen { @@ -780,48 +834,209 @@ fn a_dialog_the_task_worked_in_is_in_the_way_of_the_next_step() { use crate::agentic::flow::front::Front; let at = Some("https://flights.test/"); let (window, sheet) = (page_at("window", 0), page_at("sheet", 3)); + let press = node("Control 4", "button", &["Click"], &["main"], 4.0); let mut front = Front::default(); - front.act("browse https://flights.test/", false); - assert!(front.look(&window, at, true).is_none()); - front.act("click", true); - assert!( - front.look(&sheet, at, true).is_some(), - "the press opened it" - ); + front.act("browse https://flights.test/", None); + assert!(front.look(&window, at).is_none()); + front.act("click", Some(&press)); + assert!(front.look(&sheet, at).is_some(), "the press opened it"); front.next_step(); - assert!(front.opened_dialog, "the next step answers what it asks"); - front.act("click", true); - front.look(&sheet, at, true); + assert!(front.opened_dialog(), "the next step answers what it asks"); + front.act("click", Some(&press)); + front.look(&sheet, at); assert!( - front.opened_dialog, + front.opened_dialog(), "still its own within the step that works in it" ); front.next_step(); - assert!(!front.opened_dialog, "a step later, it is in the way"); - front.look(&sheet, at, true); + assert!(!front.opened_dialog(), "a step later, it is in the way"); + front.look(&sheet, at); assert!( - !front.opened_dialog, + !front.opened_dialog(), "and it does not become the task's again" ); // A scroll or the run's own housekeeping opens no dialog of the task's. for action in ["scroll", "click (clear distraction)", "click (dismiss)"] { let mut front = Front::default(); - front.act("browse https://flights.test/", false); - front.look(&window, at, true); - front.act(action, true); - assert!(front.look(&sheet, at, true).is_none(), "{action}"); - assert!(!front.opened_dialog, "{action}"); + front.act("browse https://flights.test/", None); + front.look(&window, at); + front.act(action, Some(&press)); + assert!(front.look(&sheet, at).is_none(), "{action}"); + assert!(!front.opened_dialog(), "{action}"); } // Opening an address leaves what was in front behind. let mut front = Front::default(); - front.act("browse https://flights.test/", false); - front.look(&window, at, true); - front.act("click", true); - front.look(&sheet, at, true); - front.act("browse https://flights.test/next", false); - assert!(!front.opened_dialog); + front.act("browse https://flights.test/", None); + front.look(&window, at); + front.act("click", Some(&press)); + front.look(&sheet, at); + front.act("browse https://flights.test/next", None); + assert!(!front.opened_dialog()); +} + +#[test] +fn a_dialog_at_a_runs_first_look_is_the_tasks_only_when_the_run_before_left_it() { + use crate::agentic::flow::front::Front; + let at = Some("https://flights.test/"); + let sheet = page_at("sheet", 3); + // A pop-up the page opened itself, at a rescue's first look. + let mut front = Front::new(false); + assert!(front.look(&sheet, at).is_none()); + assert!( + !front.opened_dialog(), + "the page's own, cleared like any other" + ); + // The dialog the task's run before left open. + let mut front = Front::new(true); + assert!(front.look(&sheet, at).is_some()); + assert!(front.opened_dialog(), "the task's current stage"); +} + +/// A sheet of ten grid cells named by bare numbers, under `month` when +/// given. +fn days_at(month: Option<&str>) -> Screen { + let mut screen = page_at("sheet", 3); + screen.candidates.extend((1..=10).map(|day: u32| { + node( + &day.to_string(), + "gridcell", + &["Click"], + &["main"], + f64::from(100 + day), + ) + })); + screen.context.extend(month.map(str::to_owned)); + screen +} + +#[test] +fn turning_a_calendars_month_answers_nothing_it_asks() { + // An arrow pressed is no day chosen: the calendar still asks for one, + // and nothing it covers may be pressed through it yet. + use crate::agentic::flow::front::Front; + let at = Some("https://flights.test/"); + let calendar = days_at(Some("October 2026")); + let mut front = Front::default(); + front.act("browse https://flights.test/", None); + front.look(&page_at("window", 0), at); + front.act( + "click", + Some(&node("Departure", "button", &["Click"], &["main"], 0.0)), + ); + assert!(front.look(&calendar, at).is_some(), "the press opened it"); + front.act( + "click", + Some(&node("Next month", "button", &["Click"], &["main"], 90.0)), + ); + front.look(&calendar, at); + assert!(!front.served_calendar(), "a month turned is no day chosen"); + assert!(front.opened_dialog()); + front.act( + "click", + Some(&node("5", "gridcell", &["Click"], &["main"], 105.0)), + ); + front.look(&calendar, at); + assert!(front.served_calendar(), "a day chosen serves its field"); +} + +#[test] +fn a_grid_of_bare_numbers_is_a_calendar_only_beside_a_month() { + // A seat map's cells are numbers in a grid too: pressed in, it is no + // calendar that has served its field and may be closed for a press + // behind it. + use crate::agentic::flow::front::Front; + let at = Some("https://cinema.test/"); + for (month, calendar) in [(None, false), (Some("October 2026"), true)] { + let grid = days_at(month); + let mut front = Front::default(); + front.act("browse https://cinema.test/", None); + front.look(&page_at("window", 0), at); + front.act( + "click", + Some(&node("Select seats", "button", &["Click"], &["main"], 0.0)), + ); + front.look(&grid, at); + front.act( + "click", + Some(&node("5", "gridcell", &["Click"], &["main"], 105.0)), + ); + front.look(&grid, at); + assert_eq!(front.served_calendar(), calendar, "{month:?}"); + } +} + +#[test] +fn a_calendar_reads_its_days_from_labels_that_name_their_month() { + // Day cells named whole dates, the day not first ("Thu Oct 01 2026", + // "Choose Thursday, October 22nd, 2026"), are a calendar's days too. + use crate::agentic::flow::front::Front; + let at = Some("https://stays.test/"); + let mut calendar = page_at("sheet", 3); + calendar.candidates.extend((1..=7).map(|day: u32| { + node( + &format!("Choose Thursday, October {day}th, 2026"), + "button", + &["Click"], + &["main"], + f64::from(100 + day), + ) + })); + let mut front = Front::default(); + front.act("browse https://stays.test/", None); + front.look(&page_at("window", 0), at); + front.act( + "click", + Some(&node("Check-in", "button", &["Click"], &["main"], 0.0)), + ); + front.look(&calendar, at); + // An arrow named "Next" and described "next month" turns the month. + let next = Candidate { + description: Some("next month".to_owned()), + ..node("Next", "button", &["Click"], &["main"], 90.0) + }; + front.act("click", Some(&next)); + front.look(&calendar, at); + assert!(!front.served_calendar(), "a month turned is no day chosen"); + front.act( + "click", + Some(&node( + "Choose Thursday, October 2th, 2026", + "button", + &["Click"], + &["main"], + 102.0, + )), + ); + front.look(&calendar, at); + assert!(front.served_calendar(), "its days are read as a calendar's"); +} + +#[test] +fn a_search_box_typed_into_counts_only_while_it_shows_uncovered() { + use crate::agentic::flow::act::still_shows; + let field = node("Search Lenskart", "textbox", &["SetValue"], &["main"], 0.0); + let mut screen = page_at("sheet", 0); + screen.candidates.push(field.clone()); + assert!(still_shows(&screen, &field)); + screen.candidates.last_mut().unwrap().states = vec!["covered".to_owned()]; + assert!( + !still_shows(&screen, &field), + "a dialog over it takes the keys" + ); + screen.candidates.pop(); + assert!(!still_shows(&screen, &field), "nor once it is gone"); +} + +#[test] +fn a_run_that_never_looked_hands_on_the_dialog_it_was_left() { + use crate::agentic::flow::front::Front; + assert!(Front::new(true).left_open(), "nothing changed in front"); + assert!(!Front::new(false).left_open()); + let mut front = Front::new(true); + front.look(&page_at("window", 0), Some("https://flights.test/")); + assert!(!front.left_open(), "the window is in front once it looked"); } #[tokio::test] diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/enter_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/enter_tests.rs index f05aced7..58dcf204 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/enter_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/enter_tests.rs @@ -426,3 +426,61 @@ async fn a_control_named_by_the_slot_opens_its_box_only_when_jev_agrees() { refused.result.steps[0].actions ); } + +#[test] +fn a_short_place_word_names_its_box_only_as_a_labels_first_word() { + // Live, a flight form drew its place boxes as buttons ("From DEL", + // "To BLR"), and the "to" box was never opened: "to" is too short a + // word to look for anywhere in a label ("Tap to add a return date"). + use super::enter::named_opener; + let slots = vec![tinycomputer_bus::Slot { + slot: "to".to_owned(), + text: "Mumbai".to_owned(), + }]; + let screen = Screen { + app: "browser".to_owned(), + window: None, + surface: "window".to_owned(), + candidates: vec![ + node( + "From DEL, Delhi Airport India", + "button", + &["Click"], + &["form"], + 0.0, + ), + node( + "Return Tap to add a return date", + "button", + &["Click"], + &["form"], + 1.0, + ), + node( + "To BLR, Bengaluru Airport India", + "button", + &["Click"], + &["form"], + 2.0, + ), + ], + context: Vec::new(), + unexplored: Vec::new(), + text_nodes: Vec::new(), + }; + let opener = named_opener(&screen, &slots, &BTreeSet::from([0])).unwrap(); + assert_eq!( + opener.name.as_deref(), + Some("To BLR, Bengaluru Airport India") + ); + // However the surface spells the role. + let mut screen = screen; + for candidate in &mut screen.candidates { + candidate.role = "Button".to_owned(); + } + let opener = named_opener(&screen, &slots, &BTreeSet::from([0])).unwrap(); + assert_eq!( + opener.name.as_deref(), + Some("To BLR, Bengaluru Airport India") + ); +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/grounding_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/grounding_tests.rs index fbd86b1a..65c80b12 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/grounding_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/grounding_tests.rs @@ -58,6 +58,75 @@ async fn a_large_screen_is_narrowed_by_region_before_choosing() { ); } +#[test] +fn a_small_region_is_never_cut_from_a_crowded_knockout() { + use crate::agentic::flow::ground::{Regions, knockout_groups}; + + let member = |name: String, region: &str| { + node( + &name, + "button", + &["Click"], + &["window \"Flights\"", region], + 10.0, + ) + }; + let routes = (0..450) + .map(|index| member(format!("Route {index}"), "list \"Popular routes\"")) + .collect::>(); + let airports = (0..5) + .map(|index| member(format!("Airport {index}"), "listbox \"Airports\"")) + .collect::>(); + let pool = [routes.clone(), airports.clone()].concat(); + let regions: Regions = vec![ + ("list \"Popular routes\"".to_owned(), routes), + ("listbox \"Airports\"".to_owned(), airports), + ]; + let groups = knockout_groups(&pool, Some(®ions)); + assert!(groups.len() <= ask::CAP, "{}", groups.len()); + let offered = groups + .iter() + .flat_map(|(_, group)| group) + .filter_map(|candidate| candidate.name.clone()) + .collect::>(); + for index in 0..5 { + assert!( + offered.contains(&format!("Airport {index}")), + "the small region, last on the page, is offered whole" + ); + } + // With no regions, the knockout takes the first groups in page order. + assert_eq!(knockout_groups(&pool, None).len(), ask::CAP); + + // Many small regions out of view give up their chunks before a region in + // view: live, a travellers pop-up drawn after twenty regions of links out + // of view lost its "Done". + let mut regions: Regions = (0..30) + .map(|index| { + let mut link = member(format!("Link {index}"), "contentinfo"); + link.states = vec!["offscreen".to_owned()]; + (format!("list {index}"), vec![link]) + }) + .collect(); + regions.push(( + "dialog \"Travellers\"".to_owned(), + vec![member("Done".to_owned(), "dialog \"Travellers\"")], + )); + let pool = regions + .iter() + .flat_map(|(_, members)| members.clone()) + .collect::>(); + let groups = knockout_groups(&pool, Some(®ions)); + assert_eq!(groups.len(), ask::CAP); + assert!( + groups + .iter() + .flat_map(|(_, group)| group) + .any(|candidate| candidate.name.as_deref() == Some("Done")), + "the region in view keeps its chunk" + ); +} + #[tokio::test] async fn one_crowded_region_falls_back_to_a_knockout() { let run = run_with( diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/pick_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/pick_tests.rs index 7cb730f3..d32efdda 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/pick_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/pick_tests.rs @@ -32,6 +32,37 @@ async fn pick_ranks_a_measurable_criterion_exactly_and_opens_the_winner() { } } +#[tokio::test] +async fn pick_ranks_nearness_to_a_number_by_distance() { + // Live, a store's sizes 9 and 10 were sold out, and "closest to 9", + // judged item by item, took none of 6, 7, and 8. + let run = run( + App::with(|sim| { + sim.results = vec![ + ("6", "₹255", "in stock"), + ("7", "₹255", "3 left"), + ("8", "₹255", "2 left"), + ]; + }), + json!({"app": "Mail", "steps": [ + {"pick": {"from": "the size options", "by": "closest to 9", "into": "size"}} + ]}), + ) + .await; + assert_eq!(run.app.sim().picked, ["@s:select-3"]); + assert!( + run.result.steps[0].note.contains("ranked"), + "{}", + run.result.steps[0].note + ); + assert!( + !run.requests + .iter() + .any(|request| request.questions.contains_key("record")), + "nearness to a number needs no judgement" + ); +} + /// Says an item belongs to the list picked from only when it shows `brand`. fn belongs_when(brand: &'static str) -> impl Fn(&str, &Question, &Sim) -> Option { move |id, question, _| { @@ -120,6 +151,29 @@ async fn pick_ranks_the_list_that_has_prices_not_the_longest_one() { assert!(run.result.steps[0].note.contains("ranked")); } +#[tokio::test] +async fn pick_ranks_a_list_it_can_open_not_bare_fares() { + // Live, "the cheapest flight" ranked a list of bare fares and took + // "₹ 6,054", which had nothing to open. + let app = flights(); + app.sim().fare_chips = 8; + let run = run( + app, + json!({"app": "Mail", "steps": [ + {"pick": {"from": "the flight results", "by": "lowest price", "into": "flight"}} + ]}), + ) + .await; + assert_eq!( + run.result.stop, + FlowStopReason::Completed, + "{:?}", + run.result.steps + ); + assert_eq!(run.app.sim().picked, ["@s:select-1"]); + assert!(run.result.vars["flight"].starts_with("IndiGo")); +} + #[tokio::test] async fn pick_asks_jev_when_the_criterion_needs_judgement() { let run = run_with( diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/picker_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/picker_tests.rs new file mode 100644 index 00000000..218cb7d4 --- /dev/null +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/picker_tests.rs @@ -0,0 +1,231 @@ +//! Pop-ups and calendars a step works with: a closer that goes with its +//! pop-up (a "Close" or a consent bar's "Accept all"), a calendar the task picked in closed for a press behind it, in +//! its own step or a later one, and a date picked from a calendar already +//! open. + +use super::*; + +/// Answers that press the toast's "Close" and never judge the step done. +fn press_close(id: &str, question: &Question, _: &Sim) -> Option { + match id { + "done" | "blocked" => Some(noul(0.05)), + "move" => Some(pick(question, "activate", 0.9)), + _ if id == "target" || id == "region" || id.starts_with("group_") => { + Some(pick(question, "Close", 0.9)) + } + _ => None, + } +} + +#[tokio::test] +async fn a_closer_that_goes_with_its_pop_up_ends_a_step_closing_it() { + // Live, an offer pop-up drawn without a dialog's role closed at the + // first press, and the step stalled: the judge could not tell whether + // "declining optional cookies" was done with no cookie banner shown. + let toast = || { + App::with(|sim| { + sim.quirks.insert(Quirk::PromoToast); + }) + }; + let run = run_with( + toast(), + json!({"app": "Mail", "steps": ["close any login or offer pop-up, declining optional cookies"]}), + |request| request.disabled_loops.push(FlowLoop::Attention), + press_close, + ) + .await; + assert_eq!(run.result.stop, FlowStopReason::Completed); + assert_eq!(run.app.sim().clicks, ["Close"]); + assert!( + run.result.steps[0] + .note + .contains("closed with what it was on"), + "{}", + run.result.steps[0].note + ); + let unrelated = run_with( + toast(), + json!({"app": "Mail", "steps": ["archive the message"]}), + |request| { + request.disabled_loops.push(FlowLoop::Attention); + request.max_actions = 1; + }, + press_close, + ) + .await; + assert_ne!( + unrelated.result.stop, + FlowStopReason::Completed, + "a step that closes nothing is not finished by a closer" + ); +} + +/// Answers that press the cookie bar's "Accept all" and never judge the +/// step done. +fn press_accept(id: &str, question: &Question, _: &Sim) -> Option { + match id { + "done" | "blocked" => Some(noul(0.05)), + "move" => Some(pick(question, "activate", 0.9)), + _ if id == "target" || id == "region" || id.starts_with("group_") => { + Some(pick(question, "Accept all", 0.9)) + } + _ => None, + } +} + +#[tokio::test] +async fn accepting_a_cookie_bar_ends_the_step_that_accepts_it() { + // A consent bar drawn without a dialog's role closes on "Accept all" + // as surely as a pop-up on "Close": the press going with the bar is + // the evidence the judge cannot see. + let run = run_with( + App::with(|sim| { + sim.quirks.insert(Quirk::CookieBar); + }), + json!({"app": "Mail", "steps": ["accept the cookie banner"]}), + |request| request.disabled_loops.push(FlowLoop::Attention), + press_accept, + ) + .await; + assert_eq!(run.result.stop, FlowStopReason::Completed); + assert_eq!(run.app.sim().clicks, ["Accept all"]); + assert!( + run.result.steps[0] + .note + .contains("closed with what it was on"), + "{}", + run.result.steps[0].note + ); +} + +/// Answers that open the calendar, pick a day, then press "Find flights", +/// judging the step done once that press went through. +fn pick_then_find(id: &str, question: &Question, sim: &Sim) -> Option { + let found = sim.clicks.iter().any(|click| click == "Find flights"); + let next = if sim.fields.contains_key("Departure") { + "Find flights" + } else if sim + .booking + .as_ref() + .is_some_and(|booking| booking.calendar.is_some()) + { + "12 September 2026" + } else { + "Departure" + }; + match id { + "done" => Some(noul(if found { 0.95 } else { 0.05 })), + "blocked" => Some(noul(0.05)), + "move" => Some(pick(question, "activate", 0.9)), + _ if id == "target" || id == "region" || id.starts_with("group_") => { + Some(pick(question, next, 0.9)) + } + _ => None, + } +} + +#[tokio::test] +async fn a_calendar_the_task_picked_in_is_closed_for_a_press_behind_it() { + // Live, a calendar the task opened stayed in front of the guests and + // Search buttons once both dates were picked, and every press behind + // it was refused as lying behind the task's own dialog. + let run = run_with( + App::with(|sim| { + sim.booking = Some(Booking::default()); + sim.quirks.insert(Quirk::CalendarStaysOpen); + }), + json!({"app": "Mail", "steps": ["pick 12 September 2026 as the departure date, then press Find flights"]}), + |request| request.disabled_loops.push(FlowLoop::Attention), + pick_then_find, + ) + .await; + assert_eq!( + run.result.stop, + FlowStopReason::Completed, + "{:?}", + run.result.steps + ); + let sim = run.app.sim(); + assert_eq!(sim.presses, ["escape"]); + assert_eq!( + sim.clicks, + ["Departure", "12 September 2026", "Find flights"] + ); + assert!( + run.result.steps[0] + .actions + .iter() + .any(|action| action.action == "press escape (uncover)") + ); +} + +#[tokio::test] +async fn a_calendar_a_step_before_picked_in_is_closed_for_a_press_behind_it() { + // Live, the dates were picked a step each, and the calendar left open + // in front of the Search button was handed back to the page with the + // next step: every press behind it was refused, and no step closed it. + let run = run_with( + App::with(|sim| { + sim.booking = Some(Booking::default()); + sim.quirks.insert(Quirk::CalendarStaysOpen); + }), + json!({"app": "Mail", "steps": [ + "pick 12 September 2026 as the departure date", + "press Find flights" + ]}), + |request| request.disabled_loops.push(FlowLoop::Attention), + |id, question, sim| { + if id != "done" { + return pick_then_find(id, question, sim); + } + let finished = if text_of(question, "step").contains("find flights") { + sim.clicks.iter().any(|click| click == "Find flights") + } else { + sim.fields.contains_key("Departure") + }; + Some(noul(if finished { 0.95 } else { 0.05 })) + }, + ) + .await; + assert_eq!( + run.result.stop, + FlowStopReason::Completed, + "{:?}", + run.result.steps + ); + let sim = run.app.sim(); + assert_eq!(sim.presses, ["escape"]); + assert_eq!( + sim.clicks, + ["Departure", "12 September 2026", "Find flights"] + ); +} + +#[tokio::test] +async fn a_date_whose_calendar_is_open_is_picked_without_pressing_its_button() { + // Live, a check-out date's step found the calendar open from the + // check-in, pressed its button, which closed it, four times, and never + // picked the day. + let run = run_with( + App::with(|sim| { + sim.booking = Some(Booking { + calendar: Some(8), + ..Booking::default() + }); + sim.quirks.insert(Quirk::CalendarStaysOpen); + }), + json!({"app": "Mail", "steps": [{"enter": {"departure date": "12 September 2026"}}]}), + |request| request.disabled_loops.push(FlowLoop::Attention), + |_, _, _| None, + ) + .await; + assert_eq!( + run.result.stop, + FlowStopReason::Completed, + "{:?}", + run.result.steps + ); + let sim = run.app.sim(); + assert_eq!(sim.clicks, ["12 September 2026"]); + assert_eq!(sim.fields["Departure"], "12 September 2026"); +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/places.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/places.rs index 29da5249..41753e44 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/places.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/places.rs @@ -47,8 +47,16 @@ pub(super) struct Places { /// still to come, as a ride app's did live ("Allow location access", /// "Search in a different city"). pub(super) starters: bool, + /// Whether each box shows only once its own button is pressed ("From + /// …", "To …"), one at a time, as a flight form draws them. + pub(super) behind_buttons: bool, + /// The box whose button was pressed last (`behind_buttons`). + pub(super) door: Option, } +/// The buttons that show the boxes of a form drawn `behind_buttons`. +const DOORS: [&str; 2] = ["From", "To"]; + /// The places suggested for `typed`: each one that holds every typed word. fn suggested(typed: &str) -> Vec<&'static str> { let words = typed @@ -77,6 +85,25 @@ pub(super) fn places_widget( ) { let form = [root, "group \"Get a ride\""]; for (index, name) in PLACE_BOXES.iter().enumerate() { + if places.behind_buttons { + // The button shows a place once one is picked, not what is typed. + let shown = sim + .fields + .get(*name) + .filter(|_| places.picked.contains(*name)) + .cloned() + .unwrap_or_else(|| "Select a place".to_owned()); + candidates.push(node( + &format!("{} {shown}", DOORS[index]), + "button", + &["Click"], + &form, + 190.0 + 40.0 * f64::from(u8::try_from(index).unwrap()), + )); + if places.door.as_deref() != Some(*name) { + continue; + } + } let mut field = node( name, "textbox", @@ -147,11 +174,12 @@ pub(super) fn await_place_rows(sim: &mut Sim) -> bool { } /// Closes the open list, dropping its box's text unless a suggestion was -/// picked for it. +/// picked for it; a box shown behind its button closes with it. pub(super) fn drop_unpicked(sim: &mut Sim) { let Some(places) = sim.places.as_mut() else { return; }; + places.door = None; if let Some(open) = places.open.take() && !places.picked.contains(&open) { @@ -165,6 +193,27 @@ pub(super) fn drop_unpicked(sim: &mut Sim) { /// whichever that is; a press anywhere else but the box moves the focus on, /// so the list closes and drops the box's unpicked text, as a page does. pub(super) fn press_place(sim: &mut Sim, name: &str) -> bool { + let door = sim + .places + .as_ref() + .filter(|places| places.behind_buttons) + .and_then(|_| { + DOORS + .iter() + .position(|door| name.starts_with(&format!("{door} "))) + }); + if let Some(index) = door { + // Another box's button moves the focus on, as a press anywhere else + // does: the open list closes and drops its box's unpicked text. + let open = sim.places.as_ref().and_then(|places| places.open.clone()); + if open.as_deref() != Some(PLACE_BOXES[index]) { + drop_unpicked(sim); + } + if let Some(places) = sim.places.as_mut() { + places.door = Some(PLACE_BOXES[index].to_owned()); + } + return true; + } let Some(open) = sim.places.as_ref().and_then(|places| places.open.clone()) else { return false; }; @@ -183,6 +232,7 @@ pub(super) fn press_place(sim: &mut Sim, name: &str) -> bool { if let Some(places) = sim.places.as_mut() { places.picked.insert(open.clone()); places.open = None; + places.door = None; } sim.fields.insert(open, place.to_owned()); true diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/screens.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/screens.rs index 5779d16c..fe1476a4 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/screens.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/screens.rs @@ -131,6 +131,7 @@ pub(super) struct Booking { /// What pressing `name` does to the booking form. pub(super) fn press_booking(sim: &mut Sim, name: &str) { + let stays_open = sim.has(Quirk::CalendarStaysOpen); let Some(booking) = sim.booking.as_mut() else { return; }; @@ -142,6 +143,8 @@ pub(super) fn press_booking(sim: &mut Sim, name: &str) { sim.fields .insert("Destination".to_owned(), "Srinagar, SXR".to_owned()); } + // A calendar that stays open is closed by its own button again. + "Departure" if stays_open && booking.calendar.is_some() => booking.calendar = None, "Departure" => booking.calendar = Some(8), "Next Month" => booking.calendar = booking.calendar.map(|month| (month + 1) % 12), // The calendar's own aggregated-label container also ends with @@ -150,7 +153,9 @@ pub(super) fn press_booking(sim: &mut Sim, name: &str) { // a bug that let production code ground and press that container // instead of a day must not be able to pass this simulated test. day if booking.calendar.is_some() && is_single_day_label(day) => { - booking.calendar = None; + if !stays_open { + booking.calendar = None; + } sim.fields.insert("Departure".to_owned(), day.to_owned()); } _ => {} @@ -257,6 +262,13 @@ pub(super) fn booking_widget( } } candidates.push(node("Departure", "button", &["Click"], &widget, 120.0)); + if sim.has(Quirk::CalendarStaysOpen) { + let mut find = node("Find flights", "button", &["Click"], &widget, 200.0); + if booking.calendar.is_some() { + find.states = vec!["covered".to_owned()]; + } + candidates.push(find); + } if let Some(month) = booking.calendar { // The date field's own label lists the whole open calendar. let listing = (1..=28) @@ -380,6 +392,19 @@ pub(super) fn result_cards( ..Candidate::default() }); } + for chip in 0..sim.fare_chips { + text_nodes.push(Candidate { + role: "text".to_owned(), + value: Some(json!(format!("₹ {}", 1_000 + chip))), + path: vec![ + root.to_owned(), + "list \"Fares\"".to_owned(), + format!("listitem #{}", chip + 1), + ], + order: 300 + chip * 10, + ..Candidate::default() + }); + } for day in 0..sim.date_strip { let path = vec![ root.to_owned(), @@ -421,6 +446,11 @@ pub(super) fn overlays(sim: &Sim, root: &str, candidates: &mut Vec) { candidates.push(node("Close", "button", &["Click"], &toast, 700.0)); candidates.push(node("Learn more", "link", &["Click"], &toast, 720.0)); } + if sim.has(Quirk::CookieBar) { + let bar = [root, "region \"We use cookies\""]; + candidates.push(node("Accept all", "button", &["Click"], &bar, 700.0)); + candidates.push(node("Cookie policy", "link", &["Click"], &bar, 720.0)); + } if sim.has(Quirk::ConsentBanner) { for candidate in candidates.iter_mut() { candidate.states = vec!["covered".to_owned()]; diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs index 645678c5..f22ef657 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs @@ -35,6 +35,12 @@ pub(super) enum Quirk { DisabledArchive, /// A promo toast with a Close button sits over the page until closed. PromoToast, + /// A cookie bar drawn without a dialog's role, its "Accept all" button + /// the way out, shows until accepted; like the promo toast, it covers + /// no control. + CookieBar, + /// The inbox's search, once open, fills the window as a sheet. + SearchSheet, /// A consent banner lies over the page as a popover: every other click /// is refused as covered, Escape leaves it, and its "Allow Selection" /// or "Allow all" closes it. @@ -56,6 +62,9 @@ pub(super) enum Quirk { SearchBehindLink, /// The search behind "Search mail" shows. SearchOpen, + /// The booking calendar is a dialog that stays open once a day is + /// picked, covering a "Find flights" button, until Escape closes it. + CalendarStaysOpen, } #[derive(Debug, Default)] @@ -76,6 +85,9 @@ pub(super) struct Sim { pub(super) selected_result: Option, /// Days in a date strip above the results, a longer list than they are. pub(super) date_strip: usize, + /// Bare fares in a list above the results, cheaper than every flight, + /// with nothing to open. + pub(super) fare_chips: usize, pub(super) extra_buttons: usize, /// A booking form with an autocomplete destination and a calendar. pub(super) booking: Option, @@ -240,9 +252,8 @@ impl App { } overlays(&sim, &root, &mut candidates); let text_nodes = result_cards(&sim, &root, &mut candidates); - let mut surface = "window".to_owned(); + let surface = surface_of(&sim); if sim.obstacle { - surface = "sheet".to_owned(); obstacle_sheet(&sim, &mut candidates); } Screen { @@ -259,6 +270,23 @@ impl App { } } +/// What the simulated page shows in front: the obstacle's sheet, a +/// calendar that stays open as a dialog, or the window. +fn surface_of(sim: &Sim) -> String { + if sim.obstacle || (sim.has(Quirk::SearchSheet) && sim.has(Quirk::SearchOpen)) { + "sheet".to_owned() + } else if sim.has(Quirk::CalendarStaysOpen) + && sim + .booking + .as_ref() + .is_some_and(|booking| booking.calendar.is_some()) + { + "dialog".to_owned() + } else { + "window".to_owned() + } +} + /// The reply to a click on `name` that something on the simulated page /// refuses or takes over, or `None` when the click goes through: an /// unclickable page, a consent banner whose own buttons close it, or a @@ -290,6 +318,15 @@ fn refused_click(sim: &mut Sim, name: &str) -> Option { } return Some(covered("consent")); } + if name == "Find flights" + && sim.has(Quirk::CalendarStaysOpen) + && sim + .booking + .as_ref() + .is_some_and(|booking| booking.calendar.is_some()) + { + return Some(covered("calendar")); + } sim.has(Quirk::Drawer).then(|| covered("drawer")) } @@ -355,6 +392,10 @@ impl AgentBackend for App { sim.quirks.remove(&Quirk::PromoToast); return DesktopResponse::ok("click", json!({})); } + if name == "Accept all" && sim.has(Quirk::CookieBar) { + sim.quirks.remove(&Quirk::CookieBar); + return DesktopResponse::ok("click", json!({})); + } if sim.page().is_some() { press_shop(&mut sim, &name); return sim.located("click"); @@ -459,6 +500,11 @@ impl AgentBackend for App { sim.obstacle = false; sim.quirks.remove(&Quirk::Drawer); drop_unpicked(&mut sim); + if sim.has(Quirk::CalendarStaysOpen) + && let Some(booking) = sim.booking.as_mut() + { + booking.calendar = None; + } } _ => {} } diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/suggestion_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/suggestion_tests.rs index 93a0c9b4..9e33338f 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/suggestion_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/suggestion_tests.rs @@ -456,3 +456,102 @@ fn a_box_that_appears_beside_the_suggestions_is_never_taken_for_one() { [row] ); } + +#[test] +fn a_matching_row_already_shown_counts_only_in_view_and_in_front() { + // A place box's own list is in view and in front: live, a footer link + // "hotels in Goa" at the foot of the page was pressed as the place typed. + // A new row still counts out of view, as one its list scrolled away. + use super::steps::fresh_rows; + let place_box = node("Where to?", "textbox", &["Click", "SetValue"], &[], 0.0); + let row = |name: &str, role: &str, states: &[&str], y: f64| { + let mut row = node(name, role, &["Click"], &[], y); + row.states = states.iter().map(|state| (*state).to_owned()).collect(); + row + }; + let before = vec![ + place_box.clone(), + row("Goa beaches", "button", &[], 1.0), + row("hotels in Goa", "link", &["offscreen"], 2.0), + row("Goa villas", "link", &["covered"], 3.0), + ]; + let shown = before + .iter() + .map(|candidate| (candidate.role.as_str(), candidate.name.as_deref())) + .collect::>(); + let mut candidates = before.clone(); + candidates.push(row( + "Goa International Airport", + "option", + &["offscreen"], + 4.0, + )); + let screen = Screen { + app: "browser".to_owned(), + window: None, + surface: "window".to_owned(), + candidates, + context: Vec::new(), + unexplored: Vec::new(), + text_nodes: Vec::new(), + }; + let fresh = fresh_rows(&screen, &shown, &place_box, "Goa", &[], true); + assert_eq!( + fresh + .iter() + .filter_map(|candidate| candidate.name.as_deref()) + .collect::>(), + ["Goa beaches", "Goa International Airport"] + ); +} + +#[tokio::test] +async fn enter_opens_each_place_box_behind_its_own_button_in_turn() { + // Live, a flight form drew its place boxes as buttons ("From DEL", "To + // BLR"), each box showing only once its button is pressed. The "to" box + // was never opened, and the place was pressed in a link at the foot of + // the page instead. + let run = run_with( + App::with(|sim| { + sim.places = Some(Places { + behind_buttons: true, + ..Places::default() + }); + }), + json!({"app": "Mail", "steps": [ + {"enter": {"from": "Connaught Place", "to": "Indira Gandhi International Airport"}} + ]}), + |_| {}, + |id, question, sim| { + if id.starts_with("slot_") { + let purpose = text_of(question, "purpose"); + let field = if purpose.contains("the from ") { + "Pickup location" + } else { + "Dropoff location" + }; + return Some(pick(question, field, 0.9)); + } + suggesting(id, question, sim, 0.9) + }, + ) + .await; + assert_eq!( + run.result.stop, + FlowStopReason::Completed, + "{:?}", + run.result.steps + ); + let sim = run.app.sim(); + assert_eq!( + sim.fields["Pickup location"], + "Connaught Place New Delhi, Delhi, India" + ); + assert_eq!( + sim.fields["Dropoff location"], + "Indira Gandhi International Airport New Delhi, Delhi, India", + "{:?} {:?}", + run.result.steps[0].actions, + sim.clicks + ); +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/front.rs b/crates/tinycomputer-engine/src/agentic/flow/front.rs index 7e4676c4..c7103f53 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/front.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/front.rs @@ -2,12 +2,18 @@ //! there: a dialog the task opened is the flow's next stage, never cleared //! as a distraction or an obstacle, in this step or the next. -use super::view::Screen; +use super::{ + steps::names_a_month, + view::{Candidate, Screen}, +}; /// Controls a layer must cover, beyond what was covered before the press /// that opened it, before it counts as in front of the page. pub(super) const LAYER_COVERS: usize = 3; +/// Day cells nothing covers that make what is in front a calendar: a week. +const CALENDAR_DAYS: usize = 7; + /// The history note for a dialog left open by the run before a rescue. const LEFT_OPEN: &str = "a dialog the task opened before is still in front: it is the task's current stage, so work within it"; @@ -29,6 +35,26 @@ pub(super) enum Acted { Typed, } +/// Whose the dialog in front is, and how far the task has worked in it. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(super) enum Dialog { + /// None of the task's: the window, or what the page itself put in + /// front. + #[default] + Page, + /// The task's own, opened by its press (or left open by its run before) + /// and not pressed in since: the question it asks first, such as a + /// format or a quantity. Live, such a dialog was closed at the start of + /// the following step. + Asking, + /// The task's own, pressed in since it opened. + Answered, + /// The task's own, worked in by a step before this one: no longer its + /// question, and in the way of the next step, but in front until the + /// window is (a calendar left open after its day was chosen). + Served, +} + /// What was in front on the last look, and how the run's own actions /// brought it there. #[derive(Debug, Clone)] @@ -38,16 +64,13 @@ pub(super) struct Front { pub(super) surface: String, /// What the run did since the last look. pub(super) acted: Acted, - /// Whether the dialog in front was opened by the run's own press (a - /// question a booking or purchase button asks first, such as a format - /// or a quantity). Live, such a dialog was closed at the start of the - /// following step. - pub(super) opened_dialog: bool, - /// Whether the run has neither looked nor browsed yet: a dialog in - /// front at a run's first look, before any browsing, was left there by - /// the task's run before it (a rescue continues where that run - /// stopped), and counts as opened by the task. - pub(super) fresh: bool, + /// Whose the dialog in front is ([`Dialog`]). + pub(super) dialog: Dialog, + /// What the run's first look takes a dialog in front for, until it has + /// looked or browsed: the task's own when the task's run before this one + /// left its own dialog there (`true`; a rescue continues where that run + /// stopped), else the page's. + first_look: Option, /// How many controls something covered on the last look with nothing /// in front: what a sticky header always covers, which a layer a press /// opens must add to before it counts. @@ -56,11 +79,8 @@ pub(super) struct Front { /// either changed went to another page, and what covers that page is /// the page's own, not an answer the press asked for. pub(super) looked_at: (Option, Option), - /// Whether the run has pressed or typed inside the task's dialog since - /// it opened: such a dialog has served the step that worked in it, and - /// a later step finds it in the way rather than asking (a calendar left - /// open after its day was chosen). - pub(super) answered: bool, + /// Whether the last look showed a calendar ([`holds_calendar`]). + pub(super) calendar: bool, } /// What the run's own housekeeping does, never a press of the task's: a @@ -76,37 +96,61 @@ const HOUSEKEEPING: &[&str] = &[ impl Default for Front { fn default() -> Self { + Self::new(false) + } +} + +impl Front { + /// A run's front before its first look: `inherited` when the task's run + /// before this one left its own dialog in front. + pub(super) fn new(inherited: bool) -> Self { Self { surface: "window".to_owned(), acted: Acted::Nothing, - opened_dialog: false, - fresh: true, + dialog: Dialog::Page, + first_look: Some(inherited), covered_base: 0, looked_at: (None, None), - answered: false, + calendar: false, } } -} -impl Front { + /// Whether the dialog in front is the task's own and still its stage: + /// opened by its press, or left open by its run before, and not yet + /// handed back by a step that worked in it. + pub(super) fn opened_dialog(&self) -> bool { + matches!(self.dialog, Dialog::Asking | Dialog::Answered) + } + + /// Whether the run leaves the task's own dialog in front, for the task's + /// next run ([`tinycomputer_bus::RunFlowRequest::dialog_left_open`]): one + /// it opened, or the one its run before left there while it has not + /// looked since. A run that ends before its first look changed nothing + /// in front. + pub(super) fn left_open(&self) -> bool { + self.opened_dialog() || self.first_look == Some(true) + } + /// Notes one action, `action` as the run logs it (`click`, `fill …`, - /// `browse …`, `click (dismiss)`), on an element when `targeted`. + /// `browse …`, `click (dismiss)`), on `target` when it had one. /// /// Only the task's own presses and typing count: a scroll moves no /// question into view, and the run's housekeeping (a distraction - /// cleared, an undo) answers none. Opening an address or an application - /// leaves whatever was in front behind. - pub(super) fn act(&mut self, action: &str, targeted: bool) { + /// cleared, an undo) answers none. Nor does turning a calendar's month + /// (`turns_the_month`): the calendar still asks for its day. Opening an + /// address or an application leaves whatever was in front behind. + pub(super) fn act(&mut self, action: &str, target: Option<&Candidate>) { + let targeted = target.is_some(); if action.starts_with("browse ") { *self = Self { - fresh: false, + first_look: None, ..Self::default() }; return; } if action.starts_with("launch ") { *self = Self { - fresh: self.fresh, + first_look: self.first_look, ..Self::default() }; return; @@ -117,48 +161,50 @@ impl Front { let typing = action.starts_with("fill") || action.starts_with("type"); self.acted = if typing { Acted::Typed } else { Acted::Pressed }; } - if pressing && self.opened_dialog { - self.answered = true; + if pressing && self.dialog == Dialog::Asking && !target.is_some_and(turns_the_month) { + self.dialog = Dialog::Answered; } } + /// Whether the dialog in front is a calendar the task opened and has + /// pressed in since, in this step or one before: a picker that has + /// served its field, in the way of a press behind it rather than asking + /// anything. Live, a calendar stayed in front of the guests and Search + /// buttons once both dates were picked, a step each, and every press + /// behind it was refused. + pub(super) fn served_calendar(&self) -> bool { + matches!(self.dialog, Dialog::Answered | Dialog::Served) && self.calendar + } + /// Begins a step: a dialog the task opened and then worked in has /// served the step before, and is no longer the task's own; one the /// last step's last press opened still asks its question (a format /// dialog a booking button raised). pub(super) fn next_step(&mut self) { - if self.answered { - self.opened_dialog = false; - self.answered = false; + if self.dialog == Dialog::Answered { + self.dialog = Dialog::Served; } } /// Takes in a look at `screen`, at the address `location`: what is in /// front, and whether the task opened it. The note for the history - /// when the dialog in front became the task's. Only a `browsing` run - /// takes a dialog at its first look as the task's: a browser task's - /// first run always browses first, so a dialog then is a rescue's - /// inheritance, while an application can open with its own alert. - pub(super) fn look( - &mut self, - screen: &Screen, - location: Option<&str>, - browsing: bool, - ) -> Option<&'static str> { - let left_open = self.fresh && browsing; - self.fresh = false; + /// when the dialog in front became the task's. A dialog at a run's first + /// look is the task's only when the task's run before this one left its + /// own dialog in front ([`Front::new`]): live, a sign-up the page opened + /// on load, and a site's menu drawer, were taken for the task's at a + /// rescue's first look, and every move past them was refused. + pub(super) fn look(&mut self, screen: &Screen, location: Option<&str>) -> Option<&'static str> { + let left_open = self.first_look.take() == Some(true); + self.calendar = holds_calendar(screen); let front = self.front_of(screen, location); let note = if front == "window" { - self.opened_dialog = false; - self.answered = false; + self.dialog = Dialog::Page; None - } else if left_open && !self.opened_dialog { - self.opened_dialog = true; - self.answered = false; + } else if left_open && !self.opened_dialog() { + self.dialog = Dialog::Asking; Some(LEFT_OPEN) } else if self.acted != Acted::Nothing && self.surface == "window" { - self.opened_dialog = true; - self.answered = false; + self.dialog = Dialog::Asking; Some(OPENED) } else { None @@ -209,3 +255,77 @@ fn covered_count(screen: &Screen) -> usize { }) .count() } + +/// Whether `target` only turns a calendar's month ("next month", "Previous +/// month", or a "Next" described "next month"), which answers nothing the +/// calendar asks. +fn turns_the_month(target: &Candidate) -> bool { + [target.name.as_deref(), target.description.as_deref()] + .into_iter() + .flatten() + .any(|text| { + let words = text + .split(|character: char| !character.is_alphanumeric()) + .filter(|word| !word.is_empty()) + .map(str::to_lowercase) + .collect::>(); + let said = words.join(" "); + words.len() <= 4 && (said.contains("next month") || said.contains("previous month")) + }) +} + +/// The day of the month `word` names: 1 to 31, with a leading zero or an +/// ordinal ending ("01", "22nd"). +fn day_of(word: &str) -> Option { + let digits = ["st", "nd", "rd", "th"] + .iter() + .find_map(|ending| { + word.to_ascii_lowercase() + .strip_suffix(ending) + .map(str::to_owned) + }) + .unwrap_or_else(|| word.to_owned()); + digits + .parse::() + .ok() + .filter(|day| (1..=31).contains(day)) +} + +/// Whether `screen` shows a calendar: [`CALENDAR_DAYS`] or more day numbers +/// nothing covers, each beside a month's name ("1 September 2026", or "1" +/// described "Thu Oct 01 2026"), or a grid cell on a screen that names a +/// month somewhere: a seat map's or a table's cells are bare numbers too. +/// A short label that names a month holds its day anywhere ("Thu Oct 01 +/// 2026", "Choose Thursday, October 22nd, 2026"). +fn holds_calendar(screen: &Screen) -> bool { + let month_shown = screen + .candidates + .iter() + .chain(&screen.text_nodes) + .flat_map(|node| [node.name.as_deref(), node.description.as_deref()]) + .flatten() + .chain(screen.context.iter().map(String::as_str)) + .any(names_a_month); + screen + .candidates + .iter() + .filter(|candidate| { + let name = candidate.name.as_deref().unwrap_or_default(); + let day = name.split_whitespace().next().and_then(day_of).is_some() + || (names_a_month(name) + && name.split_whitespace().count() <= 6 + && name + .split(|character: char| !character.is_alphanumeric()) + .any(|word| day_of(word).is_some())); + let dated = (month_shown && candidate.role.eq_ignore_ascii_case("gridcell")) + || names_a_month(name) + || candidate.description.as_deref().is_some_and(names_a_month); + let covered = candidate + .states + .iter() + .any(|state| state.eq_ignore_ascii_case("covered")); + day && dated && !covered + }) + .count() + >= CALENDAR_DAYS +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/ground/mod.rs b/crates/tinycomputer-engine/src/agentic/flow/ground/mod.rs index 8afcb48e..e959d12c 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/ground/mod.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/ground/mod.rs @@ -32,6 +32,9 @@ mod decide; mod narrow; +#[cfg(test)] +pub(super) use narrow::knockout_groups; + use std::collections::BTreeMap; use tinyinference_decisions::{Answer, EvaluationRequest}; diff --git a/crates/tinycomputer-engine/src/agentic/flow/ground/narrow.rs b/crates/tinycomputer-engine/src/agentic/flow/ground/narrow.rs index f19d8a6e..f44d7e50 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/ground/narrow.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/ground/narrow.rs @@ -16,7 +16,7 @@ use crate::agentic::flow::{ view::{Candidate, Screen, distinct, label, named_first}, }; -use super::{AGREED, BRANCH_MARGIN, First, Grounded, Opening, split, winners}; +use super::{AGREED, BRANCH_MARGIN, First, Grounded, Opening, Regions, split, winners}; impl FlowRun<'_, B> { /// Picks the element of `pool` that serves `purpose`, or `None` when no @@ -127,25 +127,7 @@ impl FlowRun<'_, B> { /// round trip of its own. fn narrowing(&self, screen: &Screen, purpose: &str, pool: &[Candidate]) -> First { let regions = split(pool, 0).map(|(_, regions)| regions); - let aligned = regions.as_ref().map(|regions| { - regions - .iter() - .enumerate() - .flat_map(|(index, (_, members))| { - members - .chunks(CAP) - .map(move |chunk| (Some(index), chunk.to_vec())) - }) - .collect::>() - }); - let groups = match aligned { - Some(groups) if groups.len() <= CAP => groups, - _ => pool - .chunks(CAP) - .take(CAP) - .map(|chunk| (None, chunk.to_vec())) - .collect(), - }; + let groups = knockout_groups(pool, regions.as_ref()); let mut questions = Questions::default(); for (index, (_, group)) in groups.iter().enumerate() { questions = questions.with( @@ -328,3 +310,60 @@ impl FlowRun<'_, B> { kept } } + +/// The groups of one knockout over `pool`: along `regions` when the screen +/// has them, each region in [`CAP`]-sized chunks, at most [`CAP`] chunks in +/// all; otherwise the pool's first [`CAP`] chunks. When the regions need more +/// chunks than that, the regions with nothing in view give up their last +/// chunks first, the largest first, then the largest of the rest, so a small +/// region (the list in front, a dialog's rows) is offered whole: live, the +/// rows of an airport list that came after a page's 400 route links were cut +/// off unseen, and so was a travellers pop-up's "Done" drawn after twenty +/// regions of links out of view. +pub(in crate::agentic::flow) fn knockout_groups( + pool: &[Candidate], + regions: Option<&Regions>, +) -> Vec<(Option, Vec)> { + let Some(regions) = regions else { + return pool + .chunks(CAP) + .take(CAP) + .map(|chunk| (None, chunk.to_vec())) + .collect(); + }; + let mut kept = regions + .iter() + .map(|(_, members)| members.len().div_ceil(CAP)) + .collect::>(); + let unseen = regions + .iter() + .map(|(_, members)| { + !members + .iter() + .any(|member| denoise::tier(member) == denoise::Tier::InView) + }) + .collect::>(); + while kept.iter().sum::() > CAP { + let Some(largest) = kept + .iter() + .enumerate() + .filter(|(_, chunks)| **chunks > 0) + .max_by_key(|(index, chunks)| (unseen[*index], **chunks)) + .map(|(index, _)| index) + else { + break; + }; + kept[largest] -= 1; + } + regions + .iter() + .zip(kept) + .enumerate() + .flat_map(|(index, ((_, members), chunks))| { + members + .chunks(CAP) + .take(chunks) + .map(move |chunk| (Some(index), chunk.to_vec())) + }) + .collect() +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/look.rs b/crates/tinycomputer-engine/src/agentic/flow/look.rs index ddeef4b6..528e2d10 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/look.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/look.rs @@ -37,8 +37,7 @@ impl FlowRun<'_, B> { match observed { Ok(screen) => { self.blind_looks = 0; - let browsing = self.app.eq_ignore_ascii_case("browser"); - if let Some(note) = self.front.look(&screen, self.location.as_deref(), browsing) { + if let Some(note) = self.front.look(&screen, self.location.as_deref()) { self.history.push(note.to_owned()); } Ok(screen) diff --git a/crates/tinycomputer-engine/src/agentic/flow/mod.rs b/crates/tinycomputer-engine/src/agentic/flow/mod.rs index f0b4c6c5..4ab977ea 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/mod.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/mod.rs @@ -286,6 +286,9 @@ pub(super) struct FlowRun<'r, B> { /// field holding text the flow typed shows no choice the page made /// (`steps::already_holds`). pub(super) typed: BTreeSet, + /// The field the run's last action typed into, while nothing else has + /// acted since but waits: the focus is still in it. + pub(super) typed_last: Option, /// How much the run deliberates before acting on a decision. deliberation: Deliberation, /// Every framing's own answer to each question, under the original @@ -314,7 +317,8 @@ pub(super) struct FlowRun<'r, B> { /// Escape or a close that did not clear it once will not the next time. pub(super) step_cleared: BTreeSet, /// What is in front, and whether the run's own press put it there - /// (`front.rs`). + /// (`front.rs`); built knowing whether the task's run before this one + /// left its own dialog in front ([`RunFlowRequest::dialog_left_open`]). pub(in crate::agentic::flow) front: Front, } diff --git a/crates/tinycomputer-engine/src/agentic/flow/run.rs b/crates/tinycomputer-engine/src/agentic/flow/run.rs index a592a7f3..889b6a2a 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/run.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/run.rs @@ -132,6 +132,7 @@ impl<'r, B: AgentBackend + Sync> FlowRun<'r, B> { read: request.collected.keys().cloned().collect(), refused: BTreeSet::new(), typed: BTreeSet::new(), + typed_last: None, deliberation: request.deliberation, ballots: BTreeMap::new(), asked: BTreeMap::new(), @@ -140,7 +141,7 @@ impl<'r, B: AgentBackend + Sync> FlowRun<'r, B> { expecting: None, step_location: None, step_cleared: BTreeSet::new(), - front: Front::default(), + front: Front::new(request.dialog_left_open), } } @@ -346,6 +347,7 @@ impl<'r, B: AgentBackend + Sync> FlowRun<'r, B> { actions: self.actions, metrics: self.metrics, trace: self.trace, + dialog_left_open: self.front.left_open(), }, ) } diff --git a/crates/tinycomputer-engine/src/agentic/flow/steps/choose.rs b/crates/tinycomputer-engine/src/agentic/flow/steps/choose.rs index 3fcb3c54..001b87a1 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/steps/choose.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/steps/choose.rs @@ -15,9 +15,11 @@ use crate::agentic::flow::{ use super::{ LOCATE_FLOOR, + date::looks_like_date, matching::{ - already_chosen, already_holds, clickable, closest, editable, held_text, is_checked, - is_one_option, lists_more_than, mentions, one_option, plainest, redacted, within, + already_chosen, already_holds, clickable, closest, date_shown_in, editable, held_text, + is_checked, is_one_option, lists_more_than, mentions, one_option, plainest, redacted, + within, }, }; @@ -166,22 +168,7 @@ impl FlowRun<'_, B> { if !private && let Some(ended) = self.made_already(&screen, what, option, attempt) { return Ok(ended); } - let pool = clickable(&screen.candidates) - .into_iter() - .filter(|candidate| !is_destructive(candidate, &screen, &self.stop_before)) - .collect::>(); - // A field that holds the typed option is where it was typed, - // not one of the options it offers. - let pool = closest( - pool.into_iter() - .filter(|candidate| { - mentions(candidate, option) - && !editable(candidate) - && !lists_more_than(candidate, option) - }) - .collect(), - ); - let pool = within(pool, what); + let pool = option_pool(&screen, what, option, into_focus, &self.stop_before); // Matches that all name one option leave nothing to judge; a // private option is never judged, since Jev is not told it. An // option no control names is a description ("the lowest fare"), @@ -275,6 +262,7 @@ impl FlowRun<'_, B> { (attempt == 0) .then(|| already_holds(screen, option, &self.typed)) .flatten() + .or_else(|| date_shown_in(screen, what, option)) .map(|holder| format!("{} already shows {option:?}", label(&holder))) })?; self.history.push(shown); @@ -310,3 +298,49 @@ impl FlowRun<'_, B> { .filter(|grounded| grounded.confidence >= LOCATE_FLOOR)) } } + +/// The controls on `screen` that could be `option` in `what`: pressable and +/// not irreversible, naming the option, no box it was typed into, the +/// closest matches, within `what` when the page says where; for an `enter` +/// value with no box (`into_focus` unset), only those the screen shows. +fn option_pool( + screen: &Screen, + what: &str, + option: &str, + into_focus: bool, + stop_before: &[String], +) -> Vec { + let pool = clickable(&screen.candidates) + .into_iter() + .filter(|candidate| !is_destructive(candidate, screen, stop_before)) + .collect::>(); + // A field that holds the typed option is where it was typed, + // not one of the options it offers. + let pool = closest( + pool.into_iter() + .filter(|candidate| { + mentions(candidate, option) + && !editable(candidate) + && !lists_more_than(candidate, option) + }) + .collect(), + ); + let pool = within(pool, what); + // An `enter` value with no box to type it into is picked from + // what the screen shows, a date aside (a calendar can scroll its + // days out of its own view): live, the only "Mumbai" was a link + // out of view at the foot of the page, pressed as the place to + // fly to, and the search went from Mumbai instead. + if into_focus || looks_like_date(option) { + pool + } else { + pool.into_iter() + .filter(|candidate| { + !candidate + .states + .iter() + .any(|state| state.eq_ignore_ascii_case("offscreen")) + }) + .collect() + } +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/steps/condition.rs b/crates/tinycomputer-engine/src/agentic/flow/steps/condition.rs index b8e242a6..2377ca94 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/steps/condition.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/steps/condition.rs @@ -180,7 +180,7 @@ impl FlowRun<'_, B> { } // A dialog the task opened asks its question first (a format, a // quantity): what lies past it will not show while it waits. - if self.front.opened_dialog && check >= 1 { + if self.front.opened_dialog() && check >= 1 { return Err(Halt::Failed( "a dialog the task opened is waiting for an answer, so nothing past it shows: choose what it asks, then continue" .to_owned(), diff --git a/crates/tinycomputer-engine/src/agentic/flow/steps/date.rs b/crates/tinycomputer-engine/src/agentic/flow/steps/date.rs index 793a9474..7d3ed3f6 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/steps/date.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/steps/date.rs @@ -22,6 +22,15 @@ const MONTHS: &[&str] = &[ "december", ]; +/// Whether `text` names a month, in full or cut to three letters or more +/// ("Oct", "Sept"), as a calendar's day cells and headings do. +pub(in crate::agentic::flow) fn names_a_month(text: &str) -> bool { + text.split(|character: char| !character.is_alphabetic()) + .filter(|word| word.chars().count() >= 3) + .map(str::to_lowercase) + .any(|word| MONTHS.iter().any(|month| month.starts_with(&word))) +} + /// Whether `option` names a calendar day: a month name and a day number that /// is a real day of that month (a year, when given, decides February's 28th /// against its 29th). `February 31` or `April 31` names no such day, and is diff --git a/crates/tinycomputer-engine/src/agentic/flow/steps/list.rs b/crates/tinycomputer-engine/src/agentic/flow/steps/list.rs index 75697c51..dd04fa52 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/steps/list.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/steps/list.rs @@ -6,7 +6,7 @@ use std::collections::BTreeMap; use serde_json::json; use tinycomputer_bus::{FlowLoop, JevOperation, PickStep, ReadStep, StepOutcome}; use tinycomputer_core::surface::{Group, result_families}; -use tinycomputer_core::{Criterion, Record, rank}; +use tinycomputer_core::{Criterion, Record, closest_to, rank, rank_closest}; use tinyinference_decisions::Answer; use crate::agentic::flow::{ @@ -23,18 +23,19 @@ use super::{ impl FlowRun<'_, B> { /// Picks the best of a list of results by `pick.by`, stores its text, - /// and opens it. A criterion over prices, times, durations, or stops is - /// ranked exactly, and the first ranked item that Jev confirms belongs - /// to `pick.from` is taken: the ranking reads only its measure, and - /// live, "the cheapest of the results rated 4 stars or more" took a - /// 3.1-star item of another brand. Anything else, or a ranking none of - /// whose leaders belongs, is judged by Jev among the records. + /// and opens it. A criterion over prices, times, durations, or stops, + /// or nearness to a number ("closest to 9"), is ranked exactly, and the + /// first ranked item that Jev confirms belongs to `pick.from` is taken: + /// the ranking reads only its measure, and live, "the cheapest of the + /// results rated 4 stars or more" took a 3.1-star item of another brand. + /// Anything else, or a ranking none of whose leaders belongs, is judged + /// by Jev among the records. pub(super) async fn pick(&mut self, log: &mut StepLog, pick: &PickStep) -> Result { let from = substitute_safe(&pick.from, &self.vars, &self.facts); let by = substitute_safe(&pick.by, &self.vars, &self.facts); let mut screen = self.look().await?; self.explore(&mut screen).await; - let families = result_families(&screen); + let families = openable(result_families(&screen)); if families.is_empty() { return Err(Halt::Failed(format!("no list of {from} is showing"))); } @@ -47,22 +48,25 @@ impl FlowRun<'_, B> { let condition = first_meeting(&by); let criterion = Criterion::parse(&by).or_else(|| condition.as_ref().map(|_| Criterion::First)); - let ranked = match criterion { + let ranked = match (closest_to(&by), criterion) { + // Nearness to a number ranks by distance, within the list `from` + // names, since numbers show in every list. Live, a store's sizes + // 9 and 10 were sold out, and "closest to 9", judged item by + // item, took none of 6, 7, and 8. + (Some(target), _) => { + let groups = self.named_list(log, &screen, &from, &families).await?; + rank_closest(&records_of(groups), target).map(|order| (groups, order)) + } // The list's own order fits every list on the page, so the one // `from` names is asked for first, as a judged pick does. - Some(order @ (Criterion::First | Criterion::Last)) => { - let list = if families.len() > 1 { - self.judge_list(log, &screen, &from, &families).await? - } else { - 0 - }; - let groups = &families[list]; + (None, Some(order @ (Criterion::First | Criterion::Last))) => { + let groups = self.named_list(log, &screen, &from, &families).await?; rank(&records_of(groups), order).map(|ranking| (groups, ranking)) } - Some(criterion) => families.iter().find_map(|groups| { + (None, Some(criterion)) => families.iter().find_map(|groups| { rank(&records_of(groups), criterion).map(|order| (groups, order)) }), - None => None, + (None, None) => None, }; let meets = condition.map_or_else(|| from.clone(), |condition| format!("{from}, {condition}")); @@ -78,12 +82,7 @@ impl FlowRun<'_, B> { } else { // Several lists show (a chat list beside the open chat's // messages): judge within the one `from` names. - let list = if families.len() > 1 { - self.judge_list(log, &screen, &from, &families).await? - } else { - 0 - }; - let groups = &families[list]; + let groups = self.named_list(log, &screen, &from, &families).await?; let best = self.judge_pick(log, &screen, &from, &by, groups).await?; (groups, best, "judged") }; @@ -197,6 +196,23 @@ impl FlowRun<'_, B> { )) } + /// The list of `families` that `from` names: the only one, or the one + /// Jev chooses when several show. + async fn named_list<'f>( + &mut self, + log: &mut StepLog, + screen: &crate::agentic::flow::view::Screen, + from: &str, + families: &'f [Vec], + ) -> Result<&'f Vec, Halt> { + let list = if families.len() > 1 { + self.judge_list(log, screen, from, families).await? + } else { + 0 + }; + Ok(&families[list]) + } + /// Asks Jev which of the lists showing is `what`, each shown by its /// first [`LIST_PREVIEW`] items, among the first [`MAX_LISTS`]. A list /// not clearly chosen falls back to the one Jev leaned to @@ -341,7 +357,29 @@ impl FlowRun<'_, B> { } } -/// Each card's text as a record, its fields numbered in reading order. +/// The lists of `families` a pick can open an item of: those whose items +/// mostly hold something to press, when any list's do. A pick opens what +/// it takes, and a list of bare fares (a strip above the flights, or each +/// card's price read apart from it) has nothing to open: live, "the +/// cheapest flight" ranked such a list and took "₹ 6,054". +fn openable(families: Vec>) -> Vec> { + let opens = |groups: &Vec| { + groups + .iter() + .filter(|group| group.primary.is_some()) + .count() + * 2 + > groups.len() + }; + if families.iter().any(opens) { + families.into_iter().filter(opens).collect() + } else { + families + } +} + +/// Each card's text as a record, its fields numbered in reading order, and +/// numbered so that their keys sort in that order too. fn records_of(groups: &[Group]) -> Vec { groups .iter() @@ -350,7 +388,7 @@ fn records_of(groups: &[Group]) -> Vec { .fields .iter() .enumerate() - .map(|(index, text)| (format!("field {index}"), text.clone())) + .map(|(index, text)| (format!("field {index:03}"), text.clone())) .collect(), }) .collect() diff --git a/crates/tinycomputer-engine/src/agentic/flow/steps/matching.rs b/crates/tinycomputer-engine/src/agentic/flow/steps/matching.rs index ec09e337..b3d08c2f 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/steps/matching.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/steps/matching.rs @@ -29,6 +29,9 @@ pub(super) fn held_text(screen: &Screen) -> Vec<(Candidate, String)> { /// The matches whose labels say little besides the option: a container /// whose label strings together everything inside it (a calendar button /// named with every day of the month) is dropped when a plainer match exists. +/// A list's option is one option however much its label says: live, an +/// airport row ("BOM Mumbai, India … 3 Nearby Airports found") was dropped +/// for a footer link that only said "Mumbai". pub(in crate::agentic::flow) fn closest(matches: Vec) -> Vec { let length = |candidate: &Candidate| candidate.name.as_deref().map_or(0, str::len); let Some(shortest) = matches.iter().map(length).min() else { @@ -36,7 +39,10 @@ pub(in crate::agentic::flow) fn closest(matches: Vec) -> Vec Option { + if !looks_like_date(option) { + return None; + } + let wanted = date_words(option); + let named = plain(what) + .split(' ') + .filter(|word| word.chars().count() > 3 && *word != "date") + .map(str::to_owned) + .collect::>(); + if named.is_empty() { + return None; + } + screen + .candidates + .iter() + .find(|candidate| { + let shown = [ + candidate.name.as_deref(), + candidate.value.as_ref().and_then(serde_json::Value::as_str), + ] + .into_iter() + .flatten() + .collect::>() + .join(" "); + let words = plain(&shown); + named + .iter() + .any(|word| words.split(' ').any(|said| said == word)) + && shows_date(&shown, &wanted) + }) + .cloned() +} + /// Roles a page marks as the one chosen among its siblings. const SELECTABLE_ROLES: &[&str] = &["tab", "radio", "radiobutton", "option", "menuitemradio"]; @@ -309,15 +358,36 @@ pub(super) fn within(pool: Vec, what: &str) -> Vec { if regional.is_empty() { pool } else { regional } } +/// Words that place one thing against another ("from Delhi to Mumbai"), +/// too common in labels to say which region an option is in. +const PLACING_WORDS: &[&str] = &[ + "to", "from", "via", "at", "in", "on", "for", "of", "by", "with", +]; + /// Whether `candidate` sits inside — or itself names — the region `what` /// describes. A page rarely echoes a description such as "the outbound /// flight list" on an option's own label, so this also checks the option's /// ancestor labels (`path`), which the snapshot records outermost first. +/// +/// A region named by a placing word alone ("to", "from") holds only what +/// sits under a container whose name begins with it, never a control that +/// names it itself (the "To" box's own button is no option of its list): +/// live, "to" kept a page's "Delhi to Mumbai flights" links and dropped +/// the airport list. pub(in crate::agentic::flow) fn in_region(candidate: &Candidate, what: &str) -> bool { let wanted = plain(what); if wanted.is_empty() { return true; } + if PLACING_WORDS.contains(&wanted.as_str()) { + return candidate.path.iter().any(|ancestor| { + let name = ancestor + .split_once('"') + .map_or("", |(_, quoted)| quoted.trim_end_matches('"')); + let name = plain(name); + name == wanted || name.starts_with(&format!("{wanted} ")) + }); + } let names = |text: &str| format!(" {} ", plain(text)).contains(&format!(" {wanted} ")); [candidate.name.as_deref(), candidate.description.as_deref()] .into_iter() diff --git a/crates/tinycomputer-engine/src/agentic/flow/steps/mod.rs b/crates/tinycomputer-engine/src/agentic/flow/steps/mod.rs index 1cd28bd7..1b226d85 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/steps/mod.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/steps/mod.rs @@ -21,13 +21,15 @@ mod stop; mod suggestion; mod typing; +pub(super) use date::{looks_like_date, names_a_month}; pub(super) use matching::left_unchosen; #[cfg(test)] pub(super) use { - date::{date_words, looks_like_date, shows_date}, + date::{date_words, shows_date}, list::{first_meeting, leaning}, matching::{ - already_chosen, already_holds, closest, in_region, lists_more_than, redacted, search_text, + already_chosen, already_holds, closest, date_shown_in, in_region, lists_more_than, + redacted, search_text, }, read::chosen_together, read::readable, diff --git a/crates/tinycomputer-engine/src/agentic/flow/steps/suggestion.rs b/crates/tinycomputer-engine/src/agentic/flow/steps/suggestion.rs index d4d91bf0..116e04d3 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/steps/suggestion.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/steps/suggestion.rs @@ -334,7 +334,10 @@ pub(in crate::agentic::flow) fn suggests(slot: &str, _field: &Candidate) -> bool /// press. For a `place` box, a row that was already showing counts too when /// it matches the text: a ride app lists popular places as soon as its box /// has the focus, and live, the place typed was among them, so nothing new -/// appeared and nothing was picked. +/// appeared and nothing was picked. Such a row is in view and in front, as +/// a list under the box is: live, a footer link "hotels in Goa", off screen +/// at the foot of the page, was pressed as the place typed, and the box's +/// own rows were never waited for. pub(in crate::agentic::flow) fn fresh_rows( screen: &Screen, shown: &BTreeSet<(&str, Option<&str>)>, @@ -347,8 +350,12 @@ pub(in crate::agentic::flow) fn fresh_rows( .into_iter() .filter(|candidate| { let new = !shown.contains(&(candidate.role.as_str(), candidate.name.as_deref())); - let matches = - place && (mentions(candidate, text) || shares_most_words(candidate, text)); + let in_view = !candidate.states.iter().any(|state| { + state.eq_ignore_ascii_case("offscreen") || state.eq_ignore_ascii_case("covered") + }); + let matches = place + && in_view + && (mentions(candidate, text) || shares_most_words(candidate, text)); let a_box = candidate .available_actions .iter() diff --git a/crates/tinycomputer-engine/src/task/budget.rs b/crates/tinycomputer-engine/src/task/budget.rs index cdb6039c..3a821266 100644 --- a/crates/tinycomputer-engine/src/task/budget.rs +++ b/crates/tinycomputer-engine/src/task/budget.rs @@ -65,6 +65,7 @@ pub(super) fn run_request( trace: state.trace, // What earlier runs saved, so a resumed run remembers it. collected: state.reads.clone(), + dialog_left_open: state.dialog_left_open, ..RunFlowRequest::default() }, state.constraints.clone(), diff --git a/crates/tinycomputer-engine/src/task/drive.rs b/crates/tinycomputer-engine/src/task/drive.rs index ea9ec2c0..b4f03708 100644 --- a/crates/tinycomputer-engine/src/task/drive.rs +++ b/crates/tinycomputer-engine/src/task/drive.rs @@ -178,6 +178,9 @@ pub(super) async fn drive(cell: Arc, runner: Arc, runs: Ve if let Some(rescue) = run.rescue.and_then(|index| state.rescues.get_mut(index)) { rescue.outcome = rescue_outcome(&reached, rescue.steps.len()); } + state.dialog_left_open = result + .as_ref() + .is_some_and(|result| result.dialog_left_open); if let Some(result) = result { state.spent.actions = state.spent.actions.saturating_add(result.actions); state.spent.model_calls = diff --git a/crates/tinycomputer-engine/src/task/store.rs b/crates/tinycomputer-engine/src/task/store.rs index 72a83412..926bc49b 100644 --- a/crates/tinycomputer-engine/src/task/store.rs +++ b/crates/tinycomputer-engine/src/task/store.rs @@ -47,6 +47,11 @@ pub(super) struct State { pub(super) exchanges: Vec, pub(super) learned: Vec, pub(super) reads: BTreeMap, + /// Whether the task's last run left its own dialog in front, for the + /// next run to work within ([`RunFlowRequest::dialog_left_open`]). + /// + /// [`RunFlowRequest::dialog_left_open`]: tinycomputer_bus::RunFlowRequest::dialog_left_open + pub(super) dialog_left_open: bool, pub(super) finished: usize, pub(super) resume: Option, /// What every run of this task has spent so far, so an approval or a @@ -112,6 +117,7 @@ impl Tasks { exchanges: Vec::new(), learned: Vec::new(), reads: BTreeMap::new(), + dialog_left_open: false, finished: 0, resume: None, spent: Spent::default(), diff --git a/crates/tinycomputer-engine/src/task/task_tests.rs b/crates/tinycomputer-engine/src/task/task_tests.rs index 85849548..faee5b3d 100644 --- a/crates/tinycomputer-engine/src/task/task_tests.rs +++ b/crates/tinycomputer-engine/src/task/task_tests.rs @@ -188,6 +188,7 @@ fn finished_run( actions: 3, metrics: JevMetrics::default(), trace: Vec::new(), + dialog_left_open: false, }; DesktopResponse::ok("run-flow", serde_json::to_value(result).unwrap()) } diff --git a/crates/tinycomputer-engine/src/task/task_tests/rescue_tests.rs b/crates/tinycomputer-engine/src/task/task_tests/rescue_tests.rs index ae6e91b1..c08b9dbd 100644 --- a/crates/tinycomputer-engine/src/task/task_tests/rescue_tests.rs +++ b/crates/tinycomputer-engine/src/task/task_tests/rescue_tests.rs @@ -378,3 +378,31 @@ async fn a_skip_resumes_at_the_next_step_with_the_guard_kept() { assert_eq!(rescue.covers, 1); assert_eq!(rescue.outcome, RescueOutcome::Recovered); } + +#[tokio::test] +async fn a_rescue_takes_the_dialog_in_front_as_the_tasks_only_when_the_run_before_left_it() { + let rest = || finished_run(FlowStopReason::Completed, vec![], &[], None); + // The failed run left the task's own dialog in front: its rescue works + // within it. + let mut left_open = failed_at_step_two(); + left_open.data.as_mut().unwrap()["dialog_left_open"] = json!(true); + let (tasks, script, _model) = rescued(vec![left_open, rest()], &[Ok(ONE_STEP)]); + let view = begin(&tasks, TaskBudget::default()); + settle(&tasks, &view.id).await; + let requests = script.requests.lock().unwrap().clone(); + assert_eq!(requests.len(), 2); + assert!( + !requests[0].dialog_left_open, + "a first run inherits nothing" + ); + assert!(requests[1].dialog_left_open); + + // One that left none: a dialog in front at the rescue's first look is + // the page's own (a sign-up it opened, a menu), not the task's. + let (tasks, script, _model) = rescued(vec![failed_at_step_two(), rest()], &[Ok(ONE_STEP)]); + let view = begin(&tasks, TaskBudget::default()); + settle(&tasks, &view.id).await; + let requests = script.requests.lock().unwrap().clone(); + assert_eq!(requests.len(), 2); + assert!(!requests[1].dialog_left_open); +} diff --git a/crates/tinycomputer/tests/public_api_tests.rs b/crates/tinycomputer/tests/public_api_tests.rs index 3b1cf339..8f8047ac 100644 --- a/crates/tinycomputer/tests/public_api_tests.rs +++ b/crates/tinycomputer/tests/public_api_tests.rs @@ -34,7 +34,7 @@ fn the_crate_re_exports_the_contract_rather_than_redefining_it() { #[test] fn the_shipped_contract_binds_to_itself() { assert!(is_compatible(CONTRACT_VERSION)); - assert_eq!(CONTRACT_VERSION, (2, 8)); + assert_eq!(CONTRACT_VERSION, (2, 9)); } #[test] diff --git a/docs/crates/tinycomputer-browser/interacting.md b/docs/crates/tinycomputer-browser/interacting.md index 1f2d8991..72ae369c 100644 --- a/docs/crates/tinycomputer-browser/interacting.md +++ b/docs/crates/tinycomputer-browser/interacting.md @@ -54,6 +54,15 @@ than clicking through it and reporting success on a click that landed nowhere useful. That refusal message is what `reply::classify` turns into `Error::NotActionable`. +A selector's element that shows at all is not scrolled, though, and its +press lands on its middle whether the window shows that point or not, with +success reported. So before pressing a control sight found, the surface +brings it to the middle of the window when its middle lies outside the +window (`INTO_VIEW_JS` in `surface/uncover.rs`), and leaves a control in +view where it is. Live, a store's "Add to cart" sat at the window's foot +with its middle below it, and was pressed again and again with nothing +happening. A ref of the tree is brought into view by agent-browser itself. + ### Click-through on result cards A refusal because something is "covered" is not always the end of the diff --git a/docs/crates/tinycomputer-browser/sight.md b/docs/crates/tinycomputer-browser/sight.md index a5f813a5..9d897916 100644 --- a/docs/crates/tinycomputer-browser/sight.md +++ b/docs/crates/tinycomputer-browser/sight.md @@ -64,7 +64,8 @@ Sight decides what is a control by behavior, not by role: those (so a `` inside a button does not become a second control just because it inherits the button's pointer cursor). - A small element (under a quarter of the window) whose React props hold a - press handler (`onClick`, `onPress`) is a control too: a page can wire a + press handler (`onClick`, `onPress`), or whose Preact listeners (`l` once + minified, `_listeners`) hold a click one, is a control too: a page can wire a plain `div` to a click with neither a cursor nor a tab stop. A handler for the mouse going down alone (a carousel's track) makes no control, and a control made only by such a handler hides nothing pressable inside it. @@ -73,12 +74,20 @@ Sight decides what is a control by behavior, not by role: `aria-disabled="true"` and a class name that *ends* in `disabled`. That last rule exists because plenty of date pickers grey out a past day purely through a CSS class, `rdrDay rdrDayDisabled` on a real calendar widget, - with no ARIA attribute at all. + with no ARIA attribute at all. A class behind a variant prefix + (`placeholder:text-disabled`, `disabled:opacity-50`) is never read this + way, nor for `selected` below: it styles a part or a state the element may + not be in. - Chosen controls read `selected` by the same reasoning: besides `aria-selected`, `aria-current`, `aria-checked`, and `aria-pressed`, a class name ending in `-selected` or `-checked` (never `unselected` or `not-selected`) marks one, as a store's picked size (`size-buttons-size-button-selected`) carries no ARIA state at all. +- A control repeated on every card ("ADD" on each product), inside a card + that is a control itself, is described by that card's name (`in Maggi + Double Masala 95 g ₹20`) when it has no description of its own: by its + name alone, every copy is the same, and live, "add the first Maggi" + pressed the first card's "ADD", on a ramen above the Maggi. The IndiGo case above is the sharpest illustration of the next rule: **a claimed text box that takes no text is a button, not a text box.** Sight diff --git a/docs/crates/tinycomputer-engine/flow/step-kinds.md b/docs/crates/tinycomputer-engine/flow/step-kinds.md index 4d17c6fc..a25d7b9f 100644 --- a/docs/crates/tinycomputer-engine/flow/step-kinds.md +++ b/docs/crates/tinycomputer-engine/flow/step-kinds.md @@ -121,7 +121,11 @@ The exact rankings are: - lowest or highest price, - earliest or latest time, - fewest stops, -- shortest duration. +- shortest duration, +- nearest to a number ("closest to 9", "nearest to size 42"): each card's + first number, by its distance, within the list `from` names (asked for + first when several show, as for "first"), since numbers show in every + list. When it parses ("lowest price"), the parsers in `tinycomputer-core/src/records/` read prices with currency symbols, clock diff --git a/docs/crates/tinycomputer-engine/flow/the-do-loop.md b/docs/crates/tinycomputer-engine/flow/the-do-loop.md index 10f1c811..1adc4406 100644 --- a/docs/crates/tinycomputer-engine/flow/the-do-loop.md +++ b/docs/crates/tinycomputer-engine/flow/the-do-loop.md @@ -152,7 +152,10 @@ They are written in macOS spelling (`cmd+n`) and each surface translates them: the desktop sends `ctrl+n` on Windows and Linux, the browser sends `Meta+n` or `Control+n`. Return is refused while a sheet or alert is showing, because there it presses that dialog's default button, which -might not be the one the step wants. +might not be the one the step wants, unless the run's last action typed +into a search box (`is_search_box`): Return there runs the search. Live, a +store's search opened as a full-window sheet, and pressing Enter in it +was refused 154 times. ### The destructive check diff --git a/docs/technical/decision-loops.md b/docs/technical/decision-loops.md index 3b049df3..bc238dab 100644 --- a/docs/technical/decision-loops.md +++ b/docs/technical/decision-loops.md @@ -272,7 +272,8 @@ folder, find, reply, settings, back, next field, confirm (Return), and dismiss spelling (`cmd+n`); each surface translates them, so the desktop sends `ctrl+n` on Windows and Linux and the browser sends `Meta+n` or `Control+n`. Return is refused while a sheet or alert is showing, because there it presses -the default button. +the default button, unless the run's last action typed into a search box, +where Return runs the search. Before any click, `is_destructive` runs. It refuses the click, and fails the step, when: diff --git a/docs/technical/decision-thresholds.md b/docs/technical/decision-thresholds.md index 3f47667d..73e60847 100644 --- a/docs/technical/decision-thresholds.md +++ b/docs/technical/decision-thresholds.md @@ -49,6 +49,7 @@ Change a constant and its row together. | `MAX_REPEAT_PRESSES` | 3 | `act/mod.rs` | presses of one control (by label and place, so a toggle's two looks count as one) or one key in one `do` step after which it is struck off for the step; a scroll is no press. Pressing a named control also strikes off its copies on the other items of its list (same label, another card of the same list), once the next look shows the press changed the screen, unless the step says all, every, each, or both, or chooses several items (a choosing verb in its first two words and a count of 2 to 20, or a number word, within three words before a plural: "choose 2 adjacent seats"); undoing that press lifts its own copies again | | `MAX_IDLE_SCROLLS` | 1 | `act/mod.rs` | scrolls that showed nothing new after which a step's "scroll" move is taken as "activate": the screen already lists what lies below the fold | | `LAYER_COVERS` | 3 | `front.rs` | controls something drawn over the window must cover, beyond what was covered before the press that opened it, on the same page, before it counts as a dialog the task opened (`surface` `layer`); a step that pressed inside such a dialog hands it back at the next step, and opening an address forgets it | +| `CALENDAR_DAYS` | 7 | `front.rs` | day cells nothing covers (a grid cell, or a day number beside a month's name) that make what is in front a calendar: a press refused as covered behind a calendar the task opened and has pressed in since, in that step or one before, closes it and presses again, where any other dialog of the task's is worked within | | `FRONT_CONTROLS` | 8 | `act/turns.rs` | most controls of the task's dialog in front a failed step's note names, so a rescue answers with one of them | | `STEADY_HOLD` / `STEADY_CHECKS` | 0.65 / 3 | `steps/mod.rs` | belief a `wait_for` condition must keep, on checks in a row of one unchanged screen, to be taken as held under `DONE` | | `HEDGE_AFTER` / `HEDGE_AFTER_LARGE` | 4 s / 5 s | `hedge.rs` | how long a framing runs before a copy of it is sent and the first answer of the two taken; the longer wait is for a request of `HEDGE_LARGE_BYTES` (32 KB) or more. Sage gets no copy | diff --git a/docs/technical/jev-harness.md b/docs/technical/jev-harness.md index c0e051d3..7d7d8e3d 100644 --- a/docs/technical/jev-harness.md +++ b/docs/technical/jev-harness.md @@ -76,7 +76,9 @@ framing of every request together: one round trip. In order: 1. **Build.** A step's code builds the questions (`ask/questions.rs`) and the shared state (`ask::state`, in `ask/screen_state.rs`: app, window, surface, current step, visible text, - up to 120 elements, the last eight history lines, and field contents when + up to 120 elements (on a longer screen, those in view first, up to three + quarters of them, then the rest as the page orders them, all listed in + screen order), the last eight history lines, and field contents when `include_values` is set). Screen text is always wrapped as `untrusted_accessibility_data`. 2. **Budget.** The run's call budget is checked first; a spent budget stops diff --git a/docs/technical/specs/desktop-module-contract.md b/docs/technical/specs/desktop-module-contract.md index 4114c46f..8d1a4013 100644 --- a/docs/technical/specs/desktop-module-contract.md +++ b/docs/technical/specs/desktop-module-contract.md @@ -41,9 +41,13 @@ engine's argument types, the permission preflight, and the bus surface. `open_jev` and `sage` decision providers, `JevConfig.fast`, `JevConfiguration.fast`, the planner's `LanguageModelProvider` routes, and `Capabilities.decision_model`, `planner_model`, `rescue_model`, and - `output_model`, all optional) return an + `output_model`; 2.9 added `RunFlowRequest.dialog_left_open` and + `FlowRunResult.dialog_left_open`; all optional) return an `AgentResponse` instead — see [`unified-agent.md`](unified-agent.md). They - share this interface because a TinyBus module exports one interface. + share this interface because a TinyBus module exports one interface. A + task hands `dialog_left_open` from one run's result to its next run's + request: a run takes a dialog in front at its first look as the task's + own stage only when the run before it left that dialog open. - The thirteen browser members (2.6) close the list, each prefixed `Browser` (`tinycomputer_bus::browser::names`): sessions, navigate, snapshot, perform, read, evaluate, screenshot, held outputs, and downloads. A member diff --git a/docs/technical/specs/unified-agent.md b/docs/technical/specs/unified-agent.md index c2703bd3..7e37cba6 100644 --- a/docs/technical/specs/unified-agent.md +++ b/docs/technical/specs/unified-agent.md @@ -104,7 +104,8 @@ These are additive to the grammar in [`jev-intent-flows.md`](jev-intent-flows.md - `in "" {steps}` - `extract {what, fields, into}` - `pick {from, by, into}` (implemented): chooses the best result card and - opens it; prices, times, durations, and stops are ranked exactly + opens it; prices, times, durations, stops, and nearness to a number + ("closest to 9") are ranked exactly - `ask [slots]` - `checkpoint ""`