diff --git a/.env.example b/.env.example index 249fb8e2..c3ccee45 100644 --- a/.env.example +++ b/.env.example @@ -47,6 +47,12 @@ TINYCOMPUTER_LAB_SELF_EMAIL= # How the live browser examples read a page: by sight (the default), or # `tree` for the accessibility tree alone (docs/technical/specs/browser-sight.md). # TINYCOMPUTER_BROWSER_PERCEPTION=tree +# How a page settles after an action: prompt (default) or steady. +# TINYCOMPUTER_BROWSER_SETTLE=steady +# task_live: plan inside StartTask as OpenHuman does; the browser opens +# while it plans, at the one address the task names, unless PRELAUNCH is 0. +# TASK_PLAN=in-task +# TINYCOMPUTER_BROWSER_PRELAUNCH=0 # The on-screen cursor's overlay helper, when it is not beside the module. # TINYCOMPUTER_CURSOR_OVERLAY=/path/to/tinycomputer-cursor-overlay @@ -79,6 +85,9 @@ TINYCOMPUTER_LAB_SELF_EMAIL= # default 5). # TINYCOMPUTER_RESCUE_MODEL= # TASK_RESCUES=5 +# task_live: a file of elements earlier runs learned; read before the task, +# saved after it. +# TASK_MEMORY=target/task-live/memory/amazon.json # The reasoning model task_live shapes a finished task's answer with # (default openai/gpt-6-luna on OpenRouter), and a JSON TaskOutput file # asking for it. diff --git a/MODULE.md b/MODULE.md index c5276b41..29e8b271 100644 --- a/MODULE.md +++ b/MODULE.md @@ -61,8 +61,12 @@ is optional: base URL; and `rescue_route` (`api_key`, `provider`, `endpoint_url`, `sdk_name`) gives the rescuer a route and key of its own; - `browser`: how every browser launches — `executable` (the Chrome or - Chromium binary), `user_agent`, `args` (an array of launch arguments), and - `perception` (`sight`, the default, or `tree`: how a task reads a page). + Chromium binary), `user_agent`, `args` (an array of launch arguments), + `perception` (`sight`, the default, or `tree`: how a task reads a page), + `settle` (`prompt`, the default, or `steady`: how long a page is let + settle after an action), and `prelaunch` (a boolean, `true` by default: + open a browser-only task's browser while it is planned, at the one web + address the task's text names, if it names one). Configuration is delivered as sensitive host-control traffic and is never shown to monitors. diff --git a/crates/tinycomputer-browser/src/fake/mod.rs b/crates/tinycomputer-browser/src/fake/mod.rs index 63bf297b..dd4f3389 100644 --- a/crates/tinycomputer-browser/src/fake/mod.rs +++ b/crates/tinycomputer-browser/src/fake/mod.rs @@ -10,13 +10,15 @@ use serde_json::{Value, json}; use crate::engine::{Engine, Launcher, Reply}; type Script = dyn Fn(&Value) -> Option + Send + Sync; +type Stall = dyn Fn(&Value) -> bool + Send + Sync; /// Records every command; answers from an optional override, else like the -/// engine would. +/// engine would, and never answers a command it is told to stall on. #[derive(Clone)] pub(crate) struct Fake { sent: Arc>>, script: Arc"# + ) + }; + for (host, banner, shown) in [ + ("display: contents", "", true), + ("display: block", "", true), + ("display: contents", "display: none", false), + ] { + let Some(reading) = live_reading(&page(host, banner)).await else { + return; + }; + assert_eq!(reading["unreachable"], 0, "{reading}"); + let shadows = reading["shadows"].as_array().unwrap(); + assert_eq!( + shadows.len(), + usize::from(shown), + "host {host:?}, banner {banner:?}: {reading}" + ); + if shown { + assert_eq!( + shadows[0]["label"], "popover \"We value your privacy\"", + "{reading}" + ); + } + } + // The tree read under the host offers the banner's buttons, and only + // them. + let (browser, info) = live_page(&page("display: contents", "")) + .await + .expect("a live run opens the page"); + let reading = browser + .command( + &info.id, + json!({"action": "evaluate", "script": script(None)}), + ) + .await + .unwrap()["result"] + .clone(); + let host = reading["shadows"][0]["id"].as_str().unwrap().to_owned(); + let subtree = browser + .snapshot( + &info.id, + tinycomputer_bus::browser::SnapshotRequest { + selector: Some(format!("[data-tc-seen=\"{host}\"]")), + ..tinycomputer_bus::browser::SnapshotRequest::default() + }, + ) + .await + .unwrap(); + browser.close_session(&info.id).await.unwrap(); + assert!(subtree.tree.contains("Allow Selection"), "{}", subtree.tree); + assert!(!subtree.tree.contains("Add To Cart"), "{}", subtree.tree); +} + +#[cfg(feature = "agent-browser")] +#[tokio::test] +async fn live_what_a_shadow_host_slots_is_left_to_the_tree() { + // The tree read under a host holds the host and what the page puts in + // its slots: sight reads neither, or each would be offered twice. A + // hidden dialog in the shadow root names no layer, and the banner's + // `aria-labelledby` resolves inside the shadow root. + let page = r#"
+
+ "#; + let Some((browser, info)) = live_page(page).await else { + return; + }; + let reading = browser + .command( + &info.id, + json!({"action": "evaluate", "script": script(None)}), + ) + .await + .unwrap()["result"] + .clone(); + let names = shown_names(&reading); + assert!(names.iter().any(|name| name == "Add To Cart"), "{names:?}"); + assert!( + !names.iter().any(|name| name == "Slotted choice"), + "{names:?}" + ); + let shadows = reading["shadows"].as_array().unwrap(); + assert_eq!(shadows.len(), 1, "{reading}"); + assert_eq!( + shadows[0]["label"], "popover \"Cookie choices\"", + "{reading}" + ); + let host = shadows[0]["id"].as_str().unwrap().to_owned(); + let subtree = browser + .snapshot( + &info.id, + tinycomputer_bus::browser::SnapshotRequest { + selector: Some(format!("[data-tc-seen=\"{host}\"]")), + ..tinycomputer_bus::browser::SnapshotRequest::default() + }, + ) + .await + .unwrap(); + browser.close_session(&info.id).await.unwrap(); + for read in ["Allow Selection", "Slotted choice"] { + assert!(subtree.tree.contains(read), "{read}: {}", subtree.tree); + } +} + #[cfg(feature = "agent-browser")] #[tokio::test] async fn live_hidden_elements_are_dropped() { diff --git a/crates/tinycomputer-browser/src/surface/surface_tests/operations_tests.rs b/crates/tinycomputer-browser/src/surface/surface_tests/operations_tests.rs index eee0ac6e..85de0465 100644 --- a/crates/tinycomputer-browser/src/surface/surface_tests/operations_tests.rs +++ b/crates/tinycomputer-browser/src/surface/surface_tests/operations_tests.rs @@ -229,7 +229,8 @@ fn pressing_launching_settling_and_navigating() { assert_eq!(surface.press("", "").error.unwrap().code, "INVALID_KEY"); let launched = surface.launch("browser"); assert_eq!(launched.data.unwrap()["running"], true); - surface.settle(); + // Settling steadily, as a host may still ask for. + surface.clone().with_settle(crate::Settle::Steady).settle(); let idle = fake.last("waitforloadstate"); assert_eq!( (idle["state"].as_str(), idle["timeout"].as_u64()), @@ -549,3 +550,276 @@ fn a_navigation_that_timed_out_on_a_drawn_page_is_taken_as_open() { let reply = blank.surface.navigate("https://shop.test/search?q=milk"); assert!(!reply.ok, "nothing drawn yet: the timeout stands"); } + +#[test] +fn a_page_opened_early_is_not_loaded_again_until_it_is_read() { + // A shop whose page has drawn by the time it is asked. + let shop = || { + Fake::scripted(|command| { + let script = command["script"].as_str().unwrap_or_default(); + (command["action"] == "evaluate" && script.contains("readyState")).then(|| { + ok(&json!({"result": { + "url": "https://www.shop.test/", + "title": "Shop", + "drawn": true, + }})) + }) + }) + }; + let loads = |fake: &Fake| { + fake.actions() + .iter() + .filter(|action| *action == "navigate") + .count() + }; + + let early = harness("open-at", shop()); + assert!(early.surface.open_at("https://shop.test")); + assert_eq!(loads(&early.fake), 1); + assert_eq!( + early.fake.last("navigate")["timeout"], + 10_000, + "a plan drafted sooner waits for it no longer" + ); + // The plan's first step browses there, by another of its addresses. + let reply = early.surface.navigate("https://www.shop.test/"); + assert!(reply.ok, "{:?}", reply.error); + assert_eq!(reply.data.unwrap()["title"], "Shop"); + assert_eq!(loads(&early.fake), 1, "loaded once"); + // Asked again, it loads again, with the session's own deadline. + assert!(early.surface.navigate("https://shop.test").ok); + assert_eq!(loads(&early.fake), 2); + assert!(early.fake.last("navigate")["timeout"].is_null()); + + // Once the page is read, or another page is asked for, it loads. + let read = harness("open-at-read", shop()); + assert!(read.surface.open_at("https://shop.test")); + assert!(read.surface.observe("browser", None, Depth::Full).is_ok()); + assert!(read.surface.navigate("https://shop.test").ok); + assert_eq!(loads(&read.fake), 2); + let elsewhere = harness("open-at-elsewhere", shop()); + assert!(elsewhere.surface.open_at("https://shop.test")); + assert!(elsewhere.surface.navigate("https://shop.test/cart").ok); + assert_eq!(loads(&elsewhere.fake), 2); + + // A page that has drawn nothing yet is loaded as asked. + let blank = harness("open-at-blank", Fake::new()); + assert!(blank.surface.open_at("https://shop.test")); + assert!(blank.surface.navigate("https://shop.test").ok); + assert_eq!(loads(&blank.fake), 2); + + // A page that would not load is not kept, and a surface let go opens + // none. + let refused = harness( + "open-at-refused", + Fake::scripted(|command| { + (command["action"] == "navigate") + .then(|| failure("Domain 'shop.test' is not in the allowed domains list")) + }), + ); + assert!(!refused.surface.open_at("https://shop.test")); + let closed = harness("open-at-closed", shop()); + closed.surface.close(); + assert!(!closed.surface.open_at("https://shop.test")); + assert_eq!( + closed.fake.actions().len(), + 0, + "{:?}", + closed.fake.actions() + ); + // Let go after it opened early, it keeps no page for a later session. + let let_go = harness("open-at-let-go", shop()); + assert!(let_go.surface.open_at("https://shop.test")); + let_go.surface.close(); + // A new session shows the shop drawn: a page kept would skip the load. + assert!(let_go.surface.launch("browser").ok); + assert!(let_go.surface.navigate("https://shop.test").ok); + assert_eq!(loads(&let_go.fake), 2); +} + +#[test] +fn a_prompt_settle_counts_quiet_from_the_start_and_waits_only_while_the_page_changes() { + // Prompt is how a surface settles unless told otherwise. + let Harness { fake, surface, .. } = harness("prompt-settle", page_fake()); + surface.settle(); + let quiet = fake.last("waitforloadstate"); + assert_eq!( + (quiet["state"].as_str(), quiet["timeout"].as_u64()), + (Some("networkquiet"), Some(1_000)) + ); + let still = fake.last("evaluate"); + let script = still["script"].as_str().unwrap(); + assert!(script.contains("MutationObserver"), "{script}"); + assert!( + script.contains("setTimeout(done, 400)"), + "capped at SETTLE_MS: {script}" + ); + assert!(script.contains(">= 120"), "still for STILL_MS: {script}"); + assert!( + script.contains("getAnimations"), + "waits out CSS animations: {script}" + ); + assert!( + script.contains("if (finished) return"), + "stops looking at frames once settled: {script}" + ); + assert!( + script.contains("element.shadowRoot"), + "watches shadow roots too: {script}" + ); + assert!( + !fake.actions().iter().any(|action| action == "wait"), + "no fixed pause: {:?}", + fake.actions() + ); +} + +#[test] +fn a_brief_settle_waits_only_while_the_page_changes() { + // A launch or Escape fetched nothing: no wait for the network. + let brief = harness("brief-settle", page_fake()); + brief.surface.settle_briefly(); + assert!( + !brief + .fake + .actions() + .iter() + .any(|action| action == "waitforloadstate" || action == "wait"), + "{:?}", + brief.fake.actions() + ); + let still = brief.fake.last("evaluate"); + assert!( + still["script"].as_str().unwrap().contains("getAnimations"), + "the page is still watched until it stops changing" + ); + // Settling steadily settles in full, as it always did. + let steady = harness("brief-steady", page_fake()); + steady + .surface + .clone() + .with_settle(crate::Settle::Steady) + .settle_briefly(); + assert_eq!(steady.fake.last("waitforloadstate")["state"], "networkidle"); + assert_eq!(steady.fake.last("wait")["timeout"], 400); +} + +#[test] +fn a_wait_for_a_change_ends_at_the_pages_first_change_or_its_time() { + let Harness { fake, surface, .. } = harness("await-change", page_fake()); + assert!(surface.open()); + assert!(surface.await_change(1_000), "the page changed"); + let watch = fake.last("evaluate"); + let script = watch["script"].as_str().unwrap(); + assert!(script.contains("MutationObserver"), "{script}"); + assert!( + script.contains("element.shadowRoot"), + "watches shadow roots too: {script}" + ); + assert!( + script.contains("done(false), 1000"), + "still once the time given passes: {script}" + ); + assert!( + script.contains("'data-tc-'"), + "sight's own marks are no change: {script}" + ); + assert!( + !fake.actions().iter().any(|action| action == "wait"), + "no fixed pause: {:?}", + fake.actions() + ); + + let still = harness( + "await-still", + Fake::scripted(|command| { + (command["action"] == "evaluate").then(|| ok(&json!({"result": false}))) + }), + ); + assert!(still.surface.open()); + assert!(!still.surface.await_change(1_000), "the page stayed still"); + + // A watch that cannot run says the page may have changed, so a caller + // looks again as it would after a pause. + let unwatched = harness("await-unwatched", Fake::new()); + assert!(unwatched.surface.open()); + assert!( + unwatched.surface.await_change(1_000), + "no answer of its own" + ); + // With no page open, there is nothing to watch and nothing is opened. + let closed = harness("await-closed", Fake::new()); + assert!(closed.surface.await_change(1_000), "no session to watch"); + assert!( + closed.fake.actions().is_empty(), + "{:?}", + closed.fake.actions() + ); +} + +#[test] +fn a_watch_or_wait_the_page_never_answers_is_given_up_on() { + // Live, an evaluate sent while a page was being replaced waited out the + // browser's 30 s deadline. The harness is kept whole: its runtime runs + // the deadlines. + let watched = harness( + "watch-stalled", + page_fake().stalling(|command| command["action"] == "evaluate"), + ); + assert!(watched.surface.open()); + let started = std::time::Instant::now(); + assert!( + watched.surface.await_change(10), + "a watch given up on may have seen a change" + ); + watched.surface.settle(); + assert!( + started.elapsed() < std::time::Duration::from_secs(5), + "{:?}", + started.elapsed() + ); + + let waited = harness( + "quiet-stalled", + page_fake().stalling(|command| command["action"] == "waitforloadstate"), + ); + assert!(waited.surface.open()); + let started = std::time::Instant::now(); + waited.surface.settle(); + assert!( + started.elapsed() < std::time::Duration::from_secs(5), + "{:?}", + started.elapsed() + ); + let still = waited.fake.last("evaluate"); + assert!( + still["script"].as_str().unwrap().contains("getAnimations"), + "the page is still watched once the wait is given up on" + ); +} + +#[test] +fn a_surface_opens_its_session_when_asked_rather_than_at_first_use() { + let Harness { fake, surface, .. } = harness("open-early", page_fake()); + assert!(surface.session().is_none()); + assert!(surface.open()); + assert!(surface.session().is_some()); + assert!(fake.actions().iter().any(|action| action == "launch")); + let launches = fake.actions().len(); + assert!(surface.open(), "already open"); + assert_eq!(fake.actions().len(), launches, "nothing launched twice"); + + let refused = Fake::scripted(|command| { + (command["action"] == "launch").then(|| failure("Chrome not found")) + }); + let Harness { surface, .. } = harness("open-refused", refused); + assert!(!surface.open()); + + // A surface let go before its early open began is not opened: a task + // cancelled while planning holds no browser. + let Harness { fake, surface, .. } = harness("open-closed", page_fake()); + surface.close(); + assert!(!surface.open()); + assert!(surface.session().is_none()); + assert!(fake.actions().is_empty(), "{:?}", fake.actions()); +} diff --git a/crates/tinycomputer-browser/src/surface/surface_tests/perception_tests.rs b/crates/tinycomputer-browser/src/surface/surface_tests/perception_tests.rs index 15484078..0a35d321 100644 --- a/crates/tinycomputer-browser/src/surface/surface_tests/perception_tests.rs +++ b/crates/tinycomputer-browser/src/surface/surface_tests/perception_tests.rs @@ -120,3 +120,131 @@ fn the_tree_is_read_when_sight_fails_or_is_turned_off() { assert!(!fake.actions().iter().any(|action| action == "evaluate")); assert!(format!("{surface:?}").contains("Tree")); } + +/// A page read by sight whose `shadows` show controls: a covered "Add To +/// Cart", and under a host the tree reads a consent banner's buttons, or +/// fails to when `subtree_fails`. +fn shadowed_fake(shadows: serde_json::Value, subtree_fails: bool) -> Fake { + Fake::scripted(move |command| match command["action"].as_str().unwrap() { + "evaluate" + if command["script"] + .as_str() + .unwrap() + .contains("__tinycomputerSeen") => + { + Some(ok(&json!({"result": { + "ok": true, + "title": "Glasses", + "surface": "window", + "unreachable": 0, + "shadows": shadows, + "denoised": {"ads": 0, "empty": 0, "hidden": 0}, + "nodes": [ + {"id": "1", "role": "button", "name": "Add To Cart", "states": ["covered"], "path": ["main"]}, + {"text": "Limited Period Offer", "path": ["main"]} + ] + }}))) + } + "snapshot" if command.get("selector").is_some() && subtree_fails => { + Some(failure("no such element")) + } + "snapshot" if command.get("selector").is_some() => Some(ok(&json!({ + "snapshot": "- generic\n - paragraph\n - StaticText \"We value your privacy\"\n - button \"Allow Selection\" [ref=e2]\n - button \"Allow all\" [ref=e3]", + "refs": { + "e2": {"role": "button", "name": "Allow Selection"}, + "e3": {"role": "button", "name": "Allow all"} + } + }))), + _ => None, + }) +} + +#[test] +fn a_shadow_roots_controls_are_read_by_the_tree_beside_sight() { + // Live, a consent banner in a shadow root lay over "Add To Cart": sight + // could not read it, and giving the whole page to the tree read the + // rest of the page worse. + let banner = json!([{"id": "9", "label": "popover \"We value your privacy\""}]); + let Harness { fake, surface, .. } = harness("shadow-merged", shadowed_fake(banner, false)); + let screen = surface.observe("", None, Depth::Full).unwrap(); + let names = screen + .candidates + .iter() + .map(|candidate| { + ( + candidate.ref_id.as_str(), + candidate.name.as_deref().unwrap_or_default(), + ) + }) + .collect::>(); + assert_eq!( + names, + [ + ("seen:1", "Add To Cart"), + ("e2", "Allow Selection"), + ("e3", "Allow all") + ] + ); + let allow = &screen.candidates[1]; + assert_eq!( + allow.path[0], "popover \"We value your privacy\"", + "{:?}", + allow.path + ); + assert!( + allow.order > screen.candidates[0].order, + "read after sight's nodes" + ); + assert!( + screen + .context + .iter() + .any(|line| line.contains("We value your privacy")) + ); + let subtree = fake.last("snapshot"); + assert_eq!(subtree["selector"], r#"[data-tc-seen="9"]"#, "{subtree}"); + + // A shadow root that draws no layer keeps the tree's own places. + let plain = json!([{"id": "9", "label": null}]); + let Harness { surface, .. } = harness("shadow-plain", shadowed_fake(plain, false)); + let screen = surface.observe("", None, Depth::Full).unwrap(); + assert!( + !screen.candidates[1] + .path + .first() + .is_some_and(|label| label.starts_with("popover")), + "{:?}", + screen.candidates[1].path + ); +} + +#[test] +fn the_tree_reads_the_page_when_two_shadow_roots_show_or_one_cannot_be_read() { + let two = json!([{"id": "9", "label": null}, {"id": "10", "label": null}]); + let Harness { fake, surface, .. } = harness("shadow-two", shadowed_fake(two, false)); + let screen = surface.observe("", None, Depth::Full).unwrap(); + assert!( + screen + .candidates + .iter() + .all(|candidate| !candidate.ref_id.starts_with("seen:")) + ); + assert!( + fake.last("snapshot").get("selector").is_none(), + "the whole page" + ); + + let one = json!([{"id": "9", "label": null}]); + let Harness { fake, surface, .. } = harness("shadow-failed", shadowed_fake(one, true)); + let screen = surface.observe("", None, Depth::Full).unwrap(); + assert!( + screen + .candidates + .iter() + .all(|candidate| !candidate.ref_id.starts_with("seen:")) + ); + assert!( + fake.last("snapshot").get("selector").is_none(), + "the whole page" + ); +} diff --git a/crates/tinycomputer-browser/src/surface/tabs.rs b/crates/tinycomputer-browser/src/surface/tabs.rs index 820a3cfa..0a09209c 100644 --- a/crates/tinycomputer-browser/src/surface/tabs.rs +++ b/crates/tinycomputer-browser/src/surface/tabs.rs @@ -2,8 +2,9 @@ //! page that drew before it finished loading as open. use serde_json::{Value, json}; +use tinycomputer_bus::browser::SessionId; -use super::{BrowserSurface, sight}; +use super::{BrowserSurface, READ_TIMEOUT, sight}; /// Where the page is, what it is called, and whether it has drawn words. const DRAWN_JS: &str = r"(() => ({ @@ -80,22 +81,31 @@ const SAME_TAB_JS: &str = r"(element => { })"; impl BrowserSurface { - /// The page's address and title when the session shows `url` drawn + /// The page's address and title when session `id` shows `url` drawn /// with words, though its navigation timed out waiting for `load`: a /// heavy page keeps fetching long after it can be read (live, a store's /// results page). `None` when it shows another page, or nothing yet. - pub(super) fn drawn_page(&self, url: &str) -> Option<(String, String)> { - let id = self.ensure_session().ok()?; + pub(super) fn drawn_page(&self, id: &SessionId, url: &str) -> Option<(String, String)> { + self.shown_page(id) + .filter(|(shown, _)| place(shown) == place(url)) + } + + /// The address and title of the page session `id` shows, once it has + /// drawn words; `None` while it shows nothing yet. + pub(super) fn shown_page(&self, id: &SessionId) -> Option<(String, String)> { + let reading = self + .browser + .command(id, json!({"action": "evaluate", "script": DRAWN_JS})); + // Within a deadline: a call sent while a page is being replaced (a + // redirect, a challenge's reload) can wait out the browser's own 30 s. let data = self - .block( - self.browser - .command(&id, json!({"action": "evaluate", "script": DRAWN_JS})), - ) + .block(async { tokio::time::timeout(READ_TIMEOUT, reading).await }) + .ok()? .ok()?; let page = data.get("result")?; let shown = page.get("url").and_then(Value::as_str)?; let drawn = page.get("drawn").and_then(Value::as_bool).unwrap_or(false); - (drawn && place(shown) == place(url)).then(|| { + drawn.then(|| { ( shown.to_owned(), page.get("title") diff --git a/crates/tinycomputer-browser/src/surface/watch.rs b/crates/tinycomputer-browser/src/surface/watch.rs new file mode 100644 index 00000000..0079822e --- /dev/null +++ b/crates/tinycomputer-browser/src/surface/watch.rs @@ -0,0 +1,78 @@ +//! The scripts that watch a page change: whether it has gone still after an +//! action, and whether it changes at all within a given time. Each watches +//! the document and every open shadow root in it, where a web component +//! draws its rows, and each runs within a deadline of its own. + +use std::time::Duration; + +use super::{SETTLE_MS, STILL_MS}; + +/// How much longer than its own cap a watch may take to answer before it +/// is given up on: an evaluate sent while a page was being replaced waited +/// out the browser's 30 s deadline live. +const SLACK_MS: u64 = 500; + +/// The deadline for a watch, or a wait, whose own cap is `ms`. +pub(super) fn deadline(ms: u64) -> Duration { + Duration::from_millis(ms + SLACK_MS) +} + +/// Starts `watcher` on the document and on every open shadow root in it, +/// shadow roots inside shadow roots too. +const OBSERVE: &str = r"const observe = watcher => { + const options = { subtree: true, childList: true, attributes: true, characterData: true }; + const within = root => { + watcher.observe(root, options); + for (const element of root.querySelectorAll('*')) if (element.shadowRoot) within(element.shadowRoot); + }; + within(document); +};"; + +/// A promise that resolves `true` at the page's first change of its own — +/// an element or words added, removed, or rewritten, or an element's look +/// changed, but never a `data-tc-` mark sight leaves — or `false` once `ms` +/// pass with none: a list a box fetches for the text typed shows as soon as +/// it is drawn, and a still page costs `ms` once. +pub(super) fn change_script(ms: u64) -> String { + format!( + r"new Promise(resolve => {{ + {OBSERVE} + const pages = record => record.type !== 'attributes' || !String(record.attributeName).startsWith('data-tc-'); + const watcher = new MutationObserver(records => {{ if (records.some(pages)) done(true); }}); + const done = changed => {{ watcher.disconnect(); clearTimeout(cap); resolve(changed); }}; + const cap = setTimeout(() => done(false), {ms}); + observe(watcher); +}})" + ) +} + +/// A promise that resolves once the page has gone [`STILL_MS`] without a DOM +/// change, has no finite CSS animation or transition running, and has drawn +/// at least two frames, or after [`SETTLE_MS`] at most: a banner or menu +/// fading out (which changes no DOM node) has time to finish, an unchanging +/// page costs about two frames, and an endless spinner is not waited for. +/// Once it resolves, it stops looking at frames. +pub(super) fn still_script() -> String { + format!( + r"new Promise(resolve => {{ + {OBSERVE} + let last = performance.now(); + let frames = 0; + let finished = false; + const watcher = new MutationObserver(() => {{ last = performance.now(); }}); + const done = () => {{ finished = true; watcher.disconnect(); clearTimeout(cap); resolve(true); }}; + const cap = setTimeout(done, {SETTLE_MS}); + observe(watcher); + const moving = () => typeof document.getAnimations === 'function' + && document.getAnimations().some(animation => animation.playState === 'running' + && Number.isFinite(animation.effect?.getComputedTiming?.().endTime ?? Infinity)); + const frame = () => {{ + if (finished) return; + frames += 1; + if (frames >= 2 && performance.now() - last >= {STILL_MS} && !moving()) done(); + else requestAnimationFrame(frame); + }}; + requestAnimationFrame(frame); +}})" + ) +} diff --git a/crates/tinycomputer-bus/src/agent/types/request.rs b/crates/tinycomputer-bus/src/agent/types/request.rs index d27bc5d3..f6904c94 100644 --- a/crates/tinycomputer-bus/src/agent/types/request.rs +++ b/crates/tinycomputer-bus/src/agent/types/request.rs @@ -142,7 +142,9 @@ pub enum PaymentMode { pub struct TaskBudget { /// Actions across every surface. pub max_actions: Option, - /// Jev evaluations. Every framing of a voted decision counts as one. + /// Jev evaluations. Every framing of a voted decision counts as one. A + /// task given this cap does not warm Jev's connections while it is + /// planned, as warming them would spend calls the cap does not count. pub max_model_calls: Option, /// How many ways each decision is asked before its answers are averaged; /// the module's default when unset. diff --git a/crates/tinycomputer-bus/src/agentic/types/config.rs b/crates/tinycomputer-bus/src/agentic/types/config.rs index 97456141..414416cf 100644 --- a/crates/tinycomputer-bus/src/agentic/types/config.rs +++ b/crates/tinycomputer-bus/src/agentic/types/config.rs @@ -60,8 +60,8 @@ pub struct JevConfig { /// [`JevProvider::default_model`]. Sage takes no model selection and /// ignores it. pub model: Option, - /// Per-attempt HTTP timeout. Absent means the client default. Ignored by - /// Sage. + /// Per-attempt HTTP timeout. Absent means the module's default: 10 + /// seconds. Ignored by Sage. pub timeout_ms: Option, /// Additional transient retries. Absent means the module's default: four, /// waiting 1, 2, 4, then 8 seconds between attempts. Ignored by Sage. diff --git a/crates/tinycomputer-bus/src/agentic/types/result.rs b/crates/tinycomputer-bus/src/agentic/types/result.rs index 3dc1f3f2..3ac6bfe3 100644 --- a/crates/tinycomputer-bus/src/agentic/types/result.rs +++ b/crates/tinycomputer-bus/src/agentic/types/result.rs @@ -165,13 +165,17 @@ pub enum JevStopReason { /// Aggregate provider measurements for one result. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct JevMetrics { - /// Jev evaluations performed. + /// Jev evaluations performed. A decision ended on a quorum counts the + /// framings it did not wait for too: they still run, and are charged. pub calls: u32, - /// HTTP attempts including retries. + /// HTTP attempts including retries, of the evaluations waited for. pub attempts: u32, - /// Total provider latency in milliseconds. + /// Total provider latency in milliseconds, of the evaluations waited + /// for. pub latency_ms: u64, - /// Provider-reported input tokens. + /// Provider-reported input tokens, of the evaluations waited for. Those + /// of framings a quorum did not wait for end after their decision, and + /// only the journal records them. pub input_tokens: u64, /// Provider-reported output tokens. pub output_tokens: u64, diff --git a/crates/tinycomputer-bus/src/flow/guide.md b/crates/tinycomputer-bus/src/flow/guide.md index 1f4f6c02..6b3791e2 100644 --- a/crates/tinycomputer-bus/src/flow/guide.md +++ b/crates/tinycomputer-bus/src/flow/guide.md @@ -39,7 +39,7 @@ do. | `do` | `{"do": "start a new note"}` | Same as a plain string. | | `enter` | `{"enter": {"subject": "Hi"}}` | Put each text into the field its key describes; a box that suggests matches as you type (a location, a city) has the matching suggestion picked. | | `choose` | `{"choose": {"what": "the font list", "option": "Helvetica"}}` | Pick an option in a list, menu, or popup, or in a group of option buttons (a size, a colour, a quantity, a day in a strip of dates). | -| `read` | `{"read": {"what": "the newest message's subject", "into": "subject"}}` | Store visible text in a variable. | +| `read` | `{"read": {"what": "the newest message's subject", "into": "subject"}}` | Store visible text in a variable. Read the price of one item before any step raises its count: after that, its line and the cart show the total for all of them. | | `extract` | `{"extract": {"what": "the flight results", "into": "flights"}}` | Store every item of a list, as JSON rows of their text, in a variable. | | `pick` | `{"pick": {"from": "the flight results", "by": "lowest price", "into": "flight"}}` | Choose the best of a list of results (cards or rows, each an item to open) and open it; `into` stores its text. Prices, times, durations, and stops are compared exactly. | | `verify` | `{"verify": "the draft shows a recipient"}` | Fail the flow unless this holds. | @@ -155,7 +155,9 @@ do. buttons only once the item is in the cart, so to buy more than one, add the item first and raise its count in the next step; a − count + stepper where the add button was means the item is in the cart with - that count. A + that count. To report the price of one, `read` it before raising the + count: after that, the item's line and the cart show the total for all + of them, and a cart often shows no price for one. A `pick` opens a whole result card; to press one of several buttons inside the cards (a time or a slot listed under each place), use a plain step that names it ("press the earliest time listed"). A dialog's headings diff --git a/crates/tinycomputer-core/src/surface/mod.rs b/crates/tinycomputer-core/src/surface/mod.rs index 4ca1e48f..6fe04a97 100644 --- a/crates/tinycomputer-core/src/surface/mod.rs +++ b/crates/tinycomputer-core/src/surface/mod.rs @@ -68,12 +68,33 @@ pub trait Surface: Clone + Send + 'static { /// Launches `app`, or brings it forward when it is already running. fn launch(&self, app: &str) -> DesktopResponse; - /// Gives the application a moment to finish reacting: after every action, - /// before the next look, and before a value is read back — a page that - /// closes a banner a beat after the click, or a token field turning an - /// address into a token. + /// Gives the application a moment to finish reacting: after an action + /// (one that fetches nothing settles briefly instead, + /// [`Surface::settle_briefly`]), before the next look, and before a value + /// is read back — a page that closes a banner a beat after the click, or + /// a token field turning an address into a token. fn settle(&self) {} + /// Settles after an action that fetches nothing — launching what is + /// already open, Escape closing a layer — so only the application's + /// own movement is waited out. A surface that cannot tell such an + /// action apart settles as after any other. + fn settle_briefly(&self) { + self.settle(); + } + + /// Waits, for up to the given milliseconds, for the application to + /// change by itself — a list of suggestions a box fetches for the text + /// just typed, the rest of a page arriving — and says whether it did, so + /// a caller watching for something to appear stops once nothing moves. + /// + /// A surface that cannot watch for a change pauses as a `Wait` does, + /// however long it was given, and says it may have changed. + fn await_change(&self, _ms: u64) -> bool { + let _paused = self.execute(JevOperation::Wait, None, None); + true + } + /// Loads `url`, for a surface that has addresses. /// /// A desktop application has none, so the default refuses with diff --git a/crates/tinycomputer-core/src/surface/surface_tests/delivery_tests.rs b/crates/tinycomputer-core/src/surface/surface_tests/delivery_tests.rs index 535a3bd9..c2de727a 100644 --- a/crates/tinycomputer-core/src/surface/surface_tests/delivery_tests.rs +++ b/crates/tinycomputer-core/src/surface/surface_tests/delivery_tests.rs @@ -75,6 +75,54 @@ impl Surface for TextBackend { } } +/// [`TextBackend`], counting how often it settles in full. +#[derive(Clone, Default)] +struct Settling { + text: TextBackend, + settled: Arc, +} + +impl Surface for Settling { + fn observe( + &self, + app: &str, + root: Option<&str>, + depth: Depth, + ) -> Result> { + self.text.observe(app, root, depth) + } + + fn execute( + &self, + operation: JevOperation, + target: Option, + text: Option, + ) -> DesktopResponse { + self.text.execute(operation, target, text) + } + + fn read_value(&self, target: &Candidate) -> Option { + self.text.read_value(target) + } + + fn paste(&self, app: &str, target: &Candidate, text: &str) -> DesktopResponse { + self.text.paste(app, target, text) + } + + fn press(&self, app: &str, combo: &str) -> DesktopResponse { + self.text.press(app, combo) + } + + fn launch(&self, app: &str) -> DesktopResponse { + self.text.launch(app) + } + + fn settle(&self) { + self.settled + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + } +} + fn field() -> Candidate { Candidate { ref_id: "@s:e1".to_owned(), @@ -189,6 +237,18 @@ fn text_that_never_arrives_is_reported_as_not_delivered() { #[test] fn a_surface_settles_instantly_and_has_no_addresses_unless_it_says_otherwise() { Surface::settle(&TextBackend::default()); + // Settling briefly is settling in full, unless the surface can tell them + // apart: the desktop's own settle runs after a launch or Escape. + let settling = Settling::default(); + settling.settle_briefly(); + assert_eq!( + settling.settled.load(std::sync::atomic::Ordering::SeqCst), + 1 + ); + assert!( + Surface::await_change(&TextBackend::default(), 1_000), + "one that cannot watch pauses and says it may have changed" + ); let refused = Surface::navigate(&TextBackend::default(), "https://example.com"); assert_eq!(refused.error.unwrap().code, "ACTION_NOT_SUPPORTED"); let refused = Surface::back(&TextBackend::default(), "Mail"); diff --git a/crates/tinycomputer-engine/src/agentic/agentic_tests.rs b/crates/tinycomputer-engine/src/agentic/agentic_tests.rs index 98cf7ef2..df4f071b 100644 --- a/crates/tinycomputer-engine/src/agentic/agentic_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/agentic_tests.rs @@ -12,6 +12,7 @@ mod policy_tests; mod resolve_tests; mod scope_tests; mod waiting_tests; +mod warm_tests; use std::{ collections::{BTreeMap, VecDeque}, @@ -364,6 +365,7 @@ fn runtime_recording( }, pending: Arc::new(Mutex::new(std::collections::HashMap::new())), journal: super::journal::Journal::default(), + copies: Arc::default(), }, requests, ) diff --git a/crates/tinycomputer-engine/src/agentic/agentic_tests/resolve_tests.rs b/crates/tinycomputer-engine/src/agentic/agentic_tests/resolve_tests.rs index c6df3f0e..265010cb 100644 --- a/crates/tinycomputer-engine/src/agentic/agentic_tests/resolve_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/agentic_tests/resolve_tests.rs @@ -45,6 +45,23 @@ fn jev_calls_ride_out_a_provider_outage_unless_told_otherwise() { assert_eq!(told.initial_backoff, RETRY.initial_backoff); } +#[test] +fn a_jev_attempt_gives_up_after_ten_seconds_unless_told_otherwise() { + // Live, a request nothing came back for waited the client's own 30 s + // before its retry answered in under a second. + let mut request = JevConfig::new("key"); + assert_eq!( + client_config(&request).timeout, + std::time::Duration::from_secs(10) + ); + request.timeout_ms = Some(30_000); + assert_eq!( + client_config(&request).timeout, + std::time::Duration::from_secs(30), + "a configured timeout wins" + ); +} + #[test] fn each_provider_selects_its_decision_model() { let configured = |value: serde_json::Value| { diff --git a/crates/tinycomputer-engine/src/agentic/agentic_tests/warm_tests.rs b/crates/tinycomputer-engine/src/agentic/agentic_tests/warm_tests.rs new file mode 100644 index 00000000..d87fbf29 --- /dev/null +++ b/crates/tinycomputer-engine/src/agentic/agentic_tests/warm_tests.rs @@ -0,0 +1,110 @@ +//! Tests for warming a runtime's connections while a task's plan is drafted. + +use super::*; +use crate::agentic::runtime::{FIRST_TURN, WARM_TIMEOUT}; + +/// An evaluator nothing ever comes back from. +struct Silent; + +impl Evaluator for Silent { + fn evaluate<'a>( + &'a self, + _request: &'a tinyinference_decisions::EvaluationRequest, + ) -> std::pin::Pin< + Box< + dyn std::future::Future< + Output = Result< + tinyinference_decisions::EvaluationResult, + tinyinference_decisions::EvaluationFailure, + >, + > + Send + + 'a, + >, + > { + Box::pin(std::future::pending()) + } +} + +fn ready() -> tinyinference_decisions::EvaluationResult { + evaluation(json!({ + "model": "typesafe/jev-1.13-20260917", + "answers": {"ready": {"type": "noul", "noul": 0.9}}, + "usage": {"input_tokens": 10, "output_tokens": 2} + })) +} + +#[tokio::test] +async fn a_warm_up_asks_one_small_question_for_each_call_of_a_first_turn() { + let (runtime, requests) = runtime_recording(vec![ready(); 14]); + runtime.warm(7).await; + let requests = requests.lock().unwrap(); + assert_eq!( + requests.len(), + 14, + "the judging and grounding's opening, each in every framing, all at once" + ); + for request in requests.iter() { + assert_eq!(request.model, "jev-latest", "the runtime's own model"); + assert_eq!( + request.questions.keys().collect::>(), + ["ready"], + "{request:?}" + ); + assert!(matches!( + request.questions["ready"], + tinyinference_decisions::Question::Noul(_) + )); + } +} + +#[tokio::test] +async fn a_warm_up_opens_no_more_than_a_first_turn_asks_at_once() { + let (runtime, requests) = runtime_recording(vec![ready(); 18]); + runtime.warm(50).await; + assert_eq!( + requests.lock().unwrap().len(), + 18, + "MAX_VOTES framings of each first-turn request at most" + ); + let (runtime, requests) = runtime_recording(vec![ready(); 2]); + runtime.warm(0).await; + assert_eq!( + requests.lock().unwrap().len(), + usize::try_from(FIRST_TURN).unwrap(), + "one framing of each at least" + ); +} + +#[tokio::test] +async fn sage_is_not_warmed() { + let (mut runtime, requests) = runtime_recording(Vec::new()); + runtime.configuration.provider = JevProvider::Sage; + runtime.warm(7).await; + assert!(requests.lock().unwrap().is_empty()); +} + +#[tokio::test(start_paused = true)] +async fn a_warm_up_nothing_answers_is_given_up_on() { + let (mut runtime, _requests) = runtime_recording(Vec::new()); + runtime.client = Arc::new(Silent); + let started = tokio::time::Instant::now(); + runtime.warm(7).await; + assert_eq!(started.elapsed(), WARM_TIMEOUT); +} + +#[tokio::test] +async fn a_warm_up_is_journaled_with_the_task() { + let dir = std::env::temp_dir().join(format!("tinycomputer-warm-{}", std::process::id())); + let (runtime, _requests) = runtime_recording(vec![ready(); 14]); + let runtime = runtime.with_journal(&dir).journaled_as("task-t-1"); + runtime.warm(7).await; + let journal = std::fs::read_to_string(dir.join("task-t-1").join("journal.jsonl")).unwrap(); + let steps = journal + .lines() + .map(|line| serde_json::from_str::(line).unwrap()) + .filter(|event| event["event"] == "exchange") + .map(|event| event["step"].as_str().unwrap().to_owned()) + .collect::>(); + std::fs::remove_dir_all(&dir).unwrap(); + assert_eq!(steps, vec!["warm-up"; 14]); +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/README.md b/crates/tinycomputer-engine/src/agentic/flow/README.md index e430bcf7..72fdd10b 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/README.md +++ b/crates/tinycomputer-engine/src/agentic/flow/README.md @@ -40,6 +40,7 @@ evidence, checked after acting, and undone and retried when wrong. | `view/` | re-exports the screen model and digest from `tinycomputer-core::surface`; keeps the flow's policy: the act threshold, which controls it must not press, and `named_first` | | `backend/` | `AgentBackend` (the core `Surface` trait, which `Desktop` implements in `tinycomputer-desktop/src/surface/`) and the async wrappers that call it off the executor | | `vote.rs` | framings, ballots, and their tally | +| `quorum.rs` | a decision merged without its last two framings once the rest agree plainly | | `flow_tests.rs` | the harness every flow test runs through; `flow_tests/` holds a simulated mail app and web shop (`simulator.rs`, `screens.rs`), an oracle Jev that answers from their state (`oracle.rs`), and each topic's tests in `_tests.rs`, deliberation's scenarios in `deliberation_tests.rs` | ## Operational constraints diff --git a/crates/tinycomputer-engine/src/agentic/flow/act/mod.rs b/crates/tinycomputer-engine/src/agentic/flow/act/mod.rs index 659b1caf..600ca55d 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/act/mod.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/act/mod.rs @@ -21,8 +21,8 @@ //! a screen that returns to where it was two turns ago bans both presses. //! //! The loop's pieces: `turns` runs it, `judge` reads each turn's screen, -//! `moves` makes the chosen move, and `recover` undoes a turn that went -//! wrong. This root holds the thresholds and the state they share. +//! `moves` makes the chosen move, `uncover` presses again a target the page +//! refused as covered, and `recover` undoes a turn that went wrong. This root holds the thresholds and the state they share. mod copies; mod dialog; @@ -33,6 +33,7 @@ mod judge; mod moves; mod recover; mod turns; +mod uncover; pub(super) use judge::Judgement; diff --git a/crates/tinycomputer-engine/src/agentic/flow/act/moves.rs b/crates/tinycomputer-engine/src/agentic/flow/act/moves.rs index 72f465e7..0e6ce49f 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/act/moves.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/act/moves.rs @@ -15,7 +15,7 @@ use crate::agentic::flow::{ }; use super::{ - Expected, MAX_REPEAT_PRESSES, Move, activate_purpose, covered, creates_new, dialog::in_dialog, + Expected, MAX_REPEAT_PRESSES, Move, activate_purpose, creates_new, dialog::in_dialog, judge::Judgement, }; @@ -228,7 +228,7 @@ impl FlowRun<'_, B> { } let expected = self.expect(log, operation, &target, screen); let reply = self - .press_uncovering(log, verb, &target, jev_operation) + .press_uncovering(log, verb, &target, jev_operation, intent) .await?; self.history .push(format!("{verb} {} ok={}", label(&target), reply.ok)); @@ -238,55 +238,6 @@ impl FlowRun<'_, B> { Ok(Some((target, expected))) } - /// Performs `operation` on an already-vetted `target`. When the click - /// is refused because something covers it — a drawer, a menu, or a - /// result card's own click layer — presses Escape once and tries the - /// same target again. Escape never chooses a new element. - pub(in crate::agentic::flow) async fn press_uncovering( - &mut self, - log: &mut StepLog, - verb: &str, - target: &Candidate, - operation: JevOperation, - ) -> Result { - let chosen = target.clone(); - let reply = self - .act(log, verb, Some(target), move |backend| { - backend.execute(operation, Some(chosen), None) - }) - .await?; - if !covered(&reply) { - return Ok(reply); - } - // A dialog in front is the page's question (a format, a quantity), - // not a popover in the way: Escape would close it, and pressing what - // lies behind it leaves the flow it began (live, a movie's language - // link behind its booking dialog led to a listing of other films). - // A layer drawn over the window is such a question only when the - // task's own press opened it; a calendar left open is in the way. - if self.front.opened_dialog || !matches!(self.front.surface.as_str(), "window" | "layer") { - self.history.push(format!( - "{} lies behind the dialog in front; act within the dialog instead", - label(target) - )); - return Ok(reply); - } - let app = self.app.clone(); - self.act(log, "press escape (uncover)", None, move |backend| { - backend.press(&app, "escape") - }) - .await?; - self.history.push(format!( - "{} was covered by something; pressed escape to close it", - label(target) - )); - let retried = target.clone(); - self.act(log, verb, Some(target), move |backend| { - backend.execute(operation, Some(retried), None) - }) - .await - } - /// Dismisses whatever is blocking the step, choosing only safe controls. pub(super) async fn clear_obstacle( &mut self, diff --git a/crates/tinycomputer-engine/src/agentic/flow/act/turns.rs b/crates/tinycomputer-engine/src/agentic/flow/act/turns.rs index 06898b29..896a42b6 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/act/turns.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/act/turns.rs @@ -4,7 +4,7 @@ use std::time::Instant; use serde_json::json; -use tinycomputer_bus::StepOutcome; +use tinycomputer_bus::{FlowLoop, StepOutcome}; use crate::agentic::flow::{ Ended, FlowRun, Halt, StepLog, @@ -83,10 +83,8 @@ impl FlowRun<'_, B> { state.turn = Some((turn, Instant::now(), self.decisions, self.rounds)); log.turns = log.turns.saturating_add(1); let screen = self.look().await?; - self.note_change(state, &screen)?; - self.note_oscillation(log, state, &screen); - if state.first.is_none() { - state.first = Some(screen.clone()); + if self.note_turn(log, state, &screen) { + return self.stalled(log, intent).await; } if let Some(ended) = state .last @@ -274,15 +272,29 @@ impl FlowRun<'_, B> { } } + /// Notes what the turn's look shows: what the last action changed, an + /// oscillation, and the step's first screen; `true` once [`STALL_TURNS`] + /// turns in a row changed nothing. + fn note_turn(&mut self, log: &mut StepLog, state: &mut DoState, screen: &Screen) -> bool { + if self.note_change(state, screen) { + return true; + } + self.note_oscillation(log, state, screen); + if state.first.is_none() { + state.first = Some(screen.clone()); + } + false + } + /// Records what the last action changed, banning an element that changed - /// nothing and failing the step after [`STALL_TURNS`] such turns. + /// nothing; `true` once [`STALL_TURNS`] such turns ran in a row. /// /// A wait that changes nothing is not a stall: the page has settled, and /// Jev is told so. It is not let wait again after [`MAX_IDLE_WAITS`] of /// them, which leaves it to judge or act on the page as it stands. - fn note_change(&mut self, state: &mut DoState, screen: &Screen) -> Result<(), Halt> { + fn note_change(&mut self, state: &mut DoState, screen: &Screen) -> bool { let Some(previous) = &state.last else { - return Ok(()); + return false; }; let changed = fingerprint(&previous.before) != fingerprint(screen); let note = change_note(&previous.before, screen, changed); @@ -296,7 +308,7 @@ impl FlowRun<'_, B> { "waited: the page has finished loading and nothing changed, so waiting longer will not change it" .to_owned(), ); - return Ok(()); + return false; } else { state.unchanged = state.unchanged.saturating_add(1); if previous.scrolled { @@ -318,17 +330,35 @@ impl FlowRun<'_, B> { if let Some(struck) = copies::strike_pending(state, changed) { self.history.push(struck); } - // A step whose work the page did by itself (a search box that lists - // results as it is typed in) has nothing left to press: the note - // says so, so a rescue skips it rather than retry it. Live, four - // rescues looked for a search button a live search does not have. - if state.unchanged >= STALL_TURNS { - return Err(Halt::Failed( - "the last three actions changed nothing on screen; if the screen already shows what this step was for, its work is done" - .to_owned(), + state.unchanged >= STALL_TURNS + } + + /// Ends a step whose last [`STALL_TURNS`] actions changed nothing: done, + /// when the screen already shows what the step was for (asked only when + /// the completion loop is on), else failed. + /// + /// A step whose work the page did by itself (a search box that lists + /// results as it is typed in) has nothing left to press. Live, rescues + /// looked for a search button a live search does not have, and 25 rescues + /// of 189 on one day found the step's work already done, each after + /// ~18 s; the question here costs one decision. A failure's note still + /// says so, so a rescue skips the step rather than retry it. + async fn stalled(&mut self, log: &mut StepLog, intent: &str) -> Result { + let condition = format!( + "the screen already shows the result that the step {intent:?} is meant to bring about" + ); + // Whether a step is done is the completion loop's question: a run + // that turned it off fails a stalled step outright, as before. + if self.enabled(FlowLoop::Completion) && self.holds(log, &condition).await? >= DONE { + return Ok(Ended::new( + StepOutcome::AlreadyDone, + "the last three actions changed nothing, and the screen already shows what this step was for", )); } - Ok(()) + Err(Halt::Failed( + "the last three actions changed nothing on screen; if the screen already shows what this step was for, its work is done" + .to_owned(), + )) } } diff --git a/crates/tinycomputer-engine/src/agentic/flow/act/uncover.rs b/crates/tinycomputer-engine/src/agentic/flow/act/uncover.rs new file mode 100644 index 00000000..5032fb1d --- /dev/null +++ b/crates/tinycomputer-engine/src/agentic/flow/act/uncover.rs @@ -0,0 +1,92 @@ +//! Pressing a target the page refuses as covered: closing what lies over +//! it, then pressing the same target once more. + +use tinycomputer_bus::JevOperation; + +use crate::agentic::flow::{ + FlowRun, Halt, StepLog, + attention::front_closer, + backend::AgentBackend, + view::{Candidate, label, signature}, +}; + +use super::covered; + +impl FlowRun<'_, B> { + /// Performs `operation` on an already-vetted `target`. When the click + /// is refused because something covers it — a drawer, a menu, a consent + /// banner, or a result card's own click layer — closes what covers it + /// once and tries the same target again: with the least committal + /// control of a layer in front ([`front_closer`]; never a layer the + /// step's `intent` names, or the one `target` sits in), or else with + /// Escape. Neither chooses a new target. + pub(in crate::agentic::flow) async fn press_uncovering( + &mut self, + log: &mut StepLog, + verb: &str, + target: &Candidate, + operation: JevOperation, + intent: &str, + ) -> Result { + let chosen = target.clone(); + let reply = self + .act(log, verb, Some(target), move |backend| { + backend.execute(operation, Some(chosen), None) + }) + .await?; + if !covered(&reply) { + return Ok(reply); + } + // A dialog in front is the page's question (a format, a quantity), + // not a popover in the way: Escape would close it, and pressing what + // lies behind it leaves the flow it began (live, a movie's language + // link behind its booking dialog led to a listing of other films). + // A layer drawn over the window is such a question only when the + // task's own press opened it; a calendar left open is in the way. + if self.front.opened_dialog || !matches!(self.front.surface.as_str(), "window" | "layer") { + self.history.push(format!( + "{} lies behind the dialog in front; act within the dialog instead", + label(target) + )); + return Ok(reply); + } + // A layer in front that closes with a control of its own (a consent + // banner's "Allow Selection") is closed with it: Escape leaves such a + // banner where it is. + let screen = self.look().await?; + if let Some(closer) = front_closer( + &screen, + target, + intent, + &self.stop_before, + &self.step_cleared, + ) { + self.step_cleared.insert(signature(&closer)); + let pressed = closer.clone(); + self.act(log, "click (uncover)", Some(&closer), move |backend| { + backend.execute(JevOperation::Click, Some(pressed), None) + }) + .await?; + self.history.push(format!( + "{} was covered by a layer in front; pressed {} to close it", + label(target), + label(&closer) + )); + } else { + let app = self.app.clone(); + self.act(log, "press escape (uncover)", None, move |backend| { + backend.press(&app, "escape") + }) + .await?; + self.history.push(format!( + "{} was covered by something; pressed escape to close it", + label(target) + )); + } + let retried = target.clone(); + self.act(log, verb, Some(target), move |backend| { + backend.execute(operation, Some(retried), None) + }) + .await + } +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/action.rs b/crates/tinycomputer-engine/src/agentic/flow/action.rs index 00578fec..b70f9eeb 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/action.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/action.rs @@ -43,6 +43,7 @@ impl FlowRun<'_, B> { } let note = match (&reply.error, &reply.data) { (Some(error), _) => error.code.clone(), + (None, Some(_)) if still(&reply) => "nothing changed".to_owned(), (None, Some(data)) => data .get("path") .and_then(serde_json::Value::as_str) @@ -63,11 +64,18 @@ impl FlowRun<'_, B> { note, }); let settle_started = Instant::now(); - if reply.ok { + // A wait that saw the surface stay still has nothing to settle. + let settles = reply.ok && !still(&reply); + if settles { // Let the surface finish reacting, so the next look sees what the // action did rather than the moment before it took effect. - self.backend_call(|backend| { - backend.settle(); + let briefly = fetches_nothing(action); + self.backend_call(move |backend| { + if briefly { + backend.settle_briefly(); + } else { + backend.settle(); + } DesktopResponse::ok("settle", serde_json::json!({})) }) .await; @@ -81,12 +89,30 @@ impl FlowRun<'_, B> { "ok": reply.ok, "note": record.map(|record| record.note.as_str()), "wall_ms": acted_ms, - "settle_ms": if reply.ok { millis(settle_started.elapsed()) } else { 0 }, + "settle_ms": if settles { millis(settle_started.elapsed()) } else { 0 }, }) }); Ok(reply) } + /// Waits up to `ms` for the surface to change by itself + /// (`Surface::await_change`), as one `wait` action charged to the budget + /// and the step log: settled when the surface changed, and `false` when + /// it stayed still, so a caller watching for something to appear stops. + pub(in crate::agentic::flow) async fn await_change( + &mut self, + log: &mut StepLog, + ms: u64, + ) -> Result { + let reply = self + .act(log, "wait", None, move |backend| { + let changed = backend.await_change(ms); + DesktopResponse::ok("wait", json!({ "still": !changed })) + }) + .await?; + Ok(!still(&reply)) + } + async fn backend_call(&self, call: F) -> DesktopResponse where F: FnOnce(B) -> DesktopResponse + Send + 'static, @@ -94,3 +120,23 @@ impl FlowRun<'_, B> { blocking(self.backend.clone(), call).await } } + +/// Whether `action` fetches nothing the next look must wait for, so the +/// surface settles briefly after it (`Surface::settle_briefly`): a launch, +/// which leaves an open page as it is, or Escape closing a layer. Live, 3% +/// of launches and 12% of Escapes settled with a request of the page still +/// running, against 39% of fills (fetching suggestions the next look reads) +/// and 70% of clicks, which settle in full. +fn fetches_nothing(action: &str) -> bool { + action == "launch" || action.starts_with("launch ") || action.starts_with("press escape") +} + +/// Whether `reply` is a wait's that saw the surface stay still. +fn still(reply: &DesktopResponse) -> bool { + reply + .data + .as_ref() + .and_then(|data| data.get("still")) + .and_then(Value::as_bool) + .unwrap_or(false) +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/attention/attention_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/attention/attention_tests.rs index 57d0cd09..ab1b2a40 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/attention/attention_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/attention/attention_tests.rs @@ -4,7 +4,7 @@ use std::collections::BTreeSet; -use super::{ESCAPED, MAX_DISTRACTION_SIZE, MAX_DISTRACTIONS, find::distractions}; +use super::{ESCAPED, MAX_DISTRACTION_SIZE, MAX_DISTRACTIONS, find::distractions, front_closer}; use crate::agentic::flow::view::{Candidate, Screen, signature}; fn button(name: &str, path: &[&str]) -> Candidate { @@ -269,3 +269,73 @@ fn something_covering_what_the_step_needs_is_cleared_with_escape() { .is_empty() ); } + +#[test] +fn a_covered_press_closes_a_layer_in_front_but_never_its_own() { + // A size popover the step works in, with a toast lying over its rows. + let sizes = ["main", "popover \"Sizes\""]; + let toast = ["alert \"Saved to your wishlist\""]; + let target = button("Size M", &sizes); + let mut candidates = content(); + candidates.extend([target.clone(), button("Close", &sizes)]); + let with_toast = |mut candidates: Vec| { + candidates.push(button("Close", &toast)); + screen(candidates) + }; + let closer = front_closer( + &with_toast(candidates.clone()), + &target, + "choose size M", + &[], + &BTreeSet::new(), + ) + .unwrap(); + assert_eq!(closer.path, toast, "the toast's, not the popover's"); + + // With only the step's own layer in front, nothing is closed. + let own = front_closer( + &screen(candidates.clone()), + &target, + "choose size M", + &[], + &BTreeSet::new(), + ); + assert!(own.is_none(), "{own:?}"); + + // Nor is the target itself, the toast's own button. + let close_toast = button("Close", &toast); + let itself = front_closer( + &with_toast(content()), + &close_toast, + "close the toast", + &[], + &BTreeSet::new(), + ); + assert!(itself.is_none(), "{itself:?}"); + + // A layer the step names is the step's. + let mut candidates = content(); + candidates.extend(consent().into_iter().map(|mut control| { + control.path = vec!["dialog \"Cookie consent\"".to_owned()]; + control + })); + let named = front_closer( + &screen(candidates.clone()), + &candidates[0], + "accept the cookie consent", + &[], + &BTreeSet::new(), + ); + assert!(named.is_none(), "{named:?}"); + let other = front_closer( + &screen(candidates.clone()), + &candidates[0], + "search for flights", + &[], + &BTreeSet::new(), + ); + assert_eq!( + other.and_then(|closer| closer.name).as_deref(), + Some("Reject all") + ); +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/attention/find.rs b/crates/tinycomputer-engine/src/agentic/flow/attention/find.rs index 88e94a85..007cc318 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/attention/find.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/attention/find.rs @@ -218,6 +218,7 @@ pub(in crate::agentic::flow) fn distractions( .map(|(_, member)| label(member)) .collect(), closer: Some(closer.clone()), + front, }, )); } @@ -305,6 +306,7 @@ fn covering(screen: &Screen, intent: &[String], cleared: &BTreeSet) -> O name: "something open over the page".to_owned(), shows: needed.into_iter().chain(front).take(6).collect(), closer: None, + front: true, }) } diff --git a/crates/tinycomputer-engine/src/agentic/flow/attention/mod.rs b/crates/tinycomputer-engine/src/agentic/flow/attention/mod.rs index 5a66d5f0..c9abb494 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/attention/mod.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/attention/mod.rs @@ -14,7 +14,10 @@ //! Only when there is a candidate is Jev asked, with one Choice, and only a //! clearly agreed pick (`evidence/`) is cleared, with the region's //! least-committal control: rejecting or essential-only first, closing next, -//! accepting last. +//! accepting last. One exception asks no one: when the page refuses a press +//! as covered, the least-committal control of a layer in front, other than +//! the step's own, is pressed before the press is tried again +//! ([`front_closer`]). //! //! `find` finds the distractions without asking anyone, and `clear` asks //! Jev and clears the one it picks. @@ -24,7 +27,7 @@ mod find; use std::collections::BTreeSet; -use super::view::Candidate; +use super::view::{Candidate, Screen, signature}; /// Most distractions one attention question offers. pub(super) const MAX_DISTRACTIONS: usize = 4; @@ -49,6 +52,31 @@ pub(super) struct Distraction { /// for something that covers the page with no control of its own, which /// Escape clears. pub(super) closer: Option, + /// Whether it lies in front of the page (the digest's front regions). + pub(super) front: bool, +} + +/// The control that closes a layer in front of the page, the least +/// committal its region holds (a consent banner's "Allow Selection" before +/// its "Allow all"), when one does and it was not pressed in this step: what +/// a press of `target` the page refused as covered clears before trying +/// again. Live, a consent banner lay over "Add To Cart", Escape left it +/// there, and every press was refused. A layer the step's `intent` names, +/// or the one `target` itself sits in (a popover a toast lies over), is the +/// step's, and is never closed this way. +pub(super) fn front_closer( + screen: &Screen, + target: &Candidate, + intent: &str, + stop_before: &[String], + cleared: &BTreeSet, +) -> Option { + let pressed = signature(target); + find::distractions(screen, intent, stop_before, cleared) + .into_iter() + .filter(|distraction| distraction.front) + .filter_map(|distraction| distraction.closer) + .find(|closer| signature(closer) != pressed && !target.path.starts_with(&closer.path)) } /// The key a step's Escape at something covering the page is remembered diff --git a/crates/tinycomputer-engine/src/agentic/flow/decide.rs b/crates/tinycomputer-engine/src/agentic/flow/decide.rs index 800d217c..ac40955d 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/decide.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/decide.rs @@ -9,7 +9,8 @@ use tinycomputer_core::Facts; use tinyinference_decisions::{Answer, EvaluationRequest, Question}; use super::{ - FlowRun, Halt, MAX_REQUEST_BYTES, StepLog, ask, backend::AgentBackend, brief::clip, vote, + FlowRun, Halt, MAX_REQUEST_BYTES, StepLog, ask, backend::AgentBackend, brief::clip, hedge, + quorum, vote, }; use crate::agentic::{journal::millis, merge_metrics, provider_error}; @@ -18,8 +19,10 @@ impl FlowRun<'_, B> { /// /// The request is briefed and masked first, then asked in as many /// framings as the run votes with — concurrently, each one charged as an - /// evaluation — and the answers are averaged. On a web page it also - /// carries a page-kind question, whose answer briefs the next request. + /// evaluation — and the answers are averaged: every framing's, or those + /// in once all but two of seven or more agree plainly (`quorum.rs`). On + /// a web page it also carries a page-kind question, whose answer briefs + /// the next request. pub(in crate::agentic::flow) async fn ask( &mut self, log: &mut StepLog, @@ -72,20 +75,27 @@ impl FlowRun<'_, B> { // A part none of whose framings answered leaves its questions // without an answer, which fails the decision as a whole. let mut unanswered = false; - for (framings, handles) in framings.into_iter().zip(handles) { + let mut left = 0_u32; + let mut asked_in = BTreeMap::new(); + for ((part, framings), handles) in parts.iter().zip(framings).zip(handles) { let before = answered.len(); - for (framing, handle) in framings.into_iter().zip(handles) { - match handle.await { - Ok(Ok(evaluation)) => { - merge_metrics(&mut self.metrics, &evaluation); - log.calls = log.calls.saturating_add(1); - answered.push((framing, evaluation.response.answers)); - } - Ok(Err(error)) => { - failure.get_or_insert(error); - } - Err(_) => {} - } + for id in part.questions.keys() { + asked_in.insert(id.clone(), framings.len()); + } + let size = quorum::size(framings.len()); + let gathered = quorum::gather(framings, handles, size).await; + for (framing, evaluation) in gathered.answered { + merge_metrics(&mut self.metrics, &evaluation); + log.calls = log.calls.saturating_add(1); + answered.push((framing, evaluation.response.answers)); + } + // Framings a quorum did not wait for still run, and are + // charged as made: their tokens are not known yet. + self.metrics.calls = self.metrics.calls.saturating_add(gathered.left); + log.calls = log.calls.saturating_add(gathered.left); + left = left.saturating_add(gathered.left); + if let Some(error) = gathered.failure { + failure.get_or_insert(error); } unanswered |= answered.len() == before; } @@ -99,6 +109,7 @@ impl FlowRun<'_, B> { let ballots = vote::ballots(&answered); let merged = vote::tally(&ballots); self.ballots.extend(ballots); + self.asked.extend(asked_in); merged } }; @@ -111,6 +122,7 @@ impl FlowRun<'_, B> { "questions": request.questions.keys().collect::>(), "framings": votes, "answered": answered.len(), + "left": left, "batched": batched, "parts": parts.len(), "request_bytes": largest(&parts), @@ -199,7 +211,7 @@ impl FlowRun<'_, B> { .collect() } - /// Sends every framing to Jev at once. + /// Sends every framing to Jev at once, each one [`hedged`](hedge::hedged). pub(super) fn spawn( &self, framings: &[vote::Framing], @@ -217,7 +229,7 @@ impl FlowRun<'_, B> { let runtime = self.runtime.clone(); let step = self.step.clone(); let request = framing.request.clone(); - tokio::spawn(async move { runtime.evaluate(Some(&step), &request).await }) + tokio::spawn(async move { hedge::hedged(&runtime, &step, &request).await }) }) .collect() } @@ -331,7 +343,7 @@ pub(in crate::agentic::flow) fn largest(parts: &[EvaluationRequest]) -> usize { } /// The size of `request`, in bytes of JSON. -fn bytes(request: &EvaluationRequest) -> usize { +pub(super) fn bytes(request: &EvaluationRequest) -> usize { serde_json::to_vec(request).map_or(0, |json| json.len()) } diff --git a/crates/tinycomputer-engine/src/agentic/flow/escalate/belief.rs b/crates/tinycomputer-engine/src/agentic/flow/escalate/belief.rs index 145fc66f..26e9499b 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/escalate/belief.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/escalate/belief.rs @@ -29,7 +29,7 @@ impl FlowRun<'_, B> { let asked = request .questions .keys() - .map(|id| self.ballot(id).len()) + .map(|id| self.asked_in(id)) .max() .unwrap_or_default(); let from = u32::try_from(asked).unwrap_or(u32::MAX); @@ -68,6 +68,10 @@ impl FlowRun<'_, B> { for (id, ballot) in fresh.clone() { self.ballots.entry(id).or_default().extend(ballot); } + let widened_to = usize::try_from(to).unwrap_or(usize::MAX); + for id in parts.iter().flat_map(|part| part.questions.keys()) { + self.asked.insert(id.clone(), widened_to); + } self.runtime.journal.record("decision", || { json!({ "step": self.step, diff --git a/crates/tinycomputer-engine/src/agentic/flow/escalate/mod.rs b/crates/tinycomputer-engine/src/agentic/flow/escalate/mod.rs index 6ce83fdd..912ba494 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/escalate/mod.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/escalate/mod.rs @@ -120,6 +120,16 @@ impl FlowRun<'_, B> { self.ballots.get(id).map_or(&[], Vec::as_slice) } + /// How many framings `id` was asked in, by the latest decision that + /// asked it and any widening since: more than its ballot holds when the + /// decision ended on a quorum. + pub(super) fn asked_in(&self, id: &str) -> usize { + self.asked + .get(id) + .copied() + .unwrap_or_else(|| self.ballot(id).len()) + } + /// Each framing's own reading of `belief`, from the latest ballots. fn framed(&self, belief: &Belief<'_>) -> Vec { let ids = [Some(belief.yes), Some(belief.no), belief.top] diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests.rs index 48512226..e39d5cd5 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests.rs @@ -27,9 +27,11 @@ mod do_loop_tests; mod end_to_end_tests; mod enter_tests; mod grounding_tests; +mod hedge_tests; mod helpers_tests; mod journal_tests; mod pick_tests; +mod quorum_tests; mod reflection_tests; mod split_tests; mod step_kinds_tests; @@ -80,9 +82,9 @@ use super::{ vote, wide, }; -fn runtime(oracle: Oracle) -> JevRuntime { +fn runtime(client: impl Evaluator + 'static) -> JevRuntime { JevRuntime { - client: Arc::new(oracle), + client: Arc::new(client), configuration: tinycomputer_bus::JevConfiguration { provider: tinycomputer_bus::JevProvider::OpenRouter, model: "jev-latest".to_owned(), @@ -91,6 +93,7 @@ fn runtime(oracle: Oracle) -> JevRuntime { }, pending: Arc::default(), journal: crate::agentic::journal::Journal::default(), + copies: Arc::default(), } } @@ -122,6 +125,7 @@ async fn run_with( }, pending: Arc::default(), journal: crate::agentic::journal::Journal::default(), + copies: Arc::default(), }; // One framing per decision, so every test that counts requests counts // decisions; voting has its own tests. diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs index cd64c13d..cd69140c 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs @@ -109,6 +109,45 @@ async fn a_covered_click_closes_what_covers_it_and_tries_again() { .collect::>(), ["click", "press escape (uncover)", "click"], ); + // The launch and Escape fetch nothing and settle briefly; the refused + // click does not settle, and the one that went through settles in full. + assert_eq!(sim.trail, ["settle briefly", "settle briefly", "settle"]); +} + +#[tokio::test] +async fn a_covered_click_closes_the_banner_in_front_with_its_own_button() { + // Live, a consent banner lay over "Add To Cart"; Escape left it there, + // and every press was refused. Its least committal button closes it. + let run = run_with( + App::quirky(Quirk::ConsentBanner), + json!({"app": "Mail", "steps": ["start a new email message"]}), + |request| request.disabled_loops.push(FlowLoop::Attention), + |id, question, _| (id == "move").then(|| pick(question, "activate", 0.9)), + ) + .await; + assert_eq!( + run.result.stop, + FlowStopReason::Completed, + "{:?}", + run.result.steps + ); + let sim = run.app.sim(); + assert!(sim.compose_open); + assert!(sim.presses.is_empty(), "no Escape: {:?}", sim.presses); + assert!( + sim.clicks.contains(&"Allow Selection".to_owned()) + && !sim.clicks.contains(&"Allow all".to_owned()), + "{:?}", + sim.clicks + ); + let actions = &run.result.steps[0].actions; + assert_eq!( + actions + .iter() + .map(|action| action.action.as_str()) + .collect::>(), + ["click", "click (uncover)", "click"], + ); } #[tokio::test] @@ -157,6 +196,40 @@ async fn actions_that_change_nothing_fail_the_step() { assert!(run.result.steps[0].note.contains("changed nothing")); } +#[tokio::test] +async fn a_stalled_step_whose_result_already_shows_is_done() { + // Live, "press Enter to search" pressed Enter three times over results a + // live search had already listed, failed, and a rescue found its work + // done ~18 s later: 25 of a day's 189 rescues were such steps. + let run = run_with( + App::quirky(Quirk::Frozen), + json!({"app": "Mail", "steps": ["press the search button"]}), + |_| {}, + |id, question, _| match id { + "done" => Some(noul(0.05)), + "move" => Some(pick(question, "activate", 0.9)), + "holds" if text_of(question, "condition").contains("already shows the result") => { + Some(noul(0.95)) + } + _ => None, + }, + ) + .await; + let step = &run.result.steps[0]; + assert_eq!( + run.result.stop, + FlowStopReason::Completed, + "{:?}", + run.result.steps + ); + assert_eq!(step.outcome, StepOutcome::AlreadyDone, "{}", step.note); + assert!( + step.note.contains("already shows what this step was for"), + "{}", + step.note + ); +} + #[tokio::test] async fn an_irreversible_control_is_refused_inside_an_ordinary_step() { let run = run_with( diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/hedge_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/hedge_tests.rs new file mode 100644 index 00000000..6847b8fb --- /dev/null +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/hedge_tests.rs @@ -0,0 +1,226 @@ +//! Hedging a framing: one still running past its hedge delay gets a copy, +//! and the first answer of the two counts. + +use super::*; +use crate::agentic::flow::hedge::hedged; + +/// A Jev that answers each call after the wait scripted for it, in call +/// order, or fails it. +struct Paced { + calls: Mutex, + script: Vec<(Duration, bool)>, +} + +impl Evaluator for Paced { + fn evaluate<'a>( + &'a self, + _request: &'a EvaluationRequest, + ) -> Pin> + Send + 'a>> + { + let call = { + let mut calls = self.calls.lock().unwrap(); + *calls += 1; + *calls - 1 + }; + let (wait, fails) = self.script.get(call).copied().unwrap_or_default(); + Box::pin(async move { + tokio::time::sleep(wait).await; + if fails { + return Err(EvaluationFailure { + error: Box::new(tinyinference_decisions::Error::RateLimited), + attempts: 1, + latency: wait, + }); + } + Ok(EvaluationResult { + response: EvaluationResponse { + model: format!("call {call}"), + answers: BTreeMap::new(), + usage: tinyinference_decisions::Usage::default(), + }, + request_id: None, + attempts: 1, + latency: wait, + }) + }) + } +} + +fn paced(script: &[(u64, bool)]) -> (JevRuntime, Arc) { + paced_as(tinycomputer_bus::JevProvider::OpenRouter, script) +} + +fn paced_as( + provider: tinycomputer_bus::JevProvider, + script: &[(u64, bool)], +) -> (JevRuntime, Arc) { + let paced = Arc::new(Paced { + calls: Mutex::new(0), + script: script + .iter() + .map(|(ms, fails)| (Duration::from_millis(*ms), *fails)) + .collect(), + }); + let runtime = JevRuntime { + client: paced.clone(), + configuration: tinycomputer_bus::JevConfiguration { + provider, + model: "jev-latest".to_owned(), + endpoint_url: None, + fast: false, + }, + pending: Arc::default(), + journal: crate::agentic::journal::Journal::default(), + copies: Arc::default(), + }; + (runtime, paced) +} + +/// A request of about `bytes` bytes. +fn request(bytes: usize) -> EvaluationRequest { + ask::request( + "jev-latest", + json!({"visible_text": "x".repeat(bytes)}), + ask::Questions::default(), + ) +} + +#[tokio::test(start_paused = true)] +async fn a_stalled_framing_is_answered_by_its_copy() { + // Live, one framing of a burst stalled 12–32 s while its siblings + // answered in under a second. + let (runtime, paced) = paced(&[(30_000, false), (600, false)]); + let started = tokio::time::Instant::now(); + let answer = hedged(&runtime, "1", &request(1_000)).await.unwrap(); + assert_eq!(answer.response.model, "call 1", "the copy answered"); + assert_eq!(answer.attempts, 2, "the copy is an attempt of its own"); + assert_eq!(started.elapsed(), Duration::from_millis(4_600)); + assert_eq!(*paced.calls.lock().unwrap(), 2); +} + +#[tokio::test(start_paused = true)] +async fn a_framing_that_answers_in_time_gets_no_copy() { + // Live, a slow evening's calls took up to 3.4 s and still answered. + let (runtime, paced) = paced(&[(3_900, false)]); + let answer = hedged(&runtime, "1", &request(1_000)).await.unwrap(); + assert_eq!( + (answer.response.model.as_str(), answer.attempts), + ("call 0", 1) + ); + assert_eq!(*paced.calls.lock().unwrap(), 1); +} + +#[tokio::test(start_paused = true)] +async fn a_large_request_waits_longer_before_its_copy() { + // A 32 KB request's p99.9 was 3.9 s live: at 4.9 s it still gets no copy. + let (runtime, paced) = paced(&[(4_900, false)]); + let answer = hedged(&runtime, "1", &request(40_000)).await.unwrap(); + assert_eq!(answer.attempts, 1); + assert_eq!(*paced.calls.lock().unwrap(), 1); +} + +#[tokio::test(start_paused = true)] +async fn a_failed_copy_gives_way_and_two_failures_fail() { + // The copy fails at once; the slow first answer still counts. + let (runtime, _) = paced(&[(6_000, false), (0, true)]); + let started = tokio::time::Instant::now(); + let answer = hedged(&runtime, "1", &request(1_000)).await.unwrap(); + assert_eq!(answer.response.model, "call 0"); + assert_eq!(started.elapsed(), Duration::from_millis(6_000)); + + // The first fails after its copy was sent; the copy's answer counts. + let (runtime, _) = paced(&[(4_500, true), (1_000, false)]); + let answer = hedged(&runtime, "1", &request(1_000)).await.unwrap(); + assert_eq!(answer.response.model, "call 1"); + + let (runtime, _) = paced(&[(4_500, true), (1_000, true)]); + assert!(hedged(&runtime, "1", &request(1_000)).await.is_err()); +} + +#[tokio::test(start_paused = true)] +async fn a_sage_framing_gets_no_copy() { + // Sage's calls take 5–6 s as a rule: a copy would double them. + let (runtime, paced) = paced_as( + tinycomputer_bus::JevProvider::Sage, + &[(6_000, false), (600, false)], + ); + let started = tokio::time::Instant::now(); + let answer = hedged(&runtime, "1", &request(1_000)).await.unwrap(); + assert_eq!( + (answer.response.model.as_str(), answer.attempts), + ("call 0", 1) + ); + assert_eq!(started.elapsed(), Duration::from_millis(6_000)); + assert_eq!(*paced.calls.lock().unwrap(), 1); +} + +#[tokio::test(start_paused = true)] +async fn no_more_than_two_copies_are_in_flight() { + // Three framings stall together, as on a slow gateway: two get a copy, + // the third waits for its own answer. + let (runtime, paced) = paced(&[ + (30_000, false), + (30_000, false), + (30_000, false), + (600, false), + (600, false), + ]); + let asked = request(1_000); + let ask = || hedged(&runtime, "1", &asked); + let (one, two, three) = tokio::join!(ask(), ask(), ask()); + let mut attempts = [one, two, three].map(|answer| answer.unwrap().attempts); + // Which framing's delay ends first is the timer's to say. + attempts.sort_unstable(); + assert_eq!(attempts, [1, 2, 2]); + assert_eq!(*paced.calls.lock().unwrap(), 5); + assert_eq!( + runtime.copies.load(std::sync::atomic::Ordering::Acquire), + 0, + "every place given back" + ); +} + +#[tokio::test(start_paused = true)] +async fn the_journal_names_the_answer_that_counted() { + let scratch = std::env::temp_dir().join(format!( + "tinycomputer-hedge-journal-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let won = |script: &[(u64, bool)]| { + let (mut runtime, _) = paced(script); + runtime.journal = crate::agentic::journal::Journal::at(&scratch).fresh("hedge"); + let dir = runtime.journal.run_dir().unwrap(); + async move { + let _ = hedged(&runtime, "1", &request(1_000)).await; + let journal = std::fs::read_to_string(dir.join(crate::JOURNAL_FILE)).unwrap(); + let event: Value = journal + .lines() + .map(|line| serde_json::from_str::(line).unwrap()) + .find(|event| event["event"] == "hedge") + .unwrap(); + ( + event["won"].as_str().unwrap().to_owned(), + event["ok"].as_bool().unwrap(), + ) + } + }; + // The first finished first, failing; the copy's answer counted. + assert_eq!( + won(&[(4_500, true), (1_000, false)]).await, + ("copy".to_owned(), true) + ); + // The copy failed at once; the first's answer counted. + assert_eq!( + won(&[(6_000, false), (0, true)]).await, + ("first".to_owned(), true) + ); + assert_eq!( + won(&[(4_500, true), (1_000, true)]).await, + ("neither".to_owned(), false) + ); + let _ = std::fs::remove_dir_all(&scratch); +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/places.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/places.rs index 53a31431..29da5249 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/places.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/places.rs @@ -20,6 +20,13 @@ pub(super) const PLACES: [&str; 4] = [ /// The heading a panel of suggestions opens with (`Places::panel`). const PANEL_HEADING: &str = "Select a pickup point Choose where your driver meets you"; +/// The rows a box lists of its own while its matches are fetched +/// (`Places::starters`): none of them names a place typed. +const STARTER_ROWS: [&str; 2] = [ + "Allow location access It provides your pickup address", + "Search in a different city", +]; + /// The ride form's state. #[derive(Debug, Default)] pub(super) struct Places { @@ -31,6 +38,15 @@ pub(super) struct Places { /// button, its rows unread and its name stringing them all together, as /// a store's delivery-area popover was read live. pub(super) panel: bool, + /// Waits for a change a box's list takes to show once typed into, as a + /// page that fetches its rows draws them late. + pub(super) late: u8, + /// Waits still to come before the open list shows its rows. + pub(super) pending: u8, + /// Whether the open list shows rows of its own while its matches are + /// still to come, as a ride app's did live ("Allow location access", + /// "Search in a different city"). + pub(super) starters: bool, } /// The places suggested for `typed`: each one that holds every typed word. @@ -71,9 +87,14 @@ pub(super) fn places_widget( field.value = sim.fields.get(*name).map(|value| json!(value)); candidates.push(field); } - if let Some(open) = &places.open { + let list = [root, "group \"Get a ride\"", "listbox \"Suggestions\""]; + if places.starters && places.open.is_some() && places.pending > 0 { + for row in STARTER_ROWS { + candidates.push(node(row, "option", &["Click"], &list, 300.0)); + } + } + if let Some(open) = places.open.as_ref().filter(|_| places.pending == 0) { let typed = sim.fields.get(open).cloned().unwrap_or_default(); - let list = [root, "group \"Get a ride\"", "listbox \"Suggestions\""]; if places.panel { let rows = suggested(&typed); if !rows.is_empty() { @@ -107,11 +128,24 @@ fn type_place(sim: &mut Sim, name: &str, text: String) { if let Some(places) = sim.places.as_mut() { places.open = Some(name.to_owned()); places.picked.remove(name); + places.pending = places.late; } sim.focused = Some(name.to_owned()); sim.fields.insert(name.to_owned(), text); } +/// A wait for the page to change: the open list draws its rows one wait +/// nearer; nothing else on the ride form changes by itself. +pub(super) fn await_place_rows(sim: &mut Sim) -> bool { + match sim.places.as_mut() { + Some(places) if places.open.is_some() && places.pending > 0 => { + places.pending -= 1; + true + } + _ => false, + } +} + /// Closes the open list, dropping its box's text unless a suggestion was /// picked for it. pub(super) fn drop_unpicked(sim: &mut Sim) { diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/quorum_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/quorum_tests.rs new file mode 100644 index 00000000..4eb0920e --- /dev/null +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/quorum_tests.rs @@ -0,0 +1,486 @@ +//! Ending a decision on a quorum: once all but its last two framings have +//! answered and agree plainly, the rest are not waited for. + +use super::*; +use crate::agentic::flow::quorum::{self, QUORUM_TOP, SURE_NO, SURE_YES}; +use crate::agentic::flow::{FlowRun, StepLog}; + +/// A target Choice whose options each framing relabels and reorders, a +/// yes/no, and the page kind. +fn request() -> EvaluationRequest { + let choice = |options: &[&str], labelled: bool| { + Question::Choice(tinyinference_decisions::Choice { + instructions: json!({"question": "which"}), + criteria: options + .iter() + .enumerate() + .map(|(index, option)| { + if labelled { + ((index + 1).to_string(), Some(json!(option))) + } else { + ((*option).to_owned(), None) + } + }) + .collect(), + }) + }; + EvaluationRequest { + state: json!("a product page"), + model: "jev-latest".to_owned(), + questions: BTreeMap::from([ + ( + "target".to_owned(), + choice(&["Add to Cart", "Buy Now", "Wishlist"], true), + ), + ( + "done".to_owned(), + Question::Noul(tinyinference_decisions::Noul { + instructions: json!({"question": "done?"}), + criteria: None, + }), + ), + ("page_kind".to_owned(), choice(&["product", "cart"], false)), + ]), + } +} + +/// One framing's answers: `target` (an option's text) at `sure`, `done` +/// at `done`, and the page kind `kind` at a weak 0.6, in the framing's own +/// keys. +fn answers( + framing: &vote::Framing, + target: &str, + sure: f64, + done: f64, + kind: &str, +) -> BTreeMap { + let Question::Choice(choice) = &framing.request.questions["target"] else { + panic!("target is a choice"); + }; + let key = choice + .criteria + .iter() + .find(|(_, text)| text.as_ref() == Some(&json!(target))) + .map(|(key, _)| key.clone()) + .unwrap(); + let others = (1.0 - sure) / 2.0; + let other = if kind == "product" { "cart" } else { "product" }; + BTreeMap::from([ + ( + "target".to_owned(), + Answer::Choice(ChoiceAnswer { + choice: key.clone(), + probabilities: choice + .criteria + .keys() + .map(|option| (option.clone(), if *option == key { sure } else { others })) + .collect(), + confidence: sure, + }), + ), + ("done".to_owned(), noul(done)), + ( + "page_kind".to_owned(), + Answer::Choice(ChoiceAnswer { + choice: kind.to_owned(), + probabilities: BTreeMap::from([(kind.to_owned(), 0.6), (other.to_owned(), 0.4)]), + confidence: 0.6, + }), + ), + ]) +} + +/// Whether five framings answering as `each` says, framing by framing, +/// settle the decision. +fn settles(each: impl Fn(usize) -> (&'static str, f64, f64, &'static str)) -> bool { + let framings = vote::framings(&request(), 7); + let answered = (0..5) + .map(|index| { + let (target, sure, done, kind) = each(index); + (index, answers(&framings[index], target, sure, done, kind)) + }) + .collect::>(); + quorum::settled( + &framings, + answered.iter().map(|(index, answers)| (*index, answers)), + 5, + ) +} + +#[test] +fn only_a_decision_asked_seven_ways_or_more_ends_on_a_quorum() { + assert_eq!(quorum::size(7), Some(5)); + assert_eq!(quorum::size(9), Some(7)); + assert_eq!(quorum::size(6), None); + assert_eq!(quorum::size(1), None); +} + +#[test] +fn five_plain_answers_settle_a_decision_however_each_framing_labels_it() { + // Each framing names "Add to Cart" by a key of its own; the page kind + // needs only to be the same. + assert!(settles(|_| ("Add to Cart", 0.95, 0.99, "product"))); + assert!(settles(|_| ("Add to Cart", QUORUM_TOP, SURE_NO, "product"))); + assert!(settles(|_| ("Add to Cart", 0.95, SURE_YES, "product"))); +} + +#[test] +fn a_decision_any_answer_leaves_open_is_not_settled() { + let weak_pick = |index: usize| { + ( + "Add to Cart", + if index == 3 { 0.85 } else { 0.95 }, + 0.99, + "product", + ) + }; + assert!(!settles(weak_pick), "a pick under QUORUM_TOP"); + let other_pick = |index: usize| { + ( + if index == 2 { "Buy Now" } else { "Add to Cart" }, + 0.95, + 0.99, + "product", + ) + }; + assert!(!settles(other_pick), "a framing picked another option"); + let unsure = |index: usize| { + ( + "Add to Cart", + 0.95, + if index == 4 { 0.95 } else { 0.99 }, + "product", + ) + }; + assert!(!settles(unsure), "a yes/no short of SURE_YES"); + let split = |index: usize| { + ( + "Add to Cart", + 0.95, + if index == 0 { 0.02 } else { 0.99 }, + "product", + ) + }; + assert!(!settles(split), "a yes/no answered both ways"); + let page = |index: usize| { + ( + "Add to Cart", + 0.95, + 0.99, + if index == 1 { "cart" } else { "product" }, + ) + }; + assert!(!settles(page), "the page kind read two ways"); +} + +#[test] +fn a_decision_short_of_its_quorum_is_not_settled() { + let framings = vote::framings(&request(), 7); + let plain = |index: usize| answers(&framings[index], "Add to Cart", 0.95, 0.99, "product"); + let four = (0..4) + .map(|index| (index, plain(index))) + .collect::>(); + assert!(!quorum::settled( + &framings, + four.iter().map(|(index, answers)| (*index, answers)), + 5 + )); + // Five answered, but one left a question out. + let mut five = (0..5) + .map(|index| (index, plain(index))) + .collect::>(); + five[2].1.remove("done"); + assert!(!quorum::settled( + &framings, + five.iter().map(|(index, answers)| (*index, answers)), + 5 + )); +} + +/// Framing tasks for `framings`, framing `index` answering after `plan`'s +/// wait with its answers, or failing; each counts itself in `finished` as +/// it ends. +fn paced( + framings: &[vote::Framing], + plan: &[(u64, Option<(&'static str, f64)>)], + finished: &Arc, +) -> Vec>> { + framings + .iter() + .zip(plan) + .map(|(framing, (wait, answer))| { + let answers = + answer.map(|(target, done)| answers(framing, target, 0.95, done, "product")); + let (wait, finished) = (Duration::from_millis(*wait), Arc::clone(finished)); + tokio::spawn(async move { + tokio::time::sleep(wait).await; + finished.fetch_add(1, Ordering::SeqCst); + answers + .map(|answers| EvaluationResult { + response: EvaluationResponse { + model: "typesafe/jev-test".to_owned(), + answers, + usage: tinyinference_decisions::Usage::default(), + }, + request_id: None, + attempts: 1, + latency: wait, + }) + .ok_or_else(|| EvaluationFailure { + error: Box::new(tinyinference_decisions::Error::RateLimited), + attempts: 1, + latency: wait, + }) + }) + }) + .collect() +} + +#[tokio::test(start_paused = true)] +async fn a_decision_ends_once_five_plain_answers_are_in_and_the_rest_still_run() { + let framings = vote::framings(&request(), 7); + let finished = Arc::new(AtomicU64::new(0)); + let agreeing = Some(("Add to Cart", 0.99)); + let plan = [ + (500, agreeing), + (100, agreeing), + (300, agreeing), + (200, agreeing), + (400, agreeing), + (900, agreeing), + (3_000, agreeing), + ]; + let handles = paced(&framings, &plan, &finished); + let started = tokio::time::Instant::now(); + let gathered = quorum::gather(framings, handles, quorum::size(7)).await; + assert_eq!( + started.elapsed(), + Duration::from_millis(500), + "the fifth answer" + ); + assert_eq!(gathered.left, 2); + assert!(gathered.failure.is_none()); + // In framing order, as the ballots read them. + assert_eq!( + gathered + .answered + .iter() + .map(|(_, evaluation)| evaluation.latency.as_millis()) + .collect::>(), + [500, 100, 300, 200, 400] + ); + // The two left are not cut off: each runs to its end. + tokio::time::sleep(Duration::from_secs(3)).await; + assert_eq!(finished.load(Ordering::SeqCst), 7); +} + +#[tokio::test(start_paused = true)] +async fn a_decision_waits_for_every_framing_unless_five_agree_plainly() { + let finished = Arc::new(AtomicU64::new(0)); + let agreeing = Some(("Add to Cart", 0.99)); + let started = tokio::time::Instant::now(); + // One of the first five in picks another option: all seven are waited + // for. (One that answers after five agreeing ones is not.) + let framings = vote::framings(&request(), 7); + let mut plan = [(100, agreeing); 7]; + plan[2] = (50, Some(("Buy Now", 0.99))); + plan[6] = (3_000, agreeing); + let handles = paced(&framings, &plan, &finished); + let gathered = quorum::gather(framings, handles, quorum::size(7)).await; + assert_eq!(started.elapsed(), Duration::from_millis(3_000)); + assert_eq!((gathered.answered.len(), gathered.left), (7, 0)); + + // A failed framing is no answer; five others are a quorum still. + let started = tokio::time::Instant::now(); + let framings = vote::framings(&request(), 7); + let mut plan = [(100, agreeing); 7]; + plan[0] = (50, None); + plan[6] = (3_000, agreeing); + let handles = paced(&framings, &plan, &finished); + let gathered = quorum::gather(framings, handles, quorum::size(7)).await; + assert_eq!(started.elapsed(), Duration::from_millis(100)); + assert!(gathered.failure.is_some()); + assert_eq!((gathered.answered.len(), gathered.left), (5, 1)); + + // Asked fewer than seven ways, a decision waits for every framing. + let started = tokio::time::Instant::now(); + let framings = vote::framings(&request(), 6); + let mut plan = [(100, agreeing); 6]; + plan[5] = (2_000, agreeing); + let handles = paced(&framings, &plan, &finished); + let gathered = quorum::gather(framings, handles, quorum::size(6)).await; + assert_eq!(started.elapsed(), Duration::from_millis(2_000)); + assert_eq!((gathered.answered.len(), gathered.left), (6, 0)); +} + +/// The oracle, with each decision's framings answering after the waits in +/// `pace`, in the order they are asked. +struct Staggered { + oracle: Oracle, + pace: [u64; 7], + calls: Mutex, +} + +impl Evaluator for Staggered { + fn evaluate<'a>( + &'a self, + request: &'a EvaluationRequest, + ) -> Pin> + Send + 'a>> + { + let call = { + let mut calls = self.calls.lock().unwrap(); + *calls += 1; + *calls - 1 + }; + let wait = Duration::from_millis(self.pace[call % self.pace.len()]); + Box::pin(async move { + tokio::time::sleep(wait).await; + self.oracle.evaluate(request).await + }) + } +} + +/// A `verify` run asked seven ways, its last two framings 2 s behind the +/// rest, with Jev `sure` the inbox shows; how long it took, its result, and +/// its journal's decisions. +async fn verify_staggered(sure: f64) -> (Duration, FlowRunResult, Vec) { + let scratch = std::env::temp_dir().join(format!( + "tinycomputer-quorum-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let app = App::with(|_| {}); + let staggered = Staggered { + oracle: Oracle { + app: app.clone(), + hook: Box::new(move |id: &str, _: &Question, _: &Sim| { + (id == "holds").then(|| noul(sure)) + }), + requests: Mutex::new(Vec::new()), + fail: false, + }, + pace: [10, 10, 10, 10, 10, 2_000, 2_000], + calls: Mutex::new(0), + }; + let mut runtime = runtime(staggered); + runtime.journal = crate::agentic::journal::Journal::at(&scratch).fresh("quorum"); + let dir = runtime.journal.run_dir().unwrap(); + let started = tokio::time::Instant::now(); + let reply = run_flow_with( + app, + &runtime, + RunFlowRequest { + flow: serde_json::from_value(json!({ + "app": "Mail", + "steps": [{"verify": "the inbox shows"}] + })) + .unwrap(), + votes: 7, + ..RunFlowRequest::default() + }, + ) + .await; + let took = started.elapsed(); + let decisions = std::fs::read_to_string(dir.join(crate::JOURNAL_FILE)) + .unwrap() + .lines() + .map(|line| serde_json::from_str::(line).unwrap()) + .filter(|event| event["event"] == "decision") + .collect(); + let _ = std::fs::remove_dir_all(&scratch); + ( + took, + serde_json::from_value(reply.data.unwrap()).unwrap(), + decisions, + ) +} + +#[tokio::test(start_paused = true)] +async fn a_run_whose_framings_agree_plainly_does_not_wait_for_its_slowest() { + let (took, result, decisions) = verify_staggered(0.99).await; + assert_eq!(result.stop, FlowStopReason::Completed, "{:?}", result.steps); + assert!(took < Duration::from_secs(2), "{took:?}"); + assert_ne!(decisions.len(), 0); + for decision in &decisions { + assert_eq!( + ( + &decision["framings"], + &decision["answered"], + &decision["left"] + ), + (&json!(7), &json!(5), &json!(2)), + "{decision}" + ); + } + // Every framing sent is a call made, waited for or not. + assert_eq!( + usize::try_from(result.metrics.calls).unwrap(), + decisions.len() * 7 + ); + + // Jev only fairly sure: every decision waits for all seven. + let (took, result, decisions) = verify_staggered(0.9).await; + assert_eq!(result.stop, FlowStopReason::Completed, "{:?}", result.steps); + assert!(took >= Duration::from_secs(2), "{took:?}"); + assert!(decisions.iter().all(|decision| decision["left"] == 0)); +} + +#[tokio::test(start_paused = true)] +async fn a_decision_ended_on_a_quorum_widens_past_every_framing_it_asked() { + // A press nothing undoes is vouched for, and the vouching always widens. + let app = App::with(|_| {}); + let staggered = Staggered { + oracle: Oracle { + app: app.clone(), + hook: Box::new(|id: &str, _: &Question, _: &Sim| match id { + "is_0" => Some(noul(0.99)), + "only_near_0" => Some(noul(0.02)), + _ => None, + }), + requests: Mutex::new(Vec::new()), + fail: false, + }, + pace: [10, 10, 10, 10, 10, 2_000, 2_000], + calls: Mutex::new(0), + }; + let runtime = runtime(staggered); + let request = RunFlowRequest { + flow: serde_json::from_value(json!({ + "app": "Mail", + "steps": [{"stop_before": "sending the email"}] + })) + .unwrap(), + votes: 7, + ..RunFlowRequest::default() + }; + let mut run = FlowRun::new(app, &runtime, &request); + let mut log = StepLog::default(); + let yes_no = |question: &str| { + Question::Noul(tinyinference_decisions::Noul { + instructions: json!({"question": question}), + criteria: None, + }) + }; + let vouching = EvaluationRequest { + state: json!("the draft, its Send button in view"), + model: "jev-latest".to_owned(), + questions: BTreeMap::from([ + ("is_0".to_owned(), yes_no("is it the Send button?")), + ("only_near_0".to_owned(), yes_no("is it only near it?")), + ]), + }; + run.ask(&mut log, vouching.clone()).await.unwrap(); + assert_eq!( + (run.ballot("is_0").len(), run.asked_in("is_0")), + (5, 7), + "ended on a quorum" + ); + run.widen(&mut log, &vouching).await.unwrap().unwrap(); + // The eighth and ninth framings: not the sixth and seventh again, which + // were asked and left. + assert_eq!((run.ballot("is_0").len(), run.asked_in("is_0")), (7, 9)); + assert_eq!(log.calls, 9, "seven, and two more"); +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/screens.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/screens.rs index 89b5c475..5779d16c 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/screens.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/screens.rs @@ -421,6 +421,20 @@ pub(super) fn overlays(sim: &Sim, root: &str, candidates: &mut Vec) { candidates.push(node("Close", "button", &["Click"], &toast, 700.0)); candidates.push(node("Learn more", "link", &["Click"], &toast, 720.0)); } + if sim.has(Quirk::ConsentBanner) { + for candidate in candidates.iter_mut() { + candidate.states = vec!["covered".to_owned()]; + } + let banner = [root, "popover \"We value your privacy\""]; + candidates.push(node( + "Allow Selection", + "button", + &["Click"], + &banner, + 740.0, + )); + candidates.push(node("Allow all", "button", &["Click"], &banner, 760.0)); + } } /// The inbox's search behind its "Search mail" link, beside a "Contact us" diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs index 895475f6..645678c5 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs @@ -35,6 +35,10 @@ pub(super) enum Quirk { DisabledArchive, /// A promo toast with a Close button sits over the page until closed. PromoToast, + /// A consent banner lies over the page as a popover: every other click + /// is refused as covered, Escape leaves it, and its "Allow Selection" + /// or "Allow all" closes it. + ConsentBanner, /// Text typed with no target lands at the end of the field typed into /// last, as a browser keeps the focus there. FocusStays, @@ -95,6 +99,9 @@ pub(super) struct Sim { /// The field typed or pasted into last: where the focus stays. pub(super) focused: Option, pub(super) quirks: BTreeSet, + /// The calls that touch no element, in order: each `settle` and `settle + /// briefly`, and each `await_change` as `changed` or `still`. + pub(super) trail: Vec<&'static str>, } impl Sim { @@ -252,6 +259,40 @@ impl App { } } +/// The reply to a click on `name` that something on the simulated page +/// refuses or takes over, or `None` when the click goes through: an +/// unclickable page, a consent banner whose own buttons close it, or a +/// drawer over everything. +fn refused_click(sim: &mut Sim, name: &str) -> Option { + let covered = |by: &str| { + DesktopResponse::err( + "click", + tinycomputer_bus::DesktopError::new( + "NOT_ACTIONABLE", + format!("Element '@s:{name}' is covered by at its click point"), + ), + ) + }; + if sim.has(Quirk::Unclickable) { + return Some(DesktopResponse::err( + "click", + tinycomputer_bus::DesktopError::new( + "NOT_ACTIONABLE", + "Element exists but is not visible.", + ), + )); + } + if sim.has(Quirk::ConsentBanner) { + if name.starts_with("Allow ") { + sim.clicks.push(name.to_owned()); + sim.quirks.remove(&Quirk::ConsentBanner); + return Some(DesktopResponse::ok("click", json!({}))); + } + return Some(covered("consent")); + } + sim.has(Quirk::Drawer).then(|| covered("drawer")) +} + impl AgentBackend for App { fn observe( &self, @@ -298,23 +339,10 @@ impl AgentBackend for App { .as_ref() .and_then(|target| target.name.clone()) .unwrap_or_default(); - if sim.has(Quirk::Unclickable) && operation == JevOperation::Click { - return DesktopResponse::err( - "click", - tinycomputer_bus::DesktopError::new( - "NOT_ACTIONABLE", - "Element exists but is not visible.", - ), - ); - } - if sim.has(Quirk::Drawer) && operation == JevOperation::Click { - return DesktopResponse::err( - "click", - tinycomputer_bus::DesktopError::new( - "NOT_ACTIONABLE", - format!("Element '@s:{name}' is covered by at its click point"), - ), - ); + if operation == JevOperation::Click + && let Some(reply) = refused_click(&mut sim, &name) + { + return reply; } if is_city_row(target.as_ref()) && operation == JevOperation::TypeText { return not_a_text_field(); @@ -394,6 +422,21 @@ impl AgentBackend for App { Some(self.sim().fields.get(&name).cloned().unwrap_or_default()) } + fn await_change(&self, _ms: u64) -> bool { + let mut sim = self.sim(); + let changed = await_place_rows(&mut sim); + sim.trail.push(if changed { "changed" } else { "still" }); + changed + } + + fn settle(&self) { + self.sim().trail.push("settle"); + } + + fn settle_briefly(&self) { + self.sim().trail.push("settle briefly"); + } + fn paste(&self, _app: &str, target: &Candidate, text: &str) -> DesktopResponse { if is_city_row(Some(target)) { return not_a_text_field(); diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/suggestion_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/suggestion_tests.rs index e15780d3..93a0c9b4 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/suggestion_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/suggestion_tests.rs @@ -85,6 +85,111 @@ async fn enter_picks_the_suggestion_an_autocomplete_box_lists_for_the_typed_text assert!(asked_for_a_suggestion(&run)); } +/// The notes of a step's waits, in order. +fn waits(run: &Run) -> Vec { + run.result.steps[0] + .actions + .iter() + .filter(|action| action.action == "wait") + .map(|action| action.note.clone()) + .collect() +} + +#[tokio::test] +async fn a_place_box_whose_rows_come_late_is_looked_at_again_as_they_show() { + // Live, a ride app's rows came after the first look. Each wait ends as + // the page changes, and the rows are picked once drawn. + for late in [1, 2] { + let run = run_with( + App::with(|sim| { + sim.places = Some(Places { + late, + ..Places::default() + }); + }), + json!({"app": "Mail", "steps": [{"enter": {"pickup location": "Connaught Place"}}]}), + |_| {}, + ride, + ) + .await; + assert_eq!( + run.result.stop, + FlowStopReason::Completed, + "{:?}", + run.result.steps + ); + assert_eq!( + run.app.sim().fields["Pickup location"], + "Connaught Place New Delhi, Delhi, India", + "rows drawn after {late} waits" + ); + assert_eq!(waits(&run), vec![String::new(); usize::from(late)]); + // A wait that saw the page change is settled, as any action is. + let trail = run.app.sim().trail.clone(); + let watched = trail.iter().position(|call| *call == "changed").unwrap(); + assert_eq!(trail.get(watched + 1), Some(&"settle"), "{trail:?}"); + } +} + +#[tokio::test] +async fn a_place_box_waits_past_its_own_rows_for_the_place_typed() { + // Live, a ride app's pickup box first listed rows of its own ("Allow + // location access", "Search in a different city"), and a look made as + // soon as the page went still saw only those: no row named the place, + // none was picked, and the pickup was never set. The box is looked at + // again until a row names the place. + let run = run_with( + App::with(|sim| { + sim.places = Some(Places { + late: 1, + starters: true, + ..Places::default() + }); + }), + json!({"app": "Mail", "steps": [{"enter": {"pickup location": "Connaught Place"}}]}), + |_| {}, + ride, + ) + .await; + assert_eq!( + run.result.stop, + FlowStopReason::Completed, + "{:?}", + run.result.steps + ); + assert_eq!( + run.app.sim().fields["Pickup location"], + "Connaught Place New Delhi, Delhi, India" + ); + assert_eq!(waits(&run), [""], "one wait, ended by the rows showing"); +} + +#[tokio::test] +async fn a_place_box_on_a_page_that_stays_still_is_waited_on_once() { + // Live, an address and a city box on a plain form waited twice each for + // a list that never came. A page that stayed still lists nothing more. + let run = run_with( + App::with(|sim| sim.places = Some(Places::default())), + json!({"app": "Mail", "steps": [{"enter": {"pickup location": "Nowhere Lane"}}]}), + |_| {}, + ride, + ) + .await; + assert_eq!( + run.result.stop, + FlowStopReason::Completed, + "{:?}", + run.result.steps + ); + assert_eq!(run.app.sim().fields["Pickup location"], "Nowhere Lane"); + assert_eq!(waits(&run), ["nothing changed"]); + assert!(!asked_for_a_suggestion(&run)); + // A wait that saw the page stay still has nothing to settle. + let trail = run.app.sim().trail.clone(); + let watched = trail.iter().position(|call| *call == "still").unwrap(); + assert_ne!(trail.get(watched + 1), Some(&"settle"), "{trail:?}"); +} + #[tokio::test] async fn a_place_box_asks_again_for_the_row_naming_its_place_in_other_words() { // An unsure pick (0.45) is not pressed as such. A place box, though, diff --git a/crates/tinycomputer-engine/src/agentic/flow/hedge.rs b/crates/tinycomputer-engine/src/agentic/flow/hedge.rs new file mode 100644 index 00000000..042a0d1d --- /dev/null +++ b/crates/tinycomputer-engine/src/agentic/flow/hedge.rs @@ -0,0 +1,119 @@ +//! Hedging a framing: one still unanswered past its hedge delay is sent +//! again, and the first answer of the two counts. + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{Duration, Instant}; + +use serde_json::json; +use tinycomputer_bus::JevProvider; +use tinyinference_decisions::{EvaluationFailure, EvaluationRequest, EvaluationResult}; + +use super::decide::bytes; +use crate::agentic::{JevRuntime, journal::millis}; + +/// How long a framing runs before a copy of it is sent and the first answer +/// of the two taken. Live, one framing of a burst stalled 12–32 s (the +/// gateway gave up after ~10 s, or nothing came back before the client's +/// timeout) as its siblings answered in under 1 s. Calls on a slow evening +/// took up to 3.4 s and still answered: a copy sent at 2.5 s lost the race +/// 15 times in 16, so copies wait for 4 s, past what a slow answer takes. +const HEDGE_AFTER: Duration = Duration::from_millis(4_000); + +/// [`HEDGE_AFTER`] for a request of [`HEDGE_LARGE_BYTES`] or more, whose +/// p99.9 was 3.9 s live. +const HEDGE_AFTER_LARGE: Duration = Duration::from_millis(5_000); + +/// Size from which a request waits [`HEDGE_AFTER_LARGE`] for its first +/// answer. +const HEDGE_LARGE_BYTES: usize = 32 * 1024; + +/// Most copies a runtime has in flight at once. A stall is rare (21 calls of +/// 36,459 live), so many framings outliving their delay together means a +/// slow or failing gateway, which a copy of each would only load more (the +/// client may be waiting out its retry delay): past this many, a framing +/// waits for its own answer. +const HEDGE_COPIES: usize = 2; + +/// Asks `request` once, and once more when no answer has come within its +/// hedge delay ([`HEDGE_AFTER`], or [`HEDGE_AFTER_LARGE`] for a large +/// request), taking whichever copy answers first. A copy that fails gives +/// way to the other; when both fail, the first failure is returned. The +/// answer that counts carries the extra attempt, and the journal records a +/// `hedge` event. Sage gets no copy, and no copy is sent while +/// [`HEDGE_COPIES`] are in flight. +pub(in crate::agentic::flow) async fn hedged( + runtime: &JevRuntime, + step: &str, + request: &EvaluationRequest, +) -> Result { + let first = runtime.evaluate(Some(step), request); + // Sage's calls take 5–6 s as a rule (`docs/technical/evals/ + // 2026-09-29-sage.md`): a copy would double its calls and its cost, and + // rarely answer first. + if runtime.configuration.provider == JevProvider::Sage { + return first.await; + } + let delay = if bytes(request) >= HEDGE_LARGE_BYTES { + HEDGE_AFTER_LARGE + } else { + HEDGE_AFTER + }; + tokio::pin!(first); + if let Ok(outcome) = tokio::time::timeout(delay, &mut first).await { + return outcome; + } + let Some(_held) = Held::claim(&runtime.copies) else { + return first.await; + }; + let sent = Instant::now(); + let copy = runtime.evaluate(Some(step), request); + tokio::pin!(copy); + let (outcome, copy_first) = tokio::select! { + outcome = &mut first => (outcome, false), + outcome = &mut copy => (outcome, true), + }; + // Which answer counts: `Some(true)` the copy's, `None` neither. + let (outcome, counted) = match outcome { + Ok(evaluation) => (Ok(evaluation), Some(copy_first)), + Err(failure) => match if copy_first { first.await } else { copy.await } { + Ok(evaluation) => (Ok(evaluation), Some(!copy_first)), + Err(_) => (Err(failure), None), + }, + }; + runtime.journal.record("hedge", || { + json!({ + "step": step, + "after_ms": millis(delay), + "won": match counted { + Some(true) => "copy", + Some(false) => "first", + None => "neither", + }, + "ok": outcome.is_ok(), + "wall_ms": millis(delay + sent.elapsed()), + }) + }); + outcome.map(|mut evaluation| { + evaluation.attempts = evaluation.attempts.saturating_add(1); + evaluation + }) +} + +/// A copy in flight, counted against [`HEDGE_COPIES`] until it is dropped. +struct Held<'a>(&'a AtomicUsize); + +impl<'a> Held<'a> { + /// A place for one more copy, unless [`HEDGE_COPIES`] are in flight. + fn claim(copies: &'a AtomicUsize) -> Option { + let held = Self(copies); + // Counted first, so two claims at once cannot both slip under the + // limit; one over it gives its place back as it is dropped. + (copies.fetch_add(1, Ordering::AcqRel) < HEDGE_COPIES).then_some(held) + } +} + +impl Drop for Held<'_> { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::AcqRel); + } +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/mod.rs b/crates/tinycomputer-engine/src/agentic/flow/mod.rs index 281a0ac6..f0b4c6c5 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/mod.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/mod.rs @@ -55,9 +55,11 @@ mod evidence; mod expect; mod front; mod ground; +mod hedge; mod ledger; mod look; mod memory; +mod quorum; mod reflect; mod run; mod steps; @@ -68,6 +70,7 @@ mod vote; mod wide; pub(crate) use validate::{check as check_flow, missing_inputs}; +pub(in crate::agentic) use vote::MAX_VOTES; use std::{ collections::{BTreeMap, BTreeSet}, @@ -289,6 +292,11 @@ pub(super) struct FlowRun<'r, B> { /// keys, from the latest decision that asked it: the evidence a /// deliberating decision reads (`evidence/`) and widens (`escalate`). ballots: BTreeMap>, + /// How many framings each question was asked in, by the latest decision + /// that asked it and any widening since. A decision ended on a quorum + /// holds fewer answers than that, and `escalate` must widen past every + /// framing asked, not only those in the ballot. + asked: BTreeMap, /// The address the surface last reported, on a surface that has them: /// a checkpoint's location, and how a navigation is noticed. pub(super) location: Option, diff --git a/crates/tinycomputer-engine/src/agentic/flow/quorum.rs b/crates/tinycomputer-engine/src/agentic/flow/quorum.rs new file mode 100644 index 00000000..af2ff428 --- /dev/null +++ b/crates/tinycomputer-engine/src/agentic/flow/quorum.rs @@ -0,0 +1,203 @@ +//! Ending a decision on a quorum: once all but its last two framings have +//! answered, and agree so plainly that the last two almost never change +//! what the loops or the evidence gate make of the answers, the decision is +//! merged without waiting for them. +//! +//! A decision waits for its slowest framing. Replayed over 13,669 decisions +//! asked seven ways, from 245 live runs, a quorum of five ended 12% of them, +//! a median 0.14 s before the seventh answer and about 2 s a run, and every +//! one read the same on every threshold, choice floor, and evidence gate as +//! all seven framings did, but for two whose mean moved by under 0.002 across +//! the edge of the evidence band. A quorum of four agreed with all seven only +//! 99.2% of the time: its stragglers dissented. +//! +//! The framings left are not cancelled: each runs to its end, its exchange +//! journaled, so its connection goes back to the pool for the next decision. + +use std::collections::BTreeMap; + +use tinyinference_decisions::{Answer, EvaluationFailure, EvaluationResult}; +use tokio::{sync::mpsc, task::JoinHandle}; + +use super::{decide::PAGE_KIND, vote}; + +/// Framings a decision ends without, at most, once the rest agree. +pub(super) const QUORUM_LEFT: usize = 2; +/// Fewest framings a decision must be asked in to end on a quorum. +pub(super) const QUORUM_VOTES: usize = 7; +/// Least probability every framing gives the option a Choice or Score +/// ranks first, the same option in each. +pub(super) const QUORUM_TOP: f64 = 0.9; +/// A yes/no every framing answers at or above this, or every one at or +/// below [`SURE_NO`], lies the evidence band's 0.12 beyond every threshold +/// the loops read one yes/no against (0.20 to 0.85). A belief that pairs it +/// with its negation or a coverage can still fall within a band, and is +/// widened then as it would be after all the framings. +pub(super) const SURE_YES: f64 = 0.97; +/// See [`SURE_YES`]. +pub(super) const SURE_NO: f64 = 0.08; + +/// One framing's evaluation, as its task ends. +type Evaluated = Result; + +/// A framing's index and how its task ended. +type Finished = (usize, Result); + +/// What one part's framings came back with. +pub(super) struct Gathered { + /// Each framing that answered, with its evaluation, in framing order. + pub(super) answered: Vec<(vote::Framing, EvaluationResult)>, + /// The first failure heard, if any framing failed. + pub(super) failure: Option, + /// Framings a quorum ended the wait without. + pub(super) left: u32, +} + +/// The framings heard from so far. +#[derive(Default)] +struct Heard { + /// Those that answered, by index, in the order they finished. + answered: Vec<(usize, EvaluationResult)>, + failure: Option, + count: usize, +} + +impl Heard { + fn take(&mut self, (index, outcome): Finished) { + self.count += 1; + match outcome { + Ok(Ok(evaluation)) => self.answered.push((index, evaluation)), + Ok(Err(error)) => { + self.failure.get_or_insert(error); + } + Err(_) => {} + } + } +} + +/// How many answers can end a decision asked in `framings` framings: all +/// but [`QUORUM_LEFT`] of them, when it is asked in [`QUORUM_VOTES`] or more; +/// `None` when it waits for every framing. +pub(super) fn size(framings: usize) -> Option { + (framings >= QUORUM_VOTES).then(|| framings - QUORUM_LEFT) +} + +/// Waits for the framings in `handles`, asked as `framings`, in the order +/// they finish: for all of them, or, with a quorum `size`, until that many +/// have answered and settle every question ([`settled`]). An answer already +/// in when the quorum is reached is used all the same; only the waiting is +/// cut short. +pub(super) async fn gather( + framings: Vec, + handles: Vec>, + size: Option, +) -> Gathered { + let count = handles.len(); + let mut finished = in_order_of_finish(handles); + let mut heard = Heard::default(); + while let Some(next) = finished.recv().await { + heard.take(next); + while let Ok(next) = finished.try_recv() { + heard.take(next); + } + let answers = heard + .answered + .iter() + .map(|(index, evaluation)| (*index, &evaluation.response.answers)); + if size + .is_some_and(|size| heard.answered.len() >= size && settled(&framings, answers, size)) + { + break; + } + } + let Heard { + mut answered, + failure, + count: heard, + } = heard; + answered.sort_unstable_by_key(|(index, _)| *index); + let mut framings = framings.into_iter().map(Some).collect::>(); + Gathered { + answered: answered + .into_iter() + .filter_map(|(index, evaluation)| Some((framings.get_mut(index)?.take()?, evaluation))) + .collect(), + failure, + left: u32::try_from(count - heard).unwrap_or(u32::MAX), + } +} + +/// Each of `handles`' outcomes, with its index, as its task ends. Every +/// task is awaited to its end, whether or not anything still listens. +fn in_order_of_finish(handles: Vec>) -> mpsc::UnboundedReceiver { + let (sender, finished) = mpsc::unbounded_channel(); + for (index, handle) in handles.into_iter().enumerate() { + let sender = sender.clone(); + tokio::spawn(async move { + let _heard = sender.send((index, handle.await)); + }); + } + finished +} + +/// Whether the `answered` framings, by their index in `framings`, settle +/// every question the framings asked: each answered by at least `size` of +/// them, every yes/no [sure](sure) and every Choice and Score ranking the +/// same option first at [`QUORUM_TOP`] or more. The page-kind question, +/// which only briefs the next request, needs only the same first option. +pub(super) fn settled<'a>( + framings: &'a [vote::Framing], + answered: impl Iterator)> + Clone, + size: usize, +) -> bool { + let ballots = vote::ballots_at(framings, answered); + framings.first().is_some_and(|framing| { + framing.request.questions.keys().all(|id| { + ballots.get(id).is_some_and(|ballot| { + ballot.len() >= size + && match ballot.first() { + _ if id == PAGE_KIND => same_top(ballot, 0.0), + Some(Answer::Noul(_)) => sure(ballot), + Some(_) => same_top(ballot, QUORUM_TOP), + None => false, + } + }) + }) + }) +} + +/// Whether every answer in `ballot` is a yes/no at or above [`SURE_YES`], +/// or every one at or below [`SURE_NO`]. +fn sure(ballot: &[Answer]) -> bool { + let nouls = ballot + .iter() + .map(|answer| match answer { + Answer::Noul(noul) => Some(noul.noul), + _ => None, + }) + .collect::>>(); + nouls.is_some_and(|nouls| { + nouls.iter().all(|noul| *noul >= SURE_YES) || nouls.iter().all(|noul| *noul <= SURE_NO) + }) +} + +/// Whether every answer in `ballot` ranks the same option first, each with +/// at least `floor`. +fn same_top(ballot: &[Answer], floor: f64) -> bool { + let mut tops = ballot.iter().map(|answer| { + let probabilities = match answer { + Answer::Choice(choice) => &choice.probabilities, + Answer::Score(score) => &score.probabilities, + Answer::Noul(_) => return None, + }; + probabilities + .iter() + .max_by(|left, right| left.1.total_cmp(right.1)) + .filter(|(_, probability)| **probability >= floor) + .map(|(option, _)| option) + }); + let Some(Some(first)) = tops.next() else { + return false; + }; + tops.all(|top| top == Some(first)) +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/run.rs b/crates/tinycomputer-engine/src/agentic/flow/run.rs index 46099b89..a592a7f3 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/run.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/run.rs @@ -134,6 +134,7 @@ impl<'r, B: AgentBackend + Sync> FlowRun<'r, B> { typed: BTreeSet::new(), deliberation: request.deliberation, ballots: BTreeMap::new(), + asked: BTreeMap::new(), location: None, frontier: Vec::new(), expecting: None, diff --git a/crates/tinycomputer-engine/src/agentic/flow/steps/list.rs b/crates/tinycomputer-engine/src/agentic/flow/steps/list.rs index 30c87747..75697c51 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/steps/list.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/steps/list.rs @@ -114,7 +114,7 @@ impl FlowRun<'_, B> { return Ok(self.picked_as_selected(&picked, &from, &by, &summary, groups.len())); } let reply = self - .press_uncovering(log, "click", &primary, JevOperation::Click) + .press_uncovering(log, "click", &primary, JevOperation::Click, &from) .await?; if !reply.ok { let why = reply.error.as_ref().map_or_else( diff --git a/crates/tinycomputer-engine/src/agentic/flow/steps/suggestion.rs b/crates/tinycomputer-engine/src/agentic/flow/steps/suggestion.rs index 6eec20d3..d4d91bf0 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/steps/suggestion.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/steps/suggestion.rs @@ -72,17 +72,24 @@ impl FlowRun<'_, B> { let mut fresh = fresh_rows(&screen, &shown, field, text, &self.stop_before, place); // A box that suggests places lists them once the page has fetched // them: live, a ride app's rows came after the first look, and the - // pickup typed was never set, so no ride showed. + // pickup typed was never set, so no ride showed. Rows that name + // nothing typed ("Allow location access", "Search in a different + // city") are the box's own, shown while its matches are fetched, so + // they are waited past too: live, a look made as soon as the page + // went still saw only those. The wait ends as the page changes, and + // a page that stayed still lists nothing more: live, an address and + // a city box on a plain form waited 2.3-4.2 s each for a list that + // never comes. for _ in 0..LATE_LOOKS { - if !fresh.is_empty() || !place { + if !place || fresh.iter().any(|row| names_typed(row, text)) { break; } - self.act(log, "wait", None, |backend| { - backend.execute(JevOperation::Wait, None, None) - }) - .await?; + let changed = self.await_change(log, LATE_LOOK_MS).await?; screen = self.look().await?; fresh = fresh_rows(&screen, &shown, field, text, &self.stop_before, place); + if !changed { + break; + } } let mentioned = fresh .iter() @@ -276,9 +283,19 @@ pub(in crate::agentic::flow) fn shares_most_words(candidate: &Candidate, text: & shared >= 2 && shared * 2 >= words.len() } -/// Looks again, a wait apart, for the rows a place box lists late. +/// Whether `row` names the text typed: all of it, or most of its words. +fn names_typed(row: &Candidate, text: &str) -> bool { + mentions(row, text) || shares_most_words(row, text) +} + +/// Looks again, after a wait for the page to change, for the rows a place +/// box lists late: until one names the text typed. const LATE_LOOKS: u32 = 2; +/// Longest one wait for a place box's late rows: the page is looked at again +/// as soon as it changes, and a still page lists nothing more. +const LATE_LOOK_MS: u64 = 1_000; + /// Words of a slot that name a place, whose box lists matches as it is /// typed in. const PLACE_WORDS: &[&str] = &[ diff --git a/crates/tinycomputer-engine/src/agentic/flow/vote.rs b/crates/tinycomputer-engine/src/agentic/flow/vote.rs index c517b5f6..0d176cea 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/vote.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/vote.rs @@ -31,7 +31,7 @@ use serde_json::Value; use tinyinference_decisions::{Answer, ChoiceAnswer, EvaluationRequest, NoulAnswer, Question}; /// Most framings one decision is asked in. -pub(super) const MAX_VOTES: u32 = 9; +pub(in crate::agentic) const MAX_VOTES: u32 = 9; /// A perspective added to each framing after the first, in turn. const PERSPECTIVES: [&str; 4] = [ @@ -169,14 +169,30 @@ fn keys_for(count: usize, index: usize) -> Vec { pub(super) fn ballots( answered: &[(Framing, BTreeMap)], ) -> BTreeMap> { + ballots_of(&answered.iter().map(|(framing, answers)| (framing, answers))) +} + +/// The [`ballots`] of the framings `answered` so far, each named by its +/// index in `framings`, in the order given. +pub(super) fn ballots_at<'a>( + framings: &'a [Framing], + answered: impl Iterator)> + Clone, +) -> BTreeMap> { + ballots_of(&answered.filter_map(|(index, answers)| Some((framings.get(index)?, answers)))) +} + +fn ballots_of<'a, I>(answered: &I) -> BTreeMap> +where + I: Iterator)> + Clone, +{ answered - .iter() + .clone() .flat_map(|(framing, _)| framing.request.questions.keys()) .collect::>() .into_iter() .map(|id| { let answers = answered - .iter() + .clone() .filter_map(|(framing, answers)| { Some(original(framing, id, answers.get(id)?.clone())) }) diff --git a/crates/tinycomputer-engine/src/agentic/journal/README.md b/crates/tinycomputer-engine/src/agentic/journal/README.md index 40721e6a..e1702b9b 100644 --- a/crates/tinycomputer-engine/src/agentic/journal/README.md +++ b/crates/tinycomputer-engine/src/agentic/journal/README.md @@ -24,11 +24,16 @@ event table, reading a run with `jev_journal`, finding latency — is the journal on or off. - **After masking.** The flow journals the request after `FlowRun::mask`, so secrets appear only as `${name}`. +- **A task's whole time.** The task controller times what happens between + its flows — planning, each rescue, each wait for a person — and hands the + event to its `FlowRunner::journal`; the module's runner writes it with + `JevRuntime::journal_event` into the task's `task-…` file, or for + `PlanTask`, which plans before a task exists, into a run of its own. ## Public surface `JevRuntime::with_journal`, `JevRuntime::journaled_as`, -`JevRuntime::journal_dir`, and the constants `JOURNAL_ENV`, +`JevRuntime::journal_dir`, `JevRuntime::journal_event`, and the constants `JOURNAL_ENV`, `JOURNAL_DEFAULT_DIR`, and `JOURNAL_FILE`, all re-exported from the crate root. The reader lives in `tinycomputer-examples` (`src/journal/`, the `jev_journal` binary), since only developers read journals. diff --git a/crates/tinycomputer-engine/src/agentic/journal/journal_tests.rs b/crates/tinycomputer-engine/src/agentic/journal/journal_tests.rs index 44850413..09b4b7e5 100644 --- a/crates/tinycomputer-engine/src/agentic/journal/journal_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/journal/journal_tests.rs @@ -220,3 +220,24 @@ fn millis_saturate() { assert_eq!(millis(Duration::from_micros(2500)), 2); assert_eq!(millis(Duration::MAX), u64::MAX); } + +#[test] +fn a_fresh_run_is_named_for_its_kind_and_opened_only_when_the_journal_is_on() { + let scratch = Scratch::new("fresh"); + let root = Journal::at(&scratch.0); + assert!(!root.is_open(), "a root is no run"); + let plan = root.fresh("plan"); + assert!(plan.is_open()); + let dir = plan.run_dir().unwrap(); + assert!( + dir.file_name() + .unwrap() + .to_string_lossy() + .contains("Z-plan-"), + "{}", + dir.display() + ); + plan.record("plan", || json!({"wall_ms": 12})); + assert_eq!(events(&dir)[0]["wall_ms"], 12); + assert!(!Journal::default().fresh("plan").is_open(), "off stays off"); +} diff --git a/crates/tinycomputer-engine/src/agentic/journal/mod.rs b/crates/tinycomputer-engine/src/agentic/journal/mod.rs index 7fd649ff..4b3d3ef4 100644 --- a/crates/tinycomputer-engine/src/agentic/journal/mod.rs +++ b/crates/tinycomputer-engine/src/agentic/journal/mod.rs @@ -128,10 +128,29 @@ impl Journal { } } + /// Whether a run is open to write to. + pub(crate) fn is_open(&self) -> bool { + self.run.is_some() + } + + /// Whether the journal is on: it has a folder to write runs under. + pub(crate) fn is_on(&self) -> bool { + self.root.is_some() + } + + /// This journal, writing to a new run named for the time and `kind`. A + /// journal that is off stays off. + pub(crate) fn fresh(&self, kind: &str) -> Self { + if !self.is_on() { + return self.clone(); + } + self.named(&fresh_id(kind)) + } + /// This journal with a run begun: a `run` event in the current run, or, /// when there is none yet, in a new one named for the time and `kind`. pub(crate) fn begin(&self, kind: &str, label: &str, model: &str) -> Self { - let journal = if self.run.is_some() { + let journal = if self.run.is_some() || !self.is_on() { self.clone() } else { self.named(&fresh_id(kind)) diff --git a/crates/tinycomputer-engine/src/agentic/runtime.rs b/crates/tinycomputer-engine/src/agentic/runtime.rs index efa3284b..41c7bb09 100644 --- a/crates/tinycomputer-engine/src/agentic/runtime.rs +++ b/crates/tinycomputer-engine/src/agentic/runtime.rs @@ -5,16 +5,17 @@ use std::{ collections::HashMap, future::Future, pin::Pin, - sync::{Arc, Mutex}, + sync::{Arc, Mutex, atomic::AtomicUsize}, time::Duration, }; use tinycomputer_bus::{DesktopError, JevConfig, JevConfiguration, JevProvider}; use tinyinference_decisions::{ Client, ClientConfig, Error as JevError, EvaluationFailure, EvaluationRequest, - EvaluationResult, RetryPolicy, + EvaluationResult, Noul, Question, RetryPolicy, }; +use super::flow::MAX_VOTES; use super::journal::Journal; use super::pending::PendingRun; use super::sage; @@ -29,6 +30,20 @@ pub(super) const RETRY: RetryPolicy = RetryPolicy { max_backoff: Duration::from_secs(8), }; +/// How long one Jev attempt may take when the configuration does not say. +/// Live, the slowest answer took 8.6 s, and a request nothing came back for +/// waited the client's own 30 s before its retry answered in under 1 s. +pub(super) const ATTEMPT_TIMEOUT: Duration = Duration::from_secs(10); + +/// The longest [`JevRuntime::warm`] waits for its answers before it gives +/// its calls up. +pub(super) const WARM_TIMEOUT: Duration = Duration::from_secs(10); + +/// Requests a step's first turn asks at once, each in every framing: its +/// judging, and grounding's opening for the move it almost always makes +/// (`flow::act::judge`). +pub(super) const FIRST_TURN: u32 = 2; + /// Configured Jev transport and non-secret policy metadata. #[derive(Clone)] pub struct JevRuntime { @@ -36,6 +51,9 @@ pub struct JevRuntime { pub(super) configuration: JevConfiguration, pub(super) pending: Arc>>, pub(super) journal: Journal, + /// Hedged copies of framings in flight, shared by every flow run on + /// this runtime. + pub(super) copies: Arc, } impl std::fmt::Debug for JevRuntime { @@ -46,6 +64,7 @@ impl std::fmt::Debug for JevRuntime { .field("configuration", &self.configuration) .field("pending", &"[redacted]") .field("journal", &self.journal) + .field("copies", &self.copies) .finish() } } @@ -91,6 +110,7 @@ impl JevRuntime { }, pending: Arc::new(Mutex::new(HashMap::new())), journal: Journal::from_env(), + copies: Arc::default(), }) } @@ -133,6 +153,7 @@ impl JevRuntime { }, pending: Arc::new(Mutex::new(HashMap::new())), journal: Journal::from_env(), + copies: Arc::default(), }) } @@ -166,6 +187,31 @@ impl JevRuntime { } } + /// Writes one event of `kind` with `fields` to the debug journal: into + /// this runtime's open run (see [`JevRuntime::journaled_as`]), or, with + /// none open, into a new run named for the time and `kind`. It is for + /// time a task spends outside its flows, such as planning, a rescue, or + /// waiting on a person, so the task's journal accounts for all of its + /// time. `fields` is only built when recording: nothing is, and no run + /// is opened, when the journal is off. + pub fn journal_event(&self, kind: &str, fields: impl FnOnce() -> serde_json::Value) { + if !self.journaling() { + return; + } + let journal = if self.journal.is_open() { + self.journal.clone() + } else { + self.journal.fresh(kind) + }; + journal.record(kind, fields); + } + + /// Whether this runtime's debug journal is on. + #[must_use] + pub fn journaling(&self) -> bool { + self.journal.is_on() + } + /// The directory this runtime's current run journal is written to, if /// the journal is on and a run has begun. #[must_use] @@ -186,6 +232,32 @@ impl JevRuntime { } } + /// Opens connections to Jev for the decisions to come, while a task's + /// plan is drafted: one for each call of a step's first turn, which asks + /// its judging and grounding's opening together (`FIRST_TURN`), each in + /// `votes` framings (at most 9, `MAX_VOTES`). Each connection is opened + /// with a one-question evaluation, all at once, journaled as `warm-up`; + /// its answer is dropped, and those still out after 10 s + /// (`WARM_TIMEOUT`) are given up on. A decision asks its framings all + /// at once, each on a connection of its own; live, a task's first + /// decision took 330 ms more a call than its later ones, opening them. + /// Sage, whose calls take seconds, is not warmed. + pub async fn warm(&self, votes: u32) { + if self.configuration.provider == JevProvider::Sage { + return; + } + let request = Arc::new(warm_up(&self.configuration.model)); + let mut calls = tokio::task::JoinSet::new(); + for _ in 0..votes.clamp(1, MAX_VOTES).saturating_mul(FIRST_TURN) { + let (runtime, request) = (self.clone(), Arc::clone(&request)); + calls.spawn(async move { + let _answer = runtime.evaluate(Some("warm-up"), &request).await; + }); + } + // Calls still out when the time is up are dropped with the set. + let _warmed = tokio::time::timeout(WARM_TIMEOUT, calls.join_all()).await; + } + /// Asks Jev one request, journaling the exchange against `step`. pub(super) async fn evaluate( &self, @@ -198,6 +270,23 @@ impl JevRuntime { } } +/// The smallest request `model` answers: one yes/no question about nothing +/// on any screen. +fn warm_up(model: &str) -> EvaluationRequest { + EvaluationRequest { + state: serde_json::json!("A connection check."), + model: model.to_owned(), + questions: [( + "ready".to_owned(), + Question::Noul(Noul { + instructions: serde_json::json!("The state is a connection check."), + criteria: None, + }), + )] + .into(), + } +} + pub(super) trait Evaluator: Send + Sync { fn evaluate<'a>( &'a self, @@ -270,8 +359,9 @@ pub(super) fn trusted_endpoint(provider: JevProvider, endpoint: &str) -> bool { } /// The HTTP client configuration for a Jev `request`: its provider's -/// route, endpoint, timeout, and attribution, retrying as [`RETRY`] unless -/// the request sets its own number of retries. +/// route, endpoint, timeout ([`ATTEMPT_TIMEOUT`] unless the request sets +/// one), and attribution, retrying as [`RETRY`] unless the request sets its +/// own number of retries. pub(super) fn client_config(request: &JevConfig) -> ClientConfig { let mut config = match request.provider { JevProvider::TypeSafe => ClientConfig::new(request.api_key()), @@ -283,9 +373,9 @@ pub(super) fn client_config(request: &JevConfig) -> ClientConfig { if let Some(endpoint) = &request.endpoint_url { config = config.with_endpoint_url(endpoint); } - if let Some(timeout_ms) = request.timeout_ms { - config.timeout = Duration::from_millis(timeout_ms); - } + config.timeout = request + .timeout_ms + .map_or(ATTEMPT_TIMEOUT, Duration::from_millis); config.retry = RETRY; if let Some(max_retries) = request.max_retries { config.retry.max_retries = max_retries; diff --git a/crates/tinycomputer-engine/src/lib.rs b/crates/tinycomputer-engine/src/lib.rs index 48b57f79..d4786a89 100644 --- a/crates/tinycomputer-engine/src/lib.rs +++ b/crates/tinycomputer-engine/src/lib.rs @@ -41,7 +41,7 @@ pub use agentic::{ JOURNAL_DEFAULT_DIR, JOURNAL_ENV, JOURNAL_FILE, JevRuntime, flow_guide, resolve_intent, run_flow, run_goal, validate_flow, }; -pub use planner::{Completion, LanguageModel, Planner, REPAIRS, Role, Turn}; +pub use planner::{Completion, LanguageModel, ModelUse, Planner, REPAIRS, Role, Turn}; #[cfg(feature = "planner")] pub use planner::{ ModelRoute, OPEN_ROUTER_BASE_URL, OUTPUT_MODEL, PLANNER_MODEL, PlannerConfig, RESCUE_MODEL, @@ -50,7 +50,8 @@ pub use planner::{ pub use rescue::{Briefing, Guidance, MAX_RESCUE_STEPS, MAX_RESCUES, Rescuer, SCREEN_CHARS}; pub use shape::{Harvest, RECORDS_CHARS, Shaper}; pub use task::{ - CaptureFuture, FlowFuture, FlowRunner, MAX_AWAIT_MS, MAX_TASKS, Tasks, TextFuture, capabilities, + CaptureFuture, FlowFuture, FlowRunner, MAX_AWAIT_MS, MAX_TASKS, PrepareFuture, Tasks, + TextFuture, capabilities, }; pub use tinycomputer_bus::DesktopResponse; use tinycomputer_desktop::Desktop; diff --git a/crates/tinycomputer-engine/src/planner/hosted.rs b/crates/tinycomputer-engine/src/planner/hosted.rs index a59e45b9..ecb817dd 100644 --- a/crates/tinycomputer-engine/src/planner/hosted.rs +++ b/crates/tinycomputer-engine/src/planner/hosted.rs @@ -6,14 +6,19 @@ //! feature. The keys arrive in the module's private configuration and never //! leave this adapter. +use std::future::Future; use std::sync::Arc; +use std::time::Duration; use tinycomputer_bus::agent::LanguageModelProvider; use tinyinference_llm::model::{ ReasoningConfig, ReasoningEffort, ResponseFormat, collect_model_stream, }; use tinyinference_llm::providers::openai::OpenAiModel; -use tinyinference_llm::{ChatModel, Message, ModelRequest, ProviderKind, ProviderSpec}; +use tinyinference_llm::{ + ChatModel, Error, Message, ModelRequest, ProviderKind, ProviderSpec, classify_provider_error, + classify_provider_failure, +}; use super::config::{ModelRoute, OUTPUT_MODEL, PLANNER_MODEL, PlannerConfig, RESCUE_MODEL}; use super::{Completion, LanguageModel, Planner, Role, Turn}; @@ -142,14 +147,54 @@ impl LanguageModel for Hosted { // for 60 seconds, and a reasoning model's whole reply can take // longer. A route that ignores streaming still answers in one // piece. - let stream = model - .stream(&(), request) - .await - .map_err(|error| error.to_string())?; - let response = collect_model_stream(stream) - .await - .map_err(|error| error.to_string())?; - Ok(Message::Assistant(response.message).text()) + with_retries(|| { + let model = model.clone(); + let request = request.clone(); + async move { + let stream = model.stream(&(), request).await?; + let response = collect_model_stream(stream).await?; + Ok(Message::Assistant(response.message).text()) + } + }) + .await + .map_err(|error| error.to_string()) }) } } + +/// How many times one model call is tried when it fails in passing — a +/// gateway's 502, a rate limit, a dropped connection — waiting 1, 2, then 4 +/// seconds between tries. Live, one 502 from Tiny Humans' gateway ended a +/// task at its plan, 3 s in. +const MODEL_TRIES: u32 = 4; + +/// Runs `call` until it answers, fails in a way another try cannot mend, or +/// has failed [`MODEL_TRIES`] times. +pub(super) async fn with_retries(mut call: C) -> Result +where + C: FnMut() -> F, + F: Future>, +{ + let mut tries = 1; + loop { + match call().await { + Err(error) if tries < MODEL_TRIES && passing(&error) => { + tokio::time::sleep(Duration::from_secs(1 << (tries - 1))).await; + tries += 1; + } + outcome => return outcome, + } + } +} + +/// Whether `error` may pass on another try: a server error, a rate limit +/// that is not a spending cap, or a dropped connection, but never a +/// refused key, a request the route rejects, or a reply that would not +/// parse. +pub(super) fn passing(error: &Error) -> bool { + match error { + Error::Provider(provider) => classify_provider_error(provider).is_retryable(), + Error::Model(message) => classify_provider_failure(None, None, message).is_retryable(), + _ => false, + } +} diff --git a/crates/tinycomputer-engine/src/planner/mod.rs b/crates/tinycomputer-engine/src/planner/mod.rs index 8f5655f6..3bddd6fe 100644 --- a/crates/tinycomputer-engine/src/planner/mod.rs +++ b/crates/tinycomputer-engine/src/planner/mod.rs @@ -50,6 +50,29 @@ pub enum Role { Assistant, } +/// What one plan or rescue used of its model: the calls it made, the first +/// and each repair of a refused answer, and the bytes the first call sent. +/// The task journals it, so a run shows what its planning and rescues cost. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct ModelUse { + /// Calls made: the first, and one per repair. A hosted call tried again + /// after a passing failure (`hosted.rs`) counts once: its tries, and + /// the waits between them, are inside it. + pub calls: u32, + /// Bytes of text in the first call's turns. + pub sent_bytes: usize, +} + +impl ModelUse { + /// The use of a conversation about to be sent for the first time. + pub(crate) fn starting(turns: &[Turn]) -> Self { + Self { + calls: 0, + sent_bytes: turns.iter().map(|turn| turn.text.len()).sum(), + } + } +} + /// One message in a planning conversation. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Turn { @@ -152,6 +175,20 @@ impl Planner { secret_names: &[String], surfaces: &[SurfaceKind], ) -> Result { + self.plan_measured(task, fact_names, secret_names, surfaces) + .await + .0 + } + + /// [`Planner::plan`], with what it used of its model, whether or not a + /// plan came back. + pub async fn plan_measured( + &self, + task: &str, + fact_names: &[String], + secret_names: &[String], + surfaces: &[SurfaceKind], + ) -> (Result, ModelUse) { let surfaces = if surfaces.is_empty() { "the web browser and desktop applications".to_owned() } else { @@ -200,9 +237,14 @@ impl Planner { ), ), ]; + let mut used = ModelUse::starting(&turns); let mut last = String::new(); for _ in 0..=REPAIRS { - let reply = self.model.complete(&turns).await?; + used.calls += 1; + let reply = match self.model.complete(&turns).await { + Ok(reply) => reply, + Err(error) => return (Err(error), used), + }; turns.push(Turn::new(Role::Assistant, reply.clone())); let problem = match parse(&reply) { Ok(flow) => { @@ -212,7 +254,7 @@ impl Planner { .filter(|error| !error.contains("` is not defined in `vars`")) .collect::>(); if errors.is_empty() { - return Ok(plan_for(flow, &known, &secrets)); + return (Ok(plan_for(flow, &known, &secrets)), used); } format!("That flow is invalid:\n- {}", errors.join("\n- ")) } @@ -224,7 +266,10 @@ impl Planner { format!("{problem}\nReply with the corrected flow only, as one JSON object."), )); } - Err(format!("the planner did not produce a valid flow: {last}")) + ( + Err(format!("the planner did not produce a valid flow: {last}")), + used, + ) } } diff --git a/crates/tinycomputer-engine/src/planner/planner_tests.rs b/crates/tinycomputer-engine/src/planner/planner_tests.rs index 5bd1b70d..1e057c28 100644 --- a/crates/tinycomputer-engine/src/planner/planner_tests.rs +++ b/crates/tinycomputer-engine/src/planner/planner_tests.rs @@ -7,7 +7,7 @@ use std::sync::{Arc, Mutex}; use tinycomputer_bus::agent::{InputKind, SurfaceKind}; -use super::{Completion, LanguageModel, Planner, REPAIRS, Role, Turn}; +use super::{Completion, LanguageModel, ModelUse, Planner, REPAIRS, Role, Turn}; /// Answers from a queue and records every conversation it was shown. #[derive(Default)] @@ -173,6 +173,36 @@ async fn a_plan_that_never_validates_or_a_failed_model_is_an_error() { assert!(planner.plan("x", &[], &[], &[]).await.is_err()); } +#[tokio::test] +async fn a_measured_plan_counts_each_call_and_what_the_first_one_sent() { + let (planner, model) = scripted(&[ + Ok("I would search for flights."), + Ok(r#"{"app": "Mail", "steps": ["start a new email message"]}"#), + ]); + let (plan, used) = planner.plan_measured("write an email", &[], &[], &[]).await; + assert_eq!(plan.unwrap().flow.app, "Mail"); + let first = model.seen.lock().unwrap()[0].clone(); + assert_eq!( + used, + ModelUse { + calls: 2, + sent_bytes: first.iter().map(|turn| turn.text.len()).sum(), + }, + "one repair after the refused answer" + ); + + let (planner, _) = scripted(&[Ok("{"), Err("rate limited")]); + let (plan, used) = planner.plan_measured("x", &[], &[], &[]).await; + assert_eq!(plan.unwrap_err(), "rate limited"); + assert_eq!(used.calls, 2, "a failed call still counts"); + + let never = [Ok(r#"{"app": "", "steps": []}"#); REPAIRS + 1]; + let (planner, _) = scripted(&never); + let (plan, used) = planner.plan_measured("x", &[], &[], &[]).await; + assert!(plan.is_err()); + assert_eq!(used.calls, u32::try_from(REPAIRS).unwrap() + 1); +} + #[cfg(feature = "planner")] #[tokio::test] async fn the_open_router_planner_needs_a_key_and_never_prints_it() { @@ -237,5 +267,7 @@ async fn the_open_router_planner_needs_a_key_and_never_prints_it() { assert!(!failed.contains("secret-key")); } +#[cfg(feature = "planner")] +mod retry_tests; #[cfg(feature = "planner")] mod route_tests; diff --git a/crates/tinycomputer-engine/src/planner/planner_tests/retry_tests.rs b/crates/tinycomputer-engine/src/planner/planner_tests/retry_tests.rs new file mode 100644 index 00000000..a9cdd7ee --- /dev/null +++ b/crates/tinycomputer-engine/src/planner/planner_tests/retry_tests.rs @@ -0,0 +1,96 @@ +//! Tests for trying a hosted model call again: a gateway's passing failure +//! is tried again, a refusal is not, and the tries are bounded. + +use std::cell::Cell; + +use tinyinference_llm::Error; +use tinyinference_llm::model::ProviderError; + +use super::super::hosted::{passing, with_retries}; + +fn bad_gateway() -> Error { + Error::Model("tinyhumans returned HTTP 502: error code: 502".to_owned()) +} + +#[tokio::test(start_paused = true)] +async fn a_model_call_is_tried_again_after_a_gateway_error() { + // Live, one 502 from Tiny Humans' gateway ended a task at its plan. + let tries = Cell::new(0); + let started = tokio::time::Instant::now(); + let answer = with_retries(|| { + tries.set(tries.get() + 1); + let failing = tries.get() < 3; + async move { + if failing { + Err(bad_gateway()) + } else { + Ok("the plan") + } + } + }) + .await; + assert_eq!(answer.unwrap(), "the plan"); + assert_eq!(tries.get(), 3); + assert_eq!( + started.elapsed(), + std::time::Duration::from_secs(3), + "1 s, then 2 s apart" + ); +} + +#[tokio::test(start_paused = true)] +async fn a_model_call_that_keeps_failing_gives_up_after_its_tries() { + let tries = Cell::new(0); + let answer = with_retries(|| { + tries.set(tries.get() + 1); + async { Err::<(), _>(bad_gateway()) } + }) + .await; + assert!(answer.is_err()); + assert_eq!(tries.get(), 4); +} + +#[tokio::test(start_paused = true)] +async fn a_refused_model_call_is_not_tried_again() { + for refusal in [ + Error::Model("tinyhumans returned HTTP 401: invalid key".to_owned()), + Error::Validation("network-backed model calls are denied".to_owned()), + ] { + let message = refusal.to_string(); + let mut refusal = Some(refusal); + let tries = Cell::new(0); + let answer = with_retries(|| { + tries.set(tries.get() + 1); + let error = refusal.take().expect("asked once"); + async move { Err::<(), _>(error) } + }) + .await; + assert!(answer.is_err()); + assert_eq!(tries.get(), 1, "{message}"); + } +} + +#[test] +fn a_failure_passes_by_its_kind_and_what_the_provider_says() { + assert!(passing(&bad_gateway())); + assert!(passing(&Error::Model( + "connection reset by peer".to_owned() + ))); + let unavailable = ProviderError { + provider: "openai".to_owned(), + status: Some(503), + message: "upstream unavailable".to_owned(), + retryable: true, + ..ProviderError::default() + }; + assert!(passing(&Error::Provider(Box::new(unavailable.clone())))); + let final_word = ProviderError { + retryable: false, + ..unavailable + }; + assert!( + !passing(&Error::Provider(Box::new(final_word))), + "a provider that says it will not pass is believed" + ); + assert!(!passing(&Error::Unsupported("tools".to_owned()))); +} diff --git a/crates/tinycomputer-engine/src/rescue/mod.rs b/crates/tinycomputer-engine/src/rescue/mod.rs index 4ffc382d..02e81fbb 100644 --- a/crates/tinycomputer-engine/src/rescue/mod.rs +++ b/crates/tinycomputer-engine/src/rescue/mod.rs @@ -27,7 +27,7 @@ use std::sync::Arc; use tinycomputer_bus::agent::{LanguageModelConfiguration, Rescue}; use tinycomputer_bus::{FLOW_GUIDE, Flow, FlowStep, StepReport}; -use crate::planner::{LanguageModel, REPAIRS, Role, Turn}; +use crate::planner::{LanguageModel, ModelUse, REPAIRS, Role, Turn}; use judge::judge; pub(crate) use judge::resumed; use render::render; @@ -181,16 +181,30 @@ impl Rescuer { /// Why no guidance came back: the model failed, or its answer stayed /// invalid after [`REPAIRS`] repairs. pub async fn guide(&self, briefing: &Briefing) -> Result { + self.guide_measured(briefing).await.0 + } + + /// [`Rescuer::guide`], with what it used of its model, whether or not + /// guidance came back: a call refused as invalid guidance costs a repair. + pub async fn guide_measured( + &self, + briefing: &Briefing, + ) -> (Result, ModelUse) { let mut turns = vec![ Turn::new(Role::System, format!("{PROTOCOL}\n\n{FLOW_GUIDE}")), Turn::new(Role::User, render(briefing)), ]; + let mut used = ModelUse::starting(&turns); let mut last = String::new(); for _ in 0..=REPAIRS { - let reply = self.model.complete(&turns).await?; + used.calls += 1; + let reply = match self.model.complete(&turns).await { + Ok(reply) => reply, + Err(error) => return (Err(error), used), + }; turns.push(Turn::new(Role::Assistant, reply.clone())); let problem = match judge(&reply, briefing) { - Ok(guidance) => return Ok(guidance), + Ok(guidance) => return (Ok(guidance), used), Err(problem) => problem, }; last.clone_from(&problem); @@ -199,7 +213,10 @@ impl Rescuer { format!("{problem}\nReply with the corrected answer only, as one JSON object."), )); } - Err(format!("the rescuer gave no valid guidance: {last}")) + ( + Err(format!("the rescuer gave no valid guidance: {last}")), + used, + ) } } diff --git a/crates/tinycomputer-engine/src/rescue/rescue_tests.rs b/crates/tinycomputer-engine/src/rescue/rescue_tests.rs index 2ed52ee4..e04db5aa 100644 --- a/crates/tinycomputer-engine/src/rescue/rescue_tests.rs +++ b/crates/tinycomputer-engine/src/rescue/rescue_tests.rs @@ -195,6 +195,32 @@ async fn invalid_guidance_is_sent_back_with_what_is_wrong() { ); } +#[tokio::test] +async fn a_measured_rescue_counts_each_call_refused_guidance_included() { + let unknown = + r#"{"action": "retry", "reason": "x", "steps": [{"enter": {"name": "${full name}"}}]}"#; + let (rescuer, model) = scripted(&[Ok(unknown), Ok(FIX)]); + let (guidance, used) = rescuer.guide_measured(&briefing()).await; + assert!(matches!(guidance, Ok(Guidance::Retry { .. }))); + assert_eq!(used.calls, 2, "the refused guidance cost a repair"); + let first = model.seen.lock().unwrap()[0].clone(); + assert_eq!( + used.sent_bytes, + first.iter().map(|turn| turn.text.len()).sum::() + ); + + let (rescuer, _) = scripted(&[Err("the model is down")]); + let (guidance, used) = rescuer.guide_measured(&briefing()).await; + assert_eq!(guidance.unwrap_err(), "the model is down"); + assert_eq!(used.calls, 1); + + let never = [Ok("nonsense"); REPAIRS + 1]; + let (rescuer, _) = scripted(&never); + let (guidance, used) = rescuer.guide_measured(&briefing()).await; + assert!(guidance.is_err()); + assert_eq!(used.calls, u32::try_from(REPAIRS).unwrap() + 1); +} + #[test] fn the_briefing_shows_earlier_rescues_and_cuts_a_long_screen() { let mut briefing = briefing(); diff --git a/crates/tinycomputer-engine/src/task/controller.rs b/crates/tinycomputer-engine/src/task/controller.rs index f9ed1381..66812c79 100644 --- a/crates/tinycomputer-engine/src/task/controller.rs +++ b/crates/tinycomputer-engine/src/task/controller.rs @@ -116,15 +116,21 @@ impl Tasks { let Some(planner) = &self.planner else { return AgentResponse::err(no_planner()); }; - match planner - .plan( + let started = std::time::Instant::now(); + let (outcome, used) = planner + .plan_measured( &request.task, &request.fact_names, &request.secret_facts, &request.surfaces, ) - .await - { + .await; + // No task exists yet, so the plan journals to a run of its own. + let drafting = started.elapsed(); + self.runner.journal(None, "plan", &|| { + super::timing::planned(&outcome, used, drafting, planner.configuration()) + }); + match outcome { Ok(plan) => AgentResponse::ok(plan), Err(reason) => AgentResponse::err(AgentError::new( "PLAN_FAILED", @@ -261,7 +267,15 @@ impl Tasks { return no_such_task(&request.id); }; let status = cell.view.borrow().status.clone(); - match status { + let waited = cell + .state + .lock() + .ok() + .and_then(|state| state.waiting_since) + .map(|since| since.elapsed()); + let id = request.id.clone(); + let paused = state_name(&status); + let reply = match status { TaskStatus::NeedsInput { .. } => self.supply(&cell, request), TaskStatus::NeedsApproval { .. } => self.decide(&cell, request.approve), TaskStatus::NeedsHuman { .. } => self.retry(&cell), @@ -274,7 +288,19 @@ impl Tasks { "call AwaitTask until the task asks for something", true, )), + }; + // The wait is over only once the task has left it: an answer that is + // refused, or that still leaves values missing, keeps it waiting. + let left = cell + .state + .lock() + .is_ok_and(|state| state.waiting_since.is_none()); + if let (Some(waited), true) = (waited, left) { + self.runner.journal(Some(&id), "resume", &|| { + super::timing::resumed(paused, waited) + }); } + reply } /// Stops a task, and lets go of whatever surface it still holds. diff --git a/crates/tinycomputer-engine/src/task/drive.rs b/crates/tinycomputer-engine/src/task/drive.rs index fa61e10d..ea9ec2c0 100644 --- a/crates/tinycomputer-engine/src/task/drive.rs +++ b/crates/tinycomputer-engine/src/task/drive.rs @@ -5,7 +5,7 @@ use std::collections::VecDeque; use std::sync::Arc; use std::time::{Duration, Instant}; -use tinycomputer_bus::agent::TaskStatus; +use tinycomputer_bus::agent::{SurfaceKind, TaskConstraints, TaskStatus}; use super::artifact::{capture, captured}; use super::budget::{elapsed_budget_failed, run_request, stop_task}; @@ -14,7 +14,7 @@ use super::interpret::{Next, finished, run_outcome}; use super::publish::{publish, records, stopped_summary}; use super::recovery::{rescue_outcome, rescued}; use super::store::{Cell, Run}; -use super::{FlowRunner, SHAPE_TIMEOUT_MS}; +use super::{DEFAULT_VOTES, FlowRunner, SHAPE_TIMEOUT_MS}; use crate::shape::Harvest; /// Plans the task, then runs the plan — or asks for what it needs first. @@ -25,19 +25,68 @@ pub(super) async fn plan_then_drive( task: String, surfaces: Vec, ) { - let (names, secrets) = cell.state.lock().map_or_else( - |_| (Vec::new(), Vec::new()), + let (names, secrets, constraints, votes, capped) = cell.state.lock().map_or_else( + |_| { + ( + Vec::new(), + Vec::new(), + TaskConstraints::default(), + DEFAULT_VOTES, + false, + ) + }, |state| { let owned = |names: Vec<&str>| names.into_iter().map(str::to_owned).collect::>(); ( owned(state.facts.names()), owned(state.facts.secret_names()), + state.constraints.clone(), + state.budget.votes.unwrap_or(DEFAULT_VOTES), + state.budget.max_model_calls.is_some(), ) }, ); - let plan = match planner.plan(&task, &names, &secrets, &surfaces).await { + let id = cell.view.borrow().id.clone(); + // Jev's connections open while the plan is drafted, so the first + // decision need not open them; nothing waits for this. A budget that + // caps the task's Jev calls is not spent on calls it does not count. + if !capped { + tokio::spawn(runner.warm(&id, votes)); + } + let started = Instant::now(); + // Timed on its own: a browser slower to open than the plan is to draft + // is not planning time. + let planning = async { + let drafted = planner + .plan_measured(&task, &names, &secrets, &surfaces) + .await; + (drafted, started.elapsed()) + }; + // A browser-only task's browser opens while the plan is drafted, so the + // first step need not wait for it: whatever the plan says, it runs there. + // The one page the task names loads in it meanwhile, for a first step + // that browses there. + let ((outcome, used), drafting) = if constraints.surfaces == [SurfaceKind::Browser] { + let page = super::page::named_page(&task); + let preparing = async { + runner.prepare(&id, &constraints).await; + if let Some(page) = &page { + runner.open_page(&id, page).await; + } + }; + tokio::join!(planning, preparing).0 + } else { + planning.await + }; + runner.journal(Some(&id), "plan", &|| { + super::timing::planned(&outcome, used, drafting, planner.configuration()) + }); + let plan = match outcome { Ok(plan) => plan, Err(reason) => { + // A browser opened while planning has nothing left to do; it is + // let go before the failure is told, as a finished task's is. + runner.release(&id); publish( &cell, TaskStatus::Failed { @@ -71,6 +120,10 @@ pub(super) async fn plan_then_drive( ) .await; } else { + // A browser opened while planning is let go while the task waits on + // a person, who may take long or never answer; the run that follows + // opens one again. + runner.release(&id); publish( &cell, TaskStatus::NeedsInput { diff --git a/crates/tinycomputer-engine/src/task/mod.rs b/crates/tinycomputer-engine/src/task/mod.rs index fe15bdae..368f1b96 100644 --- a/crates/tinycomputer-engine/src/task/mod.rs +++ b/crates/tinycomputer-engine/src/task/mod.rs @@ -46,10 +46,12 @@ mod errors; mod human; mod interpret; mod names; +mod page; mod publish; mod recovery; mod resume; mod store; +mod timing; use std::future::Future; use std::pin::Pin; @@ -72,6 +74,10 @@ pub type TextFuture = Pin> + Send>>; /// task's surface could take one. pub type CaptureFuture = Pin> + Send>>; +/// The future [`FlowRunner::prepare`] returns, once the surfaces are ready +/// or could not be made so; a task goes on either way. +pub type PrepareFuture = Pin + Send>>; + /// Runs a task's flows, on surfaces that live as long as the task. pub trait FlowRunner: Send + Sync + 'static { /// Runs `request` for `task` within `constraints`, returning `RunFlow`'s @@ -106,6 +112,46 @@ pub trait FlowRunner: Send + Sync + 'static { /// Lets go of whatever the task held, once it has ended. fn release(&self, _task: &TaskId) {} + + /// Gets the task's surfaces ready while its plan is drafted, so its + /// first step does not wait for them: called alongside the planner for + /// a task that runs on the browser alone. [`FlowRunner::release`] may + /// run while the future is in flight, or after it was dropped with a + /// cancelled task: what it opens then must be let go too. Does nothing + /// by default. + fn prepare(&self, _task: &TaskId, _constraints: &TaskConstraints) -> PrepareFuture { + Box::pin(async {}) + } + + /// Loads `url`, the one web page the task's text names, in the browser + /// [`FlowRunner::prepare`] got ready, while the plan is drafted: a first + /// step that browses there finds it loaded. Called after `prepare`, in + /// the same wait, for a task that runs on the browser alone. Does + /// nothing by default. + fn open_page(&self, _task: &TaskId, _url: &str) -> PrepareFuture { + Box::pin(async {}) + } + + /// Opens the connections the task's first decision, asked `votes` ways, + /// will use, while its plan is drafted (`JevRuntime::warm`). Started + /// with the planner for every task and never waited for: the task goes + /// on whether it finishes or not. Does nothing by default. + fn warm(&self, _task: &TaskId, _votes: u32) -> PrepareFuture { + Box::pin(async {}) + } + + /// Writes an `event` of the time a task spends outside its flows + /// (`plan`, `rescue`, `resume`) to the debug journal: the task's own, + /// or for `PlanTask`, which plans before any task exists (`task` is + /// `None`), a run of its own. `fields` is only called when the event is + /// written: nothing is built by default, or when the journal is off. + fn journal( + &self, + _task: Option<&TaskId>, + _event: &str, + _fields: &dyn Fn() -> serde_json::Value, + ) { + } } /// How many tasks the controller holds; finished ones are dropped first. diff --git a/crates/tinycomputer-engine/src/task/page.rs b/crates/tinycomputer-engine/src/task/page.rs new file mode 100644 index 00000000..5f8330bd --- /dev/null +++ b/crates/tinycomputer-engine/src/task/page.rs @@ -0,0 +1,85 @@ +//! The one web page a task's own words send its browser to, which the +//! browser can load while the plan is drafted ([`FlowRunner::open_page`]). +//! +//! [`FlowRunner::open_page`]: super::FlowRunner::open_page + +/// Punctuation that can follow an address in a sentence, and is no part of +/// it. +const TRAILING: &[char] = &['.', ',', ';', ':', '!', '?', ')', ']', '}', '\'', '"']; + +/// Punctuation that can come between an address and the words before it. +const OPENING: &[char] = &['(', '[', '<', '"', '\'', ':']; + +/// Words that, right before an address, send the browser there: the task +/// starts on that page, rather than reading, checking, or passing on an +/// address it only mentions. +const SENDS_TO: &[&str] = &[ + "go to", + "goto", + "open", + "visit", + "navigate to", + "browse to", + "head to", + "start at", + "start on", + "on", + "at", +]; + +/// The one web address `task` names, written out with its scheme +/// (`https://…` or `http://…`) and without the punctuation after it, when +/// the words before it send the browser there and it carries no query or +/// fragment, which can hold a token a load would spend. `None` when it names +/// none, or several, which leave no one page to start on. An address written +/// twice, with a trailing slash or without, is one. +pub(super) fn named_page(task: &str) -> Option { + let mut named: Vec<(&str, &str)> = Vec::new(); + let mut from = 0; + while let Some(found) = scheme_at(&task[from..]) { + let start = from + found; + let end = task[start..] + .find(char::is_whitespace) + .map_or(task.len(), |length| start + length); + from = end; + let address = task[start..end].trim_end_matches(TRAILING); + let has_host = address + .split_once("://") + .is_some_and(|(_, rest)| !rest.trim_start_matches('/').is_empty()); + let seen = named + .iter() + .any(|(seen, _)| seen.trim_end_matches('/') == address.trim_end_matches('/')); + if has_host && !seen { + named.push((address, &task[..start])); + } + } + match named.as_slice() { + [(page, before)] if sent_to(before) && !page.contains(['?', '#']) => { + Some((*page).to_owned()) + } + _ => None, + } +} + +/// Where the next `https://` or `http://` in `text` starts. +fn scheme_at(text: &str) -> Option { + [text.find("https://"), text.find("http://")] + .into_iter() + .flatten() + .min() +} + +/// Whether `before`, a task's words up to an address, ends with words that +/// send the browser there. +fn sent_to(before: &str) -> bool { + let before = before + .trim_end_matches(|character: char| { + character.is_whitespace() || OPENING.contains(&character) + }) + .to_lowercase(); + SENDS_TO.iter().any(|words| { + before + .strip_suffix(words) + .is_some_and(|ahead| ahead.is_empty() || ahead.ends_with(char::is_whitespace)) + }) +} diff --git a/crates/tinycomputer-engine/src/task/publish.rs b/crates/tinycomputer-engine/src/task/publish.rs index 6c513929..d0580dff 100644 --- a/crates/tinycomputer-engine/src/task/publish.rs +++ b/crates/tinycomputer-engine/src/task/publish.rs @@ -25,7 +25,22 @@ pub(super) fn stopped_summary(status: &TaskStatus) -> String { /// Updates a task's view: status, summary, progress, step, and next calls. pub(super) fn publish(cell: &Cell, status: TaskStatus, summary: &str) { - let (progress, step) = cell.state.lock().map_or((0.0, None), |state| { + let waits = matches!( + status, + TaskStatus::NeedsInput { .. } + | TaskStatus::NeedsApproval { .. } + | TaskStatus::NeedsHuman { .. } + ); + let (progress, step) = cell.state.lock().map_or((0.0, None), |mut state| { + // The wait starts when the task first asks; publishing the same + // pause again does not restart it. + state.waiting_since = if waits { + state + .waiting_since + .or_else(|| Some(std::time::Instant::now())) + } else { + None + }; let total = state.flow.steps.len().max(1); let fraction = |count: usize| f32::from(u16::try_from(count).unwrap_or(u16::MAX)); let progress = fraction(state.finished.min(total)) / fraction(total); diff --git a/crates/tinycomputer-engine/src/task/recovery.rs b/crates/tinycomputer-engine/src/task/recovery.rs index 1b61adcd..8ed8a3a6 100644 --- a/crates/tinycomputer-engine/src/task/recovery.rs +++ b/crates/tinycomputer-engine/src/task/recovery.rs @@ -4,7 +4,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::time::{Duration, Instant}; -use tinycomputer_bus::agent::{Rescue, RescueOutcome, TaskStatus}; +use tinycomputer_bus::agent::{Rescue, RescueOutcome, TaskId, TaskStatus}; use tinycomputer_bus::{Flow, FlowStep, StepOutcome, StepReport}; use tinycomputer_core::Facts; @@ -12,8 +12,9 @@ use super::brief::brief; use super::names::{fact_names, known_names}; use super::publish::publish; use super::store::{Cell, Run}; +use super::timing; use super::{FlowRunner, RESCUE_TIMEOUT_MS}; -use crate::rescue::{Briefing, Guidance, MAX_RESCUES, resumed}; +use crate::rescue::{Briefing, Guidance, MAX_RESCUES, Rescuer, resumed}; /// A recoverable failure of a top-level step is first rescued: `Err` holds /// the run the guidance makes, to run next. Anything else, or a rescue that @@ -159,13 +160,9 @@ pub(super) async fn rescue( failed + 1 ), ); - let started = Instant::now(); let wait = time_left.map_or(RESCUE_TIMEOUT_MS, |left| left.min(RESCUE_TIMEOUT_MS)); - let answer = tokio::time::timeout(Duration::from_millis(wait), rescuer.guide(&briefing)) - .await - .unwrap_or_else(|_| Err("the rescuer took too long".to_owned())); - let spent_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX); - let (record, guided) = record(failed, briefing.failure.clone(), answer); + let (record, guided, spent_ms) = + ask(runner, &id, &rescuer, &briefing, wait, (attempt, limit)).await; let guided = guided.map(|steps| resumed(&briefing, steps, record.covers)); let reason = facts.redact(&record.reason); let index = { @@ -191,6 +188,45 @@ pub(super) async fn rescue( }) } +/// Asks `rescuer` about `briefing`, waiting at most `wait` ms, journals how +/// it went as rescue `attempt` of `limit`, and records it: the record, the +/// guidance's steps when it gave any, and the milliseconds asking took. +async fn ask( + runner: &dyn FlowRunner, + id: &TaskId, + rescuer: &Rescuer, + briefing: &Briefing, + wait: u64, + (attempt, limit): (usize, u32), +) -> (Rescue, Option>, u64) { + let started = Instant::now(); + let (answer, used) = match tokio::time::timeout( + Duration::from_millis(wait), + rescuer.guide_measured(briefing), + ) + .await + { + Ok((answer, used)) => (answer, Some(used)), + Err(_) => (Err("the rescuer took too long".to_owned()), None), + }; + let took = started.elapsed(); + let outcome = timing::answered(&answer, used.is_none()); + let (record, guided) = record(briefing.failed, briefing.failure.clone(), answer); + runner.journal(Some(id), "rescue", &|| { + timing::rescued(&timing::Rescued { + attempt, + limit, + took, + used, + outcome, + record: &record, + model: rescuer.configuration(), + }) + }); + let spent_ms = u64::try_from(took.as_millis()).unwrap_or(u64::MAX); + (record, guided, spent_ms) +} + /// The record of a rescue of step `failed`, and the guidance's steps when /// it gave any. pub(super) fn record( diff --git a/crates/tinycomputer-engine/src/task/store.rs b/crates/tinycomputer-engine/src/task/store.rs index 458cadfa..72a83412 100644 --- a/crates/tinycomputer-engine/src/task/store.rs +++ b/crates/tinycomputer-engine/src/task/store.rs @@ -59,6 +59,9 @@ pub(super) struct State { pub(super) output: Option, /// Screenshots taken each time a run stopped, oldest first. pub(super) artifacts: Vec, + /// When the task began waiting for an answer it is still waiting for, + /// to journal how long the person took. + pub(super) waiting_since: Option, } /// A task's cumulative spend against its [`TaskBudget`], across every run. @@ -115,6 +118,7 @@ impl Tasks { rescues: Vec::new(), output: request.output.clone(), artifacts: Vec::new(), + waiting_since: None, }), worker: Mutex::new(None), rescuer: self.rescuer.clone(), diff --git a/crates/tinycomputer-engine/src/task/task_tests.rs b/crates/tinycomputer-engine/src/task/task_tests.rs index 6800b48d..85849548 100644 --- a/crates/tinycomputer-engine/src/task/task_tests.rs +++ b/crates/tinycomputer-engine/src/task/task_tests.rs @@ -13,11 +13,13 @@ mod describe_tests; mod errors_tests; mod human_tests; mod output_tests; +mod page_tests; mod plan_tests; mod rescue_tests; mod runner_tests; mod start_tests; mod status_tests; +mod timing_tests; use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::sync::{Arc, Mutex}; @@ -50,8 +52,18 @@ struct Script { shot: Mutex>, /// A capture that never answers, like a hung surface. stuck: std::sync::atomic::AtomicBool, - /// `capture` and `release` calls, in the order they arrived. + /// `capture`, `release`, `prepare` and `open_page` calls, in the order + /// they arrived. events: Mutex>, + /// What the task journaled outside its flows, in order. + journaled: Mutex, String, serde_json::Value)>>, + /// Tasks whose surfaces were got ready while they were planned. + prepared: Mutex>, + /// Tasks whose Jev connections were warmed while they were planned, + /// with the votes asked for. The warm-up never ends. + warmed: Mutex>, + /// The pages loaded while tasks were planned, with their task. + opened: Mutex>, } impl FlowRunner for Script { @@ -89,6 +101,46 @@ impl FlowRunner for Script { self.events.lock().unwrap().push("release"); self.released.lock().unwrap().push(task.clone()); } + + fn prepare(&self, task: &TaskId, _constraints: &TaskConstraints) -> super::PrepareFuture { + self.events.lock().unwrap().push("prepare"); + self.prepared.lock().unwrap().push(task.clone()); + Box::pin(async {}) + } + + fn open_page(&self, task: &TaskId, url: &str) -> super::PrepareFuture { + self.events.lock().unwrap().push("open_page"); + self.opened + .lock() + .unwrap() + .push((task.clone(), url.to_owned())); + Box::pin(async {}) + } + + fn warm(&self, task: &TaskId, votes: u32) -> super::PrepareFuture { + self.warmed.lock().unwrap().push((task.clone(), votes)); + Box::pin(std::future::pending()) + } + + fn journal(&self, task: Option<&TaskId>, event: &str, fields: &dyn Fn() -> serde_json::Value) { + self.journaled + .lock() + .unwrap() + .push((task.cloned(), event.to_owned(), fields())); + } +} + +/// The `event`s the task journaled outside its flows, with the task each +/// went to. +fn journaled(script: &Script, event: &str) -> Vec<(Option, serde_json::Value)> { + script + .journaled + .lock() + .unwrap() + .iter() + .filter(|(_, kind, _)| kind == event) + .map(|(task, _, fields)| (task.clone(), fields.clone())) + .collect() } fn controller(replies: Vec) -> (Tasks, Arc