From 50dc081512e03a2e607f4007a8aa99b12fda603f Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 7 Oct 2026 14:10:36 +0530 Subject: [PATCH 01/44] Journal a task's whole time and split it in jev_journal A task's journal covered only its flows: the planner's call before the first flow, each rescue between flows, and a person's answer to a pause left gaps that read as unexplained time (38% of a live batch). The task controller now times them and hands plan, rescue, and resume events to a new FlowRunner::journal hook; the module's runner writes them into the task's file, or for PlanTask, which plans before a task exists, into a run of its own. Planner::plan_measured and Rescuer::guide_measured count the model calls each took, repairs of refused answers included. jev_journal gains --split, which reads whole tasks by their timestamps (each run restarts elapsed_ms) and splits wall time into planning, rescues, waits, and flows (Jev, settling, acting, reading), with per-call and per-decision latency and what the slowest call adds to each round; and --compare, the medians of two sets of tasks side by side. No change to what a task does: the hook does nothing by default, and nothing when the journal is off. --- .../src/agentic/journal/README.md | 7 +- .../src/agentic/journal/journal_tests.rs | 21 + .../src/agentic/journal/mod.rs | 11 + .../src/agentic/runtime.rs | 15 + crates/tinycomputer-engine/src/lib.rs | 2 +- crates/tinycomputer-engine/src/planner/mod.rs | 49 +- .../src/planner/planner_tests.rs | 32 +- crates/tinycomputer-engine/src/rescue/mod.rs | 25 +- .../src/rescue/rescue_tests.rs | 26 + .../src/task/controller.rs | 28 +- crates/tinycomputer-engine/src/task/drive.rs | 12 +- crates/tinycomputer-engine/src/task/mod.rs | 8 + .../tinycomputer-engine/src/task/publish.rs | 17 +- .../tinycomputer-engine/src/task/recovery.rs | 54 +- crates/tinycomputer-engine/src/task/store.rs | 4 + .../src/task/task_tests.rs | 23 + .../src/task/task_tests/human_tests.rs | 4 + .../src/task/task_tests/plan_tests.rs | 28 +- .../src/task/task_tests/rescue_tests.rs | 16 +- .../src/task/task_tests/timing_tests.rs | 76 +++ crates/tinycomputer-engine/src/task/timing.rs | 94 +++ .../src/bin/jev_journal.rs | 89 ++- .../src/journal/journal_tests.rs | 2 + .../src/journal/journal_tests/split_tests.rs | 188 ++++++ .../tinycomputer-examples/src/journal/mod.rs | 6 +- .../tinycomputer-examples/src/journal/runs.rs | 24 + .../src/journal/split.rs | 537 ++++++++++++++++++ .../tinycomputer/src/tinybus_module/runner.rs | 13 + .../tinybus_module_tests/tasks_tests.rs | 48 ++ docs/technical/jev-journal.md | 46 +- 30 files changed, 1466 insertions(+), 39 deletions(-) create mode 100644 crates/tinycomputer-engine/src/task/task_tests/timing_tests.rs create mode 100644 crates/tinycomputer-engine/src/task/timing.rs create mode 100644 crates/tinycomputer-examples/src/journal/journal_tests/split_tests.rs create mode 100644 crates/tinycomputer-examples/src/journal/split.rs diff --git a/crates/tinycomputer-engine/src/agentic/journal/README.md b/crates/tinycomputer-engine/src/agentic/journal/README.md index 40721e6a..e1702b9b 100644 --- a/crates/tinycomputer-engine/src/agentic/journal/README.md +++ b/crates/tinycomputer-engine/src/agentic/journal/README.md @@ -24,11 +24,16 @@ event table, reading a run with `jev_journal`, finding latency — is the journal on or off. - **After masking.** The flow journals the request after `FlowRun::mask`, so secrets appear only as `${name}`. +- **A task's whole time.** The task controller times what happens between + its flows — planning, each rescue, each wait for a person — and hands the + event to its `FlowRunner::journal`; the module's runner writes it with + `JevRuntime::journal_event` into the task's `task-…` file, or for + `PlanTask`, which plans before a task exists, into a run of its own. ## Public surface `JevRuntime::with_journal`, `JevRuntime::journaled_as`, -`JevRuntime::journal_dir`, and the constants `JOURNAL_ENV`, +`JevRuntime::journal_dir`, `JevRuntime::journal_event`, and the constants `JOURNAL_ENV`, `JOURNAL_DEFAULT_DIR`, and `JOURNAL_FILE`, all re-exported from the crate root. The reader lives in `tinycomputer-examples` (`src/journal/`, the `jev_journal` binary), since only developers read journals. diff --git a/crates/tinycomputer-engine/src/agentic/journal/journal_tests.rs b/crates/tinycomputer-engine/src/agentic/journal/journal_tests.rs index 44850413..09b4b7e5 100644 --- a/crates/tinycomputer-engine/src/agentic/journal/journal_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/journal/journal_tests.rs @@ -220,3 +220,24 @@ fn millis_saturate() { assert_eq!(millis(Duration::from_micros(2500)), 2); assert_eq!(millis(Duration::MAX), u64::MAX); } + +#[test] +fn a_fresh_run_is_named_for_its_kind_and_opened_only_when_the_journal_is_on() { + let scratch = Scratch::new("fresh"); + let root = Journal::at(&scratch.0); + assert!(!root.is_open(), "a root is no run"); + let plan = root.fresh("plan"); + assert!(plan.is_open()); + let dir = plan.run_dir().unwrap(); + assert!( + dir.file_name() + .unwrap() + .to_string_lossy() + .contains("Z-plan-"), + "{}", + dir.display() + ); + plan.record("plan", || json!({"wall_ms": 12})); + assert_eq!(events(&dir)[0]["wall_ms"], 12); + assert!(!Journal::default().fresh("plan").is_open(), "off stays off"); +} diff --git a/crates/tinycomputer-engine/src/agentic/journal/mod.rs b/crates/tinycomputer-engine/src/agentic/journal/mod.rs index 7fd649ff..4ac4e5ba 100644 --- a/crates/tinycomputer-engine/src/agentic/journal/mod.rs +++ b/crates/tinycomputer-engine/src/agentic/journal/mod.rs @@ -128,6 +128,17 @@ impl Journal { } } + /// Whether a run is open to write to. + pub(crate) fn is_open(&self) -> bool { + self.run.is_some() + } + + /// This journal, writing to a new run named for the time and `kind`. A + /// journal that is off stays off. + pub(crate) fn fresh(&self, kind: &str) -> Self { + self.named(&fresh_id(kind)) + } + /// This journal with a run begun: a `run` event in the current run, or, /// when there is none yet, in a new one named for the time and `kind`. pub(crate) fn begin(&self, kind: &str, label: &str, model: &str) -> Self { diff --git a/crates/tinycomputer-engine/src/agentic/runtime.rs b/crates/tinycomputer-engine/src/agentic/runtime.rs index efa3284b..ae0220c6 100644 --- a/crates/tinycomputer-engine/src/agentic/runtime.rs +++ b/crates/tinycomputer-engine/src/agentic/runtime.rs @@ -166,6 +166,21 @@ impl JevRuntime { } } + /// Writes one event of `kind` with `fields` to the debug journal: into + /// this runtime's open run (see [`JevRuntime::journaled_as`]), or, with + /// none open, into a new run named for the time and `kind`. It is for + /// time a task spends outside its flows, such as planning, a rescue, or + /// waiting on a person, so the task's journal accounts for all of its + /// time. Does nothing when the journal is off. + pub fn journal_event(&self, kind: &str, fields: serde_json::Value) { + let journal = if self.journal.is_open() { + self.journal.clone() + } else { + self.journal.fresh(kind) + }; + journal.record(kind, || fields); + } + /// The directory this runtime's current run journal is written to, if /// the journal is on and a run has begun. #[must_use] diff --git a/crates/tinycomputer-engine/src/lib.rs b/crates/tinycomputer-engine/src/lib.rs index 48b57f79..1a76320d 100644 --- a/crates/tinycomputer-engine/src/lib.rs +++ b/crates/tinycomputer-engine/src/lib.rs @@ -41,7 +41,7 @@ pub use agentic::{ JOURNAL_DEFAULT_DIR, JOURNAL_ENV, JOURNAL_FILE, JevRuntime, flow_guide, resolve_intent, run_flow, run_goal, validate_flow, }; -pub use planner::{Completion, LanguageModel, Planner, REPAIRS, Role, Turn}; +pub use planner::{Completion, LanguageModel, ModelUse, Planner, REPAIRS, Role, Turn}; #[cfg(feature = "planner")] pub use planner::{ ModelRoute, OPEN_ROUTER_BASE_URL, OUTPUT_MODEL, PLANNER_MODEL, PlannerConfig, RESCUE_MODEL, diff --git a/crates/tinycomputer-engine/src/planner/mod.rs b/crates/tinycomputer-engine/src/planner/mod.rs index 8f5655f6..00f47577 100644 --- a/crates/tinycomputer-engine/src/planner/mod.rs +++ b/crates/tinycomputer-engine/src/planner/mod.rs @@ -50,6 +50,27 @@ pub enum Role { Assistant, } +/// What one plan or rescue used of its model: the calls it made, the first +/// and each repair of a refused answer, and the bytes the first call sent. +/// The task journals it, so a run shows what its planning and rescues cost. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct ModelUse { + /// Calls made: the first, and one per repair. + pub calls: u32, + /// Bytes of text in the first call's turns. + pub sent_bytes: usize, +} + +impl ModelUse { + /// The use of a conversation about to be sent for the first time. + pub(crate) fn starting(turns: &[Turn]) -> Self { + Self { + calls: 0, + sent_bytes: turns.iter().map(|turn| turn.text.len()).sum(), + } + } +} + /// One message in a planning conversation. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Turn { @@ -152,6 +173,20 @@ impl Planner { secret_names: &[String], surfaces: &[SurfaceKind], ) -> Result { + self.plan_measured(task, fact_names, secret_names, surfaces) + .await + .0 + } + + /// [`Planner::plan`], with what it used of its model, whether or not a + /// plan came back. + pub async fn plan_measured( + &self, + task: &str, + fact_names: &[String], + secret_names: &[String], + surfaces: &[SurfaceKind], + ) -> (Result, ModelUse) { let surfaces = if surfaces.is_empty() { "the web browser and desktop applications".to_owned() } else { @@ -200,9 +235,14 @@ impl Planner { ), ), ]; + let mut used = ModelUse::starting(&turns); let mut last = String::new(); for _ in 0..=REPAIRS { - let reply = self.model.complete(&turns).await?; + used.calls += 1; + let reply = match self.model.complete(&turns).await { + Ok(reply) => reply, + Err(error) => return (Err(error), used), + }; turns.push(Turn::new(Role::Assistant, reply.clone())); let problem = match parse(&reply) { Ok(flow) => { @@ -212,7 +252,7 @@ impl Planner { .filter(|error| !error.contains("` is not defined in `vars`")) .collect::>(); if errors.is_empty() { - return Ok(plan_for(flow, &known, &secrets)); + return (Ok(plan_for(flow, &known, &secrets)), used); } format!("That flow is invalid:\n- {}", errors.join("\n- ")) } @@ -224,7 +264,10 @@ impl Planner { format!("{problem}\nReply with the corrected flow only, as one JSON object."), )); } - Err(format!("the planner did not produce a valid flow: {last}")) + ( + Err(format!("the planner did not produce a valid flow: {last}")), + used, + ) } } diff --git a/crates/tinycomputer-engine/src/planner/planner_tests.rs b/crates/tinycomputer-engine/src/planner/planner_tests.rs index 5bd1b70d..a6d4e1ec 100644 --- a/crates/tinycomputer-engine/src/planner/planner_tests.rs +++ b/crates/tinycomputer-engine/src/planner/planner_tests.rs @@ -7,7 +7,7 @@ use std::sync::{Arc, Mutex}; use tinycomputer_bus::agent::{InputKind, SurfaceKind}; -use super::{Completion, LanguageModel, Planner, REPAIRS, Role, Turn}; +use super::{Completion, LanguageModel, ModelUse, Planner, REPAIRS, Role, Turn}; /// Answers from a queue and records every conversation it was shown. #[derive(Default)] @@ -173,6 +173,36 @@ async fn a_plan_that_never_validates_or_a_failed_model_is_an_error() { assert!(planner.plan("x", &[], &[], &[]).await.is_err()); } +#[tokio::test] +async fn a_measured_plan_counts_each_call_and_what_the_first_one_sent() { + let (planner, model) = scripted(&[ + Ok("I would search for flights."), + Ok(r#"{"app": "Mail", "steps": ["start a new email message"]}"#), + ]); + let (plan, used) = planner.plan_measured("write an email", &[], &[], &[]).await; + assert_eq!(plan.unwrap().flow.app, "Mail"); + let first = model.seen.lock().unwrap()[0].clone(); + assert_eq!( + used, + ModelUse { + calls: 2, + sent_bytes: first.iter().map(|turn| turn.text.len()).sum(), + }, + "one repair after the refused answer" + ); + + let (planner, _) = scripted(&[Ok("{"), Err("rate limited")]); + let (plan, used) = planner.plan_measured("x", &[], &[], &[]).await; + assert_eq!(plan.unwrap_err(), "rate limited"); + assert_eq!(used.calls, 2, "a failed call still counts"); + + let never = [Ok(r#"{"app": "", "steps": []}"#); REPAIRS + 1]; + let (planner, _) = scripted(&never); + let (plan, used) = planner.plan_measured("x", &[], &[], &[]).await; + assert!(plan.is_err()); + assert_eq!(used.calls, u32::try_from(REPAIRS).unwrap() + 1); +} + #[cfg(feature = "planner")] #[tokio::test] async fn the_open_router_planner_needs_a_key_and_never_prints_it() { diff --git a/crates/tinycomputer-engine/src/rescue/mod.rs b/crates/tinycomputer-engine/src/rescue/mod.rs index 4ffc382d..02e81fbb 100644 --- a/crates/tinycomputer-engine/src/rescue/mod.rs +++ b/crates/tinycomputer-engine/src/rescue/mod.rs @@ -27,7 +27,7 @@ use std::sync::Arc; use tinycomputer_bus::agent::{LanguageModelConfiguration, Rescue}; use tinycomputer_bus::{FLOW_GUIDE, Flow, FlowStep, StepReport}; -use crate::planner::{LanguageModel, REPAIRS, Role, Turn}; +use crate::planner::{LanguageModel, ModelUse, REPAIRS, Role, Turn}; use judge::judge; pub(crate) use judge::resumed; use render::render; @@ -181,16 +181,30 @@ impl Rescuer { /// Why no guidance came back: the model failed, or its answer stayed /// invalid after [`REPAIRS`] repairs. pub async fn guide(&self, briefing: &Briefing) -> Result { + self.guide_measured(briefing).await.0 + } + + /// [`Rescuer::guide`], with what it used of its model, whether or not + /// guidance came back: a call refused as invalid guidance costs a repair. + pub async fn guide_measured( + &self, + briefing: &Briefing, + ) -> (Result, ModelUse) { let mut turns = vec![ Turn::new(Role::System, format!("{PROTOCOL}\n\n{FLOW_GUIDE}")), Turn::new(Role::User, render(briefing)), ]; + let mut used = ModelUse::starting(&turns); let mut last = String::new(); for _ in 0..=REPAIRS { - let reply = self.model.complete(&turns).await?; + used.calls += 1; + let reply = match self.model.complete(&turns).await { + Ok(reply) => reply, + Err(error) => return (Err(error), used), + }; turns.push(Turn::new(Role::Assistant, reply.clone())); let problem = match judge(&reply, briefing) { - Ok(guidance) => return Ok(guidance), + Ok(guidance) => return (Ok(guidance), used), Err(problem) => problem, }; last.clone_from(&problem); @@ -199,7 +213,10 @@ impl Rescuer { format!("{problem}\nReply with the corrected answer only, as one JSON object."), )); } - Err(format!("the rescuer gave no valid guidance: {last}")) + ( + Err(format!("the rescuer gave no valid guidance: {last}")), + used, + ) } } diff --git a/crates/tinycomputer-engine/src/rescue/rescue_tests.rs b/crates/tinycomputer-engine/src/rescue/rescue_tests.rs index 2ed52ee4..e04db5aa 100644 --- a/crates/tinycomputer-engine/src/rescue/rescue_tests.rs +++ b/crates/tinycomputer-engine/src/rescue/rescue_tests.rs @@ -195,6 +195,32 @@ async fn invalid_guidance_is_sent_back_with_what_is_wrong() { ); } +#[tokio::test] +async fn a_measured_rescue_counts_each_call_refused_guidance_included() { + let unknown = + r#"{"action": "retry", "reason": "x", "steps": [{"enter": {"name": "${full name}"}}]}"#; + let (rescuer, model) = scripted(&[Ok(unknown), Ok(FIX)]); + let (guidance, used) = rescuer.guide_measured(&briefing()).await; + assert!(matches!(guidance, Ok(Guidance::Retry { .. }))); + assert_eq!(used.calls, 2, "the refused guidance cost a repair"); + let first = model.seen.lock().unwrap()[0].clone(); + assert_eq!( + used.sent_bytes, + first.iter().map(|turn| turn.text.len()).sum::() + ); + + let (rescuer, _) = scripted(&[Err("the model is down")]); + let (guidance, used) = rescuer.guide_measured(&briefing()).await; + assert_eq!(guidance.unwrap_err(), "the model is down"); + assert_eq!(used.calls, 1); + + let never = [Ok("nonsense"); REPAIRS + 1]; + let (rescuer, _) = scripted(&never); + let (guidance, used) = rescuer.guide_measured(&briefing()).await; + assert!(guidance.is_err()); + assert_eq!(used.calls, u32::try_from(REPAIRS).unwrap() + 1); +} + #[test] fn the_briefing_shows_earlier_rescues_and_cuts_a_long_screen() { let mut briefing = briefing(); diff --git a/crates/tinycomputer-engine/src/task/controller.rs b/crates/tinycomputer-engine/src/task/controller.rs index f9ed1381..4a0cb613 100644 --- a/crates/tinycomputer-engine/src/task/controller.rs +++ b/crates/tinycomputer-engine/src/task/controller.rs @@ -116,15 +116,22 @@ impl Tasks { let Some(planner) = &self.planner else { return AgentResponse::err(no_planner()); }; - match planner - .plan( + let started = std::time::Instant::now(); + let (outcome, used) = planner + .plan_measured( &request.task, &request.fact_names, &request.secret_facts, &request.surfaces, ) - .await - { + .await; + // No task exists yet, so the plan journals to a run of its own. + self.runner.journal( + None, + "plan", + super::timing::planned(&outcome, used, started.elapsed(), planner.configuration()), + ); + match outcome { Ok(plan) => AgentResponse::ok(plan), Err(reason) => AgentResponse::err(AgentError::new( "PLAN_FAILED", @@ -261,6 +268,19 @@ impl Tasks { return no_such_task(&request.id); }; let status = cell.view.borrow().status.clone(); + let waited = cell + .state + .lock() + .ok() + .and_then(|state| state.waiting_since) + .map(|since| since.elapsed()); + if let Some(waited) = waited { + self.runner.journal( + Some(&request.id), + "resume", + super::timing::resumed(state_name(&status), waited), + ); + } match status { TaskStatus::NeedsInput { .. } => self.supply(&cell, request), TaskStatus::NeedsApproval { .. } => self.decide(&cell, request.approve), diff --git a/crates/tinycomputer-engine/src/task/drive.rs b/crates/tinycomputer-engine/src/task/drive.rs index fa61e10d..c20af1b8 100644 --- a/crates/tinycomputer-engine/src/task/drive.rs +++ b/crates/tinycomputer-engine/src/task/drive.rs @@ -35,7 +35,17 @@ pub(super) async fn plan_then_drive( ) }, ); - let plan = match planner.plan(&task, &names, &secrets, &surfaces).await { + let started = Instant::now(); + let (outcome, used) = planner + .plan_measured(&task, &names, &secrets, &surfaces) + .await; + let id = cell.view.borrow().id.clone(); + runner.journal( + Some(&id), + "plan", + super::timing::planned(&outcome, used, started.elapsed(), planner.configuration()), + ); + let plan = match outcome { Ok(plan) => plan, Err(reason) => { publish( diff --git a/crates/tinycomputer-engine/src/task/mod.rs b/crates/tinycomputer-engine/src/task/mod.rs index fe15bdae..e9bb0d52 100644 --- a/crates/tinycomputer-engine/src/task/mod.rs +++ b/crates/tinycomputer-engine/src/task/mod.rs @@ -50,6 +50,7 @@ mod publish; mod recovery; mod resume; mod store; +mod timing; use std::future::Future; use std::pin::Pin; @@ -106,6 +107,13 @@ pub trait FlowRunner: Send + Sync + 'static { /// Lets go of whatever the task held, once it has ended. fn release(&self, _task: &TaskId) {} + + /// Writes an `event` of the time a task spends outside its flows + /// (`plan`, `rescue`, `resume`) to the debug journal: the task's own, + /// or for `PlanTask`, which plans before any task exists (`task` is + /// `None`), a run of its own. Does nothing by default, and nothing when + /// the journal is off. + fn journal(&self, _task: Option<&TaskId>, _event: &str, _fields: serde_json::Value) {} } /// How many tasks the controller holds; finished ones are dropped first. diff --git a/crates/tinycomputer-engine/src/task/publish.rs b/crates/tinycomputer-engine/src/task/publish.rs index 6c513929..d0580dff 100644 --- a/crates/tinycomputer-engine/src/task/publish.rs +++ b/crates/tinycomputer-engine/src/task/publish.rs @@ -25,7 +25,22 @@ pub(super) fn stopped_summary(status: &TaskStatus) -> String { /// Updates a task's view: status, summary, progress, step, and next calls. pub(super) fn publish(cell: &Cell, status: TaskStatus, summary: &str) { - let (progress, step) = cell.state.lock().map_or((0.0, None), |state| { + let waits = matches!( + status, + TaskStatus::NeedsInput { .. } + | TaskStatus::NeedsApproval { .. } + | TaskStatus::NeedsHuman { .. } + ); + let (progress, step) = cell.state.lock().map_or((0.0, None), |mut state| { + // The wait starts when the task first asks; publishing the same + // pause again does not restart it. + state.waiting_since = if waits { + state + .waiting_since + .or_else(|| Some(std::time::Instant::now())) + } else { + None + }; let total = state.flow.steps.len().max(1); let fraction = |count: usize| f32::from(u16::try_from(count).unwrap_or(u16::MAX)); let progress = fraction(state.finished.min(total)) / fraction(total); diff --git a/crates/tinycomputer-engine/src/task/recovery.rs b/crates/tinycomputer-engine/src/task/recovery.rs index 1b61adcd..7002297a 100644 --- a/crates/tinycomputer-engine/src/task/recovery.rs +++ b/crates/tinycomputer-engine/src/task/recovery.rs @@ -4,7 +4,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::time::{Duration, Instant}; -use tinycomputer_bus::agent::{Rescue, RescueOutcome, TaskStatus}; +use tinycomputer_bus::agent::{Rescue, RescueOutcome, TaskId, TaskStatus}; use tinycomputer_bus::{Flow, FlowStep, StepOutcome, StepReport}; use tinycomputer_core::Facts; @@ -12,8 +12,9 @@ use super::brief::brief; use super::names::{fact_names, known_names}; use super::publish::publish; use super::store::{Cell, Run}; +use super::timing; use super::{FlowRunner, RESCUE_TIMEOUT_MS}; -use crate::rescue::{Briefing, Guidance, MAX_RESCUES, resumed}; +use crate::rescue::{Briefing, Guidance, MAX_RESCUES, Rescuer, resumed}; /// A recoverable failure of a top-level step is first rescued: `Err` holds /// the run the guidance makes, to run next. Anything else, or a rescue that @@ -159,13 +160,9 @@ pub(super) async fn rescue( failed + 1 ), ); - let started = Instant::now(); let wait = time_left.map_or(RESCUE_TIMEOUT_MS, |left| left.min(RESCUE_TIMEOUT_MS)); - let answer = tokio::time::timeout(Duration::from_millis(wait), rescuer.guide(&briefing)) - .await - .unwrap_or_else(|_| Err("the rescuer took too long".to_owned())); - let spent_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX); - let (record, guided) = record(failed, briefing.failure.clone(), answer); + let (record, guided, spent_ms) = + ask(runner, &id, &rescuer, &briefing, wait, (attempt, limit)).await; let guided = guided.map(|steps| resumed(&briefing, steps, record.covers)); let reason = facts.redact(&record.reason); let index = { @@ -191,6 +188,47 @@ pub(super) async fn rescue( }) } +/// Asks `rescuer` about `briefing`, waiting at most `wait` ms, journals how +/// it went as rescue `attempt` of `limit`, and records it: the record, the +/// guidance's steps when it gave any, and the milliseconds asking took. +async fn ask( + runner: &dyn FlowRunner, + id: &TaskId, + rescuer: &Rescuer, + briefing: &Briefing, + wait: u64, + (attempt, limit): (usize, u32), +) -> (Rescue, Option>, u64) { + let started = Instant::now(); + let (answer, used) = match tokio::time::timeout( + Duration::from_millis(wait), + rescuer.guide_measured(briefing), + ) + .await + { + Ok((answer, used)) => (answer, Some(used)), + Err(_) => (Err("the rescuer took too long".to_owned()), None), + }; + let took = started.elapsed(); + let outcome = timing::answered(&answer, used.is_none()); + let (record, guided) = record(briefing.failed, briefing.failure.clone(), answer); + runner.journal( + Some(id), + "rescue", + timing::rescued(&timing::Rescued { + attempt, + limit, + took, + used, + outcome, + record: &record, + model: rescuer.configuration(), + }), + ); + let spent_ms = u64::try_from(took.as_millis()).unwrap_or(u64::MAX); + (record, guided, spent_ms) +} + /// The record of a rescue of step `failed`, and the guidance's steps when /// it gave any. pub(super) fn record( diff --git a/crates/tinycomputer-engine/src/task/store.rs b/crates/tinycomputer-engine/src/task/store.rs index 458cadfa..72a83412 100644 --- a/crates/tinycomputer-engine/src/task/store.rs +++ b/crates/tinycomputer-engine/src/task/store.rs @@ -59,6 +59,9 @@ pub(super) struct State { pub(super) output: Option, /// Screenshots taken each time a run stopped, oldest first. pub(super) artifacts: Vec, + /// When the task began waiting for an answer it is still waiting for, + /// to journal how long the person took. + pub(super) waiting_since: Option, } /// A task's cumulative spend against its [`TaskBudget`], across every run. @@ -115,6 +118,7 @@ impl Tasks { rescues: Vec::new(), output: request.output.clone(), artifacts: Vec::new(), + waiting_since: None, }), worker: Mutex::new(None), rescuer: self.rescuer.clone(), diff --git a/crates/tinycomputer-engine/src/task/task_tests.rs b/crates/tinycomputer-engine/src/task/task_tests.rs index 6800b48d..bc7bd09e 100644 --- a/crates/tinycomputer-engine/src/task/task_tests.rs +++ b/crates/tinycomputer-engine/src/task/task_tests.rs @@ -18,6 +18,7 @@ mod rescue_tests; mod runner_tests; mod start_tests; mod status_tests; +mod timing_tests; use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::sync::{Arc, Mutex}; @@ -52,6 +53,8 @@ struct Script { stuck: std::sync::atomic::AtomicBool, /// `capture` and `release` calls, in the order they arrived. events: Mutex>, + /// What the task journaled outside its flows, in order. + journaled: Mutex, String, serde_json::Value)>>, } impl FlowRunner for Script { @@ -89,6 +92,26 @@ impl FlowRunner for Script { self.events.lock().unwrap().push("release"); self.released.lock().unwrap().push(task.clone()); } + + fn journal(&self, task: Option<&TaskId>, event: &str, fields: serde_json::Value) { + self.journaled + .lock() + .unwrap() + .push((task.cloned(), event.to_owned(), fields)); + } +} + +/// The `event`s the task journaled outside its flows, with the task each +/// went to. +fn journaled(script: &Script, event: &str) -> Vec<(Option, serde_json::Value)> { + script + .journaled + .lock() + .unwrap() + .iter() + .filter(|(_, kind, _)| kind == event) + .map(|(task, _, fields)| (task.clone(), fields.clone())) + .collect() } fn controller(replies: Vec) -> (Tasks, Arc"# + ) + }; + for (host, banner, unreachable) in [ + ("display: contents", "", 1), + ("display: block", "", 1), + ("display: contents", "display: none", 0), + ] { + let Some(reading) = live_reading(&page(host, banner)).await else { + return; + }; + assert_eq!( + reading["unreachable"], unreachable, + "host {host:?}, banner {banner:?}: {reading}" + ); + } + // The tree the surface reads instead offers the banner's buttons. + let tree = live_tree(&page("display: contents", "")) + .await + .expect("a live run reads the tree too"); + for button in ["Allow Selection", "Allow all", "Add To Cart"] { + assert!(tree.contains(button), "{button} in {tree}"); + } +} + #[cfg(feature = "agent-browser")] #[tokio::test] async fn live_hidden_elements_are_dropped() { diff --git a/docs/crates/tinycomputer-browser/sight.md b/docs/crates/tinycomputer-browser/sight.md index 03dd017a..7553c852 100644 --- a/docs/crates/tinycomputer-browser/sight.md +++ b/docs/crates/tinycomputer-browser/sight.md @@ -248,7 +248,9 @@ Sight gives way to the accessibility tree (`tree.rs`, see below) when the reading fails outright, or when it sees a control inside a shadow root, or behind a frame that covers a large share of the viewport, both cases where a plain CSS selector from the top-level page cannot address the element -sight found. Reading through the tree in those cases is deliberately +sight found. A shadow root's host need not draw a box of its own: one laid +out as `display: contents` has none, and a consent banner's host was one, +so a shadow root counts once its host or any of its controls shows. Reading through the tree in those cases is deliberately unglamorous: it is the same fallback the crate always had, just demoted from "the only way" to "the way out when sight cannot help." From ee4567641ac826a0e640cd2683b4e19eaed443c2 Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 7 Oct 2026 17:20:01 +0530 Subject: [PATCH 07/44] Wait for a place box's late suggestions only while the page changes After text goes into a place box (an address, a city, a pickup), the flow looked twice more for suggestions the page lists late, each look a fixed 500 ms pause plus a full settle. A plain form box lists none, so every address and city field on BlazeDemo's passenger form paid 2.3 s (4.2 s with steady settling) for a list that never comes, four waits a run. Surface::await_change waits for the surface to change by itself and says whether it did. The browser watches the page with a MutationObserver that ends at the first change of its own (never sight's data-tc- marks), or after LATE_LOOK_MS = 1000 ms with none; a surface that cannot watch, the desktop's, pauses as a Wait did and says it may have changed. The flow looks again after each wait, as before, but a wait that saw the page stay still is not settled, notes "nothing changed", and ends the looking: a still page lists nothing more. Rows that arrive late are looked at as soon as they are drawn. The simulated ride form now draws its rows a number of waits late; the new flow tests pick rows that come one and two waits late, and wait once, not twice, beside a box that lists nothing. --- .../src/surface/operations.rs | 30 +++++++++ .../surface/surface_tests/operations_tests.rs | 45 +++++++++++++ crates/tinycomputer-core/src/surface/mod.rs | 12 ++++ .../surface/surface_tests/delivery_tests.rs | 4 ++ .../src/agentic/flow/action.rs | 35 +++++++++- .../src/agentic/flow/flow_tests/places.rs | 20 +++++- .../src/agentic/flow/flow_tests/simulator.rs | 4 ++ .../flow/flow_tests/suggestion_tests.rs | 64 +++++++++++++++++++ .../src/agentic/flow/steps/suggestion.rs | 20 ++++-- .../tinycomputer-engine/src/workspace/mod.rs | 8 +++ .../src/workspace/workspace_tests.rs | 23 +++++++ .../tinycomputer-browser/interacting.md | 8 +++ .../tinycomputer-core/surfaces-and-screens.md | 9 +++ docs/crates/tinycomputer-engine/workspace.md | 2 +- docs/technical/architecture.md | 1 + docs/technical/decision-thresholds.md | 3 +- docs/technical/jev-journal.md | 2 +- 17 files changed, 278 insertions(+), 12 deletions(-) diff --git a/crates/tinycomputer-browser/src/surface/operations.rs b/crates/tinycomputer-browser/src/surface/operations.rs index 50b1408b..c57de5fd 100644 --- a/crates/tinycomputer-browser/src/surface/operations.rs +++ b/crates/tinycomputer-browser/src/surface/operations.rs @@ -258,6 +258,19 @@ impl Surface for BrowserSurface { let _settled = self.perform("wait", pause(SETTLE_MS)); } + fn await_change(&self, ms: u64) -> bool { + let Ok(id) = self.ensure_session() else { + return true; + }; + self.block(self.browser.command( + &id, + json!({"action": "evaluate", "script": change_script(ms)}), + )) + .ok() + .and_then(|data| data.get("result").and_then(Value::as_bool)) + .unwrap_or(true) + } + fn navigate(&self, url: &str) -> DesktopResponse { let page = self .ensure_session() @@ -342,6 +355,23 @@ pub(crate) fn browser_key(combo: &str, platform: Platform) -> String { .join("+") } +/// A promise that resolves `true` at the page's first change of its own — +/// an element or words added, removed, or rewritten, or an element's look +/// changed, but never a `data-tc-` mark sight leaves — or `false` once `ms` +/// pass with none: a list a box fetches for the text typed shows as soon as +/// it is drawn, and a still page costs `ms` once. +fn change_script(ms: u64) -> String { + format!( + r"new Promise(resolve => {{ + const pages = record => record.type !== 'attributes' || !String(record.attributeName).startsWith('data-tc-'); + const watcher = new MutationObserver(records => {{ if (records.some(pages)) done(true); }}); + const done = changed => {{ watcher.disconnect(); clearTimeout(cap); resolve(changed); }}; + const cap = setTimeout(() => done(false), {ms}); + watcher.observe(document, {{ subtree: true, childList: true, attributes: true, characterData: true }}); +}})" + ) +} + /// A promise that resolves once the page has gone [`STILL_MS`] without a DOM /// change, has no finite CSS animation or transition running, and has drawn /// at least two frames, or after [`SETTLE_MS`] at most: a banner or menu diff --git a/crates/tinycomputer-browser/src/surface/surface_tests/operations_tests.rs b/crates/tinycomputer-browser/src/surface/surface_tests/operations_tests.rs index 418a3424..e254660a 100644 --- a/crates/tinycomputer-browser/src/surface/surface_tests/operations_tests.rs +++ b/crates/tinycomputer-browser/src/surface/surface_tests/operations_tests.rs @@ -580,6 +580,51 @@ fn a_prompt_settle_counts_quiet_from_the_start_and_waits_only_while_the_page_cha ); } +#[test] +fn a_wait_for_a_change_ends_at_the_pages_first_change_or_its_time() { + let Harness { fake, surface, .. } = harness("await-change", page_fake()); + assert!(surface.await_change(1_000), "the page changed"); + let watch = fake.last("evaluate"); + let script = watch["script"].as_str().unwrap(); + assert!(script.contains("MutationObserver"), "{script}"); + assert!( + script.contains("done(false), 1000"), + "still once the time given passes: {script}" + ); + assert!( + script.contains("'data-tc-'"), + "sight's own marks are no change: {script}" + ); + assert!( + !fake.actions().iter().any(|action| action == "wait"), + "no fixed pause: {:?}", + fake.actions() + ); + + let still = harness( + "await-still", + Fake::scripted(|command| { + (command["action"] == "evaluate").then(|| ok(&json!({"result": false}))) + }), + ); + assert!(!still.surface.await_change(1_000), "the page stayed still"); + + // A watch that cannot run says the page may have changed, so a caller + // looks again as it would after a pause. + let unwatched = harness("await-unwatched", Fake::new()); + assert!( + unwatched.surface.await_change(1_000), + "no answer of its own" + ); + let closed = harness( + "await-closed", + Fake::scripted(|command| { + (command["action"] == "launch").then(|| failure("Chrome not found")) + }), + ); + assert!(closed.surface.await_change(1_000), "no session to watch"); +} + #[test] fn a_surface_opens_its_session_when_asked_rather_than_at_first_use() { let Harness { fake, surface, .. } = harness("open-early", page_fake()); diff --git a/crates/tinycomputer-core/src/surface/mod.rs b/crates/tinycomputer-core/src/surface/mod.rs index 4ca1e48f..2a25f910 100644 --- a/crates/tinycomputer-core/src/surface/mod.rs +++ b/crates/tinycomputer-core/src/surface/mod.rs @@ -74,6 +74,18 @@ pub trait Surface: Clone + Send + 'static { /// address into a token. fn settle(&self) {} + /// Waits, for up to the given milliseconds, for the application to + /// change by itself — a list of suggestions a box fetches for the text + /// just typed, the rest of a page arriving — and says whether it did, so + /// a caller watching for something to appear stops once nothing moves. + /// + /// A surface that cannot watch for a change pauses as a `Wait` does, + /// however long it was given, and says it may have changed. + fn await_change(&self, _ms: u64) -> bool { + let _paused = self.execute(JevOperation::Wait, None, None); + true + } + /// Loads `url`, for a surface that has addresses. /// /// A desktop application has none, so the default refuses with diff --git a/crates/tinycomputer-core/src/surface/surface_tests/delivery_tests.rs b/crates/tinycomputer-core/src/surface/surface_tests/delivery_tests.rs index 535a3bd9..95e24dae 100644 --- a/crates/tinycomputer-core/src/surface/surface_tests/delivery_tests.rs +++ b/crates/tinycomputer-core/src/surface/surface_tests/delivery_tests.rs @@ -189,6 +189,10 @@ fn text_that_never_arrives_is_reported_as_not_delivered() { #[test] fn a_surface_settles_instantly_and_has_no_addresses_unless_it_says_otherwise() { Surface::settle(&TextBackend::default()); + assert!( + Surface::await_change(&TextBackend::default(), 1_000), + "one that cannot watch pauses and says it may have changed" + ); let refused = Surface::navigate(&TextBackend::default(), "https://example.com"); assert_eq!(refused.error.unwrap().code, "ACTION_NOT_SUPPORTED"); let refused = Surface::back(&TextBackend::default(), "Mail"); diff --git a/crates/tinycomputer-engine/src/agentic/flow/action.rs b/crates/tinycomputer-engine/src/agentic/flow/action.rs index 00578fec..5d009ce5 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/action.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/action.rs @@ -43,6 +43,7 @@ impl FlowRun<'_, B> { } let note = match (&reply.error, &reply.data) { (Some(error), _) => error.code.clone(), + (None, Some(_)) if still(&reply) => "nothing changed".to_owned(), (None, Some(data)) => data .get("path") .and_then(serde_json::Value::as_str) @@ -63,7 +64,9 @@ impl FlowRun<'_, B> { note, }); let settle_started = Instant::now(); - if reply.ok { + // A wait that saw the surface stay still has nothing to settle. + let settles = reply.ok && !still(&reply); + if settles { // Let the surface finish reacting, so the next look sees what the // action did rather than the moment before it took effect. self.backend_call(|backend| { @@ -81,12 +84,30 @@ impl FlowRun<'_, B> { "ok": reply.ok, "note": record.map(|record| record.note.as_str()), "wall_ms": acted_ms, - "settle_ms": if reply.ok { millis(settle_started.elapsed()) } else { 0 }, + "settle_ms": if settles { millis(settle_started.elapsed()) } else { 0 }, }) }); Ok(reply) } + /// Waits up to `ms` for the surface to change by itself + /// (`Surface::await_change`), as one `wait` action charged to the budget + /// and the step log: settled when the surface changed, and `false` when + /// it stayed still, so a caller watching for something to appear stops. + pub(in crate::agentic::flow) async fn await_change( + &mut self, + log: &mut StepLog, + ms: u64, + ) -> Result { + let reply = self + .act(log, "wait", None, move |backend| { + let changed = backend.await_change(ms); + DesktopResponse::ok("wait", json!({ "still": !changed })) + }) + .await?; + Ok(!still(&reply)) + } + async fn backend_call(&self, call: F) -> DesktopResponse where F: FnOnce(B) -> DesktopResponse + Send + 'static, @@ -94,3 +115,13 @@ impl FlowRun<'_, B> { blocking(self.backend.clone(), call).await } } + +/// Whether `reply` is a wait's that saw the surface stay still. +fn still(reply: &DesktopResponse) -> bool { + reply + .data + .as_ref() + .and_then(|data| data.get("still")) + .and_then(Value::as_bool) + .unwrap_or(false) +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/places.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/places.rs index 53a31431..165e3ccd 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/places.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/places.rs @@ -31,6 +31,11 @@ pub(super) struct Places { /// button, its rows unread and its name stringing them all together, as /// a store's delivery-area popover was read live. pub(super) panel: bool, + /// Waits for a change a box's list takes to show once typed into, as a + /// page that fetches its rows draws them late. + pub(super) late: u8, + /// Waits still to come before the open list shows its rows. + pub(super) pending: u8, } /// The places suggested for `typed`: each one that holds every typed word. @@ -71,7 +76,7 @@ pub(super) fn places_widget( field.value = sim.fields.get(*name).map(|value| json!(value)); candidates.push(field); } - if let Some(open) = &places.open { + if let Some(open) = places.open.as_ref().filter(|_| places.pending == 0) { let typed = sim.fields.get(open).cloned().unwrap_or_default(); let list = [root, "group \"Get a ride\"", "listbox \"Suggestions\""]; if places.panel { @@ -107,11 +112,24 @@ fn type_place(sim: &mut Sim, name: &str, text: String) { if let Some(places) = sim.places.as_mut() { places.open = Some(name.to_owned()); places.picked.remove(name); + places.pending = places.late; } sim.focused = Some(name.to_owned()); sim.fields.insert(name.to_owned(), text); } +/// A wait for the page to change: the open list draws its rows one wait +/// nearer; nothing else on the ride form changes by itself. +pub(super) fn await_place_rows(sim: &mut Sim) -> bool { + match sim.places.as_mut() { + Some(places) if places.open.is_some() && places.pending > 0 => { + places.pending -= 1; + true + } + _ => false, + } +} + /// Closes the open list, dropping its box's text unless a suggestion was /// picked for it. pub(super) fn drop_unpicked(sim: &mut Sim) { diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs index 895475f6..5695a682 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs @@ -394,6 +394,10 @@ impl AgentBackend for App { Some(self.sim().fields.get(&name).cloned().unwrap_or_default()) } + fn await_change(&self, _ms: u64) -> bool { + await_place_rows(&mut self.sim()) + } + fn paste(&self, _app: &str, target: &Candidate, text: &str) -> DesktopResponse { if is_city_row(Some(target)) { return not_a_text_field(); diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/suggestion_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/suggestion_tests.rs index e15780d3..f6c90e93 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/suggestion_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/suggestion_tests.rs @@ -85,6 +85,70 @@ async fn enter_picks_the_suggestion_an_autocomplete_box_lists_for_the_typed_text assert!(asked_for_a_suggestion(&run)); } +/// The notes of a step's waits, in order. +fn waits(run: &Run) -> Vec { + run.result.steps[0] + .actions + .iter() + .filter(|action| action.action == "wait") + .map(|action| action.note.clone()) + .collect() +} + +#[tokio::test] +async fn a_place_box_whose_rows_come_late_is_looked_at_again_as_they_show() { + // Live, a ride app's rows came after the first look. Each wait ends as + // the page changes, and the rows are picked once drawn. + for late in [1, 2] { + let run = run_with( + App::with(|sim| { + sim.places = Some(Places { + late, + ..Places::default() + }); + }), + json!({"app": "Mail", "steps": [{"enter": {"pickup location": "Connaught Place"}}]}), + |_| {}, + ride, + ) + .await; + assert_eq!( + run.result.stop, + FlowStopReason::Completed, + "{:?}", + run.result.steps + ); + assert_eq!( + run.app.sim().fields["Pickup location"], + "Connaught Place New Delhi, Delhi, India", + "rows drawn after {late} waits" + ); + assert_eq!(waits(&run), vec![String::new(); usize::from(late)]); + } +} + +#[tokio::test] +async fn a_place_box_on_a_page_that_stays_still_is_waited_on_once() { + // Live, an address and a city box on a plain form waited twice each for + // a list that never came. A page that stayed still lists nothing more. + let run = run_with( + App::with(|sim| sim.places = Some(Places::default())), + json!({"app": "Mail", "steps": [{"enter": {"pickup location": "Nowhere Lane"}}]}), + |_| {}, + ride, + ) + .await; + assert_eq!( + run.result.stop, + FlowStopReason::Completed, + "{:?}", + run.result.steps + ); + assert_eq!(run.app.sim().fields["Pickup location"], "Nowhere Lane"); + assert_eq!(waits(&run), ["nothing changed"]); + assert!(!asked_for_a_suggestion(&run)); +} + #[tokio::test] async fn a_place_box_asks_again_for_the_row_naming_its_place_in_other_words() { // An unsure pick (0.45) is not pressed as such. A place box, though, diff --git a/crates/tinycomputer-engine/src/agentic/flow/steps/suggestion.rs b/crates/tinycomputer-engine/src/agentic/flow/steps/suggestion.rs index 6eec20d3..3b1124d8 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/steps/suggestion.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/steps/suggestion.rs @@ -72,17 +72,20 @@ impl FlowRun<'_, B> { let mut fresh = fresh_rows(&screen, &shown, field, text, &self.stop_before, place); // A box that suggests places lists them once the page has fetched // them: live, a ride app's rows came after the first look, and the - // pickup typed was never set, so no ride showed. + // pickup typed was never set, so no ride showed. The wait ends as + // the page changes, and a page that stayed still lists nothing more: + // live, an address and a city box on a plain form waited 2.3-4.2 s + // each for a list that never comes. for _ in 0..LATE_LOOKS { if !fresh.is_empty() || !place { break; } - self.act(log, "wait", None, |backend| { - backend.execute(JevOperation::Wait, None, None) - }) - .await?; + let changed = self.await_change(log, LATE_LOOK_MS).await?; screen = self.look().await?; fresh = fresh_rows(&screen, &shown, field, text, &self.stop_before, place); + if !changed { + break; + } } let mentioned = fresh .iter() @@ -276,9 +279,14 @@ pub(in crate::agentic::flow) fn shares_most_words(candidate: &Candidate, text: & shared >= 2 && shared * 2 >= words.len() } -/// Looks again, a wait apart, for the rows a place box lists late. +/// Looks again, after a wait for the page to change, for the rows a place +/// box lists late. const LATE_LOOKS: u32 = 2; +/// Longest one wait for a place box's late rows: the page is looked at again +/// as soon as it changes, and a still page lists nothing more. +const LATE_LOOK_MS: u64 = 1_000; + /// Words of a slot that name a place, whose box lists matches as it is /// typed in. const PLACE_WORDS: &[&str] = &[ diff --git a/crates/tinycomputer-engine/src/workspace/mod.rs b/crates/tinycomputer-engine/src/workspace/mod.rs index f2ca9bbc..d7fdd9e8 100644 --- a/crates/tinycomputer-engine/src/workspace/mod.rs +++ b/crates/tinycomputer-engine/src/workspace/mod.rs @@ -275,6 +275,14 @@ impl Surface for Workspace { } } + fn await_change(&self, ms: u64) -> bool { + match (self.active_browser(), &self.desktop) { + (Some(browser), _) => browser.await_change(ms), + (None, Some(desktop)) => desktop.await_change(ms), + (None, None) => false, + } + } + fn navigate(&self, url: &str) -> DesktopResponse { let Some(browser) = &self.browser else { return no_browser("navigate"); diff --git a/crates/tinycomputer-engine/src/workspace/workspace_tests.rs b/crates/tinycomputer-engine/src/workspace/workspace_tests.rs index 345c1601..f1b17e99 100644 --- a/crates/tinycomputer-engine/src/workspace/workspace_tests.rs +++ b/crates/tinycomputer-engine/src/workspace/workspace_tests.rs @@ -107,6 +107,11 @@ impl Surface for Recorder { self.note("settle"); } + fn await_change(&self, _ms: u64) -> bool { + self.note("await_change"); + true + } + fn navigate(&self, _url: &str) -> DesktopResponse { self.note("navigate") } @@ -189,6 +194,24 @@ fn unnamed_calls_follow_the_side_last_observed_or_opened() { ); } +#[test] +fn a_wait_for_a_change_watches_the_active_side() { + let (workspace, calls) = workspace(true); + assert!(workspace.await_change(1_000)); + workspace.navigate("https://flights.test"); + assert!(workspace.await_change(1_000)); + assert_eq!( + drain(&calls), + [ + "desktop:await_change", + "browser:navigate", + "browser:await_change" + ] + ); + let bare: Workspace = Workspace::new(None, None); + assert!(!bare.await_change(1_000), "nothing to change"); +} + #[test] fn going_back_follows_the_active_side() { let (workspace, calls) = workspace(true); diff --git a/docs/crates/tinycomputer-browser/interacting.md b/docs/crates/tinycomputer-browser/interacting.md index 97055d08..c351d718 100644 --- a/docs/crates/tinycomputer-browser/interacting.md +++ b/docs/crates/tinycomputer-browser/interacting.md @@ -186,6 +186,14 @@ receive window, and then pauses `SETTLE_MS` regardless, giving a banner or menu that is mid-animation time to finish closing: about 1.6 s an action, live. +`Surface::await_change(ms)` watches the page rather than pausing: one +`evaluate` whose `MutationObserver` resolves `true` at the page's first change +of its own (an element or words added, removed, or rewritten, or an +element's look changed, never a `data-tc-` mark sight leaves), or `false` +once `ms` pass with none. A flow waiting for a place box's late suggestions +settles and looks again as soon as the page changes, and stops waiting once +it stays still; a watch that cannot run says the page may have changed. + ## Going back `Surface::back` maps straight to `Action::Back`, which agent-browser diff --git a/docs/crates/tinycomputer-core/surfaces-and-screens.md b/docs/crates/tinycomputer-core/surfaces-and-screens.md index a7280863..2a62fb69 100644 --- a/docs/crates/tinycomputer-core/surfaces-and-screens.md +++ b/docs/crates/tinycomputer-core/surfaces-and-screens.md @@ -34,6 +34,14 @@ engine calls it after every action, before the next observation, and before reading a value back, so a surface's own idea of "how long is a moment" stays in one place rather than being copied into every caller. +And `await_change(ms)`, which waits up to `ms` for the application to change +by itself and says whether it did. A flow uses it while it watches for +something to appear, such as the suggestions a place box lists for the text +just typed: it looks again as soon as the page changes, and stops once the +page stays still. By default it pauses as a `Wait` does and says the +application may have changed; `tinycomputer-browser` watches the page's DOM +instead. + Every member returns a `DesktopResponse`, never a plain `Result`. That is a deliberate rule of the whole repository, not just this trait: a denied permission or a stale reference is a result a caller can act on (retry, @@ -53,6 +61,7 @@ pub trait Surface: Clone + Send + 'static { fn press(&self, app: &str, combo: &str) -> DesktopResponse; fn launch(&self, app: &str) -> DesktopResponse; fn settle(&self) {} + fn await_change(&self, ms: u64) -> bool { /* pauses, and says it may have */ } fn navigate(&self, url: &str) -> DesktopResponse { /* refuses by default */ } fn back(&self, app: &str) -> DesktopResponse { /* refuses by default */ } } diff --git a/docs/crates/tinycomputer-engine/workspace.md b/docs/crates/tinycomputer-engine/workspace.md index ddacda42..0f981fda 100644 --- a/docs/crates/tinycomputer-engine/workspace.md +++ b/docs/crates/tinycomputer-engine/workspace.md @@ -79,7 +79,7 @@ Every method on `Surface` is handled, but not all the same way: | `launch` | By the named application; success makes that side active. | | `navigate` | Always the browser (there is no desktop equivalent); failure never activates it. | | `back` | The active side if it is the browser, else the desktop; there is no browser fallback if the desktop is active and has no browser. | -| `settle` | The active side, or the desktop if the browser is not active. | +| `settle`, `await_change` | The active side, or the desktop if the browser is not active; with neither, nothing changes. | ## Reading what is on screen without acting diff --git a/docs/technical/architecture.md b/docs/technical/architecture.md index 1b127960..b5af555f 100644 --- a/docs/technical/architecture.md +++ b/docs/technical/architecture.md @@ -97,6 +97,7 @@ pub trait Surface: Clone + Send + 'static { fn press(&self, app: &str, combo: &str) -> DesktopResponse; fn launch(&self, app: &str) -> DesktopResponse; fn settle(&self) {} + fn await_change(&self, ms: u64) -> bool { /* pauses, and says it may have */ } fn navigate(&self, url: &str) -> DesktopResponse { /* ACTION_NOT_SUPPORTED */ } } ``` diff --git a/docs/technical/decision-thresholds.md b/docs/technical/decision-thresholds.md index 5f2d09e8..38f0dc5c 100644 --- a/docs/technical/decision-thresholds.md +++ b/docs/technical/decision-thresholds.md @@ -51,7 +51,8 @@ Change a constant and its row together. | `LAYER_COVERS` | 3 | `front.rs` | controls something drawn over the window must cover, beyond what was covered before the press that opened it, on the same page, before it counts as a dialog the task opened (`surface` `layer`); a step that pressed inside such a dialog hands it back at the next step, and opening an address forgets it | | `FRONT_CONTROLS` | 8 | `act/turns.rs` | most controls of the task's dialog in front a failed step's note names, so a rescue answers with one of them | | `STEADY_HOLD` / `STEADY_CHECKS` | 0.65 / 3 | `steps/mod.rs` | belief a `wait_for` condition must keep, on checks in a row of one unchanged screen, to be taken as held under `DONE` | -| `LATE_LOOKS` | 2 | `steps/suggestion.rs` | looks again, a wait apart, for the suggestions a place or search box lists late, before its text is left as typed | +| `LATE_LOOKS` | 2 | `steps/suggestion.rs` | looks again, after a wait for the page to change, for the suggestions a place box lists late, before its text is left as typed; a page that stayed still through a wait lists nothing more | +| `LATE_LOOK_MS` | 1000 ms | `steps/suggestion.rs` | longest one of those waits: it ends as soon as the page changes (`Surface::await_change`) | | `BARE_CHARS` | 3 | `tinycomputer-core` `surface/groups.rs` | most letters and digits each field of a card may show for a list of such cards to be bare markers (carousel dots, size chips, page numbers), never results | | `CARD_LINK_CHARS` | 20 | `tinycomputer-core` `surface/groups.rs` | least characters (with a word in them) a link, option, radio, or button must show for a run of three or more under one parent to be a list of cards that are one control each | diff --git a/docs/technical/jev-journal.md b/docs/technical/jev-journal.md index 7c2707f5..57c67747 100644 --- a/docs/technical/jev-journal.md +++ b/docs/technical/jev-journal.md @@ -65,7 +65,7 @@ has `""`, and goal and intent runs carry their goal or intent text. | `turn` | a `do` turn ends | `step`, `turn`, `decisions` (made in that turn), `rounds` (round trips they took: a batch is one), `wall_ms` | | `survey` | the wide strategy surveys a crowded screen | `step`, `regions` asked about, `most_relevant` (region ids), `distractions` | | `observe` | a flow reads the screen | `step`, `part` (`screen` or `subtree`), `wall_ms`, `ok`, `candidates`, `unexplored` | -| `action` | a flow acts | `step`, `action`, `target`, `ok`, `note`, `wall_ms`, `settle_ms` | +| `action` | a flow acts | `step`, `action`, `target`, `ok`, `note`, `wall_ms`, `settle_ms`; a `wait` for the page to change that saw it stay still notes "nothing changed" and is not settled | | `reflect` | a `choose` that pressed something is reflected on | `step`, `held` (calibrated belief the choice shows), `attempt` (`first` or `after_repair`); `contradicted` when a selected sibling settled it without Jev | | `attention` | a turn or step asks what needs attention first | `step`, `distractions` (container names), `choice`, `verdict` | | `evidence` | a deliberating decision is weighed | `step`, `site` (`target`, `done`, `holds`), `p`, `margin`, `agreement`, `spread`, `framings`, `verdict` (`accept`, `deliberate`, `abstain`) | From 720961ba92cdfef30f75c9dc437cf6cf94ba9873 Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 7 Oct 2026 17:20:53 +0530 Subject: [PATCH 08/44] Tell planners to read the price of one before raising the count MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Asked for the price of one packet after raising its count to 2, Zepto runs read the cart line's ₹40, the total for both: the cart shows no price for one, and the product page's ₹20 lay behind the cart drawer. The flow guide, which the planner and the rescuer both read, now says to read the price of one before the count is raised. --- crates/tinycomputer-bus/src/flow/guide.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/tinycomputer-bus/src/flow/guide.md b/crates/tinycomputer-bus/src/flow/guide.md index 1f4f6c02..d958de26 100644 --- a/crates/tinycomputer-bus/src/flow/guide.md +++ b/crates/tinycomputer-bus/src/flow/guide.md @@ -155,7 +155,9 @@ do. buttons only once the item is in the cart, so to buy more than one, add the item first and raise its count in the next step; a − count + stepper where the add button was means the item is in the cart with - that count. A + that count. To report the price of one, `read` it before raising the + count: after that, the item's line and the cart show the total for all + of them, and a cart often shows no price for one. A `pick` opens a whole result card; to press one of several buttons inside the cards (a time or a slot listed under each place), use a plain step that names it ("press the earliest time listed"). A dialog's headings From 7d1783e189ad99c12ed90e7790525a7998b6d83b Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 7 Oct 2026 17:49:59 +0530 Subject: [PATCH 09/44] Try a planner, rescuer or shaper call again when it fails in passing One HTTP 502 from Tiny Humans' gateway failed a whole BlazeDemo task at its plan, 3 s in: a hosted model call that failed was never tried again. It now is, up to 4 times, 1, 2, then 4 seconds apart, when the failure can pass: a server error, a rate limit that is not a spending cap, or a dropped connection (tinyinference-llm's own classification). A refused key, a rejected request, and the test guard's refusal are not retried. --- .../tinycomputer-engine/src/planner/hosted.rs | 63 ++++++++++-- .../src/planner/planner_tests.rs | 2 + .../src/planner/planner_tests/retry_tests.rs | 96 +++++++++++++++++++ docs/crates/tinycomputer-engine/planner.md | 9 +- 4 files changed, 158 insertions(+), 12 deletions(-) create mode 100644 crates/tinycomputer-engine/src/planner/planner_tests/retry_tests.rs diff --git a/crates/tinycomputer-engine/src/planner/hosted.rs b/crates/tinycomputer-engine/src/planner/hosted.rs index a59e45b9..ecb817dd 100644 --- a/crates/tinycomputer-engine/src/planner/hosted.rs +++ b/crates/tinycomputer-engine/src/planner/hosted.rs @@ -6,14 +6,19 @@ //! feature. The keys arrive in the module's private configuration and never //! leave this adapter. +use std::future::Future; use std::sync::Arc; +use std::time::Duration; use tinycomputer_bus::agent::LanguageModelProvider; use tinyinference_llm::model::{ ReasoningConfig, ReasoningEffort, ResponseFormat, collect_model_stream, }; use tinyinference_llm::providers::openai::OpenAiModel; -use tinyinference_llm::{ChatModel, Message, ModelRequest, ProviderKind, ProviderSpec}; +use tinyinference_llm::{ + ChatModel, Error, Message, ModelRequest, ProviderKind, ProviderSpec, classify_provider_error, + classify_provider_failure, +}; use super::config::{ModelRoute, OUTPUT_MODEL, PLANNER_MODEL, PlannerConfig, RESCUE_MODEL}; use super::{Completion, LanguageModel, Planner, Role, Turn}; @@ -142,14 +147,54 @@ impl LanguageModel for Hosted { // for 60 seconds, and a reasoning model's whole reply can take // longer. A route that ignores streaming still answers in one // piece. - let stream = model - .stream(&(), request) - .await - .map_err(|error| error.to_string())?; - let response = collect_model_stream(stream) - .await - .map_err(|error| error.to_string())?; - Ok(Message::Assistant(response.message).text()) + with_retries(|| { + let model = model.clone(); + let request = request.clone(); + async move { + let stream = model.stream(&(), request).await?; + let response = collect_model_stream(stream).await?; + Ok(Message::Assistant(response.message).text()) + } + }) + .await + .map_err(|error| error.to_string()) }) } } + +/// How many times one model call is tried when it fails in passing — a +/// gateway's 502, a rate limit, a dropped connection — waiting 1, 2, then 4 +/// seconds between tries. Live, one 502 from Tiny Humans' gateway ended a +/// task at its plan, 3 s in. +const MODEL_TRIES: u32 = 4; + +/// Runs `call` until it answers, fails in a way another try cannot mend, or +/// has failed [`MODEL_TRIES`] times. +pub(super) async fn with_retries(mut call: C) -> Result +where + C: FnMut() -> F, + F: Future>, +{ + let mut tries = 1; + loop { + match call().await { + Err(error) if tries < MODEL_TRIES && passing(&error) => { + tokio::time::sleep(Duration::from_secs(1 << (tries - 1))).await; + tries += 1; + } + outcome => return outcome, + } + } +} + +/// Whether `error` may pass on another try: a server error, a rate limit +/// that is not a spending cap, or a dropped connection, but never a +/// refused key, a request the route rejects, or a reply that would not +/// parse. +pub(super) fn passing(error: &Error) -> bool { + match error { + Error::Provider(provider) => classify_provider_error(provider).is_retryable(), + Error::Model(message) => classify_provider_failure(None, None, message).is_retryable(), + _ => false, + } +} diff --git a/crates/tinycomputer-engine/src/planner/planner_tests.rs b/crates/tinycomputer-engine/src/planner/planner_tests.rs index a6d4e1ec..1e057c28 100644 --- a/crates/tinycomputer-engine/src/planner/planner_tests.rs +++ b/crates/tinycomputer-engine/src/planner/planner_tests.rs @@ -267,5 +267,7 @@ async fn the_open_router_planner_needs_a_key_and_never_prints_it() { assert!(!failed.contains("secret-key")); } +#[cfg(feature = "planner")] +mod retry_tests; #[cfg(feature = "planner")] mod route_tests; diff --git a/crates/tinycomputer-engine/src/planner/planner_tests/retry_tests.rs b/crates/tinycomputer-engine/src/planner/planner_tests/retry_tests.rs new file mode 100644 index 00000000..a9cdd7ee --- /dev/null +++ b/crates/tinycomputer-engine/src/planner/planner_tests/retry_tests.rs @@ -0,0 +1,96 @@ +//! Tests for trying a hosted model call again: a gateway's passing failure +//! is tried again, a refusal is not, and the tries are bounded. + +use std::cell::Cell; + +use tinyinference_llm::Error; +use tinyinference_llm::model::ProviderError; + +use super::super::hosted::{passing, with_retries}; + +fn bad_gateway() -> Error { + Error::Model("tinyhumans returned HTTP 502: error code: 502".to_owned()) +} + +#[tokio::test(start_paused = true)] +async fn a_model_call_is_tried_again_after_a_gateway_error() { + // Live, one 502 from Tiny Humans' gateway ended a task at its plan. + let tries = Cell::new(0); + let started = tokio::time::Instant::now(); + let answer = with_retries(|| { + tries.set(tries.get() + 1); + let failing = tries.get() < 3; + async move { + if failing { + Err(bad_gateway()) + } else { + Ok("the plan") + } + } + }) + .await; + assert_eq!(answer.unwrap(), "the plan"); + assert_eq!(tries.get(), 3); + assert_eq!( + started.elapsed(), + std::time::Duration::from_secs(3), + "1 s, then 2 s apart" + ); +} + +#[tokio::test(start_paused = true)] +async fn a_model_call_that_keeps_failing_gives_up_after_its_tries() { + let tries = Cell::new(0); + let answer = with_retries(|| { + tries.set(tries.get() + 1); + async { Err::<(), _>(bad_gateway()) } + }) + .await; + assert!(answer.is_err()); + assert_eq!(tries.get(), 4); +} + +#[tokio::test(start_paused = true)] +async fn a_refused_model_call_is_not_tried_again() { + for refusal in [ + Error::Model("tinyhumans returned HTTP 401: invalid key".to_owned()), + Error::Validation("network-backed model calls are denied".to_owned()), + ] { + let message = refusal.to_string(); + let mut refusal = Some(refusal); + let tries = Cell::new(0); + let answer = with_retries(|| { + tries.set(tries.get() + 1); + let error = refusal.take().expect("asked once"); + async move { Err::<(), _>(error) } + }) + .await; + assert!(answer.is_err()); + assert_eq!(tries.get(), 1, "{message}"); + } +} + +#[test] +fn a_failure_passes_by_its_kind_and_what_the_provider_says() { + assert!(passing(&bad_gateway())); + assert!(passing(&Error::Model( + "connection reset by peer".to_owned() + ))); + let unavailable = ProviderError { + provider: "openai".to_owned(), + status: Some(503), + message: "upstream unavailable".to_owned(), + retryable: true, + ..ProviderError::default() + }; + assert!(passing(&Error::Provider(Box::new(unavailable.clone())))); + let final_word = ProviderError { + retryable: false, + ..unavailable + }; + assert!( + !passing(&Error::Provider(Box::new(final_word))), + "a provider that says it will not pass is believed" + ); + assert!(!passing(&Error::Unsupported("tools".to_owned()))); +} diff --git a/docs/crates/tinycomputer-engine/planner.md b/docs/crates/tinycomputer-engine/planner.md index 8a87ea52..943c8a4f 100644 --- a/docs/crates/tinycomputer-engine/planner.md +++ b/docs/crates/tinycomputer-engine/planner.md @@ -91,9 +91,12 @@ route. This is the only file in the crate that links a text-generating model at all: the keys it is given in the module's private configuration never leave this one adapter. Every call is streamed and gathered into one reply: Tiny Humans' gateway answers HTTP 504 to a request that sends nothing back -for 60 seconds, and a reasoning model's whole reply can take longer. It -builds all three of the crate's language-model helpers, not just -the planner: +for 60 seconds, and a reasoning model's whole reply can take longer. A call +that fails in passing (a server error, a rate limit that is not a spending +cap, a dropped connection) is tried up to 4 times, 1, 2, then 4 seconds +apart: live, one 502 from the gateway ended a task at its plan. A refused key +or a rejected request is not tried again. It builds all three of the crate's +language-model helpers, not just the planner: ```rust pub fn open_router(config: &PlannerConfig) -> Result From 21d7bddc214d0df26da3d7c8d75713fbdd987192 Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 7 Oct 2026 17:53:55 +0530 Subject: [PATCH 10/44] Send a copy of a stalled framing, and give a Jev attempt 10 s In the 7 October runs, 21 of 36,459 Jev calls needed a retry: one framing of a burst stalled while its siblings answered in about 0.6 s, until the gateway gave up after ~10 s (12 s with the retry) or nothing came back before the client's 30 s timeout (31.6 s). Every decision waits for all of its framings, so about one run in four lost 12-32 s to one of them. A framing that has not answered after HEDGE_AFTER (2.5 s; 3.5 s for a request of 32 KB or more, whose p99 was 3.1 s) is now sent once more, and whichever copy answers first counts; a copy that fails gives way to the other. Fewer than 0.3% of calls ran that long otherwise, so the copies cost little. The answer that counts carries the extra attempt, and the journal records a `hedge` event. Every framing goes through it, the evidence gate's included. A Jev attempt now gives up after 10 s unless jev.timeout_ms says otherwise (the slowest answer seen took 8.6 s), so a request both copies of which stall is retried after 10 s rather than 30 s. --- .../agentic/agentic_tests/resolve_tests.rs | 17 +++ .../src/agentic/flow/decide.rs | 79 ++++++++++- .../src/agentic/flow/flow_tests.rs | 1 + .../agentic/flow/flow_tests/hedge_tests.rs | 129 ++++++++++++++++++ .../src/agentic/runtime.rs | 16 ++- docs/crates/tinycomputer/configuration.md | 10 +- docs/technical/decision-thresholds.md | 1 + docs/technical/jev-harness.md | 2 +- docs/technical/jev-journal.md | 1 + 9 files changed, 242 insertions(+), 14 deletions(-) create mode 100644 crates/tinycomputer-engine/src/agentic/flow/flow_tests/hedge_tests.rs diff --git a/crates/tinycomputer-engine/src/agentic/agentic_tests/resolve_tests.rs b/crates/tinycomputer-engine/src/agentic/agentic_tests/resolve_tests.rs index c6df3f0e..265010cb 100644 --- a/crates/tinycomputer-engine/src/agentic/agentic_tests/resolve_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/agentic_tests/resolve_tests.rs @@ -45,6 +45,23 @@ fn jev_calls_ride_out_a_provider_outage_unless_told_otherwise() { assert_eq!(told.initial_backoff, RETRY.initial_backoff); } +#[test] +fn a_jev_attempt_gives_up_after_ten_seconds_unless_told_otherwise() { + // Live, a request nothing came back for waited the client's own 30 s + // before its retry answered in under a second. + let mut request = JevConfig::new("key"); + assert_eq!( + client_config(&request).timeout, + std::time::Duration::from_secs(10) + ); + request.timeout_ms = Some(30_000); + assert_eq!( + client_config(&request).timeout, + std::time::Duration::from_secs(30), + "a configured timeout wins" + ); +} + #[test] fn each_provider_selects_its_decision_model() { let configured = |value: serde_json::Value| { diff --git a/crates/tinycomputer-engine/src/agentic/flow/decide.rs b/crates/tinycomputer-engine/src/agentic/flow/decide.rs index 800d217c..eee46fef 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/decide.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/decide.rs @@ -1,17 +1,36 @@ //! Asking Jev: every request is briefed, masked, fitted to size, and voted //! on through one door. -use std::{collections::BTreeMap, time::Instant}; +use std::{ + collections::BTreeMap, + time::{Duration, Instant}, +}; use serde_json::{Value, json}; use tinycomputer_bus::{FlowLoop, FlowStopReason, JevExchange}; use tinycomputer_core::Facts; -use tinyinference_decisions::{Answer, EvaluationRequest, Question}; +use tinyinference_decisions::{ + Answer, EvaluationFailure, EvaluationRequest, EvaluationResult, Question, +}; use super::{ FlowRun, Halt, MAX_REQUEST_BYTES, StepLog, ask, backend::AgentBackend, brief::clip, vote, }; -use crate::agentic::{journal::millis, merge_metrics, provider_error}; +use crate::agentic::{JevRuntime, journal::millis, merge_metrics, provider_error}; + +/// How long a framing runs before a copy of it is sent and the first answer +/// of the two taken. Live, a call's p99 was 2.0 s, while one framing of a +/// burst stalled 12–32 s (the gateway gave up after ~10 s, or nothing came +/// back before the client's timeout) as its siblings answered in under 1 s. +const HEDGE_AFTER: Duration = Duration::from_millis(2_500); + +/// [`HEDGE_AFTER`] for a request of [`HEDGE_LARGE_BYTES`] or more, whose +/// p99 was 3.1 s live. +const HEDGE_AFTER_LARGE: Duration = Duration::from_millis(3_500); + +/// Size from which a request waits [`HEDGE_AFTER_LARGE`] for its first +/// answer. +const HEDGE_LARGE_BYTES: usize = 32 * 1024; impl FlowRun<'_, B> { /// Asks Jev one request, charging it to the run and the step. @@ -199,7 +218,7 @@ impl FlowRun<'_, B> { .collect() } - /// Sends every framing to Jev at once. + /// Sends every framing to Jev at once, each one [`hedged`]. pub(super) fn spawn( &self, framings: &[vote::Framing], @@ -217,7 +236,7 @@ impl FlowRun<'_, B> { let runtime = self.runtime.clone(); let step = self.step.clone(); let request = framing.request.clone(); - tokio::spawn(async move { runtime.evaluate(Some(&step), &request).await }) + tokio::spawn(async move { hedged(&runtime, &step, &request).await }) }) .collect() } @@ -258,6 +277,56 @@ type Sent = ( /// The id of the page-kind question a request on a web page carries. pub(super) const PAGE_KIND: &str = "page_kind"; +/// Asks `request` once, and once more when no answer has come within its +/// hedge delay ([`HEDGE_AFTER`], or [`HEDGE_AFTER_LARGE`] for a large +/// request), taking whichever copy answers first. A copy that fails gives +/// way to the other; when both fail, the first failure is returned. The +/// answer that counts carries the extra attempt, and the journal records a +/// `hedge` event. +pub(in crate::agentic::flow) async fn hedged( + runtime: &JevRuntime, + step: &str, + request: &EvaluationRequest, +) -> Result { + let delay = if bytes(request) >= HEDGE_LARGE_BYTES { + HEDGE_AFTER_LARGE + } else { + HEDGE_AFTER + }; + let first = runtime.evaluate(Some(step), request); + tokio::pin!(first); + if let Ok(outcome) = tokio::time::timeout(delay, &mut first).await { + return outcome; + } + let sent = Instant::now(); + let copy = runtime.evaluate(Some(step), request); + tokio::pin!(copy); + let (outcome, copy_won) = tokio::select! { + outcome = &mut first => (outcome, false), + outcome = &mut copy => (outcome, true), + }; + let outcome = match outcome { + Ok(evaluation) => Ok(evaluation), + Err(failure) => { + let other = if copy_won { first.await } else { copy.await }; + other.or(Err(failure)) + } + }; + runtime.journal.record("hedge", || { + json!({ + "step": step, + "after_ms": millis(delay), + "won": if copy_won { "copy" } else { "first" }, + "ok": outcome.is_ok(), + "wall_ms": millis(delay + sent.elapsed()), + }) + }); + outcome.map(|mut evaluation| { + evaluation.attempts = evaluation.attempts.saturating_add(1); + evaluation + }) +} + /// `request` cut by its questions into requests of at most `limit` bytes of /// JSON, each carrying the whole state and as many of the questions, in /// order, as fit beside it. Jev evaluates every question on its own against diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests.rs index 48512226..749942f6 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests.rs @@ -27,6 +27,7 @@ mod do_loop_tests; mod end_to_end_tests; mod enter_tests; mod grounding_tests; +mod hedge_tests; mod helpers_tests; mod journal_tests; mod pick_tests; diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/hedge_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/hedge_tests.rs new file mode 100644 index 00000000..63f0d5cd --- /dev/null +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/hedge_tests.rs @@ -0,0 +1,129 @@ +//! Hedging a framing: one still running past its hedge delay gets a copy, +//! and the first answer of the two counts. + +use super::*; +use crate::agentic::flow::decide::hedged; + +/// A Jev that answers each call after the wait scripted for it, in call +/// order, or fails it. +struct Paced { + calls: Mutex, + script: Vec<(Duration, bool)>, +} + +impl Evaluator for Paced { + fn evaluate<'a>( + &'a self, + _request: &'a EvaluationRequest, + ) -> Pin> + Send + 'a>> + { + let call = { + let mut calls = self.calls.lock().unwrap(); + *calls += 1; + *calls - 1 + }; + let (wait, fails) = self.script.get(call).copied().unwrap_or_default(); + Box::pin(async move { + tokio::time::sleep(wait).await; + if fails { + return Err(EvaluationFailure { + error: Box::new(tinyinference_decisions::Error::RateLimited), + attempts: 1, + latency: wait, + }); + } + Ok(EvaluationResult { + response: EvaluationResponse { + model: format!("call {call}"), + answers: BTreeMap::new(), + usage: tinyinference_decisions::Usage::default(), + }, + request_id: None, + attempts: 1, + latency: wait, + }) + }) + } +} + +fn paced(script: &[(u64, bool)]) -> (JevRuntime, Arc) { + let paced = Arc::new(Paced { + calls: Mutex::new(0), + script: script + .iter() + .map(|(ms, fails)| (Duration::from_millis(*ms), *fails)) + .collect(), + }); + let runtime = JevRuntime { + client: paced.clone(), + configuration: tinycomputer_bus::JevConfiguration { + provider: tinycomputer_bus::JevProvider::OpenRouter, + model: "jev-latest".to_owned(), + endpoint_url: None, + fast: false, + }, + pending: Arc::default(), + journal: crate::agentic::journal::Journal::default(), + }; + (runtime, paced) +} + +/// A request of about `bytes` bytes. +fn request(bytes: usize) -> EvaluationRequest { + ask::request( + "jev-latest", + json!({"visible_text": "x".repeat(bytes)}), + ask::Questions::default(), + ) +} + +#[tokio::test(start_paused = true)] +async fn a_stalled_framing_is_answered_by_its_copy() { + // Live, one framing of a burst stalled 12–32 s while its siblings + // answered in under a second. + let (runtime, paced) = paced(&[(30_000, false), (600, false)]); + let started = tokio::time::Instant::now(); + let answer = hedged(&runtime, "1", &request(1_000)).await.unwrap(); + assert_eq!(answer.response.model, "call 1", "the copy answered"); + assert_eq!(answer.attempts, 2, "the copy is an attempt of its own"); + assert_eq!(started.elapsed(), Duration::from_millis(3_100)); + assert_eq!(*paced.calls.lock().unwrap(), 2); +} + +#[tokio::test(start_paused = true)] +async fn a_framing_that_answers_in_time_gets_no_copy() { + let (runtime, paced) = paced(&[(2_400, false)]); + let answer = hedged(&runtime, "1", &request(1_000)).await.unwrap(); + assert_eq!( + (answer.response.model.as_str(), answer.attempts), + ("call 0", 1) + ); + assert_eq!(*paced.calls.lock().unwrap(), 1); +} + +#[tokio::test(start_paused = true)] +async fn a_large_request_waits_longer_before_its_copy() { + // A 32 KB request's p99 was 3.1 s live: at 3 s it still gets no copy. + let (runtime, paced) = paced(&[(3_000, false)]); + let answer = hedged(&runtime, "1", &request(40_000)).await.unwrap(); + assert_eq!(answer.attempts, 1); + assert_eq!(*paced.calls.lock().unwrap(), 1); +} + +#[tokio::test(start_paused = true)] +async fn a_failed_copy_gives_way_and_two_failures_fail() { + // The copy fails at once; the slow first answer still counts. + let (runtime, _) = paced(&[(4_000, false), (0, true)]); + let started = tokio::time::Instant::now(); + let answer = hedged(&runtime, "1", &request(1_000)).await.unwrap(); + assert_eq!(answer.response.model, "call 0"); + assert_eq!(started.elapsed(), Duration::from_millis(4_000)); + + // The first fails after its copy was sent; the copy's answer counts. + let (runtime, _) = paced(&[(3_000, true), (1_000, false)]); + let answer = hedged(&runtime, "1", &request(1_000)).await.unwrap(); + assert_eq!(answer.response.model, "call 1"); + + let (runtime, _) = paced(&[(3_000, true), (1_000, true)]); + assert!(hedged(&runtime, "1", &request(1_000)).await.is_err()); +} diff --git a/crates/tinycomputer-engine/src/agentic/runtime.rs b/crates/tinycomputer-engine/src/agentic/runtime.rs index ae0220c6..2c4bc3cc 100644 --- a/crates/tinycomputer-engine/src/agentic/runtime.rs +++ b/crates/tinycomputer-engine/src/agentic/runtime.rs @@ -29,6 +29,11 @@ pub(super) const RETRY: RetryPolicy = RetryPolicy { max_backoff: Duration::from_secs(8), }; +/// How long one Jev attempt may take when the configuration does not say. +/// Live, the slowest answer took 8.6 s, and a request nothing came back for +/// waited the client's own 30 s before its retry answered in under 1 s. +pub(super) const ATTEMPT_TIMEOUT: Duration = Duration::from_secs(10); + /// Configured Jev transport and non-secret policy metadata. #[derive(Clone)] pub struct JevRuntime { @@ -285,8 +290,9 @@ pub(super) fn trusted_endpoint(provider: JevProvider, endpoint: &str) -> bool { } /// The HTTP client configuration for a Jev `request`: its provider's -/// route, endpoint, timeout, and attribution, retrying as [`RETRY`] unless -/// the request sets its own number of retries. +/// route, endpoint, timeout ([`ATTEMPT_TIMEOUT`] unless the request sets +/// one), and attribution, retrying as [`RETRY`] unless the request sets its +/// own number of retries. pub(super) fn client_config(request: &JevConfig) -> ClientConfig { let mut config = match request.provider { JevProvider::TypeSafe => ClientConfig::new(request.api_key()), @@ -298,9 +304,9 @@ pub(super) fn client_config(request: &JevConfig) -> ClientConfig { if let Some(endpoint) = &request.endpoint_url { config = config.with_endpoint_url(endpoint); } - if let Some(timeout_ms) = request.timeout_ms { - config.timeout = Duration::from_millis(timeout_ms); - } + config.timeout = request + .timeout_ms + .map_or(ATTEMPT_TIMEOUT, Duration::from_millis); config.retry = RETRY; if let Some(max_retries) = request.max_retries { config.retry.max_retries = max_retries; diff --git a/docs/crates/tinycomputer/configuration.md b/docs/crates/tinycomputer/configuration.md index e1a7af8e..c27af8fb 100644 --- a/docs/crates/tinycomputer/configuration.md +++ b/docs/crates/tinycomputer/configuration.md @@ -54,7 +54,7 @@ controls the *browser's* visibility for that one task. "provider": "open_router", "model": "jev-latest", "endpoint_url": null, - "timeout_ms": 30000, + "timeout_ms": 10000, "max_retries": 2, "sdk_name": "my-host" } @@ -79,8 +79,12 @@ OpenJEV and Sage have no Tiny Humans proxy route, so a host that wants its decisions to go through Tiny Humans uses `tiny_humans_open_router`. `sdk_name` is sent only to the Tiny Humans proxy. -A provider's server error (HTTP 5xx) or rate limit (429) is retried, waiting -1, 2, 4, then 8 seconds between attempts, or as long as the provider asks. +Each attempt may take `timeout_ms`, 10 seconds unless set: live, the slowest +answer took 8.6 s. A framing that has not answered after 2.5 s (3.5 s for a +request of 32 KB or more) is also sent once more, and whichever copy answers +first counts. A provider's server error (HTTP 5xx) or rate limit (429) is +retried, waiting 1, 2, 4, then 8 seconds between attempts, or as long as the +provider asks. `max_retries` sets how many retries follow the first attempt; it defaults to four, about 15 seconds in all, so a gateway's brief outage does not end a run. diff --git a/docs/technical/decision-thresholds.md b/docs/technical/decision-thresholds.md index 38f0dc5c..50925e5b 100644 --- a/docs/technical/decision-thresholds.md +++ b/docs/technical/decision-thresholds.md @@ -51,6 +51,7 @@ Change a constant and its row together. | `LAYER_COVERS` | 3 | `front.rs` | controls something drawn over the window must cover, beyond what was covered before the press that opened it, on the same page, before it counts as a dialog the task opened (`surface` `layer`); a step that pressed inside such a dialog hands it back at the next step, and opening an address forgets it | | `FRONT_CONTROLS` | 8 | `act/turns.rs` | most controls of the task's dialog in front a failed step's note names, so a rescue answers with one of them | | `STEADY_HOLD` / `STEADY_CHECKS` | 0.65 / 3 | `steps/mod.rs` | belief a `wait_for` condition must keep, on checks in a row of one unchanged screen, to be taken as held under `DONE` | +| `HEDGE_AFTER` / `HEDGE_AFTER_LARGE` | 2.5 s / 3.5 s | `decide.rs` | how long a framing runs before a copy of it is sent and the first answer of the two taken; the longer wait is for a request of `HEDGE_LARGE_BYTES` (32 KB) or more | | `LATE_LOOKS` | 2 | `steps/suggestion.rs` | looks again, after a wait for the page to change, for the suggestions a place box lists late, before its text is left as typed; a page that stayed still through a wait lists nothing more | | `LATE_LOOK_MS` | 1000 ms | `steps/suggestion.rs` | longest one of those waits: it ends as soon as the page changes (`Surface::await_change`) | | `BARE_CHARS` | 3 | `tinycomputer-core` `surface/groups.rs` | most letters and digits each field of a card may show for a list of such cards to be bare markers (carousel dots, size chips, page numbers), never results | diff --git a/docs/technical/jev-harness.md b/docs/technical/jev-harness.md index c3514b1d..8abee735 100644 --- a/docs/technical/jev-harness.md +++ b/docs/technical/jev-harness.md @@ -155,7 +155,7 @@ round trip's *slowest* framing, plus the action, plus settling. The levers: | Lever | Effect on latency | Effect on accuracy | |---|---|---| | `strategy` | `wide` asks one request per `do` turn instead of two to seven in sequence | the digest, survey, and memory show more of what matters; measure with the lab's `--strategy` | -| `votes` | a decision waits for its slowest framing: more framings, longer tail | more framings average out position and phrasing bias | +| `votes` | a decision waits for its slowest framing: more framings, longer tail; a framing slower than 2.5 s (3.5 s at 32 KB or more) gets a copy, and the first answer counts | more framings average out position and phrasing bias | | request size | Jev's latency grows with input tokens; a big element list is the usual cause | trimming can drop the element that was needed | | grounding memory | a remembered element is confirmed with one Noul instead of narrowing | none when the hint is right | | `disabled_loops` | each loop off removes a question or a whole decision | measure it before shipping it off | diff --git a/docs/technical/jev-journal.md b/docs/technical/jev-journal.md index 57c67747..479c344e 100644 --- a/docs/technical/jev-journal.md +++ b/docs/technical/jev-journal.md @@ -65,6 +65,7 @@ has `""`, and goal and intent runs carry their goal or intent text. | `turn` | a `do` turn ends | `step`, `turn`, `decisions` (made in that turn), `rounds` (round trips they took: a batch is one), `wall_ms` | | `survey` | the wide strategy surveys a crowded screen | `step`, `regions` asked about, `most_relevant` (region ids), `distractions` | | `observe` | a flow reads the screen | `step`, `part` (`screen` or `subtree`), `wall_ms`, `ok`, `candidates`, `unexplored` | +| `hedge` | a framing ran past its hedge delay and a copy was sent | `step`, `after_ms` (the delay), `won` (`first` or `copy`), `ok`, `wall_ms` | | `action` | a flow acts | `step`, `action`, `target`, `ok`, `note`, `wall_ms`, `settle_ms`; a `wait` for the page to change that saw it stay still notes "nothing changed" and is not settled | | `reflect` | a `choose` that pressed something is reflected on | `step`, `held` (calibrated belief the choice shows), `attempt` (`first` or `after_repair`); `contradicted` when a selected sibling settled it without Jev | | `attention` | a turn or step asks what needs attention first | `step`, `distractions` (container names), `choice`, `verdict` | From 3562a53c2f56a78d3b5e581f6b32be3eb0a4c5aa Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 7 Oct 2026 17:59:12 +0530 Subject: [PATCH 11/44] Bump agent-browser for a networkquiet that counts only page changes The networkquiet wait now counts only requests that can change what the page shows (the page's own document, scripts, stylesheets, XHR and fetch data), so analytics pings and other frames' documents that never report finishing no longer hold every settle to its cap. --- vendor/agent-browser | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/agent-browser b/vendor/agent-browser index 5efb6334..ce670649 160000 --- a/vendor/agent-browser +++ b/vendor/agent-browser @@ -1 +1 @@ -Subproject commit 5efb6334918b9a5ee9117ca6d43a68989c89b7cf +Subproject commit ce670649419d3fa7f28b2334f9e4b8159e5d0e12 From 27bec2991dec63b75f6b2932fa70a15234571240 Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 7 Oct 2026 17:59:12 +0530 Subject: [PATCH 12/44] Cap a prompt settle's network wait at 1 s On Amazon every action that opened a page sent 100+ requests for over 2 s, so prompt settling always ran its network wait to the 2 s cap, then its 400 ms stillness cap, about 2.4 s an action. What the task needed was on screen by then for a long time: the results after 0.87-1.04 s, a product's title and Add to Cart after 0.96-1.24 s, the cart's subtotal after 0.66-0.81 s (six measured page loads). Settle::Prompt now waits at most QUIET_MS = 1 s for the requests that change the page, then for the page to stop changing as before, so such a page is read after about 1.4 s. Steady settling keeps its 2 s. --- crates/tinycomputer-browser/src/surface/mod.rs | 17 +++++++++++++---- .../src/surface/operations.rs | 6 ++++-- .../surface/surface_tests/operations_tests.rs | 2 +- docs/crates/tinycomputer-browser/interacting.md | 15 ++++++++++----- docs/crates/tinycomputer/configuration.md | 2 +- 5 files changed, 29 insertions(+), 13 deletions(-) diff --git a/crates/tinycomputer-browser/src/surface/mod.rs b/crates/tinycomputer-browser/src/surface/mod.rs index 7e4717aa..786df16f 100644 --- a/crates/tinycomputer-browser/src/surface/mod.rs +++ b/crates/tinycomputer-browser/src/surface/mod.rs @@ -51,6 +51,13 @@ const SETTLE_MS: u64 = 400; /// that polls forever is never idle, so this is a cap, not an expectation. const NETWORK_IDLE_MS: u64 = 2_000; +/// The longest [`Settle::Prompt`] waits for the requests that change the +/// page to end. Live on Amazon, each action that opened a page sent 100+ +/// requests for over 2 s, while what the task needed (the results, a +/// product's title and Add to Cart, the cart's subtotal) showed after +/// 0.7–1.2 s. +const QUIET_MS: u64 = 1_000; + /// The longest one reading of the page may take, by sight or as a tree. A /// reading sent while a page was being replaced waited out the browser's /// own deadline live, 30 s for sight and again for the tree, so one look @@ -80,10 +87,12 @@ pub enum Settle { /// counted only after a first quiet receive window, so at least about /// 1.1 s — then pause [`SETTLE_MS`] more. Steady, - /// Wait for the network to go quiet, counting the 500 ms from the start, - /// then only until the page stops changing: no DOM change for - /// [`STILL_MS`] and no finite CSS animation running, over at least two - /// drawn frames, at most [`SETTLE_MS`]. + /// Wait for the network to go quiet, counting the 500 ms from the start + /// and only the requests that can change the page (its document, + /// scripts, stylesheets, fetched data), at most [`QUIET_MS`]; then only + /// until the page stops changing: no DOM change for [`STILL_MS`] and no + /// finite CSS animation running, over at least two drawn frames, at most + /// [`SETTLE_MS`]. /// An idle page is read again after about 0.6 s instead of 1.6 s; a busy /// one still waits for its requests. The default: over 44 live runs it /// cost no run its outcome. diff --git a/crates/tinycomputer-browser/src/surface/operations.rs b/crates/tinycomputer-browser/src/surface/operations.rs index c57de5fd..96d3adaa 100644 --- a/crates/tinycomputer-browser/src/surface/operations.rs +++ b/crates/tinycomputer-browser/src/surface/operations.rs @@ -14,7 +14,9 @@ use crate::error::Error; use super::envelope::{failure, not_a_text_field, reply}; use super::sight; use super::{BrowserSurface, Perception}; -use super::{NETWORK_IDLE_MS, READ_TIMEOUT, SETTLE_MS, SKELETON_DEPTH, STILL_MS, Settle, tree}; +use super::{ + NETWORK_IDLE_MS, QUIET_MS, READ_TIMEOUT, SETTLE_MS, SKELETON_DEPTH, STILL_MS, Settle, tree, +}; impl Surface for BrowserSurface { fn observe( @@ -240,7 +242,7 @@ impl Surface for BrowserSurface { if let Ok(id) = self.ensure_session() { let _quiet = self.block(self.browser.command( &id, - json!({"action": "waitforloadstate", "state": "networkquiet", "timeout": NETWORK_IDLE_MS}), + json!({"action": "waitforloadstate", "state": "networkquiet", "timeout": QUIET_MS}), )); let _still = self.block( self.browser diff --git a/crates/tinycomputer-browser/src/surface/surface_tests/operations_tests.rs b/crates/tinycomputer-browser/src/surface/surface_tests/operations_tests.rs index e254660a..3a3ce87c 100644 --- a/crates/tinycomputer-browser/src/surface/surface_tests/operations_tests.rs +++ b/crates/tinycomputer-browser/src/surface/surface_tests/operations_tests.rs @@ -559,7 +559,7 @@ fn a_prompt_settle_counts_quiet_from_the_start_and_waits_only_while_the_page_cha let quiet = fake.last("waitforloadstate"); assert_eq!( (quiet["state"].as_str(), quiet["timeout"].as_u64()), - (Some("networkquiet"), Some(2_000)) + (Some("networkquiet"), Some(1_000)) ); let still = fake.last("evaluate"); let script = still["script"].as_str().unwrap(); diff --git a/docs/crates/tinycomputer-browser/interacting.md b/docs/crates/tinycomputer-browser/interacting.md index c351d718..9d19e15c 100644 --- a/docs/crates/tinycomputer-browser/interacting.md +++ b/docs/crates/tinycomputer-browser/interacting.md @@ -166,14 +166,19 @@ appearing anywhere on the page, an element reaching a given state wait_for` refuses if none of the three is given. `Surface::settle`, called before a decision loop reads the page again, -waits (bounded, up to `NETWORK_IDLE_MS` = 2 seconds) for the page's network -to go quiet, then only while the page is still changing. A page that polls -constantly in the background never goes properly quiet, so the network wait -is a cap, not a guarantee. +waits (bounded, up to `QUIET_MS` = 1 second) for the requests that change +the page to end, then only while the page is still changing. A page that +polls constantly in the background never goes properly quiet, so the +network wait is a cap, not a guarantee. That is `Settle::Prompt`, the default (the module's `browser.settle`). It waits for the engine's `networkquiet`, which counts its 500 ms of quiet -from the start, and then resolves once no DOM change has happened for +from the start and counts only requests that can change what the page +shows: the page's own document, scripts, stylesheets, and fetched data, not +analytics pings, pictures, fonts, media, or other frames' documents (some of +which never report finishing at all). Live on Amazon, every action that +opened a page sent 100+ requests for over 2 s, while what the task needed +showed after 0.7–1.2 s. It then resolves once no DOM change has happened for `STILL_MS` = 120 milliseconds and no finite CSS animation or transition is running (a menu fading out changes no DOM node), over at least two drawn frames, and after `SETTLE_MS` = 400 milliseconds at most. An endless spinner diff --git a/docs/crates/tinycomputer/configuration.md b/docs/crates/tinycomputer/configuration.md index c27af8fb..8f7d8817 100644 --- a/docs/crates/tinycomputer/configuration.md +++ b/docs/crates/tinycomputer/configuration.md @@ -236,7 +236,7 @@ How the module launches every browser it opens — each task's, and each | `user_agent` | the `User-Agent` every launched browser sends; booking sites turn away a browser that announces itself as headless | | `args` | extra launch arguments, as an array of strings | | `perception` | how a task reads a page: `sight` (the default) reads the rendered page as a person sees it, `tree` the accessibility tree alone ([`browser-sight.md`](../../technical/specs/browser-sight.md)) | -| `settle` | how a task lets a page settle after an action before reading it again: `prompt` (the default) counts the network's 500 ms of quiet from the start and then waits only while the page is still changing (at most 400 ms), so an idle page is read again after about 0.6 s; `steady` waits for the network to go idle, at least about 1.1 s, then 400 ms more | +| `settle` | how a task lets a page settle after an action before reading it again: `prompt` (the default) counts the network's 500 ms of quiet from the start, counting only requests that can change the page and for at most 1 s, and then waits only while the page is still changing (at most 400 ms), so an idle page is read again after about 0.6 s; `steady` waits for the network to go idle, at least about 1.1 s, then 400 ms more | | `prelaunch` | `true` (the default) opens a browser-only task's browser while `StartTask` plans it (a `task` with no `flow`), so the first step does not wait for the launch; `false` opens it at the first step | Most installs never need any of this: leave `browser` out entirely and the From e188e4e6eb5829db17695af5edbd7283772c9a55 Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 7 Oct 2026 17:59:33 +0530 Subject: [PATCH 13/44] Repeat the price-of-one rule in the read step's own row of the guide MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In the live check after the rule first went in, 3 of 4 plans read the price of one before raising the count, but one Zepto plan still read it in the cart and got ₹40, the line for two. The rule now also sits in the `read` row of the step table, where a planner looks when it writes a read. --- crates/tinycomputer-bus/src/flow/guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tinycomputer-bus/src/flow/guide.md b/crates/tinycomputer-bus/src/flow/guide.md index d958de26..6b3791e2 100644 --- a/crates/tinycomputer-bus/src/flow/guide.md +++ b/crates/tinycomputer-bus/src/flow/guide.md @@ -39,7 +39,7 @@ do. | `do` | `{"do": "start a new note"}` | Same as a plain string. | | `enter` | `{"enter": {"subject": "Hi"}}` | Put each text into the field its key describes; a box that suggests matches as you type (a location, a city) has the matching suggestion picked. | | `choose` | `{"choose": {"what": "the font list", "option": "Helvetica"}}` | Pick an option in a list, menu, or popup, or in a group of option buttons (a size, a colour, a quantity, a day in a strip of dates). | -| `read` | `{"read": {"what": "the newest message's subject", "into": "subject"}}` | Store visible text in a variable. | +| `read` | `{"read": {"what": "the newest message's subject", "into": "subject"}}` | Store visible text in a variable. Read the price of one item before any step raises its count: after that, its line and the cart show the total for all of them. | | `extract` | `{"extract": {"what": "the flight results", "into": "flights"}}` | Store every item of a list, as JSON rows of their text, in a variable. | | `pick` | `{"pick": {"from": "the flight results", "by": "lowest price", "into": "flight"}}` | Choose the best of a list of results (cards or rows, each an item to open) and open it; `into` stores its text. Prices, times, durations, and stops are compared exactly. | | `verify` | `{"verify": "the draft shows a recipient"}` | Fail the flow unless this holds. | From 3f7caf4fc1af7da087c25e9039bd8127d82a42be Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 7 Oct 2026 18:09:00 +0530 Subject: [PATCH 14/44] Read one shadow root's controls from the tree beside sight Handing the whole page to the tree whenever a shadow root showed controls made Lenskart readable but worse to work with. While its consent banner was up, the tree read the search box unnamed, so Enter did not search; it cut the page at its element budget; and it read an image carousel's dots as sizes. The banner reached Jev in 1,330 calls of one run and was still never closed. Sight now keeps reading the page. For a shadow root that shows controls (its host's box or any control's), it marks the host and names the layer the shadow content draws, from its fixed or dialog element's label, heading, or first words: `popover "We value your privacy"`. The surface reads the tree under that host alone and adds its controls and text after everything sight read, under that label, so the digest sees a layer in front and the attention pass a privacy card with "Allow Selection" to close it with. A tree snapshot's refs last until the next snapshot, so with two such shadow roots, or when the subtree cannot be read, the tree reads the whole page as before. --- .../tinycomputer-browser/src/surface/mod.rs | 59 +++++++- .../src/surface/sight/mod.rs | 45 +++++- .../src/surface/sight/sight.js | 26 +++- .../surface/sight/sight_tests/live_tests.rs | 70 ++++++---- .../surface/surface_tests/perception_tests.rs | 128 ++++++++++++++++++ docs/crates/tinycomputer-browser/sight.md | 20 ++- docs/crates/tinycomputer-browser/surface.md | 4 +- docs/technical/specs/browser-sight.md | 3 +- 8 files changed, 310 insertions(+), 45 deletions(-) diff --git a/crates/tinycomputer-browser/src/surface/mod.rs b/crates/tinycomputer-browser/src/surface/mod.rs index 786df16f..c943d56d 100644 --- a/crates/tinycomputer-browser/src/surface/mod.rs +++ b/crates/tinycomputer-browser/src/surface/mod.rs @@ -29,7 +29,7 @@ use std::sync::{Arc, Mutex}; use serde_json::json; use tinycomputer_bus::DesktopResponse; -use tinycomputer_bus::browser::{Action, SessionId, SessionOptions}; +use tinycomputer_bus::browser::{Action, SessionId, SessionOptions, SnapshotRequest}; use tinycomputer_core::Platform; use tinycomputer_core::surface::Screen; use tinycomputer_cursor::ScreenCursor; @@ -255,11 +255,66 @@ impl BrowserSurface { .ok()? .ok()?; let result = reply.get("result")?; - let screen = sight::screen(result)?; + let mut screen = sight::screen(result)?; self.keep_denoised(sight::denoised(result)); + match sight::shadows(result).as_slice() { + [] => {} + [shadow] => self.read_shadow(&id, shadow, &mut screen)?, + // A tree snapshot's refs last until the next one: one host's + // subtree can be read beside sight, not two. + _ => return None, + } Some(screen) } + /// Adds to `screen` what the tree reads under `shadow`'s host: the + /// controls a selector cannot reach, under the label of the layer they + /// draw, after everything sight read. `None` when the subtree cannot be + /// read, so the tree reads the whole page instead. + fn read_shadow( + &self, + id: &SessionId, + shadow: &sight::Shadow, + screen: &mut Screen, + ) -> Option<()> { + let request = SnapshotRequest { + selector: Some(sight::selector(&shadow.host)), + ..SnapshotRequest::default() + }; + let reading = self.browser.snapshot(id, request); + let snapshot = self + .block(async { tokio::time::timeout(READ_TIMEOUT, reading).await }) + .ok()? + .ok()?; + let part = tree::screen(&snapshot.tree, &snapshot.title); + let after = screen + .candidates + .iter() + .chain(&screen.text_nodes) + .map(|node| node.order + 1) + .max() + .unwrap_or_default(); + let placed = |mut node: tinycomputer_core::surface::Candidate| { + if let Some(label) = &shadow.label { + node.path.insert(0, label.clone()); + } + node.order += after; + node + }; + screen + .candidates + .extend(part.candidates.into_iter().map(placed)); + screen + .text_nodes + .extend(part.text_nodes.into_iter().map(placed)); + for line in part.context { + if !screen.context.contains(&line) { + screen.context.push(line); + } + } + Some(()) + } + fn keep_denoised(&self, denoised: Denoised) { if let Ok(mut kept) = self.denoised.lock() { *kept = denoised; diff --git a/crates/tinycomputer-browser/src/surface/sight/mod.rs b/crates/tinycomputer-browser/src/surface/sight/mod.rs index 1405c025..f008e86c 100644 --- a/crates/tinycomputer-browser/src/surface/sight/mod.rs +++ b/crates/tinycomputer-browser/src/surface/sight/mod.rs @@ -51,10 +51,12 @@ //! removes or replaces leaves its ref pointing at nothing, and acting on it //! fails rather than reaching whatever took its place. //! -//! Sight gives way to the tree when it cannot reach what it sees: a control -//! inside a shadow root (shown, even when its host draws no box of its own), -//! or a large frame in front, which a CSS selector from the page cannot -//! address. +//! What a CSS selector from the page cannot address is read through the +//! tree. A shadow root that shows controls (even one whose host draws no +//! box of its own) has its host's subtree read by the tree and merged into +//! the reading, under the label of the layer it draws; sight gives way to +//! the tree for the whole page when two shadow roots show controls, or a +//! large frame is in front. use serde_json::{Value, json}; use tinycomputer_core::surface::{Candidate, Screen}; @@ -149,6 +151,41 @@ pub(crate) fn denoised(result: &Value) -> Denoised { } } +/// A shadow root that shows controls, which a selector from the page cannot +/// address: its host's ref, and the label of the layer it draws over the +/// page, if it draws one (`popover "We value your privacy"`). +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Shadow { + /// The host's ref, minted by sight. + pub(crate) host: String, + /// The container label its controls are read under, if any. + pub(crate) label: Option, +} + +/// The shadow roots a reading saw showing controls, in page order. +#[must_use] +pub(crate) fn shadows(result: &Value) -> Vec { + result + .get("shadows") + .and_then(Value::as_array) + .map(|shadows| { + shadows + .iter() + .filter_map(|shadow| { + let id = shadow.get("id").and_then(Value::as_str)?; + Some(Shadow { + host: format!("{PREFIX}{id}"), + label: shadow + .get("label") + .and_then(Value::as_str) + .map(str::to_owned), + }) + }) + .collect() + }) + .unwrap_or_default() +} + /// What `evaluate` returned as a [`Screen`]; `None` when the reading failed /// or saw a control it cannot reach, so the tree is read instead. #[must_use] diff --git a/crates/tinycomputer-browser/src/surface/sight/sight.js b/crates/tinycomputer-browser/src/surface/sight/sight.js index c496b8c2..a8f2b0f3 100644 --- a/crates/tinycomputer-browser/src/surface/sight/sight.js +++ b/crates/tinycomputer-browser/src/surface/sight/sight.js @@ -906,6 +906,28 @@ const controls = host.shadowRoot.querySelectorAll('a[href], button, input, select, textarea, [role], [tabindex]'); return controls.length > 0 && (shown(host) || [...controls].some(shown)); }; + // The shadow roots that show controls, each as its host's ref, which a + // selector can address, and the label of the layer it draws, if any: the + // controls the tree reads under the host keep the place they show in. + // Live, a consent banner was a fixed layer over the page. + const shadows = []; + const firstWords = (element) => { + const walker = document.createTreeWalker(element, NodeFilter.SHOW_TEXT); + for (let node = walker.nextNode(); node; node = walker.nextNode()) { + const text = squash(node.data); + if (text.split(' ').length >= 3 && node.parentElement && shown(node.parentElement)) return clip(text, 60); + } + return ''; + }; + const shadowLabel = (host) => { + for (const element of host.shadowRoot.querySelectorAll('*')) { + const floating = layer(element); + if (!floating) continue; + const named = labelOf(element) || heading(element) || firstWords(element); + return named ? `${floating} ${JSON.stringify(named)}` : floating; + } + return null; + }; let texts = 0; const insideControl = (element) => { for (let parent = element.parentElement; parent; parent = parent.parentElement) { @@ -1001,7 +1023,7 @@ } if (element.shadowRoot && showsShadowControls(element)) { if (dropped) tally(dropped); - else unreachable += 1; + else shadows.push({ id: mark(element), label: shadowLabel(element) }); } if (tag(element) === 'iframe' && shown(element) && !offscreen(element) && inFront(element)) { const rect = box(element); @@ -1095,5 +1117,5 @@ if (floating === 'alertdialog') { surface = 'alert'; break; } if (floating === 'dialog') { surface = 'sheet'; break; } } - return { ok: true, title: document.title, surface, unreachable, nodes, denoised }; + return { ok: true, title: document.title, surface, unreachable, shadows, nodes, denoised }; }) diff --git a/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_tests.rs b/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_tests.rs index 06a0e419..2b187576 100644 --- a/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_tests.rs +++ b/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_tests.rs @@ -77,23 +77,6 @@ async fn live_results(html: &str, scripts: &[String]) -> Option Option { - let (browser, info) = live_page(html).await?; - let snapshot = browser - .snapshot( - &info.id, - tinycomputer_bus::browser::SnapshotRequest::default(), - ) - .await - .expect("the fixture's tree is read"); - browser.close_session(&info.id).await.unwrap(); - Some(snapshot.tree) -} - /// The names of the controls and the words of the text a reading returned. #[cfg(feature = "agent-browser")] fn shown_names(reading: &serde_json::Value) -> Vec { @@ -309,7 +292,7 @@ async fn live_consent_banners_are_kept() { #[cfg(feature = "agent-browser")] #[tokio::test] -async fn live_a_shadow_roots_shown_controls_give_way_to_the_tree_whatever_its_host_draws() { +async fn live_a_shadow_root_that_shows_controls_is_handed_to_the_tree_under_its_layer() { // Live, a consent banner's host was drawn as `display: contents`, with // no box of its own, and its buttons went unread while the banner lay // over the page. A block host whose banner is fixed draws no box either. @@ -324,26 +307,55 @@ async fn live_a_shadow_roots_shown_controls_give_way_to_the_tree_whatever_its_ho "# ) }; - for (host, banner, unreachable) in [ - ("display: contents", "", 1), - ("display: block", "", 1), - ("display: contents", "display: none", 0), + for (host, banner, shown) in [ + ("display: contents", "", true), + ("display: block", "", true), + ("display: contents", "display: none", false), ] { let Some(reading) = live_reading(&page(host, banner)).await else { return; }; + assert_eq!(reading["unreachable"], 0, "{reading}"); + let shadows = reading["shadows"].as_array().unwrap(); assert_eq!( - reading["unreachable"], unreachable, + shadows.len(), + usize::from(shown), "host {host:?}, banner {banner:?}: {reading}" ); + if shown { + assert_eq!( + shadows[0]["label"], "popover \"We value your privacy\"", + "{reading}" + ); + } } - // The tree the surface reads instead offers the banner's buttons. - let tree = live_tree(&page("display: contents", "")) + // The tree read under the host offers the banner's buttons, and only + // them. + let (browser, info) = live_page(&page("display: contents", "")) .await - .expect("a live run reads the tree too"); - for button in ["Allow Selection", "Allow all", "Add To Cart"] { - assert!(tree.contains(button), "{button} in {tree}"); - } + .expect("a live run opens the page"); + let reading = browser + .command( + &info.id, + json!({"action": "evaluate", "script": script(None)}), + ) + .await + .unwrap()["result"] + .clone(); + let host = reading["shadows"][0]["id"].as_str().unwrap().to_owned(); + let subtree = browser + .snapshot( + &info.id, + tinycomputer_bus::browser::SnapshotRequest { + selector: Some(format!("[data-tc-seen=\"{host}\"]")), + ..tinycomputer_bus::browser::SnapshotRequest::default() + }, + ) + .await + .unwrap(); + browser.close_session(&info.id).await.unwrap(); + assert!(subtree.tree.contains("Allow Selection"), "{}", subtree.tree); + assert!(!subtree.tree.contains("Add To Cart"), "{}", subtree.tree); } #[cfg(feature = "agent-browser")] diff --git a/crates/tinycomputer-browser/src/surface/surface_tests/perception_tests.rs b/crates/tinycomputer-browser/src/surface/surface_tests/perception_tests.rs index 15484078..0a35d321 100644 --- a/crates/tinycomputer-browser/src/surface/surface_tests/perception_tests.rs +++ b/crates/tinycomputer-browser/src/surface/surface_tests/perception_tests.rs @@ -120,3 +120,131 @@ fn the_tree_is_read_when_sight_fails_or_is_turned_off() { assert!(!fake.actions().iter().any(|action| action == "evaluate")); assert!(format!("{surface:?}").contains("Tree")); } + +/// A page read by sight whose `shadows` show controls: a covered "Add To +/// Cart", and under a host the tree reads a consent banner's buttons, or +/// fails to when `subtree_fails`. +fn shadowed_fake(shadows: serde_json::Value, subtree_fails: bool) -> Fake { + Fake::scripted(move |command| match command["action"].as_str().unwrap() { + "evaluate" + if command["script"] + .as_str() + .unwrap() + .contains("__tinycomputerSeen") => + { + Some(ok(&json!({"result": { + "ok": true, + "title": "Glasses", + "surface": "window", + "unreachable": 0, + "shadows": shadows, + "denoised": {"ads": 0, "empty": 0, "hidden": 0}, + "nodes": [ + {"id": "1", "role": "button", "name": "Add To Cart", "states": ["covered"], "path": ["main"]}, + {"text": "Limited Period Offer", "path": ["main"]} + ] + }}))) + } + "snapshot" if command.get("selector").is_some() && subtree_fails => { + Some(failure("no such element")) + } + "snapshot" if command.get("selector").is_some() => Some(ok(&json!({ + "snapshot": "- generic\n - paragraph\n - StaticText \"We value your privacy\"\n - button \"Allow Selection\" [ref=e2]\n - button \"Allow all\" [ref=e3]", + "refs": { + "e2": {"role": "button", "name": "Allow Selection"}, + "e3": {"role": "button", "name": "Allow all"} + } + }))), + _ => None, + }) +} + +#[test] +fn a_shadow_roots_controls_are_read_by_the_tree_beside_sight() { + // Live, a consent banner in a shadow root lay over "Add To Cart": sight + // could not read it, and giving the whole page to the tree read the + // rest of the page worse. + let banner = json!([{"id": "9", "label": "popover \"We value your privacy\""}]); + let Harness { fake, surface, .. } = harness("shadow-merged", shadowed_fake(banner, false)); + let screen = surface.observe("", None, Depth::Full).unwrap(); + let names = screen + .candidates + .iter() + .map(|candidate| { + ( + candidate.ref_id.as_str(), + candidate.name.as_deref().unwrap_or_default(), + ) + }) + .collect::>(); + assert_eq!( + names, + [ + ("seen:1", "Add To Cart"), + ("e2", "Allow Selection"), + ("e3", "Allow all") + ] + ); + let allow = &screen.candidates[1]; + assert_eq!( + allow.path[0], "popover \"We value your privacy\"", + "{:?}", + allow.path + ); + assert!( + allow.order > screen.candidates[0].order, + "read after sight's nodes" + ); + assert!( + screen + .context + .iter() + .any(|line| line.contains("We value your privacy")) + ); + let subtree = fake.last("snapshot"); + assert_eq!(subtree["selector"], r#"[data-tc-seen="9"]"#, "{subtree}"); + + // A shadow root that draws no layer keeps the tree's own places. + let plain = json!([{"id": "9", "label": null}]); + let Harness { surface, .. } = harness("shadow-plain", shadowed_fake(plain, false)); + let screen = surface.observe("", None, Depth::Full).unwrap(); + assert!( + !screen.candidates[1] + .path + .first() + .is_some_and(|label| label.starts_with("popover")), + "{:?}", + screen.candidates[1].path + ); +} + +#[test] +fn the_tree_reads_the_page_when_two_shadow_roots_show_or_one_cannot_be_read() { + let two = json!([{"id": "9", "label": null}, {"id": "10", "label": null}]); + let Harness { fake, surface, .. } = harness("shadow-two", shadowed_fake(two, false)); + let screen = surface.observe("", None, Depth::Full).unwrap(); + assert!( + screen + .candidates + .iter() + .all(|candidate| !candidate.ref_id.starts_with("seen:")) + ); + assert!( + fake.last("snapshot").get("selector").is_none(), + "the whole page" + ); + + let one = json!([{"id": "9", "label": null}]); + let Harness { fake, surface, .. } = harness("shadow-failed", shadowed_fake(one, true)); + let screen = surface.observe("", None, Depth::Full).unwrap(); + assert!( + screen + .candidates + .iter() + .all(|candidate| !candidate.ref_id.starts_with("seen:")) + ); + assert!( + fake.last("snapshot").get("selector").is_none(), + "the whole page" + ); +} diff --git a/docs/crates/tinycomputer-browser/sight.md b/docs/crates/tinycomputer-browser/sight.md index 7553c852..7459391f 100644 --- a/docs/crates/tinycomputer-browser/sight.md +++ b/docs/crates/tinycomputer-browser/sight.md @@ -245,12 +245,20 @@ signal, not something a flow decides on. ## What sight can't reach Sight gives way to the accessibility tree (`tree.rs`, see below) when the -reading fails outright, or when it sees a control inside a shadow root, or -behind a frame that covers a large share of the viewport, both cases where -a plain CSS selector from the top-level page cannot address the element -sight found. A shadow root's host need not draw a box of its own: one laid -out as `display: contents` has none, and a consent banner's host was one, -so a shadow root counts once its host or any of its controls shows. Reading through the tree in those cases is deliberately +reading fails outright, or when it sees a frame that covers a large share of +the viewport, or controls inside two shadow roots: cases where a plain CSS +selector from the top-level page cannot address the element sight found. + +One shadow root that shows controls is read beside sight instead. Sight +marks its host and names the layer its controls draw (`popover "We value +your privacy"`), and the surface reads the tree under the host alone and +adds its controls, under that label, after everything sight read. A tree +snapshot's refs last until the next snapshot, so only one host's subtree +can be read beside sight. A shadow root counts once its host or any of its +controls shows: a host laid out as `display: contents` has no box of its +own, and live, a consent banner's host was one. Before, its buttons went +unread while the banner lay over "Add To Cart"; giving the whole page to +the tree read the rest of the page worse. Reading through the tree is deliberately unglamorous: it is the same fallback the crate always had, just demoted from "the only way" to "the way out when sight cannot help." diff --git a/docs/crates/tinycomputer-browser/surface.md b/docs/crates/tinycomputer-browser/surface.md index 3011c121..c36ba33c 100644 --- a/docs/crates/tinycomputer-browser/surface.md +++ b/docs/crates/tinycomputer-browser/surface.md @@ -42,7 +42,9 @@ tradeoff rather than a smell. `Perception::Sight` (the default) or `Perception::Tree`. `observe` tries sight first when it is enabled, and only reads the accessibility tree snapshot when sight is turned off, fails outright, or hits something it -cannot address (a shadow root, a large frame in front). See +cannot address (two shadow roots showing controls, a large frame in front); +one shadow root's controls are read from the tree under its host and added +to sight's reading. See [sight.md](sight.md) for what each of those actually does. ## Executing an operation diff --git a/docs/technical/specs/browser-sight.md b/docs/technical/specs/browser-sight.md index 8357c074..756a354a 100644 --- a/docs/technical/specs/browser-sight.md +++ b/docs/technical/specs/browser-sight.md @@ -40,7 +40,8 @@ a caret. - Vision: nothing reads pixels. An icon with no words, no alternative text, and no telling class stays unnamed. - Shadow roots and frames, which a CSS selector from the page cannot reach: - the tree is read instead (below). + one shadow root's controls are read from the tree under its host, beside + sight; otherwise the tree reads the page (below). - Changing agent-browser. Sight runs through its existing `evaluate` command, and acts through its existing CSS-selector targets. From ba5ad7ef2cea831c56f96603c9e2e7f79dfcc6aa Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 7 Oct 2026 18:09:00 +0530 Subject: [PATCH 15/44] Close the layer in front with its own button when a click is refused A press the page refused as covered was answered with Escape and one more try. Escape leaves a consent banner where it is: on Lenskart every press of "Add To Cart" and "Frame Size" under the banner was refused until the run failed. When a layer in front holds a control that closes it, the least committal one ("Allow Selection" before "Allow all", as the attention pass ranks them) is now pressed instead, as `click (uncover)`, and the same target tried again; otherwise Escape, as before. The simulator gains a consent banner Escape does not close. --- .../src/agentic/flow/act/moves.rs | 39 ++++++++---- .../src/agentic/flow/attention/find.rs | 2 + .../src/agentic/flow/attention/mod.rs | 21 ++++++- .../agentic/flow/flow_tests/do_loop_tests.rs | 36 +++++++++++ .../src/agentic/flow/flow_tests/screens.rs | 14 +++++ .../src/agentic/flow/flow_tests/simulator.rs | 59 +++++++++++++------ docs/technical/decision-loops.md | 10 +++- 7 files changed, 150 insertions(+), 31 deletions(-) diff --git a/crates/tinycomputer-engine/src/agentic/flow/act/moves.rs b/crates/tinycomputer-engine/src/agentic/flow/act/moves.rs index 72f465e7..fadd8034 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/act/moves.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/act/moves.rs @@ -9,9 +9,10 @@ use tinycomputer_bus::{JevOperation, StepOutcome}; use crate::agentic::flow::{ Ended, FlowRun, Halt, StepLog, ask::{self, Questions, chosen}, + attention::front_closer, backend::AgentBackend, memory::{learn, remember}, - view::{Candidate, Screen, element_kind, is_banned, is_destructive, label}, + view::{Candidate, Screen, element_kind, is_banned, is_destructive, label, signature}, }; use super::{ @@ -271,15 +272,33 @@ impl FlowRun<'_, B> { )); return Ok(reply); } - let app = self.app.clone(); - self.act(log, "press escape (uncover)", None, move |backend| { - backend.press(&app, "escape") - }) - .await?; - self.history.push(format!( - "{} was covered by something; pressed escape to close it", - label(target) - )); + // A layer in front that closes with a control of its own (a consent + // banner's "Allow Selection") is closed with it: Escape leaves such a + // banner where it is. + let screen = self.look().await?; + if let Some(closer) = front_closer(&screen, &self.stop_before, &self.step_cleared) { + self.step_cleared.insert(signature(&closer)); + let pressed = closer.clone(); + self.act(log, "click (uncover)", Some(&closer), move |backend| { + backend.execute(JevOperation::Click, Some(pressed), None) + }) + .await?; + self.history.push(format!( + "{} was covered by a layer in front; pressed {} to close it", + label(target), + label(&closer) + )); + } else { + let app = self.app.clone(); + self.act(log, "press escape (uncover)", None, move |backend| { + backend.press(&app, "escape") + }) + .await?; + self.history.push(format!( + "{} was covered by something; pressed escape to close it", + label(target) + )); + } let retried = target.clone(); self.act(log, verb, Some(target), move |backend| { backend.execute(operation, Some(retried), None) diff --git a/crates/tinycomputer-engine/src/agentic/flow/attention/find.rs b/crates/tinycomputer-engine/src/agentic/flow/attention/find.rs index 88e94a85..007cc318 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/attention/find.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/attention/find.rs @@ -218,6 +218,7 @@ pub(in crate::agentic::flow) fn distractions( .map(|(_, member)| label(member)) .collect(), closer: Some(closer.clone()), + front, }, )); } @@ -305,6 +306,7 @@ fn covering(screen: &Screen, intent: &[String], cleared: &BTreeSet) -> O name: "something open over the page".to_owned(), shows: needed.into_iter().chain(front).take(6).collect(), closer: None, + front: true, }) } diff --git a/crates/tinycomputer-engine/src/agentic/flow/attention/mod.rs b/crates/tinycomputer-engine/src/agentic/flow/attention/mod.rs index 5a66d5f0..cb4057b8 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/attention/mod.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/attention/mod.rs @@ -24,7 +24,7 @@ mod find; use std::collections::BTreeSet; -use super::view::Candidate; +use super::view::{Candidate, Screen}; /// Most distractions one attention question offers. pub(super) const MAX_DISTRACTIONS: usize = 4; @@ -49,6 +49,25 @@ pub(super) struct Distraction { /// for something that covers the page with no control of its own, which /// Escape clears. pub(super) closer: Option, + /// Whether it lies in front of the page (the digest's front regions). + pub(super) front: bool, +} + +/// The control that closes a layer in front of the page, the least +/// committal its region holds (a consent banner's "Allow Selection" before +/// its "Allow all"), when one does and it was not pressed in this step: what +/// a press the page refused as covered clears before trying again. Live, a +/// consent banner lay over "Add To Cart", Escape left it there, and every +/// press was refused. +pub(super) fn front_closer( + screen: &Screen, + stop_before: &[String], + cleared: &BTreeSet, +) -> Option { + find::distractions(screen, "", stop_before, cleared) + .into_iter() + .filter(|distraction| distraction.front) + .find_map(|distraction| distraction.closer) } /// The key a step's Escape at something covering the page is remembered diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs index cd64c13d..c7319a6c 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs @@ -111,6 +111,42 @@ async fn a_covered_click_closes_what_covers_it_and_tries_again() { ); } +#[tokio::test] +async fn a_covered_click_closes_the_banner_in_front_with_its_own_button() { + // Live, a consent banner lay over "Add To Cart"; Escape left it there, + // and every press was refused. Its least committal button closes it. + let run = run_with( + App::quirky(Quirk::ConsentBanner), + json!({"app": "Mail", "steps": ["start a new email message"]}), + |request| request.disabled_loops.push(FlowLoop::Attention), + |id, question, _| (id == "move").then(|| pick(question, "activate", 0.9)), + ) + .await; + assert_eq!( + run.result.stop, + FlowStopReason::Completed, + "{:?}", + run.result.steps + ); + let sim = run.app.sim(); + assert!(sim.compose_open); + assert!(sim.presses.is_empty(), "no Escape: {:?}", sim.presses); + assert!( + sim.clicks.contains(&"Allow Selection".to_owned()) + && !sim.clicks.contains(&"Allow all".to_owned()), + "{:?}", + sim.clicks + ); + let actions = &run.result.steps[0].actions; + assert_eq!( + actions + .iter() + .map(|action| action.action.as_str()) + .collect::>(), + ["click", "click (uncover)", "click"], + ); +} + #[tokio::test] async fn a_regression_is_undone_and_the_element_is_not_tried_again() { let run = run_with( diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/screens.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/screens.rs index 89b5c475..5779d16c 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/screens.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/screens.rs @@ -421,6 +421,20 @@ pub(super) fn overlays(sim: &Sim, root: &str, candidates: &mut Vec) { candidates.push(node("Close", "button", &["Click"], &toast, 700.0)); candidates.push(node("Learn more", "link", &["Click"], &toast, 720.0)); } + if sim.has(Quirk::ConsentBanner) { + for candidate in candidates.iter_mut() { + candidate.states = vec!["covered".to_owned()]; + } + let banner = [root, "popover \"We value your privacy\""]; + candidates.push(node( + "Allow Selection", + "button", + &["Click"], + &banner, + 740.0, + )); + candidates.push(node("Allow all", "button", &["Click"], &banner, 760.0)); + } } /// The inbox's search behind its "Search mail" link, beside a "Contact us" diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs index 5695a682..10c181c0 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs @@ -35,6 +35,10 @@ pub(super) enum Quirk { DisabledArchive, /// A promo toast with a Close button sits over the page until closed. PromoToast, + /// A consent banner lies over the page as a popover: every other click + /// is refused as covered, Escape leaves it, and its "Allow Selection" + /// or "Allow all" closes it. + ConsentBanner, /// Text typed with no target lands at the end of the field typed into /// last, as a browser keeps the focus there. FocusStays, @@ -252,6 +256,40 @@ impl App { } } +/// The reply to a click on `name` that something on the simulated page +/// refuses or takes over, or `None` when the click goes through: an +/// unclickable page, a consent banner whose own buttons close it, or a +/// drawer over everything. +fn refused_click(sim: &mut Sim, name: &str) -> Option { + let covered = |by: &str| { + DesktopResponse::err( + "click", + tinycomputer_bus::DesktopError::new( + "NOT_ACTIONABLE", + format!("Element '@s:{name}' is covered by at its click point"), + ), + ) + }; + if sim.has(Quirk::Unclickable) { + return Some(DesktopResponse::err( + "click", + tinycomputer_bus::DesktopError::new( + "NOT_ACTIONABLE", + "Element exists but is not visible.", + ), + )); + } + if sim.has(Quirk::ConsentBanner) { + if name.starts_with("Allow ") { + sim.clicks.push(name.to_owned()); + sim.quirks.remove(&Quirk::ConsentBanner); + return Some(DesktopResponse::ok("click", json!({}))); + } + return Some(covered("consent")); + } + sim.has(Quirk::Drawer).then(|| covered("drawer")) +} + impl AgentBackend for App { fn observe( &self, @@ -298,23 +336,10 @@ impl AgentBackend for App { .as_ref() .and_then(|target| target.name.clone()) .unwrap_or_default(); - if sim.has(Quirk::Unclickable) && operation == JevOperation::Click { - return DesktopResponse::err( - "click", - tinycomputer_bus::DesktopError::new( - "NOT_ACTIONABLE", - "Element exists but is not visible.", - ), - ); - } - if sim.has(Quirk::Drawer) && operation == JevOperation::Click { - return DesktopResponse::err( - "click", - tinycomputer_bus::DesktopError::new( - "NOT_ACTIONABLE", - format!("Element '@s:{name}' is covered by at its click point"), - ), - ); + if operation == JevOperation::Click + && let Some(reply) = refused_click(&mut sim, &name) + { + return reply; } if is_city_row(target.as_ref()) && operation == JevOperation::TypeText { return not_a_text_field(); diff --git a/docs/technical/decision-loops.md b/docs/technical/decision-loops.md index 965e54cf..23e4d7d2 100644 --- a/docs/technical/decision-loops.md +++ b/docs/technical/decision-loops.md @@ -298,9 +298,13 @@ own link, so the link is "covered" by the card itself; the browser surface then clicks through at the link's position, but only when the exact target (matched by name, and on the page, by the one element under that point with that label) sits in the same card as the cover and no dialog is involved. -Anything else comes back covered, and the runtime presses Escape once and -retries the *same* already-vetted target — in a `do` step's click and in -`pick`'s alike. Escape never chooses a new element, so nothing exposed by +Anything else comes back covered, and the runtime closes what lies over it +and retries the *same* already-vetted target — in a `do` step's click and in +`pick`'s alike. A layer in front with a control that closes it (a consent +banner's "Allow Selection") is closed with its least committal one, as the +attention pass would; otherwise the runtime presses Escape once. Live, +Escape left a consent banner over "Add To Cart" and every press was refused. +Neither chooses a new element for the step, so nothing exposed by dismissing whatever covered the click is ever pressed without going through grounding and `is_destructive` again on a later turn. From 5a5518395d05512ef8f05184993adec9f4c2e187 Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 7 Oct 2026 18:34:20 +0530 Subject: [PATCH 16/44] Send a framing's copy after 4 s, not 2.5 s On a slow evening the Tiny Humans route answered calls in up to 3.4 s (median 0.91 s, against 0.72 s earlier in the day), and a copy sent at 2.5 s lost the race to its original 15 times in 16: 5% more calls for nothing. A framing now gets its copy after 4 s, or 5 s for a request of 32 KB or more (p99.9 3.9 s), past what a slow but live answer takes; a stuck one still answers in about 5 s rather than 12-32 s. --- .../src/agentic/flow/decide.rs | 14 ++++++++------ .../src/agentic/flow/flow_tests/hedge_tests.rs | 17 +++++++++-------- docs/crates/tinycomputer/configuration.md | 2 +- docs/technical/decision-thresholds.md | 2 +- docs/technical/jev-harness.md | 2 +- 5 files changed, 20 insertions(+), 17 deletions(-) diff --git a/crates/tinycomputer-engine/src/agentic/flow/decide.rs b/crates/tinycomputer-engine/src/agentic/flow/decide.rs index eee46fef..ea7fe984 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/decide.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/decide.rs @@ -19,14 +19,16 @@ use super::{ use crate::agentic::{JevRuntime, journal::millis, merge_metrics, provider_error}; /// How long a framing runs before a copy of it is sent and the first answer -/// of the two taken. Live, a call's p99 was 2.0 s, while one framing of a -/// burst stalled 12–32 s (the gateway gave up after ~10 s, or nothing came -/// back before the client's timeout) as its siblings answered in under 1 s. -const HEDGE_AFTER: Duration = Duration::from_millis(2_500); +/// of the two taken. Live, one framing of a burst stalled 12–32 s (the +/// gateway gave up after ~10 s, or nothing came back before the client's +/// timeout) as its siblings answered in under 1 s. Calls on a slow evening +/// took up to 3.4 s and still answered: a copy sent at 2.5 s lost the race +/// 15 times in 16, so copies wait for 4 s, past what a slow answer takes. +const HEDGE_AFTER: Duration = Duration::from_millis(4_000); /// [`HEDGE_AFTER`] for a request of [`HEDGE_LARGE_BYTES`] or more, whose -/// p99 was 3.1 s live. -const HEDGE_AFTER_LARGE: Duration = Duration::from_millis(3_500); +/// p99.9 was 3.9 s live. +const HEDGE_AFTER_LARGE: Duration = Duration::from_millis(5_000); /// Size from which a request waits [`HEDGE_AFTER_LARGE`] for its first /// answer. diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/hedge_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/hedge_tests.rs index 63f0d5cd..59253588 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/hedge_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/hedge_tests.rs @@ -86,13 +86,14 @@ async fn a_stalled_framing_is_answered_by_its_copy() { let answer = hedged(&runtime, "1", &request(1_000)).await.unwrap(); assert_eq!(answer.response.model, "call 1", "the copy answered"); assert_eq!(answer.attempts, 2, "the copy is an attempt of its own"); - assert_eq!(started.elapsed(), Duration::from_millis(3_100)); + assert_eq!(started.elapsed(), Duration::from_millis(4_600)); assert_eq!(*paced.calls.lock().unwrap(), 2); } #[tokio::test(start_paused = true)] async fn a_framing_that_answers_in_time_gets_no_copy() { - let (runtime, paced) = paced(&[(2_400, false)]); + // Live, a slow evening's calls took up to 3.4 s and still answered. + let (runtime, paced) = paced(&[(3_900, false)]); let answer = hedged(&runtime, "1", &request(1_000)).await.unwrap(); assert_eq!( (answer.response.model.as_str(), answer.attempts), @@ -103,8 +104,8 @@ async fn a_framing_that_answers_in_time_gets_no_copy() { #[tokio::test(start_paused = true)] async fn a_large_request_waits_longer_before_its_copy() { - // A 32 KB request's p99 was 3.1 s live: at 3 s it still gets no copy. - let (runtime, paced) = paced(&[(3_000, false)]); + // A 32 KB request's p99.9 was 3.9 s live: at 4.9 s it still gets no copy. + let (runtime, paced) = paced(&[(4_900, false)]); let answer = hedged(&runtime, "1", &request(40_000)).await.unwrap(); assert_eq!(answer.attempts, 1); assert_eq!(*paced.calls.lock().unwrap(), 1); @@ -113,17 +114,17 @@ async fn a_large_request_waits_longer_before_its_copy() { #[tokio::test(start_paused = true)] async fn a_failed_copy_gives_way_and_two_failures_fail() { // The copy fails at once; the slow first answer still counts. - let (runtime, _) = paced(&[(4_000, false), (0, true)]); + let (runtime, _) = paced(&[(6_000, false), (0, true)]); let started = tokio::time::Instant::now(); let answer = hedged(&runtime, "1", &request(1_000)).await.unwrap(); assert_eq!(answer.response.model, "call 0"); - assert_eq!(started.elapsed(), Duration::from_millis(4_000)); + assert_eq!(started.elapsed(), Duration::from_millis(6_000)); // The first fails after its copy was sent; the copy's answer counts. - let (runtime, _) = paced(&[(3_000, true), (1_000, false)]); + let (runtime, _) = paced(&[(4_500, true), (1_000, false)]); let answer = hedged(&runtime, "1", &request(1_000)).await.unwrap(); assert_eq!(answer.response.model, "call 1"); - let (runtime, _) = paced(&[(3_000, true), (1_000, true)]); + let (runtime, _) = paced(&[(4_500, true), (1_000, true)]); assert!(hedged(&runtime, "1", &request(1_000)).await.is_err()); } diff --git a/docs/crates/tinycomputer/configuration.md b/docs/crates/tinycomputer/configuration.md index 8f7d8817..9054de0c 100644 --- a/docs/crates/tinycomputer/configuration.md +++ b/docs/crates/tinycomputer/configuration.md @@ -80,7 +80,7 @@ decisions to go through Tiny Humans uses `tiny_humans_open_router`. `sdk_name` is sent only to the Tiny Humans proxy. Each attempt may take `timeout_ms`, 10 seconds unless set: live, the slowest -answer took 8.6 s. A framing that has not answered after 2.5 s (3.5 s for a +answer took 8.6 s. A framing that has not answered after 4 s (5 s for a request of 32 KB or more) is also sent once more, and whichever copy answers first counts. A provider's server error (HTTP 5xx) or rate limit (429) is retried, waiting 1, 2, 4, then 8 seconds between attempts, or as long as the diff --git a/docs/technical/decision-thresholds.md b/docs/technical/decision-thresholds.md index 50925e5b..81567e9f 100644 --- a/docs/technical/decision-thresholds.md +++ b/docs/technical/decision-thresholds.md @@ -51,7 +51,7 @@ Change a constant and its row together. | `LAYER_COVERS` | 3 | `front.rs` | controls something drawn over the window must cover, beyond what was covered before the press that opened it, on the same page, before it counts as a dialog the task opened (`surface` `layer`); a step that pressed inside such a dialog hands it back at the next step, and opening an address forgets it | | `FRONT_CONTROLS` | 8 | `act/turns.rs` | most controls of the task's dialog in front a failed step's note names, so a rescue answers with one of them | | `STEADY_HOLD` / `STEADY_CHECKS` | 0.65 / 3 | `steps/mod.rs` | belief a `wait_for` condition must keep, on checks in a row of one unchanged screen, to be taken as held under `DONE` | -| `HEDGE_AFTER` / `HEDGE_AFTER_LARGE` | 2.5 s / 3.5 s | `decide.rs` | how long a framing runs before a copy of it is sent and the first answer of the two taken; the longer wait is for a request of `HEDGE_LARGE_BYTES` (32 KB) or more | +| `HEDGE_AFTER` / `HEDGE_AFTER_LARGE` | 4 s / 5 s | `decide.rs` | how long a framing runs before a copy of it is sent and the first answer of the two taken; the longer wait is for a request of `HEDGE_LARGE_BYTES` (32 KB) or more | | `LATE_LOOKS` | 2 | `steps/suggestion.rs` | looks again, after a wait for the page to change, for the suggestions a place box lists late, before its text is left as typed; a page that stayed still through a wait lists nothing more | | `LATE_LOOK_MS` | 1000 ms | `steps/suggestion.rs` | longest one of those waits: it ends as soon as the page changes (`Surface::await_change`) | | `BARE_CHARS` | 3 | `tinycomputer-core` `surface/groups.rs` | most letters and digits each field of a card may show for a list of such cards to be bare markers (carousel dots, size chips, page numbers), never results | diff --git a/docs/technical/jev-harness.md b/docs/technical/jev-harness.md index 8abee735..99539725 100644 --- a/docs/technical/jev-harness.md +++ b/docs/technical/jev-harness.md @@ -155,7 +155,7 @@ round trip's *slowest* framing, plus the action, plus settling. The levers: | Lever | Effect on latency | Effect on accuracy | |---|---|---| | `strategy` | `wide` asks one request per `do` turn instead of two to seven in sequence | the digest, survey, and memory show more of what matters; measure with the lab's `--strategy` | -| `votes` | a decision waits for its slowest framing: more framings, longer tail; a framing slower than 2.5 s (3.5 s at 32 KB or more) gets a copy, and the first answer counts | more framings average out position and phrasing bias | +| `votes` | a decision waits for its slowest framing: more framings, longer tail; a framing slower than 4 s (5 s at 32 KB or more) gets a copy, and the first answer counts | more framings average out position and phrasing bias | | request size | Jev's latency grows with input tokens; a big element list is the usual cause | trimming can drop the element that was needed | | grounding memory | a remembered element is confirmed with one Noul instead of narrowing | none when the hint is right | | `disabled_loops` | each loop off removes a question or a whole decision | measure it before shipping it off | From c0a48ed9a81e2123a48504f408aecc4947c51236 Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 7 Oct 2026 18:34:20 +0530 Subject: [PATCH 17/44] End a stalled step as already done when its result shows A `do` step whose last three actions changed nothing failed outright, even when the page had already done its work: "press Enter to search" pressed Enter three times over results a live search had listed as the query was typed. On 7 October, 25 of 189 rescues found such a step's work already done, each ~18 s after the failure. Such a step now asks one question first: does the screen already show the result the step is meant to bring about? When it clearly does (DONE), the step ends AlreadyDone; otherwise it fails with the same note as before, so a rescue still skips rather than retries it. --- .../src/agentic/flow/act/turns.rs | 61 +++++++++++++------ .../agentic/flow/flow_tests/do_loop_tests.rs | 34 +++++++++++ docs/catching-mistakes.md | 5 +- .../tinycomputer-engine/flow/the-do-loop.md | 10 ++- 4 files changed, 88 insertions(+), 22 deletions(-) diff --git a/crates/tinycomputer-engine/src/agentic/flow/act/turns.rs b/crates/tinycomputer-engine/src/agentic/flow/act/turns.rs index 06898b29..ae44097c 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/act/turns.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/act/turns.rs @@ -83,10 +83,8 @@ impl FlowRun<'_, B> { state.turn = Some((turn, Instant::now(), self.decisions, self.rounds)); log.turns = log.turns.saturating_add(1); let screen = self.look().await?; - self.note_change(state, &screen)?; - self.note_oscillation(log, state, &screen); - if state.first.is_none() { - state.first = Some(screen.clone()); + if self.note_turn(log, state, &screen) { + return self.stalled(log, intent).await; } if let Some(ended) = state .last @@ -274,15 +272,29 @@ impl FlowRun<'_, B> { } } + /// Notes what the turn's look shows: what the last action changed, an + /// oscillation, and the step's first screen; `true` once [`STALL_TURNS`] + /// turns in a row changed nothing. + fn note_turn(&mut self, log: &mut StepLog, state: &mut DoState, screen: &Screen) -> bool { + if self.note_change(state, screen) { + return true; + } + self.note_oscillation(log, state, screen); + if state.first.is_none() { + state.first = Some(screen.clone()); + } + false + } + /// Records what the last action changed, banning an element that changed - /// nothing and failing the step after [`STALL_TURNS`] such turns. + /// nothing; `true` once [`STALL_TURNS`] such turns ran in a row. /// /// A wait that changes nothing is not a stall: the page has settled, and /// Jev is told so. It is not let wait again after [`MAX_IDLE_WAITS`] of /// them, which leaves it to judge or act on the page as it stands. - fn note_change(&mut self, state: &mut DoState, screen: &Screen) -> Result<(), Halt> { + fn note_change(&mut self, state: &mut DoState, screen: &Screen) -> bool { let Some(previous) = &state.last else { - return Ok(()); + return false; }; let changed = fingerprint(&previous.before) != fingerprint(screen); let note = change_note(&previous.before, screen, changed); @@ -296,7 +308,7 @@ impl FlowRun<'_, B> { "waited: the page has finished loading and nothing changed, so waiting longer will not change it" .to_owned(), ); - return Ok(()); + return false; } else { state.unchanged = state.unchanged.saturating_add(1); if previous.scrolled { @@ -318,17 +330,32 @@ impl FlowRun<'_, B> { if let Some(struck) = copies::strike_pending(state, changed) { self.history.push(struck); } - // A step whose work the page did by itself (a search box that lists - // results as it is typed in) has nothing left to press: the note - // says so, so a rescue skips it rather than retry it. Live, four - // rescues looked for a search button a live search does not have. - if state.unchanged >= STALL_TURNS { - return Err(Halt::Failed( - "the last three actions changed nothing on screen; if the screen already shows what this step was for, its work is done" - .to_owned(), + state.unchanged >= STALL_TURNS + } + + /// Ends a step whose last [`STALL_TURNS`] actions changed nothing: done, + /// when the screen already shows what the step was for, else failed. + /// + /// A step whose work the page did by itself (a search box that lists + /// results as it is typed in) has nothing left to press. Live, rescues + /// looked for a search button a live search does not have, and 25 rescues + /// of 189 on one day found the step's work already done, each after + /// ~18 s; the question here costs one decision. A failure's note still + /// says so, so a rescue skips the step rather than retry it. + async fn stalled(&mut self, log: &mut StepLog, intent: &str) -> Result { + let condition = format!( + "the screen already shows the result that the step {intent:?} is meant to bring about" + ); + if self.holds(log, &condition).await? >= DONE { + return Ok(Ended::new( + StepOutcome::AlreadyDone, + "the last three actions changed nothing, and the screen already shows what this step was for", )); } - Ok(()) + Err(Halt::Failed( + "the last three actions changed nothing on screen; if the screen already shows what this step was for, its work is done" + .to_owned(), + )) } } diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs index c7319a6c..9a67c8c2 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs @@ -193,6 +193,40 @@ async fn actions_that_change_nothing_fail_the_step() { assert!(run.result.steps[0].note.contains("changed nothing")); } +#[tokio::test] +async fn a_stalled_step_whose_result_already_shows_is_done() { + // Live, "press Enter to search" pressed Enter three times over results a + // live search had already listed, failed, and a rescue found its work + // done ~18 s later: 25 of a day's 189 rescues were such steps. + let run = run_with( + App::quirky(Quirk::Frozen), + json!({"app": "Mail", "steps": ["press the search button"]}), + |_| {}, + |id, question, _| match id { + "done" => Some(noul(0.05)), + "move" => Some(pick(question, "activate", 0.9)), + "holds" if text_of(question, "condition").contains("already shows the result") => { + Some(noul(0.95)) + } + _ => None, + }, + ) + .await; + let step = &run.result.steps[0]; + assert_eq!( + run.result.stop, + FlowStopReason::Completed, + "{:?}", + run.result.steps + ); + assert_eq!(step.outcome, StepOutcome::AlreadyDone, "{}", step.note); + assert!( + step.note.contains("already shows what this step was for"), + "{}", + step.note + ); +} + #[tokio::test] async fn an_irreversible_control_is_refused_inside_an_ordinary_step() { let run = run_with( diff --git a/docs/catching-mistakes.md b/docs/catching-mistakes.md index 497000d5..caf58ca1 100644 --- a/docs/catching-mistakes.md +++ b/docs/catching-mistakes.md @@ -13,8 +13,9 @@ with every look, so it only notices real changes. - **Nothing changed.** The control that was pressed is banned for the rest of the step, so it isn't pressed again. After three actions in a row that - change nothing, the step fails with "the last three actions changed nothing - on screen". + change nothing, Jev is asked whether the screen already shows what the step + was for. If it does, the step counts as already done; if not, the step fails + with "the last three actions changed nothing on screen". - **Something changed.** A short note goes into the history Jev sees, like "the window is now New Message; appeared: textfield To:". That's how Jev learns what its last choice did. diff --git a/docs/crates/tinycomputer-engine/flow/the-do-loop.md b/docs/crates/tinycomputer-engine/flow/the-do-loop.md index 479e7f57..1aa54c19 100644 --- a/docs/crates/tinycomputer-engine/flow/the-do-loop.md +++ b/docs/crates/tinycomputer-engine/flow/the-do-loop.md @@ -28,9 +28,13 @@ refs, the ids each snapshot mints fresh: a fingerprint that included them would look different on every single turn, and stall detection would never fire. If nothing changed, the element that was just pressed is banned for the rest of the step, so the loop does not click the same dead button -twice. Three turns in a row with no change (`STALL_TURNS`) fail the step -outright, with the note "the last three actions changed nothing on -screen." +twice. After three turns in a row with no change (`STALL_TURNS`), one +question asks whether the screen already shows the result the step is +meant to bring about (a search box that listed results as it was typed in +leaves "press search" nothing to do). If it clearly does (`DONE`), the step +ends `AlreadyDone`; otherwise it fails with the note "the last three actions +changed nothing on screen". Live, 25 of a day's 189 rescues found such a +step's work already done, each ~18 s later. ### 2. Judge From 4b7490ffe0017b4e22a4bdc330122dc4ca339f6e Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 7 Oct 2026 18:37:37 +0530 Subject: [PATCH 18/44] Let task_live remember learned elements between runs A task's report lists the elements it grounded (`learned`), meant to be passed back as StartTask's `memory` so a later run confirms a remembered element with one yes or no rather than searching for it. Neither task_live nor OpenHuman passed any. TASK_MEMORY names a JSON file of grounding hints: the task starts with them, and what it learns is saved back, a newer hint replacing an older one for the same element. Unset, every run starts fresh, as benchmark batches should. --- .env.example | 3 + .../src/bin/task_live/main.rs | 55 ++++++++++++++++++- .../src/bin/task_live/main_tests.rs | 53 +++++++++++++++++- .../tinycomputer-examples/live-tasks.md | 1 + 4 files changed, 109 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index d97677c1..ea58bee3 100644 --- a/.env.example +++ b/.env.example @@ -85,6 +85,9 @@ TINYCOMPUTER_LAB_SELF_EMAIL= # default 5). # TINYCOMPUTER_RESCUE_MODEL= # TASK_RESCUES=5 +# task_live: a file of elements earlier runs learned; read before the task, +# saved after it. +# TASK_MEMORY=target/task-live/memory/amazon.json # The reasoning model task_live shapes a finished task's answer with # (default openai/gpt-6-luna on OpenRouter), and a JSON TaskOutput file # asking for it. diff --git a/crates/tinycomputer-examples/src/bin/task_live/main.rs b/crates/tinycomputer-examples/src/bin/task_live/main.rs index 53439d86..a799c2b2 100644 --- a/crates/tinycomputer-examples/src/bin/task_live/main.rs +++ b/crates/tinycomputer-examples/src/bin/task_live/main.rs @@ -69,6 +69,11 @@ //! approve or decline an irreversible action, get past a login or captcha //! in the browser window and press Enter, type a detail the task lacks, //! and finish on a payment page before the browser closes. +//! - `TASK_MEMORY` — optional: a JSON file of grounding hints. The task +//! starts with the elements earlier runs learned there (`StartTask`'s +//! `memory`), so a remembered one is confirmed with a yes or no instead of +//! searched for, and what this run learns is saved back, newer hints +//! replacing older ones for the same element. //! - `TASK_HEADED` — optional: `1` shows the browser the task launches //! instead of running it headless. A headed browser needs a display, so //! such a run is on the host. @@ -86,10 +91,10 @@ use std::path::PathBuf; use std::time::Duration; use serde_json::{Value, json}; -use tinycomputer_bus::Flow; use tinycomputer_bus::agent::{ PlanTaskRequest, StartTaskRequest, SurfaceKind, TaskBudget, TaskConstraints, TaskOutput, }; +use tinycomputer_bus::{Flow, GroundingHint}; use tinycomputer_examples::host::{Host, LabError, jev_config, module_path}; use tinycomputer_examples::task::{Person, Terminal, conclude, follow, passed}; @@ -119,6 +124,11 @@ async fn main() -> Result<(), LabError> { // Sessions open before the task are not its own, and `conclude` leaves // them alone. let before = host.browser_sessions().await?; + let memory_file = std::env::var("TASK_MEMORY").ok().map(PathBuf::from); + let memory = match &memory_file { + Some(path) => read_memory(path)?, + None => Vec::new(), + }; let view = host .start_task(&StartTaskRequest { task: Some(task.clone()), @@ -143,7 +153,7 @@ async fn main() -> Result<(), LabError> { }, trace: true, output, - ..StartTaskRequest::default() + memory, }) .await?; let limit = Duration::from_secs( @@ -161,6 +171,9 @@ async fn main() -> Result<(), LabError> { if in_task { record_plan(&out)?; } + if let Some(path) = &memory_file { + remember(&out, path)?; + } host.shutdown(); if passed(&view.status) { println!( @@ -374,5 +387,43 @@ fn record_plan(out: &std::path::Path) -> Result<(), LabError> { Ok(()) } +/// The grounding hints saved at `path`: none when it does not exist yet. +fn read_memory(path: &std::path::Path) -> Result, LabError> { + match std::fs::read_to_string(path) { + Ok(text) => Ok(serde_json::from_str(&text)?), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(Vec::new()), + Err(error) => Err(error.into()), + } +} + +/// `kept` with what a run `learned`: a hint for the same element (the same +/// application and key) gives way to the newer one, and the rest keep their +/// order. +fn merge_memory(kept: Vec, learned: Vec) -> Vec { + let mut merged = kept + .into_iter() + .filter(|hint| { + !learned + .iter() + .any(|new| new.app == hint.app && new.key == hint.key) + }) + .collect::>(); + merged.extend(learned); + merged +} + +/// Saves what the task's report learned into the memory at `path`, beside +/// what it already held. +fn remember(out: &std::path::Path, path: &std::path::Path) -> Result<(), LabError> { + let report: Value = serde_json::from_str(&std::fs::read_to_string(out.join("report.json"))?)?; + let learned: Vec = match report.get("learned") { + Some(learned) => serde_json::from_value(learned.clone())?, + None => Vec::new(), + }; + let merged = merge_memory(read_memory(path)?, learned); + std::fs::write(path, serde_json::to_string_pretty(&merged)?)?; + Ok(()) +} + #[cfg(test)] mod main_tests; diff --git a/crates/tinycomputer-examples/src/bin/task_live/main_tests.rs b/crates/tinycomputer-examples/src/bin/task_live/main_tests.rs index a1057ab0..26b0ec66 100644 --- a/crates/tinycomputer-examples/src/bin/task_live/main_tests.rs +++ b/crates/tinycomputer-examples/src/bin/task_live/main_tests.rs @@ -5,7 +5,7 @@ use std::collections::BTreeMap; use serde_json::json; use tinycomputer_examples::host::LabError; -use super::{TINY_HUMANS_MODEL, routes}; +use super::{TINY_HUMANS_MODEL, merge_memory, read_memory, remember, routes}; /// A variable lookup over `pairs`, in place of the process environment. fn lookup(pairs: &[(&str, &str)]) -> impl Fn(&str) -> Option { @@ -89,3 +89,54 @@ fn sage_takes_the_decisions_on_either_route() -> Result<(), LabError> { ); Ok(()) } + +fn hint(key: &str, name: &str) -> tinycomputer_bus::GroundingHint { + tinycomputer_bus::GroundingHint { + app: "browser".to_owned(), + key: key.to_owned(), + role: "button".to_owned(), + name: Some(name.to_owned()), + path: Vec::new(), + } +} + +#[test] +fn a_run_s_learned_elements_replace_the_same_ones_and_keep_the_rest() { + let kept = vec![hint("search", "Go"), hint("add to cart", "Add to Cart")]; + let learned = vec![ + hint("add to cart", "Add to Bag"), + hint("open the cart", "Cart"), + ]; + let merged = merge_memory(kept, learned); + let names = merged + .iter() + .map(|hint| hint.name.as_deref().unwrap_or_default()) + .collect::>(); + assert_eq!(names, ["Go", "Add to Bag", "Cart"]); +} + +#[test] +fn memory_is_read_from_its_file_and_a_run_s_learned_elements_are_saved_to_it() +-> Result<(), LabError> { + let dir = std::env::temp_dir().join(format!("task-live-memory-{}", std::process::id())); + std::fs::create_dir_all(&dir)?; + let path = dir.join("memory.json"); + assert!( + read_memory(&path)?.is_empty(), + "no file yet: nothing learned" + ); + + std::fs::write( + dir.join("report.json"), + serde_json::to_string(&json!({"learned": [hint("search", "Go")]}))?, + )?; + remember(&dir, &path)?; + assert_eq!(read_memory(&path)?, vec![hint("search", "Go")]); + + // A report that learned nothing keeps what the memory held. + std::fs::write(dir.join("report.json"), "{}")?; + remember(&dir, &path)?; + assert_eq!(read_memory(&path)?.len(), 1); + std::fs::remove_dir_all(&dir)?; + Ok(()) +} diff --git a/docs/crates/tinycomputer-examples/live-tasks.md b/docs/crates/tinycomputer-examples/live-tasks.md index 8c44c570..fba0fc14 100644 --- a/docs/crates/tinycomputer-examples/live-tasks.md +++ b/docs/crates/tinycomputer-examples/live-tasks.md @@ -120,6 +120,7 @@ they control. | `TINYCOMPUTER_FLOW_DELIBERATION` | `deep` | `deep`, `standard`, or `off` | | `TASK_MAX_MINUTES` | `20` | the task is cancelled after this long | | `TASK_RESCUES` | `5` | how many failed steps a reasoning model may rescue (`0` turns rescues off); see [rescue](../../rescue.md) | +| `TASK_MEMORY` | unset | a JSON file of grounding hints: the task starts with the elements earlier runs learned (`StartTask`'s `memory`), so a remembered one is confirmed rather than searched for, and what this run learns is saved back; unset, every run starts fresh | | `TINYCOMPUTER_RESCUE_MODEL` | `openai/gpt-6-luna` (`openrouter/deepseek/deepseek-v4-flash` with `TINYHUMANS_TOKEN`) | the model that performs a rescue | | `TINYCOMPUTER_PLANNER_MODEL` | the engine's default (`openrouter/deepseek/deepseek-v4-flash` with `TINYHUMANS_TOKEN`) | the model asked to plan the flow | From ec84f76855d6e31ef343c093f292c0279250c99d Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 7 Oct 2026 18:45:43 +0530 Subject: [PATCH 19/44] Bump agent-browser so a shadow host's subtree reads through its shadow root Reading one shadow root's controls beside sight snapshots the tree under its host. On Lenskart that snapshot failed (agent-browser described the host's subtree without piercing its shadow root, so no accessibility node was found), and the surface fell back to reading the whole page through the tree, as before the change. agent-browser d8e93d8 describes the subtree through shadow roots. The live fixture now puts a stylesheet beside the banner in its shadow root, as the live banner had. --- .../src/surface/sight/sight_tests/live_tests.rs | 6 +++++- vendor/agent-browser | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_tests.rs b/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_tests.rs index 2b187576..b3b51ee1 100644 --- a/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_tests.rs +++ b/crates/tinycomputer-browser/src/surface/sight/sight_tests/live_tests.rs @@ -296,13 +296,17 @@ async fn live_a_shadow_root_that_shows_controls_is_handed_to_the_tree_under_its_ // Live, a consent banner's host was drawn as `display: contents`, with // no box of its own, and its buttons went unread while the banner lay // over the page. A block host whose banner is fixed draws no box either. + // Its shadow root holds a stylesheet beside the banner, as live: the tree + // read under the host then finds the banner only through the shadow + // root. let page = |host: &str, banner: &str| { format!( r#"
"# ) diff --git a/vendor/agent-browser b/vendor/agent-browser index ce670649..d8e93d88 160000 --- a/vendor/agent-browser +++ b/vendor/agent-browser @@ -1 +1 @@ -Subproject commit ce670649419d3fa7f28b2334f9e4b8159e5d0e12 +Subproject commit d8e93d8861d11bac957cabf41ce5a65fe8553559 From 7f19611a40087c32330338da80920c03f77ea1d5 Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 7 Oct 2026 20:04:28 +0530 Subject: [PATCH 20/44] Bump agent-browser so networkquiet waits for what an action sent agent-browser now keeps each page session's page-changing requests that have not finished, and a networkquiet wait starts with those sent in the last 3 s: a click's own navigation or fetch, which starts before the wait can subscribe, is waited for, up to QUIET_MS, instead of the page being read as it was before the action. The bump also brings the networkquiet docs, help and MCP entries, and the subtree doc comment fix. Settle's docs say so, and give their waits as values: rustdoc refuses links from a public enum to private constants, which failed the docs job. --- crates/tinycomputer-browser/src/surface/mod.rs | 11 ++++++----- vendor/agent-browser | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/crates/tinycomputer-browser/src/surface/mod.rs b/crates/tinycomputer-browser/src/surface/mod.rs index c943d56d..e2e47fe3 100644 --- a/crates/tinycomputer-browser/src/surface/mod.rs +++ b/crates/tinycomputer-browser/src/surface/mod.rs @@ -85,14 +85,15 @@ pub enum Perception { pub enum Settle { /// Wait for the network to go idle — 500 ms with nothing in flight, /// counted only after a first quiet receive window, so at least about - /// 1.1 s — then pause [`SETTLE_MS`] more. + /// 1.1 s, and at most 2 s (`NETWORK_IDLE_MS`) — then pause 400 ms + /// (`SETTLE_MS`) more. Steady, /// Wait for the network to go quiet, counting the 500 ms from the start /// and only the requests that can change the page (its document, - /// scripts, stylesheets, fetched data), at most [`QUIET_MS`]; then only - /// until the page stops changing: no DOM change for [`STILL_MS`] and no - /// finite CSS animation running, over at least two drawn frames, at most - /// [`SETTLE_MS`]. + /// scripts, stylesheets, fetched data), those the action sent included, + /// at most 1 s (`QUIET_MS`); then only until the page stops changing: + /// no DOM change for 120 ms (`STILL_MS`) and no finite CSS animation + /// running, over at least two drawn frames, at most 400 ms (`SETTLE_MS`). /// An idle page is read again after about 0.6 s instead of 1.6 s; a busy /// one still waits for its requests. The default: over 44 live runs it /// cost no run its outcome. diff --git a/vendor/agent-browser b/vendor/agent-browser index d8e93d88..ebf10fb3 160000 --- a/vendor/agent-browser +++ b/vendor/agent-browser @@ -1 +1 @@ -Subproject commit d8e93d8861d11bac957cabf41ce5a65fe8553559 +Subproject commit ebf10fb3685d3b8ec3728e681d7d53eb7186a5a7 From b13b5853b8a9c63b99826967d6965bc7a18333e4 Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 7 Oct 2026 20:05:10 +0530 Subject: [PATCH 21/44] Give the browser's settle and change watches deadlines The prompt settle's stillness script and await_change's watch were sent with no deadline of their own, right after the actions that replace pages, and an evaluate sent while a page is replaced waited out the browser's 30 s live. Each call now has its own cap plus 500 ms (watch::deadline): a quiet wait given up on still lets the page be watched, and a watch given up on says the page may have changed. Both watches now observe the page's open shadow roots as well as the document, so a suggestion list a web component draws counts as a change. The stillness script stops asking for frames once it has resolved, and await_change with no page open opens none. The scripts move from operations.rs to their own watch.rs, and the test engine can stall a command to prove the deadlines hold. --- crates/tinycomputer-browser/src/fake/mod.rs | 18 +++- .../tinycomputer-browser/src/surface/mod.rs | 1 + .../src/surface/operations.rs | 83 +++++-------------- .../surface/surface_tests/operations_tests.rs | 69 +++++++++++++-- .../tinycomputer-browser/src/surface/watch.rs | 78 +++++++++++++++++ .../tinycomputer-browser/interacting.md | 14 ++-- docs/crates/tinycomputer-browser/surface.md | 6 +- 7 files changed, 195 insertions(+), 74 deletions(-) create mode 100644 crates/tinycomputer-browser/src/surface/watch.rs diff --git a/crates/tinycomputer-browser/src/fake/mod.rs b/crates/tinycomputer-browser/src/fake/mod.rs index 63bf297b..dd4f3389 100644 --- a/crates/tinycomputer-browser/src/fake/mod.rs +++ b/crates/tinycomputer-browser/src/fake/mod.rs @@ -10,13 +10,15 @@ use serde_json::{Value, json}; use crate::engine::{Engine, Launcher, Reply}; type Script = dyn Fn(&Value) -> Option + Send + Sync; +type Stall = dyn Fn(&Value) -> bool + Send + Sync; /// Records every command; answers from an optional override, else like the -/// engine would. +/// engine would, and never answers a command it is told to stall on. #[derive(Clone)] pub(crate) struct Fake { sent: Arc>>, script: Arc"#; + let Some((browser, info)) = live_page(page).await else { + return; + }; + let reading = browser + .command( + &info.id, + json!({"action": "evaluate", "script": script(None)}), + ) + .await + .unwrap()["result"] + .clone(); + let names = shown_names(&reading); + assert!(names.iter().any(|name| name == "Add To Cart"), "{names:?}"); + assert!( + !names.iter().any(|name| name == "Slotted choice"), + "{names:?}" + ); + let shadows = reading["shadows"].as_array().unwrap(); + assert_eq!(shadows.len(), 1, "{reading}"); + assert_eq!( + shadows[0]["label"], "popover \"Cookie choices\"", + "{reading}" + ); + let host = shadows[0]["id"].as_str().unwrap().to_owned(); + let subtree = browser + .snapshot( + &info.id, + tinycomputer_bus::browser::SnapshotRequest { + selector: Some(format!("[data-tc-seen=\"{host}\"]")), + ..tinycomputer_bus::browser::SnapshotRequest::default() + }, + ) + .await + .unwrap(); + browser.close_session(&info.id).await.unwrap(); + for read in ["Allow Selection", "Slotted choice"] { + assert!(subtree.tree.contains(read), "{read}: {}", subtree.tree); + } +} + #[cfg(feature = "agent-browser")] #[tokio::test] async fn live_hidden_elements_are_dropped() { diff --git a/docs/crates/tinycomputer-browser/sight.md b/docs/crates/tinycomputer-browser/sight.md index 7459391f..a5f813a5 100644 --- a/docs/crates/tinycomputer-browser/sight.md +++ b/docs/crates/tinycomputer-browser/sight.md @@ -43,8 +43,9 @@ to the page except a marker attribute on elements it has already seen `Perception::Sight` is the default. `BrowserSurface::observe` tries sight first, and only reads the accessibility tree when sight fails outright or -runs into something it cannot address with a CSS selector, such as a shadow -root, or a large frame sitting in front of the content. `Perception::Tree`, +runs into something it cannot address with a CSS selector, such as two +shadow roots showing controls, or a large frame sitting in front of the +content. `Perception::Tree`, set with `BrowserSurface::with_perception`, skips sight entirely and always reads through the tree; the live examples expose this as `TINYCOMPUTER_BROWSER_PERCEPTION=tree`. @@ -250,9 +251,12 @@ the viewport, or controls inside two shadow roots: cases where a plain CSS selector from the top-level page cannot address the element sight found. One shadow root that shows controls is read beside sight instead. Sight -marks its host and names the layer its controls draw (`popover "We value -your privacy"`), and the surface reads the tree under the host alone and -adds its controls, under that label, after everything sight read. A tree +marks its host and names the first shown layer its controls draw (`popover +"We value your privacy"`, with an `aria-labelledby` resolved inside the +shadow root), and the surface reads the tree under the host alone and adds +its controls, under that label, after everything sight read. The tree reads +the host itself and what the page puts in its slots too, so sight leaves +both to it: nothing is offered twice. A tree snapshot's refs last until the next snapshot, so only one host's subtree can be read beside sight. A shadow root counts once its host or any of its controls shows: a host laid out as `display: contents` has no box of its diff --git a/docs/technical/specs/browser-sight.md b/docs/technical/specs/browser-sight.md index 756a354a..e36485a2 100644 --- a/docs/technical/specs/browser-sight.md +++ b/docs/technical/specs/browser-sight.md @@ -108,9 +108,12 @@ a caret. bounding box, value read, and scoped observation. An element the page removes takes its mark with it, so a stale ref fails rather than reaching what replaced it. -8. **Fallback.** When the reading fails, or sees a control inside a shadow - root or a frame of a fifth of the viewport in front, the surface reads the - accessibility tree for that observation, as before. +8. **Fallback.** When the reading fails, sees controls inside two shadow + roots or a frame of a fifth of the viewport in front, or the tree cannot + read the one shadow root's host, the surface reads the accessibility tree + for that observation, as before. One shadow root is read beside sight: + the tree reads its host, its controls, and what the page puts in its + slots, which sight leaves out so that nothing is offered twice. 9. **Denoising.** Noise is left out before anything is returned; see [Denoising](#denoising). From 983c0cfae94f882f47da34eb12136f7bda4772c8 Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 7 Oct 2026 20:05:39 +0530 Subject: [PATCH 23/44] Let go of an early browser while its task waits, or once it is cancelled A browser-only task whose plan asks for values published needs_input with the browser it opened while planning still open, holding one of the browser's sessions for as long as the person took, or for good if the host never answered. It is now released first; the run that follows opens one again. A task cancelled while its early open had not begun yet could leak a session: release found none to close, and the open then launched one nobody owned. A closed BrowserSurface now refuses an early open, checked under the session lock, so either the open sees the close or the close waits for the open and closes what it made. --- .../tinycomputer-browser/src/surface/mod.rs | 25 +++++++++++++++++-- .../surface/surface_tests/operations_tests.rs | 8 ++++++ crates/tinycomputer-engine/src/task/drive.rs | 4 +++ crates/tinycomputer-engine/src/task/mod.rs | 5 +++- .../src/task/task_tests/plan_tests.rs | 5 ++++ docs/technical/tasks.md | 5 +++- 6 files changed, 48 insertions(+), 4 deletions(-) diff --git a/crates/tinycomputer-browser/src/surface/mod.rs b/crates/tinycomputer-browser/src/surface/mod.rs index a0f7f596..306dd2d2 100644 --- a/crates/tinycomputer-browser/src/surface/mod.rs +++ b/crates/tinycomputer-browser/src/surface/mod.rs @@ -26,6 +26,7 @@ mod watch; pub use sight::Denoised; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use serde_json::json; @@ -119,6 +120,9 @@ pub struct BrowserSurface { perception: Perception, settle: Settle, denoised: Arc>, + /// Set once the surface is let go ([`BrowserSurface::close`]): it is + /// then not opened early again. + closed: Arc, } impl std::fmt::Debug for BrowserSurface { @@ -154,6 +158,7 @@ impl BrowserSurface { perception: Perception::default(), settle: Settle::default(), denoised: Arc::new(Mutex::new(Denoised::default())), + closed: Arc::new(AtomicBool::new(false)), } } @@ -185,10 +190,13 @@ impl BrowserSurface { /// Opens the session now, if it is not open yet, rather than at the first /// call that needs it: a task that will run on the browser can open it - /// while its plan is drafted. Whether the session is open. + /// while its plan is drafted. Whether the session is open. A surface + /// already let go ([`BrowserSurface::close`]) is not opened: a task + /// cancelled while its browser was being opened early is left holding + /// none. #[must_use] pub fn open(&self) -> bool { - self.ensure_session().is_ok() + self.session_slot(true).is_ok() } /// The session this surface drives, once one is open. @@ -211,6 +219,9 @@ impl BrowserSurface { /// Closes the session, if one is open, without waiting for it: safe to /// call from async code, where a blocking surface call is not. pub fn close(&self) { + // Before the slot is taken: an early open that has not begun yet + // finds it set, and one under way finishes first and is closed. + self.closed.store(true, Ordering::Release); let Some(id) = self .session .lock() @@ -230,6 +241,13 @@ impl BrowserSurface { } fn ensure_session(&self) -> Result { + self.session_slot(false) + } + + /// The open session, opening one if there is none; when `early`, not on + /// a surface already let go. `closed` is read under the slot's lock, so + /// an early open and a close cannot both miss each other. + fn session_slot(&self, early: bool) -> Result { let mut slot = self .session .lock() @@ -237,6 +255,9 @@ impl BrowserSurface { if let Some(id) = slot.as_ref() { return Ok(id.clone()); } + if early && self.closed.load(Ordering::Acquire) { + return Err(Error::failed("the browser surface was let go")); + } let info = self.block(self.browser.open_session(self.options.clone()))?; *slot = Some(info.id.clone()); Ok(info.id) diff --git a/crates/tinycomputer-browser/src/surface/surface_tests/operations_tests.rs b/crates/tinycomputer-browser/src/surface/surface_tests/operations_tests.rs index acec80b4..11dcd378 100644 --- a/crates/tinycomputer-browser/src/surface/surface_tests/operations_tests.rs +++ b/crates/tinycomputer-browser/src/surface/surface_tests/operations_tests.rs @@ -698,4 +698,12 @@ fn a_surface_opens_its_session_when_asked_rather_than_at_first_use() { }); let Harness { surface, .. } = harness("open-refused", refused); assert!(!surface.open()); + + // A surface let go before its early open began is not opened: a task + // cancelled while planning holds no browser. + let Harness { fake, surface, .. } = harness("open-closed", page_fake()); + surface.close(); + assert!(!surface.open()); + assert!(surface.session().is_none()); + assert!(fake.actions().is_empty(), "{:?}", fake.actions()); } diff --git a/crates/tinycomputer-engine/src/task/drive.rs b/crates/tinycomputer-engine/src/task/drive.rs index 252dfaab..46dba653 100644 --- a/crates/tinycomputer-engine/src/task/drive.rs +++ b/crates/tinycomputer-engine/src/task/drive.rs @@ -90,6 +90,10 @@ pub(super) async fn plan_then_drive( ) .await; } else { + // A browser opened while planning is let go while the task waits on + // a person, who may take long or never answer; the run that follows + // opens one again. + runner.release(&id); publish( &cell, TaskStatus::NeedsInput { diff --git a/crates/tinycomputer-engine/src/task/mod.rs b/crates/tinycomputer-engine/src/task/mod.rs index b1e5a0c4..69de41a4 100644 --- a/crates/tinycomputer-engine/src/task/mod.rs +++ b/crates/tinycomputer-engine/src/task/mod.rs @@ -114,7 +114,10 @@ pub trait FlowRunner: Send + Sync + 'static { /// Gets the task's surfaces ready while its plan is drafted, so its /// first step does not wait for them: called alongside the planner for - /// a task that runs on the browser alone. Does nothing by default. + /// a task that runs on the browser alone. [`FlowRunner::release`] may + /// run while the future is in flight, or after it was dropped with a + /// cancelled task: what it opens then must be let go too. Does nothing + /// by default. fn prepare(&self, _task: &TaskId, _constraints: &TaskConstraints) -> PrepareFuture { Box::pin(async {}) } diff --git a/crates/tinycomputer-engine/src/task/task_tests/plan_tests.rs b/crates/tinycomputer-engine/src/task/task_tests/plan_tests.rs index d15caab0..02d50066 100644 --- a/crates/tinycomputer-engine/src/task/task_tests/plan_tests.rs +++ b/crates/tinycomputer-engine/src/task/task_tests/plan_tests.rs @@ -85,6 +85,11 @@ async fn a_plan_that_needs_values_asks_and_a_failed_plan_says_so() { assert!( matches!(waiting.status, TaskStatus::NeedsInput { ref fields } if fields[0].name == "phone") ); + assert_eq!( + *script.released.lock().unwrap(), + std::slice::from_ref(&started.id), + "a task waiting on a person holds no browser" + ); assert!( tasks .continue_task(ContinueTaskRequest { diff --git a/docs/technical/tasks.md b/docs/technical/tasks.md index 1ead045f..7214cb73 100644 --- a/docs/technical/tasks.md +++ b/docs/technical/tasks.md @@ -278,7 +278,10 @@ A `StartTask` with a `task` and no `flow` plans inside the task. When the task may run only on the browser, its runner is asked to get the browser ready meanwhile (`FlowRunner::prepare`); unless the module's `browser.prelaunch` is off, the session opens while the plan is drafted, so -the first step does not wait for Chrome to start. +the first step does not wait for Chrome to start. A plan that asks for +values lets the browser go while the task waits (`needs_input`), and the +run that follows opens it again; a task cancelled while its browser was +still opening is left holding none. ## Rescues From c7503c0d133ffa4cc73b6e3d52f9e7e4e113afa5 Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 7 Oct 2026 20:05:52 +0530 Subject: [PATCH 24/44] Hedge no Sage call, and send at most two copies at once Sage's calls take 5-6 s as a rule, past any hedge delay: almost every framing would have been copied, adding half again to its calls and cost for answers that rarely came first. Sage now gets no copy. Many framings outliving their delay together means a slow or failing gateway, where the client may be waiting out its own retry delay; a copy of each only loaded it more. A runtime now has at most HEDGE_COPIES (2) copies in flight; past that, a framing waits for its own answer. The hedge event's won now names the answer that counted (first, copy, or neither), not the copy that finished first. Hedging moves from decide.rs into its own module, and the harness doc's settle row says how the browser settles now. --- .../src/agentic/agentic_tests.rs | 1 + .../src/agentic/flow/decide.rs | 85 ++----------- .../src/agentic/flow/flow_tests.rs | 2 + .../agentic/flow/flow_tests/hedge_tests.rs | 100 ++++++++++++++- .../src/agentic/flow/hedge.rs | 119 ++++++++++++++++++ .../src/agentic/flow/mod.rs | 1 + .../src/agentic/runtime.rs | 8 +- docs/technical/decision-thresholds.md | 3 +- docs/technical/jev-harness.md | 4 +- docs/technical/jev-journal.md | 2 +- 10 files changed, 240 insertions(+), 85 deletions(-) create mode 100644 crates/tinycomputer-engine/src/agentic/flow/hedge.rs diff --git a/crates/tinycomputer-engine/src/agentic/agentic_tests.rs b/crates/tinycomputer-engine/src/agentic/agentic_tests.rs index 98cf7ef2..484befd4 100644 --- a/crates/tinycomputer-engine/src/agentic/agentic_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/agentic_tests.rs @@ -364,6 +364,7 @@ fn runtime_recording( }, pending: Arc::new(Mutex::new(std::collections::HashMap::new())), journal: super::journal::Journal::default(), + copies: Arc::default(), }, requests, ) diff --git a/crates/tinycomputer-engine/src/agentic/flow/decide.rs b/crates/tinycomputer-engine/src/agentic/flow/decide.rs index ea7fe984..6287ebd8 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/decide.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/decide.rs @@ -1,38 +1,17 @@ //! Asking Jev: every request is briefed, masked, fitted to size, and voted //! on through one door. -use std::{ - collections::BTreeMap, - time::{Duration, Instant}, -}; +use std::{collections::BTreeMap, time::Instant}; use serde_json::{Value, json}; use tinycomputer_bus::{FlowLoop, FlowStopReason, JevExchange}; use tinycomputer_core::Facts; -use tinyinference_decisions::{ - Answer, EvaluationFailure, EvaluationRequest, EvaluationResult, Question, -}; +use tinyinference_decisions::{Answer, EvaluationRequest, Question}; use super::{ - FlowRun, Halt, MAX_REQUEST_BYTES, StepLog, ask, backend::AgentBackend, brief::clip, vote, + FlowRun, Halt, MAX_REQUEST_BYTES, StepLog, ask, backend::AgentBackend, brief::clip, hedge, vote, }; -use crate::agentic::{JevRuntime, journal::millis, merge_metrics, provider_error}; - -/// How long a framing runs before a copy of it is sent and the first answer -/// of the two taken. Live, one framing of a burst stalled 12–32 s (the -/// gateway gave up after ~10 s, or nothing came back before the client's -/// timeout) as its siblings answered in under 1 s. Calls on a slow evening -/// took up to 3.4 s and still answered: a copy sent at 2.5 s lost the race -/// 15 times in 16, so copies wait for 4 s, past what a slow answer takes. -const HEDGE_AFTER: Duration = Duration::from_millis(4_000); - -/// [`HEDGE_AFTER`] for a request of [`HEDGE_LARGE_BYTES`] or more, whose -/// p99.9 was 3.9 s live. -const HEDGE_AFTER_LARGE: Duration = Duration::from_millis(5_000); - -/// Size from which a request waits [`HEDGE_AFTER_LARGE`] for its first -/// answer. -const HEDGE_LARGE_BYTES: usize = 32 * 1024; +use crate::agentic::{journal::millis, merge_metrics, provider_error}; impl FlowRun<'_, B> { /// Asks Jev one request, charging it to the run and the step. @@ -220,7 +199,7 @@ impl FlowRun<'_, B> { .collect() } - /// Sends every framing to Jev at once, each one [`hedged`]. + /// Sends every framing to Jev at once, each one [`hedged`](hedge::hedged). pub(super) fn spawn( &self, framings: &[vote::Framing], @@ -238,7 +217,7 @@ impl FlowRun<'_, B> { let runtime = self.runtime.clone(); let step = self.step.clone(); let request = framing.request.clone(); - tokio::spawn(async move { hedged(&runtime, &step, &request).await }) + tokio::spawn(async move { hedge::hedged(&runtime, &step, &request).await }) }) .collect() } @@ -279,56 +258,6 @@ type Sent = ( /// The id of the page-kind question a request on a web page carries. pub(super) const PAGE_KIND: &str = "page_kind"; -/// Asks `request` once, and once more when no answer has come within its -/// hedge delay ([`HEDGE_AFTER`], or [`HEDGE_AFTER_LARGE`] for a large -/// request), taking whichever copy answers first. A copy that fails gives -/// way to the other; when both fail, the first failure is returned. The -/// answer that counts carries the extra attempt, and the journal records a -/// `hedge` event. -pub(in crate::agentic::flow) async fn hedged( - runtime: &JevRuntime, - step: &str, - request: &EvaluationRequest, -) -> Result { - let delay = if bytes(request) >= HEDGE_LARGE_BYTES { - HEDGE_AFTER_LARGE - } else { - HEDGE_AFTER - }; - let first = runtime.evaluate(Some(step), request); - tokio::pin!(first); - if let Ok(outcome) = tokio::time::timeout(delay, &mut first).await { - return outcome; - } - let sent = Instant::now(); - let copy = runtime.evaluate(Some(step), request); - tokio::pin!(copy); - let (outcome, copy_won) = tokio::select! { - outcome = &mut first => (outcome, false), - outcome = &mut copy => (outcome, true), - }; - let outcome = match outcome { - Ok(evaluation) => Ok(evaluation), - Err(failure) => { - let other = if copy_won { first.await } else { copy.await }; - other.or(Err(failure)) - } - }; - runtime.journal.record("hedge", || { - json!({ - "step": step, - "after_ms": millis(delay), - "won": if copy_won { "copy" } else { "first" }, - "ok": outcome.is_ok(), - "wall_ms": millis(delay + sent.elapsed()), - }) - }); - outcome.map(|mut evaluation| { - evaluation.attempts = evaluation.attempts.saturating_add(1); - evaluation - }) -} - /// `request` cut by its questions into requests of at most `limit` bytes of /// JSON, each carrying the whole state and as many of the questions, in /// order, as fit beside it. Jev evaluates every question on its own against @@ -402,7 +331,7 @@ pub(in crate::agentic::flow) fn largest(parts: &[EvaluationRequest]) -> usize { } /// The size of `request`, in bytes of JSON. -fn bytes(request: &EvaluationRequest) -> usize { +pub(super) fn bytes(request: &EvaluationRequest) -> usize { serde_json::to_vec(request).map_or(0, |json| json.len()) } diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests.rs index 749942f6..3dd89893 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests.rs @@ -92,6 +92,7 @@ fn runtime(oracle: Oracle) -> JevRuntime { }, pending: Arc::default(), journal: crate::agentic::journal::Journal::default(), + copies: Arc::default(), } } @@ -123,6 +124,7 @@ async fn run_with( }, pending: Arc::default(), journal: crate::agentic::journal::Journal::default(), + copies: Arc::default(), }; // One framing per decision, so every test that counts requests counts // decisions; voting has its own tests. diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/hedge_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/hedge_tests.rs index 59253588..6847b8fb 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/hedge_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/hedge_tests.rs @@ -2,7 +2,7 @@ //! and the first answer of the two counts. use super::*; -use crate::agentic::flow::decide::hedged; +use crate::agentic::flow::hedge::hedged; /// A Jev that answers each call after the wait scripted for it, in call /// order, or fails it. @@ -47,6 +47,13 @@ impl Evaluator for Paced { } fn paced(script: &[(u64, bool)]) -> (JevRuntime, Arc) { + paced_as(tinycomputer_bus::JevProvider::OpenRouter, script) +} + +fn paced_as( + provider: tinycomputer_bus::JevProvider, + script: &[(u64, bool)], +) -> (JevRuntime, Arc) { let paced = Arc::new(Paced { calls: Mutex::new(0), script: script @@ -57,13 +64,14 @@ fn paced(script: &[(u64, bool)]) -> (JevRuntime, Arc) { let runtime = JevRuntime { client: paced.clone(), configuration: tinycomputer_bus::JevConfiguration { - provider: tinycomputer_bus::JevProvider::OpenRouter, + provider, model: "jev-latest".to_owned(), endpoint_url: None, fast: false, }, pending: Arc::default(), journal: crate::agentic::journal::Journal::default(), + copies: Arc::default(), }; (runtime, paced) } @@ -128,3 +136,91 @@ async fn a_failed_copy_gives_way_and_two_failures_fail() { let (runtime, _) = paced(&[(4_500, true), (1_000, true)]); assert!(hedged(&runtime, "1", &request(1_000)).await.is_err()); } + +#[tokio::test(start_paused = true)] +async fn a_sage_framing_gets_no_copy() { + // Sage's calls take 5–6 s as a rule: a copy would double them. + let (runtime, paced) = paced_as( + tinycomputer_bus::JevProvider::Sage, + &[(6_000, false), (600, false)], + ); + let started = tokio::time::Instant::now(); + let answer = hedged(&runtime, "1", &request(1_000)).await.unwrap(); + assert_eq!( + (answer.response.model.as_str(), answer.attempts), + ("call 0", 1) + ); + assert_eq!(started.elapsed(), Duration::from_millis(6_000)); + assert_eq!(*paced.calls.lock().unwrap(), 1); +} + +#[tokio::test(start_paused = true)] +async fn no_more_than_two_copies_are_in_flight() { + // Three framings stall together, as on a slow gateway: two get a copy, + // the third waits for its own answer. + let (runtime, paced) = paced(&[ + (30_000, false), + (30_000, false), + (30_000, false), + (600, false), + (600, false), + ]); + let asked = request(1_000); + let ask = || hedged(&runtime, "1", &asked); + let (one, two, three) = tokio::join!(ask(), ask(), ask()); + let mut attempts = [one, two, three].map(|answer| answer.unwrap().attempts); + // Which framing's delay ends first is the timer's to say. + attempts.sort_unstable(); + assert_eq!(attempts, [1, 2, 2]); + assert_eq!(*paced.calls.lock().unwrap(), 5); + assert_eq!( + runtime.copies.load(std::sync::atomic::Ordering::Acquire), + 0, + "every place given back" + ); +} + +#[tokio::test(start_paused = true)] +async fn the_journal_names_the_answer_that_counted() { + let scratch = std::env::temp_dir().join(format!( + "tinycomputer-hedge-journal-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let won = |script: &[(u64, bool)]| { + let (mut runtime, _) = paced(script); + runtime.journal = crate::agentic::journal::Journal::at(&scratch).fresh("hedge"); + let dir = runtime.journal.run_dir().unwrap(); + async move { + let _ = hedged(&runtime, "1", &request(1_000)).await; + let journal = std::fs::read_to_string(dir.join(crate::JOURNAL_FILE)).unwrap(); + let event: Value = journal + .lines() + .map(|line| serde_json::from_str::(line).unwrap()) + .find(|event| event["event"] == "hedge") + .unwrap(); + ( + event["won"].as_str().unwrap().to_owned(), + event["ok"].as_bool().unwrap(), + ) + } + }; + // The first finished first, failing; the copy's answer counted. + assert_eq!( + won(&[(4_500, true), (1_000, false)]).await, + ("copy".to_owned(), true) + ); + // The copy failed at once; the first's answer counted. + assert_eq!( + won(&[(6_000, false), (0, true)]).await, + ("first".to_owned(), true) + ); + assert_eq!( + won(&[(4_500, true), (1_000, true)]).await, + ("neither".to_owned(), false) + ); + let _ = std::fs::remove_dir_all(&scratch); +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/hedge.rs b/crates/tinycomputer-engine/src/agentic/flow/hedge.rs new file mode 100644 index 00000000..042a0d1d --- /dev/null +++ b/crates/tinycomputer-engine/src/agentic/flow/hedge.rs @@ -0,0 +1,119 @@ +//! Hedging a framing: one still unanswered past its hedge delay is sent +//! again, and the first answer of the two counts. + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{Duration, Instant}; + +use serde_json::json; +use tinycomputer_bus::JevProvider; +use tinyinference_decisions::{EvaluationFailure, EvaluationRequest, EvaluationResult}; + +use super::decide::bytes; +use crate::agentic::{JevRuntime, journal::millis}; + +/// How long a framing runs before a copy of it is sent and the first answer +/// of the two taken. Live, one framing of a burst stalled 12–32 s (the +/// gateway gave up after ~10 s, or nothing came back before the client's +/// timeout) as its siblings answered in under 1 s. Calls on a slow evening +/// took up to 3.4 s and still answered: a copy sent at 2.5 s lost the race +/// 15 times in 16, so copies wait for 4 s, past what a slow answer takes. +const HEDGE_AFTER: Duration = Duration::from_millis(4_000); + +/// [`HEDGE_AFTER`] for a request of [`HEDGE_LARGE_BYTES`] or more, whose +/// p99.9 was 3.9 s live. +const HEDGE_AFTER_LARGE: Duration = Duration::from_millis(5_000); + +/// Size from which a request waits [`HEDGE_AFTER_LARGE`] for its first +/// answer. +const HEDGE_LARGE_BYTES: usize = 32 * 1024; + +/// Most copies a runtime has in flight at once. A stall is rare (21 calls of +/// 36,459 live), so many framings outliving their delay together means a +/// slow or failing gateway, which a copy of each would only load more (the +/// client may be waiting out its retry delay): past this many, a framing +/// waits for its own answer. +const HEDGE_COPIES: usize = 2; + +/// Asks `request` once, and once more when no answer has come within its +/// hedge delay ([`HEDGE_AFTER`], or [`HEDGE_AFTER_LARGE`] for a large +/// request), taking whichever copy answers first. A copy that fails gives +/// way to the other; when both fail, the first failure is returned. The +/// answer that counts carries the extra attempt, and the journal records a +/// `hedge` event. Sage gets no copy, and no copy is sent while +/// [`HEDGE_COPIES`] are in flight. +pub(in crate::agentic::flow) async fn hedged( + runtime: &JevRuntime, + step: &str, + request: &EvaluationRequest, +) -> Result { + let first = runtime.evaluate(Some(step), request); + // Sage's calls take 5–6 s as a rule (`docs/technical/evals/ + // 2026-09-29-sage.md`): a copy would double its calls and its cost, and + // rarely answer first. + if runtime.configuration.provider == JevProvider::Sage { + return first.await; + } + let delay = if bytes(request) >= HEDGE_LARGE_BYTES { + HEDGE_AFTER_LARGE + } else { + HEDGE_AFTER + }; + tokio::pin!(first); + if let Ok(outcome) = tokio::time::timeout(delay, &mut first).await { + return outcome; + } + let Some(_held) = Held::claim(&runtime.copies) else { + return first.await; + }; + let sent = Instant::now(); + let copy = runtime.evaluate(Some(step), request); + tokio::pin!(copy); + let (outcome, copy_first) = tokio::select! { + outcome = &mut first => (outcome, false), + outcome = &mut copy => (outcome, true), + }; + // Which answer counts: `Some(true)` the copy's, `None` neither. + let (outcome, counted) = match outcome { + Ok(evaluation) => (Ok(evaluation), Some(copy_first)), + Err(failure) => match if copy_first { first.await } else { copy.await } { + Ok(evaluation) => (Ok(evaluation), Some(!copy_first)), + Err(_) => (Err(failure), None), + }, + }; + runtime.journal.record("hedge", || { + json!({ + "step": step, + "after_ms": millis(delay), + "won": match counted { + Some(true) => "copy", + Some(false) => "first", + None => "neither", + }, + "ok": outcome.is_ok(), + "wall_ms": millis(delay + sent.elapsed()), + }) + }); + outcome.map(|mut evaluation| { + evaluation.attempts = evaluation.attempts.saturating_add(1); + evaluation + }) +} + +/// A copy in flight, counted against [`HEDGE_COPIES`] until it is dropped. +struct Held<'a>(&'a AtomicUsize); + +impl<'a> Held<'a> { + /// A place for one more copy, unless [`HEDGE_COPIES`] are in flight. + fn claim(copies: &'a AtomicUsize) -> Option { + let held = Self(copies); + // Counted first, so two claims at once cannot both slip under the + // limit; one over it gives its place back as it is dropped. + (copies.fetch_add(1, Ordering::AcqRel) < HEDGE_COPIES).then_some(held) + } +} + +impl Drop for Held<'_> { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::AcqRel); + } +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/mod.rs b/crates/tinycomputer-engine/src/agentic/flow/mod.rs index 281a0ac6..6fb08c05 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/mod.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/mod.rs @@ -55,6 +55,7 @@ mod evidence; mod expect; mod front; mod ground; +mod hedge; mod ledger; mod look; mod memory; diff --git a/crates/tinycomputer-engine/src/agentic/runtime.rs b/crates/tinycomputer-engine/src/agentic/runtime.rs index 2c4bc3cc..e81662de 100644 --- a/crates/tinycomputer-engine/src/agentic/runtime.rs +++ b/crates/tinycomputer-engine/src/agentic/runtime.rs @@ -5,7 +5,7 @@ use std::{ collections::HashMap, future::Future, pin::Pin, - sync::{Arc, Mutex}, + sync::{Arc, Mutex, atomic::AtomicUsize}, time::Duration, }; @@ -41,6 +41,9 @@ pub struct JevRuntime { pub(super) configuration: JevConfiguration, pub(super) pending: Arc>>, pub(super) journal: Journal, + /// Hedged copies of framings in flight, shared by every flow run on + /// this runtime. + pub(super) copies: Arc, } impl std::fmt::Debug for JevRuntime { @@ -51,6 +54,7 @@ impl std::fmt::Debug for JevRuntime { .field("configuration", &self.configuration) .field("pending", &"[redacted]") .field("journal", &self.journal) + .field("copies", &self.copies) .finish() } } @@ -96,6 +100,7 @@ impl JevRuntime { }, pending: Arc::new(Mutex::new(HashMap::new())), journal: Journal::from_env(), + copies: Arc::default(), }) } @@ -138,6 +143,7 @@ impl JevRuntime { }, pending: Arc::new(Mutex::new(HashMap::new())), journal: Journal::from_env(), + copies: Arc::default(), }) } diff --git a/docs/technical/decision-thresholds.md b/docs/technical/decision-thresholds.md index 81567e9f..3193bd59 100644 --- a/docs/technical/decision-thresholds.md +++ b/docs/technical/decision-thresholds.md @@ -51,7 +51,8 @@ Change a constant and its row together. | `LAYER_COVERS` | 3 | `front.rs` | controls something drawn over the window must cover, beyond what was covered before the press that opened it, on the same page, before it counts as a dialog the task opened (`surface` `layer`); a step that pressed inside such a dialog hands it back at the next step, and opening an address forgets it | | `FRONT_CONTROLS` | 8 | `act/turns.rs` | most controls of the task's dialog in front a failed step's note names, so a rescue answers with one of them | | `STEADY_HOLD` / `STEADY_CHECKS` | 0.65 / 3 | `steps/mod.rs` | belief a `wait_for` condition must keep, on checks in a row of one unchanged screen, to be taken as held under `DONE` | -| `HEDGE_AFTER` / `HEDGE_AFTER_LARGE` | 4 s / 5 s | `decide.rs` | how long a framing runs before a copy of it is sent and the first answer of the two taken; the longer wait is for a request of `HEDGE_LARGE_BYTES` (32 KB) or more | +| `HEDGE_AFTER` / `HEDGE_AFTER_LARGE` | 4 s / 5 s | `hedge.rs` | how long a framing runs before a copy of it is sent and the first answer of the two taken; the longer wait is for a request of `HEDGE_LARGE_BYTES` (32 KB) or more. Sage gets no copy | +| `HEDGE_COPIES` | 2 | `hedge.rs` | most copies one runtime has in flight: past it, a framing waits for its own answer, so a slow or failing gateway is not sent a copy of every call | | `LATE_LOOKS` | 2 | `steps/suggestion.rs` | looks again, after a wait for the page to change, for the suggestions a place box lists late, before its text is left as typed; a page that stayed still through a wait lists nothing more | | `LATE_LOOK_MS` | 1000 ms | `steps/suggestion.rs` | longest one of those waits: it ends as soon as the page changes (`Surface::await_change`) | | `BARE_CHARS` | 3 | `tinycomputer-core` `surface/groups.rs` | most letters and digits each field of a card may show for a list of such cards to be bare markers (carousel dots, size chips, page numbers), never results | diff --git a/docs/technical/jev-harness.md b/docs/technical/jev-harness.md index 99539725..eb37f3f6 100644 --- a/docs/technical/jev-harness.md +++ b/docs/technical/jev-harness.md @@ -155,11 +155,11 @@ round trip's *slowest* framing, plus the action, plus settling. The levers: | Lever | Effect on latency | Effect on accuracy | |---|---|---| | `strategy` | `wide` asks one request per `do` turn instead of two to seven in sequence | the digest, survey, and memory show more of what matters; measure with the lab's `--strategy` | -| `votes` | a decision waits for its slowest framing: more framings, longer tail; a framing slower than 4 s (5 s at 32 KB or more) gets a copy, and the first answer counts | more framings average out position and phrasing bias | +| `votes` | a decision waits for its slowest framing: more framings, longer tail; a framing slower than 4 s (5 s at 32 KB or more) gets a copy, and the first answer counts (no copy on Sage, and two in flight at most) | more framings average out position and phrasing bias | | request size | Jev's latency grows with input tokens; a big element list is the usual cause | trimming can drop the element that was needed | | grounding memory | a remembered element is confirmed with one Noul instead of narrowing | none when the hint is right | | `disabled_loops` | each loop off removes a question or a whole decision | measure it before shipping it off | -| `settle` | fixed per action on the desktop, network-idle on the browser | too short and the next look sees the old screen | +| `settle` | fixed per action on the desktop; on the browser, `prompt` (the default) waits at most 1 s for the requests that change the page, then ≤ 400 ms for it to go still, and `steady` for network idle (≤ 2 s) and 400 ms more | too short and the next look sees the old screen | | observation | an accessibility snapshot of a large window is slow; `explore` adds more | a budgeted view can miss the target | Measure before changing any of them. The debug journal diff --git a/docs/technical/jev-journal.md b/docs/technical/jev-journal.md index 479c344e..6602e2d8 100644 --- a/docs/technical/jev-journal.md +++ b/docs/technical/jev-journal.md @@ -65,7 +65,7 @@ has `""`, and goal and intent runs carry their goal or intent text. | `turn` | a `do` turn ends | `step`, `turn`, `decisions` (made in that turn), `rounds` (round trips they took: a batch is one), `wall_ms` | | `survey` | the wide strategy surveys a crowded screen | `step`, `regions` asked about, `most_relevant` (region ids), `distractions` | | `observe` | a flow reads the screen | `step`, `part` (`screen` or `subtree`), `wall_ms`, `ok`, `candidates`, `unexplored` | -| `hedge` | a framing ran past its hedge delay and a copy was sent | `step`, `after_ms` (the delay), `won` (`first` or `copy`), `ok`, `wall_ms` | +| `hedge` | a framing ran past its hedge delay and a copy was sent (never for Sage, nor past two copies in flight) | `step`, `after_ms` (the delay), `won` (whose answer counted: `first`, `copy`, or `neither` when both failed), `ok`, `wall_ms` | | `action` | a flow acts | `step`, `action`, `target`, `ok`, `note`, `wall_ms`, `settle_ms`; a `wait` for the page to change that saw it stay still notes "nothing changed" and is not settled | | `reflect` | a `choose` that pressed something is reflected on | `step`, `held` (calibrated belief the choice shows), `attempt` (`first` or `after_repair`); `contradicted` when a selected sibling settled it without Jev | | `attention` | a turn or step asks what needs attention first | `step`, `distractions` (container names), `choice`, `verdict` | From 16679071c3d270874554d1478e3a7828bd6d8f95 Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 7 Oct 2026 20:06:10 +0530 Subject: [PATCH 25/44] Never close the covered target's own layer When a press was refused as covered, front_closer took the least committal closer of the first layer in front, asked with no intent and never told the target. A toast over a size popover's rows could get the popover's own close pressed, and a chat bubble over a consent banner's Accept all the banner's Reject all: either closes the layer the step works in, and the retried press then fails. It now skips the layer the target sits in, the target itself, and any layer the step's intent names, as the attention pass does. The covered press moves into act/uncover.rs, and its docs and the attention module's say what it presses; decision-loops.md is tightened around it so the file does not grow. --- .../src/agentic/flow/act/mod.rs | 5 +- .../src/agentic/flow/act/moves.rs | 74 +-------------- .../src/agentic/flow/act/uncover.rs | 92 +++++++++++++++++++ .../agentic/flow/attention/attention_tests.rs | 72 ++++++++++++++- .../src/agentic/flow/attention/mod.rs | 23 +++-- .../src/agentic/flow/steps/list.rs | 2 +- docs/technical/decision-loops.md | 28 +++--- 7 files changed, 198 insertions(+), 98 deletions(-) create mode 100644 crates/tinycomputer-engine/src/agentic/flow/act/uncover.rs diff --git a/crates/tinycomputer-engine/src/agentic/flow/act/mod.rs b/crates/tinycomputer-engine/src/agentic/flow/act/mod.rs index 659b1caf..600ca55d 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/act/mod.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/act/mod.rs @@ -21,8 +21,8 @@ //! a screen that returns to where it was two turns ago bans both presses. //! //! The loop's pieces: `turns` runs it, `judge` reads each turn's screen, -//! `moves` makes the chosen move, and `recover` undoes a turn that went -//! wrong. This root holds the thresholds and the state they share. +//! `moves` makes the chosen move, `uncover` presses again a target the page +//! refused as covered, and `recover` undoes a turn that went wrong. This root holds the thresholds and the state they share. mod copies; mod dialog; @@ -33,6 +33,7 @@ mod judge; mod moves; mod recover; mod turns; +mod uncover; pub(super) use judge::Judgement; diff --git a/crates/tinycomputer-engine/src/agentic/flow/act/moves.rs b/crates/tinycomputer-engine/src/agentic/flow/act/moves.rs index fadd8034..0e6ce49f 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/act/moves.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/act/moves.rs @@ -9,14 +9,13 @@ use tinycomputer_bus::{JevOperation, StepOutcome}; use crate::agentic::flow::{ Ended, FlowRun, Halt, StepLog, ask::{self, Questions, chosen}, - attention::front_closer, backend::AgentBackend, memory::{learn, remember}, - view::{Candidate, Screen, element_kind, is_banned, is_destructive, label, signature}, + view::{Candidate, Screen, element_kind, is_banned, is_destructive, label}, }; use super::{ - Expected, MAX_REPEAT_PRESSES, Move, activate_purpose, covered, creates_new, dialog::in_dialog, + Expected, MAX_REPEAT_PRESSES, Move, activate_purpose, creates_new, dialog::in_dialog, judge::Judgement, }; @@ -229,7 +228,7 @@ impl FlowRun<'_, B> { } let expected = self.expect(log, operation, &target, screen); let reply = self - .press_uncovering(log, verb, &target, jev_operation) + .press_uncovering(log, verb, &target, jev_operation, intent) .await?; self.history .push(format!("{verb} {} ok={}", label(&target), reply.ok)); @@ -239,73 +238,6 @@ impl FlowRun<'_, B> { Ok(Some((target, expected))) } - /// Performs `operation` on an already-vetted `target`. When the click - /// is refused because something covers it — a drawer, a menu, or a - /// result card's own click layer — presses Escape once and tries the - /// same target again. Escape never chooses a new element. - pub(in crate::agentic::flow) async fn press_uncovering( - &mut self, - log: &mut StepLog, - verb: &str, - target: &Candidate, - operation: JevOperation, - ) -> Result { - let chosen = target.clone(); - let reply = self - .act(log, verb, Some(target), move |backend| { - backend.execute(operation, Some(chosen), None) - }) - .await?; - if !covered(&reply) { - return Ok(reply); - } - // A dialog in front is the page's question (a format, a quantity), - // not a popover in the way: Escape would close it, and pressing what - // lies behind it leaves the flow it began (live, a movie's language - // link behind its booking dialog led to a listing of other films). - // A layer drawn over the window is such a question only when the - // task's own press opened it; a calendar left open is in the way. - if self.front.opened_dialog || !matches!(self.front.surface.as_str(), "window" | "layer") { - self.history.push(format!( - "{} lies behind the dialog in front; act within the dialog instead", - label(target) - )); - return Ok(reply); - } - // A layer in front that closes with a control of its own (a consent - // banner's "Allow Selection") is closed with it: Escape leaves such a - // banner where it is. - let screen = self.look().await?; - if let Some(closer) = front_closer(&screen, &self.stop_before, &self.step_cleared) { - self.step_cleared.insert(signature(&closer)); - let pressed = closer.clone(); - self.act(log, "click (uncover)", Some(&closer), move |backend| { - backend.execute(JevOperation::Click, Some(pressed), None) - }) - .await?; - self.history.push(format!( - "{} was covered by a layer in front; pressed {} to close it", - label(target), - label(&closer) - )); - } else { - let app = self.app.clone(); - self.act(log, "press escape (uncover)", None, move |backend| { - backend.press(&app, "escape") - }) - .await?; - self.history.push(format!( - "{} was covered by something; pressed escape to close it", - label(target) - )); - } - let retried = target.clone(); - self.act(log, verb, Some(target), move |backend| { - backend.execute(operation, Some(retried), None) - }) - .await - } - /// Dismisses whatever is blocking the step, choosing only safe controls. pub(super) async fn clear_obstacle( &mut self, diff --git a/crates/tinycomputer-engine/src/agentic/flow/act/uncover.rs b/crates/tinycomputer-engine/src/agentic/flow/act/uncover.rs new file mode 100644 index 00000000..5032fb1d --- /dev/null +++ b/crates/tinycomputer-engine/src/agentic/flow/act/uncover.rs @@ -0,0 +1,92 @@ +//! Pressing a target the page refuses as covered: closing what lies over +//! it, then pressing the same target once more. + +use tinycomputer_bus::JevOperation; + +use crate::agentic::flow::{ + FlowRun, Halt, StepLog, + attention::front_closer, + backend::AgentBackend, + view::{Candidate, label, signature}, +}; + +use super::covered; + +impl FlowRun<'_, B> { + /// Performs `operation` on an already-vetted `target`. When the click + /// is refused because something covers it — a drawer, a menu, a consent + /// banner, or a result card's own click layer — closes what covers it + /// once and tries the same target again: with the least committal + /// control of a layer in front ([`front_closer`]; never a layer the + /// step's `intent` names, or the one `target` sits in), or else with + /// Escape. Neither chooses a new target. + pub(in crate::agentic::flow) async fn press_uncovering( + &mut self, + log: &mut StepLog, + verb: &str, + target: &Candidate, + operation: JevOperation, + intent: &str, + ) -> Result { + let chosen = target.clone(); + let reply = self + .act(log, verb, Some(target), move |backend| { + backend.execute(operation, Some(chosen), None) + }) + .await?; + if !covered(&reply) { + return Ok(reply); + } + // A dialog in front is the page's question (a format, a quantity), + // not a popover in the way: Escape would close it, and pressing what + // lies behind it leaves the flow it began (live, a movie's language + // link behind its booking dialog led to a listing of other films). + // A layer drawn over the window is such a question only when the + // task's own press opened it; a calendar left open is in the way. + if self.front.opened_dialog || !matches!(self.front.surface.as_str(), "window" | "layer") { + self.history.push(format!( + "{} lies behind the dialog in front; act within the dialog instead", + label(target) + )); + return Ok(reply); + } + // A layer in front that closes with a control of its own (a consent + // banner's "Allow Selection") is closed with it: Escape leaves such a + // banner where it is. + let screen = self.look().await?; + if let Some(closer) = front_closer( + &screen, + target, + intent, + &self.stop_before, + &self.step_cleared, + ) { + self.step_cleared.insert(signature(&closer)); + let pressed = closer.clone(); + self.act(log, "click (uncover)", Some(&closer), move |backend| { + backend.execute(JevOperation::Click, Some(pressed), None) + }) + .await?; + self.history.push(format!( + "{} was covered by a layer in front; pressed {} to close it", + label(target), + label(&closer) + )); + } else { + let app = self.app.clone(); + self.act(log, "press escape (uncover)", None, move |backend| { + backend.press(&app, "escape") + }) + .await?; + self.history.push(format!( + "{} was covered by something; pressed escape to close it", + label(target) + )); + } + let retried = target.clone(); + self.act(log, verb, Some(target), move |backend| { + backend.execute(operation, Some(retried), None) + }) + .await + } +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/attention/attention_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/attention/attention_tests.rs index 57d0cd09..ab1b2a40 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/attention/attention_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/attention/attention_tests.rs @@ -4,7 +4,7 @@ use std::collections::BTreeSet; -use super::{ESCAPED, MAX_DISTRACTION_SIZE, MAX_DISTRACTIONS, find::distractions}; +use super::{ESCAPED, MAX_DISTRACTION_SIZE, MAX_DISTRACTIONS, find::distractions, front_closer}; use crate::agentic::flow::view::{Candidate, Screen, signature}; fn button(name: &str, path: &[&str]) -> Candidate { @@ -269,3 +269,73 @@ fn something_covering_what_the_step_needs_is_cleared_with_escape() { .is_empty() ); } + +#[test] +fn a_covered_press_closes_a_layer_in_front_but_never_its_own() { + // A size popover the step works in, with a toast lying over its rows. + let sizes = ["main", "popover \"Sizes\""]; + let toast = ["alert \"Saved to your wishlist\""]; + let target = button("Size M", &sizes); + let mut candidates = content(); + candidates.extend([target.clone(), button("Close", &sizes)]); + let with_toast = |mut candidates: Vec| { + candidates.push(button("Close", &toast)); + screen(candidates) + }; + let closer = front_closer( + &with_toast(candidates.clone()), + &target, + "choose size M", + &[], + &BTreeSet::new(), + ) + .unwrap(); + assert_eq!(closer.path, toast, "the toast's, not the popover's"); + + // With only the step's own layer in front, nothing is closed. + let own = front_closer( + &screen(candidates.clone()), + &target, + "choose size M", + &[], + &BTreeSet::new(), + ); + assert!(own.is_none(), "{own:?}"); + + // Nor is the target itself, the toast's own button. + let close_toast = button("Close", &toast); + let itself = front_closer( + &with_toast(content()), + &close_toast, + "close the toast", + &[], + &BTreeSet::new(), + ); + assert!(itself.is_none(), "{itself:?}"); + + // A layer the step names is the step's. + let mut candidates = content(); + candidates.extend(consent().into_iter().map(|mut control| { + control.path = vec!["dialog \"Cookie consent\"".to_owned()]; + control + })); + let named = front_closer( + &screen(candidates.clone()), + &candidates[0], + "accept the cookie consent", + &[], + &BTreeSet::new(), + ); + assert!(named.is_none(), "{named:?}"); + let other = front_closer( + &screen(candidates.clone()), + &candidates[0], + "search for flights", + &[], + &BTreeSet::new(), + ); + assert_eq!( + other.and_then(|closer| closer.name).as_deref(), + Some("Reject all") + ); +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/attention/mod.rs b/crates/tinycomputer-engine/src/agentic/flow/attention/mod.rs index cb4057b8..c9abb494 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/attention/mod.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/attention/mod.rs @@ -14,7 +14,10 @@ //! Only when there is a candidate is Jev asked, with one Choice, and only a //! clearly agreed pick (`evidence/`) is cleared, with the region's //! least-committal control: rejecting or essential-only first, closing next, -//! accepting last. +//! accepting last. One exception asks no one: when the page refuses a press +//! as covered, the least-committal control of a layer in front, other than +//! the step's own, is pressed before the press is tried again +//! ([`front_closer`]). //! //! `find` finds the distractions without asking anyone, and `clear` asks //! Jev and clears the one it picks. @@ -24,7 +27,7 @@ mod find; use std::collections::BTreeSet; -use super::view::{Candidate, Screen}; +use super::view::{Candidate, Screen, signature}; /// Most distractions one attention question offers. pub(super) const MAX_DISTRACTIONS: usize = 4; @@ -56,18 +59,24 @@ pub(super) struct Distraction { /// The control that closes a layer in front of the page, the least /// committal its region holds (a consent banner's "Allow Selection" before /// its "Allow all"), when one does and it was not pressed in this step: what -/// a press the page refused as covered clears before trying again. Live, a -/// consent banner lay over "Add To Cart", Escape left it there, and every -/// press was refused. +/// a press of `target` the page refused as covered clears before trying +/// again. Live, a consent banner lay over "Add To Cart", Escape left it +/// there, and every press was refused. A layer the step's `intent` names, +/// or the one `target` itself sits in (a popover a toast lies over), is the +/// step's, and is never closed this way. pub(super) fn front_closer( screen: &Screen, + target: &Candidate, + intent: &str, stop_before: &[String], cleared: &BTreeSet, ) -> Option { - find::distractions(screen, "", stop_before, cleared) + let pressed = signature(target); + find::distractions(screen, intent, stop_before, cleared) .into_iter() .filter(|distraction| distraction.front) - .find_map(|distraction| distraction.closer) + .filter_map(|distraction| distraction.closer) + .find(|closer| signature(closer) != pressed && !target.path.starts_with(&closer.path)) } /// The key a step's Escape at something covering the page is remembered diff --git a/crates/tinycomputer-engine/src/agentic/flow/steps/list.rs b/crates/tinycomputer-engine/src/agentic/flow/steps/list.rs index 30c87747..75697c51 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/steps/list.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/steps/list.rs @@ -114,7 +114,7 @@ impl FlowRun<'_, B> { return Ok(self.picked_as_selected(&picked, &from, &by, &summary, groups.len())); } let reply = self - .press_uncovering(log, "click", &primary, JevOperation::Click) + .press_uncovering(log, "click", &primary, JevOperation::Click, &from) .await?; if !reply.ok { let why = reply.error.as_ref().map_or_else( diff --git a/docs/technical/decision-loops.md b/docs/technical/decision-loops.md index 23e4d7d2..9f52bbbd 100644 --- a/docs/technical/decision-loops.md +++ b/docs/technical/decision-loops.md @@ -298,26 +298,22 @@ own link, so the link is "covered" by the card itself; the browser surface then clicks through at the link's position, but only when the exact target (matched by name, and on the page, by the one element under that point with that label) sits in the same card as the cover and no dialog is involved. -Anything else comes back covered, and the runtime closes what lies over it -and retries the *same* already-vetted target — in a `do` step's click and in -`pick`'s alike. A layer in front with a control that closes it (a consent -banner's "Allow Selection") is closed with its least committal one, as the -attention pass would; otherwise the runtime presses Escape once. Live, -Escape left a consent banner over "Add To Cart" and every press was refused. -Neither chooses a new element for the step, so nothing exposed by +Anything else comes back covered: a front layer's least committal control (a +consent banner's "Allow Selection"; never the target's own layer or one the +step names), or else Escape, is pressed once, and the *same* vetted target +retried (`do` and `pick` alike), so nothing exposed by dismissing whatever covered the click is ever pressed without going through grounding and `is_destructive` again on a later turn. A dismissal the completion judge would otherwise never see ends the step -immediately: when the last action pressed a control whose own words the -step's intent names ("Accept Essential Only" for a step about accepting -cookies), or the intent asks to dismiss, close, accept, decline, reject, or -skip a banner, dialog, popup, cookie notice, modal, overlay, or prompt, and -the screen has returned to the application's own window, the step ends as -`Done` — a closed overlay leaves no trace afterward for the judge to read. -The same check runs once more after the very last turn, so a dismissal that -lands on the last permitted turn is not reported as failed for want of -another look. +immediately: when the last action pressed a control whose own words the step's +intent names ("Accept Essential Only" for a step about accepting cookies), or +the intent asks to dismiss, close, accept, decline, reject, or skip a banner, +dialog, popup, cookie notice, modal, overlay, or prompt, and the screen has +returned to the application's own window, the step ends as `Done` — a closed +overlay leaves no trace afterward for the judge to read. The same check runs +once more after the very last turn, so a dismissal that lands on the last +permitted turn is not reported as failed for want of another look. After eight turns without an end, the runtime looks one last time. If the completion estimate reaches 0.75 the step is `Done`; otherwise it fails with From 8fa166abc769b30ecb6b953ebef48fd8fafe3286 Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 7 Oct 2026 20:06:27 +0530 Subject: [PATCH 26/44] Ask whether a stalled step is done only with the completion loop on Whether a step is done is the completion loop's question, and every other completion question in the do loop is gated on it; the stalled-step check asked holds() even when a run turned that loop off. Such a run now fails a stalled step outright, as before the check. decision-loops.md, decision-thresholds.md (STALL_TURNS and DONE) and jev-questions.md still described a stall as an outright failure, and now say what happens instead. --- crates/tinycomputer-engine/src/agentic/flow/act/turns.rs | 9 ++++++--- docs/crates/tinycomputer-engine/flow/the-do-loop.md | 4 ++-- docs/technical/decision-loops.md | 7 ++++--- docs/technical/decision-thresholds.md | 4 ++-- docs/technical/jev-questions.md | 2 +- 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/crates/tinycomputer-engine/src/agentic/flow/act/turns.rs b/crates/tinycomputer-engine/src/agentic/flow/act/turns.rs index ae44097c..896a42b6 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/act/turns.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/act/turns.rs @@ -4,7 +4,7 @@ use std::time::Instant; use serde_json::json; -use tinycomputer_bus::StepOutcome; +use tinycomputer_bus::{FlowLoop, StepOutcome}; use crate::agentic::flow::{ Ended, FlowRun, Halt, StepLog, @@ -334,7 +334,8 @@ impl FlowRun<'_, B> { } /// Ends a step whose last [`STALL_TURNS`] actions changed nothing: done, - /// when the screen already shows what the step was for, else failed. + /// when the screen already shows what the step was for (asked only when + /// the completion loop is on), else failed. /// /// A step whose work the page did by itself (a search box that lists /// results as it is typed in) has nothing left to press. Live, rescues @@ -346,7 +347,9 @@ impl FlowRun<'_, B> { let condition = format!( "the screen already shows the result that the step {intent:?} is meant to bring about" ); - if self.holds(log, &condition).await? >= DONE { + // Whether a step is done is the completion loop's question: a run + // that turned it off fails a stalled step outright, as before. + if self.enabled(FlowLoop::Completion) && self.holds(log, &condition).await? >= DONE { return Ok(Ended::new( StepOutcome::AlreadyDone, "the last three actions changed nothing, and the screen already shows what this step was for", diff --git a/docs/crates/tinycomputer-engine/flow/the-do-loop.md b/docs/crates/tinycomputer-engine/flow/the-do-loop.md index 1aa54c19..8a10092b 100644 --- a/docs/crates/tinycomputer-engine/flow/the-do-loop.md +++ b/docs/crates/tinycomputer-engine/flow/the-do-loop.md @@ -29,8 +29,8 @@ would look different on every single turn, and stall detection would never fire. If nothing changed, the element that was just pressed is banned for the rest of the step, so the loop does not click the same dead button twice. After three turns in a row with no change (`STALL_TURNS`), one -question asks whether the screen already shows the result the step is -meant to bring about (a search box that listed results as it was typed in +question (unless the run turned the completion loop off) asks whether the +screen already shows the result the step is meant to bring about (a search box that listed results as it was typed in leaves "press search" nothing to do). If it clearly does (`DONE`), the step ends `AlreadyDone`; otherwise it fails with the note "the last three actions changed nothing on screen". Live, 25 of a day's 189 rescues found such a diff --git a/docs/technical/decision-loops.md b/docs/technical/decision-loops.md index 9f52bbbd..9b6dd8df 100644 --- a/docs/technical/decision-loops.md +++ b/docs/technical/decision-loops.md @@ -186,9 +186,10 @@ The runtime compares a fingerprint of the screen before and after the last action. The fingerprint leaves refs out on purpose: every snapshot mints new refs, so a fingerprint that included them would see a change on every turn and stall detection would never fire. If nothing changed, the element that was -pressed is banned for the rest of the step. Three turns in a row with no change -(`STALL_TURNS`) fail the step with "the last three actions changed nothing on -screen". +pressed is banned for the rest of the step. After three turns in a row with no +change (`STALL_TURNS`), Jev is asked whether the screen already shows what the +step was for (`holds`; not with the completion loop off): at `DONE` the step is +`AlreadyDone`, else it fails: "the last three actions changed nothing on screen". ### 2. Judge diff --git a/docs/technical/decision-thresholds.md b/docs/technical/decision-thresholds.md index 3193bd59..018f7eb3 100644 --- a/docs/technical/decision-thresholds.md +++ b/docs/technical/decision-thresholds.md @@ -7,7 +7,7 @@ Change a constant and its row together. | Constant | Value | Where | Meaning | |---|---|---|---| -| `DONE` | 0.75 | `act/mod.rs` | completion that ends a step after acting; also the bar for `verify`, `wait_for`, `if`, `repeat_until` | +| `DONE` | 0.75 | `act/mod.rs` | completion that ends a step after acting; also the bar for `verify`, `wait_for`, `if`, `repeat_until`, and for a stalled `do` step's screen already showing its result | | `ALREADY_DONE` | 0.85 | `act/mod.rs` | completion that skips a step before acting | | `BLOCKED` | 0.70 | `act/mod.rs` | obstacle probability that triggers dismissal | | `LEANS_DONE` | 0.50 | `act/mod.rs` | completion under which a `finished` move is overruled after acting | @@ -36,7 +36,7 @@ Change a constant and its row together. | `REPAIR_TURNS` | 4 | `reflect.rs` | turns one reflection repair may spend | | `MAX_ACTIONS` / `MAX_CALLS` | 120 / 10000 | `mod.rs` | per-run caps on actions and Jev calls | | `MAX_VOTES` | 9 | `vote.rs` | most framings one decision is asked in; a deliberated decision is widened up to it | -| `STALL_TURNS` / `MAX_IDLE_WAITS` | 3 / 2 | `act/mod.rs` | unchanged turns before a step fails; idle waits before Jev may not wait again | +| `STALL_TURNS` / `MAX_IDLE_WAITS` | 3 / 2 | `act/mod.rs` | unchanged turns before a step ends: `AlreadyDone` when the screen already shows its result (at `DONE`, with the completion loop on), else failed; idle waits before Jev may not wait again | | `MAX_OBSTACLES` / `MAX_UNDOS` | 2 / 2 | `act/mod.rs` | obstacles dismissed and undos run per step at most | | `FIELD_ERROR` | 0.70 | `enter/mod.rs` | field-error probability that makes a slot be entered again | | `NOT_ASKED` | 0.35 | `enter/mod.rs` | "the form asks for it" probability under which a slot with no field is taken as not asked for | diff --git a/docs/technical/jev-questions.md b/docs/technical/jev-questions.md index 2f9d4e7c..16b12fc6 100644 --- a/docs/technical/jev-questions.md +++ b/docs/technical/jev-questions.md @@ -195,7 +195,7 @@ request is also re-asked in more framings, which changes no id. Slot names go to Jev. Slot values never do. -### Conditions (`steps/condition.rs::holds`, for `verify`, `wait_for`, `if`, `repeat_until`, and `stop_before`'s after-check) +### Conditions (`steps/condition.rs::holds`, for `verify`, `wait_for`, `if`, `repeat_until`, `stop_before`'s after-check, and a stalled `do` step's already-done check) | Id | Type | Given | Answer used as | |---|---|---|---| From 577b2ce2cd5a0ac6aeee3ed41b45653cd3a51f85 Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 7 Oct 2026 20:06:32 +0530 Subject: [PATCH 27/44] Check that a wait which saw the page stay still is not settled The simulator now keeps a trail of its settle and await_change calls. The late-suggestion tests check that a wait that saw a change is settled like any action, and one that saw a still page is not. --- .../src/agentic/flow/flow_tests/simulator.rs | 12 +++++++++++- .../src/agentic/flow/flow_tests/suggestion_tests.rs | 8 ++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs index 10c181c0..5cf021e4 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs @@ -99,6 +99,9 @@ pub(super) struct Sim { /// The field typed or pasted into last: where the focus stays. pub(super) focused: Option, pub(super) quirks: BTreeSet, + /// The calls that touch no element, in order: each `settle`, and each + /// `await_change` as `changed` or `still`. + pub(super) trail: Vec<&'static str>, } impl Sim { @@ -420,7 +423,14 @@ impl AgentBackend for App { } fn await_change(&self, _ms: u64) -> bool { - await_place_rows(&mut self.sim()) + let mut sim = self.sim(); + let changed = await_place_rows(&mut sim); + sim.trail.push(if changed { "changed" } else { "still" }); + changed + } + + fn settle(&self) { + self.sim().trail.push("settle"); } fn paste(&self, _app: &str, target: &Candidate, text: &str) -> DesktopResponse { diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/suggestion_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/suggestion_tests.rs index f6c90e93..d6c18dc6 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/suggestion_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/suggestion_tests.rs @@ -124,6 +124,10 @@ async fn a_place_box_whose_rows_come_late_is_looked_at_again_as_they_show() { "rows drawn after {late} waits" ); assert_eq!(waits(&run), vec![String::new(); usize::from(late)]); + // A wait that saw the page change is settled, as any action is. + let trail = run.app.sim().trail.clone(); + let watched = trail.iter().position(|call| *call == "changed").unwrap(); + assert_eq!(trail.get(watched + 1), Some(&"settle"), "{trail:?}"); } } @@ -147,6 +151,10 @@ async fn a_place_box_on_a_page_that_stays_still_is_waited_on_once() { assert_eq!(run.app.sim().fields["Pickup location"], "Nowhere Lane"); assert_eq!(waits(&run), ["nothing changed"]); assert!(!asked_for_a_suggestion(&run)); + // A wait that saw the page stay still has nothing to settle. + let trail = run.app.sim().trail.clone(); + let watched = trail.iter().position(|call| *call == "still").unwrap(); + assert_ne!(trail.get(watched + 1), Some(&"settle"), "{trail:?}"); } #[tokio::test] From 99ad17a58bab96f450d0627f14b1150332cf5c5e Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 7 Oct 2026 20:06:43 +0530 Subject: [PATCH 28/44] Build nothing for the journal when it is off The plan, rescue and resume events were built for every task, journal on or off, and journal_event drew a run id even when off: the journal's own rule is to build nothing then. FlowRunner::journal and journal_event now take a closure, called only when an event is written, and the module's runner and JevRuntime::journal_event return before anything is built when the journal is off (JevRuntime::journaling). A resume was journaled before ContinueTask's answer was checked, so a refused answer, or one still missing values, recorded a wait the task had not left, and --split counted it twice. It is journaled once the task has left the wait. A plan's wall_ms is timed on the planner alone: a browser slower to open than the plan was counted as planning. ModelUse's calls are documented as excluding a hosted call's own retries. --- .../src/agentic/journal/mod.rs | 10 +++++- .../src/agentic/runtime.rs | 16 ++++++++-- crates/tinycomputer-engine/src/planner/mod.rs | 4 ++- .../src/task/controller.rs | 32 +++++++++++-------- crates/tinycomputer-engine/src/task/drive.rs | 19 +++++++---- crates/tinycomputer-engine/src/task/mod.rs | 12 +++++-- .../tinycomputer-engine/src/task/recovery.rs | 8 ++--- .../src/task/task_tests.rs | 4 +-- .../src/task/task_tests/plan_tests.rs | 11 +++++++ .../src/task/task_tests/runner_tests.rs | 4 +-- .../tinycomputer/src/tinybus_module/runner.rs | 4 +-- .../tinybus_module_tests/tasks_tests.rs | 10 ++++-- docs/technical/jev-journal.md | 2 +- 13 files changed, 93 insertions(+), 43 deletions(-) diff --git a/crates/tinycomputer-engine/src/agentic/journal/mod.rs b/crates/tinycomputer-engine/src/agentic/journal/mod.rs index 4ac4e5ba..4b3d3ef4 100644 --- a/crates/tinycomputer-engine/src/agentic/journal/mod.rs +++ b/crates/tinycomputer-engine/src/agentic/journal/mod.rs @@ -133,16 +133,24 @@ impl Journal { self.run.is_some() } + /// Whether the journal is on: it has a folder to write runs under. + pub(crate) fn is_on(&self) -> bool { + self.root.is_some() + } + /// This journal, writing to a new run named for the time and `kind`. A /// journal that is off stays off. pub(crate) fn fresh(&self, kind: &str) -> Self { + if !self.is_on() { + return self.clone(); + } self.named(&fresh_id(kind)) } /// This journal with a run begun: a `run` event in the current run, or, /// when there is none yet, in a new one named for the time and `kind`. pub(crate) fn begin(&self, kind: &str, label: &str, model: &str) -> Self { - let journal = if self.run.is_some() { + let journal = if self.run.is_some() || !self.is_on() { self.clone() } else { self.named(&fresh_id(kind)) diff --git a/crates/tinycomputer-engine/src/agentic/runtime.rs b/crates/tinycomputer-engine/src/agentic/runtime.rs index e81662de..ce08af8d 100644 --- a/crates/tinycomputer-engine/src/agentic/runtime.rs +++ b/crates/tinycomputer-engine/src/agentic/runtime.rs @@ -182,14 +182,24 @@ impl JevRuntime { /// none open, into a new run named for the time and `kind`. It is for /// time a task spends outside its flows, such as planning, a rescue, or /// waiting on a person, so the task's journal accounts for all of its - /// time. Does nothing when the journal is off. - pub fn journal_event(&self, kind: &str, fields: serde_json::Value) { + /// time. `fields` is only built when recording: nothing is, and no run + /// is opened, when the journal is off. + pub fn journal_event(&self, kind: &str, fields: impl FnOnce() -> serde_json::Value) { + if !self.journaling() { + return; + } let journal = if self.journal.is_open() { self.journal.clone() } else { self.journal.fresh(kind) }; - journal.record(kind, || fields); + journal.record(kind, fields); + } + + /// Whether this runtime's debug journal is on. + #[must_use] + pub fn journaling(&self) -> bool { + self.journal.is_on() } /// The directory this runtime's current run journal is written to, if diff --git a/crates/tinycomputer-engine/src/planner/mod.rs b/crates/tinycomputer-engine/src/planner/mod.rs index 00f47577..3bddd6fe 100644 --- a/crates/tinycomputer-engine/src/planner/mod.rs +++ b/crates/tinycomputer-engine/src/planner/mod.rs @@ -55,7 +55,9 @@ pub enum Role { /// The task journals it, so a run shows what its planning and rescues cost. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct ModelUse { - /// Calls made: the first, and one per repair. + /// Calls made: the first, and one per repair. A hosted call tried again + /// after a passing failure (`hosted.rs`) counts once: its tries, and + /// the waits between them, are inside it. pub calls: u32, /// Bytes of text in the first call's turns. pub sent_bytes: usize, diff --git a/crates/tinycomputer-engine/src/task/controller.rs b/crates/tinycomputer-engine/src/task/controller.rs index 4a0cb613..66812c79 100644 --- a/crates/tinycomputer-engine/src/task/controller.rs +++ b/crates/tinycomputer-engine/src/task/controller.rs @@ -126,11 +126,10 @@ impl Tasks { ) .await; // No task exists yet, so the plan journals to a run of its own. - self.runner.journal( - None, - "plan", - super::timing::planned(&outcome, used, started.elapsed(), planner.configuration()), - ); + let drafting = started.elapsed(); + self.runner.journal(None, "plan", &|| { + super::timing::planned(&outcome, used, drafting, planner.configuration()) + }); match outcome { Ok(plan) => AgentResponse::ok(plan), Err(reason) => AgentResponse::err(AgentError::new( @@ -274,14 +273,9 @@ impl Tasks { .ok() .and_then(|state| state.waiting_since) .map(|since| since.elapsed()); - if let Some(waited) = waited { - self.runner.journal( - Some(&request.id), - "resume", - super::timing::resumed(state_name(&status), waited), - ); - } - match status { + let id = request.id.clone(); + let paused = state_name(&status); + let reply = match status { TaskStatus::NeedsInput { .. } => self.supply(&cell, request), TaskStatus::NeedsApproval { .. } => self.decide(&cell, request.approve), TaskStatus::NeedsHuman { .. } => self.retry(&cell), @@ -294,7 +288,19 @@ impl Tasks { "call AwaitTask until the task asks for something", true, )), + }; + // The wait is over only once the task has left it: an answer that is + // refused, or that still leaves values missing, keeps it waiting. + let left = cell + .state + .lock() + .is_ok_and(|state| state.waiting_since.is_none()); + if let (Some(waited), true) = (waited, left) { + self.runner.journal(Some(&id), "resume", &|| { + super::timing::resumed(paused, waited) + }); } + reply } /// Stops a task, and lets go of whatever surface it still holds. diff --git a/crates/tinycomputer-engine/src/task/drive.rs b/crates/tinycomputer-engine/src/task/drive.rs index 46dba653..9589f7dd 100644 --- a/crates/tinycomputer-engine/src/task/drive.rs +++ b/crates/tinycomputer-engine/src/task/drive.rs @@ -38,19 +38,24 @@ pub(super) async fn plan_then_drive( ); let id = cell.view.borrow().id.clone(); let started = Instant::now(); - let planning = planner.plan_measured(&task, &names, &secrets, &surfaces); + // Timed on its own: a browser slower to open than the plan is to draft + // is not planning time. + let planning = async { + let drafted = planner + .plan_measured(&task, &names, &secrets, &surfaces) + .await; + (drafted, started.elapsed()) + }; // A browser-only task's browser opens while the plan is drafted, so the // first step need not wait for it: whatever the plan says, it runs there. - let (outcome, used) = if constraints.surfaces == [SurfaceKind::Browser] { + let ((outcome, used), drafting) = if constraints.surfaces == [SurfaceKind::Browser] { tokio::join!(planning, runner.prepare(&id, &constraints)).0 } else { planning.await }; - runner.journal( - Some(&id), - "plan", - super::timing::planned(&outcome, used, started.elapsed(), planner.configuration()), - ); + runner.journal(Some(&id), "plan", &|| { + super::timing::planned(&outcome, used, drafting, planner.configuration()) + }); let plan = match outcome { Ok(plan) => plan, Err(reason) => { diff --git a/crates/tinycomputer-engine/src/task/mod.rs b/crates/tinycomputer-engine/src/task/mod.rs index 69de41a4..4606181c 100644 --- a/crates/tinycomputer-engine/src/task/mod.rs +++ b/crates/tinycomputer-engine/src/task/mod.rs @@ -125,9 +125,15 @@ pub trait FlowRunner: Send + Sync + 'static { /// Writes an `event` of the time a task spends outside its flows /// (`plan`, `rescue`, `resume`) to the debug journal: the task's own, /// or for `PlanTask`, which plans before any task exists (`task` is - /// `None`), a run of its own. Does nothing by default, and nothing when - /// the journal is off. - fn journal(&self, _task: Option<&TaskId>, _event: &str, _fields: serde_json::Value) {} + /// `None`), a run of its own. `fields` is only called when the event is + /// written: nothing is built by default, or when the journal is off. + fn journal( + &self, + _task: Option<&TaskId>, + _event: &str, + _fields: &dyn Fn() -> serde_json::Value, + ) { + } } /// How many tasks the controller holds; finished ones are dropped first. diff --git a/crates/tinycomputer-engine/src/task/recovery.rs b/crates/tinycomputer-engine/src/task/recovery.rs index 7002297a..8ed8a3a6 100644 --- a/crates/tinycomputer-engine/src/task/recovery.rs +++ b/crates/tinycomputer-engine/src/task/recovery.rs @@ -212,9 +212,7 @@ async fn ask( let took = started.elapsed(); let outcome = timing::answered(&answer, used.is_none()); let (record, guided) = record(briefing.failed, briefing.failure.clone(), answer); - runner.journal( - Some(id), - "rescue", + runner.journal(Some(id), "rescue", &|| { timing::rescued(&timing::Rescued { attempt, limit, @@ -223,8 +221,8 @@ async fn ask( outcome, record: &record, model: rescuer.configuration(), - }), - ); + }) + }); let spent_ms = u64::try_from(took.as_millis()).unwrap_or(u64::MAX); (record, guided, spent_ms) } diff --git a/crates/tinycomputer-engine/src/task/task_tests.rs b/crates/tinycomputer-engine/src/task/task_tests.rs index b9526877..76d42adc 100644 --- a/crates/tinycomputer-engine/src/task/task_tests.rs +++ b/crates/tinycomputer-engine/src/task/task_tests.rs @@ -100,11 +100,11 @@ impl FlowRunner for Script { Box::pin(async {}) } - fn journal(&self, task: Option<&TaskId>, event: &str, fields: serde_json::Value) { + fn journal(&self, task: Option<&TaskId>, event: &str, fields: &dyn Fn() -> serde_json::Value) { self.journaled .lock() .unwrap() - .push((task.cloned(), event.to_owned(), fields)); + .push((task.cloned(), event.to_owned(), fields())); } } diff --git a/crates/tinycomputer-engine/src/task/task_tests/plan_tests.rs b/crates/tinycomputer-engine/src/task/task_tests/plan_tests.rs index 02d50066..d61a040d 100644 --- a/crates/tinycomputer-engine/src/task/task_tests/plan_tests.rs +++ b/crates/tinycomputer-engine/src/task/task_tests/plan_tests.rs @@ -90,6 +90,17 @@ async fn a_plan_that_needs_values_asks_and_a_failed_plan_says_so() { std::slice::from_ref(&started.id), "a task waiting on a person holds no browser" ); + // An answer that still leaves a value missing keeps the task waiting: + // no wait is over yet. + assert!( + tasks + .continue_task(ContinueTaskRequest { + id: started.id.clone(), + ..ContinueTaskRequest::default() + }) + .ok + ); + assert_eq!(journaled(&script, "resume").len(), 0, "no resume yet"); assert!( tasks .continue_task(ContinueTaskRequest { diff --git a/crates/tinycomputer-engine/src/task/task_tests/runner_tests.rs b/crates/tinycomputer-engine/src/task/task_tests/runner_tests.rs index 6ced4efe..0dfea938 100644 --- a/crates/tinycomputer-engine/src/task/task_tests/runner_tests.rs +++ b/crates/tinycomputer-engine/src/task/task_tests/runner_tests.rs @@ -28,8 +28,8 @@ async fn a_runner_that_only_runs_flows_reads_nothing_and_holds_nothing() { RunsOnly.release(&task); // Nothing to get ready, and no journal to write to. RunsOnly.prepare(&task, &TaskConstraints::default()).await; - RunsOnly.journal(Some(&task), "plan", serde_json::json!({"wall_ms": 1})); - RunsOnly.journal(None, "plan", serde_json::json!({})); + RunsOnly.journal(Some(&task), "plan", &|| serde_json::json!({"wall_ms": 1})); + RunsOnly.journal(None, "plan", &|| serde_json::json!({})); assert_eq!( RunsOnly.visible_text(&task).await, [] as [std::string::String; 0] diff --git a/crates/tinycomputer/src/tinybus_module/runner.rs b/crates/tinycomputer/src/tinybus_module/runner.rs index 3821fd26..042f2e1b 100644 --- a/crates/tinycomputer/src/tinybus_module/runner.rs +++ b/crates/tinycomputer/src/tinybus_module/runner.rs @@ -159,8 +159,8 @@ impl FlowRunner for WorkspaceRunner { }) } - fn journal(&self, task: Option<&TaskId>, event: &str, fields: serde_json::Value) { - let Some(runtime) = self.jev.as_ref() else { + fn journal(&self, task: Option<&TaskId>, event: &str, fields: &dyn Fn() -> serde_json::Value) { + let Some(runtime) = self.jev.as_ref().filter(|runtime| runtime.journaling()) else { return; }; match task { diff --git a/crates/tinycomputer/src/tinybus_module/tinybus_module_tests/tasks_tests.rs b/crates/tinycomputer/src/tinybus_module/tinybus_module_tests/tasks_tests.rs index e13a97fc..bcc5a281 100644 --- a/crates/tinycomputer/src/tinybus_module/tinybus_module_tests/tasks_tests.rs +++ b/crates/tinycomputer/src/tinybus_module/tinybus_module_tests/tasks_tests.rs @@ -184,8 +184,12 @@ async fn the_runner_journals_time_outside_a_tasks_flows_into_its_journal() { Some(jev), browser(), ); - runner.journal(Some(&TaskId::new("t-1")), "rescue", json!({"wall_ms": 7})); - runner.journal(None, "plan", json!({"wall_ms": 9})); + runner.journal( + Some(&TaskId::new("t-1")), + "rescue", + &|| json!({"wall_ms": 7}), + ); + runner.journal(None, "plan", &|| json!({"wall_ms": 9})); let task = std::fs::read_to_string(scratch.join("task-t-1").join(JOURNAL_FILE)).unwrap(); assert!(task.contains(r#""event":"rescue""#), "{task}"); @@ -203,7 +207,7 @@ async fn the_runner_journals_time_outside_a_tasks_flows_into_its_journal() { // With no Jev runtime there is no journal to write to. crate::tinybus_module::runner::WorkspaceRunner::new(crate::Desktop::new(), None, browser()) - .journal(None, "plan", json!({})); + .journal(None, "plan", &|| json!({})); } #[tokio::test] diff --git a/docs/technical/jev-journal.md b/docs/technical/jev-journal.md index 6602e2d8..8bcd0aad 100644 --- a/docs/technical/jev-journal.md +++ b/docs/technical/jev-journal.md @@ -80,7 +80,7 @@ has `""`, and goal and intent runs carry their goal or intent text. | `denoise` | a `do` step's screen oscillates | `step`, `oscillation` (the presses banned) | | `step` | a flow step ends | `step`, `kind`, `text`, `outcome`, `note`, `turns`, `jev_calls`, `actions`, `loops`, `confidence`, `wall_ms` | | `end` | a flow run ends | `stop`, `wall_ms`, `actions`, `metrics`, `learned` | -| `plan` | the planner drafts a task's flow (`PlanTask`, or `StartTask` with a task) | `wall_ms`, `calls` (model calls, repairs included), `sent_bytes` (the first call's text), `model`, `ok`; on success `steps`, `questions`; on failure `error` | +| `plan` | the planner drafts a task's flow (`PlanTask`, or `StartTask` with a task) | `wall_ms`, `calls` (model calls, repairs included; a call tried again after a passing failure counts once, its waits in `wall_ms`), `sent_bytes` (the first call's text), `model`, `ok`; on success `steps`, `questions`; on failure `error` | | `rescue` | the rescuer answers for a failed step | `step` (from 1), `attempt`, `limit`, `wall_ms`, `calls` and `sent_bytes` (null when it gave no answer in time), `outcome` (`guided`, `gave_up`, `error`, `timeout`), `steps` (guidance steps), `covers`, `model` | | `resume` | a person answers a paused task (`ContinueTask`) | `state` it waited at (`needs_input`, `needs_approval`, `needs_human`), `waited_ms` since it first asked | From b14d3d3a5fb64f1a0dec912fd4080bb3d805527c Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 7 Oct 2026 20:06:48 +0530 Subject: [PATCH 29/44] task_live: make the memory's folder, and refuse switches it cannot read TASK_MEMORY's own example, target/task-live/memory/amazon.json, names a folder nothing made: the whole task ran, then saving what it learned failed and the run ended with an error before its pass or fail line. The folder is now made when missing. TASK_PLAN other than in-task, and TINYCOMPUTER_BROWSER_PRELAUNCH other than 0 or 1, were silently ignored; they are now refused. A FLOW_FILE run with TASK_PLAN=in-task no longer saves the given flow as a plan drafted inside the task. What a run keeps beside its report moves into its own module, so main.rs stays under 400 lines. --- .../src/bin/task_live/main.rs | 100 ++++++------------ .../src/bin/task_live/main_tests.rs | 64 +++-------- .../src/bin/task_live/saved/mod.rs | 71 +++++++++++++ .../src/bin/task_live/saved/saved_tests.rs | 63 +++++++++++ .../tinycomputer-examples/live-tasks.md | 6 +- 5 files changed, 189 insertions(+), 115 deletions(-) create mode 100644 crates/tinycomputer-examples/src/bin/task_live/saved/mod.rs create mode 100644 crates/tinycomputer-examples/src/bin/task_live/saved/saved_tests.rs diff --git a/crates/tinycomputer-examples/src/bin/task_live/main.rs b/crates/tinycomputer-examples/src/bin/task_live/main.rs index a799c2b2..0411db7b 100644 --- a/crates/tinycomputer-examples/src/bin/task_live/main.rs +++ b/crates/tinycomputer-examples/src/bin/task_live/main.rs @@ -91,13 +91,17 @@ use std::path::PathBuf; use std::time::Duration; use serde_json::{Value, json}; +use tinycomputer_bus::Flow; use tinycomputer_bus::agent::{ PlanTaskRequest, StartTaskRequest, SurfaceKind, TaskBudget, TaskConstraints, TaskOutput, }; -use tinycomputer_bus::{Flow, GroundingHint}; use tinycomputer_examples::host::{Host, LabError, jev_config, module_path}; use tinycomputer_examples::task::{Person, Terminal, conclude, follow, passed}; +mod saved; + +use saved::{read_memory, record_plan, remember}; + #[tokio::main] async fn main() -> Result<(), LabError> { let task = std::fs::read_to_string(env("TASK_FILE")?)?; @@ -111,15 +115,17 @@ async fn main() -> Result<(), LabError> { }; let kind = surface_kind()?; - let host = Host::load(&module_path(), module_config()?).await?; // `TASK_PLAN=in-task` leaves planning to `StartTask`, as `OpenHuman` does, // so the module can open the browser while it plans. - let in_task = std::env::var("TASK_PLAN").is_ok_and(|value| value.trim() == "in-task"); + let in_task = in_task(std::env::var("TASK_PLAN").ok().as_deref())?; + let host = Host::load(&module_path(), module_config()?).await?; let flow = match std::env::var("FLOW_FILE") { Ok(path) => Some(serde_json::from_str(&std::fs::read_to_string(path)?)?), Err(_) if in_task => None, Err(_) => Some(plan(&host, &task, &facts, &secret_facts, kind, &out).await?), }; + // A given flow is no plan the task drafted. + let drafted_in_task = flow.is_none(); // The task travels with the flow, so every Jev question is briefed on it. // Sessions open before the task are not its own, and `conclude` leaves // them alone. @@ -168,7 +174,7 @@ async fn main() -> Result<(), LabError> { .then_some(&Terminal as &dyn Person); let view = follow(&host, view, &BTreeMap::new(), limit, person).await?; conclude(&host, &view, &before, &out).await?; - if in_task { + if drafted_in_task { record_plan(&out)?; } if let Some(path) = &memory_file { @@ -199,17 +205,8 @@ fn module_config() -> Result { browser.insert(field.to_owned(), json!(value.trim())); } } - match optional("TINYCOMPUTER_BROWSER_PRELAUNCH") - .as_deref() - .map(str::trim) - { - Some("0") => { - browser.insert("prelaunch".to_owned(), json!(false)); - } - Some("1") => { - browser.insert("prelaunch".to_owned(), json!(true)); - } - _ => {} + if let Some(prelaunch) = prelaunch(optional("TINYCOMPUTER_BROWSER_PRELAUNCH").as_deref())? { + browser.insert("prelaunch".to_owned(), json!(prelaunch)); } for (variable, field) in [ ("TINYCOMPUTER_BROWSER_PERCEPTION", "perception"), @@ -341,6 +338,29 @@ fn read_facts(text: &str) -> Result<(BTreeMap, Vec), Lab Ok((facts, secret)) } +/// Whether `TASK_PLAN`'s `value` leaves planning to the task: `in-task` +/// does, and no value does not. +fn in_task(value: Option<&str>) -> Result { + match value.map(str::trim) { + None | Some("") => Ok(false), + Some("in-task") => Ok(true), + Some(other) => Err(format!("TASK_PLAN must be `in-task`, not `{other}`").into()), + } +} + +/// The module's `prelaunch` from `TINYCOMPUTER_BROWSER_PRELAUNCH`'s `value`: +/// `0` or `1`, or the module's own default when unset. +fn prelaunch(value: Option<&str>) -> Result, LabError> { + match value.map(str::trim) { + None | Some("") => Ok(None), + Some("0") => Ok(Some(false)), + Some("1") => Ok(Some(true)), + Some(other) => { + Err(format!("TINYCOMPUTER_BROWSER_PRELAUNCH must be `0` or `1`, not `{other}`").into()) + } + } +} + fn env(name: &str) -> Result { std::env::var(name).map_err(|_| format!("{name} is not set")) } @@ -375,55 +395,5 @@ async fn plan( Ok(plan.flow) } -/// Writes the flow a task planned for itself (`TASK_PLAN=in-task`) to -/// `plan.json` beside its report, as a plan drafted first would be. -fn record_plan(out: &std::path::Path) -> Result<(), LabError> { - let report: Value = serde_json::from_str(&std::fs::read_to_string(out.join("report.json"))?)?; - if let Some(flow) = report.get("flow").filter(|flow| !flow.is_null()) { - let text = serde_json::to_string_pretty(flow)?; - println!("plan (drafted inside the task):\n{text}"); - std::fs::write(out.join("plan.json"), text)?; - } - Ok(()) -} - -/// The grounding hints saved at `path`: none when it does not exist yet. -fn read_memory(path: &std::path::Path) -> Result, LabError> { - match std::fs::read_to_string(path) { - Ok(text) => Ok(serde_json::from_str(&text)?), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(Vec::new()), - Err(error) => Err(error.into()), - } -} - -/// `kept` with what a run `learned`: a hint for the same element (the same -/// application and key) gives way to the newer one, and the rest keep their -/// order. -fn merge_memory(kept: Vec, learned: Vec) -> Vec { - let mut merged = kept - .into_iter() - .filter(|hint| { - !learned - .iter() - .any(|new| new.app == hint.app && new.key == hint.key) - }) - .collect::>(); - merged.extend(learned); - merged -} - -/// Saves what the task's report learned into the memory at `path`, beside -/// what it already held. -fn remember(out: &std::path::Path, path: &std::path::Path) -> Result<(), LabError> { - let report: Value = serde_json::from_str(&std::fs::read_to_string(out.join("report.json"))?)?; - let learned: Vec = match report.get("learned") { - Some(learned) => serde_json::from_value(learned.clone())?, - None => Vec::new(), - }; - let merged = merge_memory(read_memory(path)?, learned); - std::fs::write(path, serde_json::to_string_pretty(&merged)?)?; - Ok(()) -} - #[cfg(test)] mod main_tests; diff --git a/crates/tinycomputer-examples/src/bin/task_live/main_tests.rs b/crates/tinycomputer-examples/src/bin/task_live/main_tests.rs index 26b0ec66..6272c844 100644 --- a/crates/tinycomputer-examples/src/bin/task_live/main_tests.rs +++ b/crates/tinycomputer-examples/src/bin/task_live/main_tests.rs @@ -1,11 +1,12 @@ -//! Tests for the `task_live` binary: which routes Jev and the planner take. +//! Tests for the `task_live` binary: which routes Jev and the planner take, +//! and how its switches are read. use std::collections::BTreeMap; use serde_json::json; use tinycomputer_examples::host::LabError; -use super::{TINY_HUMANS_MODEL, merge_memory, read_memory, remember, routes}; +use super::{TINY_HUMANS_MODEL, in_task, prelaunch, routes}; /// A variable lookup over `pairs`, in place of the process environment. fn lookup(pairs: &[(&str, &str)]) -> impl Fn(&str) -> Option { @@ -90,53 +91,22 @@ fn sage_takes_the_decisions_on_either_route() -> Result<(), LabError> { Ok(()) } -fn hint(key: &str, name: &str) -> tinycomputer_bus::GroundingHint { - tinycomputer_bus::GroundingHint { - app: "browser".to_owned(), - key: key.to_owned(), - role: "button".to_owned(), - name: Some(name.to_owned()), - path: Vec::new(), - } -} - #[test] -fn a_run_s_learned_elements_replace_the_same_ones_and_keep_the_rest() { - let kept = vec![hint("search", "Go"), hint("add to cart", "Add to Cart")]; - let learned = vec![ - hint("add to cart", "Add to Bag"), - hint("open the cart", "Cart"), - ]; - let merged = merge_memory(kept, learned); - let names = merged - .iter() - .map(|hint| hint.name.as_deref().unwrap_or_default()) - .collect::>(); - assert_eq!(names, ["Go", "Add to Bag", "Cart"]); -} - -#[test] -fn memory_is_read_from_its_file_and_a_run_s_learned_elements_are_saved_to_it() --> Result<(), LabError> { - let dir = std::env::temp_dir().join(format!("task-live-memory-{}", std::process::id())); - std::fs::create_dir_all(&dir)?; - let path = dir.join("memory.json"); +fn task_plan_and_prelaunch_take_only_the_values_they_name() -> Result<(), LabError> { + assert!(!in_task(None)?); + assert!(!in_task(Some(" "))?); + assert!(in_task(Some(" in-task\n"))?); + assert!(in_task(Some("first")).is_err(), "a typo is not ignored"); + assert_eq!(prelaunch(None)?, None); + assert_eq!(prelaunch(Some("0"))?, Some(false)); + assert_eq!(prelaunch(Some(" 1 "))?, Some(true)); + let refused = match prelaunch(Some("false")) { + Ok(value) => return Err(format!("`false` was read as {value:?}").into()), + Err(error) => error.to_string(), + }; assert!( - read_memory(&path)?.is_empty(), - "no file yet: nothing learned" + refused.contains("TINYCOMPUTER_BROWSER_PRELAUNCH"), + "{refused}" ); - - std::fs::write( - dir.join("report.json"), - serde_json::to_string(&json!({"learned": [hint("search", "Go")]}))?, - )?; - remember(&dir, &path)?; - assert_eq!(read_memory(&path)?, vec![hint("search", "Go")]); - - // A report that learned nothing keeps what the memory held. - std::fs::write(dir.join("report.json"), "{}")?; - remember(&dir, &path)?; - assert_eq!(read_memory(&path)?.len(), 1); - std::fs::remove_dir_all(&dir)?; Ok(()) } diff --git a/crates/tinycomputer-examples/src/bin/task_live/saved/mod.rs b/crates/tinycomputer-examples/src/bin/task_live/saved/mod.rs new file mode 100644 index 00000000..d7a8f461 --- /dev/null +++ b/crates/tinycomputer-examples/src/bin/task_live/saved/mod.rs @@ -0,0 +1,71 @@ +//! What a run keeps beside its report once the task stops: the plan the +//! task drafted for itself (`TASK_PLAN=in-task`), and the elements it +//! learned (`TASK_MEMORY`). + +use std::path::Path; + +use serde_json::Value; +use tinycomputer_bus::GroundingHint; +use tinycomputer_examples::host::LabError; + +/// Writes the flow a task planned for itself (`TASK_PLAN=in-task`) to +/// `plan.json` beside its report, as a plan drafted first would be. +pub(super) fn record_plan(out: &Path) -> Result<(), LabError> { + let report: Value = serde_json::from_str(&std::fs::read_to_string(out.join("report.json"))?)?; + if let Some(flow) = report.get("flow").filter(|flow| !flow.is_null()) { + let text = serde_json::to_string_pretty(flow)?; + println!("plan (drafted inside the task):\n{text}"); + std::fs::write(out.join("plan.json"), text)?; + } + Ok(()) +} + +/// The grounding hints saved at `path`: none when it does not exist yet. +pub(super) fn read_memory(path: &Path) -> Result, LabError> { + match std::fs::read_to_string(path) { + Ok(text) => Ok(serde_json::from_str(&text)?), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(Vec::new()), + Err(error) => Err(error.into()), + } +} + +/// `kept` with what a run `learned`: a hint for the same element (the same +/// application and key) gives way to the newer one, and the rest keep their +/// order. +pub(super) fn merge_memory( + kept: Vec, + learned: Vec, +) -> Vec { + let mut merged = kept + .into_iter() + .filter(|hint| { + !learned + .iter() + .any(|new| new.app == hint.app && new.key == hint.key) + }) + .collect::>(); + merged.extend(learned); + merged +} + +/// Saves what the task's report learned into the memory at `path`, beside +/// what it already held; the memory's folder is made when missing. +pub(super) fn remember(out: &Path, path: &Path) -> Result<(), LabError> { + let report: Value = serde_json::from_str(&std::fs::read_to_string(out.join("report.json"))?)?; + let learned: Vec = match report.get("learned") { + Some(learned) => serde_json::from_value(learned.clone())?, + None => Vec::new(), + }; + let merged = merge_memory(read_memory(path)?, learned); + if let Some(folder) = path + .parent() + .filter(|folder| !folder.as_os_str().is_empty()) + { + std::fs::create_dir_all(folder)?; + } + std::fs::write(path, serde_json::to_string_pretty(&merged)?)?; + Ok(()) +} + +#[cfg(test)] +mod saved_tests; diff --git a/crates/tinycomputer-examples/src/bin/task_live/saved/saved_tests.rs b/crates/tinycomputer-examples/src/bin/task_live/saved/saved_tests.rs new file mode 100644 index 00000000..3c379a8e --- /dev/null +++ b/crates/tinycomputer-examples/src/bin/task_live/saved/saved_tests.rs @@ -0,0 +1,63 @@ +//! Tests for what a `task_live` run keeps: its learned elements, merged +//! into the memory file and saved back. + +use serde_json::json; +use tinycomputer_examples::host::LabError; + +use super::{merge_memory, read_memory, remember}; + +fn hint(key: &str, name: &str) -> tinycomputer_bus::GroundingHint { + tinycomputer_bus::GroundingHint { + app: "browser".to_owned(), + key: key.to_owned(), + role: "button".to_owned(), + name: Some(name.to_owned()), + path: Vec::new(), + } +} + +#[test] +fn a_run_s_learned_elements_replace_the_same_ones_and_keep_the_rest() { + let kept = vec![hint("search", "Go"), hint("add to cart", "Add to Cart")]; + let learned = vec![ + hint("add to cart", "Add to Bag"), + hint("open the cart", "Cart"), + ]; + let merged = merge_memory(kept, learned); + let names = merged + .iter() + .map(|hint| hint.name.as_deref().unwrap_or_default()) + .collect::>(); + assert_eq!(names, ["Go", "Add to Bag", "Cart"]); +} + +#[test] +fn memory_is_read_from_its_file_and_a_run_s_learned_elements_are_saved_to_it() +-> Result<(), LabError> { + let dir = std::env::temp_dir().join(format!("task-live-memory-{}", std::process::id())); + std::fs::create_dir_all(&dir)?; + let path = dir.join("memory.json"); + assert!( + read_memory(&path)?.is_empty(), + "no file yet: nothing learned" + ); + + std::fs::write( + dir.join("report.json"), + serde_json::to_string(&json!({"learned": [hint("search", "Go")]}))?, + )?; + remember(&dir, &path)?; + assert_eq!(read_memory(&path)?, vec![hint("search", "Go")]); + + // A report that learned nothing keeps what the memory held. + std::fs::write(dir.join("report.json"), "{}")?; + remember(&dir, &path)?; + assert_eq!(read_memory(&path)?.len(), 1); + + // A memory in a folder not made yet is saved there. + let nested = dir.join("memory").join("shop.json"); + remember(&dir, &nested)?; + assert_eq!(read_memory(&nested)?.len(), 0); + std::fs::remove_dir_all(&dir)?; + Ok(()) +} diff --git a/docs/crates/tinycomputer-examples/live-tasks.md b/docs/crates/tinycomputer-examples/live-tasks.md index fba0fc14..e61e174b 100644 --- a/docs/crates/tinycomputer-examples/live-tasks.md +++ b/docs/crates/tinycomputer-examples/live-tasks.md @@ -114,13 +114,13 @@ they control. | Variable | Default | For | |---|---|---| | `FLOW_FILE` | unset | run this flow instead of asking the planner for one | -| `TASK_PLAN` | unset | `in-task` hands the task to `StartTask` to plan, as OpenHuman does, rather than planning it first with `PlanTask`; the plan is printed and saved to `plan.json` when the task stops | +| `TASK_PLAN` | unset | `in-task` hands the task to `StartTask` to plan, as OpenHuman does, rather than planning it first with `PlanTask`; the plan is printed and saved to `plan.json` when the task stops (not with `FLOW_FILE`, which gives the flow). Any other value is refused | | `TASK_OUT` | `target/task-live` | where the plan, report, and a final screenshot are written | | `TINYCOMPUTER_FLOW_STRATEGY` | `narrow` | `narrow` or `wide` asking; see [`specs/jev-wide-turns.md`](../../technical/specs/jev-wide-turns.md) | | `TINYCOMPUTER_FLOW_DELIBERATION` | `deep` | `deep`, `standard`, or `off` | | `TASK_MAX_MINUTES` | `20` | the task is cancelled after this long | | `TASK_RESCUES` | `5` | how many failed steps a reasoning model may rescue (`0` turns rescues off); see [rescue](../../rescue.md) | -| `TASK_MEMORY` | unset | a JSON file of grounding hints: the task starts with the elements earlier runs learned (`StartTask`'s `memory`), so a remembered one is confirmed rather than searched for, and what this run learns is saved back; unset, every run starts fresh | +| `TASK_MEMORY` | unset | a JSON file of grounding hints: the task starts with the elements earlier runs learned (`StartTask`'s `memory`), so a remembered one is confirmed rather than searched for, and what this run learns is saved back (its folder made when missing); unset, every run starts fresh | | `TINYCOMPUTER_RESCUE_MODEL` | `openai/gpt-6-luna` (`openrouter/deepseek/deepseek-v4-flash` with `TINYHUMANS_TOKEN`) | the model that performs a rescue | | `TINYCOMPUTER_PLANNER_MODEL` | the engine's default (`openrouter/deepseek/deepseek-v4-flash` with `TINYHUMANS_TOKEN`) | the model asked to plan the flow | @@ -132,7 +132,7 @@ they control. | `TINYCOMPUTER_BROWSER_USER_AGENT` | the user agent it announces | | `TINYCOMPUTER_BROWSER_ARGS` | space-separated extra launch arguments | | `TINYCOMPUTER_BROWSER_PERCEPTION` | `sight` (default) or `tree`: how pages are read | -| `TINYCOMPUTER_BROWSER_PRELAUNCH` | `0` opens the browser at the first step rather than while the task plans itself (with `TASK_PLAN=in-task`), as it does by default; `1` asks for the default (the module's `browser.prelaunch`) | +| `TINYCOMPUTER_BROWSER_PRELAUNCH` | `0` opens the browser at the first step rather than while the task plans itself (with `TASK_PLAN=in-task`), as it does by default; `1` asks for the default (the module's `browser.prelaunch`); any other value is refused | | `TINYCOMPUTER_BROWSER_SETTLE` | `prompt` (default) or `steady`: how long a page is let settle after an action (see the module's `browser.settle`) | | `TINYCOMPUTER_BROWSER_ENDPOINT` | attach to a running Chrome (e.g. `http://127.0.0.1:9222`) instead of launching one | | `TASK_HEADED` | `1` shows the browser the task launches instead of running it headless; a headed run needs a display, so it runs on the host | From 6016ea989ed0354dccd9d8d4730348f65317f60e Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 7 Oct 2026 20:06:56 +0530 Subject: [PATCH 30/44] Split jev_journal's split module by what it does split.rs had grown to 537 lines. Building a split stays in split/mod.rs; rendering it for a terminal moves to split/render.rs, and reading an event's time and merging spans of it to split/time.rs. Nothing changes in what it prints. --- .../src/journal/{split.rs => split/mod.rs} | 261 +----------------- .../src/journal/split/render.rs | 183 ++++++++++++ .../src/journal/split/time.rs | 80 ++++++ 3 files changed, 271 insertions(+), 253 deletions(-) rename crates/tinycomputer-examples/src/journal/{split.rs => split/mod.rs} (57%) create mode 100644 crates/tinycomputer-examples/src/journal/split/render.rs create mode 100644 crates/tinycomputer-examples/src/journal/split/time.rs diff --git a/crates/tinycomputer-examples/src/journal/split.rs b/crates/tinycomputer-examples/src/journal/split/mod.rs similarity index 57% rename from crates/tinycomputer-examples/src/journal/split.rs rename to crates/tinycomputer-examples/src/journal/split/mod.rs index 100d6348..c0ba6400 100644 --- a/crates/tinycomputer-examples/src/journal/split.rs +++ b/crates/tinycomputer-examples/src/journal/split/mod.rs @@ -7,12 +7,17 @@ //! and overlapping intervals (the framings of one decision, a batch of //! decisions) are counted once. -use std::fmt::Write as _; - use serde::{Deserialize, Serialize}; use serde_json::Value; -use super::{number, percentile, seconds}; +use super::{number, percentile}; + +mod render; +mod time; + +pub use render::{render_compare, render_split, render_table}; +pub use time::at_ms; +use time::{Spans, length, millis, minus, union}; /// Where one task's time went, in ms unless named otherwise. #[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)] @@ -89,8 +94,6 @@ pub struct Split { pub action_p90_ms: u64, } -type Spans = Vec<(i64, i64)>; - /// Where the time of the task journaled in `events` went. `events` may hold /// several runs, from one file or several (a plan journaled before its task /// existed), in any order. @@ -287,251 +290,3 @@ pub fn median(splits: &[Split]) -> Split { } serde_json::from_value(Value::Object(middle)).unwrap_or_default() } - -/// `split` for a terminal. -#[must_use] -pub fn render_split(split: &Split) -> String { - let share = |ms: u64| (ms * 100).checked_div(split.wall_ms).unwrap_or_default(); - let line = |name: &str, ms: u64, note: String| { - let line = format!("{name:<10}{} {:>3}% {note}", seconds(ms), share(ms)); - format!("{}\n", line.trim_end()) - }; - let mut out = format!("{:<10}{}\n", "wall", seconds(split.wall_ms)); - out += &line( - "planning", - split.planning_ms, - format!( - "{} plan(s), {} model call(s)", - split.plans, split.plan_calls - ), - ); - out += &line( - "rescues", - split.rescue_ms, - format!( - "{} rescue(s), {} model call(s)", - split.rescues, split.rescue_calls - ), - ); - out += &line( - "person", - split.person_ms, - format!("{} wait(s)", split.waits), - ); - out += &line("flows", split.flow_ms, String::new()); - out += &line( - " jev", - split.jev_ms, - format!( - "{} decisions, {} calls ({} failed); per call p50 {} ms, p90 {} ms; per decision p50 {} ms, p90 {} ms; the slowest call adds {} ms a round", - split.decisions, - split.calls, - split.failed_calls, - split.call_p50_ms, - split.call_p90_ms, - split.decision_p50_ms, - split.decision_p90_ms, - split.slowest_extra_ms - ), - ); - out += &line(" settle", split.settle_ms, String::new()); - out += &line(" act", split.act_ms, format!("{} actions", split.actions)); - out += &line(" observe", split.observe_ms, String::new()); - out += &line(" other", split.flow_other_ms, String::new()); - out += &line( - "other", - split.other_ms, - "starting, shaping, and the gaps between runs".to_owned(), - ); - let _ = writeln!( - out, - "units step p50 {:.1}s, p90 {:.1}s; do turn p50 {:.1}s, p90 {:.1}s; action with settling p50 {:.1}s, p90 {:.1}s", - secs(split.step_p50_ms), - secs(split.step_p90_ms), - secs(split.turn_p50_ms), - secs(split.turn_p90_ms), - secs(split.action_p50_ms), - secs(split.action_p90_ms) - ); - let _ = writeln!( - out, - "tokens {} in, {} out", - split.input_tokens, split.output_tokens - ); - out -} - -/// One line per named split, then their medians, for a terminal. -#[must_use] -pub fn render_table(splits: &[(String, Split)]) -> String { - let mut out = format!( - "{:<44} {:>7} {:>7} {:>7} {:>7} {:>7} {:>7} {:>6} {:>6} {:>12} {:>12} {:>7}\n", - "run", - "wall", - "plan", - "rescue", - "person", - "jev", - "settle", - "calls", - "decs", - "call p50/90", - "dec p50/90", - "slow+" - ); - let row = |name: &str, split: &Split| { - format!( - "{:<44} {:>6.1}s {:>6.1}s {:>6.1}s {:>6.1}s {:>6.1}s {:>6.1}s {:>6} {:>6} {:>12} {:>12} {:>5} ms\n", - super::clip(name, 44), - secs(split.wall_ms), - secs(split.planning_ms), - secs(split.rescue_ms), - secs(split.person_ms), - secs(split.jev_ms), - secs(split.settle_ms), - split.calls, - split.decisions, - format!("{}/{}", split.call_p50_ms, split.call_p90_ms), - format!("{}/{}", split.decision_p50_ms, split.decision_p90_ms), - split.slowest_extra_ms - ) - }; - for (name, split) in splits { - out += &row(name, split); - } - let all = splits - .iter() - .map(|(_, split)| split.clone()) - .collect::>(); - out += &row(&format!("median of {}", splits.len()), &median(&all)); - out -} - -/// The medians of two sets of runs side by side, with how `after` differs -/// from `before`, for a terminal. -#[must_use] -pub fn render_compare(before: &[Split], after: &[Split]) -> String { - let (a, b) = (median(before), median(after)); - let (a_fields, b_fields) = ( - serde_json::to_value(&a).unwrap_or_default(), - serde_json::to_value(&b).unwrap_or_default(), - ); - let mut out = format!( - "{:<20} {:>12} {:>12} {:>10}\n", - "median of runs", - format!("A ({})", before.len()), - format!("B ({})", after.len()), - "B vs A" - ); - for key in COMPARED { - let (x, y) = ( - a_fields[key].as_u64().unwrap_or_default(), - b_fields[key].as_u64().unwrap_or_default(), - ); - let change = if x == 0 { - "-".to_owned() - } else { - let percent = (i128::from(y) - i128::from(x)) * 100 / i128::from(x); - format!("{percent:+}%") - }; - let _ = writeln!(out, "{key:<20} {x:>12} {y:>12} {change:>10}"); - } - out -} - -/// The fields a comparison lists, in order. -const COMPARED: [&str; 20] = [ - "wall_ms", - "planning_ms", - "rescue_ms", - "person_ms", - "flow_ms", - "jev_ms", - "settle_ms", - "act_ms", - "observe_ms", - "rescues", - "decisions", - "calls", - "call_p50_ms", - "call_p90_ms", - "decision_p50_ms", - "decision_p90_ms", - "slowest_extra_ms", - "input_tokens", - "step_p90_ms", - "action_p50_ms", -]; - -/// `event`'s `at` as milliseconds since the Unix epoch. -#[must_use] -pub fn at_ms(event: &Value) -> Option { - // `2026-10-07T06:19:52.812Z`, as the engine writes it. - let at = event["at"].as_str()?; - let field = |range: std::ops::Range| at.get(range)?.parse::().ok(); - let (year, month, day) = (field(0..4)?, field(5..7)?, field(8..10)?); - let (hour, minute, second) = (field(11..13)?, field(14..16)?, field(17..19)?); - let millis = if at.get(19..20) == Some(".") { - field(20..23)? - } else { - 0 - }; - let days = days_from_civil(year, month, day); - Some((((days * 24 + hour) * 60 + minute) * 60 + second) * 1000 + millis) -} - -/// Days from 1970-01-01 to the proleptic Gregorian date, after Howard -/// Hinnant's `days_from_civil`. -fn days_from_civil(year: i64, month: i64, day: i64) -> i64 { - let year = if month <= 2 { year - 1 } else { year }; - let era = year.div_euclid(400); - let yoe = year - era * 400; - let doy = (153 * (month + if month > 2 { -3 } else { 9 }) + 2) / 5 + day - 1; - let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; - era * 146_097 + doe - 719_468 -} - -fn union(mut spans: Spans) -> Spans { - spans.sort_unstable(); - let mut merged: Spans = Vec::new(); - for (start, end) in spans { - match merged.last_mut() { - Some(last) if start <= last.1 => last.1 = last.1.max(end), - _ => merged.push((start, end)), - } - } - merged -} - -fn length(spans: &[(i64, i64)]) -> u64 { - spans.iter().map(|(start, end)| millis(end - start)).sum() -} - -/// `spans` with every moment of `cut` (a union) taken out. -fn minus(spans: &[(i64, i64)], cut: &[(i64, i64)]) -> Spans { - let mut left = Vec::new(); - for &(start, end) in spans { - let mut from = start; - for &(cut_start, cut_end) in cut { - if cut_end <= from || cut_start >= end { - continue; - } - if cut_start > from { - left.push((from, cut_start)); - } - from = from.max(cut_end); - } - if from < end { - left.push((from, end)); - } - } - left -} - -fn millis(ms: i64) -> u64 { - u64::try_from(ms).unwrap_or_default() -} - -fn secs(ms: u64) -> f64 { - std::time::Duration::from_millis(ms).as_secs_f64() -} diff --git a/crates/tinycomputer-examples/src/journal/split/render.rs b/crates/tinycomputer-examples/src/journal/split/render.rs new file mode 100644 index 00000000..9c77fda6 --- /dev/null +++ b/crates/tinycomputer-examples/src/journal/split/render.rs @@ -0,0 +1,183 @@ +//! Splits rendered for a terminal: one task's, a table of several, and two +//! sets side by side. + +use std::fmt::Write as _; + +use super::super::seconds; +use super::time::secs; +use super::{Split, median}; + +/// `split` for a terminal. +#[must_use] +pub fn render_split(split: &Split) -> String { + let share = |ms: u64| (ms * 100).checked_div(split.wall_ms).unwrap_or_default(); + let line = |name: &str, ms: u64, note: String| { + let line = format!("{name:<10}{} {:>3}% {note}", seconds(ms), share(ms)); + format!("{}\n", line.trim_end()) + }; + let mut out = format!("{:<10}{}\n", "wall", seconds(split.wall_ms)); + out += &line( + "planning", + split.planning_ms, + format!( + "{} plan(s), {} model call(s)", + split.plans, split.plan_calls + ), + ); + out += &line( + "rescues", + split.rescue_ms, + format!( + "{} rescue(s), {} model call(s)", + split.rescues, split.rescue_calls + ), + ); + out += &line( + "person", + split.person_ms, + format!("{} wait(s)", split.waits), + ); + out += &line("flows", split.flow_ms, String::new()); + out += &line( + " jev", + split.jev_ms, + format!( + "{} decisions, {} calls ({} failed); per call p50 {} ms, p90 {} ms; per decision p50 {} ms, p90 {} ms; the slowest call adds {} ms a round", + split.decisions, + split.calls, + split.failed_calls, + split.call_p50_ms, + split.call_p90_ms, + split.decision_p50_ms, + split.decision_p90_ms, + split.slowest_extra_ms + ), + ); + out += &line(" settle", split.settle_ms, String::new()); + out += &line(" act", split.act_ms, format!("{} actions", split.actions)); + out += &line(" observe", split.observe_ms, String::new()); + out += &line(" other", split.flow_other_ms, String::new()); + out += &line( + "other", + split.other_ms, + "starting, shaping, and the gaps between runs".to_owned(), + ); + let _ = writeln!( + out, + "units step p50 {:.1}s, p90 {:.1}s; do turn p50 {:.1}s, p90 {:.1}s; action with settling p50 {:.1}s, p90 {:.1}s", + secs(split.step_p50_ms), + secs(split.step_p90_ms), + secs(split.turn_p50_ms), + secs(split.turn_p90_ms), + secs(split.action_p50_ms), + secs(split.action_p90_ms) + ); + let _ = writeln!( + out, + "tokens {} in, {} out", + split.input_tokens, split.output_tokens + ); + out +} + +/// One line per named split, then their medians, for a terminal. +#[must_use] +pub fn render_table(splits: &[(String, Split)]) -> String { + let mut out = format!( + "{:<44} {:>7} {:>7} {:>7} {:>7} {:>7} {:>7} {:>6} {:>6} {:>12} {:>12} {:>7}\n", + "run", + "wall", + "plan", + "rescue", + "person", + "jev", + "settle", + "calls", + "decs", + "call p50/90", + "dec p50/90", + "slow+" + ); + let row = |name: &str, split: &Split| { + format!( + "{:<44} {:>6.1}s {:>6.1}s {:>6.1}s {:>6.1}s {:>6.1}s {:>6.1}s {:>6} {:>6} {:>12} {:>12} {:>5} ms\n", + super::super::clip(name, 44), + secs(split.wall_ms), + secs(split.planning_ms), + secs(split.rescue_ms), + secs(split.person_ms), + secs(split.jev_ms), + secs(split.settle_ms), + split.calls, + split.decisions, + format!("{}/{}", split.call_p50_ms, split.call_p90_ms), + format!("{}/{}", split.decision_p50_ms, split.decision_p90_ms), + split.slowest_extra_ms + ) + }; + for (name, split) in splits { + out += &row(name, split); + } + let all = splits + .iter() + .map(|(_, split)| split.clone()) + .collect::>(); + out += &row(&format!("median of {}", splits.len()), &median(&all)); + out +} + +/// The medians of two sets of runs side by side, with how `after` differs +/// from `before`, for a terminal. +#[must_use] +pub fn render_compare(before: &[Split], after: &[Split]) -> String { + let (a, b) = (median(before), median(after)); + let (a_fields, b_fields) = ( + serde_json::to_value(&a).unwrap_or_default(), + serde_json::to_value(&b).unwrap_or_default(), + ); + let mut out = format!( + "{:<20} {:>12} {:>12} {:>10}\n", + "median of runs", + format!("A ({})", before.len()), + format!("B ({})", after.len()), + "B vs A" + ); + for key in COMPARED { + let (x, y) = ( + a_fields[key].as_u64().unwrap_or_default(), + b_fields[key].as_u64().unwrap_or_default(), + ); + let change = if x == 0 { + "-".to_owned() + } else { + let percent = (i128::from(y) - i128::from(x)) * 100 / i128::from(x); + format!("{percent:+}%") + }; + let _ = writeln!(out, "{key:<20} {x:>12} {y:>12} {change:>10}"); + } + out +} + +/// The fields a comparison lists, in order. +const COMPARED: [&str; 20] = [ + "wall_ms", + "planning_ms", + "rescue_ms", + "person_ms", + "flow_ms", + "jev_ms", + "settle_ms", + "act_ms", + "observe_ms", + "rescues", + "decisions", + "calls", + "call_p50_ms", + "call_p90_ms", + "decision_p50_ms", + "decision_p90_ms", + "slowest_extra_ms", + "input_tokens", + "step_p90_ms", + "action_p50_ms", +]; diff --git a/crates/tinycomputer-examples/src/journal/split/time.rs b/crates/tinycomputer-examples/src/journal/split/time.rs new file mode 100644 index 00000000..b5344b4f --- /dev/null +++ b/crates/tinycomputer-examples/src/journal/split/time.rs @@ -0,0 +1,80 @@ +//! Time for a split: an event's `at`, and spans of milliseconds merged, +//! measured, and cut. + +use serde_json::Value; + +/// Intervals of milliseconds since the Unix epoch, start and end. +pub(super) type Spans = Vec<(i64, i64)>; + +/// `event`'s `at` as milliseconds since the Unix epoch. +#[must_use] +pub fn at_ms(event: &Value) -> Option { + // `2026-10-07T06:19:52.812Z`, as the engine writes it. + let at = event["at"].as_str()?; + let field = |range: std::ops::Range| at.get(range)?.parse::().ok(); + let (year, month, day) = (field(0..4)?, field(5..7)?, field(8..10)?); + let (hour, minute, second) = (field(11..13)?, field(14..16)?, field(17..19)?); + let millis = if at.get(19..20) == Some(".") { + field(20..23)? + } else { + 0 + }; + let days = days_from_civil(year, month, day); + Some((((days * 24 + hour) * 60 + minute) * 60 + second) * 1000 + millis) +} + +/// Days from 1970-01-01 to the proleptic Gregorian date, after Howard +/// Hinnant's `days_from_civil`. +fn days_from_civil(year: i64, month: i64, day: i64) -> i64 { + let year = if month <= 2 { year - 1 } else { year }; + let era = year.div_euclid(400); + let yoe = year - era * 400; + let doy = (153 * (month + if month > 2 { -3 } else { 9 }) + 2) / 5 + day - 1; + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + era * 146_097 + doe - 719_468 +} + +pub(super) fn union(mut spans: Spans) -> Spans { + spans.sort_unstable(); + let mut merged: Spans = Vec::new(); + for (start, end) in spans { + match merged.last_mut() { + Some(last) if start <= last.1 => last.1 = last.1.max(end), + _ => merged.push((start, end)), + } + } + merged +} + +pub(super) fn length(spans: &[(i64, i64)]) -> u64 { + spans.iter().map(|(start, end)| millis(end - start)).sum() +} + +/// `spans` with every moment of `cut` (a union) taken out. +pub(super) fn minus(spans: &[(i64, i64)], cut: &[(i64, i64)]) -> Spans { + let mut left = Vec::new(); + for &(start, end) in spans { + let mut from = start; + for &(cut_start, cut_end) in cut { + if cut_end <= from || cut_start >= end { + continue; + } + if cut_start > from { + left.push((from, cut_start)); + } + from = from.max(cut_end); + } + if from < end { + left.push((from, end)); + } + } + left +} + +pub(super) fn millis(ms: i64) -> u64 { + u64::try_from(ms).unwrap_or_default() +} + +pub(super) fn secs(ms: u64) -> f64 { + std::time::Duration::from_millis(ms).as_secs_f64() +} From 6d764078cad5ba0d90cb8a3c698b3bd37085ca2b Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 7 Oct 2026 20:06:56 +0530 Subject: [PATCH 31/44] Say how a task_live run journals to TASK_OUT/journal The docs said task_live writes its journal under TASK_OUT/journal, and showed try-* folders. Neither task_live nor tasks/run sets the journal's folder: a run journals there only when started with TINYCOMPUTER_JEV_JOURNAL=$TASK_OUT/journal, and try-* was one private script's naming. The docs, jev_journal's help and runs.rs now say so. --- crates/tinycomputer-examples/src/bin/jev_journal.rs | 7 ++++--- crates/tinycomputer-examples/src/journal/runs.rs | 7 ++++--- docs/technical/jev-journal.md | 11 ++++++----- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/crates/tinycomputer-examples/src/bin/jev_journal.rs b/crates/tinycomputer-examples/src/bin/jev_journal.rs index 050e1bf8..f2a3ade2 100644 --- a/crates/tinycomputer-examples/src/bin/jev_journal.rs +++ b/crates/tinycomputer-examples/src/bin/jev_journal.rs @@ -14,9 +14,10 @@ //! ``` //! //! `` is `latest`, any unique part of a run id, or a run directory. A -//! `` is a run directory, or a folder of runs read as one task (what -//! `task_live` writes under `TASK_OUT/journal`); `--split` with none splits -//! the latest run. See `docs/technical/jev-journal.md`. +//! `` is a run directory, or a folder of runs read as one task (what a +//! `task_live` run started with `TINYCOMPUTER_JEV_JOURNAL=$TASK_OUT/journal` +//! writes there); `--split` with none splits the latest run. See +//! `docs/technical/jev-journal.md`. use std::process::ExitCode; diff --git a/crates/tinycomputer-examples/src/journal/runs.rs b/crates/tinycomputer-examples/src/journal/runs.rs index 07b1f8bf..afb5256c 100644 --- a/crates/tinycomputer-examples/src/journal/runs.rs +++ b/crates/tinycomputer-examples/src/journal/runs.rs @@ -86,9 +86,10 @@ pub fn events(dir: &Path) -> std::io::Result> { } /// Every event of the task journaled at `path`: a run directory, or a -/// folder of run directories read as one story, oldest run first. `task_live` -/// writes such a folder: the plan, journaled before the task existed, and the -/// task's own runs. +/// folder of run directories read as one story, oldest run first. A +/// `task_live` run journaling to its own folder +/// (`TINYCOMPUTER_JEV_JOURNAL=$TASK_OUT/journal`) writes such a folder: the +/// plan, journaled before the task existed, and the task's own runs. /// /// # Errors /// diff --git a/docs/technical/jev-journal.md b/docs/technical/jev-journal.md index 8bcd0aad..cfbc96b4 100644 --- a/docs/technical/jev-journal.md +++ b/docs/technical/jev-journal.md @@ -139,8 +139,9 @@ by `elapsed_ms`, which each run of a task restarts, so for a task of several runs use `--split`. `--split` reads whole tasks by their `at` timestamps. A task is a run -directory, or a folder of runs read as one — what `task_live` writes under -`TASK_OUT/journal`: its `PlanTask` plan and the task's own file. It splits +directory, or a folder of runs read as one — what a `task_live` run started +with `TINYCOMPUTER_JEV_JOURNAL=$TASK_OUT/journal` writes there: its +`PlanTask` plan and the task's own file. It splits the wall time into planning, rescues, waits for a person, and flows (and within flows: Jev, settling, acting, reading the screen, the rest), counting each moment once. It also reports per-call and per-decision latency (p50, @@ -153,10 +154,10 @@ after it, with the change in percent: two builds, or two settings, run on the same tasks. ```sh -# every task_live run of a batch -jev_journal --split target/task-live/try-*/journal +# every task of a batch, each run with TINYCOMPUTER_JEV_JOURNAL=$TASK_OUT/journal +jev_journal --split target/task-live/*/journal # the runs of one build or setting against another's -jev_journal --compare before/try-*/journal --vs after/try-*/journal +jev_journal --compare before/*/journal --vs after/*/journal ``` For anything the summary does not cover, the file is plain JSON Lines: From 09c6e327e81daefb9eaea6b8c89b40b4874c65c4 Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 7 Oct 2026 20:07:06 +0530 Subject: [PATCH 32/44] Bring the settle, late-look and Jev timeout docs up to date Docs still said the browser settles on network idle (decision-loops.md, the-do-loop.md), that a place box waits a fixed beat between its late looks (filling-forms.md), and that a Jev attempt's timeout defaults to the client's (JevConfig::timeout_ms, jev-runtime.md); it is the module's 10 s. Three paragraphs of decision-loops.md are reflowed so the file does not grow past its length on main. --- .../tinycomputer-bus/src/agentic/types/config.rs | 4 ++-- .../tinycomputer-engine/flow/filling-forms.md | 6 ++++-- .../tinycomputer-engine/flow/the-do-loop.md | 7 ++++--- docs/crates/tinycomputer-engine/jev-runtime.md | 2 +- docs/technical/decision-loops.md | 16 +++++++--------- 5 files changed, 18 insertions(+), 17 deletions(-) diff --git a/crates/tinycomputer-bus/src/agentic/types/config.rs b/crates/tinycomputer-bus/src/agentic/types/config.rs index 97456141..414416cf 100644 --- a/crates/tinycomputer-bus/src/agentic/types/config.rs +++ b/crates/tinycomputer-bus/src/agentic/types/config.rs @@ -60,8 +60,8 @@ pub struct JevConfig { /// [`JevProvider::default_model`]. Sage takes no model selection and /// ignores it. pub model: Option, - /// Per-attempt HTTP timeout. Absent means the client default. Ignored by - /// Sage. + /// Per-attempt HTTP timeout. Absent means the module's default: 10 + /// seconds. Ignored by Sage. pub timeout_ms: Option, /// Additional transient retries. Absent means the module's default: four, /// waiting 1, 2, 4, then 8 seconds between attempts. Ignored by Sage. diff --git a/docs/crates/tinycomputer-engine/flow/filling-forms.md b/docs/crates/tinycomputer-engine/flow/filling-forms.md index 8c873c6b..051240c3 100644 --- a/docs/crates/tinycomputer-engine/flow/filling-forms.md +++ b/docs/crates/tinycomputer-engine/flow/filling-forms.md @@ -146,8 +146,10 @@ before the text was typed, it picks the one that matches it "blue light blocking glasses" became another product's name), so the text stays as typed; - a place box (a slot named for a place: pickup, drop, from, to, address, - city, …) is given two more looks, a wait apart, when no row showed yet - (`LATE_LOOKS`), and there a row that was already showing still counts when + city, …) is given up to two more looks when no row showed yet + (`LATE_LOOKS`), each after a wait that ends as soon as the page changes + (`LATE_LOOK_MS`, 1 s at most); a page that stayed still through a wait + lists nothing more, and is looked at once more only. There a row that was already showing still counts when it matches the text, as does a pressable box sharing at least half the text's words: a ride app lists popular places as soon as its box has the focus, and words its rows its own way ("MG Road / Shivaji Nagar Bengaluru" diff --git a/docs/crates/tinycomputer-engine/flow/the-do-loop.md b/docs/crates/tinycomputer-engine/flow/the-do-loop.md index 8a10092b..0fec516f 100644 --- a/docs/crates/tinycomputer-engine/flow/the-do-loop.md +++ b/docs/crates/tinycomputer-engine/flow/the-do-loop.md @@ -137,9 +137,10 @@ click on an answer it does not recognise, because a malformed or injected answer must fail closed rather than guess. After any action the backend reports as successful, the runtime waits for -the surface to settle (network-idle on the browser, a short pause on the -desktop) before looking again, so the next turn's screen reflects what the -action actually did rather than the moment right before it took effect. +the surface to settle (on the browser, until the requests that change the +page end and it goes still; a short pause on the desktop) before looking +again, so the next turn's screen reflects what the action actually did +rather than the moment right before it took effect. ### Shortcuts diff --git a/docs/crates/tinycomputer-engine/jev-runtime.md b/docs/crates/tinycomputer-engine/jev-runtime.md index 9b5c2695..abdf13b6 100644 --- a/docs/crates/tinycomputer-engine/jev-runtime.md +++ b/docs/crates/tinycomputer-engine/jev-runtime.md @@ -37,7 +37,7 @@ The configuration (`JevConfig`, in `tinycomputer-bus`) names: | `api_key` | The credential for that provider. Never printed; `JevRuntime`'s `Debug` implementation shows `"[configured]"` in its place. | | `endpoint_url` | An exact endpoint to use instead of the provider's own route, checked against an allow-list (see below). | | `model` | The Jev model or alias to ask for. Defaults to the provider's `JevProvider::default_model()`: `"jev-latest"`, `"openjev"` for OpenJEV, and the fixed `"levanto-sage"` for Sage. | -| `timeout_ms` / `max_retries` | Per-attempt HTTP timeout and how many transient retries the client makes: four by default (`RETRY`), waiting 1, 2, 4, then 8 seconds, since live a gateway's brief 502s ended runs after the client's own 0.3 s of waiting. Sage ignores both. | +| `timeout_ms` / `max_retries` | Per-attempt HTTP timeout (10 seconds by default, `ATTEMPT_TIMEOUT`) and how many transient retries the client makes: four by default (`RETRY`), waiting 1, 2, 4, then 8 seconds, since live a gateway's brief 502s ended runs after the client's own 0.3 s of waiting. Sage ignores both. | | `sdk_name` | Attribution sent only to the TinyHumans proxy, so it knows which host is calling. | | `fast` | Sage only: score each choice in one pass rather than one per option. | diff --git a/docs/technical/decision-loops.md b/docs/technical/decision-loops.md index 9b6dd8df..3ecde97f 100644 --- a/docs/technical/decision-loops.md +++ b/docs/technical/decision-loops.md @@ -262,9 +262,9 @@ on an answer it does not recognise, because a malformed or injected answer must fail closed. After any action the backend reports as successful, the runtime calls the -surface's `settle` before looking again — network-idle on the browser, a short -pause on the desktop — so the next turn's screen reflects what the action did -rather than the moment before it took effect. +surface's `settle` before looking again — on the browser, until the requests +that change the page end and it goes still; a short pause on the desktop — so +the next turn's screen reflects what the action did, not the moment before. The shortcut list (`act/mod.rs::SHORTCUTS`) is short and safe: new item, new folder, find, reply, settings, back, next field, confirm (Return), and dismiss @@ -354,9 +354,8 @@ the disagreement. ## `enter`: filling fields -`enter` takes a map of slot to text, such as -`{"recipient": "sam@example.com", "subject": "Friday"}`. It runs up to three -rounds: +`enter` takes a map of slot to text, such as `{"recipient": "sam@example.com", +"subject": "Friday"}`. It runs up to three rounds: 1. Look, and explore the cut-short subtrees if there are fewer editable fields than pending slots. @@ -433,9 +432,8 @@ without knowing the site. Both surfaces label repeated containers with an ordinal (`listitem #3`), so every node inside one card shares that label in its path. The list is the parent under which the most same-role ordinal containers repeat. Each container becomes a record whose fields are its visible text in -reading order, and whose primary control is the one that looks most like -"open this" (select, book, choose, view, details, continue, reserve, deal, -see). +reading order, and whose primary control is the one that looks most like "open +this" (select, book, choose, view, details, continue, reserve, deal, see). `pick` parses its `by` text into a `Criterion` when it can: lowest or highest price, earliest or latest time, fewest stops, shortest duration. The parsers in From 2c99657e250e1cf878a3335afd4763072806e26c Mon Sep 17 00:00:00 2001 From: Shanu Date: Wed, 7 Oct 2026 22:20:13 +0530 Subject: [PATCH 33/44] Wait past a place box's own rows for the ones naming the place A place box was looked at again only while it listed no new row at all. Live on Uber, the pickup box first listed rows of its own ("Allow location access", "Search in a different city"), and the prompt settle, which reads the page as soon as it goes still, looked before the matches were fetched: no row named the place, Jev rightly chose none, and the pickup was never set, so no ride options showed. The slower settle had hidden this by looking later. A place box is now looked at again, as before up to LATE_LOOKS waits that each end at the page's first change, until a row names the text typed. The simulator's ride form can list such rows of its own while its matches are pending, and the new test fails on the old condition. --- .../src/agentic/flow/flow_tests/places.rs | 18 +++++++++- .../flow/flow_tests/suggestion_tests.rs | 33 +++++++++++++++++++ .../src/agentic/flow/steps/suggestion.rs | 21 ++++++++---- .../tinycomputer-engine/flow/filling-forms.md | 5 +-- docs/technical/decision-thresholds.md | 2 +- 5 files changed, 69 insertions(+), 10 deletions(-) diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/places.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/places.rs index 165e3ccd..29da5249 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/places.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/places.rs @@ -20,6 +20,13 @@ pub(super) const PLACES: [&str; 4] = [ /// The heading a panel of suggestions opens with (`Places::panel`). const PANEL_HEADING: &str = "Select a pickup point Choose where your driver meets you"; +/// The rows a box lists of its own while its matches are fetched +/// (`Places::starters`): none of them names a place typed. +const STARTER_ROWS: [&str; 2] = [ + "Allow location access It provides your pickup address", + "Search in a different city", +]; + /// The ride form's state. #[derive(Debug, Default)] pub(super) struct Places { @@ -36,6 +43,10 @@ pub(super) struct Places { pub(super) late: u8, /// Waits still to come before the open list shows its rows. pub(super) pending: u8, + /// Whether the open list shows rows of its own while its matches are + /// still to come, as a ride app's did live ("Allow location access", + /// "Search in a different city"). + pub(super) starters: bool, } /// The places suggested for `typed`: each one that holds every typed word. @@ -76,9 +87,14 @@ pub(super) fn places_widget( field.value = sim.fields.get(*name).map(|value| json!(value)); candidates.push(field); } + let list = [root, "group \"Get a ride\"", "listbox \"Suggestions\""]; + if places.starters && places.open.is_some() && places.pending > 0 { + for row in STARTER_ROWS { + candidates.push(node(row, "option", &["Click"], &list, 300.0)); + } + } if let Some(open) = places.open.as_ref().filter(|_| places.pending == 0) { let typed = sim.fields.get(open).cloned().unwrap_or_default(); - let list = [root, "group \"Get a ride\"", "listbox \"Suggestions\""]; if places.panel { let rows = suggested(&typed); if !rows.is_empty() { diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/suggestion_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/suggestion_tests.rs index d6c18dc6..93a0c9b4 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/suggestion_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/suggestion_tests.rs @@ -131,6 +131,39 @@ async fn a_place_box_whose_rows_come_late_is_looked_at_again_as_they_show() { } } +#[tokio::test] +async fn a_place_box_waits_past_its_own_rows_for_the_place_typed() { + // Live, a ride app's pickup box first listed rows of its own ("Allow + // location access", "Search in a different city"), and a look made as + // soon as the page went still saw only those: no row named the place, + // none was picked, and the pickup was never set. The box is looked at + // again until a row names the place. + let run = run_with( + App::with(|sim| { + sim.places = Some(Places { + late: 1, + starters: true, + ..Places::default() + }); + }), + json!({"app": "Mail", "steps": [{"enter": {"pickup location": "Connaught Place"}}]}), + |_| {}, + ride, + ) + .await; + assert_eq!( + run.result.stop, + FlowStopReason::Completed, + "{:?}", + run.result.steps + ); + assert_eq!( + run.app.sim().fields["Pickup location"], + "Connaught Place New Delhi, Delhi, India" + ); + assert_eq!(waits(&run), [""], "one wait, ended by the rows showing"); +} + #[tokio::test] async fn a_place_box_on_a_page_that_stays_still_is_waited_on_once() { // Live, an address and a city box on a plain form waited twice each for diff --git a/crates/tinycomputer-engine/src/agentic/flow/steps/suggestion.rs b/crates/tinycomputer-engine/src/agentic/flow/steps/suggestion.rs index 3b1124d8..d4d91bf0 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/steps/suggestion.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/steps/suggestion.rs @@ -72,12 +72,16 @@ impl FlowRun<'_, B> { let mut fresh = fresh_rows(&screen, &shown, field, text, &self.stop_before, place); // A box that suggests places lists them once the page has fetched // them: live, a ride app's rows came after the first look, and the - // pickup typed was never set, so no ride showed. The wait ends as - // the page changes, and a page that stayed still lists nothing more: - // live, an address and a city box on a plain form waited 2.3-4.2 s - // each for a list that never comes. + // pickup typed was never set, so no ride showed. Rows that name + // nothing typed ("Allow location access", "Search in a different + // city") are the box's own, shown while its matches are fetched, so + // they are waited past too: live, a look made as soon as the page + // went still saw only those. The wait ends as the page changes, and + // a page that stayed still lists nothing more: live, an address and + // a city box on a plain form waited 2.3-4.2 s each for a list that + // never comes. for _ in 0..LATE_LOOKS { - if !fresh.is_empty() || !place { + if !place || fresh.iter().any(|row| names_typed(row, text)) { break; } let changed = self.await_change(log, LATE_LOOK_MS).await?; @@ -279,8 +283,13 @@ pub(in crate::agentic::flow) fn shares_most_words(candidate: &Candidate, text: & shared >= 2 && shared * 2 >= words.len() } +/// Whether `row` names the text typed: all of it, or most of its words. +fn names_typed(row: &Candidate, text: &str) -> bool { + mentions(row, text) || shares_most_words(row, text) +} + /// Looks again, after a wait for the page to change, for the rows a place -/// box lists late. +/// box lists late: until one names the text typed. const LATE_LOOKS: u32 = 2; /// Longest one wait for a place box's late rows: the page is looked at again diff --git a/docs/crates/tinycomputer-engine/flow/filling-forms.md b/docs/crates/tinycomputer-engine/flow/filling-forms.md index 051240c3..e3fcf8a5 100644 --- a/docs/crates/tinycomputer-engine/flow/filling-forms.md +++ b/docs/crates/tinycomputer-engine/flow/filling-forms.md @@ -146,8 +146,9 @@ before the text was typed, it picks the one that matches it "blue light blocking glasses" became another product's name), so the text stays as typed; - a place box (a slot named for a place: pickup, drop, from, to, address, - city, …) is given up to two more looks when no row showed yet - (`LATE_LOOKS`), each after a wait that ends as soon as the page changes + city, …) is given up to two more looks when no row naming the text + showed yet, such as while it lists only rows of its own ("Allow location + access") (`LATE_LOOKS`), each after a wait that ends as soon as the page changes (`LATE_LOOK_MS`, 1 s at most); a page that stayed still through a wait lists nothing more, and is looked at once more only. There a row that was already showing still counts when it matches the text, as does a pressable box sharing at least half the diff --git a/docs/technical/decision-thresholds.md b/docs/technical/decision-thresholds.md index 018f7eb3..e80b7b51 100644 --- a/docs/technical/decision-thresholds.md +++ b/docs/technical/decision-thresholds.md @@ -53,7 +53,7 @@ Change a constant and its row together. | `STEADY_HOLD` / `STEADY_CHECKS` | 0.65 / 3 | `steps/mod.rs` | belief a `wait_for` condition must keep, on checks in a row of one unchanged screen, to be taken as held under `DONE` | | `HEDGE_AFTER` / `HEDGE_AFTER_LARGE` | 4 s / 5 s | `hedge.rs` | how long a framing runs before a copy of it is sent and the first answer of the two taken; the longer wait is for a request of `HEDGE_LARGE_BYTES` (32 KB) or more. Sage gets no copy | | `HEDGE_COPIES` | 2 | `hedge.rs` | most copies one runtime has in flight: past it, a framing waits for its own answer, so a slow or failing gateway is not sent a copy of every call | -| `LATE_LOOKS` | 2 | `steps/suggestion.rs` | looks again, after a wait for the page to change, for the suggestions a place box lists late, before its text is left as typed; a page that stayed still through a wait lists nothing more | +| `LATE_LOOKS` | 2 | `steps/suggestion.rs` | looks again, after a wait for the page to change, for the suggestions a place box lists late, until a row names the text typed (rows of the box's own, such as "Allow location access", are waited past), before its text is left as typed; a page that stayed still through a wait lists nothing more | | `LATE_LOOK_MS` | 1000 ms | `steps/suggestion.rs` | longest one of those waits: it ends as soon as the page changes (`Surface::await_change`) | | `BARE_CHARS` | 3 | `tinycomputer-core` `surface/groups.rs` | most letters and digits each field of a card may show for a list of such cards to be bare markers (carousel dots, size chips, page numbers), never results | | `CARD_LINK_CHARS` | 20 | `tinycomputer-core` `surface/groups.rs` | least characters (with a word in them) a link, option, radio, or button must show for a run of three or more under one parent to be a list of cards that are one control each | From 7f00ee2d799973bb0765e407b6fbd7d4654640db Mon Sep 17 00:00:00 2001 From: Shanu Date: Thu, 8 Oct 2026 13:11:27 +0530 Subject: [PATCH 34/44] Settle briefly after a launch or Escape, which fetch nothing The flow settles the surface after every successful action. On the browser, prompt settling waits for the requests that change the page (at least its 500 ms of quiet, at most 1 s) and then for the page to go still. A launch leaves an open page as it is and Escape closes a layer: live, 3% of launches and 12% of Escapes settled with a request of the page still running, against 39% of fills (fetching the suggestions the next look reads) and 70% of clicks. Those two now call Surface::settle_briefly. The browser waits only while the page changes, 120-400 ms instead of about 0.65 s on an idle page; steady settling, and the desktop, settle in full as before. Typing, Return and every other action keep the full settle. --- .../src/surface/operations.rs | 35 +++++++++++++++---- .../surface/surface_tests/operations_tests.rs | 30 ++++++++++++++++ crates/tinycomputer-core/src/surface/mod.rs | 8 +++++ .../surface/surface_tests/delivery_tests.rs | 2 ++ .../src/agentic/flow/action.rs | 19 ++++++++-- .../agentic/flow/flow_tests/do_loop_tests.rs | 3 ++ .../src/agentic/flow/flow_tests/simulator.rs | 8 +++-- .../tinycomputer-engine/src/workspace/mod.rs | 8 +++++ .../src/workspace/workspace_tests.rs | 22 ++++++++++++ .../tinycomputer-browser/interacting.md | 9 +++++ docs/crates/tinycomputer-browser/surface.md | 5 +-- .../tinycomputer-core/surfaces-and-screens.md | 6 +++- docs/crates/tinycomputer-engine/workspace.md | 2 +- docs/technical/architecture.md | 1 + docs/technical/decision-loops.md | 8 ++--- docs/technical/jev-harness.md | 2 +- 16 files changed, 148 insertions(+), 20 deletions(-) diff --git a/crates/tinycomputer-browser/src/surface/operations.rs b/crates/tinycomputer-browser/src/surface/operations.rs index f55b8a09..4659b952 100644 --- a/crates/tinycomputer-browser/src/surface/operations.rs +++ b/crates/tinycomputer-browser/src/surface/operations.rs @@ -3,7 +3,7 @@ use serde_json::{Value, json}; use tinycomputer_bus::browser::{ - Action, NavigateRequest, ScrollDirection, SnapshotRequest, Target, WaitState, + Action, NavigateRequest, ScrollDirection, SessionId, SnapshotRequest, Target, WaitState, }; use tinycomputer_bus::{DesktopError, DesktopResponse, JevOperation}; use tinycomputer_core::surface::{Candidate, Depth, Screen, Surface, uses_pointer}; @@ -246,12 +246,7 @@ impl Surface for BrowserSurface { ); let _quiet = self .block(async { tokio::time::timeout(watch::deadline(QUIET_MS), quiet).await }); - let still = self.browser.command( - &id, - json!({"action": "evaluate", "script": watch::still_script()}), - ); - let _still = self - .block(async { tokio::time::timeout(watch::deadline(SETTLE_MS), still).await }); + self.await_still(&id); } return; } @@ -264,6 +259,18 @@ impl Surface for BrowserSurface { let _settled = self.perform("wait", pause(SETTLE_MS)); } + fn settle_briefly(&self) { + // Nothing was fetched to wait for: only the page's own movement, + // under prompt settling. Steady settling stays as it always was. + if self.settle != Settle::Prompt { + self.settle(); + return; + } + if let Ok(id) = self.ensure_session() { + self.await_still(&id); + } + } + fn await_change(&self, ms: u64) -> bool { // With no page open there is nothing to watch, and nothing to open. let Some(id) = self.session() else { @@ -301,6 +308,20 @@ impl Surface for BrowserSurface { } } +impl BrowserSurface { + /// Waits until the page stops changing (`watch::still_script`), within + /// a deadline: a call sent while a page is being replaced can wait out + /// the browser's own 30 s. + fn await_still(&self, id: &SessionId) { + let still = self.browser.command( + id, + json!({"action": "evaluate", "script": watch::still_script()}), + ); + let _still = + self.block(async { tokio::time::timeout(watch::deadline(SETTLE_MS), still).await }); + } +} + /// How the engine addresses `reference`: a ref sight minted by its mark's /// CSS selector, a tree ref as itself. pub(super) fn target(reference: &str) -> Target { diff --git a/crates/tinycomputer-browser/src/surface/surface_tests/operations_tests.rs b/crates/tinycomputer-browser/src/surface/surface_tests/operations_tests.rs index 11dcd378..5af9cc10 100644 --- a/crates/tinycomputer-browser/src/surface/surface_tests/operations_tests.rs +++ b/crates/tinycomputer-browser/src/surface/surface_tests/operations_tests.rs @@ -588,6 +588,36 @@ fn a_prompt_settle_counts_quiet_from_the_start_and_waits_only_while_the_page_cha ); } +#[test] +fn a_brief_settle_waits_only_while_the_page_changes() { + // A launch or Escape fetched nothing: no wait for the network. + let brief = harness("brief-settle", page_fake()); + brief.surface.settle_briefly(); + assert!( + !brief + .fake + .actions() + .iter() + .any(|action| action == "waitforloadstate" || action == "wait"), + "{:?}", + brief.fake.actions() + ); + let still = brief.fake.last("evaluate"); + assert!( + still["script"].as_str().unwrap().contains("getAnimations"), + "the page is still watched until it stops changing" + ); + // Settling steadily settles in full, as it always did. + let steady = harness("brief-steady", page_fake()); + steady + .surface + .clone() + .with_settle(crate::Settle::Steady) + .settle_briefly(); + assert_eq!(steady.fake.last("waitforloadstate")["state"], "networkidle"); + assert_eq!(steady.fake.last("wait")["timeout"], 400); +} + #[test] fn a_wait_for_a_change_ends_at_the_pages_first_change_or_its_time() { let Harness { fake, surface, .. } = harness("await-change", page_fake()); diff --git a/crates/tinycomputer-core/src/surface/mod.rs b/crates/tinycomputer-core/src/surface/mod.rs index 2a25f910..ee54a9f4 100644 --- a/crates/tinycomputer-core/src/surface/mod.rs +++ b/crates/tinycomputer-core/src/surface/mod.rs @@ -74,6 +74,14 @@ pub trait Surface: Clone + Send + 'static { /// address into a token. fn settle(&self) {} + /// Settles after an action that fetches nothing — launching what is + /// already open, Escape closing a layer — so only the application's + /// own movement is waited out. A surface that cannot tell such an + /// action apart settles as after any other. + fn settle_briefly(&self) { + self.settle(); + } + /// Waits, for up to the given milliseconds, for the application to /// change by itself — a list of suggestions a box fetches for the text /// just typed, the rest of a page arriving — and says whether it did, so diff --git a/crates/tinycomputer-core/src/surface/surface_tests/delivery_tests.rs b/crates/tinycomputer-core/src/surface/surface_tests/delivery_tests.rs index 95e24dae..2bdc56cf 100644 --- a/crates/tinycomputer-core/src/surface/surface_tests/delivery_tests.rs +++ b/crates/tinycomputer-core/src/surface/surface_tests/delivery_tests.rs @@ -189,6 +189,8 @@ fn text_that_never_arrives_is_reported_as_not_delivered() { #[test] fn a_surface_settles_instantly_and_has_no_addresses_unless_it_says_otherwise() { Surface::settle(&TextBackend::default()); + // Settling briefly is settling, unless the surface can tell them apart. + Surface::settle_briefly(&TextBackend::default()); assert!( Surface::await_change(&TextBackend::default(), 1_000), "one that cannot watch pauses and says it may have changed" diff --git a/crates/tinycomputer-engine/src/agentic/flow/action.rs b/crates/tinycomputer-engine/src/agentic/flow/action.rs index 5d009ce5..b70f9eeb 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/action.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/action.rs @@ -69,8 +69,13 @@ impl FlowRun<'_, B> { if settles { // Let the surface finish reacting, so the next look sees what the // action did rather than the moment before it took effect. - self.backend_call(|backend| { - backend.settle(); + let briefly = fetches_nothing(action); + self.backend_call(move |backend| { + if briefly { + backend.settle_briefly(); + } else { + backend.settle(); + } DesktopResponse::ok("settle", serde_json::json!({})) }) .await; @@ -116,6 +121,16 @@ impl FlowRun<'_, B> { } } +/// Whether `action` fetches nothing the next look must wait for, so the +/// surface settles briefly after it (`Surface::settle_briefly`): a launch, +/// which leaves an open page as it is, or Escape closing a layer. Live, 3% +/// of launches and 12% of Escapes settled with a request of the page still +/// running, against 39% of fills (fetching suggestions the next look reads) +/// and 70% of clicks, which settle in full. +fn fetches_nothing(action: &str) -> bool { + action == "launch" || action.starts_with("launch ") || action.starts_with("press escape") +} + /// Whether `reply` is a wait's that saw the surface stay still. fn still(reply: &DesktopResponse) -> bool { reply diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs index 9a67c8c2..cd69140c 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/do_loop_tests.rs @@ -109,6 +109,9 @@ async fn a_covered_click_closes_what_covers_it_and_tries_again() { .collect::>(), ["click", "press escape (uncover)", "click"], ); + // The launch and Escape fetch nothing and settle briefly; the refused + // click does not settle, and the one that went through settles in full. + assert_eq!(sim.trail, ["settle briefly", "settle briefly", "settle"]); } #[tokio::test] diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs index 5cf021e4..645678c5 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/simulator.rs @@ -99,8 +99,8 @@ pub(super) struct Sim { /// The field typed or pasted into last: where the focus stays. pub(super) focused: Option, pub(super) quirks: BTreeSet, - /// The calls that touch no element, in order: each `settle`, and each - /// `await_change` as `changed` or `still`. + /// The calls that touch no element, in order: each `settle` and `settle + /// briefly`, and each `await_change` as `changed` or `still`. pub(super) trail: Vec<&'static str>, } @@ -433,6 +433,10 @@ impl AgentBackend for App { self.sim().trail.push("settle"); } + fn settle_briefly(&self) { + self.sim().trail.push("settle briefly"); + } + fn paste(&self, _app: &str, target: &Candidate, text: &str) -> DesktopResponse { if is_city_row(Some(target)) { return not_a_text_field(); diff --git a/crates/tinycomputer-engine/src/workspace/mod.rs b/crates/tinycomputer-engine/src/workspace/mod.rs index d7fdd9e8..179dd8de 100644 --- a/crates/tinycomputer-engine/src/workspace/mod.rs +++ b/crates/tinycomputer-engine/src/workspace/mod.rs @@ -275,6 +275,14 @@ impl Surface for Workspace { } } + fn settle_briefly(&self) { + match (self.active_browser(), &self.desktop) { + (Some(browser), _) => browser.settle_briefly(), + (None, Some(desktop)) => desktop.settle_briefly(), + (None, None) => {} + } + } + fn await_change(&self, ms: u64) -> bool { match (self.active_browser(), &self.desktop) { (Some(browser), _) => browser.await_change(ms), diff --git a/crates/tinycomputer-engine/src/workspace/workspace_tests.rs b/crates/tinycomputer-engine/src/workspace/workspace_tests.rs index f1b17e99..010333f4 100644 --- a/crates/tinycomputer-engine/src/workspace/workspace_tests.rs +++ b/crates/tinycomputer-engine/src/workspace/workspace_tests.rs @@ -107,6 +107,10 @@ impl Surface for Recorder { self.note("settle"); } + fn settle_briefly(&self) { + self.note("settle_briefly"); + } + fn await_change(&self, _ms: u64) -> bool { self.note("await_change"); true @@ -212,6 +216,24 @@ fn a_wait_for_a_change_watches_the_active_side() { assert!(!bare.await_change(1_000), "nothing to change"); } +#[test] +fn a_brief_settle_settles_the_active_side() { + let (workspace, calls) = workspace(true); + workspace.settle_briefly(); + workspace.navigate("https://flights.test"); + workspace.settle_briefly(); + assert_eq!( + drain(&calls), + [ + "desktop:settle_briefly", + "browser:navigate", + "browser:settle_briefly" + ] + ); + let bare: Workspace = Workspace::new(None, None); + bare.settle_briefly(); +} + #[test] fn going_back_follows_the_active_side() { let (workspace, calls) = workspace(true); diff --git a/docs/crates/tinycomputer-browser/interacting.md b/docs/crates/tinycomputer-browser/interacting.md index b55208a5..1f2d8991 100644 --- a/docs/crates/tinycomputer-browser/interacting.md +++ b/docs/crates/tinycomputer-browser/interacting.md @@ -194,6 +194,15 @@ receive window, and then pauses `SETTLE_MS` regardless, giving a banner or menu that is mid-animation time to finish closing: about 1.6 s an action, live. +`Surface::settle_briefly` follows an action that fetches nothing. Under +`Settle::Prompt` it skips the network wait and waits only while the page +changes (120–400 ms instead of about 0.65 s on an idle page); under +`Settle::Steady` it settles in full. A flow settles briefly only after a +launch, which leaves an open page as it is, and after Escape closing a layer: +live, 3% of launches and 12% of Escapes settled with a request of the page +still running, against 39% of fills, whose suggestions the next look reads, +and 70% of clicks. + `Surface::await_change(ms)` watches the page rather than pausing: one `evaluate` whose `MutationObserver` resolves `true` at the page's first change of its own (an element or words added, removed, or rewritten, or an diff --git a/docs/crates/tinycomputer-browser/surface.md b/docs/crates/tinycomputer-browser/surface.md index 42346509..5600861e 100644 --- a/docs/crates/tinycomputer-browser/surface.md +++ b/docs/crates/tinycomputer-browser/surface.md @@ -131,8 +131,9 @@ of unopened submenus the way a desktop tree does). called before a flow reads the page again after an action, waits, bounded, for the requests that change the page to end, then only while the page is still changing (`Settle::Prompt`; `Settle::Steady` waits for the network to -go idle, then pauses a further beat regardless), as described in -[interacting.md](interacting.md#scrolling-and-waiting). +go idle, then pauses a further beat regardless). `Surface::settle_briefly`, +after a launch or Escape, skips the network wait under `Settle::Prompt`; both +are described in [interacting.md](interacting.md#scrolling-and-waiting). ## Cross-links diff --git a/docs/crates/tinycomputer-core/surfaces-and-screens.md b/docs/crates/tinycomputer-core/surfaces-and-screens.md index 2a62fb69..b99a49ab 100644 --- a/docs/crates/tinycomputer-core/surfaces-and-screens.md +++ b/docs/crates/tinycomputer-core/surfaces-and-screens.md @@ -32,7 +32,10 @@ it to give an application a moment to react: closing a banner a beat after a click, or turning a typed address into a token in an autocomplete field. The engine calls it after every action, before the next observation, and before reading a value back, so a surface's own idea of "how long is a moment" stays -in one place rather than being copied into every caller. +in one place rather than being copied into every caller. After an action that +fetches nothing (a launch, Escape) the engine calls `settle_briefly()` +instead, which settles in full unless the surface overrides it: +`tinycomputer-browser` then waits only while the page changes. And `await_change(ms)`, which waits up to `ms` for the application to change by itself and says whether it did. A flow uses it while it watches for @@ -61,6 +64,7 @@ pub trait Surface: Clone + Send + 'static { fn press(&self, app: &str, combo: &str) -> DesktopResponse; fn launch(&self, app: &str) -> DesktopResponse; fn settle(&self) {} + fn settle_briefly(&self) { /* settles */ } fn await_change(&self, ms: u64) -> bool { /* pauses, and says it may have */ } fn navigate(&self, url: &str) -> DesktopResponse { /* refuses by default */ } fn back(&self, app: &str) -> DesktopResponse { /* refuses by default */ } diff --git a/docs/crates/tinycomputer-engine/workspace.md b/docs/crates/tinycomputer-engine/workspace.md index 0f981fda..46fc1148 100644 --- a/docs/crates/tinycomputer-engine/workspace.md +++ b/docs/crates/tinycomputer-engine/workspace.md @@ -79,7 +79,7 @@ Every method on `Surface` is handled, but not all the same way: | `launch` | By the named application; success makes that side active. | | `navigate` | Always the browser (there is no desktop equivalent); failure never activates it. | | `back` | The active side if it is the browser, else the desktop; there is no browser fallback if the desktop is active and has no browser. | -| `settle`, `await_change` | The active side, or the desktop if the browser is not active; with neither, nothing changes. | +| `settle`, `settle_briefly`, `await_change` | The active side, or the desktop if the browser is not active; with neither, nothing changes. | ## Reading what is on screen without acting diff --git a/docs/technical/architecture.md b/docs/technical/architecture.md index b5af555f..60ad5da2 100644 --- a/docs/technical/architecture.md +++ b/docs/technical/architecture.md @@ -97,6 +97,7 @@ pub trait Surface: Clone + Send + 'static { fn press(&self, app: &str, combo: &str) -> DesktopResponse; fn launch(&self, app: &str) -> DesktopResponse; fn settle(&self) {} + fn settle_briefly(&self) { /* settles */ } fn await_change(&self, ms: u64) -> bool { /* pauses, and says it may have */ } fn navigate(&self, url: &str) -> DesktopResponse { /* ACTION_NOT_SUPPORTED */ } } diff --git a/docs/technical/decision-loops.md b/docs/technical/decision-loops.md index 3ecde97f..3b049df3 100644 --- a/docs/technical/decision-loops.md +++ b/docs/technical/decision-loops.md @@ -261,10 +261,10 @@ Any other answer is ignored and logged. The runtime never falls back to a click on an answer it does not recognise, because a malformed or injected answer must fail closed. -After any action the backend reports as successful, the runtime calls the -surface's `settle` before looking again — on the browser, until the requests -that change the page end and it goes still; a short pause on the desktop — so -the next turn's screen reflects what the action did, not the moment before. +After any action the backend reports as successful, the runtime settles the +surface before looking again (on the browser, until the requests that change +the page end and it goes still, or only until it is still after a launch or +Escape; a short pause on the desktop), so the next look sees what it did. The shortcut list (`act/mod.rs::SHORTCUTS`) is short and safe: new item, new folder, find, reply, settings, back, next field, confirm (Return), and dismiss diff --git a/docs/technical/jev-harness.md b/docs/technical/jev-harness.md index eb37f3f6..749f1fcf 100644 --- a/docs/technical/jev-harness.md +++ b/docs/technical/jev-harness.md @@ -159,7 +159,7 @@ round trip's *slowest* framing, plus the action, plus settling. The levers: | request size | Jev's latency grows with input tokens; a big element list is the usual cause | trimming can drop the element that was needed | | grounding memory | a remembered element is confirmed with one Noul instead of narrowing | none when the hint is right | | `disabled_loops` | each loop off removes a question or a whole decision | measure it before shipping it off | -| `settle` | fixed per action on the desktop; on the browser, `prompt` (the default) waits at most 1 s for the requests that change the page, then ≤ 400 ms for it to go still, and `steady` for network idle (≤ 2 s) and 400 ms more | too short and the next look sees the old screen | +| `settle` | fixed per action on the desktop; on the browser, `prompt` (the default) waits at most 1 s for the requests that change the page, then ≤ 400 ms for it to go still (only the latter after a launch or Escape, which fetch nothing), and `steady` for network idle (≤ 2 s) and 400 ms more | too short and the next look sees the old screen | | observation | an accessibility snapshot of a large window is slow; `explore` adds more | a budgeted view can miss the target | Measure before changing any of them. The debug journal From 544b4fb2c39434966f8401aa8ceea2fbfecf8efc Mon Sep 17 00:00:00 2001 From: Shanu Date: Thu, 8 Oct 2026 13:21:32 +0530 Subject: [PATCH 35/44] Warm Jev's connections while a task's plan is drafted A decision asks its framings all at once, each on an HTTP/1.1 connection of its own, and opening one to the gateway costs a handshake: over 84 live runs, a task's first decision took 330 ms more a call (median) than its later decisions of the same size. JevRuntime::warm(votes) sends one one-question evaluation per framing (at most MAX_VOTES), all at once, through evaluate, so each is journaled under the step "warm-up"; answers are dropped and calls still out after 10 s are given up on. Sage is not warmed. A planned task starts it with the planner through FlowRunner::warm, which does nothing by default, and never waits for it; the module runner warms its Jev runtime, journaled with the task. --- .../src/agentic/agentic_tests.rs | 1 + .../src/agentic/agentic_tests/warm_tests.rs | 98 +++++++++++++++++++ .../src/agentic/flow/mod.rs | 1 + .../src/agentic/flow/vote.rs | 2 +- .../src/agentic/runtime.rs | 49 +++++++++- crates/tinycomputer-engine/src/task/drive.rs | 17 +++- crates/tinycomputer-engine/src/task/mod.rs | 8 ++ .../src/task/task_tests.rs | 8 ++ .../src/task/task_tests/plan_tests.rs | 58 +++++++++++ .../src/task/task_tests/runner_tests.rs | 1 + .../tinycomputer/src/tinybus_module/README.md | 3 +- .../tinycomputer/src/tinybus_module/runner.rs | 9 ++ .../tinybus_module_tests/tasks_tests.rs | 26 +++++ .../crates/tinycomputer-engine/jev-runtime.md | 17 ++++ docs/technical/jev-journal.md | 2 +- docs/technical/tasks.md | 6 ++ 16 files changed, 299 insertions(+), 7 deletions(-) create mode 100644 crates/tinycomputer-engine/src/agentic/agentic_tests/warm_tests.rs diff --git a/crates/tinycomputer-engine/src/agentic/agentic_tests.rs b/crates/tinycomputer-engine/src/agentic/agentic_tests.rs index 484befd4..df4f071b 100644 --- a/crates/tinycomputer-engine/src/agentic/agentic_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/agentic_tests.rs @@ -12,6 +12,7 @@ mod policy_tests; mod resolve_tests; mod scope_tests; mod waiting_tests; +mod warm_tests; use std::{ collections::{BTreeMap, VecDeque}, diff --git a/crates/tinycomputer-engine/src/agentic/agentic_tests/warm_tests.rs b/crates/tinycomputer-engine/src/agentic/agentic_tests/warm_tests.rs new file mode 100644 index 00000000..abd0bda9 --- /dev/null +++ b/crates/tinycomputer-engine/src/agentic/agentic_tests/warm_tests.rs @@ -0,0 +1,98 @@ +//! Tests for warming a runtime's connections while a task's plan is drafted. + +use super::*; +use crate::agentic::runtime::WARM_TIMEOUT; + +/// An evaluator nothing ever comes back from. +struct Silent; + +impl Evaluator for Silent { + fn evaluate<'a>( + &'a self, + _request: &'a tinyinference_decisions::EvaluationRequest, + ) -> std::pin::Pin< + Box< + dyn std::future::Future< + Output = Result< + tinyinference_decisions::EvaluationResult, + tinyinference_decisions::EvaluationFailure, + >, + > + Send + + 'a, + >, + > { + Box::pin(std::future::pending()) + } +} + +fn ready() -> tinyinference_decisions::EvaluationResult { + evaluation(json!({ + "model": "typesafe/jev-1.13-20260917", + "answers": {"ready": {"type": "noul", "noul": 0.9}}, + "usage": {"input_tokens": 10, "output_tokens": 2} + })) +} + +#[tokio::test] +async fn a_warm_up_asks_one_small_question_for_each_framing() { + let (runtime, requests) = runtime_recording(vec![ready(); 7]); + runtime.warm(7).await; + let requests = requests.lock().unwrap(); + assert_eq!(requests.len(), 7, "one call a framing, all at once"); + for request in requests.iter() { + assert_eq!(request.model, "jev-latest", "the runtime's own model"); + assert_eq!( + request.questions.keys().collect::>(), + ["ready"], + "{request:?}" + ); + assert!(matches!( + request.questions["ready"], + tinyinference_decisions::Question::Noul(_) + )); + } +} + +#[tokio::test] +async fn a_warm_up_opens_no_more_than_a_decision_asks_at_once() { + let (runtime, requests) = runtime_recording(vec![ready(); 9]); + runtime.warm(50).await; + assert_eq!(requests.lock().unwrap().len(), 9, "MAX_VOTES at most"); + let (runtime, requests) = runtime_recording(vec![ready()]); + runtime.warm(0).await; + assert_eq!(requests.lock().unwrap().len(), 1, "one at least"); +} + +#[tokio::test] +async fn sage_is_not_warmed() { + let (mut runtime, requests) = runtime_recording(Vec::new()); + runtime.configuration.provider = JevProvider::Sage; + runtime.warm(7).await; + assert!(requests.lock().unwrap().is_empty()); +} + +#[tokio::test(start_paused = true)] +async fn a_warm_up_nothing_answers_is_given_up_on() { + let (mut runtime, _requests) = runtime_recording(Vec::new()); + runtime.client = Arc::new(Silent); + let started = tokio::time::Instant::now(); + runtime.warm(7).await; + assert_eq!(started.elapsed(), WARM_TIMEOUT); +} + +#[tokio::test] +async fn a_warm_up_is_journaled_with_the_task() { + let dir = std::env::temp_dir().join(format!("tinycomputer-warm-{}", std::process::id())); + let (runtime, _requests) = runtime_recording(vec![ready(); 7]); + let runtime = runtime.with_journal(&dir).journaled_as("task-t-1"); + runtime.warm(7).await; + let journal = std::fs::read_to_string(dir.join("task-t-1").join("journal.jsonl")).unwrap(); + let steps = journal + .lines() + .map(|line| serde_json::from_str::(line).unwrap()) + .filter(|event| event["event"] == "exchange") + .map(|event| event["step"].as_str().unwrap().to_owned()) + .collect::>(); + std::fs::remove_dir_all(&dir).unwrap(); + assert_eq!(steps, vec!["warm-up"; 7]); +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/mod.rs b/crates/tinycomputer-engine/src/agentic/flow/mod.rs index 6fb08c05..94ff5ec9 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/mod.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/mod.rs @@ -69,6 +69,7 @@ mod vote; mod wide; pub(crate) use validate::{check as check_flow, missing_inputs}; +pub(in crate::agentic) use vote::MAX_VOTES; use std::{ collections::{BTreeMap, BTreeSet}, diff --git a/crates/tinycomputer-engine/src/agentic/flow/vote.rs b/crates/tinycomputer-engine/src/agentic/flow/vote.rs index c517b5f6..069211cd 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/vote.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/vote.rs @@ -31,7 +31,7 @@ use serde_json::Value; use tinyinference_decisions::{Answer, ChoiceAnswer, EvaluationRequest, NoulAnswer, Question}; /// Most framings one decision is asked in. -pub(super) const MAX_VOTES: u32 = 9; +pub(in crate::agentic) const MAX_VOTES: u32 = 9; /// A perspective added to each framing after the first, in turn. const PERSPECTIVES: [&str; 4] = [ diff --git a/crates/tinycomputer-engine/src/agentic/runtime.rs b/crates/tinycomputer-engine/src/agentic/runtime.rs index ce08af8d..dc9e2c39 100644 --- a/crates/tinycomputer-engine/src/agentic/runtime.rs +++ b/crates/tinycomputer-engine/src/agentic/runtime.rs @@ -12,9 +12,10 @@ use std::{ use tinycomputer_bus::{DesktopError, JevConfig, JevConfiguration, JevProvider}; use tinyinference_decisions::{ Client, ClientConfig, Error as JevError, EvaluationFailure, EvaluationRequest, - EvaluationResult, RetryPolicy, + EvaluationResult, Noul, Question, RetryPolicy, }; +use super::flow::MAX_VOTES; use super::journal::Journal; use super::pending::PendingRun; use super::sage; @@ -34,6 +35,10 @@ pub(super) const RETRY: RetryPolicy = RetryPolicy { /// waited the client's own 30 s before its retry answered in under 1 s. pub(super) const ATTEMPT_TIMEOUT: Duration = Duration::from_secs(10); +/// The longest [`JevRuntime::warm`] waits for its answers before it gives +/// its calls up. +pub(super) const WARM_TIMEOUT: Duration = Duration::from_secs(10); + /// Configured Jev transport and non-secret policy metadata. #[derive(Clone)] pub struct JevRuntime { @@ -222,6 +227,31 @@ impl JevRuntime { } } + /// Opens connections to Jev for the decisions to come, while a task's + /// plan is drafted: one for each framing of a decision asked `votes` + /// ways (at most 9, `MAX_VOTES`), each with a one-question evaluation, + /// all at once. Each is journaled as `warm-up`, its answer is dropped, + /// and those still out after 10 s (`WARM_TIMEOUT`) are given up on. + /// A decision asks its framings all at once, each on a connection of its + /// own; live, a task's first decision took 330 ms more a call than its + /// later ones, opening them. Sage, whose calls take seconds, is not + /// warmed. + pub async fn warm(&self, votes: u32) { + if self.configuration.provider == JevProvider::Sage { + return; + } + let request = Arc::new(warm_up(&self.configuration.model)); + let mut calls = tokio::task::JoinSet::new(); + for _ in 0..votes.clamp(1, MAX_VOTES) { + let (runtime, request) = (self.clone(), Arc::clone(&request)); + calls.spawn(async move { + let _answer = runtime.evaluate(Some("warm-up"), &request).await; + }); + } + // Calls still out when the time is up are dropped with the set. + let _warmed = tokio::time::timeout(WARM_TIMEOUT, calls.join_all()).await; + } + /// Asks Jev one request, journaling the exchange against `step`. pub(super) async fn evaluate( &self, @@ -234,6 +264,23 @@ impl JevRuntime { } } +/// The smallest request `model` answers: one yes/no question about nothing +/// on any screen. +fn warm_up(model: &str) -> EvaluationRequest { + EvaluationRequest { + state: serde_json::json!("A connection check."), + model: model.to_owned(), + questions: [( + "ready".to_owned(), + Question::Noul(Noul { + instructions: serde_json::json!("The state is a connection check."), + criteria: None, + }), + )] + .into(), + } +} + pub(super) trait Evaluator: Send + Sync { fn evaluate<'a>( &'a self, diff --git a/crates/tinycomputer-engine/src/task/drive.rs b/crates/tinycomputer-engine/src/task/drive.rs index 9589f7dd..47e05011 100644 --- a/crates/tinycomputer-engine/src/task/drive.rs +++ b/crates/tinycomputer-engine/src/task/drive.rs @@ -14,7 +14,7 @@ use super::interpret::{Next, finished, run_outcome}; use super::publish::{publish, records, stopped_summary}; use super::recovery::{rescue_outcome, rescued}; use super::store::{Cell, Run}; -use super::{FlowRunner, SHAPE_TIMEOUT_MS}; +use super::{DEFAULT_VOTES, FlowRunner, SHAPE_TIMEOUT_MS}; use crate::shape::Harvest; /// Plans the task, then runs the plan — or asks for what it needs first. @@ -25,18 +25,29 @@ pub(super) async fn plan_then_drive( task: String, surfaces: Vec, ) { - let (names, secrets, constraints) = cell.state.lock().map_or_else( - |_| (Vec::new(), Vec::new(), TaskConstraints::default()), + let (names, secrets, constraints, votes) = cell.state.lock().map_or_else( + |_| { + ( + Vec::new(), + Vec::new(), + TaskConstraints::default(), + DEFAULT_VOTES, + ) + }, |state| { let owned = |names: Vec<&str>| names.into_iter().map(str::to_owned).collect::>(); ( owned(state.facts.names()), owned(state.facts.secret_names()), state.constraints.clone(), + state.budget.votes.unwrap_or(DEFAULT_VOTES), ) }, ); let id = cell.view.borrow().id.clone(); + // Jev's connections open while the plan is drafted, so the first + // decision need not open them; nothing waits for this. + tokio::spawn(runner.warm(&id, votes)); let started = Instant::now(); // Timed on its own: a browser slower to open than the plan is to draft // is not planning time. diff --git a/crates/tinycomputer-engine/src/task/mod.rs b/crates/tinycomputer-engine/src/task/mod.rs index 4606181c..143a6a46 100644 --- a/crates/tinycomputer-engine/src/task/mod.rs +++ b/crates/tinycomputer-engine/src/task/mod.rs @@ -122,6 +122,14 @@ pub trait FlowRunner: Send + Sync + 'static { Box::pin(async {}) } + /// Opens the connections the task's first decision, asked `votes` ways, + /// will use, while its plan is drafted (`JevRuntime::warm`). Started + /// with the planner for every task and never waited for: the task goes + /// on whether it finishes or not. Does nothing by default. + fn warm(&self, _task: &TaskId, _votes: u32) -> PrepareFuture { + Box::pin(async {}) + } + /// Writes an `event` of the time a task spends outside its flows /// (`plan`, `rescue`, `resume`) to the debug journal: the task's own, /// or for `PlanTask`, which plans before any task exists (`task` is diff --git a/crates/tinycomputer-engine/src/task/task_tests.rs b/crates/tinycomputer-engine/src/task/task_tests.rs index 76d42adc..dbf424b6 100644 --- a/crates/tinycomputer-engine/src/task/task_tests.rs +++ b/crates/tinycomputer-engine/src/task/task_tests.rs @@ -57,6 +57,9 @@ struct Script { journaled: Mutex, String, serde_json::Value)>>, /// Tasks whose surfaces were got ready while they were planned. prepared: Mutex>, + /// Tasks whose Jev connections were warmed while they were planned, + /// with the votes asked for. The warm-up never ends. + warmed: Mutex>, } impl FlowRunner for Script { @@ -100,6 +103,11 @@ impl FlowRunner for Script { Box::pin(async {}) } + fn warm(&self, task: &TaskId, votes: u32) -> super::PrepareFuture { + self.warmed.lock().unwrap().push((task.clone(), votes)); + Box::pin(std::future::pending()) + } + fn journal(&self, task: Option<&TaskId>, event: &str, fields: &dyn Fn() -> serde_json::Value) { self.journaled .lock() diff --git a/crates/tinycomputer-engine/src/task/task_tests/plan_tests.rs b/crates/tinycomputer-engine/src/task/task_tests/plan_tests.rs index d61a040d..2c72ea93 100644 --- a/crates/tinycomputer-engine/src/task/task_tests/plan_tests.rs +++ b/crates/tinycomputer-engine/src/task/task_tests/plan_tests.rs @@ -242,3 +242,61 @@ async fn a_browser_only_task_gets_its_browser_ready_while_it_is_planned() { std::slice::from_ref(&started.id) ); } + +#[tokio::test] +async fn a_task_warms_jev_while_it_is_planned_and_never_waits_for_it() { + // The script's warm-up never ends: the task finishes all the same. + let flow = r#"{"app": "Mail", "steps": ["start a new email message"]}"#; + let (tasks, script) = planned( + vec![finished_run(FlowStopReason::Completed, vec![], &[], None)], + Ok(flow), + ); + let started = tasks + .start(&StartTaskRequest { + task: Some("start an email".to_owned()), + ..StartTaskRequest::default() + }) + .data + .unwrap(); + assert!(matches!( + settle(&tasks, &started.id).await.status, + TaskStatus::Done { .. } + )); + assert_eq!(*script.warmed.lock().unwrap(), [(started.id.clone(), 7)]); + + // Asked as many ways as the task's budget says. + let (tasks, script) = planned( + vec![finished_run(FlowStopReason::Completed, vec![], &[], None)], + Ok(flow), + ); + let started = tasks + .start(&StartTaskRequest { + task: Some("start an email".to_owned()), + budget: tinycomputer_bus::agent::TaskBudget { + votes: Some(3), + ..tinycomputer_bus::agent::TaskBudget::default() + }, + ..StartTaskRequest::default() + }) + .data + .unwrap(); + settle(&tasks, &started.id).await; + assert_eq!(*script.warmed.lock().unwrap(), [(started.id.clone(), 3)]); + + // A flow handed over whole is not planned, and not warmed. + let (tasks, script) = controller(vec![finished_run( + FlowStopReason::Completed, + vec![], + &[], + None, + )]); + let started = tasks + .start(&StartTaskRequest { + flow: Some(serde_json::from_str(flow).unwrap()), + ..StartTaskRequest::default() + }) + .data + .unwrap(); + settle(&tasks, &started.id).await; + assert!(script.warmed.lock().unwrap().is_empty()); +} diff --git a/crates/tinycomputer-engine/src/task/task_tests/runner_tests.rs b/crates/tinycomputer-engine/src/task/task_tests/runner_tests.rs index 0dfea938..e62bdb5c 100644 --- a/crates/tinycomputer-engine/src/task/task_tests/runner_tests.rs +++ b/crates/tinycomputer-engine/src/task/task_tests/runner_tests.rs @@ -28,6 +28,7 @@ async fn a_runner_that_only_runs_flows_reads_nothing_and_holds_nothing() { RunsOnly.release(&task); // Nothing to get ready, and no journal to write to. RunsOnly.prepare(&task, &TaskConstraints::default()).await; + RunsOnly.warm(&task, 7).await; RunsOnly.journal(Some(&task), "plan", &|| serde_json::json!({"wall_ms": 1})); RunsOnly.journal(None, "plan", &|| serde_json::json!({})); assert_eq!( diff --git a/crates/tinycomputer/src/tinybus_module/README.md b/crates/tinycomputer/src/tinybus_module/README.md index de5a7a23..1a374494 100644 --- a/crates/tinycomputer/src/tinybus_module/README.md +++ b/crates/tinycomputer/src/tinybus_module/README.md @@ -57,7 +57,8 @@ The module takes its configuration from the loader as a JSON object, parsed into `trace_strict` and `headed` as booleans. `browser` sets how every browser launches — `executable`, `user_agent`, `args`, a task's page `perception`, how a page `settle`s after an action, and whether to -`prelaunch` a planned browser task's browser — and `cursor` sets the +`prelaunch` a planned browser task's browser (a planned task also warms +the `jev` runtime's connections meanwhile) — and `cursor` sets the agent's on-screen cursor: a pace (`off`, `brisk`, `natural`, `calm`) or `{pace, overlay}` with the overlay helper's path. An optional `jev` object configures the diff --git a/crates/tinycomputer/src/tinybus_module/runner.rs b/crates/tinycomputer/src/tinybus_module/runner.rs index 042f2e1b..c2dc9a5f 100644 --- a/crates/tinycomputer/src/tinybus_module/runner.rs +++ b/crates/tinycomputer/src/tinybus_module/runner.rs @@ -159,6 +159,15 @@ impl FlowRunner for WorkspaceRunner { }) } + fn warm(&self, task: &TaskId, votes: u32) -> PrepareFuture { + let Some(runtime) = self.jev.as_ref() else { + return Box::pin(async {}); + }; + // Journaled with the task's flows (see `run`). + let runtime = runtime.journaled_as(&format!("task-{task}")); + Box::pin(async move { runtime.warm(votes).await }) + } + fn journal(&self, task: Option<&TaskId>, event: &str, fields: &dyn Fn() -> serde_json::Value) { let Some(runtime) = self.jev.as_ref().filter(|runtime| runtime.journaling()) else { return; diff --git a/crates/tinycomputer/src/tinybus_module/tinybus_module_tests/tasks_tests.rs b/crates/tinycomputer/src/tinybus_module/tinybus_module_tests/tasks_tests.rs index bcc5a281..28487c0d 100644 --- a/crates/tinycomputer/src/tinybus_module/tinybus_module_tests/tasks_tests.rs +++ b/crates/tinycomputer/src/tinybus_module/tinybus_module_tests/tasks_tests.rs @@ -244,3 +244,29 @@ async fn the_runner_opens_no_browser_early_once_prelaunch_is_off() { .await; assert_eq!(runner.workspaces.lock().unwrap().len(), 1); } + +#[tokio::test] +async fn the_runner_warms_its_jev_runtime_for_a_task() { + use tinycomputer_bus::agent::TaskId; + use tinycomputer_engine::{FlowRunner, JevRuntime}; + + let browser = || { + std::sync::Arc::new(tinycomputer_browser::Browser::new(std::sync::Arc::new( + tinycomputer_browser::AgentBrowser, + ))) + }; + // The task's runtime is reached; Sage's calls take seconds and are not + // warmed, so nothing goes out. + let jev = JevRuntime::sage("test-key", false).unwrap(); + crate::tinybus_module::runner::WorkspaceRunner::new( + crate::Desktop::new(), + Some(jev), + browser(), + ) + .warm(&TaskId::new("t-1"), 7) + .await; + // With no Jev runtime there is nothing to warm. + crate::tinybus_module::runner::WorkspaceRunner::new(crate::Desktop::new(), None, browser()) + .warm(&TaskId::new("t-1"), 7) + .await; +} diff --git a/docs/crates/tinycomputer-engine/jev-runtime.md b/docs/crates/tinycomputer-engine/jev-runtime.md index abdf13b6..9846fd12 100644 --- a/docs/crates/tinycomputer-engine/jev-runtime.md +++ b/docs/crates/tinycomputer-engine/jev-runtime.md @@ -126,6 +126,23 @@ the journal. That is the enforcement mechanism for "one door": there is exactly one function in this crate that can produce a Jev answer, and it always logs. +## Warming connections + +A decision asks its framings all at once, each on an HTTP/1.1 connection of +its own, and opening one to the gateway costs a handshake: live, a task's +first decision took 330 ms more a call (the median over 84 runs) than the +task's later decisions of the same size. `JevRuntime::warm(votes)` opens them +ahead of time. It sends one small evaluation per framing of a decision asked +`votes` ways (at most 9, `MAX_VOTES`), all at once, each a single yes/no +question about a one-line state, through `evaluate` like any other call, so +each is journaled under the step `warm-up`. The answers are dropped, calls +still out after 10 s (`WARM_TIMEOUT`) are given up on, and Sage, whose calls +take seconds, is not warmed. + +The task controller warms while a task's plan is drafted +(`FlowRunner::warm`), so its first decision finds the connections open; the +task never waits for the warm-up. + ## Run identity and the journal A `JevRuntime` carries a journal handle (see [journal.md](journal.md)) that diff --git a/docs/technical/jev-journal.md b/docs/technical/jev-journal.md index cfbc96b4..ffceaa23 100644 --- a/docs/technical/jev-journal.md +++ b/docs/technical/jev-journal.md @@ -60,7 +60,7 @@ has `""`, and goal and intent runs carry their goal or intent text. | `event` | When | Fields | |---|---|---| | `run` | a run begins | `kind` (`flow`, `goal`, `goal-continuation`, `intent`), `label`, `model`, `pid` | -| `exchange` | every Jev call, one per framing | `step`, `questions` (ids), `request_bytes`, `request` (the exact `EvaluationRequest`), `ok`, `latency_ms`, `attempts`; on success `request_id`, `model`, `input_tokens`, `output_tokens`, `answers`; on failure `error` | +| `exchange` | every Jev call, one per framing | `step` (`warm-up` for the calls that open connections while a task is planned), `questions` (ids), `request_bytes`, `request` (the exact `EvaluationRequest`), `ok`, `latency_ms`, `attempts`; on success `request_id`, `model`, `input_tokens`, `output_tokens`, `answers`; on failure `error` | | `decision` | a flow decision is merged | `step`, `questions`, `framings`, `answered`, `batched` (requests asked in the same round trip), `parts` (requests the decision's questions were split across; 1 unless they outgrew `MAX_REQUEST_BYTES`), `request_bytes` (the largest part), `wall_ms` — what the step actually waited | | `turn` | a `do` turn ends | `step`, `turn`, `decisions` (made in that turn), `rounds` (round trips they took: a batch is one), `wall_ms` | | `survey` | the wide strategy surveys a crowded screen | `step`, `regions` asked about, `most_relevant` (region ids), `distractions` | diff --git a/docs/technical/tasks.md b/docs/technical/tasks.md index 7214cb73..82e7c4a4 100644 --- a/docs/technical/tasks.md +++ b/docs/technical/tasks.md @@ -283,6 +283,12 @@ values lets the browser go while the task waits (`needs_input`), and the run that follows opens it again; a task cancelled while its browser was still opening is left holding none. +Every task planned this way also has its runner warm Jev while the plan is +drafted (`FlowRunner::warm`): one small evaluation for each way its first +decision will be asked, so that decision finds its connections open +([`jev-runtime.md`](../crates/tinycomputer-engine/jev-runtime.md#warming-connections)). +The task never waits for the warm-up. + ## Rescues When a top-level step fails and no person is needed, the task asks a From c9970c2493513942524f59e10f933cedc03a13b2 Mon Sep 17 00:00:00 2001 From: Shanu Date: Thu, 8 Oct 2026 13:46:55 +0530 Subject: [PATCH 36/44] End a decision once all but two framings agree plainly A decision waits for its slowest framing. Asked 7 ways or more, it is now merged without its last 2 framings once those in settle every question: every Choice and Score ranks the same option first at 0.9 or more, every yes/no is at 0.97 or more in each framing or at 0.08 or less in each (the evidence band beyond every yes/no threshold), and the page kind reads alike. Replayed over 13,669 decisions asked seven ways in 245 live runs, such a quorum ended 12% of them, a median 0.14 s sooner, about 2 s a run, and every one read the same on every threshold, choice floor and evidence gate as all seven framings did, but for two whose mean moved by under 0.002 across the band's edge. A quorum of four agreed only 99.2% of the time: its stragglers dissented. Framings are taken in the order they finish, and an answer already in when the quorum is reached is used. Those left are not cancelled: they run to their end, so their connections go back to the pool, count as calls, and journal their exchanges after the decision, whose event now says how many it did not wait for (left). --- .../src/agentic/flow/README.md | 1 + .../src/agentic/flow/decide.rs | 37 +- .../src/agentic/flow/flow_tests.rs | 5 +- .../agentic/flow/flow_tests/quorum_tests.rs | 428 ++++++++++++++++++ .../src/agentic/flow/mod.rs | 1 + .../src/agentic/flow/quorum.rs | 201 ++++++++ .../src/agentic/flow/vote.rs | 20 +- .../flow/voting-and-briefing.md | 15 + docs/technical/decision-thresholds.md | 3 + docs/technical/jev-harness.md | 2 +- docs/technical/jev-journal.md | 9 +- 11 files changed, 698 insertions(+), 24 deletions(-) create mode 100644 crates/tinycomputer-engine/src/agentic/flow/flow_tests/quorum_tests.rs create mode 100644 crates/tinycomputer-engine/src/agentic/flow/quorum.rs diff --git a/crates/tinycomputer-engine/src/agentic/flow/README.md b/crates/tinycomputer-engine/src/agentic/flow/README.md index e430bcf7..72fdd10b 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/README.md +++ b/crates/tinycomputer-engine/src/agentic/flow/README.md @@ -40,6 +40,7 @@ evidence, checked after acting, and undone and retried when wrong. | `view/` | re-exports the screen model and digest from `tinycomputer-core::surface`; keeps the flow's policy: the act threshold, which controls it must not press, and `named_first` | | `backend/` | `AgentBackend` (the core `Surface` trait, which `Desktop` implements in `tinycomputer-desktop/src/surface/`) and the async wrappers that call it off the executor | | `vote.rs` | framings, ballots, and their tally | +| `quorum.rs` | a decision merged without its last two framings once the rest agree plainly | | `flow_tests.rs` | the harness every flow test runs through; `flow_tests/` holds a simulated mail app and web shop (`simulator.rs`, `screens.rs`), an oracle Jev that answers from their state (`oracle.rs`), and each topic's tests in `_tests.rs`, deliberation's scenarios in `deliberation_tests.rs` | ## Operational constraints diff --git a/crates/tinycomputer-engine/src/agentic/flow/decide.rs b/crates/tinycomputer-engine/src/agentic/flow/decide.rs index 6287ebd8..0a55252f 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/decide.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/decide.rs @@ -9,7 +9,8 @@ use tinycomputer_core::Facts; use tinyinference_decisions::{Answer, EvaluationRequest, Question}; use super::{ - FlowRun, Halt, MAX_REQUEST_BYTES, StepLog, ask, backend::AgentBackend, brief::clip, hedge, vote, + FlowRun, Halt, MAX_REQUEST_BYTES, StepLog, ask, backend::AgentBackend, brief::clip, hedge, + quorum, vote, }; use crate::agentic::{journal::millis, merge_metrics, provider_error}; @@ -18,8 +19,10 @@ impl FlowRun<'_, B> { /// /// The request is briefed and masked first, then asked in as many /// framings as the run votes with — concurrently, each one charged as an - /// evaluation — and the answers are averaged. On a web page it also - /// carries a page-kind question, whose answer briefs the next request. + /// evaluation — and the answers are averaged: every framing's, or those + /// in once all but two of seven or more agree plainly (`quorum.rs`). On + /// a web page it also carries a page-kind question, whose answer briefs + /// the next request. pub(in crate::agentic::flow) async fn ask( &mut self, log: &mut StepLog, @@ -72,20 +75,23 @@ impl FlowRun<'_, B> { // A part none of whose framings answered leaves its questions // without an answer, which fails the decision as a whole. let mut unanswered = false; + let mut left = 0_u32; for (framings, handles) in framings.into_iter().zip(handles) { let before = answered.len(); - for (framing, handle) in framings.into_iter().zip(handles) { - match handle.await { - Ok(Ok(evaluation)) => { - merge_metrics(&mut self.metrics, &evaluation); - log.calls = log.calls.saturating_add(1); - answered.push((framing, evaluation.response.answers)); - } - Ok(Err(error)) => { - failure.get_or_insert(error); - } - Err(_) => {} - } + let size = quorum::size(framings.len()); + let gathered = quorum::gather(framings, handles, size).await; + for (framing, evaluation) in gathered.answered { + merge_metrics(&mut self.metrics, &evaluation); + log.calls = log.calls.saturating_add(1); + answered.push((framing, evaluation.response.answers)); + } + // Framings a quorum did not wait for still run, and are + // charged as made: their tokens are not known yet. + self.metrics.calls = self.metrics.calls.saturating_add(gathered.left); + log.calls = log.calls.saturating_add(gathered.left); + left = left.saturating_add(gathered.left); + if let Some(error) = gathered.failure { + failure.get_or_insert(error); } unanswered |= answered.len() == before; } @@ -111,6 +117,7 @@ impl FlowRun<'_, B> { "questions": request.questions.keys().collect::>(), "framings": votes, "answered": answered.len(), + "left": left, "batched": batched, "parts": parts.len(), "request_bytes": largest(&parts), diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests.rs index 3dd89893..e39d5cd5 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests.rs @@ -31,6 +31,7 @@ mod hedge_tests; mod helpers_tests; mod journal_tests; mod pick_tests; +mod quorum_tests; mod reflection_tests; mod split_tests; mod step_kinds_tests; @@ -81,9 +82,9 @@ use super::{ vote, wide, }; -fn runtime(oracle: Oracle) -> JevRuntime { +fn runtime(client: impl Evaluator + 'static) -> JevRuntime { JevRuntime { - client: Arc::new(oracle), + client: Arc::new(client), configuration: tinycomputer_bus::JevConfiguration { provider: tinycomputer_bus::JevProvider::OpenRouter, model: "jev-latest".to_owned(), diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/quorum_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/quorum_tests.rs new file mode 100644 index 00000000..9b112cf6 --- /dev/null +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/quorum_tests.rs @@ -0,0 +1,428 @@ +//! Ending a decision on a quorum: once all but its last two framings have +//! answered and agree plainly, the rest are not waited for. + +use super::*; +use crate::agentic::flow::quorum::{self, QUORUM_TOP, SURE_NO, SURE_YES}; + +/// A target Choice whose options each framing relabels and reorders, a +/// yes/no, and the page kind. +fn request() -> EvaluationRequest { + let choice = |options: &[&str], labelled: bool| { + Question::Choice(tinyinference_decisions::Choice { + instructions: json!({"question": "which"}), + criteria: options + .iter() + .enumerate() + .map(|(index, option)| { + if labelled { + ((index + 1).to_string(), Some(json!(option))) + } else { + ((*option).to_owned(), None) + } + }) + .collect(), + }) + }; + EvaluationRequest { + state: json!("a product page"), + model: "jev-latest".to_owned(), + questions: BTreeMap::from([ + ( + "target".to_owned(), + choice(&["Add to Cart", "Buy Now", "Wishlist"], true), + ), + ( + "done".to_owned(), + Question::Noul(tinyinference_decisions::Noul { + instructions: json!({"question": "done?"}), + criteria: None, + }), + ), + ("page_kind".to_owned(), choice(&["product", "cart"], false)), + ]), + } +} + +/// One framing's answers: `target` (an option's text) at `sure`, `done` +/// at `done`, and the page kind `kind` at a weak 0.6, in the framing's own +/// keys. +fn answers( + framing: &vote::Framing, + target: &str, + sure: f64, + done: f64, + kind: &str, +) -> BTreeMap { + let Question::Choice(choice) = &framing.request.questions["target"] else { + panic!("target is a choice"); + }; + let key = choice + .criteria + .iter() + .find(|(_, text)| text.as_ref() == Some(&json!(target))) + .map(|(key, _)| key.clone()) + .unwrap(); + let others = (1.0 - sure) / 2.0; + let other = if kind == "product" { "cart" } else { "product" }; + BTreeMap::from([ + ( + "target".to_owned(), + Answer::Choice(ChoiceAnswer { + choice: key.clone(), + probabilities: choice + .criteria + .keys() + .map(|option| (option.clone(), if *option == key { sure } else { others })) + .collect(), + confidence: sure, + }), + ), + ("done".to_owned(), noul(done)), + ( + "page_kind".to_owned(), + Answer::Choice(ChoiceAnswer { + choice: kind.to_owned(), + probabilities: BTreeMap::from([(kind.to_owned(), 0.6), (other.to_owned(), 0.4)]), + confidence: 0.6, + }), + ), + ]) +} + +/// Whether five framings answering as `each` says, framing by framing, +/// settle the decision. +fn settles(each: impl Fn(usize) -> (&'static str, f64, f64, &'static str)) -> bool { + let framings = vote::framings(&request(), 7); + let answered = (0..5) + .map(|index| { + let (target, sure, done, kind) = each(index); + (index, answers(&framings[index], target, sure, done, kind)) + }) + .collect::>(); + quorum::settled( + &framings, + answered.iter().map(|(index, answers)| (*index, answers)), + 5, + ) +} + +#[test] +fn only_a_decision_asked_seven_ways_or_more_ends_on_a_quorum() { + assert_eq!(quorum::size(7), Some(5)); + assert_eq!(quorum::size(9), Some(7)); + assert_eq!(quorum::size(6), None); + assert_eq!(quorum::size(1), None); +} + +#[test] +fn five_plain_answers_settle_a_decision_however_each_framing_labels_it() { + // Each framing names "Add to Cart" by a key of its own; the page kind + // needs only to be the same. + assert!(settles(|_| ("Add to Cart", 0.95, 0.99, "product"))); + assert!(settles(|_| ("Add to Cart", QUORUM_TOP, SURE_NO, "product"))); + assert!(settles(|_| ("Add to Cart", 0.95, SURE_YES, "product"))); +} + +#[test] +fn a_decision_any_answer_leaves_open_is_not_settled() { + let weak_pick = |index: usize| { + ( + "Add to Cart", + if index == 3 { 0.85 } else { 0.95 }, + 0.99, + "product", + ) + }; + assert!(!settles(weak_pick), "a pick under QUORUM_TOP"); + let other_pick = |index: usize| { + ( + if index == 2 { "Buy Now" } else { "Add to Cart" }, + 0.95, + 0.99, + "product", + ) + }; + assert!(!settles(other_pick), "a framing picked another option"); + let unsure = |index: usize| { + ( + "Add to Cart", + 0.95, + if index == 4 { 0.95 } else { 0.99 }, + "product", + ) + }; + assert!(!settles(unsure), "a yes/no short of SURE_YES"); + let split = |index: usize| { + ( + "Add to Cart", + 0.95, + if index == 0 { 0.02 } else { 0.99 }, + "product", + ) + }; + assert!(!settles(split), "a yes/no answered both ways"); + let page = |index: usize| { + ( + "Add to Cart", + 0.95, + 0.99, + if index == 1 { "cart" } else { "product" }, + ) + }; + assert!(!settles(page), "the page kind read two ways"); +} + +#[test] +fn a_decision_short_of_its_quorum_is_not_settled() { + let framings = vote::framings(&request(), 7); + let plain = |index: usize| answers(&framings[index], "Add to Cart", 0.95, 0.99, "product"); + let four = (0..4) + .map(|index| (index, plain(index))) + .collect::>(); + assert!(!quorum::settled( + &framings, + four.iter().map(|(index, answers)| (*index, answers)), + 5 + )); + // Five answered, but one left a question out. + let mut five = (0..5) + .map(|index| (index, plain(index))) + .collect::>(); + five[2].1.remove("done"); + assert!(!quorum::settled( + &framings, + five.iter().map(|(index, answers)| (*index, answers)), + 5 + )); +} + +/// Framing tasks for `framings`, framing `index` answering after `plan`'s +/// wait with its answers, or failing; each counts itself in `finished` as +/// it ends. +fn paced( + framings: &[vote::Framing], + plan: &[(u64, Option<(&'static str, f64)>)], + finished: &Arc, +) -> Vec>> { + framings + .iter() + .zip(plan) + .map(|(framing, (wait, answer))| { + let answers = + answer.map(|(target, done)| answers(framing, target, 0.95, done, "product")); + let (wait, finished) = (Duration::from_millis(*wait), Arc::clone(finished)); + tokio::spawn(async move { + tokio::time::sleep(wait).await; + finished.fetch_add(1, Ordering::SeqCst); + answers + .map(|answers| EvaluationResult { + response: EvaluationResponse { + model: "typesafe/jev-test".to_owned(), + answers, + usage: tinyinference_decisions::Usage::default(), + }, + request_id: None, + attempts: 1, + latency: wait, + }) + .ok_or_else(|| EvaluationFailure { + error: Box::new(tinyinference_decisions::Error::RateLimited), + attempts: 1, + latency: wait, + }) + }) + }) + .collect() +} + +#[tokio::test(start_paused = true)] +async fn a_decision_ends_once_five_plain_answers_are_in_and_the_rest_still_run() { + let framings = vote::framings(&request(), 7); + let finished = Arc::new(AtomicU64::new(0)); + let agreeing = Some(("Add to Cart", 0.99)); + let plan = [ + (500, agreeing), + (100, agreeing), + (300, agreeing), + (200, agreeing), + (400, agreeing), + (900, agreeing), + (3_000, agreeing), + ]; + let handles = paced(&framings, &plan, &finished); + let started = tokio::time::Instant::now(); + let gathered = quorum::gather(framings, handles, quorum::size(7)).await; + assert_eq!( + started.elapsed(), + Duration::from_millis(500), + "the fifth answer" + ); + assert_eq!(gathered.left, 2); + assert!(gathered.failure.is_none()); + // In framing order, as the ballots read them. + assert_eq!( + gathered + .answered + .iter() + .map(|(_, evaluation)| evaluation.latency.as_millis()) + .collect::>(), + [500, 100, 300, 200, 400] + ); + // The two left are not cut off: each runs to its end. + tokio::time::sleep(Duration::from_secs(3)).await; + assert_eq!(finished.load(Ordering::SeqCst), 7); +} + +#[tokio::test(start_paused = true)] +async fn a_decision_waits_for_every_framing_unless_five_agree_plainly() { + let finished = Arc::new(AtomicU64::new(0)); + let agreeing = Some(("Add to Cart", 0.99)); + let started = tokio::time::Instant::now(); + // One of the first five in picks another option: all seven are waited + // for. (One that answers after five agreeing ones is not.) + let framings = vote::framings(&request(), 7); + let mut plan = [(100, agreeing); 7]; + plan[2] = (50, Some(("Buy Now", 0.99))); + plan[6] = (3_000, agreeing); + let handles = paced(&framings, &plan, &finished); + let gathered = quorum::gather(framings, handles, quorum::size(7)).await; + assert_eq!(started.elapsed(), Duration::from_millis(3_000)); + assert_eq!((gathered.answered.len(), gathered.left), (7, 0)); + + // A failed framing is no answer; five others are a quorum still. + let started = tokio::time::Instant::now(); + let framings = vote::framings(&request(), 7); + let mut plan = [(100, agreeing); 7]; + plan[0] = (50, None); + plan[6] = (3_000, agreeing); + let handles = paced(&framings, &plan, &finished); + let gathered = quorum::gather(framings, handles, quorum::size(7)).await; + assert_eq!(started.elapsed(), Duration::from_millis(100)); + assert!(gathered.failure.is_some()); + assert_eq!((gathered.answered.len(), gathered.left), (5, 1)); + + // Asked fewer than seven ways, a decision waits for every framing. + let started = tokio::time::Instant::now(); + let framings = vote::framings(&request(), 6); + let mut plan = [(100, agreeing); 6]; + plan[5] = (2_000, agreeing); + let handles = paced(&framings, &plan, &finished); + let gathered = quorum::gather(framings, handles, quorum::size(6)).await; + assert_eq!(started.elapsed(), Duration::from_millis(2_000)); + assert_eq!((gathered.answered.len(), gathered.left), (6, 0)); +} + +/// The oracle, with each decision's framings answering after the waits in +/// `pace`, in the order they are asked. +struct Staggered { + oracle: Oracle, + pace: [u64; 7], + calls: Mutex, +} + +impl Evaluator for Staggered { + fn evaluate<'a>( + &'a self, + request: &'a EvaluationRequest, + ) -> Pin> + Send + 'a>> + { + let call = { + let mut calls = self.calls.lock().unwrap(); + *calls += 1; + *calls - 1 + }; + let wait = Duration::from_millis(self.pace[call % self.pace.len()]); + Box::pin(async move { + tokio::time::sleep(wait).await; + self.oracle.evaluate(request).await + }) + } +} + +/// A `verify` run asked seven ways, its last two framings 2 s behind the +/// rest, with Jev `sure` the inbox shows; how long it took, its result, and +/// its journal's decisions. +async fn verify_staggered(sure: f64) -> (Duration, FlowRunResult, Vec) { + let scratch = std::env::temp_dir().join(format!( + "tinycomputer-quorum-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let app = App::with(|_| {}); + let staggered = Staggered { + oracle: Oracle { + app: app.clone(), + hook: Box::new(move |id: &str, _: &Question, _: &Sim| { + (id == "holds").then(|| noul(sure)) + }), + requests: Mutex::new(Vec::new()), + fail: false, + }, + pace: [10, 10, 10, 10, 10, 2_000, 2_000], + calls: Mutex::new(0), + }; + let mut runtime = runtime(staggered); + runtime.journal = crate::agentic::journal::Journal::at(&scratch).fresh("quorum"); + let dir = runtime.journal.run_dir().unwrap(); + let started = tokio::time::Instant::now(); + let reply = run_flow_with( + app, + &runtime, + RunFlowRequest { + flow: serde_json::from_value(json!({ + "app": "Mail", + "steps": [{"verify": "the inbox shows"}] + })) + .unwrap(), + votes: 7, + ..RunFlowRequest::default() + }, + ) + .await; + let took = started.elapsed(); + let decisions = std::fs::read_to_string(dir.join(crate::JOURNAL_FILE)) + .unwrap() + .lines() + .map(|line| serde_json::from_str::(line).unwrap()) + .filter(|event| event["event"] == "decision") + .collect(); + let _ = std::fs::remove_dir_all(&scratch); + ( + took, + serde_json::from_value(reply.data.unwrap()).unwrap(), + decisions, + ) +} + +#[tokio::test(start_paused = true)] +async fn a_run_whose_framings_agree_plainly_does_not_wait_for_its_slowest() { + let (took, result, decisions) = verify_staggered(0.99).await; + assert_eq!(result.stop, FlowStopReason::Completed, "{:?}", result.steps); + assert!(took < Duration::from_secs(2), "{took:?}"); + assert_ne!(decisions.len(), 0); + for decision in &decisions { + assert_eq!( + ( + &decision["framings"], + &decision["answered"], + &decision["left"] + ), + (&json!(7), &json!(5), &json!(2)), + "{decision}" + ); + } + // Every framing sent is a call made, waited for or not. + assert_eq!( + usize::try_from(result.metrics.calls).unwrap(), + decisions.len() * 7 + ); + + // Jev only fairly sure: every decision waits for all seven. + let (took, result, decisions) = verify_staggered(0.9).await; + assert_eq!(result.stop, FlowStopReason::Completed, "{:?}", result.steps); + assert!(took >= Duration::from_secs(2), "{took:?}"); + assert!(decisions.iter().all(|decision| decision["left"] == 0)); +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/mod.rs b/crates/tinycomputer-engine/src/agentic/flow/mod.rs index 94ff5ec9..be18eae8 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/mod.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/mod.rs @@ -59,6 +59,7 @@ mod hedge; mod ledger; mod look; mod memory; +mod quorum; mod reflect; mod run; mod steps; diff --git a/crates/tinycomputer-engine/src/agentic/flow/quorum.rs b/crates/tinycomputer-engine/src/agentic/flow/quorum.rs new file mode 100644 index 00000000..91e8a14d --- /dev/null +++ b/crates/tinycomputer-engine/src/agentic/flow/quorum.rs @@ -0,0 +1,201 @@ +//! Ending a decision on a quorum: once all but its last two framings have +//! answered, and agree so plainly that the last two almost never change +//! what the loops or the evidence gate make of the answers, the decision is +//! merged without waiting for them. +//! +//! A decision waits for its slowest framing. Replayed over 13,669 decisions +//! asked seven ways, from 245 live runs, a quorum of five ended 12% of them, +//! a median 0.14 s before the seventh answer and about 2 s a run, and every +//! one read the same on every threshold, choice floor, and evidence gate as +//! all seven framings did, but for two whose mean moved by under 0.002 across +//! the edge of the evidence band. A quorum of four agreed with all seven only +//! 99.2% of the time: its stragglers dissented. +//! +//! The framings left are not cancelled: each runs to its end, its exchange +//! journaled, so its connection goes back to the pool for the next decision. + +use std::collections::BTreeMap; + +use tinyinference_decisions::{Answer, EvaluationFailure, EvaluationResult}; +use tokio::{sync::mpsc, task::JoinHandle}; + +use super::{decide::PAGE_KIND, vote}; + +/// Framings a decision ends without, at most, once the rest agree. +pub(super) const QUORUM_LEFT: usize = 2; +/// Fewest framings a decision must be asked in to end on a quorum. +pub(super) const QUORUM_VOTES: usize = 7; +/// Least probability every framing gives the option a Choice or Score +/// ranks first, the same option in each. +pub(super) const QUORUM_TOP: f64 = 0.9; +/// A yes/no every framing answers at or above this, or every one at or +/// below [`SURE_NO`], lies the evidence band's 0.12 beyond every yes/no +/// threshold the loops use (0.20 to 0.85). +pub(super) const SURE_YES: f64 = 0.97; +/// See [`SURE_YES`]. +pub(super) const SURE_NO: f64 = 0.08; + +/// One framing's evaluation, as its task ends. +type Evaluated = Result; + +/// A framing's index and how its task ended. +type Finished = (usize, Result); + +/// What one part's framings came back with. +pub(super) struct Gathered { + /// Each framing that answered, with its evaluation, in framing order. + pub(super) answered: Vec<(vote::Framing, EvaluationResult)>, + /// The first failure heard, if any framing failed. + pub(super) failure: Option, + /// Framings a quorum ended the wait without. + pub(super) left: u32, +} + +/// The framings heard from so far. +#[derive(Default)] +struct Heard { + /// Those that answered, by index, in the order they finished. + answered: Vec<(usize, EvaluationResult)>, + failure: Option, + count: usize, +} + +impl Heard { + fn take(&mut self, (index, outcome): Finished) { + self.count += 1; + match outcome { + Ok(Ok(evaluation)) => self.answered.push((index, evaluation)), + Ok(Err(error)) => { + self.failure.get_or_insert(error); + } + Err(_) => {} + } + } +} + +/// How many answers can end a decision asked in `framings` framings: all +/// but [`QUORUM_LEFT`] of them, when it is asked in [`QUORUM_VOTES`] or more; +/// `None` when it waits for every framing. +pub(super) fn size(framings: usize) -> Option { + (framings >= QUORUM_VOTES).then(|| framings - QUORUM_LEFT) +} + +/// Waits for the framings in `handles`, asked as `framings`, in the order +/// they finish: for all of them, or, with a quorum `size`, until that many +/// have answered and settle every question ([`settled`]). An answer already +/// in when the quorum is reached is used all the same; only the waiting is +/// cut short. +pub(super) async fn gather( + framings: Vec, + handles: Vec>, + size: Option, +) -> Gathered { + let count = handles.len(); + let mut finished = in_order_of_finish(handles); + let mut heard = Heard::default(); + while let Some(next) = finished.recv().await { + heard.take(next); + while let Ok(next) = finished.try_recv() { + heard.take(next); + } + let answers = heard + .answered + .iter() + .map(|(index, evaluation)| (*index, &evaluation.response.answers)); + if size + .is_some_and(|size| heard.answered.len() >= size && settled(&framings, answers, size)) + { + break; + } + } + let Heard { + mut answered, + failure, + count: heard, + } = heard; + answered.sort_unstable_by_key(|(index, _)| *index); + let mut framings = framings.into_iter().map(Some).collect::>(); + Gathered { + answered: answered + .into_iter() + .filter_map(|(index, evaluation)| Some((framings.get_mut(index)?.take()?, evaluation))) + .collect(), + failure, + left: u32::try_from(count - heard).unwrap_or(u32::MAX), + } +} + +/// Each of `handles`' outcomes, with its index, as its task ends. Every +/// task is awaited to its end, whether or not anything still listens. +fn in_order_of_finish(handles: Vec>) -> mpsc::UnboundedReceiver { + let (sender, finished) = mpsc::unbounded_channel(); + for (index, handle) in handles.into_iter().enumerate() { + let sender = sender.clone(); + tokio::spawn(async move { + let _heard = sender.send((index, handle.await)); + }); + } + finished +} + +/// Whether the `answered` framings, by their index in `framings`, settle +/// every question the framings asked: each answered by at least `size` of +/// them, every yes/no [sure](sure) and every Choice and Score ranking the +/// same option first at [`QUORUM_TOP`] or more. The page-kind question, +/// which only briefs the next request, needs only the same first option. +pub(super) fn settled<'a>( + framings: &'a [vote::Framing], + answered: impl Iterator)> + Clone, + size: usize, +) -> bool { + let ballots = vote::ballots_at(framings, answered); + framings.first().is_some_and(|framing| { + framing.request.questions.keys().all(|id| { + ballots.get(id).is_some_and(|ballot| { + ballot.len() >= size + && match ballot.first() { + _ if id == PAGE_KIND => same_top(ballot, 0.0), + Some(Answer::Noul(_)) => sure(ballot), + Some(_) => same_top(ballot, QUORUM_TOP), + None => false, + } + }) + }) + }) +} + +/// Whether every answer in `ballot` is a yes/no at or above [`SURE_YES`], +/// or every one at or below [`SURE_NO`]. +fn sure(ballot: &[Answer]) -> bool { + let nouls = ballot + .iter() + .map(|answer| match answer { + Answer::Noul(noul) => Some(noul.noul), + _ => None, + }) + .collect::>>(); + nouls.is_some_and(|nouls| { + nouls.iter().all(|noul| *noul >= SURE_YES) || nouls.iter().all(|noul| *noul <= SURE_NO) + }) +} + +/// Whether every answer in `ballot` ranks the same option first, each with +/// at least `floor`. +fn same_top(ballot: &[Answer], floor: f64) -> bool { + let mut tops = ballot.iter().map(|answer| { + let probabilities = match answer { + Answer::Choice(choice) => &choice.probabilities, + Answer::Score(score) => &score.probabilities, + Answer::Noul(_) => return None, + }; + probabilities + .iter() + .max_by(|left, right| left.1.total_cmp(right.1)) + .filter(|(_, probability)| **probability >= floor) + .map(|(option, _)| option) + }); + let Some(Some(first)) = tops.next() else { + return false; + }; + tops.all(|top| top == Some(first)) +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/vote.rs b/crates/tinycomputer-engine/src/agentic/flow/vote.rs index 069211cd..0d176cea 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/vote.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/vote.rs @@ -169,14 +169,30 @@ fn keys_for(count: usize, index: usize) -> Vec { pub(super) fn ballots( answered: &[(Framing, BTreeMap)], ) -> BTreeMap> { + ballots_of(&answered.iter().map(|(framing, answers)| (framing, answers))) +} + +/// The [`ballots`] of the framings `answered` so far, each named by its +/// index in `framings`, in the order given. +pub(super) fn ballots_at<'a>( + framings: &'a [Framing], + answered: impl Iterator)> + Clone, +) -> BTreeMap> { + ballots_of(&answered.filter_map(|(index, answers)| Some((framings.get(index)?, answers)))) +} + +fn ballots_of<'a, I>(answered: &I) -> BTreeMap> +where + I: Iterator)> + Clone, +{ answered - .iter() + .clone() .flat_map(|(framing, _)| framing.request.questions.keys()) .collect::>() .into_iter() .map(|id| { let answers = answered - .iter() + .clone() .filter_map(|(framing, answers)| { Some(original(framing, id, answers.get(id)?.clone())) }) diff --git a/docs/crates/tinycomputer-engine/flow/voting-and-briefing.md b/docs/crates/tinycomputer-engine/flow/voting-and-briefing.md index 7c8c6ff3..1581a022 100644 --- a/docs/crates/tinycomputer-engine/flow/voting-and-briefing.md +++ b/docs/crates/tinycomputer-engine/flow/voting-and-briefing.md @@ -114,6 +114,21 @@ does not cost more time. When the budget is short of what full voting would need, the run votes with whatever framings still fit rather than failing outright. +A decision waits for its slowest framing only when it must (`quorum.rs`). +Asked 7 ways or more, it is merged without its last 2 framings once those +in settle every question: every Choice and Score ranks the same option +first, each at 0.9 or more (`QUORUM_TOP`); every yes/no is at 0.97 or more +in each framing, or at 0.08 or less in each (`SURE_YES`, `SURE_NO`: the +evidence band beyond every yes/no threshold); and the page kind reads +alike. Replayed over 13,669 decisions asked seven ways in 245 live runs, +such a quorum ended 12% of them, a median 0.14 s sooner (about 2 s a run), +and every one read the same on every threshold and evidence gate as all +seven framings did, but for two whose mean moved by under 0.002 across the +band's edge. A quorum of four would not have: its stragglers dissented in +1% of the decisions it would have ended. The framings left run to their end, so their +connections go back to the pool; they count as calls, and their exchanges +are journaled as they end. + Voting is the mechanism; deliberation, described on its own page, is what decides whether a decision's evidence is strong enough to stop there or needs to widen further. See [Deliberation](deliberation.md). diff --git a/docs/technical/decision-thresholds.md b/docs/technical/decision-thresholds.md index e80b7b51..87b26107 100644 --- a/docs/technical/decision-thresholds.md +++ b/docs/technical/decision-thresholds.md @@ -52,6 +52,9 @@ Change a constant and its row together. | `FRONT_CONTROLS` | 8 | `act/turns.rs` | most controls of the task's dialog in front a failed step's note names, so a rescue answers with one of them | | `STEADY_HOLD` / `STEADY_CHECKS` | 0.65 / 3 | `steps/mod.rs` | belief a `wait_for` condition must keep, on checks in a row of one unchanged screen, to be taken as held under `DONE` | | `HEDGE_AFTER` / `HEDGE_AFTER_LARGE` | 4 s / 5 s | `hedge.rs` | how long a framing runs before a copy of it is sent and the first answer of the two taken; the longer wait is for a request of `HEDGE_LARGE_BYTES` (32 KB) or more. Sage gets no copy | +| `QUORUM_VOTES` / `QUORUM_LEFT` | 7 / 2 | `quorum.rs` | a decision asked at least 7 ways is merged without its last 2 framings once those in settle every question; the 2 still run, and count as calls | +| `QUORUM_TOP` | 0.9 | `quorum.rs` | least probability every framing in gives the option a Choice or Score ranks first, the same option in each, for a quorum; the page kind needs only the same option | +| `SURE_YES` / `SURE_NO` | 0.97 / 0.08 | `quorum.rs` | a yes/no settles for a quorum when every framing in puts it at or above the first, or every one at or below the second: `UNDECIDED_BAND` beyond the highest (0.85) and lowest (0.20) yes/no thresholds | | `HEDGE_COPIES` | 2 | `hedge.rs` | most copies one runtime has in flight: past it, a framing waits for its own answer, so a slow or failing gateway is not sent a copy of every call | | `LATE_LOOKS` | 2 | `steps/suggestion.rs` | looks again, after a wait for the page to change, for the suggestions a place box lists late, until a row names the text typed (rows of the box's own, such as "Allow location access", are waited past), before its text is left as typed; a page that stayed still through a wait lists nothing more | | `LATE_LOOK_MS` | 1000 ms | `steps/suggestion.rs` | longest one of those waits: it ends as soon as the page changes (`Surface::await_change`) | diff --git a/docs/technical/jev-harness.md b/docs/technical/jev-harness.md index 749f1fcf..c0e051d3 100644 --- a/docs/technical/jev-harness.md +++ b/docs/technical/jev-harness.md @@ -155,7 +155,7 @@ round trip's *slowest* framing, plus the action, plus settling. The levers: | Lever | Effect on latency | Effect on accuracy | |---|---|---| | `strategy` | `wide` asks one request per `do` turn instead of two to seven in sequence | the digest, survey, and memory show more of what matters; measure with the lab's `--strategy` | -| `votes` | a decision waits for its slowest framing: more framings, longer tail; a framing slower than 4 s (5 s at 32 KB or more) gets a copy, and the first answer counts (no copy on Sage, and two in flight at most) | more framings average out position and phrasing bias | +| `votes` | a decision waits for its slowest framing, unless all but two of seven or more agree plainly (a quorum, `quorum.rs`): more framings, longer tail; a framing slower than 4 s (5 s at 32 KB or more) gets a copy, and the first answer counts (no copy on Sage, and two in flight at most) | more framings average out position and phrasing bias | | request size | Jev's latency grows with input tokens; a big element list is the usual cause | trimming can drop the element that was needed | | grounding memory | a remembered element is confirmed with one Noul instead of narrowing | none when the hint is right | | `disabled_loops` | each loop off removes a question or a whole decision | measure it before shipping it off | diff --git a/docs/technical/jev-journal.md b/docs/technical/jev-journal.md index ffceaa23..44285380 100644 --- a/docs/technical/jev-journal.md +++ b/docs/technical/jev-journal.md @@ -61,7 +61,7 @@ has `""`, and goal and intent runs carry their goal or intent text. |---|---|---| | `run` | a run begins | `kind` (`flow`, `goal`, `goal-continuation`, `intent`), `label`, `model`, `pid` | | `exchange` | every Jev call, one per framing | `step` (`warm-up` for the calls that open connections while a task is planned), `questions` (ids), `request_bytes`, `request` (the exact `EvaluationRequest`), `ok`, `latency_ms`, `attempts`; on success `request_id`, `model`, `input_tokens`, `output_tokens`, `answers`; on failure `error` | -| `decision` | a flow decision is merged | `step`, `questions`, `framings`, `answered`, `batched` (requests asked in the same round trip), `parts` (requests the decision's questions were split across; 1 unless they outgrew `MAX_REQUEST_BYTES`), `request_bytes` (the largest part), `wall_ms` — what the step actually waited | +| `decision` | a flow decision is merged | `step`, `questions`, `framings`, `answered`, `left` (framings a quorum did not wait for; their `exchange` lines follow when they end), `batched` (requests asked in the same round trip), `parts` (requests the decision's questions were split across; 1 unless they outgrew `MAX_REQUEST_BYTES`), `request_bytes` (the largest part), `wall_ms` — what the step actually waited | | `turn` | a `do` turn ends | `step`, `turn`, `decisions` (made in that turn), `rounds` (round trips they took: a batch is one), `wall_ms` | | `survey` | the wide strategy surveys a crowded screen | `step`, `regions` asked about, `most_relevant` (region ids), `distractions` | | `observe` | a flow reads the screen | `step`, `part` (`screen` or `subtree`), `wall_ms`, `ok`, `candidates`, `unexplored` | @@ -84,9 +84,10 @@ has `""`, and goal and intent runs carry their goal or intent text. | `rescue` | the rescuer answers for a failed step | `step` (from 1), `attempt`, `limit`, `wall_ms`, `calls` and `sent_bytes` (null when it gave no answer in time), `outcome` (`guided`, `gave_up`, `error`, `timeout`), `steps` (guidance steps), `covers`, `model` | | `resume` | a person answers a paused task (`ContinueTask`) | `state` it waited at (`needs_input`, `needs_approval`, `needs_human`), `waited_ms` since it first asked | -A voted decision writes one `exchange` per framing and then one `decision`. -The framings run concurrently, so a decision's `wall_ms` is close to its -slowest framing's `latency_ms`, not their sum. The `turn` events are where +A voted decision writes one `exchange` per framing and then one `decision`; +the framings a quorum did not wait for (`left`) write theirs after it, as +they end. The framings run concurrently, so a decision's `wall_ms` is close +to its slowest awaited framing's `latency_ms`, not their sum. The `turn` events are where the summary's decisions-per-turn come from: a turn waits for its decisions one after another, so that number, not the call count, is what a `do` step's Jev latency scales with. A parent step (`if`, From c1183d7845b3e70a3f988835bbc05c486f3ec8cf Mon Sep 17 00:00:00 2001 From: Shanu Date: Thu, 8 Oct 2026 13:46:59 +0530 Subject: [PATCH 37/44] Count a quorum's left framings in no round of --split The framings a quorum did not wait for journal their exchanges after their decision, so jev_journal --split read them as the next decision's round, and its "slowest call adds" figure took a straggler for that round's slowest. A decision's `left` exchanges that follow it now count in no round; they still count as calls. --- .../src/journal/journal_tests/split_tests.rs | 26 +++++++++++++++++++ .../src/journal/split/mod.rs | 19 +++++++++----- docs/technical/jev-journal.md | 3 ++- 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/crates/tinycomputer-examples/src/journal/journal_tests/split_tests.rs b/crates/tinycomputer-examples/src/journal/journal_tests/split_tests.rs index 1c14bdc8..4a33b02d 100644 --- a/crates/tinycomputer-examples/src/journal/journal_tests/split_tests.rs +++ b/crates/tinycomputer-examples/src/journal/journal_tests/split_tests.rs @@ -186,3 +186,29 @@ fn a_folder_of_runs_reads_as_one_task() { ); let _ = std::fs::remove_dir_all(&scratch); } + +#[test] +fn the_framings_a_quorum_left_count_in_no_round() { + let exchange = |ms: u64, latency: u64| json!({"event": "exchange", "at": at(ms), "latency_ms": latency, "ok": true}); + let decision = |ms: u64, left: u64| json!({"event": "decision", "at": at(ms), "wall_ms": 500, "left": left}); + let events = vec![ + json!({"event": "run", "at": at(0), "kind": "flow"}), + exchange(400, 400), + exchange(500, 500), + decision(500, 0), + exchange(800, 300), + exchange(800, 300), + exchange(800, 300), + decision(800, 2), + // The two the quorum left, ending after their decision. + exchange(1_900, 2_000), + exchange(2_400, 2_500), + exchange(3_000, 600), + exchange(3_100, 700), + decision(3_100, 0), + ]; + let spent = split(&events); + // Rounds of 400/500, 300/300/300 and 600/700: 100, 0 and 100 ms. + assert_eq!(spent.slowest_extra_ms, 66); + assert_eq!(spent.calls, 9, "every call made counts"); +} diff --git a/crates/tinycomputer-examples/src/journal/split/mod.rs b/crates/tinycomputer-examples/src/journal/split/mod.rs index c0ba6400..e841102b 100644 --- a/crates/tinycomputer-examples/src/journal/split/mod.rs +++ b/crates/tinycomputer-examples/src/journal/split/mod.rs @@ -242,18 +242,25 @@ impl Kinds { /// The mean of how much longer each round of calls waited for its slowest /// call than for its median one. A round is the calls journaled since the -/// previous decision: one decision's framings, or a batch's. +/// previous decision: one decision's framings, or a batch's. The framings a +/// quorum did not wait for (its `left`) journal after their decision, and +/// are no round's. fn slowest_extra(events: &[Value]) -> u64 { let mut extras = Vec::new(); let mut round = Vec::new(); + let mut late = 0; for event in events { match event["event"].as_str() { + Some("exchange") if late > 0 => late -= 1, Some("exchange") => round.push(number(event, "latency_ms")), - Some("decision") if !round.is_empty() => { - round.sort_unstable(); - let slowest = round[round.len() - 1]; - extras.push(slowest - round[(round.len() - 1) / 2]); - round.clear(); + Some("decision") => { + if !round.is_empty() { + round.sort_unstable(); + let slowest = round[round.len() - 1]; + extras.push(slowest - round[(round.len() - 1) / 2]); + round.clear(); + } + late = number(event, "left"); } Some("run") => round.clear(), _ => {} diff --git a/docs/technical/jev-journal.md b/docs/technical/jev-journal.md index 44285380..1b2c5c72 100644 --- a/docs/technical/jev-journal.md +++ b/docs/technical/jev-journal.md @@ -146,7 +146,8 @@ with `TINYCOMPUTER_JEV_JOURNAL=$TASK_OUT/journal` writes there: its the wall time into planning, rescues, waits for a person, and flows (and within flows: Jev, settling, acting, reading the screen, the rest), counting each moment once. It also reports per-call and per-decision latency (p50, -p90), how much the slowest call adds to each round of calls, decisions, +p90), how much the slowest call adds to each round of calls (a quorum's +`left` framings count in no round), decisions, calls, tokens, and step, `do` turn, and action latency (p50, p90). One task prints in full; several print one line each with their medians; `--json` prints every split and the median. From 8f8a7ae51ec7f615de26f3634e79b86fb3e9a1bc Mon Sep 17 00:00:00 2001 From: Shanu Date: Thu, 8 Oct 2026 14:07:59 +0530 Subject: [PATCH 38/44] Report Jev's cost in jev_journal --split and --compare openhuman#7000 asks the timing harness to report cost beside wall time, decisions and input tokens. A split now prices Jev's answers the way the repository's evals do: their input tokens at $0.042 per million, output free. It is kept as jev_cost_micro_usd (millionths of a dollar, so the medians stay whole numbers), printed on the tokens line and in the table, and compared. Another model's calls (Sage bills by units) are not priced, nor planning and rescues, whose tokens are not journaled. --- .../src/journal/journal_tests/split_tests.rs | 47 +++++++++++++++++++ .../src/journal/split/mod.rs | 17 +++++++ .../src/journal/split/render.rs | 28 ++++++++--- docs/technical/jev-journal.md | 9 +++- 4 files changed, 92 insertions(+), 9 deletions(-) diff --git a/crates/tinycomputer-examples/src/journal/journal_tests/split_tests.rs b/crates/tinycomputer-examples/src/journal/journal_tests/split_tests.rs index 4a33b02d..f92fe138 100644 --- a/crates/tinycomputer-examples/src/journal/journal_tests/split_tests.rs +++ b/crates/tinycomputer-examples/src/journal/journal_tests/split_tests.rs @@ -67,6 +67,7 @@ fn a_tasks_time_is_split_across_its_runs_counting_each_moment_once() { slowest_extra_ms: 250, input_tokens: 200, output_tokens: 10, + jev_cost_micro_usd: 0, actions: 1, step_p50_ms: 1700, step_p90_ms: 1700, @@ -212,3 +213,49 @@ fn the_framings_a_quorum_left_count_in_no_round() { assert_eq!(spent.slowest_extra_ms, 66); assert_eq!(spent.calls, 9, "every call made counts"); } + +#[test] +fn jevs_input_tokens_are_priced_and_another_models_are_not() { + let exchange = |model: &str, input_tokens: u64| { + json!({ + "event": "exchange", "at": at(1000), "latency_ms": 500, "ok": true, + "model": model, "input_tokens": input_tokens, "output_tokens": 40, + }) + }; + let spent = split(&[ + json!({"event": "run", "at": at(0), "kind": "flow"}), + exchange("typesafe/jev-1.13-20260917", 1_000_000), + exchange("typesafe/jev-1.13-20260917", 300_000), + // Sage bills by units, not at Jev's price. + exchange("levanto-sage", 500_000), + ]); + assert_eq!(spent.input_tokens, 1_800_000); + assert_eq!( + spent.jev_cost_micro_usd, 54_600, + "1.3 M tokens at $0.042 a million" + ); + let shown = render_split(&spent); + assert!( + shown.contains("1800000 in, 120 out; Jev cost $0.0546"), + "{shown}" + ); + let table = render_table(&[("a".to_owned(), spent.clone())]); + assert!( + table.lines().next().unwrap().ends_with("jev cost"), + "{table}" + ); + assert!( + table.lines().nth(1).unwrap().ends_with("$0.0546"), + "{table}" + ); + let halved = Split { + jev_cost_micro_usd: 27_300, + ..spent.clone() + }; + let compared = render_compare(&[spent], &[halved]); + let cost = compared + .lines() + .find(|line| line.starts_with("jev_cost_micro_usd")) + .unwrap(); + assert!(cost.ends_with("-50%"), "{cost}"); +} diff --git a/crates/tinycomputer-examples/src/journal/split/mod.rs b/crates/tinycomputer-examples/src/journal/split/mod.rs index e841102b..8135a8b6 100644 --- a/crates/tinycomputer-examples/src/journal/split/mod.rs +++ b/crates/tinycomputer-examples/src/journal/split/mod.rs @@ -19,6 +19,10 @@ pub use render::{render_compare, render_split, render_table}; pub use time::at_ms; use time::{Spans, length, millis, minus, union}; +/// Jev's price, in millionths of a dollar per million input tokens: $0.042, +/// as the repository's evals count it. Jev's output is free. +const JEV_MICRO_USD_PER_M_INPUT: u64 = 42_000; + /// Where one task's time went, in ms unless named otherwise. #[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(default)] @@ -78,6 +82,11 @@ pub struct Split { pub input_tokens: u64, /// Provider-reported output tokens. pub output_tokens: u64, + /// What Jev's answers cost, in millionths of a dollar: their input + /// tokens at $0.042 per million; Jev's output is free. Another model's + /// calls (Sage bills by units) are not priced, nor planning and rescues, + /// whose tokens are not journaled. + pub jev_cost_micro_usd: u64, /// Actions taken. pub actions: u64, /// A step's wall time, 50th and 90th percentiles. @@ -105,6 +114,7 @@ pub fn split(events: &[Value]) -> Split { (Vec::new(), Vec::new(), Vec::new(), Vec::new(), Vec::new()); let mut start = i64::MAX; let mut end = i64::MIN; + let mut jev_tokens = 0_u64; for event in events { let Some(at) = at_ms(event) else { continue; @@ -147,6 +157,12 @@ pub fn split(events: &[Value]) -> Split { split.failed_calls += u64::from(event["ok"] == Value::Bool(false)); split.input_tokens += number(event, "input_tokens"); split.output_tokens += number(event, "output_tokens"); + if event["model"] + .as_str() + .is_some_and(|model| model.contains("jev")) + { + jev_tokens += number(event, "input_tokens"); + } } "step" => steps.push(number(event, "wall_ms")), "turn" => turns.push(number(event, "wall_ms")), @@ -159,6 +175,7 @@ pub fn split(events: &[Value]) -> Split { _ => span.0, }); } + split.jev_cost_micro_usd = jev_tokens.saturating_mul(JEV_MICRO_USD_PER_M_INPUT) / 1_000_000; if start > end { return split; } diff --git a/crates/tinycomputer-examples/src/journal/split/render.rs b/crates/tinycomputer-examples/src/journal/split/render.rs index 9c77fda6..84b0060e 100644 --- a/crates/tinycomputer-examples/src/journal/split/render.rs +++ b/crates/tinycomputer-examples/src/journal/split/render.rs @@ -74,17 +74,28 @@ pub fn render_split(split: &Split) -> String { ); let _ = writeln!( out, - "tokens {} in, {} out", - split.input_tokens, split.output_tokens + "tokens {} in, {} out; Jev cost {}", + split.input_tokens, + split.output_tokens, + dollars(split.jev_cost_micro_usd) ); out } +/// `micro_usd` millionths of a dollar, to a hundredth of a cent. +fn dollars(micro_usd: u64) -> String { + format!( + "${}.{:04}", + micro_usd / 1_000_000, + micro_usd % 1_000_000 / 100 + ) +} + /// One line per named split, then their medians, for a terminal. #[must_use] pub fn render_table(splits: &[(String, Split)]) -> String { let mut out = format!( - "{:<44} {:>7} {:>7} {:>7} {:>7} {:>7} {:>7} {:>6} {:>6} {:>12} {:>12} {:>7}\n", + "{:<44} {:>7} {:>7} {:>7} {:>7} {:>7} {:>7} {:>6} {:>6} {:>12} {:>12} {:>7} {:>8}\n", "run", "wall", "plan", @@ -96,11 +107,12 @@ pub fn render_table(splits: &[(String, Split)]) -> String { "decs", "call p50/90", "dec p50/90", - "slow+" + "slow+", + "jev cost" ); let row = |name: &str, split: &Split| { format!( - "{:<44} {:>6.1}s {:>6.1}s {:>6.1}s {:>6.1}s {:>6.1}s {:>6.1}s {:>6} {:>6} {:>12} {:>12} {:>5} ms\n", + "{:<44} {:>6.1}s {:>6.1}s {:>6.1}s {:>6.1}s {:>6.1}s {:>6.1}s {:>6} {:>6} {:>12} {:>12} {:>5} ms {:>8}\n", super::super::clip(name, 44), secs(split.wall_ms), secs(split.planning_ms), @@ -112,7 +124,8 @@ pub fn render_table(splits: &[(String, Split)]) -> String { split.decisions, format!("{}/{}", split.call_p50_ms, split.call_p90_ms), format!("{}/{}", split.decision_p50_ms, split.decision_p90_ms), - split.slowest_extra_ms + split.slowest_extra_ms, + dollars(split.jev_cost_micro_usd) ) }; for (name, split) in splits { @@ -159,7 +172,7 @@ pub fn render_compare(before: &[Split], after: &[Split]) -> String { } /// The fields a comparison lists, in order. -const COMPARED: [&str; 20] = [ +const COMPARED: [&str; 21] = [ "wall_ms", "planning_ms", "rescue_ms", @@ -178,6 +191,7 @@ const COMPARED: [&str; 20] = [ "decision_p90_ms", "slowest_extra_ms", "input_tokens", + "jev_cost_micro_usd", "step_p90_ms", "action_p50_ms", ]; diff --git a/docs/technical/jev-journal.md b/docs/technical/jev-journal.md index 1b2c5c72..2bac8879 100644 --- a/docs/technical/jev-journal.md +++ b/docs/technical/jev-journal.md @@ -148,8 +148,13 @@ within flows: Jev, settling, acting, reading the screen, the rest), counting each moment once. It also reports per-call and per-decision latency (p50, p90), how much the slowest call adds to each round of calls (a quorum's `left` framings count in no round), decisions, -calls, tokens, and step, `do` turn, and action latency (p50, p90). One task prints in full; several print one -line each with their medians; `--json` prints every split and the median. +calls, tokens, Jev's cost, and step, `do` turn, and action latency (p50, +p90). Jev's cost is its answers' input tokens at $0.042 per million, the +price the evals count (OpenRouter's; Jev's output is free), as +`jev_cost_micro_usd` in millionths of a dollar; another model's calls (Sage +bills by units) are not priced, nor planning and rescues, whose tokens are +not journaled. One task prints in full; several print one line each with +their medians; `--json` prints every split and the median. `--compare` prints the medians of the tasks before `--vs` against those after it, with the change in percent: two builds, or two settings, run on From 59688af6791d862751519877506079cce31bc52e Mon Sep 17 00:00:00 2001 From: Shanu Date: Thu, 8 Oct 2026 14:24:39 +0530 Subject: [PATCH 39/44] Load the page a browser task names while its plan is drafted A browser-only task's browser already opens while its plan is drafted (browser.prelaunch). Its first step then browses to the site the task names: over 40 live runs that page took 1.9 s to load (p90 6.4 s) and 1.1 s to settle, while the plan before it took 10-30 s. In 56 of 57 live plans the first step browsed exactly the page the task's text named. When the task's text names one web address, written out with https:// or http://, the runner now loads it in the early browser in the same wait (FlowRunner::open_page, BrowserSurface::open_at). Until the page is first read or another address loads, a navigation to the same place (scheme, www. and a trailing slash aside) finds it loaded and loads nothing. A task naming several addresses or none loads none; a page not drawn yet, a page that would not load, or a surface let go meanwhile is loaded as asked, and the session's allowed origins apply as to any navigation. It rides on browser.prelaunch: off, nothing opens early. --- .env.example | 2 +- MODULE.md | 3 +- .../tinycomputer-browser/src/surface/mod.rs | 25 ++++++ .../src/surface/operations.rs | 44 ++++++++--- .../surface/surface_tests/operations_tests.rs | 78 +++++++++++++++++++ .../tinycomputer-browser/src/surface/tabs.rs | 17 ++-- crates/tinycomputer-engine/src/task/drive.rs | 11 ++- crates/tinycomputer-engine/src/task/mod.rs | 10 +++ crates/tinycomputer-engine/src/task/page.rs | 35 +++++++++ .../src/task/task_tests.rs | 11 +++ .../src/task/task_tests/page_tests.rs | 37 +++++++++ .../src/task/task_tests/plan_tests.rs | 57 ++++++++++++++ .../src/task/task_tests/runner_tests.rs | 1 + .../src/bin/task_live/main.rs | 3 +- .../tinycomputer/src/tinybus_module/config.rs | 3 +- .../tinycomputer/src/tinybus_module/runner.rs | 24 ++++++ .../tinybus_module_tests/browser_tests.rs | 2 +- .../tinybus_module_tests/tasks_tests.rs | 43 ++++++++++ docs/crates/tinycomputer-browser/surface.md | 9 +++ .../tinycomputer-examples/live-tasks.md | 2 +- docs/crates/tinycomputer/configuration.md | 2 +- docs/technical/tasks.md | 9 ++- 22 files changed, 405 insertions(+), 23 deletions(-) create mode 100644 crates/tinycomputer-engine/src/task/page.rs create mode 100644 crates/tinycomputer-engine/src/task/task_tests/page_tests.rs diff --git a/.env.example b/.env.example index ea58bee3..c3ccee45 100644 --- a/.env.example +++ b/.env.example @@ -50,7 +50,7 @@ TINYCOMPUTER_LAB_SELF_EMAIL= # How a page settles after an action: prompt (default) or steady. # TINYCOMPUTER_BROWSER_SETTLE=steady # task_live: plan inside StartTask as OpenHuman does; the browser opens -# while it plans unless PRELAUNCH is 0. +# while it plans, at the one address the task names, unless PRELAUNCH is 0. # TASK_PLAN=in-task # TINYCOMPUTER_BROWSER_PRELAUNCH=0 diff --git a/MODULE.md b/MODULE.md index d3da85e6..29e8b271 100644 --- a/MODULE.md +++ b/MODULE.md @@ -65,7 +65,8 @@ is optional: `perception` (`sight`, the default, or `tree`: how a task reads a page), `settle` (`prompt`, the default, or `steady`: how long a page is let settle after an action), and `prelaunch` (a boolean, `true` by default: - open a browser-only task's browser while it is planned). + open a browser-only task's browser while it is planned, at the one web + address the task's text names, if it names one). Configuration is delivered as sensitive host-control traffic and is never shown to monitors. diff --git a/crates/tinycomputer-browser/src/surface/mod.rs b/crates/tinycomputer-browser/src/surface/mod.rs index 306dd2d2..2b0435cb 100644 --- a/crates/tinycomputer-browser/src/surface/mod.rs +++ b/crates/tinycomputer-browser/src/surface/mod.rs @@ -123,6 +123,10 @@ pub struct BrowserSurface { /// Set once the surface is let go ([`BrowserSurface::close`]): it is /// then not opened early again. closed: Arc, + /// The address the session was opened at early + /// ([`BrowserSurface::open_at`]), until the page is first read or + /// another address is loaded: a navigation there finds it loaded. + opened_at: Arc>>, } impl std::fmt::Debug for BrowserSurface { @@ -134,6 +138,7 @@ impl std::fmt::Debug for BrowserSurface { .field("cursor", &self.cursor) .field("perception", &self.perception) .field("settle", &self.settle) + .field("opened_at", &self.opened_at) .finish_non_exhaustive() } } @@ -159,6 +164,7 @@ impl BrowserSurface { settle: Settle::default(), denoised: Arc::new(Mutex::new(Denoised::default())), closed: Arc::new(AtomicBool::new(false)), + opened_at: Arc::default(), } } @@ -199,6 +205,24 @@ impl BrowserSurface { self.session_slot(true).is_ok() } + /// Opens the session early, as [`BrowserSurface::open`] does, and loads + /// `url` in it: the page a task names, loaded while its plan is drafted. + /// Until the page is first read, a navigation to the same place (one + /// page's two addresses: `https://` or not, `www.` or not, a trailing + /// slash or not) finds it loaded and loads nothing. Whether the page + /// loaded; a surface already let go opens nothing. + #[must_use] + pub fn open_at(&self, url: &str) -> bool { + let Ok(id) = self.session_slot(true) else { + return false; + }; + let loaded = self.navigate_in(&id, url).ok; + if loaded && let Ok(mut opened) = self.opened_at.lock() { + *opened = Some(url.to_owned()); + } + loaded + } + /// The session this surface drives, once one is open. #[must_use] pub fn session(&self) -> Option { @@ -222,6 +246,7 @@ impl BrowserSurface { // Before the slot is taken: an early open that has not begun yet // finds it set, and one under way finishes first and is closed. self.closed.store(true, Ordering::Release); + let _opened = self.early_page(); let Some(id) = self .session .lock() diff --git a/crates/tinycomputer-browser/src/surface/operations.rs b/crates/tinycomputer-browser/src/surface/operations.rs index 4659b952..365c69bc 100644 --- a/crates/tinycomputer-browser/src/surface/operations.rs +++ b/crates/tinycomputer-browser/src/surface/operations.rs @@ -14,7 +14,7 @@ use crate::error::Error; use super::envelope::{failure, not_a_text_field, reply}; use super::{BrowserSurface, Perception}; use super::{NETWORK_IDLE_MS, QUIET_MS, READ_TIMEOUT, SETTLE_MS, SKELETON_DEPTH, Settle, tree}; -use super::{sight, watch}; +use super::{sight, tabs::place, watch}; impl Surface for BrowserSurface { fn observe( @@ -23,6 +23,8 @@ impl Surface for BrowserSurface { root: Option<&str>, depth: Depth, ) -> std::result::Result> { + // A page once read is no longer the one opened early. + let _opened = self.early_page(); if self.perception == Perception::Sight && let Some(mut screen) = self.see(root) { @@ -288,13 +290,34 @@ impl Surface for BrowserSurface { } fn navigate(&self, url: &str) -> DesktopResponse { + // Loaded there while the plan was drafted, and not read since. + if let Some(opened) = self.early_page() + && place(&opened) == place(url) + && let Some(id) = self.session() + && let Some((shown, title)) = self.shown_page(&id) + { + return reply("navigate", Ok(json!({"url": shown, "title": title}))); + } + match self.ensure_session() { + Ok(id) => self.navigate_in(&id, url), + Err(error) => reply("navigate", Err(error)), + } + } + + fn back(&self, _app: &str) -> DesktopResponse { + self.perform("back", Action::Back) + } +} + +impl BrowserSurface { + /// Loads `url` in session `id`. A heavy page can be read long before its + /// `load` event fires. + pub(super) fn navigate_in(&self, id: &SessionId, url: &str) -> DesktopResponse { let page = self - .ensure_session() - .and_then(|id| self.block(self.browser.navigate(&id, NavigateRequest::new(url)))) + .block(self.browser.navigate(id, NavigateRequest::new(url))) .map(|page| (page.url, page.title)); - // A heavy page can be read long before its `load` event fires. let page = match page { - Err(error @ Error::Timeout { .. }) => self.drawn_page(url).ok_or(error), + Err(error @ Error::Timeout { .. }) => self.drawn_page(id, url).ok_or(error), other => other, }; reply( @@ -303,12 +326,15 @@ impl Surface for BrowserSurface { ) } - fn back(&self, _app: &str) -> DesktopResponse { - self.perform("back", Action::Back) + /// Takes the address the session was opened at early: the first read + /// or navigation leaves that page as it was opened. + pub(super) fn early_page(&self) -> Option { + self.opened_at + .lock() + .ok() + .and_then(|mut opened| opened.take()) } -} -impl BrowserSurface { /// Waits until the page stops changing (`watch::still_script`), within /// a deadline: a call sent while a page is being replaced can wait out /// the browser's own 30 s. diff --git a/crates/tinycomputer-browser/src/surface/surface_tests/operations_tests.rs b/crates/tinycomputer-browser/src/surface/surface_tests/operations_tests.rs index 5af9cc10..9b7fb733 100644 --- a/crates/tinycomputer-browser/src/surface/surface_tests/operations_tests.rs +++ b/crates/tinycomputer-browser/src/surface/surface_tests/operations_tests.rs @@ -551,6 +551,84 @@ fn a_navigation_that_timed_out_on_a_drawn_page_is_taken_as_open() { assert!(!reply.ok, "nothing drawn yet: the timeout stands"); } +#[test] +fn a_page_opened_early_is_not_loaded_again_until_it_is_read() { + // A shop whose page has drawn by the time it is asked. + let shop = || { + Fake::scripted(|command| { + let script = command["script"].as_str().unwrap_or_default(); + (command["action"] == "evaluate" && script.contains("readyState")).then(|| { + ok(&json!({"result": { + "url": "https://www.shop.test/", + "title": "Shop", + "drawn": true, + }})) + }) + }) + }; + let loads = |fake: &Fake| { + fake.actions() + .iter() + .filter(|action| *action == "navigate") + .count() + }; + + let early = harness("open-at", shop()); + assert!(early.surface.open_at("https://shop.test")); + assert_eq!(loads(&early.fake), 1); + // The plan's first step browses there, by another of its addresses. + let reply = early.surface.navigate("https://www.shop.test/"); + assert!(reply.ok, "{:?}", reply.error); + assert_eq!(reply.data.unwrap()["title"], "Shop"); + assert_eq!(loads(&early.fake), 1, "loaded once"); + // Asked again, it loads again. + assert!(early.surface.navigate("https://shop.test").ok); + assert_eq!(loads(&early.fake), 2); + + // Once the page is read, or another page is asked for, it loads. + let read = harness("open-at-read", shop()); + assert!(read.surface.open_at("https://shop.test")); + assert!(read.surface.observe("browser", None, Depth::Full).is_ok()); + assert!(read.surface.navigate("https://shop.test").ok); + assert_eq!(loads(&read.fake), 2); + let elsewhere = harness("open-at-elsewhere", shop()); + assert!(elsewhere.surface.open_at("https://shop.test")); + assert!(elsewhere.surface.navigate("https://shop.test/cart").ok); + assert_eq!(loads(&elsewhere.fake), 2); + + // A page that has drawn nothing yet is loaded as asked. + let blank = harness("open-at-blank", Fake::new()); + assert!(blank.surface.open_at("https://shop.test")); + assert!(blank.surface.navigate("https://shop.test").ok); + assert_eq!(loads(&blank.fake), 2); + + // A page that would not load is not kept, and a surface let go opens + // none. + let refused = harness( + "open-at-refused", + Fake::scripted(|command| { + (command["action"] == "navigate") + .then(|| failure("Domain 'shop.test' is not in the allowed domains list")) + }), + ); + assert!(!refused.surface.open_at("https://shop.test")); + let closed = harness("open-at-closed", shop()); + closed.surface.close(); + assert!(!closed.surface.open_at("https://shop.test")); + assert_eq!( + closed.fake.actions().len(), + 0, + "{:?}", + closed.fake.actions() + ); + // Let go after it opened early, it keeps no page for a later session. + let let_go = harness("open-at-let-go", shop()); + assert!(let_go.surface.open_at("https://shop.test")); + let_go.surface.close(); + assert!(let_go.surface.navigate("https://shop.test").ok); + assert_eq!(loads(&let_go.fake), 2); +} + #[test] fn a_prompt_settle_counts_quiet_from_the_start_and_waits_only_while_the_page_changes() { // Prompt is how a surface settles unless told otherwise. diff --git a/crates/tinycomputer-browser/src/surface/tabs.rs b/crates/tinycomputer-browser/src/surface/tabs.rs index 820a3cfa..7adf20e7 100644 --- a/crates/tinycomputer-browser/src/surface/tabs.rs +++ b/crates/tinycomputer-browser/src/surface/tabs.rs @@ -2,6 +2,7 @@ //! page that drew before it finished loading as open. use serde_json::{Value, json}; +use tinycomputer_bus::browser::SessionId; use super::{BrowserSurface, sight}; @@ -80,22 +81,28 @@ const SAME_TAB_JS: &str = r"(element => { })"; impl BrowserSurface { - /// The page's address and title when the session shows `url` drawn + /// The page's address and title when session `id` shows `url` drawn /// with words, though its navigation timed out waiting for `load`: a /// heavy page keeps fetching long after it can be read (live, a store's /// results page). `None` when it shows another page, or nothing yet. - pub(super) fn drawn_page(&self, url: &str) -> Option<(String, String)> { - let id = self.ensure_session().ok()?; + pub(super) fn drawn_page(&self, id: &SessionId, url: &str) -> Option<(String, String)> { + self.shown_page(id) + .filter(|(shown, _)| place(shown) == place(url)) + } + + /// The address and title of the page session `id` shows, once it has + /// drawn words; `None` while it shows nothing yet. + pub(super) fn shown_page(&self, id: &SessionId) -> Option<(String, String)> { let data = self .block( self.browser - .command(&id, json!({"action": "evaluate", "script": DRAWN_JS})), + .command(id, json!({"action": "evaluate", "script": DRAWN_JS})), ) .ok()?; let page = data.get("result")?; let shown = page.get("url").and_then(Value::as_str)?; let drawn = page.get("drawn").and_then(Value::as_bool).unwrap_or(false); - (drawn && place(shown) == place(url)).then(|| { + drawn.then(|| { ( shown.to_owned(), page.get("title") diff --git a/crates/tinycomputer-engine/src/task/drive.rs b/crates/tinycomputer-engine/src/task/drive.rs index 47e05011..377ab53b 100644 --- a/crates/tinycomputer-engine/src/task/drive.rs +++ b/crates/tinycomputer-engine/src/task/drive.rs @@ -59,8 +59,17 @@ pub(super) async fn plan_then_drive( }; // A browser-only task's browser opens while the plan is drafted, so the // first step need not wait for it: whatever the plan says, it runs there. + // The one page the task names loads in it meanwhile, for a first step + // that browses there. let ((outcome, used), drafting) = if constraints.surfaces == [SurfaceKind::Browser] { - tokio::join!(planning, runner.prepare(&id, &constraints)).0 + let page = super::page::named_page(&task); + let preparing = async { + runner.prepare(&id, &constraints).await; + if let Some(page) = &page { + runner.open_page(&id, page).await; + } + }; + tokio::join!(planning, preparing).0 } else { planning.await }; diff --git a/crates/tinycomputer-engine/src/task/mod.rs b/crates/tinycomputer-engine/src/task/mod.rs index 143a6a46..368f1b96 100644 --- a/crates/tinycomputer-engine/src/task/mod.rs +++ b/crates/tinycomputer-engine/src/task/mod.rs @@ -46,6 +46,7 @@ mod errors; mod human; mod interpret; mod names; +mod page; mod publish; mod recovery; mod resume; @@ -122,6 +123,15 @@ pub trait FlowRunner: Send + Sync + 'static { Box::pin(async {}) } + /// Loads `url`, the one web page the task's text names, in the browser + /// [`FlowRunner::prepare`] got ready, while the plan is drafted: a first + /// step that browses there finds it loaded. Called after `prepare`, in + /// the same wait, for a task that runs on the browser alone. Does + /// nothing by default. + fn open_page(&self, _task: &TaskId, _url: &str) -> PrepareFuture { + Box::pin(async {}) + } + /// Opens the connections the task's first decision, asked `votes` ways, /// will use, while its plan is drafted (`JevRuntime::warm`). Started /// with the planner for every task and never waited for: the task goes diff --git a/crates/tinycomputer-engine/src/task/page.rs b/crates/tinycomputer-engine/src/task/page.rs new file mode 100644 index 00000000..0950c85a --- /dev/null +++ b/crates/tinycomputer-engine/src/task/page.rs @@ -0,0 +1,35 @@ +//! The one web page a task's own words name, which its browser can load +//! while the plan is drafted ([`FlowRunner::open_page`]). +//! +//! [`FlowRunner::open_page`]: super::FlowRunner::open_page + +/// Punctuation that can follow an address in a sentence, and is no part of +/// it. +const TRAILING: &[char] = &['.', ',', ';', ':', '!', '?', ')', ']', '}', '\'', '"']; + +/// The one web address `task` names, written out with its scheme +/// (`https://…` or `http://…`) and without the punctuation after it; `None` +/// when it names none, or several, which leave no one page to start on. An +/// address written twice, with a trailing slash or without, is one. +pub(super) fn named_page(task: &str) -> Option { + let mut named: Vec<&str> = Vec::new(); + for word in task.split_whitespace() { + let Some(start) = word.find("https://").or_else(|| word.find("http://")) else { + continue; + }; + let address = word[start..].trim_end_matches(TRAILING); + let has_host = address + .split_once("://") + .is_some_and(|(_, rest)| !rest.trim_start_matches('/').is_empty()); + let seen = named + .iter() + .any(|seen| seen.trim_end_matches('/') == address.trim_end_matches('/')); + if has_host && !seen { + named.push(address); + } + } + match named.as_slice() { + [page] => Some((*page).to_owned()), + _ => None, + } +} diff --git a/crates/tinycomputer-engine/src/task/task_tests.rs b/crates/tinycomputer-engine/src/task/task_tests.rs index dbf424b6..52628de1 100644 --- a/crates/tinycomputer-engine/src/task/task_tests.rs +++ b/crates/tinycomputer-engine/src/task/task_tests.rs @@ -13,6 +13,7 @@ mod describe_tests; mod errors_tests; mod human_tests; mod output_tests; +mod page_tests; mod plan_tests; mod rescue_tests; mod runner_tests; @@ -60,6 +61,8 @@ struct Script { /// Tasks whose Jev connections were warmed while they were planned, /// with the votes asked for. The warm-up never ends. warmed: Mutex>, + /// The pages loaded while tasks were planned, with their task. + opened: Mutex>, } impl FlowRunner for Script { @@ -103,6 +106,14 @@ impl FlowRunner for Script { Box::pin(async {}) } + fn open_page(&self, task: &TaskId, url: &str) -> super::PrepareFuture { + self.opened + .lock() + .unwrap() + .push((task.clone(), url.to_owned())); + Box::pin(async {}) + } + fn warm(&self, task: &TaskId, votes: u32) -> super::PrepareFuture { self.warmed.lock().unwrap().push((task.clone(), votes)); Box::pin(std::future::pending()) diff --git a/crates/tinycomputer-engine/src/task/task_tests/page_tests.rs b/crates/tinycomputer-engine/src/task/task_tests/page_tests.rs new file mode 100644 index 00000000..cc11ca2b --- /dev/null +++ b/crates/tinycomputer-engine/src/task/task_tests/page_tests.rs @@ -0,0 +1,37 @@ +//! Tests for which web page a task's own words name. + +use super::super::page::named_page; + +#[test] +fn a_task_names_the_one_address_it_writes_out() { + assert_eq!( + named_page("Go to https://blinkit.com. Search for Maggi.").as_deref(), + Some("https://blinkit.com") + ); + // Kept as written, a path and query too, without what closes the sentence. + assert_eq!( + named_page("Book a cab on https://www.uber.com/global/en/price-estimate/, then stop.") + .as_deref(), + Some("https://www.uber.com/global/en/price-estimate/") + ); + assert_eq!( + named_page("Open [the shop](http://shop.test/?q=boots)").as_deref(), + Some("http://shop.test/?q=boots") + ); + // Written twice, with a trailing slash or without, it is one page. + assert_eq!( + named_page("Open https://mail.test and stay on https://mail.test/").as_deref(), + Some("https://mail.test") + ); +} + +#[test] +fn a_task_naming_no_page_or_several_names_none() { + assert_eq!(named_page("Order Maggi on Blinkit"), None); + assert_eq!( + named_page("Compare https://a.test with https://b.test"), + None + ); + assert_eq!(named_page("Type https:// into the box"), None); + assert_eq!(named_page(""), None); +} diff --git a/crates/tinycomputer-engine/src/task/task_tests/plan_tests.rs b/crates/tinycomputer-engine/src/task/task_tests/plan_tests.rs index 2c72ea93..9d68a7c1 100644 --- a/crates/tinycomputer-engine/src/task/task_tests/plan_tests.rs +++ b/crates/tinycomputer-engine/src/task/task_tests/plan_tests.rs @@ -300,3 +300,60 @@ async fn a_task_warms_jev_while_it_is_planned_and_never_waits_for_it() { settle(&tasks, &started.id).await; assert!(script.warmed.lock().unwrap().is_empty()); } + +#[tokio::test] +async fn a_browser_only_task_loads_the_page_it_names_while_it_is_planned() { + use tinycomputer_bus::agent::{SurfaceKind, TaskConstraints}; + + let mail = r#"{"app": "browser", "steps": [{"browse": "https://mail.test"}]}"#; + let start = |tasks: &Tasks, task: &str, surfaces: Vec| { + tasks + .start(&StartTaskRequest { + task: Some(task.to_owned()), + constraints: TaskConstraints { + surfaces, + ..TaskConstraints::default() + }, + ..StartTaskRequest::default() + }) + .data + .unwrap() + }; + let (tasks, script) = planned( + vec![finished_run(FlowStopReason::Completed, vec![], &[], None)], + Ok(mail), + ); + let started = start( + &tasks, + "Go to https://mail.test and open my inbox.", + vec![SurfaceKind::Browser], + ); + settle(&tasks, &started.id).await; + assert_eq!( + *script.opened.lock().unwrap(), + [(started.id.clone(), "https://mail.test".to_owned())] + ); + assert_eq!( + *script.prepared.lock().unwrap(), + std::slice::from_ref(&started.id), + "after its browser" + ); + + // Two pages leave no one to start on; a task that may use the desktop + // gets no browser early, nor a page. + for (task, surfaces) in [ + ( + "Compare https://a.test with https://b.test", + vec![SurfaceKind::Browser], + ), + ("Go to https://mail.test and open my inbox.", Vec::new()), + ] { + let (tasks, script) = planned( + vec![finished_run(FlowStopReason::Completed, vec![], &[], None)], + Ok(mail), + ); + let started = start(&tasks, task, surfaces); + settle(&tasks, &started.id).await; + assert_eq!(script.opened.lock().unwrap().len(), 0, "{task}"); + } +} diff --git a/crates/tinycomputer-engine/src/task/task_tests/runner_tests.rs b/crates/tinycomputer-engine/src/task/task_tests/runner_tests.rs index e62bdb5c..21a309d9 100644 --- a/crates/tinycomputer-engine/src/task/task_tests/runner_tests.rs +++ b/crates/tinycomputer-engine/src/task/task_tests/runner_tests.rs @@ -28,6 +28,7 @@ async fn a_runner_that_only_runs_flows_reads_nothing_and_holds_nothing() { RunsOnly.release(&task); // Nothing to get ready, and no journal to write to. RunsOnly.prepare(&task, &TaskConstraints::default()).await; + RunsOnly.open_page(&task, "https://mail.test").await; RunsOnly.warm(&task, 7).await; RunsOnly.journal(Some(&task), "plan", &|| serde_json::json!({"wall_ms": 1})); RunsOnly.journal(None, "plan", &|| serde_json::json!({})); diff --git a/crates/tinycomputer-examples/src/bin/task_live/main.rs b/crates/tinycomputer-examples/src/bin/task_live/main.rs index 0411db7b..4dd05f6a 100644 --- a/crates/tinycomputer-examples/src/bin/task_live/main.rs +++ b/crates/tinycomputer-examples/src/bin/task_live/main.rs @@ -27,7 +27,8 @@ //! plan is printed and saved once the task stops. //! - `TINYCOMPUTER_BROWSER_PRELAUNCH` — optional: `0` opens a browser-only //! task's browser at its first step rather than while the task plans -//! itself (with `TASK_PLAN=in-task`), as it does by default. +//! itself (with `TASK_PLAN=in-task`), at the address the task names, as +//! it does by default. //! - `TASK_OUT` — optional: where the plan, report, and final screenshot go //! (default `target/task-live`). //! - `OUTPUT_FILE` — optional: a JSON `TaskOutput` (`instructions` and a diff --git a/crates/tinycomputer/src/tinybus_module/config.rs b/crates/tinycomputer/src/tinybus_module/config.rs index d1570db2..b1ae35ae 100644 --- a/crates/tinycomputer/src/tinybus_module/config.rs +++ b/crates/tinycomputer/src/tinybus_module/config.rs @@ -26,7 +26,8 @@ pub(crate) struct BrowserDefaults { /// default) or `steady`. pub(crate) settle: Settle, /// Whether a browser-only task's browser opens while its plan is - /// drafted (the default), rather than at its first step. + /// drafted (the default), at the one web address the task's text names + /// if it names one, rather than at its first step. pub(crate) prelaunch: bool, } diff --git a/crates/tinycomputer/src/tinybus_module/runner.rs b/crates/tinycomputer/src/tinybus_module/runner.rs index c2dc9a5f..a3089cdf 100644 --- a/crates/tinycomputer/src/tinybus_module/runner.rs +++ b/crates/tinycomputer/src/tinybus_module/runner.rs @@ -159,6 +159,30 @@ impl FlowRunner for WorkspaceRunner { }) } + fn open_page(&self, task: &TaskId, url: &str) -> PrepareFuture { + // In the browser `prepare` opened, which it does unless prelaunch is + // off. + let browser = self + .defaults + .prelaunch + .then(|| { + self.workspaces.lock().ok().and_then(|workspaces| { + workspaces + .get(task) + .and_then(|(_, browser)| browser.clone()) + }) + }) + .flatten(); + let url = url.to_owned(); + Box::pin(async move { + if let Some(browser) = browser { + // A page that will not load fails again, and is reported, + // at the step that browses there. + let _loaded = tokio::task::spawn_blocking(move || browser.open_at(&url)).await; + } + }) + } + fn warm(&self, task: &TaskId, votes: u32) -> PrepareFuture { let Some(runtime) = self.jev.as_ref() else { return Box::pin(async {}); diff --git a/crates/tinycomputer/src/tinybus_module/tinybus_module_tests/browser_tests.rs b/crates/tinycomputer/src/tinybus_module/tinybus_module_tests/browser_tests.rs index 6f11ea7e..f489ac58 100644 --- a/crates/tinycomputer/src/tinybus_module/tinybus_module_tests/browser_tests.rs +++ b/crates/tinycomputer/src/tinybus_module/tinybus_module_tests/browser_tests.rs @@ -57,7 +57,7 @@ fn answer(command: &Value) -> Value { } #[derive(Debug, Default)] -struct ScriptedLauncher(Arc>>); +pub(super) struct ScriptedLauncher(pub(super) Arc>>); impl Launcher for ScriptedLauncher { fn open(&self, _session: &str) -> Box { diff --git a/crates/tinycomputer/src/tinybus_module/tinybus_module_tests/tasks_tests.rs b/crates/tinycomputer/src/tinybus_module/tinybus_module_tests/tasks_tests.rs index 28487c0d..05f1ca87 100644 --- a/crates/tinycomputer/src/tinybus_module/tinybus_module_tests/tasks_tests.rs +++ b/crates/tinycomputer/src/tinybus_module/tinybus_module_tests/tasks_tests.rs @@ -270,3 +270,46 @@ async fn the_runner_warms_its_jev_runtime_for_a_task() { .warm(&TaskId::new("t-1"), 7) .await; } + +#[tokio::test(flavor = "multi_thread")] +async fn the_runner_loads_the_page_a_browser_task_names_in_its_early_browser() { + use tinycomputer_bus::agent::{SurfaceKind, TaskConstraints, TaskId}; + use tinycomputer_engine::FlowRunner; + + let scratch = + std::env::temp_dir().join(format!("tinycomputer-runner-page-{}", std::process::id())); + let sent = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let browser = std::sync::Arc::new(tinycomputer_browser::Browser::with_scratch( + std::sync::Arc::new(super::browser_tests::ScriptedLauncher(sent.clone())), + scratch.clone(), + )); + let mut runner = + crate::tinybus_module::runner::WorkspaceRunner::new(crate::Desktop::new(), None, browser); + let navigated = |sent: &std::sync::Arc>>| { + sent.lock() + .unwrap() + .iter() + .filter(|command| command["action"] == "navigate") + .map(|command| command["url"].clone()) + .collect::>() + }; + let browser_only = TaskConstraints { + surfaces: vec![SurfaceKind::Browser], + ..TaskConstraints::default() + }; + let task = TaskId::new("t-1"); + runner.prepare(&task, &browser_only).await; + runner.open_page(&task, "https://example.com").await; + assert_eq!(navigated(&sent), [json!("https://example.com")]); + + // A task whose browser was never made ready loads nothing, nor does one + // once prelaunch is off. + runner + .open_page(&TaskId::new("t-2"), "https://example.com") + .await; + runner.defaults.prelaunch = false; + runner.open_page(&task, "https://example.com").await; + assert_eq!(navigated(&sent).len(), 1); + runner.release(&task); + let _ = std::fs::remove_dir_all(&scratch); +} diff --git a/docs/crates/tinycomputer-browser/surface.md b/docs/crates/tinycomputer-browser/surface.md index 5600861e..ed24ab8b 100644 --- a/docs/crates/tinycomputer-browser/surface.md +++ b/docs/crates/tinycomputer-browser/surface.md @@ -135,6 +135,15 @@ go idle, then pauses a further beat regardless). `Surface::settle_briefly`, after a launch or Escape, skips the network wait under `Settle::Prompt`; both are described in [interacting.md](interacting.md#scrolling-and-waiting). +`BrowserSurface::open_at(url)` opens the session early, as `open` does, and +loads `url` in it: the page a task names, loaded while its plan is drafted. +Until the page is first read (`observe`) or another address is loaded, a +`navigate` to the same place finds it already there and loads nothing, +answering with the page's address and title as they stand; "the same place" +is `tabs::place`'s, which ignores the scheme, a leading `www.`, the fragment +and a trailing slash. A page not yet drawn, or a surface let go meanwhile, +is loaded as asked. + ## Cross-links - [sight.md](sight.md): how `observe` actually reads the page. diff --git a/docs/crates/tinycomputer-examples/live-tasks.md b/docs/crates/tinycomputer-examples/live-tasks.md index e61e174b..18d1a1b9 100644 --- a/docs/crates/tinycomputer-examples/live-tasks.md +++ b/docs/crates/tinycomputer-examples/live-tasks.md @@ -132,7 +132,7 @@ they control. | `TINYCOMPUTER_BROWSER_USER_AGENT` | the user agent it announces | | `TINYCOMPUTER_BROWSER_ARGS` | space-separated extra launch arguments | | `TINYCOMPUTER_BROWSER_PERCEPTION` | `sight` (default) or `tree`: how pages are read | -| `TINYCOMPUTER_BROWSER_PRELAUNCH` | `0` opens the browser at the first step rather than while the task plans itself (with `TASK_PLAN=in-task`), as it does by default; `1` asks for the default (the module's `browser.prelaunch`); any other value is refused | +| `TINYCOMPUTER_BROWSER_PRELAUNCH` | `0` opens the browser at the first step rather than while the task plans itself (with `TASK_PLAN=in-task`), at the one address the task names, as it does by default; `1` asks for the default (the module's `browser.prelaunch`); any other value is refused | | `TINYCOMPUTER_BROWSER_SETTLE` | `prompt` (default) or `steady`: how long a page is let settle after an action (see the module's `browser.settle`) | | `TINYCOMPUTER_BROWSER_ENDPOINT` | attach to a running Chrome (e.g. `http://127.0.0.1:9222`) instead of launching one | | `TASK_HEADED` | `1` shows the browser the task launches instead of running it headless; a headed run needs a display, so it runs on the host | diff --git a/docs/crates/tinycomputer/configuration.md b/docs/crates/tinycomputer/configuration.md index 9054de0c..2032b321 100644 --- a/docs/crates/tinycomputer/configuration.md +++ b/docs/crates/tinycomputer/configuration.md @@ -237,7 +237,7 @@ How the module launches every browser it opens — each task's, and each | `args` | extra launch arguments, as an array of strings | | `perception` | how a task reads a page: `sight` (the default) reads the rendered page as a person sees it, `tree` the accessibility tree alone ([`browser-sight.md`](../../technical/specs/browser-sight.md)) | | `settle` | how a task lets a page settle after an action before reading it again: `prompt` (the default) counts the network's 500 ms of quiet from the start, counting only requests that can change the page and for at most 1 s, and then waits only while the page is still changing (at most 400 ms), so an idle page is read again after about 0.6 s; `steady` waits for the network to go idle, at least about 1.1 s, then 400 ms more | -| `prelaunch` | `true` (the default) opens a browser-only task's browser while `StartTask` plans it (a `task` with no `flow`), so the first step does not wait for the launch; `false` opens it at the first step | +| `prelaunch` | `true` (the default) opens a browser-only task's browser while `StartTask` plans it (a `task` with no `flow`), so the first step does not wait for the launch, and loads in it the one web address the task's text names, if it names one, so a first step that browses there does not wait for the page either; `false` opens it at the first step | Most installs never need any of this: leave `browser` out entirely and the linked `agent-browser` engine looks for Chrome itself. An unknown key under diff --git a/docs/technical/tasks.md b/docs/technical/tasks.md index 82e7c4a4..9e97b7da 100644 --- a/docs/technical/tasks.md +++ b/docs/technical/tasks.md @@ -278,7 +278,14 @@ A `StartTask` with a `task` and no `flow` plans inside the task. When the task may run only on the browser, its runner is asked to get the browser ready meanwhile (`FlowRunner::prepare`); unless the module's `browser.prelaunch` is off, the session opens while the plan is drafted, so -the first step does not wait for Chrome to start. A plan that asks for +the first step does not wait for Chrome to start. When the task's text +names one web address, written out with `https://` or `http://`, the page +loads in it meanwhile (`FlowRunner::open_page`), in the same wait: a first +step that browses there finds it loaded, and loads nothing, as long as +nothing has read the page first. Over 57 live plans, 56 started by browsing +the page their task named, and that first page took 1.9 s to load (p90 +6.4 s) and 1.1 s to settle; a plan takes 10–30 s. A task naming several +addresses, or none, loads none. A plan that asks for values lets the browser go while the task waits (`needs_input`), and the run that follows opens it again; a task cancelled while its browser was still opening is left holding none. From 3713dffefb399216411af463d314a398cc7080ba Mon Sep 17 00:00:00 2001 From: Shanu Date: Thu, 8 Oct 2026 17:04:01 +0530 Subject: [PATCH 40/44] Widen a decision a quorum ended past every framing it asked Widening asked framings from the ballot's length on, but a decision ended on a quorum holds only the answers it waited for. After one asked seven ways, a widening sent the sixth and seventh framings again, which had been asked and left, put their answers in the ballot twice, and never heard the eighth and ninth; with nine votes it asked again where it would have asked nothing. FlowRun now keeps how many framings each question was asked in, set by each decision and raised by each widening, and widens from there. A press nothing undoes is vouched for with a widening every time. SURE_YES and SURE_NO keep each yes/no beyond every threshold one answer is read against, not beyond the band of a belief that pairs two answers (a yes/no with its negation, or a coverage): such a belief can still be widened, as it would be after all the framings. The docs now say so and give the quorum of four's figure from the same replay as the rest (0.8%), and JevMetrics says which of its counts take in the framings a quorum did not wait for. --- .../src/agentic/types/result.rs | 12 ++-- .../src/agentic/flow/decide.rs | 7 ++- .../src/agentic/flow/escalate/belief.rs | 6 +- .../src/agentic/flow/escalate/mod.rs | 10 ++++ .../agentic/flow/flow_tests/quorum_tests.rs | 58 +++++++++++++++++++ .../src/agentic/flow/mod.rs | 5 ++ .../src/agentic/flow/quorum.rs | 6 +- .../src/agentic/flow/run.rs | 1 + .../flow/voting-and-briefing.md | 21 ++++--- docs/technical/decision-thresholds.md | 2 +- 10 files changed, 110 insertions(+), 18 deletions(-) diff --git a/crates/tinycomputer-bus/src/agentic/types/result.rs b/crates/tinycomputer-bus/src/agentic/types/result.rs index 3dc1f3f2..3ac6bfe3 100644 --- a/crates/tinycomputer-bus/src/agentic/types/result.rs +++ b/crates/tinycomputer-bus/src/agentic/types/result.rs @@ -165,13 +165,17 @@ pub enum JevStopReason { /// Aggregate provider measurements for one result. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct JevMetrics { - /// Jev evaluations performed. + /// Jev evaluations performed. A decision ended on a quorum counts the + /// framings it did not wait for too: they still run, and are charged. pub calls: u32, - /// HTTP attempts including retries. + /// HTTP attempts including retries, of the evaluations waited for. pub attempts: u32, - /// Total provider latency in milliseconds. + /// Total provider latency in milliseconds, of the evaluations waited + /// for. pub latency_ms: u64, - /// Provider-reported input tokens. + /// Provider-reported input tokens, of the evaluations waited for. Those + /// of framings a quorum did not wait for end after their decision, and + /// only the journal records them. pub input_tokens: u64, /// Provider-reported output tokens. pub output_tokens: u64, diff --git a/crates/tinycomputer-engine/src/agentic/flow/decide.rs b/crates/tinycomputer-engine/src/agentic/flow/decide.rs index 0a55252f..ac40955d 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/decide.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/decide.rs @@ -76,8 +76,12 @@ impl FlowRun<'_, B> { // without an answer, which fails the decision as a whole. let mut unanswered = false; let mut left = 0_u32; - for (framings, handles) in framings.into_iter().zip(handles) { + let mut asked_in = BTreeMap::new(); + for ((part, framings), handles) in parts.iter().zip(framings).zip(handles) { let before = answered.len(); + for id in part.questions.keys() { + asked_in.insert(id.clone(), framings.len()); + } let size = quorum::size(framings.len()); let gathered = quorum::gather(framings, handles, size).await; for (framing, evaluation) in gathered.answered { @@ -105,6 +109,7 @@ impl FlowRun<'_, B> { let ballots = vote::ballots(&answered); let merged = vote::tally(&ballots); self.ballots.extend(ballots); + self.asked.extend(asked_in); merged } }; diff --git a/crates/tinycomputer-engine/src/agentic/flow/escalate/belief.rs b/crates/tinycomputer-engine/src/agentic/flow/escalate/belief.rs index 145fc66f..26e9499b 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/escalate/belief.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/escalate/belief.rs @@ -29,7 +29,7 @@ impl FlowRun<'_, B> { let asked = request .questions .keys() - .map(|id| self.ballot(id).len()) + .map(|id| self.asked_in(id)) .max() .unwrap_or_default(); let from = u32::try_from(asked).unwrap_or(u32::MAX); @@ -68,6 +68,10 @@ impl FlowRun<'_, B> { for (id, ballot) in fresh.clone() { self.ballots.entry(id).or_default().extend(ballot); } + let widened_to = usize::try_from(to).unwrap_or(usize::MAX); + for id in parts.iter().flat_map(|part| part.questions.keys()) { + self.asked.insert(id.clone(), widened_to); + } self.runtime.journal.record("decision", || { json!({ "step": self.step, diff --git a/crates/tinycomputer-engine/src/agentic/flow/escalate/mod.rs b/crates/tinycomputer-engine/src/agentic/flow/escalate/mod.rs index 6ce83fdd..912ba494 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/escalate/mod.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/escalate/mod.rs @@ -120,6 +120,16 @@ impl FlowRun<'_, B> { self.ballots.get(id).map_or(&[], Vec::as_slice) } + /// How many framings `id` was asked in, by the latest decision that + /// asked it and any widening since: more than its ballot holds when the + /// decision ended on a quorum. + pub(super) fn asked_in(&self, id: &str) -> usize { + self.asked + .get(id) + .copied() + .unwrap_or_else(|| self.ballot(id).len()) + } + /// Each framing's own reading of `belief`, from the latest ballots. fn framed(&self, belief: &Belief<'_>) -> Vec { let ids = [Some(belief.yes), Some(belief.no), belief.top] diff --git a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/quorum_tests.rs b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/quorum_tests.rs index 9b112cf6..4eb0920e 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/flow_tests/quorum_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/flow_tests/quorum_tests.rs @@ -3,6 +3,7 @@ use super::*; use crate::agentic::flow::quorum::{self, QUORUM_TOP, SURE_NO, SURE_YES}; +use crate::agentic::flow::{FlowRun, StepLog}; /// A target Choice whose options each framing relabels and reorders, a /// yes/no, and the page kind. @@ -426,3 +427,60 @@ async fn a_run_whose_framings_agree_plainly_does_not_wait_for_its_slowest() { assert!(took >= Duration::from_secs(2), "{took:?}"); assert!(decisions.iter().all(|decision| decision["left"] == 0)); } + +#[tokio::test(start_paused = true)] +async fn a_decision_ended_on_a_quorum_widens_past_every_framing_it_asked() { + // A press nothing undoes is vouched for, and the vouching always widens. + let app = App::with(|_| {}); + let staggered = Staggered { + oracle: Oracle { + app: app.clone(), + hook: Box::new(|id: &str, _: &Question, _: &Sim| match id { + "is_0" => Some(noul(0.99)), + "only_near_0" => Some(noul(0.02)), + _ => None, + }), + requests: Mutex::new(Vec::new()), + fail: false, + }, + pace: [10, 10, 10, 10, 10, 2_000, 2_000], + calls: Mutex::new(0), + }; + let runtime = runtime(staggered); + let request = RunFlowRequest { + flow: serde_json::from_value(json!({ + "app": "Mail", + "steps": [{"stop_before": "sending the email"}] + })) + .unwrap(), + votes: 7, + ..RunFlowRequest::default() + }; + let mut run = FlowRun::new(app, &runtime, &request); + let mut log = StepLog::default(); + let yes_no = |question: &str| { + Question::Noul(tinyinference_decisions::Noul { + instructions: json!({"question": question}), + criteria: None, + }) + }; + let vouching = EvaluationRequest { + state: json!("the draft, its Send button in view"), + model: "jev-latest".to_owned(), + questions: BTreeMap::from([ + ("is_0".to_owned(), yes_no("is it the Send button?")), + ("only_near_0".to_owned(), yes_no("is it only near it?")), + ]), + }; + run.ask(&mut log, vouching.clone()).await.unwrap(); + assert_eq!( + (run.ballot("is_0").len(), run.asked_in("is_0")), + (5, 7), + "ended on a quorum" + ); + run.widen(&mut log, &vouching).await.unwrap().unwrap(); + // The eighth and ninth framings: not the sixth and seventh again, which + // were asked and left. + assert_eq!((run.ballot("is_0").len(), run.asked_in("is_0")), (7, 9)); + assert_eq!(log.calls, 9, "seven, and two more"); +} diff --git a/crates/tinycomputer-engine/src/agentic/flow/mod.rs b/crates/tinycomputer-engine/src/agentic/flow/mod.rs index be18eae8..f0b4c6c5 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/mod.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/mod.rs @@ -292,6 +292,11 @@ pub(super) struct FlowRun<'r, B> { /// keys, from the latest decision that asked it: the evidence a /// deliberating decision reads (`evidence/`) and widens (`escalate`). ballots: BTreeMap>, + /// How many framings each question was asked in, by the latest decision + /// that asked it and any widening since. A decision ended on a quorum + /// holds fewer answers than that, and `escalate` must widen past every + /// framing asked, not only those in the ballot. + asked: BTreeMap, /// The address the surface last reported, on a surface that has them: /// a checkpoint's location, and how a navigation is noticed. pub(super) location: Option, diff --git a/crates/tinycomputer-engine/src/agentic/flow/quorum.rs b/crates/tinycomputer-engine/src/agentic/flow/quorum.rs index 91e8a14d..af2ff428 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/quorum.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/quorum.rs @@ -29,8 +29,10 @@ pub(super) const QUORUM_VOTES: usize = 7; /// ranks first, the same option in each. pub(super) const QUORUM_TOP: f64 = 0.9; /// A yes/no every framing answers at or above this, or every one at or -/// below [`SURE_NO`], lies the evidence band's 0.12 beyond every yes/no -/// threshold the loops use (0.20 to 0.85). +/// below [`SURE_NO`], lies the evidence band's 0.12 beyond every threshold +/// the loops read one yes/no against (0.20 to 0.85). A belief that pairs it +/// with its negation or a coverage can still fall within a band, and is +/// widened then as it would be after all the framings. pub(super) const SURE_YES: f64 = 0.97; /// See [`SURE_YES`]. pub(super) const SURE_NO: f64 = 0.08; diff --git a/crates/tinycomputer-engine/src/agentic/flow/run.rs b/crates/tinycomputer-engine/src/agentic/flow/run.rs index 46099b89..a592a7f3 100644 --- a/crates/tinycomputer-engine/src/agentic/flow/run.rs +++ b/crates/tinycomputer-engine/src/agentic/flow/run.rs @@ -134,6 +134,7 @@ impl<'r, B: AgentBackend + Sync> FlowRun<'r, B> { typed: BTreeSet::new(), deliberation: request.deliberation, ballots: BTreeMap::new(), + asked: BTreeMap::new(), location: None, frontier: Vec::new(), expecting: None, diff --git a/docs/crates/tinycomputer-engine/flow/voting-and-briefing.md b/docs/crates/tinycomputer-engine/flow/voting-and-briefing.md index 1581a022..2c3b7dee 100644 --- a/docs/crates/tinycomputer-engine/flow/voting-and-briefing.md +++ b/docs/crates/tinycomputer-engine/flow/voting-and-briefing.md @@ -119,15 +119,18 @@ Asked 7 ways or more, it is merged without its last 2 framings once those in settle every question: every Choice and Score ranks the same option first, each at 0.9 or more (`QUORUM_TOP`); every yes/no is at 0.97 or more in each framing, or at 0.08 or less in each (`SURE_YES`, `SURE_NO`: the -evidence band beyond every yes/no threshold); and the page kind reads -alike. Replayed over 13,669 decisions asked seven ways in 245 live runs, -such a quorum ended 12% of them, a median 0.14 s sooner (about 2 s a run), -and every one read the same on every threshold and evidence gate as all -seven framings did, but for two whose mean moved by under 0.002 across the -band's edge. A quorum of four would not have: its stragglers dissented in -1% of the decisions it would have ended. The framings left run to their end, so their -connections go back to the pool; they count as calls, and their exchanges -are journaled as they end. +evidence band beyond every threshold one yes/no is read against); and the +page kind reads alike. A belief pairing two answers, a yes/no and its +negation or a coverage, can still fall within a band; it is widened then, +past every framing the decision asked, as it would be after all of them. +Replayed over 13,669 decisions asked seven ways in 245 live runs, such a +quorum ended 12% of them, a median 0.14 s sooner (about 2 s a run), and +every one read the same on every threshold and evidence gate as all seven +framings did, but for two whose mean moved by under 0.002 across the +band's edge. A quorum of four would not have: its stragglers changed what +was read in 0.8% of the decisions it would have ended. The framings left +run to their end, so their connections go back to the pool; they count as +calls, and their exchanges are journaled as they end. Voting is the mechanism; deliberation, described on its own page, is what decides whether a decision's evidence is strong enough to stop there or diff --git a/docs/technical/decision-thresholds.md b/docs/technical/decision-thresholds.md index 87b26107..3f47667d 100644 --- a/docs/technical/decision-thresholds.md +++ b/docs/technical/decision-thresholds.md @@ -54,7 +54,7 @@ Change a constant and its row together. | `HEDGE_AFTER` / `HEDGE_AFTER_LARGE` | 4 s / 5 s | `hedge.rs` | how long a framing runs before a copy of it is sent and the first answer of the two taken; the longer wait is for a request of `HEDGE_LARGE_BYTES` (32 KB) or more. Sage gets no copy | | `QUORUM_VOTES` / `QUORUM_LEFT` | 7 / 2 | `quorum.rs` | a decision asked at least 7 ways is merged without its last 2 framings once those in settle every question; the 2 still run, and count as calls | | `QUORUM_TOP` | 0.9 | `quorum.rs` | least probability every framing in gives the option a Choice or Score ranks first, the same option in each, for a quorum; the page kind needs only the same option | -| `SURE_YES` / `SURE_NO` | 0.97 / 0.08 | `quorum.rs` | a yes/no settles for a quorum when every framing in puts it at or above the first, or every one at or below the second: `UNDECIDED_BAND` beyond the highest (0.85) and lowest (0.20) yes/no thresholds | +| `SURE_YES` / `SURE_NO` | 0.97 / 0.08 | `quorum.rs` | a yes/no settles for a quorum when every framing in puts it at or above the first, or every one at or below the second: `UNDECIDED_BAND` beyond the highest (0.85) and lowest (0.20) thresholds one yes/no is read against; a belief pairing two answers can still fall within a band, and is widened past every framing asked | | `HEDGE_COPIES` | 2 | `hedge.rs` | most copies one runtime has in flight: past it, a framing waits for its own answer, so a slow or failing gateway is not sent a copy of every call | | `LATE_LOOKS` | 2 | `steps/suggestion.rs` | looks again, after a wait for the page to change, for the suggestions a place box lists late, until a row names the text typed (rows of the box's own, such as "Allow location access", are waited past), before its text is left as typed; a page that stayed still through a wait lists nothing more | | `LATE_LOOK_MS` | 1000 ms | `steps/suggestion.rs` | longest one of those waits: it ends as soon as the page changes (`Surface::await_change`) | From 7d1ca8708017908f309f6405b1c7ac829764c4d6 Mon Sep 17 00:00:00 2001 From: Shanu Date: Thu, 8 Oct 2026 17:04:02 +0530 Subject: [PATCH 41/44] Warm every call a task's first turn makes, and none under a call cap A step's first turn asks its judging and grounding's opening together, each in every framing, so warming one connection per framing left half of that turn's calls opening their own. The warm-up now opens one for each (FIRST_TURN, 2, times the votes; at most 18). A task whose budget caps its Jev calls (max_model_calls) is not warmed: the warm-up's calls would spend from it unseen. The module runner's warm test now checks that the warm-up's calls reach the task's journal, with a runtime whose calls give up before any request can reach Jev. --- .../src/agent/types/request.rs | 4 ++- .../src/agentic/agentic_tests/warm_tests.rs | 34 +++++++++++++------ .../src/agentic/runtime.rs | 24 ++++++++----- crates/tinycomputer-engine/src/task/drive.rs | 11 ++++-- .../src/task/task_tests/plan_tests.rs | 19 +++++++++++ .../tinybus_module_tests/tasks_tests.rs | 30 +++++++++++++--- .../crates/tinycomputer-engine/jev-runtime.md | 21 +++++++----- docs/technical/tasks.md | 8 +++-- 8 files changed, 112 insertions(+), 39 deletions(-) diff --git a/crates/tinycomputer-bus/src/agent/types/request.rs b/crates/tinycomputer-bus/src/agent/types/request.rs index d27bc5d3..f6904c94 100644 --- a/crates/tinycomputer-bus/src/agent/types/request.rs +++ b/crates/tinycomputer-bus/src/agent/types/request.rs @@ -142,7 +142,9 @@ pub enum PaymentMode { pub struct TaskBudget { /// Actions across every surface. pub max_actions: Option, - /// Jev evaluations. Every framing of a voted decision counts as one. + /// Jev evaluations. Every framing of a voted decision counts as one. A + /// task given this cap does not warm Jev's connections while it is + /// planned, as warming them would spend calls the cap does not count. pub max_model_calls: Option, /// How many ways each decision is asked before its answers are averaged; /// the module's default when unset. diff --git a/crates/tinycomputer-engine/src/agentic/agentic_tests/warm_tests.rs b/crates/tinycomputer-engine/src/agentic/agentic_tests/warm_tests.rs index abd0bda9..d87fbf29 100644 --- a/crates/tinycomputer-engine/src/agentic/agentic_tests/warm_tests.rs +++ b/crates/tinycomputer-engine/src/agentic/agentic_tests/warm_tests.rs @@ -1,7 +1,7 @@ //! Tests for warming a runtime's connections while a task's plan is drafted. use super::*; -use crate::agentic::runtime::WARM_TIMEOUT; +use crate::agentic::runtime::{FIRST_TURN, WARM_TIMEOUT}; /// An evaluator nothing ever comes back from. struct Silent; @@ -34,11 +34,15 @@ fn ready() -> tinyinference_decisions::EvaluationResult { } #[tokio::test] -async fn a_warm_up_asks_one_small_question_for_each_framing() { - let (runtime, requests) = runtime_recording(vec![ready(); 7]); +async fn a_warm_up_asks_one_small_question_for_each_call_of_a_first_turn() { + let (runtime, requests) = runtime_recording(vec![ready(); 14]); runtime.warm(7).await; let requests = requests.lock().unwrap(); - assert_eq!(requests.len(), 7, "one call a framing, all at once"); + assert_eq!( + requests.len(), + 14, + "the judging and grounding's opening, each in every framing, all at once" + ); for request in requests.iter() { assert_eq!(request.model, "jev-latest", "the runtime's own model"); assert_eq!( @@ -54,13 +58,21 @@ async fn a_warm_up_asks_one_small_question_for_each_framing() { } #[tokio::test] -async fn a_warm_up_opens_no_more_than_a_decision_asks_at_once() { - let (runtime, requests) = runtime_recording(vec![ready(); 9]); +async fn a_warm_up_opens_no_more_than_a_first_turn_asks_at_once() { + let (runtime, requests) = runtime_recording(vec![ready(); 18]); runtime.warm(50).await; - assert_eq!(requests.lock().unwrap().len(), 9, "MAX_VOTES at most"); - let (runtime, requests) = runtime_recording(vec![ready()]); + assert_eq!( + requests.lock().unwrap().len(), + 18, + "MAX_VOTES framings of each first-turn request at most" + ); + let (runtime, requests) = runtime_recording(vec![ready(); 2]); runtime.warm(0).await; - assert_eq!(requests.lock().unwrap().len(), 1, "one at least"); + assert_eq!( + requests.lock().unwrap().len(), + usize::try_from(FIRST_TURN).unwrap(), + "one framing of each at least" + ); } #[tokio::test] @@ -83,7 +95,7 @@ async fn a_warm_up_nothing_answers_is_given_up_on() { #[tokio::test] async fn a_warm_up_is_journaled_with_the_task() { let dir = std::env::temp_dir().join(format!("tinycomputer-warm-{}", std::process::id())); - let (runtime, _requests) = runtime_recording(vec![ready(); 7]); + let (runtime, _requests) = runtime_recording(vec![ready(); 14]); let runtime = runtime.with_journal(&dir).journaled_as("task-t-1"); runtime.warm(7).await; let journal = std::fs::read_to_string(dir.join("task-t-1").join("journal.jsonl")).unwrap(); @@ -94,5 +106,5 @@ async fn a_warm_up_is_journaled_with_the_task() { .map(|event| event["step"].as_str().unwrap().to_owned()) .collect::>(); std::fs::remove_dir_all(&dir).unwrap(); - assert_eq!(steps, vec!["warm-up"; 7]); + assert_eq!(steps, vec!["warm-up"; 14]); } diff --git a/crates/tinycomputer-engine/src/agentic/runtime.rs b/crates/tinycomputer-engine/src/agentic/runtime.rs index dc9e2c39..41c7bb09 100644 --- a/crates/tinycomputer-engine/src/agentic/runtime.rs +++ b/crates/tinycomputer-engine/src/agentic/runtime.rs @@ -39,6 +39,11 @@ pub(super) const ATTEMPT_TIMEOUT: Duration = Duration::from_secs(10); /// its calls up. pub(super) const WARM_TIMEOUT: Duration = Duration::from_secs(10); +/// Requests a step's first turn asks at once, each in every framing: its +/// judging, and grounding's opening for the move it almost always makes +/// (`flow::act::judge`). +pub(super) const FIRST_TURN: u32 = 2; + /// Configured Jev transport and non-secret policy metadata. #[derive(Clone)] pub struct JevRuntime { @@ -228,21 +233,22 @@ impl JevRuntime { } /// Opens connections to Jev for the decisions to come, while a task's - /// plan is drafted: one for each framing of a decision asked `votes` - /// ways (at most 9, `MAX_VOTES`), each with a one-question evaluation, - /// all at once. Each is journaled as `warm-up`, its answer is dropped, - /// and those still out after 10 s (`WARM_TIMEOUT`) are given up on. - /// A decision asks its framings all at once, each on a connection of its - /// own; live, a task's first decision took 330 ms more a call than its - /// later ones, opening them. Sage, whose calls take seconds, is not - /// warmed. + /// plan is drafted: one for each call of a step's first turn, which asks + /// its judging and grounding's opening together (`FIRST_TURN`), each in + /// `votes` framings (at most 9, `MAX_VOTES`). Each connection is opened + /// with a one-question evaluation, all at once, journaled as `warm-up`; + /// its answer is dropped, and those still out after 10 s + /// (`WARM_TIMEOUT`) are given up on. A decision asks its framings all + /// at once, each on a connection of its own; live, a task's first + /// decision took 330 ms more a call than its later ones, opening them. + /// Sage, whose calls take seconds, is not warmed. pub async fn warm(&self, votes: u32) { if self.configuration.provider == JevProvider::Sage { return; } let request = Arc::new(warm_up(&self.configuration.model)); let mut calls = tokio::task::JoinSet::new(); - for _ in 0..votes.clamp(1, MAX_VOTES) { + for _ in 0..votes.clamp(1, MAX_VOTES).saturating_mul(FIRST_TURN) { let (runtime, request) = (self.clone(), Arc::clone(&request)); calls.spawn(async move { let _answer = runtime.evaluate(Some("warm-up"), &request).await; diff --git a/crates/tinycomputer-engine/src/task/drive.rs b/crates/tinycomputer-engine/src/task/drive.rs index 377ab53b..ea9ec2c0 100644 --- a/crates/tinycomputer-engine/src/task/drive.rs +++ b/crates/tinycomputer-engine/src/task/drive.rs @@ -25,13 +25,14 @@ pub(super) async fn plan_then_drive( task: String, surfaces: Vec, ) { - let (names, secrets, constraints, votes) = cell.state.lock().map_or_else( + let (names, secrets, constraints, votes, capped) = cell.state.lock().map_or_else( |_| { ( Vec::new(), Vec::new(), TaskConstraints::default(), DEFAULT_VOTES, + false, ) }, |state| { @@ -41,13 +42,17 @@ pub(super) async fn plan_then_drive( owned(state.facts.secret_names()), state.constraints.clone(), state.budget.votes.unwrap_or(DEFAULT_VOTES), + state.budget.max_model_calls.is_some(), ) }, ); let id = cell.view.borrow().id.clone(); // Jev's connections open while the plan is drafted, so the first - // decision need not open them; nothing waits for this. - tokio::spawn(runner.warm(&id, votes)); + // decision need not open them; nothing waits for this. A budget that + // caps the task's Jev calls is not spent on calls it does not count. + if !capped { + tokio::spawn(runner.warm(&id, votes)); + } let started = Instant::now(); // Timed on its own: a browser slower to open than the plan is to draft // is not planning time. diff --git a/crates/tinycomputer-engine/src/task/task_tests/plan_tests.rs b/crates/tinycomputer-engine/src/task/task_tests/plan_tests.rs index 9d68a7c1..67b7e2ad 100644 --- a/crates/tinycomputer-engine/src/task/task_tests/plan_tests.rs +++ b/crates/tinycomputer-engine/src/task/task_tests/plan_tests.rs @@ -283,6 +283,25 @@ async fn a_task_warms_jev_while_it_is_planned_and_never_waits_for_it() { settle(&tasks, &started.id).await; assert_eq!(*script.warmed.lock().unwrap(), [(started.id.clone(), 3)]); + // A budget capping its Jev calls is not spent on warming. + let (tasks, script) = planned( + vec![finished_run(FlowStopReason::Completed, vec![], &[], None)], + Ok(flow), + ); + let started = tasks + .start(&StartTaskRequest { + task: Some("start an email".to_owned()), + budget: tinycomputer_bus::agent::TaskBudget { + max_model_calls: Some(40), + ..tinycomputer_bus::agent::TaskBudget::default() + }, + ..StartTaskRequest::default() + }) + .data + .unwrap(); + settle(&tasks, &started.id).await; + assert!(script.warmed.lock().unwrap().is_empty()); + // A flow handed over whole is not planned, and not warmed. let (tasks, script) = controller(vec![finished_run( FlowStopReason::Completed, diff --git a/crates/tinycomputer/src/tinybus_module/tinybus_module_tests/tasks_tests.rs b/crates/tinycomputer/src/tinybus_module/tinybus_module_tests/tasks_tests.rs index 05f1ca87..455be3a3 100644 --- a/crates/tinycomputer/src/tinybus_module/tinybus_module_tests/tasks_tests.rs +++ b/crates/tinycomputer/src/tinybus_module/tinybus_module_tests/tasks_tests.rs @@ -255,9 +255,13 @@ async fn the_runner_warms_its_jev_runtime_for_a_task() { tinycomputer_browser::AgentBrowser, ))) }; - // The task's runtime is reached; Sage's calls take seconds and are not - // warmed, so nothing goes out. - let jev = JevRuntime::sage("test-key", false).unwrap(); + // A runtime whose every call gives up within a millisecond, before any + // request can reach Jev: each is journaled with the task all the same. + let dir = std::env::temp_dir().join(format!("tinycomputer-runner-warm-{}", std::process::id())); + let mut config = tinycomputer_bus::JevConfig::new("test-key"); + config.timeout_ms = Some(1); + config.max_retries = Some(0); + let jev = JevRuntime::configure(&config).unwrap().with_journal(&dir); crate::tinybus_module::runner::WorkspaceRunner::new( crate::Desktop::new(), Some(jev), @@ -265,7 +269,25 @@ async fn the_runner_warms_its_jev_runtime_for_a_task() { ) .warm(&TaskId::new("t-1"), 7) .await; - // With no Jev runtime there is nothing to warm. + let journal = std::fs::read_to_string(dir.join("task-t-1").join("journal.jsonl")).unwrap(); + let _ = std::fs::remove_dir_all(&dir); + let warmed = journal + .lines() + .map(|line| serde_json::from_str::(line).unwrap()) + .filter(|event| event["event"] == "exchange" && event["step"] == "warm-up") + .count(); + assert_eq!(warmed, 14, "every framing of a first turn's two requests"); + + // Sage's calls take seconds and are not warmed; with no Jev runtime + // there is nothing to warm. + let sage = JevRuntime::sage("test-key", false).unwrap(); + crate::tinybus_module::runner::WorkspaceRunner::new( + crate::Desktop::new(), + Some(sage), + browser(), + ) + .warm(&TaskId::new("t-1"), 7) + .await; crate::tinybus_module::runner::WorkspaceRunner::new(crate::Desktop::new(), None, browser()) .warm(&TaskId::new("t-1"), 7) .await; diff --git a/docs/crates/tinycomputer-engine/jev-runtime.md b/docs/crates/tinycomputer-engine/jev-runtime.md index 9846fd12..afaac145 100644 --- a/docs/crates/tinycomputer-engine/jev-runtime.md +++ b/docs/crates/tinycomputer-engine/jev-runtime.md @@ -132,16 +132,21 @@ A decision asks its framings all at once, each on an HTTP/1.1 connection of its own, and opening one to the gateway costs a handshake: live, a task's first decision took 330 ms more a call (the median over 84 runs) than the task's later decisions of the same size. `JevRuntime::warm(votes)` opens them -ahead of time. It sends one small evaluation per framing of a decision asked -`votes` ways (at most 9, `MAX_VOTES`), all at once, each a single yes/no -question about a one-line state, through `evaluate` like any other call, so -each is journaled under the step `warm-up`. The answers are dropped, calls -still out after 10 s (`WARM_TIMEOUT`) are given up on, and Sage, whose calls -take seconds, is not warmed. +ahead of time. A step's first turn asks its judging and grounding's opening +together (`FIRST_TURN`, 2), each in `votes` framings (at most 9, +`MAX_VOTES`), so the warm-up sends one small evaluation for each of those +calls, all at once: a single yes/no question about a one-line state, through +`evaluate` like any other call, so each is journaled under the step +`warm-up`. The answers are dropped, calls still out after 10 s +(`WARM_TIMEOUT`) are given up on, and Sage, whose calls take seconds, is not +warmed. Idle connections stay in the client's pool for 90 s, longer than a +plan takes to draft. The task controller warms while a task's plan is drafted -(`FlowRunner::warm`), so its first decision finds the connections open; the -task never waits for the warm-up. +(`FlowRunner::warm`), so its first turn finds the connections open; the task +never waits for the warm-up. A task whose budget caps its Jev calls +(`max_model_calls`) is not warmed: the warm-up's calls would not count +against the cap. ## Run identity and the journal diff --git a/docs/technical/tasks.md b/docs/technical/tasks.md index 9e97b7da..ee8549e6 100644 --- a/docs/technical/tasks.md +++ b/docs/technical/tasks.md @@ -291,10 +291,12 @@ run that follows opens it again; a task cancelled while its browser was still opening is left holding none. Every task planned this way also has its runner warm Jev while the plan is -drafted (`FlowRunner::warm`): one small evaluation for each way its first -decision will be asked, so that decision finds its connections open +drafted (`FlowRunner::warm`): one small evaluation for each call its first +turn will make, its judging and grounding's opening in every framing, so that +turn finds its connections open ([`jev-runtime.md`](../crates/tinycomputer-engine/jev-runtime.md#warming-connections)). -The task never waits for the warm-up. +The task never waits for the warm-up, and a task whose budget caps its Jev +calls is not warmed. ## Rescues From 99eb46aff4ed249c28c725edf611dd7564f05e8d Mon Sep 17 00:00:00 2001 From: Shanu Date: Thu, 8 Oct 2026 17:04:02 +0530 Subject: [PATCH 42/44] Leave warm-ups and a quorum's late framings out of --split's rounds jev_journal --split counted the warm-up's calls with the decisions' (calls, their percentiles, failed calls) and in the first round's slowest call; they are still priced. A quorum's left framings were skipped by position, the next exchanges after their decision whichever decision they belonged to, so in a batch a grounding framing could be skipped and the judging's straggler counted in grounding's round. A late exchange is now matched to its decision by its questions, which are among the decision's; a journal without question ids reads as before. --- .../src/journal/journal_tests/split_tests.rs | 46 +++++++++++++++++++ .../src/journal/split/mod.rs | 44 +++++++++++++++--- 2 files changed, 83 insertions(+), 7 deletions(-) diff --git a/crates/tinycomputer-examples/src/journal/journal_tests/split_tests.rs b/crates/tinycomputer-examples/src/journal/journal_tests/split_tests.rs index f92fe138..4de5a73b 100644 --- a/crates/tinycomputer-examples/src/journal/journal_tests/split_tests.rs +++ b/crates/tinycomputer-examples/src/journal/journal_tests/split_tests.rs @@ -214,6 +214,52 @@ fn the_framings_a_quorum_left_count_in_no_round() { assert_eq!(spent.calls, 9, "every call made counts"); } +#[test] +fn a_quorums_late_framings_are_told_from_the_next_rounds_by_their_questions() { + let exchange = |ms: u64, latency: u64, questions: &[&str]| json!({"event": "exchange", "at": at(ms), "latency_ms": latency, "ok": true, "questions": questions}); + let decision = |ms: u64, left: u64, questions: &[&str]| json!({"event": "decision", "at": at(ms), "wall_ms": 500, "left": left, "questions": questions}); + let judging = ["done", "not_done"]; + let events = vec![ + json!({"event": "run", "at": at(0), "kind": "flow"}), + exchange(300, 300, &judging), + exchange(400, 400, &judging), + decision(400, 2, &judging), + // Grounding's round, the judging's two late framings among its calls. + exchange(900, 400, &["target"]), + exchange(1_000, 1_000, &judging), + exchange(1_100, 600, &["target"]), + exchange(1_200, 1_200, &judging), + decision(1_100, 0, &["target"]), + ]; + let spent = split(&events); + // Rounds of 300/400 and 400/600: 100 and 200 ms. + assert_eq!(spent.slowest_extra_ms, 150); +} + +#[test] +fn a_warm_up_is_priced_but_is_no_decisions_call() { + let events = vec![ + json!({"event": "run", "at": at(0), "kind": "flow"}), + json!({"event": "exchange", "at": at(100), "latency_ms": 5_000, "ok": false, + "step": "warm-up", "model": "typesafe/jev-1.13-20260917", "input_tokens": 20}), + json!({"event": "exchange", "at": at(400), "latency_ms": 400, "ok": true, + "model": "typesafe/jev-1.13-20260917", "input_tokens": 1_000}), + json!({"event": "exchange", "at": at(500), "latency_ms": 500, "ok": true, + "model": "typesafe/jev-1.13-20260917", "input_tokens": 1_000}), + json!({"event": "decision", "at": at(500), "wall_ms": 500}), + ]; + let spent = split(&events); + assert_eq!((spent.calls, spent.failed_calls), (2, 0)); + assert_eq!( + spent.slowest_extra_ms, 100, + "the round is the decision's own" + ); + assert_eq!( + spent.input_tokens, 2_020, + "its tokens are spent all the same" + ); +} + #[test] fn jevs_input_tokens_are_priced_and_another_models_are_not() { let exchange = |model: &str, input_tokens: u64| { diff --git a/crates/tinycomputer-examples/src/journal/split/mod.rs b/crates/tinycomputer-examples/src/journal/split/mod.rs index 8135a8b6..282f6d58 100644 --- a/crates/tinycomputer-examples/src/journal/split/mod.rs +++ b/crates/tinycomputer-examples/src/journal/split/mod.rs @@ -23,6 +23,10 @@ use time::{Spans, length, millis, minus, union}; /// as the repository's evals count it. Jev's output is free. const JEV_MICRO_USD_PER_M_INPUT: u64 = 42_000; +/// The step a call opening a connection while a task is planned is +/// journaled under. +const WARM_UP: &str = "warm-up"; + /// Where one task's time went, in ms unless named otherwise. #[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(default)] @@ -153,8 +157,12 @@ pub fn split(events: &[Value]) -> Split { actions.push(number(event, "wall_ms") + number(event, "settle_ms")); } "exchange" => { - calls.push(number(event, "latency_ms")); - split.failed_calls += u64::from(event["ok"] == Value::Bool(false)); + // A warm-up opens a connection while the plan is drafted: + // priced, but no decision's call. + if event["step"] != WARM_UP { + calls.push(number(event, "latency_ms")); + split.failed_calls += u64::from(event["ok"] == Value::Bool(false)); + } split.input_tokens += number(event, "input_tokens"); split.output_tokens += number(event, "output_tokens"); if event["model"] @@ -257,19 +265,37 @@ impl Kinds { } } +/// The question ids an `exchange` or `decision` event names. +fn questions(event: &Value) -> Vec<&str> { + event["questions"] + .as_array() + .map(|ids| ids.iter().filter_map(Value::as_str).collect()) + .unwrap_or_default() +} + /// The mean of how much longer each round of calls waited for its slowest /// call than for its median one. A round is the calls journaled since the /// previous decision: one decision's framings, or a batch's. The framings a /// quorum did not wait for (its `left`) journal after their decision, and -/// are no round's. +/// are no round's: each is told from the calls around it by its questions, +/// which are among its decision's. A warm-up is no round's either. fn slowest_extra(events: &[Value]) -> u64 { let mut extras = Vec::new(); let mut round = Vec::new(); - let mut late = 0; + // Each decision with framings still out: its questions, and how many. + let mut late: Vec<(Vec<&str>, u64)> = Vec::new(); for event in events { match event["event"].as_str() { - Some("exchange") if late > 0 => late -= 1, - Some("exchange") => round.push(number(event, "latency_ms")), + Some("exchange") if event["step"] == WARM_UP => {} + Some("exchange") => { + let asked = questions(event); + match late.iter_mut().find(|(decided, left)| { + *left > 0 && asked.iter().all(|id| decided.contains(id)) + }) { + Some((_, left)) => *left -= 1, + None => round.push(number(event, "latency_ms")), + } + } Some("decision") => { if !round.is_empty() { round.sort_unstable(); @@ -277,7 +303,11 @@ fn slowest_extra(events: &[Value]) -> u64 { extras.push(slowest - round[(round.len() - 1) / 2]); round.clear(); } - late = number(event, "left"); + late.retain(|(_, left)| *left > 0); + let left = number(event, "left"); + if left > 0 { + late.push((questions(event), left)); + } } Some("run") => round.clear(), _ => {} From b2e0772103e7c15eb8b98fefdf8518eae29b8c6b Mon Sep 17 00:00:00 2001 From: Shanu Date: Thu, 8 Oct 2026 17:04:02 +0530 Subject: [PATCH 43/44] Load early only the page a task goes to, within 10 s, and journal it The early load took any one address a task's text wrote out, though a task may only mention one: to check it, read it out, or pass it on, and a link's query can carry a one-time token a load would spend. It now loads an address only when the words right before it send the browser there ("go to", "open", "visit", "start at", "on", ...) and it carries no query or fragment. The early navigation waits for its page's load at most 10 s (EARLY_LOAD_MS, beyond nine in ten live first pages), and is journaled as open_page (wall_ms, loaded), since the plan's outcome waits for it. The check that a navigation finds the early page still shown reads it within READ_TIMEOUT, and the mark of the early page names its session and is set only while the surface is not let go, so it can match no other session. --- .../tinycomputer-browser/src/surface/mod.rs | 43 ++++++---- .../src/surface/operations.rs | 28 +++---- .../surface/surface_tests/operations_tests.rs | 10 ++- .../tinycomputer-browser/src/surface/tabs.rs | 13 ++-- crates/tinycomputer-engine/src/task/page.rs | 78 +++++++++++++++---- .../src/task/task_tests.rs | 5 +- .../src/task/task_tests/page_tests.rs | 43 ++++++++-- .../src/task/task_tests/plan_tests.rs | 14 +++- .../tinycomputer/src/tinybus_module/runner.rs | 19 ++++- .../tinybus_module_tests/tasks_tests.rs | 25 +++++- docs/crates/tinycomputer-browser/surface.md | 12 +-- docs/technical/jev-journal.md | 1 + docs/technical/tasks.md | 25 +++--- 13 files changed, 243 insertions(+), 73 deletions(-) diff --git a/crates/tinycomputer-browser/src/surface/mod.rs b/crates/tinycomputer-browser/src/surface/mod.rs index 2b0435cb..8839b603 100644 --- a/crates/tinycomputer-browser/src/surface/mod.rs +++ b/crates/tinycomputer-browser/src/surface/mod.rs @@ -31,7 +31,9 @@ use std::sync::{Arc, Mutex}; use serde_json::json; use tinycomputer_bus::DesktopResponse; -use tinycomputer_bus::browser::{Action, SessionId, SessionOptions, SnapshotRequest}; +use tinycomputer_bus::browser::{ + Action, NavigateRequest, SessionId, SessionOptions, SnapshotRequest, +}; use tinycomputer_core::Platform; use tinycomputer_core::surface::Screen; use tinycomputer_cursor::ScreenCursor; @@ -108,6 +110,12 @@ pub enum Settle { /// to count as still. const STILL_MS: u64 = 120; +/// The longest an early load ([`BrowserSurface::open_at`]) waits for its +/// page's `load` event: beyond nine in ten live first pages (6.4 s). A plan +/// drafted sooner waits for it no longer, and a page not drawn by then is +/// loaded again by the step that browses there. +const EARLY_LOAD_MS: u64 = 10_000; + /// One browser session, lazily opened, as a [`Surface`]. #[derive(Clone)] pub struct BrowserSurface { @@ -123,10 +131,10 @@ pub struct BrowserSurface { /// Set once the surface is let go ([`BrowserSurface::close`]): it is /// then not opened early again. closed: Arc, - /// The address the session was opened at early - /// ([`BrowserSurface::open_at`]), until the page is first read or - /// another address is loaded: a navigation there finds it loaded. - opened_at: Arc>>, + /// The session opened early ([`BrowserSurface::open_at`]) and the + /// address loaded in it, until the page is first read or another address + /// is loaded: a navigation there, in that session, finds it loaded. + opened_at: Arc>>, } impl std::fmt::Debug for BrowserSurface { @@ -206,19 +214,28 @@ impl BrowserSurface { } /// Opens the session early, as [`BrowserSurface::open`] does, and loads - /// `url` in it: the page a task names, loaded while its plan is drafted. - /// Until the page is first read, a navigation to the same place (one - /// page's two addresses: `https://` or not, `www.` or not, a trailing - /// slash or not) finds it loaded and loads nothing. Whether the page - /// loaded; a surface already let go opens nothing. + /// `url` in it: the page a task names, loaded while its plan is drafted, + /// waiting for its `load` at most `EARLY_LOAD_MS`. Until the page is + /// first read, a navigation to the same place (one page's two + /// addresses: `https://` or not, `www.` or not, a trailing slash or not) + /// in the same session finds it loaded and loads nothing. Whether the + /// page loaded; a surface already let go opens nothing, and one let go + /// meanwhile keeps no page. #[must_use] pub fn open_at(&self, url: &str) -> bool { let Ok(id) = self.session_slot(true) else { return false; }; - let loaded = self.navigate_in(&id, url).ok; - if loaded && let Ok(mut opened) = self.opened_at.lock() { - *opened = Some(url.to_owned()); + let request = NavigateRequest { + timeout_ms: Some(EARLY_LOAD_MS), + ..NavigateRequest::new(url) + }; + let loaded = self.navigate_in(&id, request).ok; + if loaded + && !self.closed.load(Ordering::Acquire) + && let Ok(mut opened) = self.opened_at.lock() + { + *opened = Some((id, url.to_owned())); } loaded } diff --git a/crates/tinycomputer-browser/src/surface/operations.rs b/crates/tinycomputer-browser/src/surface/operations.rs index 365c69bc..2378c7c9 100644 --- a/crates/tinycomputer-browser/src/surface/operations.rs +++ b/crates/tinycomputer-browser/src/surface/operations.rs @@ -290,16 +290,17 @@ impl Surface for BrowserSurface { } fn navigate(&self, url: &str) -> DesktopResponse { - // Loaded there while the plan was drafted, and not read since. - if let Some(opened) = self.early_page() + // Loaded there while the plan was drafted, in this session, and not + // read since. + if let Some((opened_in, opened)) = self.early_page() && place(&opened) == place(url) - && let Some(id) = self.session() - && let Some((shown, title)) = self.shown_page(&id) + && self.session().as_ref() == Some(&opened_in) + && let Some((shown, title)) = self.shown_page(&opened_in) { return reply("navigate", Ok(json!({"url": shown, "title": title}))); } match self.ensure_session() { - Ok(id) => self.navigate_in(&id, url), + Ok(id) => self.navigate_in(&id, NavigateRequest::new(url)), Err(error) => reply("navigate", Err(error)), } } @@ -310,14 +311,15 @@ impl Surface for BrowserSurface { } impl BrowserSurface { - /// Loads `url` in session `id`. A heavy page can be read long before its - /// `load` event fires. - pub(super) fn navigate_in(&self, id: &SessionId, url: &str) -> DesktopResponse { + /// Loads `request`'s address in session `id`. A heavy page can be read + /// long before its `load` event fires. + pub(super) fn navigate_in(&self, id: &SessionId, request: NavigateRequest) -> DesktopResponse { + let url = request.url.clone(); let page = self - .block(self.browser.navigate(id, NavigateRequest::new(url))) + .block(self.browser.navigate(id, request)) .map(|page| (page.url, page.title)); let page = match page { - Err(error @ Error::Timeout { .. }) => self.drawn_page(id, url).ok_or(error), + Err(error @ Error::Timeout { .. }) => self.drawn_page(id, &url).ok_or(error), other => other, }; reply( @@ -326,9 +328,9 @@ impl BrowserSurface { ) } - /// Takes the address the session was opened at early: the first read - /// or navigation leaves that page as it was opened. - pub(super) fn early_page(&self) -> Option { + /// Takes the session opened early and the address loaded in it: the + /// first read or navigation leaves that page as it was opened. + pub(super) fn early_page(&self) -> Option<(SessionId, String)> { self.opened_at .lock() .ok() diff --git a/crates/tinycomputer-browser/src/surface/surface_tests/operations_tests.rs b/crates/tinycomputer-browser/src/surface/surface_tests/operations_tests.rs index 9b7fb733..85de0465 100644 --- a/crates/tinycomputer-browser/src/surface/surface_tests/operations_tests.rs +++ b/crates/tinycomputer-browser/src/surface/surface_tests/operations_tests.rs @@ -576,14 +576,20 @@ fn a_page_opened_early_is_not_loaded_again_until_it_is_read() { let early = harness("open-at", shop()); assert!(early.surface.open_at("https://shop.test")); assert_eq!(loads(&early.fake), 1); + assert_eq!( + early.fake.last("navigate")["timeout"], + 10_000, + "a plan drafted sooner waits for it no longer" + ); // The plan's first step browses there, by another of its addresses. let reply = early.surface.navigate("https://www.shop.test/"); assert!(reply.ok, "{:?}", reply.error); assert_eq!(reply.data.unwrap()["title"], "Shop"); assert_eq!(loads(&early.fake), 1, "loaded once"); - // Asked again, it loads again. + // Asked again, it loads again, with the session's own deadline. assert!(early.surface.navigate("https://shop.test").ok); assert_eq!(loads(&early.fake), 2); + assert!(early.fake.last("navigate")["timeout"].is_null()); // Once the page is read, or another page is asked for, it loads. let read = harness("open-at-read", shop()); @@ -625,6 +631,8 @@ fn a_page_opened_early_is_not_loaded_again_until_it_is_read() { let let_go = harness("open-at-let-go", shop()); assert!(let_go.surface.open_at("https://shop.test")); let_go.surface.close(); + // A new session shows the shop drawn: a page kept would skip the load. + assert!(let_go.surface.launch("browser").ok); assert!(let_go.surface.navigate("https://shop.test").ok); assert_eq!(loads(&let_go.fake), 2); } diff --git a/crates/tinycomputer-browser/src/surface/tabs.rs b/crates/tinycomputer-browser/src/surface/tabs.rs index 7adf20e7..0a09209c 100644 --- a/crates/tinycomputer-browser/src/surface/tabs.rs +++ b/crates/tinycomputer-browser/src/surface/tabs.rs @@ -4,7 +4,7 @@ use serde_json::{Value, json}; use tinycomputer_bus::browser::SessionId; -use super::{BrowserSurface, sight}; +use super::{BrowserSurface, READ_TIMEOUT, sight}; /// Where the page is, what it is called, and whether it has drawn words. const DRAWN_JS: &str = r"(() => ({ @@ -93,11 +93,14 @@ impl BrowserSurface { /// The address and title of the page session `id` shows, once it has /// drawn words; `None` while it shows nothing yet. pub(super) fn shown_page(&self, id: &SessionId) -> Option<(String, String)> { + let reading = self + .browser + .command(id, json!({"action": "evaluate", "script": DRAWN_JS})); + // Within a deadline: a call sent while a page is being replaced (a + // redirect, a challenge's reload) can wait out the browser's own 30 s. let data = self - .block( - self.browser - .command(id, json!({"action": "evaluate", "script": DRAWN_JS})), - ) + .block(async { tokio::time::timeout(READ_TIMEOUT, reading).await }) + .ok()? .ok()?; let page = data.get("result")?; let shown = page.get("url").and_then(Value::as_str)?; diff --git a/crates/tinycomputer-engine/src/task/page.rs b/crates/tinycomputer-engine/src/task/page.rs index 0950c85a..5f8330bd 100644 --- a/crates/tinycomputer-engine/src/task/page.rs +++ b/crates/tinycomputer-engine/src/task/page.rs @@ -1,5 +1,5 @@ -//! The one web page a task's own words name, which its browser can load -//! while the plan is drafted ([`FlowRunner::open_page`]). +//! The one web page a task's own words send its browser to, which the +//! browser can load while the plan is drafted ([`FlowRunner::open_page`]). //! //! [`FlowRunner::open_page`]: super::FlowRunner::open_page @@ -7,29 +7,79 @@ /// it. const TRAILING: &[char] = &['.', ',', ';', ':', '!', '?', ')', ']', '}', '\'', '"']; +/// Punctuation that can come between an address and the words before it. +const OPENING: &[char] = &['(', '[', '<', '"', '\'', ':']; + +/// Words that, right before an address, send the browser there: the task +/// starts on that page, rather than reading, checking, or passing on an +/// address it only mentions. +const SENDS_TO: &[&str] = &[ + "go to", + "goto", + "open", + "visit", + "navigate to", + "browse to", + "head to", + "start at", + "start on", + "on", + "at", +]; + /// The one web address `task` names, written out with its scheme -/// (`https://…` or `http://…`) and without the punctuation after it; `None` -/// when it names none, or several, which leave no one page to start on. An -/// address written twice, with a trailing slash or without, is one. +/// (`https://…` or `http://…`) and without the punctuation after it, when +/// the words before it send the browser there and it carries no query or +/// fragment, which can hold a token a load would spend. `None` when it names +/// none, or several, which leave no one page to start on. An address written +/// twice, with a trailing slash or without, is one. pub(super) fn named_page(task: &str) -> Option { - let mut named: Vec<&str> = Vec::new(); - for word in task.split_whitespace() { - let Some(start) = word.find("https://").or_else(|| word.find("http://")) else { - continue; - }; - let address = word[start..].trim_end_matches(TRAILING); + let mut named: Vec<(&str, &str)> = Vec::new(); + let mut from = 0; + while let Some(found) = scheme_at(&task[from..]) { + let start = from + found; + let end = task[start..] + .find(char::is_whitespace) + .map_or(task.len(), |length| start + length); + from = end; + let address = task[start..end].trim_end_matches(TRAILING); let has_host = address .split_once("://") .is_some_and(|(_, rest)| !rest.trim_start_matches('/').is_empty()); let seen = named .iter() - .any(|seen| seen.trim_end_matches('/') == address.trim_end_matches('/')); + .any(|(seen, _)| seen.trim_end_matches('/') == address.trim_end_matches('/')); if has_host && !seen { - named.push(address); + named.push((address, &task[..start])); } } match named.as_slice() { - [page] => Some((*page).to_owned()), + [(page, before)] if sent_to(before) && !page.contains(['?', '#']) => { + Some((*page).to_owned()) + } _ => None, } } + +/// Where the next `https://` or `http://` in `text` starts. +fn scheme_at(text: &str) -> Option { + [text.find("https://"), text.find("http://")] + .into_iter() + .flatten() + .min() +} + +/// Whether `before`, a task's words up to an address, ends with words that +/// send the browser there. +fn sent_to(before: &str) -> bool { + let before = before + .trim_end_matches(|character: char| { + character.is_whitespace() || OPENING.contains(&character) + }) + .to_lowercase(); + SENDS_TO.iter().any(|words| { + before + .strip_suffix(words) + .is_some_and(|ahead| ahead.is_empty() || ahead.ends_with(char::is_whitespace)) + }) +} diff --git a/crates/tinycomputer-engine/src/task/task_tests.rs b/crates/tinycomputer-engine/src/task/task_tests.rs index 52628de1..85849548 100644 --- a/crates/tinycomputer-engine/src/task/task_tests.rs +++ b/crates/tinycomputer-engine/src/task/task_tests.rs @@ -52,7 +52,8 @@ struct Script { shot: Mutex>, /// A capture that never answers, like a hung surface. stuck: std::sync::atomic::AtomicBool, - /// `capture` and `release` calls, in the order they arrived. + /// `capture`, `release`, `prepare` and `open_page` calls, in the order + /// they arrived. events: Mutex>, /// What the task journaled outside its flows, in order. journaled: Mutex, String, serde_json::Value)>>, @@ -102,11 +103,13 @@ impl FlowRunner for Script { } fn prepare(&self, task: &TaskId, _constraints: &TaskConstraints) -> super::PrepareFuture { + self.events.lock().unwrap().push("prepare"); self.prepared.lock().unwrap().push(task.clone()); Box::pin(async {}) } fn open_page(&self, task: &TaskId, url: &str) -> super::PrepareFuture { + self.events.lock().unwrap().push("open_page"); self.opened .lock() .unwrap() diff --git a/crates/tinycomputer-engine/src/task/task_tests/page_tests.rs b/crates/tinycomputer-engine/src/task/task_tests/page_tests.rs index cc11ca2b..8b1c5e8a 100644 --- a/crates/tinycomputer-engine/src/task/task_tests/page_tests.rs +++ b/crates/tinycomputer-engine/src/task/task_tests/page_tests.rs @@ -1,22 +1,27 @@ -//! Tests for which web page a task's own words name. +//! Tests for which web page a task's own words send its browser to. use super::super::page::named_page; #[test] -fn a_task_names_the_one_address_it_writes_out() { +fn a_task_names_the_one_address_it_sends_its_browser_to() { assert_eq!( named_page("Go to https://blinkit.com. Search for Maggi.").as_deref(), Some("https://blinkit.com") ); - // Kept as written, a path and query too, without what closes the sentence. + // As OpenHuman writes the address a task starts at. + assert_eq!( + named_page("Order Maggi. Start at https://www.amazon.in.").as_deref(), + Some("https://www.amazon.in") + ); + // Kept as written, a path too, without what closes the sentence. assert_eq!( named_page("Book a cab on https://www.uber.com/global/en/price-estimate/, then stop.") .as_deref(), Some("https://www.uber.com/global/en/price-estimate/") ); assert_eq!( - named_page("Open [the shop](http://shop.test/?q=boots)").as_deref(), - Some("http://shop.test/?q=boots") + named_page("Visit: (http://shop.test/deals)").as_deref(), + Some("http://shop.test/deals") ); // Written twice, with a trailing slash or without, it is one page. assert_eq!( @@ -25,6 +30,29 @@ fn a_task_names_the_one_address_it_writes_out() { ); } +#[test] +fn an_address_only_mentioned_or_one_a_load_could_spend_is_not_loaded() { + // Checked, passed on, or read, not gone to. + assert_eq!( + named_page("Check https://paypa1-secure.test/login on VirusTotal"), + None + ); + assert_eq!(named_page("Post https://shop.test in the team chat"), None); + // A query or a fragment can hold a token. + assert_eq!( + named_page("Go to https://acct.test/confirm?token=f00d to finish"), + None + ); + assert_eq!(named_page("Open https://app.test/#/cart"), None); + // "on" and "at" only as words of their own. + assert_eq!(named_page("Reason https://shop.test"), None); + assert_eq!( + named_page("Summarize the page at https://news.test?").as_deref(), + Some("https://news.test"), + "read there, so gone to" + ); +} + #[test] fn a_task_naming_no_page_or_several_names_none() { assert_eq!(named_page("Order Maggi on Blinkit"), None); @@ -32,6 +60,11 @@ fn a_task_naming_no_page_or_several_names_none() { named_page("Compare https://a.test with https://b.test"), None ); + assert_eq!( + named_page("Go to https://shop.test and paste https://acct.test/confirm"), + None, + "another address, mentioned, leaves no one page" + ); assert_eq!(named_page("Type https:// into the box"), None); assert_eq!(named_page(""), None); } diff --git a/crates/tinycomputer-engine/src/task/task_tests/plan_tests.rs b/crates/tinycomputer-engine/src/task/task_tests/plan_tests.rs index 67b7e2ad..63f39a4a 100644 --- a/crates/tinycomputer-engine/src/task/task_tests/plan_tests.rs +++ b/crates/tinycomputer-engine/src/task/task_tests/plan_tests.rs @@ -354,8 +354,18 @@ async fn a_browser_only_task_loads_the_page_it_names_while_it_is_planned() { ); assert_eq!( *script.prepared.lock().unwrap(), - std::slice::from_ref(&started.id), - "after its browser" + std::slice::from_ref(&started.id) + ); + assert_eq!( + script + .events + .lock() + .unwrap() + .iter() + .filter(|event| matches!(**event, "prepare" | "open_page")) + .collect::>(), + [&"prepare", &"open_page"], + "in the browser opened for it" ); // Two pages leave no one to start on; a task that may use the desktop diff --git a/crates/tinycomputer/src/tinybus_module/runner.rs b/crates/tinycomputer/src/tinybus_module/runner.rs index a3089cdf..0e472375 100644 --- a/crates/tinycomputer/src/tinybus_module/runner.rs +++ b/crates/tinycomputer/src/tinybus_module/runner.rs @@ -173,12 +173,29 @@ impl FlowRunner for WorkspaceRunner { }) }) .flatten(); + // Journaled with the task's flows (see `run`): the plan's outcome + // waits for this load. + let journal = self + .jev + .as_ref() + .filter(|runtime| runtime.journaling()) + .map(|runtime| runtime.journaled_as(&format!("task-{task}"))); let url = url.to_owned(); Box::pin(async move { if let Some(browser) = browser { + let started = std::time::Instant::now(); // A page that will not load fails again, and is reported, // at the step that browses there. - let _loaded = tokio::task::spawn_blocking(move || browser.open_at(&url)).await; + let loaded = tokio::task::spawn_blocking(move || browser.open_at(&url)) + .await + .unwrap_or(false); + if let Some(journal) = journal { + let wall_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX); + journal.journal_event( + "open_page", + || serde_json::json!({"wall_ms": wall_ms, "loaded": loaded}), + ); + } } }) } diff --git a/crates/tinycomputer/src/tinybus_module/tinybus_module_tests/tasks_tests.rs b/crates/tinycomputer/src/tinybus_module/tinybus_module_tests/tasks_tests.rs index 455be3a3..d403bdf7 100644 --- a/crates/tinycomputer/src/tinybus_module/tinybus_module_tests/tasks_tests.rs +++ b/crates/tinycomputer/src/tinybus_module/tinybus_module_tests/tasks_tests.rs @@ -305,8 +305,15 @@ async fn the_runner_loads_the_page_a_browser_task_names_in_its_early_browser() { std::sync::Arc::new(super::browser_tests::ScriptedLauncher(sent.clone())), scratch.clone(), )); - let mut runner = - crate::tinybus_module::runner::WorkspaceRunner::new(crate::Desktop::new(), None, browser); + // Journaled, with the task's flows. + let jev = tinycomputer_engine::JevRuntime::sage("test-key", false) + .unwrap() + .with_journal(scratch.join("journal")); + let mut runner = crate::tinybus_module::runner::WorkspaceRunner::new( + crate::Desktop::new(), + Some(jev), + browser, + ); let navigated = |sent: &std::sync::Arc>>| { sent.lock() .unwrap() @@ -323,6 +330,20 @@ async fn the_runner_loads_the_page_a_browser_task_names_in_its_early_browser() { runner.prepare(&task, &browser_only).await; runner.open_page(&task, "https://example.com").await; assert_eq!(navigated(&sent), [json!("https://example.com")]); + let journal = std::fs::read_to_string( + scratch + .join("journal") + .join("task-t-1") + .join("journal.jsonl"), + ) + .unwrap(); + let opened = journal + .lines() + .map(|line| serde_json::from_str::(line).unwrap()) + .find(|event| event["event"] == "open_page") + .unwrap(); + assert_eq!(opened["loaded"], true, "{opened}"); + assert!(opened["wall_ms"].is_u64(), "{opened}"); // A task whose browser was never made ready loads nothing, nor does one // once prelaunch is off. diff --git a/docs/crates/tinycomputer-browser/surface.md b/docs/crates/tinycomputer-browser/surface.md index ed24ab8b..baeb3ebf 100644 --- a/docs/crates/tinycomputer-browser/surface.md +++ b/docs/crates/tinycomputer-browser/surface.md @@ -136,11 +136,13 @@ after a launch or Escape, skips the network wait under `Settle::Prompt`; both are described in [interacting.md](interacting.md#scrolling-and-waiting). `BrowserSurface::open_at(url)` opens the session early, as `open` does, and -loads `url` in it: the page a task names, loaded while its plan is drafted. -Until the page is first read (`observe`) or another address is loaded, a -`navigate` to the same place finds it already there and loads nothing, -answering with the page's address and title as they stand; "the same place" -is `tabs::place`'s, which ignores the scheme, a leading `www.`, the fragment +loads `url` in it: the page a task names, loaded while its plan is drafted, +waiting for its `load` event at most 10 s (`EARLY_LOAD_MS`, beyond nine in +ten live first pages). Until the page is first read (`observe`) or another +address is loaded, a `navigate` to the same place, in the same session, +finds it already there and loads nothing, answering with the page's address +and title as they stand (read within `READ_TIMEOUT`); "the same place" is +`tabs::place`'s, which ignores the scheme, a leading `www.`, the fragment and a trailing slash. A page not yet drawn, or a surface let go meanwhile, is loaded as asked. diff --git a/docs/technical/jev-journal.md b/docs/technical/jev-journal.md index 2bac8879..45b5f856 100644 --- a/docs/technical/jev-journal.md +++ b/docs/technical/jev-journal.md @@ -80,6 +80,7 @@ has `""`, and goal and intent runs carry their goal or intent text. | `denoise` | a `do` step's screen oscillates | `step`, `oscillation` (the presses banned) | | `step` | a flow step ends | `step`, `kind`, `text`, `outcome`, `note`, `turns`, `jev_calls`, `actions`, `loops`, `confidence`, `wall_ms` | | `end` | a flow run ends | `stop`, `wall_ms`, `actions`, `metrics`, `learned` | +| `open_page` | a browser-only task's named page loads in its early browser while the plan is drafted (`FlowRunner::open_page`); the plan's outcome waits for it | `wall_ms`, `loaded` | | `plan` | the planner drafts a task's flow (`PlanTask`, or `StartTask` with a task) | `wall_ms`, `calls` (model calls, repairs included; a call tried again after a passing failure counts once, its waits in `wall_ms`), `sent_bytes` (the first call's text), `model`, `ok`; on success `steps`, `questions`; on failure `error` | | `rescue` | the rescuer answers for a failed step | `step` (from 1), `attempt`, `limit`, `wall_ms`, `calls` and `sent_bytes` (null when it gave no answer in time), `outcome` (`guided`, `gave_up`, `error`, `timeout`), `steps` (guidance steps), `covers`, `model` | | `resume` | a person answers a paused task (`ContinueTask`) | `state` it waited at (`needs_input`, `needs_approval`, `needs_human`), `waited_ms` since it first asked | diff --git a/docs/technical/tasks.md b/docs/technical/tasks.md index ee8549e6..63043e54 100644 --- a/docs/technical/tasks.md +++ b/docs/technical/tasks.md @@ -278,17 +278,20 @@ A `StartTask` with a `task` and no `flow` plans inside the task. When the task may run only on the browser, its runner is asked to get the browser ready meanwhile (`FlowRunner::prepare`); unless the module's `browser.prelaunch` is off, the session opens while the plan is drafted, so -the first step does not wait for Chrome to start. When the task's text -names one web address, written out with `https://` or `http://`, the page -loads in it meanwhile (`FlowRunner::open_page`), in the same wait: a first -step that browses there finds it loaded, and loads nothing, as long as -nothing has read the page first. Over 57 live plans, 56 started by browsing -the page their task named, and that first page took 1.9 s to load (p90 -6.4 s) and 1.1 s to settle; a plan takes 10–30 s. A task naming several -addresses, or none, loads none. A plan that asks for -values lets the browser go while the task waits (`needs_input`), and the -run that follows opens it again; a task cancelled while its browser was -still opening is left holding none. +the first step does not wait for Chrome to start. When the task's text sends +its browser to one web address, written out with `https://` or `http://` +right after words such as "go to", "open", "visit", "start at" or "on", and +carrying no query or fragment, the page loads in it meanwhile +(`FlowRunner::open_page`, journaled as `open_page`), in the same wait, for +at most 10 s: a first step that browses there finds it loaded, and loads +nothing, as long as nothing has read the page first. Over 57 live plans, 56 +started by browsing the page their task named, and that first page took 1.9 +s to load (p90 6.4 s) and 1.1 s to settle; a plan takes 10–30 s. An address +the task only mentions (one to check, read out, or pass on), one whose query +could hold a token a load would spend, and a task naming several addresses +or none load nothing. A plan that asks for values lets the browser go while +the task waits (`needs_input`), and the run that follows opens it again; a +task cancelled while its browser was still opening is left holding none. Every task planned this way also has its runner warm Jev while the plan is drafted (`FlowRunner::warm`): one small evaluation for each call its first From 7bc42399764a1e6400411b9a6c760826d682654a Mon Sep 17 00:00:00 2001 From: Shanu Date: Thu, 8 Oct 2026 17:04:02 +0530 Subject: [PATCH 44/44] Check that settling briefly settles in full unless a surface says otherwise The desktop surface relies on Surface::settle_briefly's default running its own settle after a launch or Escape; a test now counts that it does. The settle rustdoc, the do loop's page and the surfaces page no longer say that every action settles in full. --- crates/tinycomputer-core/src/surface/mod.rs | 9 +-- .../surface/surface_tests/delivery_tests.rs | 58 ++++++++++++++++++- .../tinycomputer-core/surfaces-and-screens.md | 2 +- .../tinycomputer-engine/flow/the-do-loop.md | 3 +- 4 files changed, 64 insertions(+), 8 deletions(-) diff --git a/crates/tinycomputer-core/src/surface/mod.rs b/crates/tinycomputer-core/src/surface/mod.rs index ee54a9f4..6fe04a97 100644 --- a/crates/tinycomputer-core/src/surface/mod.rs +++ b/crates/tinycomputer-core/src/surface/mod.rs @@ -68,10 +68,11 @@ pub trait Surface: Clone + Send + 'static { /// Launches `app`, or brings it forward when it is already running. fn launch(&self, app: &str) -> DesktopResponse; - /// Gives the application a moment to finish reacting: after every action, - /// before the next look, and before a value is read back — a page that - /// closes a banner a beat after the click, or a token field turning an - /// address into a token. + /// Gives the application a moment to finish reacting: after an action + /// (one that fetches nothing settles briefly instead, + /// [`Surface::settle_briefly`]), before the next look, and before a value + /// is read back — a page that closes a banner a beat after the click, or + /// a token field turning an address into a token. fn settle(&self) {} /// Settles after an action that fetches nothing — launching what is diff --git a/crates/tinycomputer-core/src/surface/surface_tests/delivery_tests.rs b/crates/tinycomputer-core/src/surface/surface_tests/delivery_tests.rs index 2bdc56cf..c2de727a 100644 --- a/crates/tinycomputer-core/src/surface/surface_tests/delivery_tests.rs +++ b/crates/tinycomputer-core/src/surface/surface_tests/delivery_tests.rs @@ -75,6 +75,54 @@ impl Surface for TextBackend { } } +/// [`TextBackend`], counting how often it settles in full. +#[derive(Clone, Default)] +struct Settling { + text: TextBackend, + settled: Arc, +} + +impl Surface for Settling { + fn observe( + &self, + app: &str, + root: Option<&str>, + depth: Depth, + ) -> Result> { + self.text.observe(app, root, depth) + } + + fn execute( + &self, + operation: JevOperation, + target: Option, + text: Option, + ) -> DesktopResponse { + self.text.execute(operation, target, text) + } + + fn read_value(&self, target: &Candidate) -> Option { + self.text.read_value(target) + } + + fn paste(&self, app: &str, target: &Candidate, text: &str) -> DesktopResponse { + self.text.paste(app, target, text) + } + + fn press(&self, app: &str, combo: &str) -> DesktopResponse { + self.text.press(app, combo) + } + + fn launch(&self, app: &str) -> DesktopResponse { + self.text.launch(app) + } + + fn settle(&self) { + self.settled + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + } +} + fn field() -> Candidate { Candidate { ref_id: "@s:e1".to_owned(), @@ -189,8 +237,14 @@ fn text_that_never_arrives_is_reported_as_not_delivered() { #[test] fn a_surface_settles_instantly_and_has_no_addresses_unless_it_says_otherwise() { Surface::settle(&TextBackend::default()); - // Settling briefly is settling, unless the surface can tell them apart. - Surface::settle_briefly(&TextBackend::default()); + // Settling briefly is settling in full, unless the surface can tell them + // apart: the desktop's own settle runs after a launch or Escape. + let settling = Settling::default(); + settling.settle_briefly(); + assert_eq!( + settling.settled.load(std::sync::atomic::Ordering::SeqCst), + 1 + ); assert!( Surface::await_change(&TextBackend::default(), 1_000), "one that cannot watch pauses and says it may have changed" diff --git a/docs/crates/tinycomputer-core/surfaces-and-screens.md b/docs/crates/tinycomputer-core/surfaces-and-screens.md index b99a49ab..8529bebb 100644 --- a/docs/crates/tinycomputer-core/surfaces-and-screens.md +++ b/docs/crates/tinycomputer-core/surfaces-and-screens.md @@ -30,7 +30,7 @@ leaves them as the default refusal. There is also `settle()`, which does nothing by default. A surface overrides it to give an application a moment to react: closing a banner a beat after a click, or turning a typed address into a token in an autocomplete field. The -engine calls it after every action, before the next observation, and before +engine calls it after an action, before the next observation, and before reading a value back, so a surface's own idea of "how long is a moment" stays in one place rather than being copied into every caller. After an action that fetches nothing (a launch, Escape) the engine calls `settle_briefly()` diff --git a/docs/crates/tinycomputer-engine/flow/the-do-loop.md b/docs/crates/tinycomputer-engine/flow/the-do-loop.md index 0fec516f..10f1c811 100644 --- a/docs/crates/tinycomputer-engine/flow/the-do-loop.md +++ b/docs/crates/tinycomputer-engine/flow/the-do-loop.md @@ -138,7 +138,8 @@ answer must fail closed rather than guess. After any action the backend reports as successful, the runtime waits for the surface to settle (on the browser, until the requests that change the -page end and it goes still; a short pause on the desktop) before looking +page end and it goes still, or only until it is still after a launch or +Escape, which fetch nothing; a short pause on the desktop) before looking again, so the next turn's screen reflects what the action actually did rather than the moment right before it took effect.