From f1c3a624ed2509a36bd87636b99de96aeb54bb62 Mon Sep 17 00:00:00 2001 From: Alfa_beda <95350284+DroBov1Tya@users.noreply.github.com> Date: Tue, 17 Nov 2026 15:02:53 +0300 Subject: [PATCH] computer: run several actions in one call A known sequence (click a field, type, press enter, look) cost one model round trip per action. computer_actions runs up to 20 click, type_text, press_keys, scroll and screenshot steps in order in one call. A batch must never get less scrutiny than its steps one by one: - every step is checked before anything is asked or done; - a step whose tool is switched off in the mode, or that a before_tool hook denies (run per step with its own tool and arguments), refuses the whole batch; - one approval lists every step by its batch number, steps a session grant covers marked; mixed kinds get no session grant; the arbiter judges each step alone; - the steps travel structured in ApprovalRequest (protocol 11) and in the request text, so every client and chat platform shows each one; the terminal pages long batches and lets a yes count only after all steps were shown. A chat platform that would cut a long request gets no yes for it: it can only deny there, the app allows it. The single tools now build the same checked step as the batch, so the two cannot ask or act differently. Every click in a batch reads the picture taken before it. press_keys refuses control characters and shows keys by their normal names. Not verified: - a live batch on a real macOS desktop (AXPress clicks, typing, keys, screenshots inside a batch, coordinates after an inner screenshot); - the same on Linux X11 and Wayland, and a Linux build; - the approval drawn in the terminal, desktop and browser clients and in Telegram with a 20-step batch. --- changes/computer-actions.md | 9 + docs/computer-use.md | 39 +++ docs/plugins.md | 6 + projects/clients/browser/src/chat/approval.rs | 15 +- .../clients/browser/src/chat/data/tests.rs | 1 + projects/clients/browser/style.css | 1 + .../clients/desktop/src/dev/screenshot.rs | 1 + .../desktop/src/pages/chat/approval.rs | 20 ++ projects/clients/terminal/src/app/mod.rs | 2 + .../terminal/src/app/thread/approval.rs | 91 +++++- .../terminal/src/ui/thread/input/approval.rs | 96 +++++- .../terminal/src/ui/thread/input/mod.rs | 4 +- projects/core/src/plugins/hooks.rs | 67 ++++ projects/core/src/plugins/platform/session.rs | 68 +++- .../core/src/runtime/agent/agent/calls.rs | 9 +- projects/core/src/runtime/thread/launcher.rs | 5 +- projects/core/src/tools/approvals.rs | 134 +++++++- projects/core/src/tools/computer/batch.rs | 199 ++++++++++++ .../core/src/tools/computer/batch_tests.rs | 291 ++++++++++++++++++ projects/core/src/tools/computer/click.rs | 112 +++---- projects/core/src/tools/computer/mod.rs | 8 +- .../core/src/tools/computer/press_keys.rs | 43 ++- projects/core/src/tools/computer/screen.rs | 58 +++- .../core/src/tools/computer/screenshot.rs | 82 ++--- projects/core/src/tools/computer/scroll.rs | 65 ++-- projects/core/src/tools/computer/step.rs | 38 +++ projects/core/src/tools/computer/type_text.rs | 60 ++-- projects/core/src/tools/mod.rs | 2 +- projects/core/src/tools/registry.rs | 94 +++++- projects/core/src/tools/tool.rs | 48 ++- projects/protocol/src/client/hello.rs | 2 +- projects/protocol/src/thread/event.rs | 1 + projects/protocol/src/tools/approval.rs | 48 +++ projects/protocol/src/tools/mod.rs | 2 +- 34 files changed, 1515 insertions(+), 206 deletions(-) create mode 100644 changes/computer-actions.md create mode 100644 projects/core/src/tools/computer/batch.rs create mode 100644 projects/core/src/tools/computer/batch_tests.rs create mode 100644 projects/core/src/tools/computer/step.rs diff --git a/changes/computer-actions.md b/changes/computer-actions.md new file mode 100644 index 00000000..3ff95295 --- /dev/null +++ b/changes/computer-actions.md @@ -0,0 +1,9 @@ +Added +Computer use: `computer_actions` runs up to 20 clicks, typing, keys, scrolls and +screenshots in one call, checked and asked for together before the first runs, with +every click read in the picture taken before the batch. Every client and chat platform +lists each step of the request; tool switches, plugin hooks and the arbiter judge each +step as if it were called alone. `press_keys` now refuses line breaks and control +characters in the keys and shows them by their normal names. A chat platform +gets a request whole or cannot allow it: one too long for its message can only be +denied there and is answered in the app. diff --git a/docs/computer-use.md b/docs/computer-use.md index 128ecb8d..2b1144f9 100644 --- a/docs/computer-use.md +++ b/docs/computer-use.md @@ -43,6 +43,45 @@ the session does not allow it. ``` `type_text` and `press_keys` go to whatever has the keyboard focus and ask every time. +Keys are a key name with modifiers joined by `+`; only plain spaces may pad them, and +the request shows them by their normal names, such as `cmd+shift+t`. + +## Several actions in one call + +`computer_actions` runs up to 20 steps of `click`, `type_text`, `press_keys`, `scroll` +and `screenshot` in order, each with the arguments it takes alone, at most four +screenshots among them: + +```json +{"steps": [ + {"tool": "click", "arguments": {"x": 320, "y": 180}}, + {"tool": "type_text", "arguments": {"text": "hello"}}, + {"tool": "press_keys", "arguments": {"keys": "enter"}}, + {"tool": "screenshot", "arguments": {}} +]} +``` + +Every step is checked before anything is asked or done; a wrong one names its number +and nothing runs. A step whose tool is switched off in the mode, or that a plugin's +`before_tool` hook denies, refuses the whole batch the same way. + +The batch is approved before any step runs, in one request that lists every step with +its number, those a session grant covers marked "already allowed". So typing and keys +in a batch are approved before the user sees where the focus lands: approve one only +when the steps before it say where that is, or ask for the steps one by one. Steps of +one kind can be allowed for the session as that kind; mixed kinds are asked for this +time only, under a neutral heading. In arbiter mode each step is judged on its own, +and one no refuses the batch. So a batch is never asked for less than its steps one +by one. The terminal grows the request to show up to twelve rows of steps and pages +the rest; a yes there pages on until every step has been shown, and counts only a +moment after the last ones appear. The desktop scrolls a long list inside the request; +a chat platform such as Telegram gets every step in its message, and a request too +long for one message (over 3500 characters) is not offered for a yes there: it can +only be denied there, and is allowed in the app. + +Every x, y is in the picture taken before the batch, even after a screenshot inside +it; after the batch the newest picture counts. The batch stops at the first failing +step and says which ran. ## Linux diff --git a/docs/plugins.md b/docs/plugins.md index f9646c51..82105eb1 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -102,6 +102,12 @@ A hook reads one JSON object on stdin: - `{"context": "…"}` adds a note the model reads: to the tool's result after a tool, or to the thread for the other events. +A call that runs other tools inside it, such as `computer_actions`, is checked first +as itself and then once per step, with that step's `tool` and `arguments` and +`"within": {"tool": "computer_actions", "step": 2}`, so a hook for `type_text` sees +typing in a batch too. A deny from any step refuses the whole call before anything is +asked or done. `after_tool` runs once, for the batch, with every step's result in it. + Printing nothing allows the action. Hooks run in the order the plugins are named. A `before_tool` hook that fails, prints something other than JSON or runs past its timeout denies the call: a check that could not run has not allowed anything. Other diff --git a/projects/clients/browser/src/chat/approval.rs b/projects/clients/browser/src/chat/approval.rs index 4f6581b9..97c82196 100644 --- a/projects/clients/browser/src/chat/approval.rs +++ b/projects/clients/browser/src/chat/approval.rs @@ -23,6 +23,8 @@ pub fn Approvals(chat: Chat) -> impl IntoView { fn ApprovalCard(request: ApprovalRequest) -> impl IntoView { let app = state(); let id = request.id; + let lines: Vec = request.steps.iter().map(|step| step.line()).collect(); + let session = request.session.is_some(); let answer = move |decision: Decision| { run(api::approve(id, decision), move |_| { // The core closes the request itself; every client sees that event. @@ -31,10 +33,19 @@ fn ApprovalCard(request: ApprovalRequest) -> impl IntoView { }; view! {
-
{request.request.describe("The agent")}
+
{request.title("The agent")}
+ // A batch lists every step, numbered: the yes covers all of them. + {(!lines.is_empty()).then(|| view! { +
+ {lines.into_iter().map(|line| view! {
{line}
}).collect_view()} +
+ })}
- + // Only what the core can keep for the session is offered, as the other clients do. + {session.then(|| view! { + + })}
diff --git a/projects/clients/browser/src/chat/data/tests.rs b/projects/clients/browser/src/chat/data/tests.rs index 9edacad0..26e89271 100644 --- a/projects/clients/browser/src/chat/data/tests.rs +++ b/projects/clients/browser/src/chat/data/tests.rs @@ -54,6 +54,7 @@ fn an_approval_shows_until_it_is_closed() { command: "rm x".into(), }, session: None, + steps: Vec::new(), }; chat.apply(ThreadEvent::Approval(request)); assert_eq!(chat.approvals.len(), 1); diff --git a/projects/clients/browser/style.css b/projects/clients/browser/style.css index aaaf89f9..e8bb8ee6 100644 --- a/projects/clients/browser/style.css +++ b/projects/clients/browser/style.css @@ -154,6 +154,7 @@ input:is([type=radio], [type=checkbox]) { accent-color: var(--accent); } .card { margin: 10px auto; } .card img { max-width: 100%; border-radius: 10px; display: block; } .approvals, .question { max-width: 820px; margin: 0 auto; width: calc(100% - 40px); } .approval, .question { border: 1px solid var(--accent); border-radius: 12px; padding: 10px 14px; margin-bottom: 8px; background: var(--surface); } +.approval-steps { margin: 6px 0 0 12px; display: flex; flex-direction: column; gap: 2px; overflow-wrap: anywhere; } .approval-actions, .row-actions { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; margin: 8px 0; } .option { display: grid; grid-template-columns: var(--s4) minmax(0, 1fr); align-items: start; gap: var(--s3); diff --git a/projects/clients/desktop/src/dev/screenshot.rs b/projects/clients/desktop/src/dev/screenshot.rs index 17d84d16..b5ffec07 100644 --- a/projects/clients/desktop/src/dev/screenshot.rs +++ b/projects/clients/desktop/src/dev/screenshot.rs @@ -730,6 +730,7 @@ fn act( command: "cargo test --workspace".into(), }, session: Some("cargo".into()), + steps: Vec::new(), }), }; if let Some(chat) = ws.shown.clone().and_then(|id| ws.chats.get_mut(&id)) { diff --git a/projects/clients/desktop/src/pages/chat/approval.rs b/projects/clients/desktop/src/pages/chat/approval.rs index 102070de..a3ebfc53 100644 --- a/projects/clients/desktop/src/pages/chat/approval.rs +++ b/projects/clients/desktop/src/pages/chat/approval.rs @@ -56,7 +56,27 @@ pub fn render(ws: &Workspace, cx: &mut Context) -> Div { .on_click(cx.listener(move |ws, _, _, cx| ws.approve(id, Decision::Deny, cx))), ); // A command is shown as code, in full: it came from the model and is about to run. + // A batch shows every step, one row each: the yes covers all of them. let what = match &request.request { + // A long batch scrolls inside the card, so the buttons stay in view. + _ if !request.steps.is_empty() => { + let steps = request.steps.iter().fold( + div() + .id(SharedString::from(format!("steps-{id}"))) + .max_h(px(240.)) + .overflow_y_scroll() + .flex() + .flex_col() + .gap_1(), + |list, step| list.child(div().pl_3().child(step.line())), + ); + div() + .flex() + .flex_col() + .gap_1() + .child(request.title("The agent")) + .child(steps) + } Request::Command { command } => { let mono = crate::ui::theme::prefs(cx) .mono_font diff --git a/projects/clients/terminal/src/app/mod.rs b/projects/clients/terminal/src/app/mod.rs index f1ac2b32..84a68a64 100644 --- a/projects/clients/terminal/src/app/mod.rs +++ b/projects/clients/terminal/src/app/mod.rs @@ -32,6 +32,8 @@ pub use panels::{ pub use panels::{Choice, Confirm, Step, Workshop, choices}; pub use state::{App, compact_number}; pub use tabs::{Tab, TabHit, TabMove}; +#[cfg(test)] +pub(crate) use thread::approval::tests::batch as approval_batch; pub use thread::approval::{ApprovalHit, Asked}; pub use thread::goal::{GoalButton, GoalHit, buttons as goal_buttons}; pub use thread::poll::{Poll, PollHit, PollTarget}; diff --git a/projects/clients/terminal/src/app/thread/approval.rs b/projects/clients/terminal/src/app/thread/approval.rs index 8e4b37f6..8378dcf3 100644 --- a/projects/clients/terminal/src/app/thread/approval.rs +++ b/projects/clients/terminal/src/app/thread/approval.rs @@ -2,6 +2,8 @@ //! everything else, the oldest first. Enter answers with the row under the cursor, Esc //! denies; the core closes a request whose call is gone. +use std::time::{Duration, Instant}; + use crossterm::event::{KeyCode, KeyEvent}; use dusk_protocol::tools::{ApprovalRequest, Decision, Request}; @@ -12,8 +14,19 @@ pub struct Asked { /// The agent that asks; `None` for the chat itself. pub agent: Option, pub cursor: usize, + /// The first row of a batch's steps in view, when they do not all fit. + pub step_scroll: usize, + /// Step rows in view on the last drawn frame. + pub step_page: usize, + /// When every step of a batch had first been on screen. Until then a yes pages on + /// instead of answering, so no step is allowed unseen. + pub steps_seen_at: Option, } +/// After the last steps of a batch first show, a yes waits this long, so a held or +/// repeated Enter that paged through does not also allow what it just showed. +pub const SETTLE: Duration = Duration::from_millis(600); + impl Asked { /// The answers offered, in order; the session one only when the core can keep it. pub fn choices(&self) -> Vec<(Decision, String)> { @@ -34,10 +47,20 @@ impl Asked { choices } + /// A yes to a batch waits until every step has been on screen for [`SETTLE`]; a no + /// and a single request never wait. + pub fn answers_now(&self, decision: Decision, now: Instant) -> bool { + decision == Decision::Deny + || self.request.steps.is_empty() + || self + .steps_seen_at + .is_some_and(|at| now.saturating_duration_since(at) >= SETTLE) + } + /// What the tool asks for, in a line. pub fn title(&self) -> String { let who = self.agent.as_deref().unwrap_or("the agent"); - self.request.request.describe(who) + self.request.title(who) } } @@ -55,6 +78,9 @@ impl App { request, agent, cursor: 0, + step_scroll: 0, + step_page: 0, + steps_seen_at: None, }); } @@ -70,6 +96,10 @@ impl App { match key.code { KeyCode::Up => asked.cursor = asked.cursor.saturating_sub(1), KeyCode::Down => asked.cursor = (asked.cursor + 1).min(last), + KeyCode::PageDown => asked.step_scroll += asked.step_page.max(1), + KeyCode::PageUp => { + asked.step_scroll = asked.step_scroll.saturating_sub(asked.step_page.max(1)) + } KeyCode::Enter => return self.answer_approval(None), KeyCode::Esc => return self.answer_approval(Some(last)), _ => {} @@ -82,12 +112,69 @@ impl App { if self.approvals.is_empty() { return Action::None; } - let asked = self.approvals.remove(0); + let asked = &mut self.approvals[0]; let choices = asked.choices(); let at = choice.unwrap_or(asked.cursor).min(choices.len() - 1); + if !asked.answers_now(choices[at].0, Instant::now()) { + // Still unseen steps: show them. Just shown: wait, as if not pressed. + if asked.steps_seen_at.is_none() { + asked.step_scroll += asked.step_page.max(1); + } + return Action::None; + } + let asked = self.approvals.remove(0); Action::Approve { id: asked.request.id, decision: choices[at].0, } } } + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use dusk_protocol::tools::ApprovalStep; + + /// A batch of `count` keyboard steps, each typing `text`. + pub(crate) fn batch(count: usize, text: &str) -> Asked { + let steps: Vec = (1..=count) + .map(|number| ApprovalStep { + number, + request: Request::Keyboard { + action: format!("type \"{text}\""), + }, + allowed: false, + }) + .collect(); + Asked { + request: ApprovalRequest { + id: 1, + request: Request::Keyboard { + action: format!("{count} computer actions"), + }, + session: None, + steps, + }, + agent: None, + cursor: 0, + step_scroll: 0, + step_page: 0, + steps_seen_at: None, + } + } + + #[test] + fn a_batch_is_allowed_only_a_while_after_every_step_was_shown() { + let now = Instant::now(); + let mut asked = batch(20, "hi"); + assert!(!asked.answers_now(Decision::Once, now)); + assert!(!asked.answers_now(Decision::Session, now)); + assert!(asked.answers_now(Decision::Deny, now)); + asked.steps_seen_at = Some(now); + // A held Enter that showed the last page does not also allow it. + assert!(!asked.answers_now(Decision::Once, now + SETTLE / 2)); + assert!(asked.answers_now(Decision::Deny, now)); + assert!(asked.answers_now(Decision::Once, now + SETTLE)); + assert!(batch(0, "").answers_now(Decision::Once, now)); + } +} diff --git a/projects/clients/terminal/src/ui/thread/input/approval.rs b/projects/clients/terminal/src/ui/thread/input/approval.rs index c33bb38f..a2d5b6cd 100644 --- a/projects/clients/terminal/src/ui/thread/input/approval.rs +++ b/projects/clients/terminal/src/ui/thread/input/approval.rs @@ -1,7 +1,7 @@ //! The question for a tool's yes, shown over the input. use super::InputView; -use crate::app::{App, ApprovalHit}; +use crate::app::{App, ApprovalHit, Asked}; use crate::ui::thread::{ theme::{ACCENT, DUSK, NEGATIVE, TEXT, bold, dim, fg}, wrap, @@ -15,11 +15,40 @@ use ratatui::{ widgets::{Block, BorderType, Clear, Paragraph}, }; -/// The border, the request, its choices and the hint. -pub(super) fn approval_rows(app: &App) -> u16 { - app.approvals - .first() - .map_or(0, |asked| asked.choices().len() as u16 + 4) +/// Rows of a batch's steps the box grows to; more are paged, never cut. +const MAX_STEP_ROWS: usize = 12; + +/// Steps sit under the title, indented past its leading space. +const STEP_INDENT: &str = " "; + +/// The border, the request, a batch's steps, its choices and the hint. +pub(super) fn approval_rows(app: &App, width: u16) -> u16 { + app.approvals.first().map_or(0, |asked| { + let steps = step_lines(asked, width.saturating_sub(2) as usize).len(); + let paged = steps > MAX_STEP_ROWS; + let shown = steps.min(MAX_STEP_ROWS) + usize::from(paged); + (asked.choices().len() + 4 + shown) as u16 + }) +} + +/// Every step of a batch, each wrapped whole to the box: the user answers for all of +/// them, so none is clipped. Control characters are drawn as `?`. +fn step_lines(asked: &Asked, width: usize) -> Vec { + let room = width.saturating_sub(STEP_INDENT.len()).max(1); + asked + .request + .steps + .iter() + .flat_map(|step| { + let line: String = step + .line() + .chars() + .map(|c| if c.is_control() { '?' } else { c }) + .collect(); + wrap::chars(&line, room) + }) + .map(|row| format!("{STEP_INDENT}{row}")) + .collect() } pub(super) const SEND: &str = " send "; @@ -53,6 +82,34 @@ impl InputView<'_> { wrap::clip(&format!(" {}", asked.title()), width), bold(TEXT), ))]; + let steps = step_lines(asked, width); + let choices = asked.choices().len(); + // What is left for steps once the title, choices and hint have their rows. + let room = (inner.height as usize).saturating_sub(choices + 2); + let paged = steps.len() > room; + let page = match paged { + true => room.saturating_sub(1).max(1), + false => steps.len(), + }; + let first = asked.step_scroll.min(steps.len().saturating_sub(page)); + let shown = steps.iter().skip(first).take(page); + lines.extend(shown.map(|row| Line::from(Span::raw(row.clone())))); + if paged { + let last = (first + page).min(steps.len()); + lines.push(Line::from(Span::styled( + wrap::clip( + &format!( + " rows {}-{last} of {} pgup/pgdn a yes shows the rest first", + first + 1, + steps.len() + ), + width, + ), + dim(), + ))); + } + let step_rows = lines.len() as u16 - 1; + let seen = first + page >= steps.len(); for (row, (decision, label)) in asked.choices().into_iter().enumerate() { let here = asked.cursor == row; let pointer = if here { " \u{203A} " } else { " " }; @@ -61,7 +118,7 @@ impl InputView<'_> { (true, _) => bold(ACCENT), (false, _) => Style::new(), }; - let y = inner.y + 1 + row as u16; + let y = inner.y + 1 + step_rows + row as u16; if y < inner.bottom() { self.app.approval_hits.push(ApprovalHit { row: y, @@ -80,5 +137,30 @@ impl InputView<'_> { dim(), ))); frame.render_widget(Paragraph::new(lines), inner); + if let Some(asked) = self.app.approvals.first_mut() { + asked.step_scroll = first; + asked.step_page = page; + if seen && asked.steps_seen_at.is_none() { + asked.steps_seen_at = Some(std::time::Instant::now()); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::app::approval_batch as batch; + + #[test] + fn every_step_is_drawn_whole_however_long() { + let text = "x".repeat(150); + let asked = batch(20, &text); + let rows = step_lines(&asked, 40); + assert!(rows.len() > MAX_STEP_ROWS); + assert!(rows.iter().all(|row| row.chars().count() <= 40)); + let drawn: String = rows.iter().map(|row| &row[STEP_INDENT.len()..]).collect(); + let whole: String = asked.request.steps.iter().map(|step| step.line()).collect(); + assert_eq!(drawn, whole); } } diff --git a/projects/clients/terminal/src/ui/thread/input/mod.rs b/projects/clients/terminal/src/ui/thread/input/mod.rs index 0b258958..29530877 100644 --- a/projects/clients/terminal/src/ui/thread/input/mod.rs +++ b/projects/clients/terminal/src/ui/thread/input/mod.rs @@ -49,7 +49,7 @@ pub struct InputView<'a> { /// Rows the box needs: the queued messages, the rule, the padding and the wrapped text. pub fn height(app: &App, width: u16) -> u16 { let (rows, _) = layout(&app.input, text_width(width)); - approval_rows(app) + approval_rows(app, width) + poll_rows(app, width) + goal_rows(app) + queue_rows(app) @@ -72,7 +72,7 @@ impl InputView<'_> { height: area.height.saturating_sub(1), ..area }; - let approval = approval_rows(self.app).min(area.height); + let approval = approval_rows(self.app, area.width).min(area.height); self.draw_approval( frame, Rect { diff --git a/projects/core/src/plugins/hooks.rs b/projects/core/src/plugins/hooks.rs index fa29f9f3..03af4673 100644 --- a/projects/core/src/plugins/hooks.rs +++ b/projects/core/src/plugins/hooks.rs @@ -108,6 +108,42 @@ impl Hooks { } } +impl Hooks { + /// The checks before a call, then before each call it runs inside, with that tool's + /// name and its own arguments, as if it were called alone. `base` is what every hook + /// is told about the thread. A deny from any of them refuses the whole call. + pub async fn before_tool( + &self, + tool: &str, + arguments: &Value, + within: &[(String, Value)], + base: &Value, + ) -> Option { + let mut details = base.clone(); + details["tool"] = json!(tool); + details["arguments"] = arguments.clone(); + if let Some(denied) = self + .run(HookEvent::BeforeTool, Some(tool), details) + .await + .denied + { + return Some(denied); + } + for (index, (name, inner)) in within.iter().enumerate() { + let number = index + 1; + let mut details = base.clone(); + details["tool"] = json!(name); + details["arguments"] = inner.clone(); + details["within"] = json!({ "tool": tool, "step": number }); + let outcome = self.run(HookEvent::BeforeTool, Some(name), details).await; + if let Some(denied) = outcome.denied { + return Some(format!("step {number} ({name}): {denied}; no step ran")); + } + } + None + } +} + fn applies(hook: &Hook, event: HookEvent, tool: Option<&str>) -> bool { hook.config.event == event && (hook.config.tools.is_empty() @@ -206,6 +242,37 @@ mod tests { .await } + #[tokio::test] + async fn every_call_inside_a_batch_meets_its_own_checks() { + let mut scoped = hook( + HookEvent::BeforeTool, + r#"if grep -q '"step":2'; then echo '{"decision":"deny","reason":"no typing"}'; fi"#, + ); + scoped.config.tools = vec!["type_text".into()]; + let hooks = Hooks::default(); + hooks.set(vec![scoped]); + let within = [ + ("click".to_owned(), json!({"x": 1, "y": 2})), + ("type_text".to_owned(), json!({"text": "hi"})), + ]; + let base = json!({ "thread": "t" }); + let denied = hooks + .before_tool("computer_actions", &json!({}), &within, &base) + .await; + assert_eq!( + denied.as_deref(), + Some("step 2 (type_text): plugin test refused: no typing; no step ran") + ); + // The same step alone in a batch of one is checked as step 1, and allowed here. + let alone = [("type_text".to_owned(), json!({"text": "hi"}))]; + assert!( + hooks + .before_tool("computer_actions", &json!({}), &alone, &base) + .await + .is_none() + ); + } + #[tokio::test] async fn a_check_before_a_tool_denies_and_fails_closed() { let denied = before_tool(r#"echo '{"decision":"deny","reason":"no"}'"#, "run_command"); diff --git a/projects/core/src/plugins/platform/session.rs b/projects/core/src/plugins/platform/session.rs index 0445a849..13ba411d 100644 --- a/projects/core/src/plugins/platform/session.rs +++ b/projects/core/src/plugins/platform/session.rs @@ -24,6 +24,18 @@ use super::{ wire::{FromPlatform, MAX_LINE_BYTES, ToPlatform}, }; +/// The longest approval text a platform is sent. Telegram cuts a message at 4096 +/// characters, and its plugin at less; a cut request would allow steps the user never +/// saw, so a longer one is not offered for a yes at all. +const MAX_APPROVAL_CHARS: usize = 3500; + +/// What the platform shows for a yes: the title and every step, or `None` when that +/// does not fit, and the request may then only be denied there. +fn approvable_text(request: &ApprovalRequest) -> Option { + let text = request.text("the agent"); + (text.chars().count() <= MAX_APPROVAL_CHARS).then_some(text) +} + /// One run of a platform's program, bridged to its thread. pub(super) struct Session<'a> { launcher: &'a Launcher, @@ -230,9 +242,20 @@ impl<'a> Session<'a> { let Some(request) = self.approvals.first() else { return Ok(()); }; - let (id, text) = (request.id, request.request.describe("the agent")); + let id = request.id; self.shown = Some(id); - self.send(ToPlatform::Approval { id, text }).await + match approvable_text(request) { + Some(text) => self.send(ToPlatform::Approval { id, text }).await, + // No button: deny still answers it, the app can still allow it. + None => { + let note = format!( + "{} The request is too long to show here in full, so it cannot be \ + allowed here: answer it in the duskcode app, or deny it with /deny.", + request.title("The agent") + ); + self.reply(¬e).await + } + } } /// Does what the user did on the platform, as if at the keyboard. @@ -261,6 +284,15 @@ impl<'a> Session<'a> { else { return self.reply("Nothing is waiting for that approval.").await; }; + let unseen = self + .approvals + .iter() + .any(|request| request.id == id && approvable_text(request).is_none()); + if allow && unseen { + return self + .reply("That request is too long to allow here: answer it in the app, or deny it.") + .await; + } let decision = if allow { Decision::Once } else { @@ -393,3 +425,35 @@ impl<'a> Session<'a> { } } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::tools::Request; + use dusk_protocol::tools::ApprovalStep; + + fn batch(steps: usize, typed: usize) -> ApprovalRequest { + let text = "x".repeat(typed); + let keyboard = |action: String| Request::Keyboard { action }; + ApprovalRequest { + id: 1, + request: keyboard(format!("{steps} computer actions")), + session: None, + steps: (1..=steps) + .map(|number| ApprovalStep { + number, + request: keyboard(format!("type \"{text}\"")), + allowed: false, + }) + .collect(), + } + } + + #[test] + fn a_request_too_long_to_show_whole_is_never_offered_for_a_yes() { + let short = approvable_text(&batch(3, 20)).expect("fits"); + assert!(short.contains("3. using the keyboard"), "{short}"); + // Twenty typed steps of 200 shown characters would be cut by the platform. + assert!(approvable_text(&batch(20, 200)).is_none()); + } +} diff --git a/projects/core/src/runtime/agent/agent/calls.rs b/projects/core/src/runtime/agent/agent/calls.rs index 02d631fa..2a7afbf0 100644 --- a/projects/core/src/runtime/agent/agent/calls.rs +++ b/projects/core/src/runtime/agent/agent/calls.rs @@ -116,9 +116,12 @@ impl Agent { .is_some_and(|g| !g.approved && g.status != GoalStatus::Done); let denied = match stop { None if !self.hooks.is_empty() => { - let details = json!({ "tool": call.name, "arguments": call.arguments }); - let checked = self.run_hooks(HookEvent::BeforeTool, Some(&call.name), details); - checked.await.denied + let within = self.tools.calls_within(&call.name, &call.arguments); + let base = json!({ "thread": self.thread_id, "depth": self.depth }); + let checked = + self.hooks + .before_tool(&call.name, &call.arguments, &within, &base); + checked.await } _ => None, }; diff --git a/projects/core/src/runtime/thread/launcher.rs b/projects/core/src/runtime/thread/launcher.rs index 119d96a8..a8342856 100644 --- a/projects/core/src/runtime/thread/launcher.rs +++ b/projects/core/src/runtime/thread/launcher.rs @@ -137,13 +137,16 @@ impl Launcher { } #[cfg(any(target_os = "macos", target_os = "linux"))] { - use crate::tools::{Click, PressKeys, Screen, Screenshot, Scroll, TypeText}; + use crate::tools::{ + Click, ComputerActions, PressKeys, Screen, Screenshot, Scroll, TypeText, + }; let screen = Arc::new(Screen::new(ArtifactStore::new(self.paths.artifacts_dir()))); tools.add(Box::new(Screenshot(Arc::clone(&screen)))); tools.add(Box::new(Click(Arc::clone(&screen)))); tools.add(Box::new(Scroll(Arc::clone(&screen)))); tools.add(Box::new(TypeText(Arc::clone(&screen)))); tools.add(Box::new(PressKeys(Arc::clone(&screen)))); + tools.add(Box::new(ComputerActions(Arc::clone(&screen)))); #[cfg(target_os = "macos")] tools.add(Box::new(crate::tools::WindowSession(screen))); } diff --git a/projects/core/src/tools/approvals.rs b/projects/core/src/tools/approvals.rs index f3f50666..42485e18 100644 --- a/projects/core/src/tools/approvals.rs +++ b/projects/core/src/tools/approvals.rs @@ -10,13 +10,13 @@ use std::{ use parking_lot::Mutex; -use anyhow::{Result, bail}; +use anyhow::{Context, Result, bail}; use tokio::sync::{mpsc, oneshot}; use super::tool::Progress; use crate::{config::ApprovalPolicy, runtime::ThreadEvent}; -pub use dusk_protocol::tools::{ApprovalRequest, Request, WindowAccess}; +pub use dusk_protocol::tools::{ApprovalRequest, ApprovalStep, Request, WindowAccess}; pub use dusk_protocol::tools::Decision; @@ -70,11 +70,92 @@ impl Approvals { if policy == ApprovalPolicy::Arbiter { return self.arbitrate(request, progress).await; } + let session = session_grant(&request); + self.prompt(request, session, Vec::new(), progress).await + } + + /// Screen actions asked as one: never less than asking each alone. A request the + /// session grants already is not asked again but still listed, with its number in + /// the batch, so the user sees every step. Steps of one kind may be allowed for the + /// session as that kind; mixed kinds are asked as the strictest of them for this + /// time only, since one session yes must not stand for another kind's. The arbiter + /// judges each step on its own. + pub async fn ask_all( + &self, + requests: Vec, + progress: Option, + policy: ApprovalPolicy, + ) -> Result<()> { + if policy == ApprovalPolicy::FullAccess { + return Ok(()); + } + if requests.len() < 2 || requests.iter().any(|r| strictness(r).is_none()) { + // A single request looks as it always has; only screen kinds merge. + for request in requests { + self.ask(request, progress.clone(), policy).await?; + } + return Ok(()); + } + let steps: Vec = { + let grants = self.grants.lock(); + requests + .into_iter() + .enumerate() + .map(|(index, request)| ApprovalStep { + number: index + 1, + allowed: grants.covers(&request), + request, + }) + .collect() + }; + let open: Vec<&ApprovalStep> = steps.iter().filter(|step| !step.allowed).collect(); + let Some(first) = open.first().map(|step| &step.request) else { + return Ok(()); + }; + if policy == ApprovalPolicy::Arbiter { + for step in &open { + let judged = self.arbitrate(step.request.clone(), progress.clone()); + judged + .await + .with_context(|| format!("step {} of the batch", step.number))?; + } + return Ok(()); + } + let same = open.iter().all(|step| same_kind(&step.request, first)); + let strictest = open + .iter() + .map(|step| &step.request) + .max_by_key(|r| strictness(r)) + .unwrap_or(first); + // The steps go in the text too, so whatever shows only the request, such as a + // message on a chat platform, still shows each step it allows. + let lines: Vec = steps.iter().map(ApprovalStep::line).collect(); + let action = format!("{} computer actions:\n{}", steps.len(), lines.join("\n")); + let request = with_action(strictest, action); + let session = if same { session_grant(&request) } else { None }; + self.prompt(request, session, steps, progress).await + } + + /// Shows the request to the user and waits. A yes for the session + /// adds a grant only when `session` offered one. + async fn prompt( + &self, + request: Request, + session: Option, + steps: Vec, + progress: Option, + ) -> Result<()> { let Some(progress) = progress else { - bail!( - "{} needs the user's approval, and no one can answer here", - request.what() - ); + let what = match steps.is_empty() { + true => request.what(), + false => steps + .iter() + .filter(|step| !step.allowed) + .map(ApprovalStep::line) + .collect::>() + .join("; "), + }; + bail!("{what} needs the user's approval, and no one can answer here"); }; let id = self.next.fetch_add(1, Ordering::Relaxed) + 1; let (sender, answer) = oneshot::channel(); @@ -85,16 +166,19 @@ impl Approvals { id, events: progress.events.clone(), }; - let session = session_grant(&request); + let offered = session.is_some(); progress.emit(ThreadEvent::Approval(ApprovalRequest { id, request: request.clone(), session, + steps, })); match answer.await { Ok(Decision::Once) => Ok(()), Ok(Decision::Session) => { - self.grants.lock().add(&request); + if offered { + self.grants.lock().add(&request); + } Ok(()) } Ok(Decision::Deny) | Err(_) => bail!( @@ -202,6 +286,40 @@ impl Grants { } } +/// How much a screen request can do, for a merged request to take the strictest kind. +/// `None` for requests that are not about the screen. +fn strictness(request: &Request) -> Option { + match request { + Request::Screen { .. } => Some(0), + Request::Window { .. } => Some(1), + Request::SystemPointer { .. } => Some(2), + Request::Keyboard { .. } => Some(3), + Request::Path { .. } | Request::Command { .. } => None, + } +} + +/// Whether one session grant would cover both. +fn same_kind(a: &Request, b: &Request) -> bool { + match (a, b) { + (Request::Window { grant: a, .. }, Request::Window { grant: b, .. }) => a == b, + _ => std::mem::discriminant(a) == std::mem::discriminant(b), + } +} + +/// `kind` asking for `action` instead. +fn with_action(kind: &Request, action: String) -> Request { + match kind { + Request::Screen { .. } => Request::Screen { action }, + Request::SystemPointer { .. } => Request::SystemPointer { action }, + Request::Keyboard { .. } => Request::Keyboard { action }, + Request::Window { grant, .. } => Request::Window { + grant: grant.clone(), + action, + }, + Request::Path { .. } | Request::Command { .. } => kind.clone(), + } +} + /// What a yes for the session covers, for the client to name: the folder of the path, /// or the program of a plain command. `None` when the session cannot be offered. pub fn session_grant(request: &Request) -> Option { diff --git a/projects/core/src/tools/computer/batch.rs b/projects/core/src/tools/computer/batch.rs new file mode 100644 index 00000000..a85ee7e1 --- /dev/null +++ b/projects/core/src/tools/computer/batch.rs @@ -0,0 +1,199 @@ +//! `computer_actions`: several screen actions in one call, so a known sequence costs one +//! round trip. Every step is checked and asked for before the first one runs, and the +//! batch is never asked for less than its steps would be one by one. + +use std::sync::Arc; + +use anyhow::{Context, Result, ensure}; +use schemars::JsonSchema; +use serde::Deserialize; +use serde_json::{Map, Value}; + +use super::{Click, PressKeys, Screenshot, Scroll, TypeText, screen::Screen, step::Step}; +use crate::{ + BoxFuture, + platform::Needs, + tools::{ + ToolResult, schema, + schema::parse_arguments, + tool::{Tool, ToolContext, ToolKind}, + }, +}; + +/// Steps in one call at most. +const MAX_STEPS: usize = 20; +/// Screenshots in one call at most, as many pictures as an MCP result may bring. +const MAX_PICTURES: usize = 4; + +pub struct ComputerActions(pub Arc); + +#[derive(Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct BatchArguments { + /// The actions, run in order; the batch stops at the first that fails. + #[schemars(length(min = 1, max = MAX_STEPS))] + steps: Vec, +} + +#[derive(Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct StepArguments { + tool: StepTool, + /// What that tool takes when called alone. + arguments: Map, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +enum StepTool { + Click, + TypeText, + PressKeys, + Scroll, + Screenshot, +} + +impl StepTool { + fn name(self) -> &'static str { + match self { + Self::Click => "click", + Self::TypeText => "type_text", + Self::PressKeys => "press_keys", + Self::Scroll => "scroll", + Self::Screenshot => "screenshot", + } + } +} + +/// The steps as the model gave them, bounded. Hooks, switches and the run all read +/// them through this one parse, so none sees other steps than the rest. +fn parse_steps(arguments: Value) -> Result> { + let BatchArguments { steps } = parse_arguments(arguments)?; + ensure!( + (1..=MAX_STEPS).contains(&steps.len()), + "give 1 to {MAX_STEPS} steps, not {}", + steps.len() + ); + let pictures = steps + .iter() + .filter(|step| step.tool == StepTool::Screenshot) + .count(); + ensure!( + pictures <= MAX_PICTURES, + "take at most {MAX_PICTURES} screenshots in one batch" + ); + Ok(steps) +} + +impl ComputerActions { + /// Every step checked, in order, before anything is asked or done. + fn steps(&self, arguments: Value) -> Result> { + let steps = parse_steps(arguments)?; + steps + .into_iter() + .enumerate() + .map(|(index, step)| { + let arguments = Value::Object(step.arguments); + let screen = &self.0; + let checked = match step.tool { + StepTool::Click => Click::step(screen, arguments), + StepTool::TypeText => TypeText::step(screen, arguments), + StepTool::PressKeys => PressKeys::step(screen, arguments), + StepTool::Scroll => Scroll::step(screen, arguments), + StepTool::Screenshot => Screenshot::step(screen, arguments), + }; + let checked = checked.with_context(|| { + format!( + "step {} ({}) is wrong, so no step ran", + index + 1, + step.tool.name() + ) + })?; + Ok((step.tool, checked)) + }) + .collect() + } +} + +/// Runs the asked-for steps in order and stops at the first failure. Every step reads x +/// and y in the picture the model saw before the batch, not in one taken meanwhile that +/// it has not seen yet; after the batch the newest picture counts again. +async fn run(screen: &Screen, steps: Vec<(StepTool, Step)>) -> ToolResult { + let total = steps.len(); + let seen = screen.coordinates(); + let mut newest = seen; + let mut result = ToolResult::text(String::new()); + let mut lines = Vec::with_capacity(total); + for (index, (tool, step)) in steps.into_iter().enumerate() { + let number = index + 1; + screen.set_coordinates(seen); + match step.run().await { + Ok(done) => { + if tool == StepTool::Screenshot { + newest = screen.coordinates(); + } + lines.push(format!("{number}. {}", done.content)); + result.artifacts.extend(done.artifacts); + } + Err(error) => { + lines.push(format!("{number}. failed: {error:#}")); + if number < total { + lines.push(format!("steps {} to {total} did not run", number + 1)); + } + result.is_error = true; + break; + } + } + } + screen.set_coordinates(newest); + result.content = lines.join("\n"); + result +} + +impl Tool for ComputerActions { + computer_tool!("computer_actions"); + + fn description(&self, _: &ToolContext) -> String { + format!( + "Run up to {MAX_STEPS} of click, type_text, press_keys, scroll and screenshot in \ + order in one call, each with the arguments it takes alone. Use it when you know \ + the coordinates and need not look between the actions, such as click a field, \ + type, press enter, then a screenshot to check. Every x, y is in the screenshot \ + taken before the batch. It asks the user once for all the steps and stops at \ + the first that fails." + ) + } + + fn parameters(&self) -> Value { + schema::of::() + } + + fn calls_within(&self, arguments: &Value) -> Result> { + let steps = parse_steps(arguments.clone())?; + Ok(steps + .into_iter() + .map(|step| (step.tool.name().to_owned(), Value::Object(step.arguments))) + .collect()) + } + + fn call<'a>(&'a self, _: Value, _: ToolContext) -> BoxFuture<'a, String> { + Box::pin(async { anyhow::bail!("computer_actions returns its steps' results") }) + } + + fn call_result<'a>( + &'a self, + arguments: Value, + context: ToolContext, + ) -> BoxFuture<'a, ToolResult> { + Box::pin(async move { + let steps = self.steps(arguments)?; + let requests = steps.iter().map(|(_, step)| step.request.clone()).collect(); + context.approve_computer(requests).await?; + Ok(run(&self.0, steps).await) + }) + } +} + +#[cfg(test)] +#[path = "batch_tests.rs"] +mod tests; diff --git a/projects/core/src/tools/computer/batch_tests.rs b/projects/core/src/tools/computer/batch_tests.rs new file mode 100644 index 00000000..18a167fb --- /dev/null +++ b/projects/core/src/tools/computer/batch_tests.rs @@ -0,0 +1,291 @@ +//! A batch is checked and asked for whole before any step runs. No test here touches +//! the desktop: steps that would are refused first, or replaced by fakes. + +use std::sync::Arc; + +use serde_json::json; +use tokio::sync::mpsc; + +use super::*; +use crate::{ + config::ApprovalPolicy, + runtime::ThreadEvent, + state::artifacts::ArtifactStore, + testing::{temp_dir, tool_context}, + tools::{ + Approvals, Decision, Judge, Request, ToolScope, Verdict, + computer::screen::View, + tool::{Progress, ToolContext}, + }, +}; + +fn batch() -> ComputerActions { + ComputerActions(Arc::new(Screen::new(ArtifactStore::new( + std::env::temp_dir(), + )))) +} + +/// A context that asks through `approvals`; with no `progress` no one can answer. +fn context(approvals: &Arc, progress: Option) -> ToolContext { + let dir = temp_dir("computer_batch"); + std::fs::create_dir_all(&dir).unwrap(); + let mut context = tool_context(&ToolScope::new(&dir, &dir).unwrap()); + context.approvals = Some(Arc::clone(approvals)); + context.progress = progress; + context +} + +fn error(result: Result) -> String { + format!("{:#}", result.expect_err("the batch should fail")) +} + +fn fake() -> Request { + Request::Screen { + action: "fake".into(), + } +} + +#[test] +fn too_many_steps_and_unknown_tools_are_refused() { + let step = json!({"tool": "press_keys", "arguments": {"keys": "enter"}}); + let steps = vec![step; MAX_STEPS + 1]; + assert!(batch().steps(json!({ "steps": steps })).is_err()); + for wrong in [ + json!({"steps": []}), + json!({"steps": [{"tool": "run_command", "arguments": {}}]}), + json!({"steps": [{"tool": "window_session", "arguments": {}}]}), + json!({"steps": [{"tool": "click", "arguments": {}, "extra": 1}]}), + json!({"steps": [], "extra": 1}), + ] { + assert!(batch().steps(wrong.clone()).is_err(), "{wrong}"); + } +} + +#[tokio::test] +async fn a_wrong_step_stops_the_batch_before_anything_is_asked() { + let approvals = Arc::new(Approvals::default()); + // A step that reached approval would fail with "no one can answer here" instead. + let arguments = json!({"steps": [ + {"tool": "press_keys", "arguments": {"keys": "enter"}}, + {"tool": "type_text", "arguments": {"text": ""}}, + ]}); + let message = error( + batch() + .call_result(arguments, context(&approvals, None)) + .await, + ); + assert!(message.contains("step 2 (type_text)"), "{message}"); + assert!(message.contains("no step ran"), "{message}"); + assert!(!message.contains("approval"), "{message}"); +} + +#[tokio::test] +async fn every_step_reads_the_picture_seen_before_the_batch() { + let screen = Arc::new(Screen::new(ArtifactStore::new(std::env::temp_dir()))); + screen.set_coordinates(Some(View::scaled(1.0))); + let shot = Arc::clone(&screen); + let reader = Arc::clone(&screen); + let steps = vec![ + ( + StepTool::Screenshot, + Step::new(fake(), move || { + shot.set_coordinates(Some(View::scaled(2.0))); + Ok(ToolResult::text("shot".into())) + }), + ), + ( + StepTool::Click, + Step::new(fake(), move || { + let at = reader.point(10.0, 10.0)?; + Ok(ToolResult::text(format!("at {}", at.x))) + }), + ), + ]; + let result = run(&screen, steps).await; + assert!(!result.is_error, "{}", result.content); + assert!(result.content.contains("2. at 10"), "{}", result.content); + // After the batch the newest picture, the one the model now sees, counts again. + assert_eq!(screen.point(10.0, 10.0).unwrap().x, 20.0); +} + +#[tokio::test] +async fn a_failed_step_stops_the_rest() { + let screen = Screen::new(ArtifactStore::new(std::env::temp_dir())); + let ran = Arc::new(parking_lot::Mutex::new(0)); + let counter = Arc::clone(&ran); + let steps = vec![ + ( + StepTool::Click, + Step::new(fake(), || anyhow::bail!("no control there")), + ), + ( + StepTool::Click, + Step::new(fake(), move || { + *counter.lock() += 1; + Ok(ToolResult::text(String::new())) + }), + ), + ]; + let result = run(&screen, steps).await; + assert!(result.is_error); + assert!(result.content.contains("1. failed: no control there")); + assert!(result.content.contains("steps 2 to 2 did not run")); + assert_eq!(*ran.lock(), 0); +} + +/// Allows the screen for the session, as a user would. +async fn allow_screen(approvals: &Approvals) { + let (events, mut seen) = mpsc::unbounded_channel(); + let progress = Progress { + call_id: "grant".into(), + events, + }; + let ask = approvals.ask(fake(), Some(progress), ApprovalPolicy::Ask); + let answer = async { + let Some(ThreadEvent::Approval(asked)) = seen.recv().await else { + panic!("missing request"); + }; + assert!(approvals.answer(asked.id, Decision::Session)); + }; + let (result, ()) = tokio::join!(ask, answer); + result.unwrap(); +} + +#[tokio::test] +async fn one_refused_step_refuses_the_batch() { + let approvals = Arc::new(Approvals::default()); + allow_screen(&approvals).await; + let arguments = json!({"steps": [ + {"tool": "screenshot", "arguments": {}}, + {"tool": "press_keys", "arguments": {"keys": "enter"}}, + ]}); + // The screenshot alone would pass on the grant; the keys still need a yes, and no + // one can answer here, so nothing runs. + let message = error( + batch() + .call_result(arguments, context(&approvals, None)) + .await, + ); + assert!(message.contains("press enter"), "{message}"); + + // Asked, the batch lists every step by its number, the allowed ones marked, so + // the click that sets the focus is seen with the typing; a no stops all. + let (events, mut seen) = mpsc::unbounded_channel(); + let progress = Progress { + call_id: "batch".into(), + events, + }; + let arguments = json!({"steps": [ + {"tool": "screenshot", "arguments": {}}, + {"tool": "type_text", "arguments": {"text": "hi"}}, + {"tool": "press_keys", "arguments": {"keys": "Shift + Enter"}}, + ]}); + let tool = batch(); + let call = tool.call_result(arguments, context(&approvals, Some(progress))); + let answer = async { + let Some(ThreadEvent::Approval(asked)) = seen.recv().await else { + panic!("missing request"); + }; + assert!(matches!(asked.request, Request::Keyboard { .. })); + let lines: Vec = asked.steps.iter().map(|step| step.line()).collect(); + assert_eq!( + lines, + [ + "1. using the screen: take a screenshot (already allowed)", + "2. using the keyboard: type \"hi\"", + "3. using the keyboard: press shift+enter", + ] + ); + assert_eq!( + asked.title("the agent"), + "the agent wants to run 3 computer actions" + ); + // A surface that reads only the request's text, or its plain text, sees every + // step too; the row drawing clients use the title, so nothing shows twice. + let described = asked.request.describe("the agent"); + let text = asked.text("the agent"); + for line in &lines { + assert!(described.contains(line.as_str()), "{described}"); + assert!(text.contains(line.as_str()), "{text}"); + } + assert!(!asked.title("the agent").contains("1.")); + assert_eq!(asked.session, None); + assert!(approvals.answer(asked.id, Decision::Deny)); + }; + let (result, ()) = tokio::join!(call, answer); + assert!(error(result).contains("denied")); +} + +/// Says no to any request that presses keys, yes to the rest. +struct NoKeys; + +impl Judge for NoKeys { + fn judge<'a>(&'a self, request: &'a Request) -> crate::BoxFuture<'a, Verdict> { + let allow = !request.what().contains("press"); + Box::pin(async move { + Ok(Verdict { + allow, + reason: "keys are not allowed".into(), + }) + }) + } +} + +#[tokio::test] +async fn the_arbiter_judges_every_step_alone() { + let approvals = Approvals::default(); + approvals.set_judge(Arc::new(NoKeys)); + let requests = vec![ + fake(), + Request::Keyboard { + action: "press enter".into(), + }, + ]; + let denied = approvals + .ask_all(requests, None, ApprovalPolicy::Arbiter) + .await + .expect_err("one step is denied"); + let message = format!("{denied:#}"); + assert!(message.starts_with("step 2 of the batch"), "{message}"); + assert!(message.contains("keys are not allowed"), "{message}"); + approvals + .ask_all(vec![fake(), fake()], None, ApprovalPolicy::Arbiter) + .await + .unwrap(); +} + +#[tokio::test] +async fn mixed_kinds_offer_no_session_grant() { + let approvals = Approvals::default(); + let requests = vec![ + fake(), + Request::SystemPointer { + action: "scroll".into(), + }, + ]; + let (events, mut seen) = mpsc::unbounded_channel(); + let progress = Progress { + call_id: "mixed".into(), + events, + }; + let ask = approvals.ask_all(requests.clone(), Some(progress), ApprovalPolicy::Ask); + let answer = async { + let Some(ThreadEvent::Approval(asked)) = seen.recv().await else { + panic!("missing request"); + }; + assert!(matches!(asked.request, Request::SystemPointer { .. })); + assert_eq!(asked.session, None); + // A client that answers "session" anyway grants nothing beyond this time. + assert!(approvals.answer(asked.id, Decision::Session)); + }; + let (result, ()) = tokio::join!(ask, answer); + result.unwrap(); + for request in requests { + assert!( + approvals + .ask(request, None, ApprovalPolicy::Ask) + .await + .is_err() + ); + } +} diff --git a/projects/core/src/tools/computer/click.rs b/projects/core/src/tools/computer/click.rs index a6174e63..80788001 100644 --- a/projects/core/src/tools/computer/click.rs +++ b/projects/core/src/tools/computer/click.rs @@ -10,12 +10,12 @@ use serde_json::Value; #[cfg(test)] use super::Desktop; -use super::{blocking, screen::Screen}; +use super::{screen::Screen, step::Step}; use crate::{ BoxFuture, platform::Needs, tools::{ - WindowAccess, schema, + Request, ToolResult, WindowAccess, schema, schema::parse_arguments, tool::{Tool, ToolContext, ToolKind}, }, @@ -65,6 +65,59 @@ struct ClickArguments { mode: PointerMode, } +impl ClickArguments { + /// Bounded before anything is asked or done. + fn parse(arguments: Value) -> Result { + let arguments: Self = parse_arguments(arguments)?; + arguments + .mode + .validate_click(arguments.right, arguments.count())?; + Ok(arguments) + } + + fn count(&self) -> i64 { + if self.double { 2 } else { 1 } + } +} + +impl Click { + /// Checked before anything is asked or done, so a batch can check every step first. + pub(super) fn step(screen: &Arc, arguments: Value) -> Result { + let arguments = ClickArguments::parse(arguments)?; + let (x, y, mode) = (arguments.x, arguments.y, arguments.mode); + let action = match mode { + PointerMode::Independent => { + format!("activate the control at {x}, {y} without moving the pointer") + } + PointerMode::System => format!("move the user's pointer and click at {x}, {y}"), + }; + let scope = screen.scope(); + let request = match (&scope, mode) { + (Some((id, label)), PointerMode::Independent) => Request::Window { + grant: Some(WindowAccess::Activate(id.clone())), + action: format!("{action} in {label}; the target app may come to the foreground"), + }, + (Some(_), PointerMode::System) => { + anyhow::bail!("system pointer is disabled in a window session") + } + (None, PointerMode::Independent) => Request::Screen { action }, + (None, PointerMode::System) => Request::SystemPointer { action }, + }; + let screen = Arc::clone(screen); + Ok(Step::new(request, move || { + let count = arguments.count(); + let expected = scope.as_ref().map(|(id, _)| id.as_str()); + screen.click(x, y, arguments.right, count, mode, expected)?; + Ok(ToolResult::text(match mode { + PointerMode::Independent => format!( + "pressed the control at {x}, {y}; the keyboard focus may not have moved" + ), + PointerMode::System => format!("clicked at {x}, {y} with the pointer"), + })) + })) + } +} + impl Tool for Click { computer_tool!("click"); @@ -93,59 +146,8 @@ impl Tool for Click { fn call<'a>(&'a self, arguments: Value, context: ToolContext) -> BoxFuture<'a, String> { Box::pin(async move { - let arguments: ClickArguments = parse_arguments(arguments)?; - let count = if arguments.double { 2 } else { 1 }; - arguments.mode.validate_click(arguments.right, count)?; - let action = match arguments.mode { - PointerMode::Independent => format!( - "activate the control at {}, {} without moving the pointer", - arguments.x, arguments.y - ), - PointerMode::System => format!( - "move the user's pointer and click at {}, {}", - arguments.x, arguments.y - ), - }; - let scope = self.0.scope(); - match (&scope, arguments.mode) { - (Some((id, label)), PointerMode::Independent) => { - context - .approve_window( - Some(WindowAccess::Activate(id.clone())), - &format!( - "{action} in {label}; the target app may come to the foreground" - ), - ) - .await? - } - (Some(_), PointerMode::System) => { - anyhow::bail!("system pointer is disabled in a window session") - } - (None, PointerMode::Independent) => context.approve_screen(&action, false).await?, - (None, PointerMode::System) => context.approve_system_pointer(&action).await?, - } - let screen = Arc::clone(&self.0); - blocking(move || { - screen.click( - arguments.x, - arguments.y, - arguments.right, - count, - arguments.mode, - scope.as_ref().map(|(id, _)| id.as_str()), - ) - }) - .await?; - match arguments.mode { - PointerMode::Independent => Ok(format!( - "pressed the control at {}, {}; the keyboard focus may not have moved", - arguments.x, arguments.y - )), - PointerMode::System => Ok(format!( - "clicked at {}, {} with the pointer", - arguments.x, arguments.y - )), - } + let step = Self::step(&self.0, arguments)?; + Ok(step.ask_and_run(&context).await?.content) }) } } diff --git a/projects/core/src/tools/computer/mod.rs b/projects/core/src/tools/computer/mod.rs index 420caa75..f197a419 100644 --- a/projects/core/src/tools/computer/mod.rs +++ b/projects/core/src/tools/computer/mod.rs @@ -10,14 +10,16 @@ //! the desktop has. //! //! The tools are the flat files here (`click`, `press_keys`, `screenshot`, `scroll`, -//! `type_text`, `window_session`) over `screen`, the screen as the model sees it, and -//! `offer`, how they are offered. `backend` is the desktop they drive; `overlay` is +//! `type_text`, `window_session`, and `batch`, several of them in one call) over +//! `screen`, the screen as the model sees it, `step`, one checked action and what it +//! asks, and `offer`, how they are offered. `backend` is the desktop they drive; `overlay` is //! dusk's own cursor on macOS. #[macro_use] mod offer; mod backend; +mod batch; mod click; #[cfg(target_os = "macos")] mod overlay; @@ -25,6 +27,7 @@ mod press_keys; mod screen; mod screenshot; mod scroll; +mod step; mod type_text; #[cfg(target_os = "macos")] mod window_session; @@ -32,6 +35,7 @@ mod window_session; use anyhow::{Context, Result}; use backend::Desktop; +pub use batch::ComputerActions; pub use click::Click; #[cfg(target_os = "macos")] pub use overlay::run_overlay_if_asked; diff --git a/projects/core/src/tools/computer/press_keys.rs b/projects/core/src/tools/computer/press_keys.rs index b812e099..f51967ae 100644 --- a/projects/core/src/tools/computer/press_keys.rs +++ b/projects/core/src/tools/computer/press_keys.rs @@ -2,16 +2,17 @@ use std::sync::Arc; +use anyhow::Result; use schemars::JsonSchema; use serde::Deserialize; use serde_json::Value; -use super::{Desktop, Screen, blocking, screen::keys}; +use super::{Desktop, Keys, Screen, screen::keys, step::Step}; use crate::{ BoxFuture, platform::Needs, tools::{ - schema, + Request, ToolResult, schema, schema::parse_arguments, tool::{Tool, ToolContext, ToolKind}, }, @@ -26,6 +27,32 @@ struct PressKeysArguments { keys: String, } +impl PressKeysArguments { + /// Bounded before anything is asked or done. + fn parse(arguments: Value) -> Result { + let arguments: Self = parse_arguments(arguments)?; + keys(&arguments.keys) + } +} + +impl PressKeys { + /// Checked before anything is asked or done, so a batch can check every step first. + pub(super) fn step(screen: &Arc, arguments: Value) -> Result { + screen.require_unscoped()?; + let combo = PressKeysArguments::parse(arguments)?; + let keys = combo.shown(); + let request = Request::Keyboard { + action: format!("press {keys}"), + }; + let screen = Arc::clone(screen); + Ok(Step::new(request, move || { + screen.unscoped(|| Desktop.press(&combo))?; + Desktop.touch(); + Ok(ToolResult::text(format!("pressed {keys}"))) + })) + } +} + impl Tool for PressKeys { computer_tool!("press_keys"); @@ -42,16 +69,8 @@ impl Tool for PressKeys { fn call<'a>(&'a self, arguments: Value, context: ToolContext) -> BoxFuture<'a, String> { Box::pin(async move { - self.0.require_unscoped()?; - let arguments: PressKeysArguments = parse_arguments(arguments)?; - let combo = keys(&arguments.keys)?; - context - .approve_screen(&format!("press {}", arguments.keys), true) - .await?; - let screen = Arc::clone(&self.0); - blocking(move || screen.unscoped(|| Desktop.press(&combo))).await?; - Desktop.touch(); - Ok(format!("pressed {}", arguments.keys)) + let step = Self::step(&self.0, arguments)?; + Ok(step.ask_and_run(&context).await?.content) }) } } diff --git a/projects/core/src/tools/computer/screen.rs b/projects/core/src/tools/computer/screen.rs index c9d11762..bcf70079 100644 --- a/projects/core/src/tools/computer/screen.rs +++ b/projects/core/src/tools/computer/screen.rs @@ -20,7 +20,7 @@ const JPEG_QUALITY: u8 = 80; /// How a picture's pixel maps to the display: `origin + pixel * scale` in points. #[derive(Debug, Clone, Copy)] -struct View { +pub(super) struct View { origin: Point, scale_x: f64, scale_y: f64, @@ -349,6 +349,17 @@ impl Screen { Ok((at, view.window)) } + /// The picture clicks and regions are read in now, if any. + pub(super) fn coordinates(&self) -> Option { + *self.view.lock() + } + + /// A batch reads every step in the picture the model saw before it, not in one it + /// took meanwhile and the model has not seen. + pub(super) fn set_coordinates(&self, view: Option) { + *self.view.lock() = view; + } + fn view(&self) -> Result { (*self.view.lock()).context("take a screenshot first: coordinates come from it") } @@ -431,6 +442,21 @@ impl View { } } +#[cfg(test)] +impl View { + /// A picture of the whole display at `scale` points a pixel. + pub(super) fn scaled(scale: f64) -> Self { + Self { + origin: Point { x: 0.0, y: 0.0 }, + scale_x: scale, + scale_y: scale, + width: 1000.0, + height: 1000.0, + window: None, + } + } +} + /// The largest window of the app named `name`, else the largest whose app or title holds /// it, so a toolbar strip never wins over the window itself; a miss names the windows /// there are. @@ -502,7 +528,9 @@ pub const KEY_NAMES: [&str; 27] = [ ]; /// `cmd+shift+t`, `enter` or `f5` as a key with its modifiers. The key comes out by its -/// one name: a lowercase letter, a digit or one of [`KEY_NAMES`]. +/// one name: a lowercase letter, a digit or one of [`KEY_NAMES`]. Only plain spaces +/// may pad a part: a line break or control character would let the text the user +/// approves look unlike the keys pressed. pub fn keys(combo: &str) -> Result { let mut keys = Keys { key: String::new(), @@ -513,7 +541,7 @@ pub fn keys(combo: &str) -> Result { }; for part in combo .split('+') - .map(|part| part.trim().to_ascii_lowercase()) + .map(|part| part.trim_matches(' ').to_ascii_lowercase()) { match part.as_str() { "cmd" | "command" | "meta" | "super" | "win" => keys.cmd = true, @@ -536,6 +564,25 @@ pub fn keys(combo: &str) -> Result { Ok(keys) } +impl Keys { + /// The combo by its normal names, such as `cmd+shift+t`, for the user to approve. + pub fn shown(&self) -> String { + let modifiers = [ + (self.cmd, "cmd"), + (self.ctrl, "ctrl"), + (self.alt, "alt"), + (self.shift, "shift"), + ]; + let mut parts: Vec<&str> = modifiers + .iter() + .filter(|(on, _)| *on) + .map(|(_, name)| *name) + .collect(); + parts.push(&self.key); + parts.join("+") + } +} + fn key_name(name: &str) -> Option { let mut chars = name.chars(); if let (Some(c), None) = (chars.next(), chars.next()) @@ -565,6 +612,11 @@ mod tests { for wrong in ["cmd", "a+b", "cmd+nope", "", "é"] { assert!(keys(wrong).is_err(), "{wrong}"); } + // What the user approves is what is pressed: no hidden lines or controls. + for hidden in ["\nenter", "enter\t", "cmd+\u{7}t", "cmd\r+t", "\u{a0}enter"] { + assert!(keys(hidden).is_err(), "{hidden:?}"); + } + assert_eq!(keys(" Shift + CMD + t ").unwrap().shown(), "cmd+shift+t"); // Every name the parser lets through, the desktop can press. for name in KEY_NAMES.iter().copied().chain(["a", "z", "0", "9"]) { assert!(Desktop.knows_key(name), "{name}"); diff --git a/projects/core/src/tools/computer/screenshot.rs b/projects/core/src/tools/computer/screenshot.rs index 774caae3..23a041f5 100644 --- a/projects/core/src/tools/computer/screenshot.rs +++ b/projects/core/src/tools/computer/screenshot.rs @@ -2,19 +2,20 @@ use std::sync::Arc; +use anyhow::Result; use schemars::JsonSchema; use serde::Deserialize; use serde_json::Value; use super::{ - blocking, screen::{Region, Screen, Target}, + step::Step, }; use crate::{ BoxFuture, platform::Needs, tools::{ - ToolResult, WindowAccess, schema, + Request, ToolResult, WindowAccess, schema, schema::parse_arguments, tool::{Tool, ToolContext, ToolKind}, }, @@ -42,6 +43,49 @@ struct RegionArguments { height: f64, } +impl ScreenshotArguments { + /// What to capture, checked before anything is asked or done. + fn target(arguments: Value) -> Result { + let arguments: Self = parse_arguments(arguments)?; + Ok(match (arguments.region, arguments.window) { + (Some(_), Some(_)) => anyhow::bail!("give a region or a window, not both"), + (Some(region), None) => Target::Region(Region { + x: region.x, + y: region.y, + width: region.width, + height: region.height, + }), + (None, Some(window)) => Target::Window(window), + (None, None) => Target::Display, + }) + } +} + +impl Screenshot { + /// Checked before anything is asked or done, so a batch can check every step first. + pub(super) fn step(screen: &Arc, arguments: Value) -> Result { + let target = ScreenshotArguments::target(arguments)?; + let scope = screen.scope(); + let request = match &scope { + Some((id, label)) => Request::Window { + grant: Some(WindowAccess::Capture(id.clone())), + action: format!("capture only {label}"), + }, + None => Request::Screen { + action: "take a screenshot".into(), + }, + }; + let screen = Arc::clone(screen); + Ok(Step::new(request, move || { + let expected = scope.as_ref().map(|(id, _)| id.as_str()); + let (artifact, text) = screen.capture(target, expected)?; + let mut result = ToolResult::text(text); + result.artifacts.push(artifact); + Ok(result) + })) + } +} + impl Tool for Screenshot { computer_tool!("screenshot"); @@ -68,38 +112,6 @@ impl Tool for Screenshot { arguments: Value, context: ToolContext, ) -> BoxFuture<'a, ToolResult> { - Box::pin(async move { - let arguments: ScreenshotArguments = parse_arguments(arguments)?; - let scope = self.0.scope(); - match &scope { - Some((id, label)) => { - context - .approve_window( - Some(WindowAccess::Capture(id.clone())), - &format!("capture only {label}"), - ) - .await? - } - None => context.approve_screen("take a screenshot", false).await?, - } - let target = match (arguments.region, arguments.window) { - (Some(_), Some(_)) => anyhow::bail!("give a region or a window, not both"), - (Some(region), None) => Target::Region(Region { - x: region.x, - y: region.y, - width: region.width, - height: region.height, - }), - (None, Some(window)) => Target::Window(window), - (None, None) => Target::Display, - }; - let screen = Arc::clone(&self.0); - let (artifact, text) = - blocking(move || screen.capture(target, scope.as_ref().map(|(id, _)| id.as_str()))) - .await?; - let mut result = ToolResult::text(text); - result.artifacts.push(artifact); - Ok(result) - }) + Box::pin(async move { Self::step(&self.0, arguments)?.ask_and_run(&context).await }) } } diff --git a/projects/core/src/tools/computer/scroll.rs b/projects/core/src/tools/computer/scroll.rs index 8985cab7..5badf033 100644 --- a/projects/core/src/tools/computer/scroll.rs +++ b/projects/core/src/tools/computer/scroll.rs @@ -2,17 +2,17 @@ use std::sync::Arc; -use anyhow::ensure; +use anyhow::{Result, ensure}; use schemars::JsonSchema; use serde::Deserialize; use serde_json::Value; -use super::{Desktop, blocking, screen::Screen}; +use super::{Desktop, screen::Screen, step::Step}; use crate::{ BoxFuture, platform::Needs, tools::{ - schema, + Request, ToolResult, schema, schema::parse_arguments, tool::{Tool, ToolContext, ToolKind}, }, @@ -36,6 +36,45 @@ struct ScrollArguments { right: i32, } +impl ScrollArguments { + /// Bounded before anything is asked or done. + fn parse(arguments: Value) -> Result { + let mut arguments: Self = parse_arguments(arguments)?; + let bounded = |pixels: i32| pixels.clamp(-MAX_SCROLL_PIXELS, MAX_SCROLL_PIXELS); + arguments.down = bounded(arguments.down); + arguments.right = bounded(arguments.right); + ensure!( + arguments.down != 0 || arguments.right != 0, + "say how far to scroll" + ); + Ok(arguments) + } +} + +impl Scroll { + /// Checked before anything is asked or done, so a batch can check every step first. + pub(super) fn step(screen: &Arc, arguments: Value) -> Result { + screen.require_unscoped()?; + let ScrollArguments { x, y, down, right } = ScrollArguments::parse(arguments)?; + let request = Request::SystemPointer { + action: format!( + "move the user's pointer to {x}, {y} and scroll {down} down, {right} right" + ), + }; + let screen = Arc::clone(screen); + Ok(Step::new(request, move || { + screen.unscoped(|| { + let at = screen.point(x, y)?; + screen.mark(at); + Desktop.scroll(at, down, right) + })?; + Ok(ToolResult::text(format!( + "scrolled {down} down and {right} right" + ))) + })) + } +} + impl Tool for Scroll { computer_tool!("scroll"); @@ -52,24 +91,8 @@ impl Tool for Scroll { fn call<'a>(&'a self, arguments: Value, context: ToolContext) -> BoxFuture<'a, String> { Box::pin(async move { - self.0.require_unscoped()?; - let arguments: ScrollArguments = parse_arguments(arguments)?; - let bounded = |pixels: i32| pixels.clamp(-MAX_SCROLL_PIXELS, MAX_SCROLL_PIXELS); - let (down, right) = (bounded(arguments.down), bounded(arguments.right)); - ensure!(down != 0 || right != 0, "say how far to scroll"); - context - .approve_system_pointer("move the user's pointer and scroll") - .await?; - let screen = Arc::clone(&self.0); - blocking(move || { - screen.unscoped(|| { - let at = screen.point(arguments.x, arguments.y)?; - screen.mark(at); - Desktop.scroll(at, down, right) - }) - }) - .await?; - Ok(format!("scrolled {down} down and {right} right")) + let step = Self::step(&self.0, arguments)?; + Ok(step.ask_and_run(&context).await?.content) }) } } diff --git a/projects/core/src/tools/computer/step.rs b/projects/core/src/tools/computer/step.rs new file mode 100644 index 00000000..b497f244 --- /dev/null +++ b/projects/core/src/tools/computer/step.rs @@ -0,0 +1,38 @@ +//! One checked screen action: what it asks the user and the work it does. A tool alone +//! and a batch build the same step, so they cannot ask or act differently. + +use anyhow::Result; + +use super::blocking; +use crate::tools::{Request, ToolResult, tool::ToolContext}; + +type Work = Box Result + Send>; + +pub(super) struct Step { + /// What the user is asked before the work runs. + pub request: Request, + work: Work, +} + +impl Step { + pub fn new( + request: Request, + work: impl FnOnce() -> Result + Send + 'static, + ) -> Self { + Self { + request, + work: Box::new(work), + } + } + + /// The work alone; the caller has asked for it. + pub async fn run(self) -> Result { + blocking(self.work).await + } + + /// A tool called on its own: asks, then acts. + pub async fn ask_and_run(self, context: &ToolContext) -> Result { + context.approve_computer(vec![self.request.clone()]).await?; + self.run().await + } +} diff --git a/projects/core/src/tools/computer/type_text.rs b/projects/core/src/tools/computer/type_text.rs index 16a1833b..5d37d3eb 100644 --- a/projects/core/src/tools/computer/type_text.rs +++ b/projects/core/src/tools/computer/type_text.rs @@ -2,17 +2,17 @@ use std::sync::Arc; -use anyhow::ensure; +use anyhow::{Result, ensure}; use schemars::JsonSchema; use serde::Deserialize; use serde_json::Value; -use super::{Desktop, Screen, blocking}; +use super::{Desktop, Screen, step::Step}; use crate::{ BoxFuture, platform::Needs, tools::{ - schema, + Request, ToolResult, schema, schema::parse_arguments, tool::{Tool, ToolContext, ToolKind}, }, @@ -32,6 +32,41 @@ struct TypeTextArguments { text: String, } +impl TypeTextArguments { + /// Bounded before anything is asked or done. + fn parse(arguments: Value) -> Result { + let arguments: Self = parse_arguments(arguments)?; + let count = arguments.text.chars().count(); + ensure!(count > 0, "the text is empty"); + ensure!( + count <= MAX_TYPED_CHARS, + "type at most {MAX_TYPED_CHARS} characters at once" + ); + Ok(arguments) + } +} + +impl TypeText { + /// Checked before anything is asked or done, so a batch can check every step first. + pub(super) fn step(screen: &Arc, arguments: Value) -> Result { + screen.require_unscoped()?; + let TypeTextArguments { text } = TypeTextArguments::parse(arguments)?; + let count = text.chars().count(); + // The user sees what will be typed, since it can run anything a shell can. + let shown: String = text.chars().take(SHOWN_CHARS).collect(); + let more = if count > SHOWN_CHARS { "..." } else { "" }; + let request = Request::Keyboard { + action: format!("type \"{}{more}\"", shown.escape_debug()), + }; + let screen = Arc::clone(screen); + Ok(Step::new(request, move || { + screen.unscoped(|| Desktop.type_text(&text))?; + Desktop.touch(); + Ok(ToolResult::text(format!("typed {count} characters"))) + })) + } +} + impl Tool for TypeText { computer_tool!("type_text"); @@ -50,23 +85,8 @@ impl Tool for TypeText { fn call<'a>(&'a self, arguments: Value, context: ToolContext) -> BoxFuture<'a, String> { Box::pin(async move { - self.0.require_unscoped()?; - let arguments: TypeTextArguments = parse_arguments(arguments)?; - let count = arguments.text.chars().count(); - ensure!(count > 0, "the text is empty"); - ensure!( - count <= MAX_TYPED_CHARS, - "type at most {MAX_TYPED_CHARS} characters at once" - ); - // The user sees what will be typed, since it can run anything a shell can. - let shown: String = arguments.text.chars().take(SHOWN_CHARS).collect(); - let more = if count > SHOWN_CHARS { "..." } else { "" }; - let action = format!("type \"{}{more}\"", shown.escape_debug()); - context.approve_screen(&action, true).await?; - let screen = Arc::clone(&self.0); - blocking(move || screen.unscoped(|| Desktop.type_text(&arguments.text))).await?; - Desktop.touch(); - Ok(format!("typed {count} characters")) + let step = Self::step(&self.0, arguments)?; + Ok(step.ask_and_run(&context).await?.content) }) } } diff --git a/projects/core/src/tools/mod.rs b/projects/core/src/tools/mod.rs index 105d3dc9..919fea0f 100644 --- a/projects/core/src/tools/mod.rs +++ b/projects/core/src/tools/mod.rs @@ -25,7 +25,7 @@ pub use computer::WindowSession; #[cfg(target_os = "macos")] pub use computer::run_overlay_if_asked; #[cfg(any(target_os = "macos", target_os = "linux"))] -pub use computer::{Click, PressKeys, Screen, Screenshot, Scroll, TypeText}; +pub use computer::{Click, ComputerActions, PressKeys, Screen, Screenshot, Scroll, TypeText}; pub use console::{Jobs, RunCommand}; pub use harness::{ AskUser, DisplayAudio, DisplayImage, GoalTool, PlanEdit, Question, ReadHistory, Remember, diff --git a/projects/core/src/tools/registry.rs b/projects/core/src/tools/registry.rs index b9753f65..4ed17bd0 100644 --- a/projects/core/src/tools/registry.rs +++ b/projects/core/src/tools/registry.rs @@ -125,6 +125,41 @@ impl ToolRegistry { kinds } + /// The calls `name` runs inside it, for hooks to judge one by one; empty for a tool + /// that runs none or arguments that do not parse, which the call itself refuses. + pub fn calls_within(&self, name: &str, arguments: &Value) -> Vec<(String, Value)> { + self.tools + .get(name) + .and_then(|tool| tool.calls_within(arguments).ok()) + .unwrap_or_default() + } + + /// A call inside another is refused as it would be alone: a tool switched off in the + /// mode does not run because a batch names it. + fn check_within( + &self, + tool: &dyn Tool, + arguments: &Value, + config: &Config, + ) -> std::result::Result<(), String> { + let within = tool + .calls_within(arguments) + .map_err(|error| format!("{error:#}"))?; + for (index, (name, _)) in within.iter().enumerate() { + let on = self + .tools + .get(name.as_str()) + .is_some_and(|inner| self.enabled(inner.as_ref(), config)); + if !on { + return Err(format!( + "step {} ({name}) is switched off, so no step ran", + index + 1 + )); + } + } + Ok(()) + } + fn enabled(&self, tool: &dyn Tool, config: &Config) -> bool { if tool.root_only() && !self.root || !self.platform.provides(tool.needs()) { return false; @@ -216,6 +251,9 @@ impl ToolRegistry { info!("tool {} called with invalid arguments", call.name); return error_result(problems); } + if let Err(problem) = self.check_within(tool.as_ref(), &arguments, config) { + return error_result(problem); + } let context = ToolContext { scope: Arc::clone(scope), config: Arc::clone(config), @@ -326,6 +364,57 @@ mod tests { assert_eq!(registry.kinds().len(), 1); } + /// A batch cannot run a tool the mode switched off: refused before anything is asked. + #[cfg(any(target_os = "macos", target_os = "linux"))] + #[tokio::test] + async fn a_batch_step_switched_off_refuses_the_batch() { + use crate::tools::{ComputerActions, PressKeys, Screen, TypeText}; + let dir = temp_dir("registry_batch_switch"); + std::fs::create_dir_all(&dir).unwrap(); + let scope = Arc::new(ToolScope::new(&dir, &dir).unwrap()); + let mut config = (*crate::testing::tool_context(&scope).config).clone(); + for (name, on) in [ + ("computer_actions", true), + ("press_keys", true), + ("type_text", false), + ] { + config.set_tool_enabled(name, on); + } + let config = Arc::new(config); + let screen = Arc::new(Screen::new(ArtifactStore::new(dir.clone()))); + let mut registry = ToolRegistry { + root: true, + ..ToolRegistry::default() + }; + registry.platform.gui = true; + registry.add(Box::new(ComputerActions(Arc::clone(&screen)))); + registry.add(Box::new(PressKeys(Arc::clone(&screen)))); + registry.add(Box::new(TypeText(screen))); + let call = ToolCall { + id: "c".into(), + name: "computer_actions".into(), + arguments: serde_json::json!({"steps": [ + {"tool": "press_keys", "arguments": {"keys": "enter"}}, + {"tool": "type_text", "arguments": {"text": "hi"}}, + ]}), + }; + let (events, mut asked) = tokio::sync::mpsc::unbounded_channel(); + let progress = Progress { + call_id: "c".into(), + events, + }; + let result = registry.call(&call, &scope, &config, progress).await; + assert!(result.is_error); + assert!( + result + .content + .contains("step 2 (type_text) is switched off"), + "{}", + result.content + ); + assert!(asked.try_recv().is_err(), "nothing may be asked"); + } + /// Providers reject a schema that is not one plain object at the top. #[tokio::test] async fn every_schema_is_one_object_providers_accept() { @@ -342,13 +431,16 @@ mod tests { tools.push(Box::new(AgentState::new(Arc::clone(&state), "t".into()))); #[cfg(any(target_os = "macos", target_os = "linux"))] { - use crate::tools::{Click, PressKeys, Screen, Screenshot, Scroll, TypeText}; + use crate::tools::{ + Click, ComputerActions, PressKeys, Screen, Screenshot, Scroll, TypeText, + }; let screen = Arc::new(Screen::new(ArtifactStore::new(dir.clone()))); tools.push(Box::new(Screenshot(Arc::clone(&screen)))); tools.push(Box::new(Click(Arc::clone(&screen)))); tools.push(Box::new(Scroll(Arc::clone(&screen)))); tools.push(Box::new(TypeText(Arc::clone(&screen)))); tools.push(Box::new(PressKeys(Arc::clone(&screen)))); + tools.push(Box::new(ComputerActions(Arc::clone(&screen)))); #[cfg(target_os = "macos")] tools.push(Box::new(crate::tools::WindowSession(screen))); } diff --git a/projects/core/src/tools/tool.rs b/projects/core/src/tools/tool.rs index 2d8564ba..cdccc928 100644 --- a/projects/core/src/tools/tool.rs +++ b/projects/core/src/tools/tool.rs @@ -74,40 +74,28 @@ impl ToolContext { Ok(resolved) } - /// The screen needs the user's yes once a session, the keyboard every time, unless - /// full access gives it. - pub async fn approve_screen(&self, action: &str, keyboard: bool) -> Result<()> { - let Some(approvals) = &self.approvals else { - return Ok(()); - }; - let action = action.to_owned(); - let request = match keyboard { - true => Request::Keyboard { action }, - false => Request::Screen { action }, - }; - let policy = self.approvals_policy().await; - approvals.ask(request, self.progress.clone(), policy).await - } - - /// A screen grant never permits physical pointer takeover. Without an approval - /// service only the explicitly selected full-access policy allows it. - pub async fn approve_system_pointer(&self, action: &str) -> Result<()> { + /// Screen actions, asked as one request: see [`Approvals::ask_all`]. The screen needs + /// the user's yes once a session, the keyboard every time, unless full access gives + /// it. A screen grant never permits the system pointer or a window, and without an + /// approval service only the explicitly selected full-access policy allows those. + pub async fn approve_computer(&self, requests: Vec) -> Result<()> { let Some(approvals) = &self.approvals else { + let guarded = requests.iter().any(|request| { + matches!( + request, + Request::SystemPointer { .. } | Request::Window { .. } + ) + }); anyhow::ensure!( - self.approvals_policy().await == ApprovalPolicy::FullAccess, - "system pointer control needs a separate approval service or explicit full access" + !guarded || self.approvals_policy().await == ApprovalPolicy::FullAccess, + "system pointer control and window use need an approval service or explicit \ + full access" ); return Ok(()); }; let policy = self.approvals_policy().await; approvals - .ask( - Request::SystemPointer { - action: action.to_owned(), - }, - self.progress.clone(), - policy, - ) + .ask_all(requests, self.progress.clone(), policy) .await } @@ -196,6 +184,12 @@ pub trait Tool: Send + Sync { true } + /// The tools this call runs inside it, each with its own arguments, so switches and + /// plugin hooks judge every one as if it were called alone. An error rejects the call. + fn calls_within(&self, _arguments: &Value) -> Result> { + Ok(Vec::new()) + } + /// An error is shown to the model as the result, so it can correct itself. fn call<'a>(&'a self, arguments: Value, context: ToolContext) -> BoxFuture<'a, String>; diff --git a/projects/protocol/src/client/hello.rs b/projects/protocol/src/client/hello.rs index 276c3504..684e6e79 100644 --- a/projects/protocol/src/client/hello.rs +++ b/projects/protocol/src/client/hello.rs @@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize}; /// Raised whenever a request, a response or an event changes shape. -pub const PROTOCOL_VERSION: u32 = 10; +pub const PROTOCOL_VERSION: u32 = 11; /// The first thing a client says. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] diff --git a/projects/protocol/src/thread/event.rs b/projects/protocol/src/thread/event.rs index e15bc639..3420936c 100644 --- a/projects/protocol/src/thread/event.rs +++ b/projects/protocol/src/thread/event.rs @@ -170,6 +170,7 @@ mod tests { command: "cargo test".into(), }, session: Some("cargo".into()), + steps: Vec::new(), })), }; let text = serde_json::to_string(&event).unwrap(); diff --git a/projects/protocol/src/tools/approval.rs b/projects/protocol/src/tools/approval.rs index 303e8b1f..5baf7a71 100644 --- a/projects/protocol/src/tools/approval.rs +++ b/projects/protocol/src/tools/approval.rs @@ -66,6 +66,54 @@ pub struct ApprovalRequest { /// What a yes for the session covers, as the core decides it: the folder of the /// path, or the program of a plain command. `None` when it cannot be offered. pub session: Option, + /// Every step of a batch of screen actions, in order; empty for a single request. + /// The user answers for all of them, so a client shows each one, never a summary. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub steps: Vec, +} + +/// One step of a batch as the user sees it before saying yes. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ApprovalStep { + /// Its place in the batch, from 1. + pub number: usize, + /// What the step would ask alone: its kind and what it does. + pub request: Request, + /// A session grant covers it already; it is listed so the user sees the whole batch. + pub allowed: bool, +} + +impl ApprovalStep { + /// The step in a line, such as `2. using the keyboard: press enter`. + pub fn line(&self) -> String { + let allowed = if self.allowed { + " (already allowed)" + } else { + "" + }; + format!("{}. {}{allowed}", self.number, self.request.what()) + } +} + +impl ApprovalRequest { + /// What `who` asks for, in a line: for a batch a neutral heading over its steps, + /// since no one kind says what all of them do. + pub fn title(&self, who: &str) -> String { + match self.steps.len() { + 0 => self.request.describe(who), + count => format!("{who} wants to run {count} computer actions"), + } + } + + /// The title and every step, a line each, for a surface that shows plain text only. + pub fn text(&self, who: &str) -> String { + let mut text = self.title(who); + for step in &self.steps { + text.push('\n'); + text.push_str(&step.line()); + } + text + } } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] diff --git a/projects/protocol/src/tools/mod.rs b/projects/protocol/src/tools/mod.rs index 8e53e3c4..70def574 100644 --- a/projects/protocol/src/tools/mod.rs +++ b/projects/protocol/src/tools/mod.rs @@ -8,7 +8,7 @@ mod question; mod result; pub use answered::{Answered, Asked}; -pub use approval::{ApprovalRequest, Decision, Request}; +pub use approval::{ApprovalRequest, ApprovalStep, Decision, Request}; pub use card::{Card, CardKind}; pub use phrase::{is_agent_tool, phrase}; pub use question::{