diff --git a/changes/computer-actions.md b/changes/computer-actions.md new file mode 100644 index 0000000..3ff9529 --- /dev/null +++ b/changes/computer-actions.md @@ -0,0 +1,9 @@ +Added +Computer use: `computer_actions` runs up to 20 clicks, typing, keys, scrolls and +screenshots in one call, checked and asked for together before the first runs, with +every click read in the picture taken before the batch. Every client and chat platform +lists each step of the request; tool switches, plugin hooks and the arbiter judge each +step as if it were called alone. `press_keys` now refuses line breaks and control +characters in the keys and shows them by their normal names. A chat platform +gets a request whole or cannot allow it: one too long for its message can only be +denied there and is answered in the app. diff --git a/docs/computer-use.md b/docs/computer-use.md index 128ecb8..2b1144f 100644 --- a/docs/computer-use.md +++ b/docs/computer-use.md @@ -43,6 +43,45 @@ the session does not allow it. ``` `type_text` and `press_keys` go to whatever has the keyboard focus and ask every time. +Keys are a key name with modifiers joined by `+`; only plain spaces may pad them, and +the request shows them by their normal names, such as `cmd+shift+t`. + +## Several actions in one call + +`computer_actions` runs up to 20 steps of `click`, `type_text`, `press_keys`, `scroll` +and `screenshot` in order, each with the arguments it takes alone, at most four +screenshots among them: + +```json +{"steps": [ + {"tool": "click", "arguments": {"x": 320, "y": 180}}, + {"tool": "type_text", "arguments": {"text": "hello"}}, + {"tool": "press_keys", "arguments": {"keys": "enter"}}, + {"tool": "screenshot", "arguments": {}} +]} +``` + +Every step is checked before anything is asked or done; a wrong one names its number +and nothing runs. A step whose tool is switched off in the mode, or that a plugin's +`before_tool` hook denies, refuses the whole batch the same way. + +The batch is approved before any step runs, in one request that lists every step with +its number, those a session grant covers marked "already allowed". So typing and keys +in a batch are approved before the user sees where the focus lands: approve one only +when the steps before it say where that is, or ask for the steps one by one. Steps of +one kind can be allowed for the session as that kind; mixed kinds are asked for this +time only, under a neutral heading. In arbiter mode each step is judged on its own, +and one no refuses the batch. So a batch is never asked for less than its steps one +by one. The terminal grows the request to show up to twelve rows of steps and pages +the rest; a yes there pages on until every step has been shown, and counts only a +moment after the last ones appear. The desktop scrolls a long list inside the request; +a chat platform such as Telegram gets every step in its message, and a request too +long for one message (over 3500 characters) is not offered for a yes there: it can +only be denied there, and is allowed in the app. + +Every x, y is in the picture taken before the batch, even after a screenshot inside +it; after the batch the newest picture counts. The batch stops at the first failing +step and says which ran. ## Linux diff --git a/docs/plugins.md b/docs/plugins.md index f9646c5..82105eb 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -102,6 +102,12 @@ A hook reads one JSON object on stdin: - `{"context": "…"}` adds a note the model reads: to the tool's result after a tool, or to the thread for the other events. +A call that runs other tools inside it, such as `computer_actions`, is checked first +as itself and then once per step, with that step's `tool` and `arguments` and +`"within": {"tool": "computer_actions", "step": 2}`, so a hook for `type_text` sees +typing in a batch too. A deny from any step refuses the whole call before anything is +asked or done. `after_tool` runs once, for the batch, with every step's result in it. + Printing nothing allows the action. Hooks run in the order the plugins are named. A `before_tool` hook that fails, prints something other than JSON or runs past its timeout denies the call: a check that could not run has not allowed anything. Other diff --git a/projects/clients/browser/src/chat/approval.rs b/projects/clients/browser/src/chat/approval.rs index 4f6581b..97c8219 100644 --- a/projects/clients/browser/src/chat/approval.rs +++ b/projects/clients/browser/src/chat/approval.rs @@ -23,6 +23,8 @@ pub fn Approvals(chat: Chat) -> impl IntoView { fn ApprovalCard(request: ApprovalRequest) -> impl IntoView { let app = state(); let id = request.id; + let lines: Vec = request.steps.iter().map(|step| step.line()).collect(); + let session = request.session.is_some(); let answer = move |decision: Decision| { run(api::approve(id, decision), move |_| { // The core closes the request itself; every client sees that event. @@ -31,10 +33,19 @@ fn ApprovalCard(request: ApprovalRequest) -> impl IntoView { }; view! {
-
{request.request.describe("The agent")}
+
{request.title("The agent")}
+ // A batch lists every step, numbered: the yes covers all of them. + {(!lines.is_empty()).then(|| view! { +
+ {lines.into_iter().map(|line| view! {
{line}
}).collect_view()} +
+ })}
- + // Only what the core can keep for the session is offered, as the other clients do. + {session.then(|| view! { + + })}
diff --git a/projects/clients/browser/src/chat/data/tests.rs b/projects/clients/browser/src/chat/data/tests.rs index 9edacad..26e8927 100644 --- a/projects/clients/browser/src/chat/data/tests.rs +++ b/projects/clients/browser/src/chat/data/tests.rs @@ -54,6 +54,7 @@ fn an_approval_shows_until_it_is_closed() { command: "rm x".into(), }, session: None, + steps: Vec::new(), }; chat.apply(ThreadEvent::Approval(request)); assert_eq!(chat.approvals.len(), 1); diff --git a/projects/clients/browser/style.css b/projects/clients/browser/style.css index aaaf89f..e8bb8ee 100644 --- a/projects/clients/browser/style.css +++ b/projects/clients/browser/style.css @@ -154,6 +154,7 @@ input:is([type=radio], [type=checkbox]) { accent-color: var(--accent); } .card { margin: 10px auto; } .card img { max-width: 100%; border-radius: 10px; display: block; } .approvals, .question { max-width: 820px; margin: 0 auto; width: calc(100% - 40px); } .approval, .question { border: 1px solid var(--accent); border-radius: 12px; padding: 10px 14px; margin-bottom: 8px; background: var(--surface); } +.approval-steps { margin: 6px 0 0 12px; display: flex; flex-direction: column; gap: 2px; overflow-wrap: anywhere; } .approval-actions, .row-actions { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; margin: 8px 0; } .option { display: grid; grid-template-columns: var(--s4) minmax(0, 1fr); align-items: start; gap: var(--s3); diff --git a/projects/clients/desktop/src/dev/screenshot.rs b/projects/clients/desktop/src/dev/screenshot.rs index 17d84d1..b5ffec0 100644 --- a/projects/clients/desktop/src/dev/screenshot.rs +++ b/projects/clients/desktop/src/dev/screenshot.rs @@ -730,6 +730,7 @@ fn act( command: "cargo test --workspace".into(), }, session: Some("cargo".into()), + steps: Vec::new(), }), }; if let Some(chat) = ws.shown.clone().and_then(|id| ws.chats.get_mut(&id)) { diff --git a/projects/clients/desktop/src/pages/chat/approval.rs b/projects/clients/desktop/src/pages/chat/approval.rs index 102070d..a3ebfc5 100644 --- a/projects/clients/desktop/src/pages/chat/approval.rs +++ b/projects/clients/desktop/src/pages/chat/approval.rs @@ -56,7 +56,27 @@ pub fn render(ws: &Workspace, cx: &mut Context) -> Div { .on_click(cx.listener(move |ws, _, _, cx| ws.approve(id, Decision::Deny, cx))), ); // A command is shown as code, in full: it came from the model and is about to run. + // A batch shows every step, one row each: the yes covers all of them. let what = match &request.request { + // A long batch scrolls inside the card, so the buttons stay in view. + _ if !request.steps.is_empty() => { + let steps = request.steps.iter().fold( + div() + .id(SharedString::from(format!("steps-{id}"))) + .max_h(px(240.)) + .overflow_y_scroll() + .flex() + .flex_col() + .gap_1(), + |list, step| list.child(div().pl_3().child(step.line())), + ); + div() + .flex() + .flex_col() + .gap_1() + .child(request.title("The agent")) + .child(steps) + } Request::Command { command } => { let mono = crate::ui::theme::prefs(cx) .mono_font diff --git a/projects/clients/terminal/src/app/mod.rs b/projects/clients/terminal/src/app/mod.rs index f1ac2b3..84a68a6 100644 --- a/projects/clients/terminal/src/app/mod.rs +++ b/projects/clients/terminal/src/app/mod.rs @@ -32,6 +32,8 @@ pub use panels::{ pub use panels::{Choice, Confirm, Step, Workshop, choices}; pub use state::{App, compact_number}; pub use tabs::{Tab, TabHit, TabMove}; +#[cfg(test)] +pub(crate) use thread::approval::tests::batch as approval_batch; pub use thread::approval::{ApprovalHit, Asked}; pub use thread::goal::{GoalButton, GoalHit, buttons as goal_buttons}; pub use thread::poll::{Poll, PollHit, PollTarget}; diff --git a/projects/clients/terminal/src/app/thread/approval.rs b/projects/clients/terminal/src/app/thread/approval.rs index 8e4b37f..8378dcf 100644 --- a/projects/clients/terminal/src/app/thread/approval.rs +++ b/projects/clients/terminal/src/app/thread/approval.rs @@ -2,6 +2,8 @@ //! everything else, the oldest first. Enter answers with the row under the cursor, Esc //! denies; the core closes a request whose call is gone. +use std::time::{Duration, Instant}; + use crossterm::event::{KeyCode, KeyEvent}; use dusk_protocol::tools::{ApprovalRequest, Decision, Request}; @@ -12,8 +14,19 @@ pub struct Asked { /// The agent that asks; `None` for the chat itself. pub agent: Option, pub cursor: usize, + /// The first row of a batch's steps in view, when they do not all fit. + pub step_scroll: usize, + /// Step rows in view on the last drawn frame. + pub step_page: usize, + /// When every step of a batch had first been on screen. Until then a yes pages on + /// instead of answering, so no step is allowed unseen. + pub steps_seen_at: Option, } +/// After the last steps of a batch first show, a yes waits this long, so a held or +/// repeated Enter that paged through does not also allow what it just showed. +pub const SETTLE: Duration = Duration::from_millis(600); + impl Asked { /// The answers offered, in order; the session one only when the core can keep it. pub fn choices(&self) -> Vec<(Decision, String)> { @@ -34,10 +47,20 @@ impl Asked { choices } + /// A yes to a batch waits until every step has been on screen for [`SETTLE`]; a no + /// and a single request never wait. + pub fn answers_now(&self, decision: Decision, now: Instant) -> bool { + decision == Decision::Deny + || self.request.steps.is_empty() + || self + .steps_seen_at + .is_some_and(|at| now.saturating_duration_since(at) >= SETTLE) + } + /// What the tool asks for, in a line. pub fn title(&self) -> String { let who = self.agent.as_deref().unwrap_or("the agent"); - self.request.request.describe(who) + self.request.title(who) } } @@ -55,6 +78,9 @@ impl App { request, agent, cursor: 0, + step_scroll: 0, + step_page: 0, + steps_seen_at: None, }); } @@ -70,6 +96,10 @@ impl App { match key.code { KeyCode::Up => asked.cursor = asked.cursor.saturating_sub(1), KeyCode::Down => asked.cursor = (asked.cursor + 1).min(last), + KeyCode::PageDown => asked.step_scroll += asked.step_page.max(1), + KeyCode::PageUp => { + asked.step_scroll = asked.step_scroll.saturating_sub(asked.step_page.max(1)) + } KeyCode::Enter => return self.answer_approval(None), KeyCode::Esc => return self.answer_approval(Some(last)), _ => {} @@ -82,12 +112,69 @@ impl App { if self.approvals.is_empty() { return Action::None; } - let asked = self.approvals.remove(0); + let asked = &mut self.approvals[0]; let choices = asked.choices(); let at = choice.unwrap_or(asked.cursor).min(choices.len() - 1); + if !asked.answers_now(choices[at].0, Instant::now()) { + // Still unseen steps: show them. Just shown: wait, as if not pressed. + if asked.steps_seen_at.is_none() { + asked.step_scroll += asked.step_page.max(1); + } + return Action::None; + } + let asked = self.approvals.remove(0); Action::Approve { id: asked.request.id, decision: choices[at].0, } } } + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use dusk_protocol::tools::ApprovalStep; + + /// A batch of `count` keyboard steps, each typing `text`. + pub(crate) fn batch(count: usize, text: &str) -> Asked { + let steps: Vec = (1..=count) + .map(|number| ApprovalStep { + number, + request: Request::Keyboard { + action: format!("type \"{text}\""), + }, + allowed: false, + }) + .collect(); + Asked { + request: ApprovalRequest { + id: 1, + request: Request::Keyboard { + action: format!("{count} computer actions"), + }, + session: None, + steps, + }, + agent: None, + cursor: 0, + step_scroll: 0, + step_page: 0, + steps_seen_at: None, + } + } + + #[test] + fn a_batch_is_allowed_only_a_while_after_every_step_was_shown() { + let now = Instant::now(); + let mut asked = batch(20, "hi"); + assert!(!asked.answers_now(Decision::Once, now)); + assert!(!asked.answers_now(Decision::Session, now)); + assert!(asked.answers_now(Decision::Deny, now)); + asked.steps_seen_at = Some(now); + // A held Enter that showed the last page does not also allow it. + assert!(!asked.answers_now(Decision::Once, now + SETTLE / 2)); + assert!(asked.answers_now(Decision::Deny, now)); + assert!(asked.answers_now(Decision::Once, now + SETTLE)); + assert!(batch(0, "").answers_now(Decision::Once, now)); + } +} diff --git a/projects/clients/terminal/src/ui/thread/input/approval.rs b/projects/clients/terminal/src/ui/thread/input/approval.rs index c33bb38..a2d5b6c 100644 --- a/projects/clients/terminal/src/ui/thread/input/approval.rs +++ b/projects/clients/terminal/src/ui/thread/input/approval.rs @@ -1,7 +1,7 @@ //! The question for a tool's yes, shown over the input. use super::InputView; -use crate::app::{App, ApprovalHit}; +use crate::app::{App, ApprovalHit, Asked}; use crate::ui::thread::{ theme::{ACCENT, DUSK, NEGATIVE, TEXT, bold, dim, fg}, wrap, @@ -15,11 +15,40 @@ use ratatui::{ widgets::{Block, BorderType, Clear, Paragraph}, }; -/// The border, the request, its choices and the hint. -pub(super) fn approval_rows(app: &App) -> u16 { - app.approvals - .first() - .map_or(0, |asked| asked.choices().len() as u16 + 4) +/// Rows of a batch's steps the box grows to; more are paged, never cut. +const MAX_STEP_ROWS: usize = 12; + +/// Steps sit under the title, indented past its leading space. +const STEP_INDENT: &str = " "; + +/// The border, the request, a batch's steps, its choices and the hint. +pub(super) fn approval_rows(app: &App, width: u16) -> u16 { + app.approvals.first().map_or(0, |asked| { + let steps = step_lines(asked, width.saturating_sub(2) as usize).len(); + let paged = steps > MAX_STEP_ROWS; + let shown = steps.min(MAX_STEP_ROWS) + usize::from(paged); + (asked.choices().len() + 4 + shown) as u16 + }) +} + +/// Every step of a batch, each wrapped whole to the box: the user answers for all of +/// them, so none is clipped. Control characters are drawn as `?`. +fn step_lines(asked: &Asked, width: usize) -> Vec { + let room = width.saturating_sub(STEP_INDENT.len()).max(1); + asked + .request + .steps + .iter() + .flat_map(|step| { + let line: String = step + .line() + .chars() + .map(|c| if c.is_control() { '?' } else { c }) + .collect(); + wrap::chars(&line, room) + }) + .map(|row| format!("{STEP_INDENT}{row}")) + .collect() } pub(super) const SEND: &str = " send "; @@ -53,6 +82,34 @@ impl InputView<'_> { wrap::clip(&format!(" {}", asked.title()), width), bold(TEXT), ))]; + let steps = step_lines(asked, width); + let choices = asked.choices().len(); + // What is left for steps once the title, choices and hint have their rows. + let room = (inner.height as usize).saturating_sub(choices + 2); + let paged = steps.len() > room; + let page = match paged { + true => room.saturating_sub(1).max(1), + false => steps.len(), + }; + let first = asked.step_scroll.min(steps.len().saturating_sub(page)); + let shown = steps.iter().skip(first).take(page); + lines.extend(shown.map(|row| Line::from(Span::raw(row.clone())))); + if paged { + let last = (first + page).min(steps.len()); + lines.push(Line::from(Span::styled( + wrap::clip( + &format!( + " rows {}-{last} of {} pgup/pgdn a yes shows the rest first", + first + 1, + steps.len() + ), + width, + ), + dim(), + ))); + } + let step_rows = lines.len() as u16 - 1; + let seen = first + page >= steps.len(); for (row, (decision, label)) in asked.choices().into_iter().enumerate() { let here = asked.cursor == row; let pointer = if here { " \u{203A} " } else { " " }; @@ -61,7 +118,7 @@ impl InputView<'_> { (true, _) => bold(ACCENT), (false, _) => Style::new(), }; - let y = inner.y + 1 + row as u16; + let y = inner.y + 1 + step_rows + row as u16; if y < inner.bottom() { self.app.approval_hits.push(ApprovalHit { row: y, @@ -80,5 +137,30 @@ impl InputView<'_> { dim(), ))); frame.render_widget(Paragraph::new(lines), inner); + if let Some(asked) = self.app.approvals.first_mut() { + asked.step_scroll = first; + asked.step_page = page; + if seen && asked.steps_seen_at.is_none() { + asked.steps_seen_at = Some(std::time::Instant::now()); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::app::approval_batch as batch; + + #[test] + fn every_step_is_drawn_whole_however_long() { + let text = "x".repeat(150); + let asked = batch(20, &text); + let rows = step_lines(&asked, 40); + assert!(rows.len() > MAX_STEP_ROWS); + assert!(rows.iter().all(|row| row.chars().count() <= 40)); + let drawn: String = rows.iter().map(|row| &row[STEP_INDENT.len()..]).collect(); + let whole: String = asked.request.steps.iter().map(|step| step.line()).collect(); + assert_eq!(drawn, whole); } } diff --git a/projects/clients/terminal/src/ui/thread/input/mod.rs b/projects/clients/terminal/src/ui/thread/input/mod.rs index 0b25895..2953087 100644 --- a/projects/clients/terminal/src/ui/thread/input/mod.rs +++ b/projects/clients/terminal/src/ui/thread/input/mod.rs @@ -49,7 +49,7 @@ pub struct InputView<'a> { /// Rows the box needs: the queued messages, the rule, the padding and the wrapped text. pub fn height(app: &App, width: u16) -> u16 { let (rows, _) = layout(&app.input, text_width(width)); - approval_rows(app) + approval_rows(app, width) + poll_rows(app, width) + goal_rows(app) + queue_rows(app) @@ -72,7 +72,7 @@ impl InputView<'_> { height: area.height.saturating_sub(1), ..area }; - let approval = approval_rows(self.app).min(area.height); + let approval = approval_rows(self.app, area.width).min(area.height); self.draw_approval( frame, Rect { diff --git a/projects/core/src/plugins/hooks.rs b/projects/core/src/plugins/hooks.rs index fa29f9f..03af467 100644 --- a/projects/core/src/plugins/hooks.rs +++ b/projects/core/src/plugins/hooks.rs @@ -108,6 +108,42 @@ impl Hooks { } } +impl Hooks { + /// The checks before a call, then before each call it runs inside, with that tool's + /// name and its own arguments, as if it were called alone. `base` is what every hook + /// is told about the thread. A deny from any of them refuses the whole call. + pub async fn before_tool( + &self, + tool: &str, + arguments: &Value, + within: &[(String, Value)], + base: &Value, + ) -> Option { + let mut details = base.clone(); + details["tool"] = json!(tool); + details["arguments"] = arguments.clone(); + if let Some(denied) = self + .run(HookEvent::BeforeTool, Some(tool), details) + .await + .denied + { + return Some(denied); + } + for (index, (name, inner)) in within.iter().enumerate() { + let number = index + 1; + let mut details = base.clone(); + details["tool"] = json!(name); + details["arguments"] = inner.clone(); + details["within"] = json!({ "tool": tool, "step": number }); + let outcome = self.run(HookEvent::BeforeTool, Some(name), details).await; + if let Some(denied) = outcome.denied { + return Some(format!("step {number} ({name}): {denied}; no step ran")); + } + } + None + } +} + fn applies(hook: &Hook, event: HookEvent, tool: Option<&str>) -> bool { hook.config.event == event && (hook.config.tools.is_empty() @@ -206,6 +242,37 @@ mod tests { .await } + #[tokio::test] + async fn every_call_inside_a_batch_meets_its_own_checks() { + let mut scoped = hook( + HookEvent::BeforeTool, + r#"if grep -q '"step":2'; then echo '{"decision":"deny","reason":"no typing"}'; fi"#, + ); + scoped.config.tools = vec!["type_text".into()]; + let hooks = Hooks::default(); + hooks.set(vec![scoped]); + let within = [ + ("click".to_owned(), json!({"x": 1, "y": 2})), + ("type_text".to_owned(), json!({"text": "hi"})), + ]; + let base = json!({ "thread": "t" }); + let denied = hooks + .before_tool("computer_actions", &json!({}), &within, &base) + .await; + assert_eq!( + denied.as_deref(), + Some("step 2 (type_text): plugin test refused: no typing; no step ran") + ); + // The same step alone in a batch of one is checked as step 1, and allowed here. + let alone = [("type_text".to_owned(), json!({"text": "hi"}))]; + assert!( + hooks + .before_tool("computer_actions", &json!({}), &alone, &base) + .await + .is_none() + ); + } + #[tokio::test] async fn a_check_before_a_tool_denies_and_fails_closed() { let denied = before_tool(r#"echo '{"decision":"deny","reason":"no"}'"#, "run_command"); diff --git a/projects/core/src/plugins/platform/session.rs b/projects/core/src/plugins/platform/session.rs index 0445a84..13ba411 100644 --- a/projects/core/src/plugins/platform/session.rs +++ b/projects/core/src/plugins/platform/session.rs @@ -24,6 +24,18 @@ use super::{ wire::{FromPlatform, MAX_LINE_BYTES, ToPlatform}, }; +/// The longest approval text a platform is sent. Telegram cuts a message at 4096 +/// characters, and its plugin at less; a cut request would allow steps the user never +/// saw, so a longer one is not offered for a yes at all. +const MAX_APPROVAL_CHARS: usize = 3500; + +/// What the platform shows for a yes: the title and every step, or `None` when that +/// does not fit, and the request may then only be denied there. +fn approvable_text(request: &ApprovalRequest) -> Option { + let text = request.text("the agent"); + (text.chars().count() <= MAX_APPROVAL_CHARS).then_some(text) +} + /// One run of a platform's program, bridged to its thread. pub(super) struct Session<'a> { launcher: &'a Launcher, @@ -230,9 +242,20 @@ impl<'a> Session<'a> { let Some(request) = self.approvals.first() else { return Ok(()); }; - let (id, text) = (request.id, request.request.describe("the agent")); + let id = request.id; self.shown = Some(id); - self.send(ToPlatform::Approval { id, text }).await + match approvable_text(request) { + Some(text) => self.send(ToPlatform::Approval { id, text }).await, + // No button: deny still answers it, the app can still allow it. + None => { + let note = format!( + "{} The request is too long to show here in full, so it cannot be \ + allowed here: answer it in the duskcode app, or deny it with /deny.", + request.title("The agent") + ); + self.reply(¬e).await + } + } } /// Does what the user did on the platform, as if at the keyboard. @@ -261,6 +284,15 @@ impl<'a> Session<'a> { else { return self.reply("Nothing is waiting for that approval.").await; }; + let unseen = self + .approvals + .iter() + .any(|request| request.id == id && approvable_text(request).is_none()); + if allow && unseen { + return self + .reply("That request is too long to allow here: answer it in the app, or deny it.") + .await; + } let decision = if allow { Decision::Once } else { @@ -393,3 +425,35 @@ impl<'a> Session<'a> { } } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::tools::Request; + use dusk_protocol::tools::ApprovalStep; + + fn batch(steps: usize, typed: usize) -> ApprovalRequest { + let text = "x".repeat(typed); + let keyboard = |action: String| Request::Keyboard { action }; + ApprovalRequest { + id: 1, + request: keyboard(format!("{steps} computer actions")), + session: None, + steps: (1..=steps) + .map(|number| ApprovalStep { + number, + request: keyboard(format!("type \"{text}\"")), + allowed: false, + }) + .collect(), + } + } + + #[test] + fn a_request_too_long_to_show_whole_is_never_offered_for_a_yes() { + let short = approvable_text(&batch(3, 20)).expect("fits"); + assert!(short.contains("3. using the keyboard"), "{short}"); + // Twenty typed steps of 200 shown characters would be cut by the platform. + assert!(approvable_text(&batch(20, 200)).is_none()); + } +} diff --git a/projects/core/src/runtime/agent/agent/calls.rs b/projects/core/src/runtime/agent/agent/calls.rs index 02d631f..2a7afbf 100644 --- a/projects/core/src/runtime/agent/agent/calls.rs +++ b/projects/core/src/runtime/agent/agent/calls.rs @@ -116,9 +116,12 @@ impl Agent { .is_some_and(|g| !g.approved && g.status != GoalStatus::Done); let denied = match stop { None if !self.hooks.is_empty() => { - let details = json!({ "tool": call.name, "arguments": call.arguments }); - let checked = self.run_hooks(HookEvent::BeforeTool, Some(&call.name), details); - checked.await.denied + let within = self.tools.calls_within(&call.name, &call.arguments); + let base = json!({ "thread": self.thread_id, "depth": self.depth }); + let checked = + self.hooks + .before_tool(&call.name, &call.arguments, &within, &base); + checked.await } _ => None, }; diff --git a/projects/core/src/runtime/thread/launcher.rs b/projects/core/src/runtime/thread/launcher.rs index 119d96a..a834285 100644 --- a/projects/core/src/runtime/thread/launcher.rs +++ b/projects/core/src/runtime/thread/launcher.rs @@ -137,13 +137,16 @@ impl Launcher { } #[cfg(any(target_os = "macos", target_os = "linux"))] { - use crate::tools::{Click, PressKeys, Screen, Screenshot, Scroll, TypeText}; + use crate::tools::{ + Click, ComputerActions, PressKeys, Screen, Screenshot, Scroll, TypeText, + }; let screen = Arc::new(Screen::new(ArtifactStore::new(self.paths.artifacts_dir()))); tools.add(Box::new(Screenshot(Arc::clone(&screen)))); tools.add(Box::new(Click(Arc::clone(&screen)))); tools.add(Box::new(Scroll(Arc::clone(&screen)))); tools.add(Box::new(TypeText(Arc::clone(&screen)))); tools.add(Box::new(PressKeys(Arc::clone(&screen)))); + tools.add(Box::new(ComputerActions(Arc::clone(&screen)))); #[cfg(target_os = "macos")] tools.add(Box::new(crate::tools::WindowSession(screen))); } diff --git a/projects/core/src/tools/approvals.rs b/projects/core/src/tools/approvals.rs index f3f5066..42485e1 100644 --- a/projects/core/src/tools/approvals.rs +++ b/projects/core/src/tools/approvals.rs @@ -10,13 +10,13 @@ use std::{ use parking_lot::Mutex; -use anyhow::{Result, bail}; +use anyhow::{Context, Result, bail}; use tokio::sync::{mpsc, oneshot}; use super::tool::Progress; use crate::{config::ApprovalPolicy, runtime::ThreadEvent}; -pub use dusk_protocol::tools::{ApprovalRequest, Request, WindowAccess}; +pub use dusk_protocol::tools::{ApprovalRequest, ApprovalStep, Request, WindowAccess}; pub use dusk_protocol::tools::Decision; @@ -70,11 +70,92 @@ impl Approvals { if policy == ApprovalPolicy::Arbiter { return self.arbitrate(request, progress).await; } + let session = session_grant(&request); + self.prompt(request, session, Vec::new(), progress).await + } + + /// Screen actions asked as one: never less than asking each alone. A request the + /// session grants already is not asked again but still listed, with its number in + /// the batch, so the user sees every step. Steps of one kind may be allowed for the + /// session as that kind; mixed kinds are asked as the strictest of them for this + /// time only, since one session yes must not stand for another kind's. The arbiter + /// judges each step on its own. + pub async fn ask_all( + &self, + requests: Vec, + progress: Option, + policy: ApprovalPolicy, + ) -> Result<()> { + if policy == ApprovalPolicy::FullAccess { + return Ok(()); + } + if requests.len() < 2 || requests.iter().any(|r| strictness(r).is_none()) { + // A single request looks as it always has; only screen kinds merge. + for request in requests { + self.ask(request, progress.clone(), policy).await?; + } + return Ok(()); + } + let steps: Vec = { + let grants = self.grants.lock(); + requests + .into_iter() + .enumerate() + .map(|(index, request)| ApprovalStep { + number: index + 1, + allowed: grants.covers(&request), + request, + }) + .collect() + }; + let open: Vec<&ApprovalStep> = steps.iter().filter(|step| !step.allowed).collect(); + let Some(first) = open.first().map(|step| &step.request) else { + return Ok(()); + }; + if policy == ApprovalPolicy::Arbiter { + for step in &open { + let judged = self.arbitrate(step.request.clone(), progress.clone()); + judged + .await + .with_context(|| format!("step {} of the batch", step.number))?; + } + return Ok(()); + } + let same = open.iter().all(|step| same_kind(&step.request, first)); + let strictest = open + .iter() + .map(|step| &step.request) + .max_by_key(|r| strictness(r)) + .unwrap_or(first); + // The steps go in the text too, so whatever shows only the request, such as a + // message on a chat platform, still shows each step it allows. + let lines: Vec = steps.iter().map(ApprovalStep::line).collect(); + let action = format!("{} computer actions:\n{}", steps.len(), lines.join("\n")); + let request = with_action(strictest, action); + let session = if same { session_grant(&request) } else { None }; + self.prompt(request, session, steps, progress).await + } + + /// Shows the request to the user and waits. A yes for the session + /// adds a grant only when `session` offered one. + async fn prompt( + &self, + request: Request, + session: Option, + steps: Vec, + progress: Option, + ) -> Result<()> { let Some(progress) = progress else { - bail!( - "{} needs the user's approval, and no one can answer here", - request.what() - ); + let what = match steps.is_empty() { + true => request.what(), + false => steps + .iter() + .filter(|step| !step.allowed) + .map(ApprovalStep::line) + .collect::>() + .join("; "), + }; + bail!("{what} needs the user's approval, and no one can answer here"); }; let id = self.next.fetch_add(1, Ordering::Relaxed) + 1; let (sender, answer) = oneshot::channel(); @@ -85,16 +166,19 @@ impl Approvals { id, events: progress.events.clone(), }; - let session = session_grant(&request); + let offered = session.is_some(); progress.emit(ThreadEvent::Approval(ApprovalRequest { id, request: request.clone(), session, + steps, })); match answer.await { Ok(Decision::Once) => Ok(()), Ok(Decision::Session) => { - self.grants.lock().add(&request); + if offered { + self.grants.lock().add(&request); + } Ok(()) } Ok(Decision::Deny) | Err(_) => bail!( @@ -202,6 +286,40 @@ impl Grants { } } +/// How much a screen request can do, for a merged request to take the strictest kind. +/// `None` for requests that are not about the screen. +fn strictness(request: &Request) -> Option { + match request { + Request::Screen { .. } => Some(0), + Request::Window { .. } => Some(1), + Request::SystemPointer { .. } => Some(2), + Request::Keyboard { .. } => Some(3), + Request::Path { .. } | Request::Command { .. } => None, + } +} + +/// Whether one session grant would cover both. +fn same_kind(a: &Request, b: &Request) -> bool { + match (a, b) { + (Request::Window { grant: a, .. }, Request::Window { grant: b, .. }) => a == b, + _ => std::mem::discriminant(a) == std::mem::discriminant(b), + } +} + +/// `kind` asking for `action` instead. +fn with_action(kind: &Request, action: String) -> Request { + match kind { + Request::Screen { .. } => Request::Screen { action }, + Request::SystemPointer { .. } => Request::SystemPointer { action }, + Request::Keyboard { .. } => Request::Keyboard { action }, + Request::Window { grant, .. } => Request::Window { + grant: grant.clone(), + action, + }, + Request::Path { .. } | Request::Command { .. } => kind.clone(), + } +} + /// What a yes for the session covers, for the client to name: the folder of the path, /// or the program of a plain command. `None` when the session cannot be offered. pub fn session_grant(request: &Request) -> Option { diff --git a/projects/core/src/tools/computer/batch.rs b/projects/core/src/tools/computer/batch.rs new file mode 100644 index 0000000..a85ee7e --- /dev/null +++ b/projects/core/src/tools/computer/batch.rs @@ -0,0 +1,199 @@ +//! `computer_actions`: several screen actions in one call, so a known sequence costs one +//! round trip. Every step is checked and asked for before the first one runs, and the +//! batch is never asked for less than its steps would be one by one. + +use std::sync::Arc; + +use anyhow::{Context, Result, ensure}; +use schemars::JsonSchema; +use serde::Deserialize; +use serde_json::{Map, Value}; + +use super::{Click, PressKeys, Screenshot, Scroll, TypeText, screen::Screen, step::Step}; +use crate::{ + BoxFuture, + platform::Needs, + tools::{ + ToolResult, schema, + schema::parse_arguments, + tool::{Tool, ToolContext, ToolKind}, + }, +}; + +/// Steps in one call at most. +const MAX_STEPS: usize = 20; +/// Screenshots in one call at most, as many pictures as an MCP result may bring. +const MAX_PICTURES: usize = 4; + +pub struct ComputerActions(pub Arc); + +#[derive(Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct BatchArguments { + /// The actions, run in order; the batch stops at the first that fails. + #[schemars(length(min = 1, max = MAX_STEPS))] + steps: Vec, +} + +#[derive(Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct StepArguments { + tool: StepTool, + /// What that tool takes when called alone. + arguments: Map, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +enum StepTool { + Click, + TypeText, + PressKeys, + Scroll, + Screenshot, +} + +impl StepTool { + fn name(self) -> &'static str { + match self { + Self::Click => "click", + Self::TypeText => "type_text", + Self::PressKeys => "press_keys", + Self::Scroll => "scroll", + Self::Screenshot => "screenshot", + } + } +} + +/// The steps as the model gave them, bounded. Hooks, switches and the run all read +/// them through this one parse, so none sees other steps than the rest. +fn parse_steps(arguments: Value) -> Result> { + let BatchArguments { steps } = parse_arguments(arguments)?; + ensure!( + (1..=MAX_STEPS).contains(&steps.len()), + "give 1 to {MAX_STEPS} steps, not {}", + steps.len() + ); + let pictures = steps + .iter() + .filter(|step| step.tool == StepTool::Screenshot) + .count(); + ensure!( + pictures <= MAX_PICTURES, + "take at most {MAX_PICTURES} screenshots in one batch" + ); + Ok(steps) +} + +impl ComputerActions { + /// Every step checked, in order, before anything is asked or done. + fn steps(&self, arguments: Value) -> Result> { + let steps = parse_steps(arguments)?; + steps + .into_iter() + .enumerate() + .map(|(index, step)| { + let arguments = Value::Object(step.arguments); + let screen = &self.0; + let checked = match step.tool { + StepTool::Click => Click::step(screen, arguments), + StepTool::TypeText => TypeText::step(screen, arguments), + StepTool::PressKeys => PressKeys::step(screen, arguments), + StepTool::Scroll => Scroll::step(screen, arguments), + StepTool::Screenshot => Screenshot::step(screen, arguments), + }; + let checked = checked.with_context(|| { + format!( + "step {} ({}) is wrong, so no step ran", + index + 1, + step.tool.name() + ) + })?; + Ok((step.tool, checked)) + }) + .collect() + } +} + +/// Runs the asked-for steps in order and stops at the first failure. Every step reads x +/// and y in the picture the model saw before the batch, not in one taken meanwhile that +/// it has not seen yet; after the batch the newest picture counts again. +async fn run(screen: &Screen, steps: Vec<(StepTool, Step)>) -> ToolResult { + let total = steps.len(); + let seen = screen.coordinates(); + let mut newest = seen; + let mut result = ToolResult::text(String::new()); + let mut lines = Vec::with_capacity(total); + for (index, (tool, step)) in steps.into_iter().enumerate() { + let number = index + 1; + screen.set_coordinates(seen); + match step.run().await { + Ok(done) => { + if tool == StepTool::Screenshot { + newest = screen.coordinates(); + } + lines.push(format!("{number}. {}", done.content)); + result.artifacts.extend(done.artifacts); + } + Err(error) => { + lines.push(format!("{number}. failed: {error:#}")); + if number < total { + lines.push(format!("steps {} to {total} did not run", number + 1)); + } + result.is_error = true; + break; + } + } + } + screen.set_coordinates(newest); + result.content = lines.join("\n"); + result +} + +impl Tool for ComputerActions { + computer_tool!("computer_actions"); + + fn description(&self, _: &ToolContext) -> String { + format!( + "Run up to {MAX_STEPS} of click, type_text, press_keys, scroll and screenshot in \ + order in one call, each with the arguments it takes alone. Use it when you know \ + the coordinates and need not look between the actions, such as click a field, \ + type, press enter, then a screenshot to check. Every x, y is in the screenshot \ + taken before the batch. It asks the user once for all the steps and stops at \ + the first that fails." + ) + } + + fn parameters(&self) -> Value { + schema::of::() + } + + fn calls_within(&self, arguments: &Value) -> Result> { + let steps = parse_steps(arguments.clone())?; + Ok(steps + .into_iter() + .map(|step| (step.tool.name().to_owned(), Value::Object(step.arguments))) + .collect()) + } + + fn call<'a>(&'a self, _: Value, _: ToolContext) -> BoxFuture<'a, String> { + Box::pin(async { anyhow::bail!("computer_actions returns its steps' results") }) + } + + fn call_result<'a>( + &'a self, + arguments: Value, + context: ToolContext, + ) -> BoxFuture<'a, ToolResult> { + Box::pin(async move { + let steps = self.steps(arguments)?; + let requests = steps.iter().map(|(_, step)| step.request.clone()).collect(); + context.approve_computer(requests).await?; + Ok(run(&self.0, steps).await) + }) + } +} + +#[cfg(test)] +#[path = "batch_tests.rs"] +mod tests; diff --git a/projects/core/src/tools/computer/batch_tests.rs b/projects/core/src/tools/computer/batch_tests.rs new file mode 100644 index 0000000..18a167f --- /dev/null +++ b/projects/core/src/tools/computer/batch_tests.rs @@ -0,0 +1,291 @@ +//! A batch is checked and asked for whole before any step runs. No test here touches +//! the desktop: steps that would are refused first, or replaced by fakes. + +use std::sync::Arc; + +use serde_json::json; +use tokio::sync::mpsc; + +use super::*; +use crate::{ + config::ApprovalPolicy, + runtime::ThreadEvent, + state::artifacts::ArtifactStore, + testing::{temp_dir, tool_context}, + tools::{ + Approvals, Decision, Judge, Request, ToolScope, Verdict, + computer::screen::View, + tool::{Progress, ToolContext}, + }, +}; + +fn batch() -> ComputerActions { + ComputerActions(Arc::new(Screen::new(ArtifactStore::new( + std::env::temp_dir(), + )))) +} + +/// A context that asks through `approvals`; with no `progress` no one can answer. +fn context(approvals: &Arc, progress: Option) -> ToolContext { + let dir = temp_dir("computer_batch"); + std::fs::create_dir_all(&dir).unwrap(); + let mut context = tool_context(&ToolScope::new(&dir, &dir).unwrap()); + context.approvals = Some(Arc::clone(approvals)); + context.progress = progress; + context +} + +fn error(result: Result) -> String { + format!("{:#}", result.expect_err("the batch should fail")) +} + +fn fake() -> Request { + Request::Screen { + action: "fake".into(), + } +} + +#[test] +fn too_many_steps_and_unknown_tools_are_refused() { + let step = json!({"tool": "press_keys", "arguments": {"keys": "enter"}}); + let steps = vec![step; MAX_STEPS + 1]; + assert!(batch().steps(json!({ "steps": steps })).is_err()); + for wrong in [ + json!({"steps": []}), + json!({"steps": [{"tool": "run_command", "arguments": {}}]}), + json!({"steps": [{"tool": "window_session", "arguments": {}}]}), + json!({"steps": [{"tool": "click", "arguments": {}, "extra": 1}]}), + json!({"steps": [], "extra": 1}), + ] { + assert!(batch().steps(wrong.clone()).is_err(), "{wrong}"); + } +} + +#[tokio::test] +async fn a_wrong_step_stops_the_batch_before_anything_is_asked() { + let approvals = Arc::new(Approvals::default()); + // A step that reached approval would fail with "no one can answer here" instead. + let arguments = json!({"steps": [ + {"tool": "press_keys", "arguments": {"keys": "enter"}}, + {"tool": "type_text", "arguments": {"text": ""}}, + ]}); + let message = error( + batch() + .call_result(arguments, context(&approvals, None)) + .await, + ); + assert!(message.contains("step 2 (type_text)"), "{message}"); + assert!(message.contains("no step ran"), "{message}"); + assert!(!message.contains("approval"), "{message}"); +} + +#[tokio::test] +async fn every_step_reads_the_picture_seen_before_the_batch() { + let screen = Arc::new(Screen::new(ArtifactStore::new(std::env::temp_dir()))); + screen.set_coordinates(Some(View::scaled(1.0))); + let shot = Arc::clone(&screen); + let reader = Arc::clone(&screen); + let steps = vec![ + ( + StepTool::Screenshot, + Step::new(fake(), move || { + shot.set_coordinates(Some(View::scaled(2.0))); + Ok(ToolResult::text("shot".into())) + }), + ), + ( + StepTool::Click, + Step::new(fake(), move || { + let at = reader.point(10.0, 10.0)?; + Ok(ToolResult::text(format!("at {}", at.x))) + }), + ), + ]; + let result = run(&screen, steps).await; + assert!(!result.is_error, "{}", result.content); + assert!(result.content.contains("2. at 10"), "{}", result.content); + // After the batch the newest picture, the one the model now sees, counts again. + assert_eq!(screen.point(10.0, 10.0).unwrap().x, 20.0); +} + +#[tokio::test] +async fn a_failed_step_stops_the_rest() { + let screen = Screen::new(ArtifactStore::new(std::env::temp_dir())); + let ran = Arc::new(parking_lot::Mutex::new(0)); + let counter = Arc::clone(&ran); + let steps = vec![ + ( + StepTool::Click, + Step::new(fake(), || anyhow::bail!("no control there")), + ), + ( + StepTool::Click, + Step::new(fake(), move || { + *counter.lock() += 1; + Ok(ToolResult::text(String::new())) + }), + ), + ]; + let result = run(&screen, steps).await; + assert!(result.is_error); + assert!(result.content.contains("1. failed: no control there")); + assert!(result.content.contains("steps 2 to 2 did not run")); + assert_eq!(*ran.lock(), 0); +} + +/// Allows the screen for the session, as a user would. +async fn allow_screen(approvals: &Approvals) { + let (events, mut seen) = mpsc::unbounded_channel(); + let progress = Progress { + call_id: "grant".into(), + events, + }; + let ask = approvals.ask(fake(), Some(progress), ApprovalPolicy::Ask); + let answer = async { + let Some(ThreadEvent::Approval(asked)) = seen.recv().await else { + panic!("missing request"); + }; + assert!(approvals.answer(asked.id, Decision::Session)); + }; + let (result, ()) = tokio::join!(ask, answer); + result.unwrap(); +} + +#[tokio::test] +async fn one_refused_step_refuses_the_batch() { + let approvals = Arc::new(Approvals::default()); + allow_screen(&approvals).await; + let arguments = json!({"steps": [ + {"tool": "screenshot", "arguments": {}}, + {"tool": "press_keys", "arguments": {"keys": "enter"}}, + ]}); + // The screenshot alone would pass on the grant; the keys still need a yes, and no + // one can answer here, so nothing runs. + let message = error( + batch() + .call_result(arguments, context(&approvals, None)) + .await, + ); + assert!(message.contains("press enter"), "{message}"); + + // Asked, the batch lists every step by its number, the allowed ones marked, so + // the click that sets the focus is seen with the typing; a no stops all. + let (events, mut seen) = mpsc::unbounded_channel(); + let progress = Progress { + call_id: "batch".into(), + events, + }; + let arguments = json!({"steps": [ + {"tool": "screenshot", "arguments": {}}, + {"tool": "type_text", "arguments": {"text": "hi"}}, + {"tool": "press_keys", "arguments": {"keys": "Shift + Enter"}}, + ]}); + let tool = batch(); + let call = tool.call_result(arguments, context(&approvals, Some(progress))); + let answer = async { + let Some(ThreadEvent::Approval(asked)) = seen.recv().await else { + panic!("missing request"); + }; + assert!(matches!(asked.request, Request::Keyboard { .. })); + let lines: Vec = asked.steps.iter().map(|step| step.line()).collect(); + assert_eq!( + lines, + [ + "1. using the screen: take a screenshot (already allowed)", + "2. using the keyboard: type \"hi\"", + "3. using the keyboard: press shift+enter", + ] + ); + assert_eq!( + asked.title("the agent"), + "the agent wants to run 3 computer actions" + ); + // A surface that reads only the request's text, or its plain text, sees every + // step too; the row drawing clients use the title, so nothing shows twice. + let described = asked.request.describe("the agent"); + let text = asked.text("the agent"); + for line in &lines { + assert!(described.contains(line.as_str()), "{described}"); + assert!(text.contains(line.as_str()), "{text}"); + } + assert!(!asked.title("the agent").contains("1.")); + assert_eq!(asked.session, None); + assert!(approvals.answer(asked.id, Decision::Deny)); + }; + let (result, ()) = tokio::join!(call, answer); + assert!(error(result).contains("denied")); +} + +/// Says no to any request that presses keys, yes to the rest. +struct NoKeys; + +impl Judge for NoKeys { + fn judge<'a>(&'a self, request: &'a Request) -> crate::BoxFuture<'a, Verdict> { + let allow = !request.what().contains("press"); + Box::pin(async move { + Ok(Verdict { + allow, + reason: "keys are not allowed".into(), + }) + }) + } +} + +#[tokio::test] +async fn the_arbiter_judges_every_step_alone() { + let approvals = Approvals::default(); + approvals.set_judge(Arc::new(NoKeys)); + let requests = vec![ + fake(), + Request::Keyboard { + action: "press enter".into(), + }, + ]; + let denied = approvals + .ask_all(requests, None, ApprovalPolicy::Arbiter) + .await + .expect_err("one step is denied"); + let message = format!("{denied:#}"); + assert!(message.starts_with("step 2 of the batch"), "{message}"); + assert!(message.contains("keys are not allowed"), "{message}"); + approvals + .ask_all(vec![fake(), fake()], None, ApprovalPolicy::Arbiter) + .await + .unwrap(); +} + +#[tokio::test] +async fn mixed_kinds_offer_no_session_grant() { + let approvals = Approvals::default(); + let requests = vec![ + fake(), + Request::SystemPointer { + action: "scroll".into(), + }, + ]; + let (events, mut seen) = mpsc::unbounded_channel(); + let progress = Progress { + call_id: "mixed".into(), + events, + }; + let ask = approvals.ask_all(requests.clone(), Some(progress), ApprovalPolicy::Ask); + let answer = async { + let Some(ThreadEvent::Approval(asked)) = seen.recv().await else { + panic!("missing request"); + }; + assert!(matches!(asked.request, Request::SystemPointer { .. })); + assert_eq!(asked.session, None); + // A client that answers "session" anyway grants nothing beyond this time. + assert!(approvals.answer(asked.id, Decision::Session)); + }; + let (result, ()) = tokio::join!(ask, answer); + result.unwrap(); + for request in requests { + assert!( + approvals + .ask(request, None, ApprovalPolicy::Ask) + .await + .is_err() + ); + } +} diff --git a/projects/core/src/tools/computer/click.rs b/projects/core/src/tools/computer/click.rs index a6174e6..8078800 100644 --- a/projects/core/src/tools/computer/click.rs +++ b/projects/core/src/tools/computer/click.rs @@ -10,12 +10,12 @@ use serde_json::Value; #[cfg(test)] use super::Desktop; -use super::{blocking, screen::Screen}; +use super::{screen::Screen, step::Step}; use crate::{ BoxFuture, platform::Needs, tools::{ - WindowAccess, schema, + Request, ToolResult, WindowAccess, schema, schema::parse_arguments, tool::{Tool, ToolContext, ToolKind}, }, @@ -65,6 +65,59 @@ struct ClickArguments { mode: PointerMode, } +impl ClickArguments { + /// Bounded before anything is asked or done. + fn parse(arguments: Value) -> Result { + let arguments: Self = parse_arguments(arguments)?; + arguments + .mode + .validate_click(arguments.right, arguments.count())?; + Ok(arguments) + } + + fn count(&self) -> i64 { + if self.double { 2 } else { 1 } + } +} + +impl Click { + /// Checked before anything is asked or done, so a batch can check every step first. + pub(super) fn step(screen: &Arc, arguments: Value) -> Result { + let arguments = ClickArguments::parse(arguments)?; + let (x, y, mode) = (arguments.x, arguments.y, arguments.mode); + let action = match mode { + PointerMode::Independent => { + format!("activate the control at {x}, {y} without moving the pointer") + } + PointerMode::System => format!("move the user's pointer and click at {x}, {y}"), + }; + let scope = screen.scope(); + let request = match (&scope, mode) { + (Some((id, label)), PointerMode::Independent) => Request::Window { + grant: Some(WindowAccess::Activate(id.clone())), + action: format!("{action} in {label}; the target app may come to the foreground"), + }, + (Some(_), PointerMode::System) => { + anyhow::bail!("system pointer is disabled in a window session") + } + (None, PointerMode::Independent) => Request::Screen { action }, + (None, PointerMode::System) => Request::SystemPointer { action }, + }; + let screen = Arc::clone(screen); + Ok(Step::new(request, move || { + let count = arguments.count(); + let expected = scope.as_ref().map(|(id, _)| id.as_str()); + screen.click(x, y, arguments.right, count, mode, expected)?; + Ok(ToolResult::text(match mode { + PointerMode::Independent => format!( + "pressed the control at {x}, {y}; the keyboard focus may not have moved" + ), + PointerMode::System => format!("clicked at {x}, {y} with the pointer"), + })) + })) + } +} + impl Tool for Click { computer_tool!("click"); @@ -93,59 +146,8 @@ impl Tool for Click { fn call<'a>(&'a self, arguments: Value, context: ToolContext) -> BoxFuture<'a, String> { Box::pin(async move { - let arguments: ClickArguments = parse_arguments(arguments)?; - let count = if arguments.double { 2 } else { 1 }; - arguments.mode.validate_click(arguments.right, count)?; - let action = match arguments.mode { - PointerMode::Independent => format!( - "activate the control at {}, {} without moving the pointer", - arguments.x, arguments.y - ), - PointerMode::System => format!( - "move the user's pointer and click at {}, {}", - arguments.x, arguments.y - ), - }; - let scope = self.0.scope(); - match (&scope, arguments.mode) { - (Some((id, label)), PointerMode::Independent) => { - context - .approve_window( - Some(WindowAccess::Activate(id.clone())), - &format!( - "{action} in {label}; the target app may come to the foreground" - ), - ) - .await? - } - (Some(_), PointerMode::System) => { - anyhow::bail!("system pointer is disabled in a window session") - } - (None, PointerMode::Independent) => context.approve_screen(&action, false).await?, - (None, PointerMode::System) => context.approve_system_pointer(&action).await?, - } - let screen = Arc::clone(&self.0); - blocking(move || { - screen.click( - arguments.x, - arguments.y, - arguments.right, - count, - arguments.mode, - scope.as_ref().map(|(id, _)| id.as_str()), - ) - }) - .await?; - match arguments.mode { - PointerMode::Independent => Ok(format!( - "pressed the control at {}, {}; the keyboard focus may not have moved", - arguments.x, arguments.y - )), - PointerMode::System => Ok(format!( - "clicked at {}, {} with the pointer", - arguments.x, arguments.y - )), - } + let step = Self::step(&self.0, arguments)?; + Ok(step.ask_and_run(&context).await?.content) }) } } diff --git a/projects/core/src/tools/computer/mod.rs b/projects/core/src/tools/computer/mod.rs index 420caa7..f197a41 100644 --- a/projects/core/src/tools/computer/mod.rs +++ b/projects/core/src/tools/computer/mod.rs @@ -10,14 +10,16 @@ //! the desktop has. //! //! The tools are the flat files here (`click`, `press_keys`, `screenshot`, `scroll`, -//! `type_text`, `window_session`) over `screen`, the screen as the model sees it, and -//! `offer`, how they are offered. `backend` is the desktop they drive; `overlay` is +//! `type_text`, `window_session`, and `batch`, several of them in one call) over +//! `screen`, the screen as the model sees it, `step`, one checked action and what it +//! asks, and `offer`, how they are offered. `backend` is the desktop they drive; `overlay` is //! dusk's own cursor on macOS. #[macro_use] mod offer; mod backend; +mod batch; mod click; #[cfg(target_os = "macos")] mod overlay; @@ -25,6 +27,7 @@ mod press_keys; mod screen; mod screenshot; mod scroll; +mod step; mod type_text; #[cfg(target_os = "macos")] mod window_session; @@ -32,6 +35,7 @@ mod window_session; use anyhow::{Context, Result}; use backend::Desktop; +pub use batch::ComputerActions; pub use click::Click; #[cfg(target_os = "macos")] pub use overlay::run_overlay_if_asked; diff --git a/projects/core/src/tools/computer/press_keys.rs b/projects/core/src/tools/computer/press_keys.rs index b812e09..f51967a 100644 --- a/projects/core/src/tools/computer/press_keys.rs +++ b/projects/core/src/tools/computer/press_keys.rs @@ -2,16 +2,17 @@ use std::sync::Arc; +use anyhow::Result; use schemars::JsonSchema; use serde::Deserialize; use serde_json::Value; -use super::{Desktop, Screen, blocking, screen::keys}; +use super::{Desktop, Keys, Screen, screen::keys, step::Step}; use crate::{ BoxFuture, platform::Needs, tools::{ - schema, + Request, ToolResult, schema, schema::parse_arguments, tool::{Tool, ToolContext, ToolKind}, }, @@ -26,6 +27,32 @@ struct PressKeysArguments { keys: String, } +impl PressKeysArguments { + /// Bounded before anything is asked or done. + fn parse(arguments: Value) -> Result { + let arguments: Self = parse_arguments(arguments)?; + keys(&arguments.keys) + } +} + +impl PressKeys { + /// Checked before anything is asked or done, so a batch can check every step first. + pub(super) fn step(screen: &Arc, arguments: Value) -> Result { + screen.require_unscoped()?; + let combo = PressKeysArguments::parse(arguments)?; + let keys = combo.shown(); + let request = Request::Keyboard { + action: format!("press {keys}"), + }; + let screen = Arc::clone(screen); + Ok(Step::new(request, move || { + screen.unscoped(|| Desktop.press(&combo))?; + Desktop.touch(); + Ok(ToolResult::text(format!("pressed {keys}"))) + })) + } +} + impl Tool for PressKeys { computer_tool!("press_keys"); @@ -42,16 +69,8 @@ impl Tool for PressKeys { fn call<'a>(&'a self, arguments: Value, context: ToolContext) -> BoxFuture<'a, String> { Box::pin(async move { - self.0.require_unscoped()?; - let arguments: PressKeysArguments = parse_arguments(arguments)?; - let combo = keys(&arguments.keys)?; - context - .approve_screen(&format!("press {}", arguments.keys), true) - .await?; - let screen = Arc::clone(&self.0); - blocking(move || screen.unscoped(|| Desktop.press(&combo))).await?; - Desktop.touch(); - Ok(format!("pressed {}", arguments.keys)) + let step = Self::step(&self.0, arguments)?; + Ok(step.ask_and_run(&context).await?.content) }) } } diff --git a/projects/core/src/tools/computer/screen.rs b/projects/core/src/tools/computer/screen.rs index c9d1176..bcf7007 100644 --- a/projects/core/src/tools/computer/screen.rs +++ b/projects/core/src/tools/computer/screen.rs @@ -20,7 +20,7 @@ const JPEG_QUALITY: u8 = 80; /// How a picture's pixel maps to the display: `origin + pixel * scale` in points. #[derive(Debug, Clone, Copy)] -struct View { +pub(super) struct View { origin: Point, scale_x: f64, scale_y: f64, @@ -349,6 +349,17 @@ impl Screen { Ok((at, view.window)) } + /// The picture clicks and regions are read in now, if any. + pub(super) fn coordinates(&self) -> Option { + *self.view.lock() + } + + /// A batch reads every step in the picture the model saw before it, not in one it + /// took meanwhile and the model has not seen. + pub(super) fn set_coordinates(&self, view: Option) { + *self.view.lock() = view; + } + fn view(&self) -> Result { (*self.view.lock()).context("take a screenshot first: coordinates come from it") } @@ -431,6 +442,21 @@ impl View { } } +#[cfg(test)] +impl View { + /// A picture of the whole display at `scale` points a pixel. + pub(super) fn scaled(scale: f64) -> Self { + Self { + origin: Point { x: 0.0, y: 0.0 }, + scale_x: scale, + scale_y: scale, + width: 1000.0, + height: 1000.0, + window: None, + } + } +} + /// The largest window of the app named `name`, else the largest whose app or title holds /// it, so a toolbar strip never wins over the window itself; a miss names the windows /// there are. @@ -502,7 +528,9 @@ pub const KEY_NAMES: [&str; 27] = [ ]; /// `cmd+shift+t`, `enter` or `f5` as a key with its modifiers. The key comes out by its -/// one name: a lowercase letter, a digit or one of [`KEY_NAMES`]. +/// one name: a lowercase letter, a digit or one of [`KEY_NAMES`]. Only plain spaces +/// may pad a part: a line break or control character would let the text the user +/// approves look unlike the keys pressed. pub fn keys(combo: &str) -> Result { let mut keys = Keys { key: String::new(), @@ -513,7 +541,7 @@ pub fn keys(combo: &str) -> Result { }; for part in combo .split('+') - .map(|part| part.trim().to_ascii_lowercase()) + .map(|part| part.trim_matches(' ').to_ascii_lowercase()) { match part.as_str() { "cmd" | "command" | "meta" | "super" | "win" => keys.cmd = true, @@ -536,6 +564,25 @@ pub fn keys(combo: &str) -> Result { Ok(keys) } +impl Keys { + /// The combo by its normal names, such as `cmd+shift+t`, for the user to approve. + pub fn shown(&self) -> String { + let modifiers = [ + (self.cmd, "cmd"), + (self.ctrl, "ctrl"), + (self.alt, "alt"), + (self.shift, "shift"), + ]; + let mut parts: Vec<&str> = modifiers + .iter() + .filter(|(on, _)| *on) + .map(|(_, name)| *name) + .collect(); + parts.push(&self.key); + parts.join("+") + } +} + fn key_name(name: &str) -> Option { let mut chars = name.chars(); if let (Some(c), None) = (chars.next(), chars.next()) @@ -565,6 +612,11 @@ mod tests { for wrong in ["cmd", "a+b", "cmd+nope", "", "é"] { assert!(keys(wrong).is_err(), "{wrong}"); } + // What the user approves is what is pressed: no hidden lines or controls. + for hidden in ["\nenter", "enter\t", "cmd+\u{7}t", "cmd\r+t", "\u{a0}enter"] { + assert!(keys(hidden).is_err(), "{hidden:?}"); + } + assert_eq!(keys(" Shift + CMD + t ").unwrap().shown(), "cmd+shift+t"); // Every name the parser lets through, the desktop can press. for name in KEY_NAMES.iter().copied().chain(["a", "z", "0", "9"]) { assert!(Desktop.knows_key(name), "{name}"); diff --git a/projects/core/src/tools/computer/screenshot.rs b/projects/core/src/tools/computer/screenshot.rs index 774caae..23a041f 100644 --- a/projects/core/src/tools/computer/screenshot.rs +++ b/projects/core/src/tools/computer/screenshot.rs @@ -2,19 +2,20 @@ use std::sync::Arc; +use anyhow::Result; use schemars::JsonSchema; use serde::Deserialize; use serde_json::Value; use super::{ - blocking, screen::{Region, Screen, Target}, + step::Step, }; use crate::{ BoxFuture, platform::Needs, tools::{ - ToolResult, WindowAccess, schema, + Request, ToolResult, WindowAccess, schema, schema::parse_arguments, tool::{Tool, ToolContext, ToolKind}, }, @@ -42,6 +43,49 @@ struct RegionArguments { height: f64, } +impl ScreenshotArguments { + /// What to capture, checked before anything is asked or done. + fn target(arguments: Value) -> Result { + let arguments: Self = parse_arguments(arguments)?; + Ok(match (arguments.region, arguments.window) { + (Some(_), Some(_)) => anyhow::bail!("give a region or a window, not both"), + (Some(region), None) => Target::Region(Region { + x: region.x, + y: region.y, + width: region.width, + height: region.height, + }), + (None, Some(window)) => Target::Window(window), + (None, None) => Target::Display, + }) + } +} + +impl Screenshot { + /// Checked before anything is asked or done, so a batch can check every step first. + pub(super) fn step(screen: &Arc, arguments: Value) -> Result { + let target = ScreenshotArguments::target(arguments)?; + let scope = screen.scope(); + let request = match &scope { + Some((id, label)) => Request::Window { + grant: Some(WindowAccess::Capture(id.clone())), + action: format!("capture only {label}"), + }, + None => Request::Screen { + action: "take a screenshot".into(), + }, + }; + let screen = Arc::clone(screen); + Ok(Step::new(request, move || { + let expected = scope.as_ref().map(|(id, _)| id.as_str()); + let (artifact, text) = screen.capture(target, expected)?; + let mut result = ToolResult::text(text); + result.artifacts.push(artifact); + Ok(result) + })) + } +} + impl Tool for Screenshot { computer_tool!("screenshot"); @@ -68,38 +112,6 @@ impl Tool for Screenshot { arguments: Value, context: ToolContext, ) -> BoxFuture<'a, ToolResult> { - Box::pin(async move { - let arguments: ScreenshotArguments = parse_arguments(arguments)?; - let scope = self.0.scope(); - match &scope { - Some((id, label)) => { - context - .approve_window( - Some(WindowAccess::Capture(id.clone())), - &format!("capture only {label}"), - ) - .await? - } - None => context.approve_screen("take a screenshot", false).await?, - } - let target = match (arguments.region, arguments.window) { - (Some(_), Some(_)) => anyhow::bail!("give a region or a window, not both"), - (Some(region), None) => Target::Region(Region { - x: region.x, - y: region.y, - width: region.width, - height: region.height, - }), - (None, Some(window)) => Target::Window(window), - (None, None) => Target::Display, - }; - let screen = Arc::clone(&self.0); - let (artifact, text) = - blocking(move || screen.capture(target, scope.as_ref().map(|(id, _)| id.as_str()))) - .await?; - let mut result = ToolResult::text(text); - result.artifacts.push(artifact); - Ok(result) - }) + Box::pin(async move { Self::step(&self.0, arguments)?.ask_and_run(&context).await }) } } diff --git a/projects/core/src/tools/computer/scroll.rs b/projects/core/src/tools/computer/scroll.rs index 8985cab..5badf03 100644 --- a/projects/core/src/tools/computer/scroll.rs +++ b/projects/core/src/tools/computer/scroll.rs @@ -2,17 +2,17 @@ use std::sync::Arc; -use anyhow::ensure; +use anyhow::{Result, ensure}; use schemars::JsonSchema; use serde::Deserialize; use serde_json::Value; -use super::{Desktop, blocking, screen::Screen}; +use super::{Desktop, screen::Screen, step::Step}; use crate::{ BoxFuture, platform::Needs, tools::{ - schema, + Request, ToolResult, schema, schema::parse_arguments, tool::{Tool, ToolContext, ToolKind}, }, @@ -36,6 +36,45 @@ struct ScrollArguments { right: i32, } +impl ScrollArguments { + /// Bounded before anything is asked or done. + fn parse(arguments: Value) -> Result { + let mut arguments: Self = parse_arguments(arguments)?; + let bounded = |pixels: i32| pixels.clamp(-MAX_SCROLL_PIXELS, MAX_SCROLL_PIXELS); + arguments.down = bounded(arguments.down); + arguments.right = bounded(arguments.right); + ensure!( + arguments.down != 0 || arguments.right != 0, + "say how far to scroll" + ); + Ok(arguments) + } +} + +impl Scroll { + /// Checked before anything is asked or done, so a batch can check every step first. + pub(super) fn step(screen: &Arc, arguments: Value) -> Result { + screen.require_unscoped()?; + let ScrollArguments { x, y, down, right } = ScrollArguments::parse(arguments)?; + let request = Request::SystemPointer { + action: format!( + "move the user's pointer to {x}, {y} and scroll {down} down, {right} right" + ), + }; + let screen = Arc::clone(screen); + Ok(Step::new(request, move || { + screen.unscoped(|| { + let at = screen.point(x, y)?; + screen.mark(at); + Desktop.scroll(at, down, right) + })?; + Ok(ToolResult::text(format!( + "scrolled {down} down and {right} right" + ))) + })) + } +} + impl Tool for Scroll { computer_tool!("scroll"); @@ -52,24 +91,8 @@ impl Tool for Scroll { fn call<'a>(&'a self, arguments: Value, context: ToolContext) -> BoxFuture<'a, String> { Box::pin(async move { - self.0.require_unscoped()?; - let arguments: ScrollArguments = parse_arguments(arguments)?; - let bounded = |pixels: i32| pixels.clamp(-MAX_SCROLL_PIXELS, MAX_SCROLL_PIXELS); - let (down, right) = (bounded(arguments.down), bounded(arguments.right)); - ensure!(down != 0 || right != 0, "say how far to scroll"); - context - .approve_system_pointer("move the user's pointer and scroll") - .await?; - let screen = Arc::clone(&self.0); - blocking(move || { - screen.unscoped(|| { - let at = screen.point(arguments.x, arguments.y)?; - screen.mark(at); - Desktop.scroll(at, down, right) - }) - }) - .await?; - Ok(format!("scrolled {down} down and {right} right")) + let step = Self::step(&self.0, arguments)?; + Ok(step.ask_and_run(&context).await?.content) }) } } diff --git a/projects/core/src/tools/computer/step.rs b/projects/core/src/tools/computer/step.rs new file mode 100644 index 0000000..b497f24 --- /dev/null +++ b/projects/core/src/tools/computer/step.rs @@ -0,0 +1,38 @@ +//! One checked screen action: what it asks the user and the work it does. A tool alone +//! and a batch build the same step, so they cannot ask or act differently. + +use anyhow::Result; + +use super::blocking; +use crate::tools::{Request, ToolResult, tool::ToolContext}; + +type Work = Box Result + Send>; + +pub(super) struct Step { + /// What the user is asked before the work runs. + pub request: Request, + work: Work, +} + +impl Step { + pub fn new( + request: Request, + work: impl FnOnce() -> Result + Send + 'static, + ) -> Self { + Self { + request, + work: Box::new(work), + } + } + + /// The work alone; the caller has asked for it. + pub async fn run(self) -> Result { + blocking(self.work).await + } + + /// A tool called on its own: asks, then acts. + pub async fn ask_and_run(self, context: &ToolContext) -> Result { + context.approve_computer(vec![self.request.clone()]).await?; + self.run().await + } +} diff --git a/projects/core/src/tools/computer/type_text.rs b/projects/core/src/tools/computer/type_text.rs index 16a1833..5d37d3e 100644 --- a/projects/core/src/tools/computer/type_text.rs +++ b/projects/core/src/tools/computer/type_text.rs @@ -2,17 +2,17 @@ use std::sync::Arc; -use anyhow::ensure; +use anyhow::{Result, ensure}; use schemars::JsonSchema; use serde::Deserialize; use serde_json::Value; -use super::{Desktop, Screen, blocking}; +use super::{Desktop, Screen, step::Step}; use crate::{ BoxFuture, platform::Needs, tools::{ - schema, + Request, ToolResult, schema, schema::parse_arguments, tool::{Tool, ToolContext, ToolKind}, }, @@ -32,6 +32,41 @@ struct TypeTextArguments { text: String, } +impl TypeTextArguments { + /// Bounded before anything is asked or done. + fn parse(arguments: Value) -> Result { + let arguments: Self = parse_arguments(arguments)?; + let count = arguments.text.chars().count(); + ensure!(count > 0, "the text is empty"); + ensure!( + count <= MAX_TYPED_CHARS, + "type at most {MAX_TYPED_CHARS} characters at once" + ); + Ok(arguments) + } +} + +impl TypeText { + /// Checked before anything is asked or done, so a batch can check every step first. + pub(super) fn step(screen: &Arc, arguments: Value) -> Result { + screen.require_unscoped()?; + let TypeTextArguments { text } = TypeTextArguments::parse(arguments)?; + let count = text.chars().count(); + // The user sees what will be typed, since it can run anything a shell can. + let shown: String = text.chars().take(SHOWN_CHARS).collect(); + let more = if count > SHOWN_CHARS { "..." } else { "" }; + let request = Request::Keyboard { + action: format!("type \"{}{more}\"", shown.escape_debug()), + }; + let screen = Arc::clone(screen); + Ok(Step::new(request, move || { + screen.unscoped(|| Desktop.type_text(&text))?; + Desktop.touch(); + Ok(ToolResult::text(format!("typed {count} characters"))) + })) + } +} + impl Tool for TypeText { computer_tool!("type_text"); @@ -50,23 +85,8 @@ impl Tool for TypeText { fn call<'a>(&'a self, arguments: Value, context: ToolContext) -> BoxFuture<'a, String> { Box::pin(async move { - self.0.require_unscoped()?; - let arguments: TypeTextArguments = parse_arguments(arguments)?; - let count = arguments.text.chars().count(); - ensure!(count > 0, "the text is empty"); - ensure!( - count <= MAX_TYPED_CHARS, - "type at most {MAX_TYPED_CHARS} characters at once" - ); - // The user sees what will be typed, since it can run anything a shell can. - let shown: String = arguments.text.chars().take(SHOWN_CHARS).collect(); - let more = if count > SHOWN_CHARS { "..." } else { "" }; - let action = format!("type \"{}{more}\"", shown.escape_debug()); - context.approve_screen(&action, true).await?; - let screen = Arc::clone(&self.0); - blocking(move || screen.unscoped(|| Desktop.type_text(&arguments.text))).await?; - Desktop.touch(); - Ok(format!("typed {count} characters")) + let step = Self::step(&self.0, arguments)?; + Ok(step.ask_and_run(&context).await?.content) }) } } diff --git a/projects/core/src/tools/mod.rs b/projects/core/src/tools/mod.rs index 105d3dc..919fea0 100644 --- a/projects/core/src/tools/mod.rs +++ b/projects/core/src/tools/mod.rs @@ -25,7 +25,7 @@ pub use computer::WindowSession; #[cfg(target_os = "macos")] pub use computer::run_overlay_if_asked; #[cfg(any(target_os = "macos", target_os = "linux"))] -pub use computer::{Click, PressKeys, Screen, Screenshot, Scroll, TypeText}; +pub use computer::{Click, ComputerActions, PressKeys, Screen, Screenshot, Scroll, TypeText}; pub use console::{Jobs, RunCommand}; pub use harness::{ AskUser, DisplayAudio, DisplayImage, GoalTool, PlanEdit, Question, ReadHistory, Remember, diff --git a/projects/core/src/tools/registry.rs b/projects/core/src/tools/registry.rs index b9753f6..4ed17bd 100644 --- a/projects/core/src/tools/registry.rs +++ b/projects/core/src/tools/registry.rs @@ -125,6 +125,41 @@ impl ToolRegistry { kinds } + /// The calls `name` runs inside it, for hooks to judge one by one; empty for a tool + /// that runs none or arguments that do not parse, which the call itself refuses. + pub fn calls_within(&self, name: &str, arguments: &Value) -> Vec<(String, Value)> { + self.tools + .get(name) + .and_then(|tool| tool.calls_within(arguments).ok()) + .unwrap_or_default() + } + + /// A call inside another is refused as it would be alone: a tool switched off in the + /// mode does not run because a batch names it. + fn check_within( + &self, + tool: &dyn Tool, + arguments: &Value, + config: &Config, + ) -> std::result::Result<(), String> { + let within = tool + .calls_within(arguments) + .map_err(|error| format!("{error:#}"))?; + for (index, (name, _)) in within.iter().enumerate() { + let on = self + .tools + .get(name.as_str()) + .is_some_and(|inner| self.enabled(inner.as_ref(), config)); + if !on { + return Err(format!( + "step {} ({name}) is switched off, so no step ran", + index + 1 + )); + } + } + Ok(()) + } + fn enabled(&self, tool: &dyn Tool, config: &Config) -> bool { if tool.root_only() && !self.root || !self.platform.provides(tool.needs()) { return false; @@ -216,6 +251,9 @@ impl ToolRegistry { info!("tool {} called with invalid arguments", call.name); return error_result(problems); } + if let Err(problem) = self.check_within(tool.as_ref(), &arguments, config) { + return error_result(problem); + } let context = ToolContext { scope: Arc::clone(scope), config: Arc::clone(config), @@ -326,6 +364,57 @@ mod tests { assert_eq!(registry.kinds().len(), 1); } + /// A batch cannot run a tool the mode switched off: refused before anything is asked. + #[cfg(any(target_os = "macos", target_os = "linux"))] + #[tokio::test] + async fn a_batch_step_switched_off_refuses_the_batch() { + use crate::tools::{ComputerActions, PressKeys, Screen, TypeText}; + let dir = temp_dir("registry_batch_switch"); + std::fs::create_dir_all(&dir).unwrap(); + let scope = Arc::new(ToolScope::new(&dir, &dir).unwrap()); + let mut config = (*crate::testing::tool_context(&scope).config).clone(); + for (name, on) in [ + ("computer_actions", true), + ("press_keys", true), + ("type_text", false), + ] { + config.set_tool_enabled(name, on); + } + let config = Arc::new(config); + let screen = Arc::new(Screen::new(ArtifactStore::new(dir.clone()))); + let mut registry = ToolRegistry { + root: true, + ..ToolRegistry::default() + }; + registry.platform.gui = true; + registry.add(Box::new(ComputerActions(Arc::clone(&screen)))); + registry.add(Box::new(PressKeys(Arc::clone(&screen)))); + registry.add(Box::new(TypeText(screen))); + let call = ToolCall { + id: "c".into(), + name: "computer_actions".into(), + arguments: serde_json::json!({"steps": [ + {"tool": "press_keys", "arguments": {"keys": "enter"}}, + {"tool": "type_text", "arguments": {"text": "hi"}}, + ]}), + }; + let (events, mut asked) = tokio::sync::mpsc::unbounded_channel(); + let progress = Progress { + call_id: "c".into(), + events, + }; + let result = registry.call(&call, &scope, &config, progress).await; + assert!(result.is_error); + assert!( + result + .content + .contains("step 2 (type_text) is switched off"), + "{}", + result.content + ); + assert!(asked.try_recv().is_err(), "nothing may be asked"); + } + /// Providers reject a schema that is not one plain object at the top. #[tokio::test] async fn every_schema_is_one_object_providers_accept() { @@ -342,13 +431,16 @@ mod tests { tools.push(Box::new(AgentState::new(Arc::clone(&state), "t".into()))); #[cfg(any(target_os = "macos", target_os = "linux"))] { - use crate::tools::{Click, PressKeys, Screen, Screenshot, Scroll, TypeText}; + use crate::tools::{ + Click, ComputerActions, PressKeys, Screen, Screenshot, Scroll, TypeText, + }; let screen = Arc::new(Screen::new(ArtifactStore::new(dir.clone()))); tools.push(Box::new(Screenshot(Arc::clone(&screen)))); tools.push(Box::new(Click(Arc::clone(&screen)))); tools.push(Box::new(Scroll(Arc::clone(&screen)))); tools.push(Box::new(TypeText(Arc::clone(&screen)))); tools.push(Box::new(PressKeys(Arc::clone(&screen)))); + tools.push(Box::new(ComputerActions(Arc::clone(&screen)))); #[cfg(target_os = "macos")] tools.push(Box::new(crate::tools::WindowSession(screen))); } diff --git a/projects/core/src/tools/tool.rs b/projects/core/src/tools/tool.rs index 2d8564b..cdccc92 100644 --- a/projects/core/src/tools/tool.rs +++ b/projects/core/src/tools/tool.rs @@ -74,40 +74,28 @@ impl ToolContext { Ok(resolved) } - /// The screen needs the user's yes once a session, the keyboard every time, unless - /// full access gives it. - pub async fn approve_screen(&self, action: &str, keyboard: bool) -> Result<()> { - let Some(approvals) = &self.approvals else { - return Ok(()); - }; - let action = action.to_owned(); - let request = match keyboard { - true => Request::Keyboard { action }, - false => Request::Screen { action }, - }; - let policy = self.approvals_policy().await; - approvals.ask(request, self.progress.clone(), policy).await - } - - /// A screen grant never permits physical pointer takeover. Without an approval - /// service only the explicitly selected full-access policy allows it. - pub async fn approve_system_pointer(&self, action: &str) -> Result<()> { + /// Screen actions, asked as one request: see [`Approvals::ask_all`]. The screen needs + /// the user's yes once a session, the keyboard every time, unless full access gives + /// it. A screen grant never permits the system pointer or a window, and without an + /// approval service only the explicitly selected full-access policy allows those. + pub async fn approve_computer(&self, requests: Vec) -> Result<()> { let Some(approvals) = &self.approvals else { + let guarded = requests.iter().any(|request| { + matches!( + request, + Request::SystemPointer { .. } | Request::Window { .. } + ) + }); anyhow::ensure!( - self.approvals_policy().await == ApprovalPolicy::FullAccess, - "system pointer control needs a separate approval service or explicit full access" + !guarded || self.approvals_policy().await == ApprovalPolicy::FullAccess, + "system pointer control and window use need an approval service or explicit \ + full access" ); return Ok(()); }; let policy = self.approvals_policy().await; approvals - .ask( - Request::SystemPointer { - action: action.to_owned(), - }, - self.progress.clone(), - policy, - ) + .ask_all(requests, self.progress.clone(), policy) .await } @@ -196,6 +184,12 @@ pub trait Tool: Send + Sync { true } + /// The tools this call runs inside it, each with its own arguments, so switches and + /// plugin hooks judge every one as if it were called alone. An error rejects the call. + fn calls_within(&self, _arguments: &Value) -> Result> { + Ok(Vec::new()) + } + /// An error is shown to the model as the result, so it can correct itself. fn call<'a>(&'a self, arguments: Value, context: ToolContext) -> BoxFuture<'a, String>; diff --git a/projects/protocol/src/client/hello.rs b/projects/protocol/src/client/hello.rs index 276c350..684e6e7 100644 --- a/projects/protocol/src/client/hello.rs +++ b/projects/protocol/src/client/hello.rs @@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize}; /// Raised whenever a request, a response or an event changes shape. -pub const PROTOCOL_VERSION: u32 = 10; +pub const PROTOCOL_VERSION: u32 = 11; /// The first thing a client says. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] diff --git a/projects/protocol/src/thread/event.rs b/projects/protocol/src/thread/event.rs index e15bc63..3420936 100644 --- a/projects/protocol/src/thread/event.rs +++ b/projects/protocol/src/thread/event.rs @@ -170,6 +170,7 @@ mod tests { command: "cargo test".into(), }, session: Some("cargo".into()), + steps: Vec::new(), })), }; let text = serde_json::to_string(&event).unwrap(); diff --git a/projects/protocol/src/tools/approval.rs b/projects/protocol/src/tools/approval.rs index 303e8b1..5baf7a7 100644 --- a/projects/protocol/src/tools/approval.rs +++ b/projects/protocol/src/tools/approval.rs @@ -66,6 +66,54 @@ pub struct ApprovalRequest { /// What a yes for the session covers, as the core decides it: the folder of the /// path, or the program of a plain command. `None` when it cannot be offered. pub session: Option, + /// Every step of a batch of screen actions, in order; empty for a single request. + /// The user answers for all of them, so a client shows each one, never a summary. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub steps: Vec, +} + +/// One step of a batch as the user sees it before saying yes. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ApprovalStep { + /// Its place in the batch, from 1. + pub number: usize, + /// What the step would ask alone: its kind and what it does. + pub request: Request, + /// A session grant covers it already; it is listed so the user sees the whole batch. + pub allowed: bool, +} + +impl ApprovalStep { + /// The step in a line, such as `2. using the keyboard: press enter`. + pub fn line(&self) -> String { + let allowed = if self.allowed { + " (already allowed)" + } else { + "" + }; + format!("{}. {}{allowed}", self.number, self.request.what()) + } +} + +impl ApprovalRequest { + /// What `who` asks for, in a line: for a batch a neutral heading over its steps, + /// since no one kind says what all of them do. + pub fn title(&self, who: &str) -> String { + match self.steps.len() { + 0 => self.request.describe(who), + count => format!("{who} wants to run {count} computer actions"), + } + } + + /// The title and every step, a line each, for a surface that shows plain text only. + pub fn text(&self, who: &str) -> String { + let mut text = self.title(who); + for step in &self.steps { + text.push('\n'); + text.push_str(&step.line()); + } + text + } } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] diff --git a/projects/protocol/src/tools/mod.rs b/projects/protocol/src/tools/mod.rs index 8e53e3c..70def57 100644 --- a/projects/protocol/src/tools/mod.rs +++ b/projects/protocol/src/tools/mod.rs @@ -8,7 +8,7 @@ mod question; mod result; pub use answered::{Answered, Asked}; -pub use approval::{ApprovalRequest, Decision, Request}; +pub use approval::{ApprovalRequest, ApprovalStep, Decision, Request}; pub use card::{Card, CardKind}; pub use phrase::{is_agent_tool, phrase}; pub use question::{