diff --git a/README.md b/README.md index d94f647..2cfae27 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ It already has your browser open right there. It just can't see it. - **Per-tab concurrency.** Two actions on the _same_ tab serialize (no races); actions on _different_ tabs run in parallel. - **Tab locking.** An agent can claim a tab so others queue behind it instead of racing (`browser_tabs { action: "lock" }`). Locks survive Chrome's service-worker recycling (`chrome.storage.session`). - **Agent-control shield.** While an agent works on a tab you see a translucent blue inner frame and your input on that tab is blocked (mouse, keyboard, wheel) — the badge shows `agent controlling the tab` and disappears when the action finishes. Locking a tab keeps a plain frame for the lock's lifetime. +- **Visible agent cursor (optional).** Switch on *Show agent cursor* in the popup's Settings and every agent mouse action (click, hover, wheel, drag — including `browser_act`) draws a blue pointer that glides to its target before the input lands — a ripple marks each click, the arrow presses in while dragging — so you can follow what the agent does. Off by default: the glide adds ~0.2–0.35 s to each mouse action on a visible tab (none in hidden tabs). It fades after a few idle seconds and stays out of the agent's own screenshots. - **Same-origin iframe piercing.** Legacy/enterprise UIs that live inside iframes (e.g. an ONT console in `iframe#mainFrame`) are reachable: all locator tools search iframe documents, and `find`/`click_text` walk every frame. - **Open-dialog rescue.** A native `alert`/`confirm`/`prompt` freezes the page's JS thread — `browser_handle_dialog` dismisses it out-of-band via CDP, no page JS needed, which also un-blocks every other tool on that tab. `browser_tabs close/focus` always work, even on a frozen tab. - **Authenticated local connection.** Token + one-time enrollment secret, so no other local process can silently drive your browser. Everything stays on localhost — no cloud, no telemetry. diff --git a/extension/handlers/agent-api.js b/extension/handlers/agent-api.js index 898a7bb..b98d395 100644 --- a/extension/handlers/agent-api.js +++ b/extension/handlers/agent-api.js @@ -11,6 +11,8 @@ import { validateActionArguments, } from '../lib/observation-v2.js'; import { SNAPSHOT_MAX_ENTRIES, SNAPSHOT_TTL_MS } from '../lib/snapshot-registry.js'; +import { PAGE_DOM_INSTALL, PAGE_DOM_VERSION } from '../lib/page-dom.js'; +import { agentCursorEnabled } from '../lib/overlay.js'; import { handleUploadFile } from './cdp.js'; const sessionKey = (sessionId) => String(sessionId || 'anonymous'); @@ -110,6 +112,16 @@ export async function handleAct(params = {}, sessionId, _agentName, signal) { }); } + // Agent cursor (opt-in, popup → Settings): the action glides it to the + // target first; it is drawn by the shared page runtime (lib/page-dom.js). + let cursor = false; + if ((params.action === 'click' || params.action === 'hover') && await agentCursorEnabled()) { + try { + await safeExec(params.tabId, PAGE_DOM_INSTALL, [PAGE_DOM_VERSION]); // no-op when current + cursor = true; + } catch { /* protected page: act reports it */ } + } + let result; try { result = await execPageV2(params.tabId, PAGE_ACT_V2, [{ @@ -118,6 +130,7 @@ export async function handleAct(params = {}, sessionId, _agentName, signal) { documentId: ownership.snapshot.documentId, routeEpoch: ownership.snapshot.routeEpoch, ttlMs: SNAPSHOT_TTL_MS, + cursor, params, }]); } catch (error) { diff --git a/extension/handlers/interaction-advanced.js b/extension/handlers/interaction-advanced.js index 61c94fa..e26b12e 100644 --- a/extension/handlers/interaction-advanced.js +++ b/extension/handlers/interaction-advanced.js @@ -5,7 +5,7 @@ */ import { resolveTab, safeExec, execDom, getFallback } from '../lib/page-exec.js'; import { withCdp } from '../lib/cdp-session.js'; -import { openShield, releaseShield } from '../lib/trusted-input.js'; +import { openShield, releaseShield, cursorTo, sleep } from '../lib/trusted-input.js'; export async function handleDialog(params) { const { tabId, action = 'accept', promptText } = params; @@ -87,11 +87,18 @@ export async function handleDrag(params) { } await openShield(tab.id); + // The agent cursor glides to the start and presses in… + await sleep(await cursorTo(tab.id, sx, sy, 'down')); return withCdp(tab.id, async (send) => { await send('Input.dispatchMouseEvent', { type: 'mousePressed', x: sx, y: sy, button: 'left', clickCount: 1, }); + // …then carries the drag: the moves are spread over its glide to the end, + // paced by the clock so CDP round-trips don't let the cursor run ahead. + const glideMs = await cursorTo(tab.id, ex, ey, 'up'); + const t0 = Date.now(); for (let i = 1; i <= steps; i++) { + if (glideMs) await sleep(t0 + (glideMs * i) / steps - Date.now()); const progress = i / steps; await send('Input.dispatchMouseEvent', { type: 'mouseMoved', diff --git a/extension/handlers/interaction.js b/extension/handlers/interaction.js index 386c6ba..15febb2 100644 --- a/extension/handlers/interaction.js +++ b/extension/handlers/interaction.js @@ -5,7 +5,7 @@ */ import { resolveTab, requireTarget, hasPoint, execDom, getFallback } from '../lib/page-exec.js'; import { autoReSnapshot } from './inspection.js'; -import { trustedSender, locateTarget, releaseShield, cdpClickAt, cdpKeyPress, cdpTypeText, keyDefinition, modifierBits, pointInfo } from '../lib/trusted-input.js'; +import { trustedSender, locateTarget, locatePointer, releaseShield, cdpClickAt, cdpKeyPress, cdpTypeText, keyDefinition, modifierBits, pointInfo, cursorTo, sleep } from '../lib/trusted-input.js'; export { handleDialog, handleDrag, handleFillForm } from './interaction-advanced.js'; @@ -52,7 +52,7 @@ async function clickAtPoint(tabId, params) { const { x, y, button = 'left' } = params; if (!BUTTONS.has(button)) throw new Error(`Unknown button ${button}`); const send = await requireCdp(tabId, 'Clicking'); - const info = await pointInfo(tabId, x, y); + const info = await pointInfo(tabId, x, y, 'click', clickCountOf(params)); try { await cdpClickAt(send, x, y, { button, clickCount: clickCountOf(params), modifiers: clickModifiers(params) }); } finally { @@ -74,10 +74,11 @@ export async function handleClick(params) { const fb = getFallback(tabId, ref); // Trusted path: a real mouse click at the element's centre over CDP, so - // focus moves, default actions run and the page sees isTrusted:true. + // focus moves, default actions run and the page sees isTrusted:true. The + // agent cursor glides there first so a person can follow the click. const send = await trustedSender(tabId, trusted); if (send && BUTTONS.has(button)) { - const loc = await locateTarget(tabId, { ref, selector, fb }); + const loc = await locatePointer(tabId, { ref, selector, fb }, 'click', clickCountOf(params)); if (loc && loc.success === false && loc.error === 'REF_GONE') return refGone(tabId, loc, ref, selector); if (loc?.success && loc.visible) { try { @@ -190,7 +191,10 @@ export async function handleType(params) { let after; try { // Not focusable by script (custom widget): click it like a user would. - if (!loc.focused) await cdpClickAt(send, loc.x, loc.y); + if (!loc.focused) { + await sleep(await cursorTo(tabId, loc.x, loc.y, 'click')); + await cdpClickAt(send, loc.x, loc.y); + } if (clear && loc.needsSelectAll) await cdpKeyPress(send, 'a', ['ctrl']); if (clear && loc.hasText && !text) await cdpKeyPress(send, 'Backspace'); await cdpTypeText(send, text); @@ -375,7 +379,7 @@ export async function handleHover(params) { const send = await trustedSender(tabId, trusted); if (send) { - const loc = await locateTarget(tabId, { ref, selector, fb }); + const loc = await locatePointer(tabId, { ref, selector, fb }, 'move'); if (loc?.success && loc.visible) { try { await send('Input.dispatchMouseEvent', { type: 'mouseMoved', x: loc.x, y: loc.y }); @@ -504,7 +508,7 @@ export async function handleClickByText(params) { // Trusted click on the parked element (same path as browser_click). const send = await trustedSender(tabId, trusted); if (send) { - const loc = await locateTarget(tabId, { ref: tempRef }); + const loc = await locatePointer(tabId, { ref: tempRef }, 'click'); if (loc?.success && loc.visible) { try { await cdpClickAt(send, loc.x, loc.y); diff --git a/extension/handlers/tabs.js b/extension/handlers/tabs.js index c68b2e3..9f5263c 100644 --- a/extension/handlers/tabs.js +++ b/extension/handlers/tabs.js @@ -13,7 +13,7 @@ import { getTabBuffer, persistSessionState, } from "../lib/state.js"; -import { showLockShield, hideLockShield } from "../lib/overlay.js"; +import { showLockShield, hideLockShield, setAgentCursorHidden, agentCursorEnabled } from "../lib/overlay.js"; import { broadcastStatus } from "../lib/connection.js"; import { lockTabUi, releaseTabUi } from "../lib/lock-ops.js"; import { withCdp, ensureViewport } from '../lib/cdp-session.js'; @@ -129,6 +129,17 @@ async function cdpScreenshot(tabId, { format, quality, scale, maxWidth, fullPage } export async function handleScreenshot(params) { + // The agent cursor stays out of the picture: it would cover what was just clicked. + const cursor = !wedgedTabs.has(params.tabId) && await agentCursorEnabled(); + if (cursor) await setAgentCursorHidden(params.tabId, true); + try { + return await captureScreenshot(params); + } finally { + if (cursor) await setAgentCursorHidden(params.tabId, false); + } +} + +async function captureScreenshot(params) { const { tabId, format = 'png', quality = 80, scale, maxWidth, fullPage = false, region } = params; const tab = await resolveTab(tabId); const protectedPage = /^(chrome|chrome-extension|devtools|edge|about):/i.test(tab.url || ''); diff --git a/extension/lib/observation-action.js b/extension/lib/observation-action.js index d5b6476..b6ca885 100644 --- a/extension/lib/observation-action.js +++ b/extension/lib/observation-action.js @@ -168,6 +168,16 @@ export async function PAGE_ACT_V2(config) { }); } } + // Agent cursor (opt-in, lib/page-dom.js): glide to the point (now in + // top-level coordinates) before acting. The events below are dispatched on + // the element itself, so only its detaching meanwhile can spoil the action. + const glideMs = config.cursor && globalThis.__bcDom?.cursor + ? globalThis.__bcDom.cursor(x, y, action === 'click' ? 'click' : 'move') + : 0; + if (glideMs > 0) { + await new Promise((resolve) => setTimeout(resolve, glideMs)); + if (!element.isConnected) return fail('STALE_STATE', 'The target detached before the action.', { ref }); + } } const eventWindow = element.ownerDocument?.defaultView || globalThis; diff --git a/extension/lib/overlay.js b/extension/lib/overlay.js index 49d51e8..e65b322 100644 --- a/extension/lib/overlay.js +++ b/extension/lib/overlay.js @@ -1,13 +1,31 @@ /** - * Page overlays (extracted from background.js). There is ONE user-visible - * element now: the lock shield — a translucent full-viewport input-capture - * layer with a blue inner frame. It serves both lifetimes: + * Page overlays (extracted from background.js). The lock shield is a + * translucent full-viewport input-capture layer with a blue inner frame. It + * serves both lifetimes: * - per-action: while an agent is actively controlling a tab (with the * running tool's name shown inside the frame — replaced the old corner * badge, per user request), and * - per-lock: for a tab lock's whole lifetime (plain frame, no label). + * The agent cursor (drawn by the shared page runtime, lib/page-dom.js) is + * switched on here and only needs hiding from screenshots. */ +/** chrome.storage.local key of the popup's "Show agent cursor" switch. */ +export const AGENT_CURSOR_KEY = 'agentCursor'; + +/** + * The agent cursor is opt-in (popup → Settings): its glide holds every mouse + * action ~200–350 ms on a visible tab. Read per action, so the switch applies + * at once; off when the setting can't be read. + */ +export async function agentCursorEnabled() { + try { + return (await chrome.storage.local.get(AGENT_CURSOR_KEY))?.[AGENT_CURSOR_KEY] === true; + } catch { + return false; + } +} + // Lock-shield: a full-viewport transparent input-capture layer + a blue inner // frame. It blocks REAL user input on the top frame; the agent's own // synthetic events bypass hit-testing by construction (handleClick / handleType @@ -137,3 +155,27 @@ export async function hideLockShield(tabId) { }); } catch {} } + +/** + * Hide or restore the agent cursor (lib/page-dom.js) around a screenshot, so + * it never covers what the agent is looking at. Cosmetic: never throws, and + * bounded so a page that stops answering can't hold the screenshot hostage. + */ +export async function setAgentCursorHidden(tabId, hidden) { + let timer; + const toggle = (async () => { + try { + await chrome.scripting.executeScript({ + target: { tabId }, + func: (h) => { + // display, not visibility: the cursor's parts reset inherited styles. + const el = document.getElementById('__bc-agent-cursor'); + if (el) el.style.display = h ? 'none' : 'block'; + }, + args: [hidden], + }); + } catch {} + })(); + await Promise.race([toggle, new Promise((r) => { timer = setTimeout(r, 1000); })]); + clearTimeout(timer); +} diff --git a/extension/lib/page-dom.js b/extension/lib/page-dom.js index 00cd0a2..0582b8f 100644 --- a/extension/lib/page-dom.js +++ b/extension/lib/page-dom.js @@ -15,7 +15,7 @@ * ambiguous is reported as gone instead of guessed. */ -export const PAGE_DOM_VERSION = 1; +export const PAGE_DOM_VERSION = 2; export function PAGE_DOM_INSTALL(version) { if (globalThis.__bcDom && globalThis.__bcDom.v === version) return false; @@ -427,11 +427,118 @@ export function PAGE_DOM_INSTALL(version) { return flatText(root, { article, max }); } + /** + * Agent cursor (user request): a visible pointer that glides to wherever the + * agent's real (CDP) mouse input lands and ripples on clicks, so a person + * watching the tab can follow the agent. Cosmetic only: pointer-events:none + * (invisible to hit-testing), never throws, fades out after a few idle + * seconds. Returns the glide's duration in ms so the caller can hold the + * real input until the cursor arrives — 0 when there is nothing to wait for + * (hidden tab, reduced motion, first appearance, no distance). + * effect: 'move' | 'click' (count rings) | 'down' / 'up' (drag press and + * release) | 'track' (snap to a re-measured point, no glide, no effect) + */ + const CURSOR_ID = '__bc-agent-cursor'; + const CURSOR_IDLE_MS = 4000; + const CURSOR_COLOR = '37,99,235'; // the agent-control frame's blue (lib/overlay.js) + + function makeCursor() { + const el = document.createElement('div'); + el.id = CURSOR_ID; + el.setAttribute('aria-hidden', 'true'); + // `all:initial` first: page CSS (resets like `svg { max-width:100% }`, + // which would collapse the arrow inside this 0×0 box) must not reach it. + el.style.cssText = 'all:initial;display:block;position:fixed;left:0;top:0;width:0;height:0;z-index:2147483647;' + + 'pointer-events:none;transition:opacity .25s;will-change:transform;'; + const NS = 'http://www.w3.org/2000/svg'; + const svg = document.createElementNS(NS, 'svg'); + svg.setAttribute('width', '25'); + svg.setAttribute('height', '30'); + svg.setAttribute('viewBox', '0 0 20 24'); + // The arrow's tip is (2, 2) at 1.25x: offset it so the tip sits exactly on the point. + // pointer-events again: `all:initial` drops the inherited none, and the tip + // sits exactly on the point the real click must reach. + svg.style.cssText = 'all:initial;pointer-events:none;position:absolute;left:-2.5px;top:-2.5px;width:25px;height:30px;overflow:visible;' + + 'transform-origin:2.5px 2.5px;transition:transform .12s;filter:drop-shadow(0 1px 2px rgba(0,0,0,.45));'; + const path = document.createElementNS(NS, 'path'); + path.setAttribute('d', 'M2 2 L2 19.5 L6.6 15.3 L9.7 22.2 L12.9 20.8 L9.9 14 L16 14 Z'); + path.style.cssText = `fill:rgb(${CURSOR_COLOR});stroke:#fff;stroke-width:1.6px;stroke-linejoin:round;`; + svg.appendChild(path); + el.appendChild(svg); + return el; + } + + function cursorEffect(el, effect, count) { + try { + const arrow = el.lastChild; + if (effect === 'down') { arrow.style.transform = 'scale(.8)'; return; } + if (effect === 'up') { arrow.style.transform = ''; return; } + // click: one ring per click (double/triple-click), and the arrow presses in. + const n = Math.min(Math.max(Number(count) || 1, 1), 3); + for (let i = 0; i < n; i++) { + const ring = document.createElement('div'); + ring.style.cssText = 'all:initial;position:absolute;left:-20px;top:-20px;width:40px;height:40px;box-sizing:border-box;' + + `border-radius:50%;border:2px solid rgb(${CURSOR_COLOR});background:rgba(${CURSOR_COLOR},.18);` + + 'pointer-events:none;opacity:0;'; + el.insertBefore(ring, arrow); // under the arrow + const a = ring.animate( + [{ transform: 'scale(.3)', opacity: 1 }, { transform: 'scale(1.5)', opacity: 0 }], + { duration: 550, delay: i * 130, easing: 'ease-out' }, + ); + a.onfinish = () => ring.remove(); + } + arrow.animate([{ transform: 'scale(1)' }, { transform: 'scale(.8)' }, { transform: 'scale(1)' }], { duration: 220 }); + } catch { /* cosmetic */ } + } + + function cursor(x, y, effect = 'move', count = 1) { + try { + const root = document.documentElement; + if (!root || !Number.isFinite(x) || !Number.isFinite(y)) return 0; + let el = document.getElementById(CURSOR_ID); + const from = el && el.__bcPos; + const fresh = !el; + if (fresh) el = makeCursor(); + // Same z-index as the agent-control shield, which is re-created per + // action: the later sibling paints on top, so stay the last one. + if (root.lastElementChild !== el) root.appendChild(el); + const to = `translate(${x}px,${y}px)`; + el.style.transform = to; + el.style.display = 'block'; // back from a screenshot (lib/overlay.js) + el.style.opacity = '1'; + el.__bcPos = { x, y }; + let still = document.visibilityState === 'hidden'; + try { still = still || matchMedia('(prefers-reduced-motion: reduce)').matches; } catch {} + let ms = 0; + if (effect === 'track') { + // Re-measured after the glide: if the target moved, jump there now. + const moved = from && Math.hypot(x - from.x, y - from.y) > 1; + if (moved && el.getAnimations) for (const a of el.getAnimations()) a.cancel(); + } else if (from && !still) { + const dist = Math.hypot(x - from.x, y - from.y); + if (dist > 2) ms = Math.round(Math.min(350, 120 + dist * 0.25)); + if (ms) { + // A drag ('up') glides linearly, in step with its evenly spaced real moves. + el.animate([{ transform: `translate(${from.x}px,${from.y}px)` }, { transform: to }], + { duration: ms, easing: effect === 'up' ? 'linear' : 'cubic-bezier(.3,.7,.4,1)' }); + } + } else if (fresh && !still) { + el.lastChild.animate([{ opacity: 0, transform: 'scale(.4)' }, { opacity: 1, transform: 'scale(1)' }], { duration: 160, easing: 'ease-out' }); + } + if (effect !== 'move' && effect !== 'track') setTimeout(() => cursorEffect(el, effect, count), ms); + clearTimeout(el.__bcIdle); + el.__bcIdle = setTimeout(() => { el.style.opacity = '0'; }, ms + CURSOR_IDLE_MS); + return ms; + } catch { + return 0; // cosmetic: never break the action it decorates + } + } + globalThis.__bcDom = Object.freeze({ v: version, clean, attr, shadowOf, frameDoc, allRoots, queryAll, isVisible, flatChildren, hasShadowHosts, roleOf, nameOf, composedText, isInteractive, preciseMatch, resolve, centerOf, elementAt, - composedContains, describe, registry, flatText, articleRoot, pageText, styleOf, connected, + composedContains, describe, registry, flatText, articleRoot, pageText, styleOf, connected, cursor, }); return true; } diff --git a/extension/lib/trusted-input.js b/extension/lib/trusted-input.js index 7ee78c6..7734052 100644 --- a/extension/lib/trusted-input.js +++ b/extension/lib/trusted-input.js @@ -10,6 +10,7 @@ */ import { ensureCdp, ensureViewport, hasCdp } from './cdp-session.js'; import { safeExec, execDom } from './page-exec.js'; +import { agentCursorEnabled } from './overlay.js'; const MOD_BITS = { alt: 1, ctrl: 2, meta: 4, shift: 8 }; @@ -106,18 +107,22 @@ export async function cdpClickAt(send, x, y, { button = 'left', clickCount = 1, } } +export const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + /** * Page-side: resolve the target through the shared DOM runtime (ref registry → * first visible selector match across shadow roots and same-origin iframes → * verified fallback), scroll it into view, optionally focus/select it, and * return its centre in TOP-level viewport coordinates (what CDP expects). * Also opens the lock shield for the agent's own trusted input for a few - * seconds, since trusted events are otherwise blocked by it. + * seconds, since trusted events are otherwise blocked by it. With `cursor` + * ({ effect, count }) the agent cursor glides to that centre (`cursorMs`). * Kept self-contained: it is serialized into the page by chrome.scripting. */ -function pageLocate(ref, sel, fb, mode) { +function pageLocate(ref, sel, fb, mode, cursor) { const D = globalThis.__bcDom; - if (!D) return { __needDom: true }; + // An older runtime (installed before an extension update) has no cursor. + if (!D || (cursor && !D.cursor)) return { __needDom: true }; let el = null; let via = 'ref'; if (ref || sel || fb) { @@ -180,6 +185,8 @@ function pageLocate(ref, sel, fb, mode) { const hit = D.elementAt(x, y); if (hit && !D.composedContains(el, hit) && !D.composedContains(hit, el)) occludedBy = D.describe(hit); } catch { /* detached mid-measure */ } + // Agent cursor (lib/page-dom.js): glide to the point the real input will hit. + const cursorMs = cursor && rect.width > 0 && rect.height > 0 && D.cursor ? D.cursor(x, y, cursor.effect, cursor.count) : 0; return { success: true, x, y, @@ -190,6 +197,7 @@ function pageLocate(ref, sel, fb, mode) { needsSelectAll: hasValue && el.value.length > 0 && !fullySelected, hasText: hasValue ? el.value.length > 0 : (el.textContent || '').length > 0, ...(occludedBy ? { occludedBy } : {}), + ...(cursorMs ? { cursorMs } : {}), ...(via !== 'ref' ? { via } : {}), // A fallback re-resolution says what it actually hit, so a wrong guess is visible. ...(via === 'fallback' ? { target: { role: D.roleOf(el), name: D.nameOf(el).slice(0, 60) } } : {}), @@ -214,17 +222,19 @@ function pageRelease() { /** * Page-side: what is at a top-level viewport point (pierces shadow roots and - * same-origin frames), and open the shield pass-through for the agent's input. + * same-origin frames), open the shield pass-through for the agent's input, + * and glide the agent cursor there. */ -function pagePointInfo(x, y) { +function pagePointInfo(x, y, effect, count) { const D = globalThis.__bcDom; - if (!D) return { __needDom: true }; + if (!D || (effect && !D.cursor)) return { __needDom: true }; window.__bcAgentInputUntil = Date.now() + 8000; const shield = document.getElementById('__bc-lock-shield'); if (shield) shield.style.pointerEvents = 'none'; const inView = x >= 0 && y >= 0 && x <= window.innerWidth && y <= window.innerHeight; + const cursorMs = effect && D.cursor ? D.cursor(x, y, effect, count) : 0; const el = D.elementAt(x, y); - if (!el) return { inView }; + if (!el) return { inView, cursorMs }; // Report the control that owns the point (e.g. the