From 8a57f17ef08af2f471a927171cd0100197903632 Mon Sep 17 00:00:00 2001 From: noiemany Date: Wed, 30 Sep 2026 01:54:14 +0300 Subject: [PATCH 01/16] fix(errors): name the selector when it matches nothing instead of "Element undefined" A selector that finds nothing (usually the page navigated or posted back) was reported as "Element undefined is gone from the DOM (feed virtualized)" plus a useless re-snapshot. It now names the selector and the page URL. Co-Authored-By: Claude Opus 5.5 --- extension/handlers/interaction.js | 15 ++++++++++----- extension/lib/trusted-input.js | 2 +- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/extension/handlers/interaction.js b/extension/handlers/interaction.js index af074db..cc11fd1 100644 --- a/extension/handlers/interaction.js +++ b/extension/handlers/interaction.js @@ -11,7 +11,12 @@ import { trustedSender, locateTarget, releaseShield, cdpClickAt, cdpKeyPress, cd export { handleDialog, handleDrag, handleFillForm } from './interaction-advanced.js'; /** Shared REF_GONE recovery: re-snapshot and hand fresh refs back (no auto-retry). */ -async function refGone(tabId, res, ref) { +async function refGone(tabId, res, ref, selector) { + // A selector that matches nothing is usually the wrong page (navigation, + // postback), not a virtualized feed — say which locator failed. + if (!(res._ref || ref) && selector) { + return { success: false, error: `No element matches selector ${selector} on the current page (${res.url || 'navigated?'}).` }; + } const fresh = await autoReSnapshot(tabId); return { success: false, @@ -36,7 +41,7 @@ export async function handleClick(params) { const send = await trustedSender(tabId, trusted); if (send && BUTTONS.has(button)) { const loc = await locateTarget(tabId, { ref, selector, fb }); - if (loc && loc.success === false && loc.error === 'REF_GONE') return refGone(tabId, loc, ref); + if (loc && loc.success === false && loc.error === 'REF_GONE') return refGone(tabId, loc, ref, selector); if (loc?.success && loc.visible) { try { await cdpClickAt(send, loc.x, loc.y, { button, clickCount: doubleClick ? 2 : 1 }); @@ -133,7 +138,7 @@ export async function handleClick(params) { // Auto-re-snapshot and embed fresh refs so the agent retries in ONE step. // We do NOT auto-retry the click: it's non-idempotent and the element that // re-appears may be a different post after the scroll shifted the feed. - if (res && res.success === false && res.error === 'REF_GONE') return refGone(tabId, res, ref); + if (res && res.success === false && res.error === 'REF_GONE') return refGone(tabId, res, ref, selector); return res; } @@ -152,7 +157,7 @@ export async function handleType(params) { const send = await trustedSender(tabId, trusted); if (send) { const loc = await locateTarget(tabId, { ref, selector, fb, mode: clear ? 'clear' : 'focus' }); - if (loc && loc.success === false && loc.error === 'REF_GONE') return refGone(tabId, loc, ref); + if (loc && loc.success === false && loc.error === 'REF_GONE') return refGone(tabId, loc, ref, selector); if (loc?.success && (loc.focused || loc.visible)) { let after; try { @@ -235,7 +240,7 @@ export async function handleType(params) { // Virtualization recovery (same as click): type target is gone, so // auto-re-snapshot and embed fresh refs. No auto-retry (non-idempotent). - if (res && res.success === false && res.error === 'REF_GONE') return refGone(tabId, res, ref); + if (res && res.success === false && res.error === 'REF_GONE') return refGone(tabId, res, ref, selector); return res; } diff --git a/extension/lib/trusted-input.js b/extension/lib/trusted-input.js index 1c8fd72..b6d7d22 100644 --- a/extension/lib/trusted-input.js +++ b/extension/lib/trusted-input.js @@ -132,7 +132,7 @@ function pageLocate(ref, sel, fb, mode) { const resolveFallback = (globalThis.__browserControllerFallbackRuntime || {}).resolveFallback || null; if (!el && fb && resolveFallback) { el = resolveFallback(fb); if (el) via = 'fallback'; } if (!el && mode === 'active') { el = document.activeElement; via = 'active'; } - if (!el) return { success: false, error: 'REF_GONE', _ref: ref }; + if (!el) return { success: false, error: 'REF_GONE', _ref: ref, url: location.href }; // Agent input pass-through for the lock shield (see overlay.js). window.__bcAgentInputUntil = Date.now() + 8000; From a268db02d02b8a92ff64ff117f80317af79eaf94 Mon Sep 17 00:00:00 2001 From: noiemany Date: Wed, 30 Sep 2026 03:46:20 +0300 Subject: [PATCH 02/16] fix(targeting): one shared resolver for every tool, find/click_text rewrite, shadow DOM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs were looked up by a data-mcp-ref attribute nothing writes any more, so every ref action fell through to the smart-selector fallback, whose first step returned the FIRST querySelector match — clicks "succeeded" on the wrong element, and hover/select/drag/fill_form/upload/press_key could not use snapshot refs at all. - lib/page-dom.js: page runtime (install-once, execDom) with ref registry → first VISIBLE selector match across open+closed shadow roots and same-origin iframes → verified fallback (unique, or nth among exact role/tag/name twins; ambiguous = REF_GONE instead of a guess). Composed hit-testing for occlusion, flat-tree text, CSS-independent names. - All tools use it: click, type, press_key, hover, select, drag, fill_form (select also matches option labels), upload_file (CDP objectId, works in shadow roots/frames), scroll. - browser_find: tokenized scoring over names/labels/attributes, role words ("search input", "... button"), shadow DOM, wrapper/echo suppression, sort before limit, optional role filter, hint when empty. - browser_click_text: case-insensitive name/composed-text match (CSS uppercase no longer breaks exact), shadow DOM, clicks the owning control with a trusted CDP click. - browser_wait: any visible match (not just the first), plus text and urlIncludes conditions. - browser_text: shadow-DOM content, mode:"article", offset paging, 100k cap. - browser_snapshot: flat tree (closed shadow roots, slots, display:contents), short refs, no isNew on the first snapshot, path-only same-origin hrefs, landmarks named by labels only, filter/depth/ref/maxChars (default 20k). - tests: page-dom resolver/find/click_text/wait/snapshot/text suites on a small fake DOM. Co-Authored-By: Claude Opus 5.5 --- extension/handlers/cdp.js | 81 ++-- extension/handlers/inspection.js | 512 ++++++++++++--------- extension/handlers/interaction-advanced.js | 78 ++-- extension/handlers/interaction.js | 273 +++++------ extension/lib/page-dom.js | 433 +++++++++++++++++ extension/lib/page-exec.js | 15 + extension/lib/trusted-input.js | 85 ++-- mcp-server/src/tools/click-text.ts | 7 +- mcp-server/src/tools/find.ts | 3 +- mcp-server/src/tools/snapshot.ts | 6 +- mcp-server/src/tools/text.ts | 6 +- mcp-server/src/tools/wait.ts | 6 +- tests/extension-agent-api.test.ts | 4 +- tests/extension-router.test.ts | 4 +- tests/helpers/fake-dom.ts | 165 +++++++ tests/legacy-ref-registry.test.ts | 7 +- tests/page-dom.test.ts | 217 +++++++++ 17 files changed, 1392 insertions(+), 510 deletions(-) create mode 100644 extension/lib/page-dom.js create mode 100644 tests/helpers/fake-dom.ts create mode 100644 tests/page-dom.test.ts diff --git a/extension/handlers/cdp.js b/extension/handlers/cdp.js index cf1a3d7..408f74b 100644 --- a/extension/handlers/cdp.js +++ b/extension/handlers/cdp.js @@ -3,7 +3,7 @@ * upload_file — the two tools that cannot be implemented with * chrome.scripting (CSP bypass / DOM.setFileInputFiles). */ -import { resolveTab, safeExec } from '../lib/page-exec.js'; +import { resolveTab, execDom, getFallback } from '../lib/page-exec.js'; import { MAX_RESULT_CHARS } from '../lib/state.js'; import { ensureCdp } from '../lib/cdp-session.js'; @@ -67,71 +67,76 @@ export async function handleRunAction(params, _sessionId, _agentName, signal) { } } +/** Main-world expression returning the node marked data-bc-upload=token (pierces open shadow roots / same-origin frames). */ +export function findMarkedExpression(token) { + return `(() => { const s = '[data-bc-upload="${token}"]'; + const q = (root, d) => { const hit = root.querySelector(s); if (hit || d > 6) return hit; + for (const el of root.querySelectorAll('*')) { + if (el.shadowRoot) { const h = q(el.shadowRoot, d + 1); if (h) return h; } + if (el.tagName === 'IFRAME') { try { const h = el.contentDocument && q(el.contentDocument, d + 1); if (h) return h; } catch (e) {} } + } + return null; }; + return q(document, 0); })()`; +} + export async function handleUploadFile(params) { const { tabId, ref, selector, filePath, files: fileList } = params; const tab = await resolveTab(tabId); const filePaths = fileList || (filePath ? [filePath] : []); if (filePaths.length === 0) throw new Error('filePath or files required'); - let sel = 'input[type="file"]'; - if (ref) sel = `[data-mcp-ref="${ref}"]`; - else if (selector) sel = selector; - - // Verify the target BEFORE the CDP round-trip: CDP's DOM.querySelector - // happily resolves any node, and DOM.setFileInputFiles on a non-file input - // fails with an opaque protocol error (or worse, on some Chrome versions, - // appears to succeed). React onChange handlers also require a change/input - // event after the files are set — CDP doesn't fire one. - const check = await safeExec(tab.id, (s) => { - const el = document.querySelector(s); + // Resolve in the page with the shared resolver (ref registry, visible-first + // selector across shadow roots / same-origin frames, verified fallback), then + // hand the node to CDP through a one-shot marker attribute. + const sel = selector || (ref ? null : 'input[type="file"]'); + const what = selector || (ref ? `ref ${ref}` : 'input[type="file"]'); + const token = `u${Date.now().toString(36)}${Math.random().toString(36).slice(2, 6)}`; + const check = await execDom(tab.id, (_ref, _sel, _fb, _token) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; + const el = D.resolve(_ref, _sel, _fb).el; if (!el) return { found: false }; + el.setAttribute('data-bc-upload', _token); return { found: true, isFileInput: el.tagName === 'INPUT' && el.type === 'file', multiple: !!el.multiple, }; - }, [sel]).catch(() => null); - if (check && check.found) { - if (!check.isFileInput) throw new Error(`Element matching ${sel} is not an .`); - if (filePaths.length > 1 && !check.multiple) { - throw new Error(`File input matching ${sel} does not accept multiple files.`); - } + }, [ref || null, sel, getFallback(tab.id, ref), token]).catch(() => null); + if (!check || !check.found) throw new Error(`File input not found: ${what}`); + if (!check.isFileInput) throw new Error(`Element matching ${what} is not an .`); + if (filePaths.length > 1 && !check.multiple) { + throw new Error(`File input matching ${what} does not accept multiple files.`); } // upload_file stays on CDP (DOM.setFileInputFiles is CDP-only). let uploaded = false; try { const send = await ensureCdp(tab.id); - await send('DOM.enable'); - const { root } = await send('DOM.getDocument'); - - const { nodeId } = await send('DOM.querySelector', { - nodeId: root.nodeId, - selector: sel, - }); - - if (!nodeId) throw new Error(`File input not found with selector: ${sel}`); - - await send('DOM.setFileInputFiles', { - files: filePaths, - nodeId, - }); + // Find the marked node wherever it lives (open shadow roots, same-origin frames). + const { result } = await send('Runtime.evaluate', { expression: findMarkedExpression(token) }); + if (!result || !result.objectId) throw new Error(`File input not found: ${what}`); + await send('DOM.setFileInputFiles', { files: filePaths, objectId: result.objectId }); uploaded = true; } finally { // Fire the events React/Vue file inputs listen for after a successful set, - // and always remove the short-lived Observation V2 handoff marker. + // and always remove the one-shot marker (and the Observation V2 handoff marker). try { - await safeExec(tab.id, (s, notify) => { - const el = document.querySelector(s); - if (!el) return; + await execDom(tab.id, (_token, notify) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; + const el = (D.queryAll(`[data-bc-upload="${_token}"]`, true) || [])[0]; + if (!el) return null; if (notify) { el.dispatchEvent(new Event('input', { bubbles: true })); el.dispatchEvent(new Event('change', { bubbles: true })); } + el.removeAttribute('data-bc-upload'); el.removeAttribute('data-bc-v2-upload'); - }, [sel, uploaded]); + return null; + }, [token, uploaded]); } catch { /* page changed — CDP outcome still determines the tool result */ } } - return { success: true, files: filePaths, selector: sel }; + return { success: true, files: filePaths, selector: what }; } diff --git a/extension/handlers/inspection.js b/extension/handlers/inspection.js index 3bea899..d67e329 100644 --- a/extension/handlers/inspection.js +++ b/extension/handlers/inspection.js @@ -2,15 +2,26 @@ * Inspection handlers (extracted from background.js): wait, scroll, snapshot, * find, text, evaluate — the read side of the toolset. */ -import { safeExec, resolveTab, getFallback } from '../lib/page-exec.js'; +import { safeExec, execDom, resolveTab, getFallback } from '../lib/page-exec.js'; import { fallbackByTab, lastSnapshotFingerprints, MAX_RESULT_CHARS, persistSessionState } from '../lib/state.js'; import { PAGE_FALLBACK_INSTALL } from '../utils/smart-selector.js'; -import { PAGE_LEGACY_REF_INSTALL } from '../utils/legacy-refs.js'; import { withCdp } from '../lib/cdp-session.js'; import { cdpEvaluate } from '../lib/cdp-evaluate.js'; +/** Default output cap for snapshots (chars of serialized tree). */ +export const SNAPSHOT_MAX_CHARS = 20_000; + +// Short refs ("s4k2-17"): a per-worker salt keeps refs from a recycled service +// worker from colliding with live ones in the page registry. +const REF_SALT = Math.random().toString(36).slice(2, 4); +let refSeq = 0; +export function nextRefPrefix(kind) { + refSeq = (refSeq + 1) % 1296; + return `${kind}${REF_SALT}${refSeq.toString(36)}-`; +} + export async function handleWait(params, _sessionId, _agentName, signal) { - const { tabId, selector, state = 'visible', timeout = 10000, delay } = params; + const { tabId, selector, state = 'visible', timeout = 10000, delay, text, urlIncludes } = params; // A promise that rejects when this call is cancelled (client gone / bridge // timeout forwarded). Long waits race against it so a cancelled call releases @@ -22,6 +33,7 @@ export async function handleWait(params, _sessionId, _agentName, signal) { }) : null; + const hasCondition = !!selector || text != null || !!urlIncludes; if (delay) { const sleep = new Promise((r) => setTimeout(r, Math.min(delay, 30000))); try { @@ -29,55 +41,79 @@ export async function handleWait(params, _sessionId, _agentName, signal) { } catch { return { success: false, error: 'aborted', waited: 0 }; } - return { success: true, waited: delay }; + return { success: true, waited: delay }; // documented: a delay ignores the conditions } - if (!selector) return { success: false, error: 'Need selector or delay' }; + if (!hasCondition) return { success: false, error: 'Need selector, text, urlIncludes or delay' }; await resolveTab(tabId); const start = Date.now(); + const what = selector || (text != null ? `text "${text}"` : `url containing "${urlIncludes}"`); while (Date.now() - start < timeout) { // Bail the moment the caller is gone so we don't pin the tab mutex for the // full timeout window after the originating agent was evicted (consistent // with handleNavigate / handleRunAction). if (signal?.aborted) return { success: false, error: 'aborted', selector, state }; - const found = await safeExec(tabId, (_sel, _state) => { - const el = document.querySelector(_sel); - if (_state === 'hidden') return !el || el.offsetParent === null; - if (_state === 'attached') return !!el; - return el && el.offsetParent !== null; - }, [selector, state]); - - if (found) return { success: true, selector, state, elapsed: Date.now() - start }; + let found; + try { + found = await execDom(tabId, (_sel, _state, _text, _url) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; + const hidden = _state === 'hidden'; + if (_url != null && !location.href.includes(_url)) return false; + if (_text != null) { + const has = D.pageText(document.body).toLowerCase().includes(String(_text).toLowerCase()); + if (hidden ? has : !has) return false; + } + if (_sel) { + // Every match across shadow roots / same-origin frames, not just the first. + const all = D.queryAll(_sel, true); + if (all === null) return { error: `Invalid CSS selector: ${_sel}` }; + if (_state === 'attached') return all.length > 0; + const anyVisible = all.some((el) => D.isVisible(el)); + return hidden ? !anyVisible : anyVisible; + } + return true; + }, [selector ?? null, state, text ?? null, urlIncludes ?? null]); + } catch { found = false; /* navigating: the next document isn't ready yet */ } + if (found && found.error) return { success: false, error: found.error }; + + if (found === true) { + return { + success: true, + ...(selector ? { selector } : {}), + ...(text != null ? { text } : {}), + ...(urlIncludes ? { urlIncludes } : {}), + state, + elapsed: Date.now() - start, + }; + } await new Promise((r) => setTimeout(r, 200)); } - return { success: false, error: `Timeout waiting for ${selector} to be ${state}` }; + return { success: false, error: `Timeout waiting for ${what} to be ${state}` }; } export async function handleScroll(params) { const { tabId, direction = 'down', amount = 500, selector, toElement, position } = params; await resolveTab(tabId); const fb = getFallback(tabId, toElement); - if (fb) await safeExec(tabId, PAGE_FALLBACK_INSTALL, []); - await safeExec(tabId, PAGE_LEGACY_REF_INSTALL, []); - return safeExec(tabId, (_dir, _amt, _sel, _toEl, _pos, _fb) => { + return execDom(tabId, (_dir, _amt, _sel, _toEl, _pos, _fb) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; if (_toEl) { - const resolveFallback = (globalThis.__browserControllerFallbackRuntime || {}).resolveFallback || null; - const resolveRef = (globalThis.__browserControllerLegacyRefRuntime || {}).resolveRef || null; - const el = (resolveRef ? resolveRef(_toEl) : null) || - document.querySelector(`[data-mcp-ref="${_toEl}"]`) || - document.querySelector(_toEl) || - (_fb && resolveFallback ? resolveFallback(_fb) : null); + // toElement accepts a ref or a CSS selector (first visible match). + let el = D.resolve(_toEl, null, _fb).el; + if (!el) { try { el = D.resolve(null, _toEl, null).el; } catch { el = null; } } if (el) { - el.scrollIntoView({ behavior: 'smooth', block: 'center' }); + el.scrollIntoView({ behavior: 'instant', block: 'center' }); return { success: true, scrolledTo: 'element' }; } return { success: false, error: 'Element not found' }; } - const target = _sel ? document.querySelector(_sel) : window; + const target = _sel ? D.resolve(null, _sel, null).el : window; if (!target) return { success: false, error: 'Scroll container not found' }; if (_pos === 'top') { @@ -118,7 +154,9 @@ export async function handleScroll(params) { * DOM with permanent data-mcp-ref attributes. */ export async function handleSnapshot(params) { - const { tabId, selector, compact = true } = params; + const { tabId, selector, ref: rootRef, depth, maxChars = SNAPSHOT_MAX_CHARS } = params; + // filter:"interactive"|"all" (Claude-in-Chrome naming) is an alias of compact. + const compact = params.filter === 'all' ? false : params.filter === 'interactive' ? true : params.compact !== false; await resolveTab(tabId); // Install the fallback page runtime first (v2 install-once pattern): the @@ -127,196 +165,185 @@ export async function handleSnapshot(params) { // extension CSP (script-src 'self', no unsafe-eval) throws in every // isolated world, which silently killed fallback capture before this fix. await safeExec(tabId, PAGE_FALLBACK_INSTALL, []); - await safeExec(tabId, PAGE_LEGACY_REF_INSTALL, []); // isNew feature: pass the fingerprints seen in the PREVIOUS snapshot so the // page function can mark newly-appeared elements. Array is serializable. - const prevFingerprints = lastSnapshotFingerprints.get(tabId) || []; - const refPrefix = `e-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}-`; + const prevFingerprints = lastSnapshotFingerprints.get(tabId) || null; + const refPrefix = nextRefPrefix('s'); - return safeExec(tabId, (_sel, _compact, _prevFingerprints, _refPrefix) => { + return execDom(tabId, (_sel, _compact, _prevFingerprints, _refPrefix, _rootRef, _depth, _maxChars) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; let refCount = 0; /** @type {Record} ref -> fallback, returned to background */ const fallbacks = {}; /** @type {string[]} fingerprints of THIS snapshot (role|name), returned to background */ const fingerprints = []; - const prevSet = new Set(_prevFingerprints); + // No previous snapshot → nothing is "new" (marking every node wasted tokens). + const prevSet = _prevFingerprints ? new Set(_prevFingerprints) : null; // Descriptor generator comes from the pre-installed page runtime. const genFallback = (globalThis.__browserControllerFallbackRuntime || {}).generateFallback || null; - const registerRef = (globalThis.__browserControllerLegacyRefRuntime || {}).registerRef || null; const skipTags = new Set(['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEMPLATE', 'SVG', 'PATH', 'BR', 'HR', 'WBR', 'META', 'LINK']); + const maxDepth = Number.isInteger(_depth) && _depth >= 0 ? _depth : Infinity; + // Output budget: stop emitting nodes once the serialized size reaches it. + let budget = Number.isInteger(_maxChars) && _maxChars > 0 ? _maxChars : Infinity; + let truncated = false; + // 'show' = render normally, 'pass' = no box of its own (display:contents, + // slots) but its children may render, false = hidden subtree. function vis(el) { - const s = getComputedStyle(el); - if (s.display === 'none' || s.visibility === 'hidden' || parseFloat(s.opacity) === 0) return false; + const s = D.styleOf(el); + if (!s || s.display === 'none') return false; + if (s.display === 'contents' || el.tagName === 'SLOT') return 'pass'; + if (s.visibility === 'hidden' || s.visibility === 'collapse' || parseFloat(s.opacity) === 0) { + // visibility is inherited but can be re-enabled below; keep walking. + return 'pass'; + } const r = el.getBoundingClientRect(); - return r.width > 0 && r.height > 0; + if (r.width > 0 && r.height > 0) return 'show'; + // Zero-size wrappers (custom-element hosts, overflow containers) can still hold visible children. + return el.childElementCount > 0 || D.shadowOf(el) ? 'pass' : false; } - function role(el) { - const r = el.getAttribute('role'); - if (r) return r; - const map = { - A: 'link', BUTTON: 'button', SELECT: 'combobox', TEXTAREA: 'textbox', IMG: 'img', - H1: 'heading', H2: 'heading', H3: 'heading', H4: 'heading', H5: 'heading', H6: 'heading', - NAV: 'navigation', MAIN: 'main', HEADER: 'banner', FOOTER: 'contentinfo', FORM: 'form', - TABLE: 'table', UL: 'list', OL: 'list', LI: 'listitem', - }; - if (el.tagName === 'INPUT') { - const t = el.type?.toLowerCase(); - if (t === 'checkbox') return 'checkbox'; - if (t === 'radio') return 'radio'; - return 'textbox'; - } - return map[el.tagName] || 'generic'; - } + const role = (el) => D.roleOf(el); + // Landmarks/regions are named only by an explicit label: their text is just + // their children's names again (token noise). + const elName = (el, r) => (landmarkRoles.has(r) && r !== 'dialog' + ? D.clean(D.attr(el, 'aria-label') || D.attr(el, 'title')) + : D.nameOf(el)).slice(0, 80); + const isInteractive = (el) => D.isInteractive(el); - function elName(el) { - const raw = ( - el.getAttribute('aria-label') || el.getAttribute('alt') || - el.getAttribute('title') || el.getAttribute('placeholder') || - '' - ).trim(); - if (raw) return raw.slice(0, 80); - const text = el.innerText; - if (!text) return ''; - const first = text.split('\n')[0].trim(); - return first.slice(0, 80); - } + const landmarkRoles = new Set(['navigation', 'main', 'banner', 'contentinfo', 'form', 'search', 'complementary', 'region', 'dialog']); - function isInteractive(el) { - const tags = ['A', 'BUTTON', 'INPUT', 'SELECT', 'TEXTAREA']; - return tags.includes(el.tagName) || el.onclick || el.getAttribute('tabindex') !== null || - el.getAttribute('role') === 'button' || el.getAttribute('role') === 'link' || - el.getAttribute('role') === 'tab' || el.getAttribute('role') === 'menuitem' || - el.getAttribute('role') === 'option' || el.getAttribute('role') === 'switch' || - el.getAttribute('contenteditable') === 'true'; + // Flat-tree children: open AND closed shadow roots, slotted content, + // same-origin iframe bodies (lib/page-dom.js flatChildren). + function childrenOf(el) { + return D.flatChildren(el).filter((c) => c.nodeType === 1); } - const landmarkRoles = new Set(['navigation', 'main', 'banner', 'contentinfo', 'form', 'search', 'complementary', 'region']); + const origin = location.origin; + function hrefOf(el) { + const h = el.href; + if (!h || typeof h !== 'string') return null; + if (h.startsWith(origin + '/')) return h.slice(origin.length); // same-origin: path only + return h; + } - // Children including shadow DOM (open roots) and same-origin iframes. - function childrenOf(el) { - const out = []; - for (const c of el.children) out.push(c); - if (el.shadowRoot) { - for (const c of el.shadowRoot.children) out.push(c); - } - // same-origin iframes: expose their document body children too. - if (el.tagName === 'IFRAME') { - try { - const doc = el.contentDocument; - if (doc && doc.body) for (const c of doc.body.children) out.push(c); - } catch { /* cross-origin: skip */ } - } - return out; + function emit(el, r, n, extra, isNewCheck) { + const ref = `${_refPrefix}${refCount++}`; + D.registry.set(ref, el); + try { if (genFallback) fallbacks[ref] = genFallback(el); } catch {} + const fp = `${r}|${n}`; + fingerprints.push(fp); + const node = { ref, role: r, ...extra }; + if (n) node.name = n; + if (isNewCheck && prevSet && !prevSet.has(fp)) node.isNew = true; + if (el.value !== undefined && el.value !== '' && typeof el.value !== 'object') node.value = String(el.value).slice(0, 200); + if (el.tagName === 'INPUT' && (el.type === 'checkbox' || el.type === 'radio')) node.checked = el.checked; + else if (D.attr(el, 'aria-checked')) node.checked = D.attr(el, 'aria-checked') === 'true'; + if (D.attr(el, 'aria-expanded')) node.expanded = D.attr(el, 'aria-expanded') === 'true'; + if (D.attr(el, 'aria-selected') === 'true') node.selected = true; + if (el.disabled) node.disabled = true; + if (el.tagName === 'A') { const h = hrefOf(el); if (h) node.href = h; } + budget -= JSON.stringify(node).length + 16; + return node; } - function buildCompact(el) { + function buildCompact(el, d) { if (!el || el.nodeType !== 1) return null; if (skipTags.has(el.tagName)) return null; - if (!vis(el)) return null; + if (budget <= 0) { truncated = true; return null; } + const v = vis(el); + if (!v) return null; - const ia = isInteractive(el); + const ia = v === 'show' && isInteractive(el); const r = role(el); - const isLandmark = landmarkRoles.has(r); - - const kids = []; - for (const c of childrenOf(el)) { - const cn = buildCompact(c); - if (cn) Array.isArray(cn) ? kids.push(...cn) : kids.push(cn); - } + const isLandmark = v === 'show' && (landmarkRoles.has(r) || (r === 'heading')); + const own = ia || isLandmark; - if (!ia && !isLandmark && r !== 'heading') { - return kids.length === 0 ? null : kids.length === 1 ? kids[0] : kids; + let node = null; + if (own) { + if (d > maxDepth) { truncated = true; return null; } + node = emit(el, r, elName(el, r), {}, true); } + const kids = []; + if (!(own && d >= maxDepth)) { + for (const c of childrenOf(el)) { + const cn = buildCompact(c, own ? d + 1 : d); + if (cn) Array.isArray(cn) ? kids.push(...cn) : kids.push(cn); + } + } else if (childrenOf(el).length) truncated = true; - const ref = `${_refPrefix}${refCount++}`; - if (registerRef) registerRef(ref, el); - const n = elName(el); - try { if (genFallback) fallbacks[ref] = genFallback(el); } catch {} - - // isNew: mark elements whose (role|name) wasn't in the previous snapshot. - const fp = `${r}|${n}`; - fingerprints.push(fp); - const isNew = !prevSet.has(fp); - - const node = { ref, role: r }; - if (n) node.name = n; - if (isNew) node.isNew = true; - if (el.value !== undefined && el.value !== '') node.value = String(el.value); - if (el.checked !== undefined) node.checked = el.checked; - if (el.disabled) node.disabled = true; - if (el.href && el.tagName === 'A') node.href = el.href; + if (!own) return kids.length === 0 ? null : kids.length === 1 ? kids[0] : kids; if (kids.length) node.children = kids; - return node; } - function buildFull(el, depth) { + function buildFull(el, d) { if (!el || el.nodeType !== 1) return null; if (skipTags.has(el.tagName)) return null; - if (!vis(el)) return null; + if (budget <= 0) { truncated = true; return null; } + const v = vis(el); + if (!v) return null; const r = role(el); - const n = elName(el); - const ia = isInteractive(el); + const ia = v === 'show' && isInteractive(el); + const n = v === 'show' ? elName(el, r) : ''; - if (r === 'generic' && !n && !ia && depth > 1) { + if (v !== 'show' || (r === 'generic' && !n && !ia && d > 1)) { const kids = []; for (const c of childrenOf(el)) { - const cn = buildFull(c, depth + 1); + const cn = buildFull(c, d + (v === 'show' ? 1 : 0)); if (cn) Array.isArray(cn) ? kids.push(...cn) : kids.push(cn); } return kids.length === 0 ? null : kids.length === 1 ? kids[0] : kids; } + if (d > maxDepth) { truncated = true; return null; } - const ref = `${_refPrefix}${refCount++}`; - if (registerRef) registerRef(ref, el); - try { if (genFallback) fallbacks[ref] = genFallback(el); } catch {} - - // isNew: mark elements whose (role|name) wasn't in the previous snapshot. - const fp = `${r}|${n}`; - fingerprints.push(fp); - const isNew = !prevSet.has(fp); - - const node = { ref, role: r }; - if (r === 'generic') node.tag = el.tagName.toLowerCase(); - if (n) node.name = n; - if (isNew) node.isNew = true; - if (el.value !== undefined && el.value !== '') node.value = String(el.value); - if (el.checked !== undefined) node.checked = el.checked; - if (el.disabled) node.disabled = true; - if (el.href && el.tagName === 'A') node.href = el.href; - + const node = emit(el, r, n, r === 'generic' ? { tag: el.tagName.toLowerCase() } : {}, true); const kids = []; for (const c of childrenOf(el)) { - const cn = buildFull(c, depth + 1); + const cn = buildFull(c, d + 1); if (cn) Array.isArray(cn) ? kids.push(...cn) : kids.push(cn); } if (kids.length) node.children = kids; - return node; } - const root = _sel ? document.querySelector(_sel) : document.body; + let root = document.body; + if (_rootRef) { + root = D.registry.get(_rootRef); + if (!D.connected(root)) return { success: false, error: `ref ${_rootRef} is gone — take a new snapshot` }; + } else if (_sel) { + const hit = D.resolve(null, _sel, null); + if (hit.error === 'INVALID_SELECTOR') return { success: false, error: `Invalid CSS selector: ${_sel}` }; + root = hit.el; + } if (!root) return { success: false, error: 'Root element not found' }; - const tree = _compact ? buildCompact(root) : buildFull(root, 0); + const tree = _compact ? buildCompact(root, 0) : buildFull(root, 0); return { success: true, url: location.href, title: document.title, compact: _compact, tree, + ...(truncated ? { + truncated: true, + hint: 'Output capped (maxChars/depth). Scope it with selector or ref (a subtree), or raise maxChars.', + } : {}), // internal: background stores these per-tab; never sent to the agent. __fallbacks: fallbacks, __fingerprints: fingerprints, }; - }, [selector, compact, prevFingerprints, refPrefix]).then((res) => { + }, [selector ?? null, compact, prevFingerprints, refPrefix, rootRef ?? null, depth ?? null, maxChars]).then((res) => { // Store the fallbacks per-tab so click/type can resolve stale refs, and - // persist them across service-worker recycles (MV3 lifetime). + // persist them across service-worker recycles (MV3 lifetime). Merged, not + // replaced: a scoped snapshot must not invalidate refs from the full one. if (res && res.__fallbacks) { - const map = new Map(Object.entries(res.__fallbacks)); + const map = fallbackByTab.get(tabId) || new Map(); + for (const [ref, fbEntry] of Object.entries(res.__fallbacks)) map.set(ref, fbEntry); + // Bound the map: keep the most recent entries. + while (map.size > 3000) map.delete(map.keys().next().value); fallbackByTab.set(tabId, map); delete res.__fallbacks; // keep it out of the agent-visible payload persistSessionState(); @@ -331,71 +358,114 @@ export async function handleSnapshot(params) { } export async function handleFind(params) { - const { tabId, query, limit = 10 } = params; + const { tabId, query, limit = 10, role } = params; await resolveTab(tabId); await safeExec(tabId, PAGE_FALLBACK_INSTALL, []); - await safeExec(tabId, PAGE_LEGACY_REF_INSTALL, []); - const refPrefix = `f-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}-`; + const refPrefix = nextRefPrefix('f'); - return safeExec(tabId, (_q, _lim, _refPrefix) => { - const qLow = _q.toLowerCase(); - const matches = []; - const fallbacks = {}; + return execDom(tabId, (_q, _lim, _refPrefix, _role) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; const genFallback = (globalThis.__browserControllerFallbackRuntime || {}).generateFallback || null; - const registerRef = (globalThis.__browserControllerLegacyRefRuntime || {}).registerRef || null; + const fallbacks = {}; - function aName(el) { - return (el.getAttribute('aria-label') || el.getAttribute('alt') || el.getAttribute('title') || - el.getAttribute('placeholder') || el.innerText?.slice(0, 200) || '').trim(); + // Words that describe the KIND of element, mapped to the roles they mean. + const ROLE_WORDS = { + button: ['button'], btn: ['button'], link: ['link'], anchor: ['link'], + input: ['textbox', 'searchbox', 'combobox', 'spinbutton'], field: ['textbox', 'searchbox', 'combobox', 'spinbutton'], + textbox: ['textbox', 'searchbox'], box: ['textbox', 'searchbox', 'combobox', 'checkbox'], textarea: ['textbox'], + searchbox: ['searchbox'], checkbox: ['checkbox'], check: ['checkbox'], radio: ['radio'], + dropdown: ['combobox', 'listbox', 'button'], select: ['combobox', 'listbox'], combobox: ['combobox'], + tab: ['tab'], menu: ['menu', 'menubar', 'button'], menuitem: ['menuitem'], option: ['option'], + heading: ['heading'], title: ['heading'], image: ['img'], img: ['img'], icon: ['img', 'button'], + dialog: ['dialog', 'alertdialog'], modal: ['dialog', 'alertdialog'], switch: ['switch'], toggle: ['switch', 'button', 'checkbox'], + slider: ['slider'], list: ['list', 'listbox'], table: ['table', 'grid'], row: ['row'], cell: ['cell', 'gridcell'], + }; + const STOP = new Set(['the', 'a', 'an', 'to', 'of', 'for', 'on', 'in', 'with', 'and', 'that', 'this', 'element', 'please']); + const words = String(_q).toLowerCase().split(/[^\p{L}\p{N}_-]+/u).filter((w) => w && !STOP.has(w)); + const roleHints = new Set(); + const content = []; + for (const w of words) { + if (ROLE_WORDS[w]) ROLE_WORDS[w].forEach((r) => roleHints.add(r)); + else content.push(w); } - - function aRole(el) { - const r = el.getAttribute('role'); - if (r) return r; - const map = { A: 'link', BUTTON: 'button', INPUT: 'input', SELECT: 'combobox', TEXTAREA: 'textbox', IMG: 'image' }; - return map[el.tagName] || el.tagName.toLowerCase(); + // "search" names the purpose AND a role. + if (words.includes('search')) roleHints.add('searchbox'); + const phrase = content.join(' '); + const wantRole = _role ? String(_role).toLowerCase() : null; + + const SKIP = new Set(['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEMPLATE', 'META', 'LINK', 'HEAD', 'HTML', 'BODY', 'BR', 'PATH']); + const cands = []; + for (const root of D.allRoots(true)) { + let els = []; + try { els = root.querySelectorAll('*'); } catch {} + for (const el of els) { + if (SKIP.has(el.tagName)) continue; + const r = D.roleOf(el); + if (r === 'none') continue; + if (wantRole && r !== wantRole) continue; + const name = D.nameOf(el).toLowerCase(); + const attrs = [el.id, D.attr(el, 'name'), D.attr(el, 'type'), D.attr(el, 'placeholder'), D.attr(el, 'data-testid'), + D.attr(el, 'title'), typeof el.className === 'string' ? el.className : ''].join(' ').toLowerCase(); + const interactive = D.isInteractive(el); + let score = 0; + let covered = 0; + for (const w of content) { + const inName = name.includes(w); + const inAttr = attrs.includes(w); + if (inName) score += new RegExp(`(^|[^\\p{L}\\p{N}])${w.replace(/[.*+?^${}()|[\]\\-]/g, '\\$&')}([^\\p{L}\\p{N}]|$)`, 'u').test(name) ? 6 : 4; + else if (inAttr) score += 3; + if (inName || inAttr) covered++; + } + if (content.length && covered === 0) continue; + if (phrase && name === phrase) score += 12; + else if (phrase && content.length > 1 && name.includes(phrase)) score += 6; + if (roleHints.size) { + if (roleHints.has(r) || (roleHints.has('searchbox') && /search/.test(attrs) && ['textbox', 'searchbox', 'combobox'].includes(r))) score += 8; + else if (!content.length) continue; + else score -= 2; + } + if (interactive) score += 4; + else if (r === 'generic') score -= 3; + // A container whose text merely CONTAINS the words is a weak match. + if (name.length > 120) score -= 4; + const coverage = content.length ? covered / content.length : 1; + if (coverage < 0.5) continue; + score = Math.round(score * coverage * 10) / 10; + if (score <= 0) continue; + cands.push({ el, r, name, score, interactive }); + } + } + cands.sort((a, b) => b.score - a.score); + // Visibility is the expensive check: only for the best-scoring pool. + const pool = []; + for (const c of cands) { + if (pool.length >= _lim * 6) break; + if (D.isVisible(c.el)) pool.push(c); } + // Drop wrappers (an ancestor scoring no better than a descendant) and echoes + // (a descendant repeating the name of the control that contains it). + const kept = pool.filter((c) => !pool.some((o) => o !== c && ( + (o.score >= c.score && D.composedContains(c.el, o.el)) + || (o.interactive && !c.interactive && o.score >= c.score && o.name === c.name && D.composedContains(o.el, c.el))))); - // Same-origin iframe piercing (field report: legacy UIs live entirely - // inside #mainFrame — the top-document walk saw none of it). - const roots = [document.body]; - (function collectFrames(doc, depth) { - if (depth >= 3) return; - for (const f of doc.querySelectorAll('iframe')) { - try { const d = f.contentDocument; if (d && d.body) { roots.push(d.body); collectFrames(d, depth + 1); } } catch {} - } - })(document, 0); - let rc = 0; - let node; - for (const root of roots) { - const walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT); - while ((node = walker.nextNode()) && matches.length < _lim * 3) { - const s = getComputedStyle(node); - const rect = node.getBoundingClientRect(); - if (s.display === 'none' || s.visibility === 'hidden' || rect.width === 0) continue; - - const n = aName(node).toLowerCase(); - const r = aRole(node).toLowerCase(); - const id = (node.id || '').toLowerCase(); - let score = 0; - if (n.includes(qLow)) score += 10; - if (r.includes(qLow)) score += 5; - if (id.includes(qLow)) score += 3; - if (score === 0) continue; - - const ref = `${_refPrefix}${rc++}`; - if (registerRef) registerRef(ref, node); - try { if (genFallback) fallbacks[ref] = genFallback(node); } catch {} + const matches = []; + kept.slice(0, _lim).forEach((c, i) => { + const ref = `${_refPrefix}${i}`; + D.registry.set(ref, c.el); + try { if (genFallback) fallbacks[ref] = genFallback(c.el); } catch {} + const rect = D.centerOf(c.el).rect; matches.push({ - ref, role: r, name: n.slice(0, 100), tag: node.tagName.toLowerCase(), score, + ref, role: c.r, name: D.nameOf(c.el).slice(0, 80), tag: c.el.tagName.toLowerCase(), score: c.score, bounds: { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) }, }); - } - } - - matches.sort((a, b) => b.score - a.score); - return { success: true, query: _q, matches: matches.slice(0, _lim), __fallbacks: fallbacks }; - }, [query, limit, refPrefix]).then((res) => { + }); + return { + success: true, query: _q, matches, + ...(matches.length === 0 ? { hint: 'No match. Try fewer/other words, a role filter, browser_snapshot, or browser_text.' } : {}), + __fallbacks: fallbacks, + }; + }, [query, limit, refPrefix, role || null]).then((res) => { if (res && res.__fallbacks) { const map = fallbackByTab.get(tabId) || new Map(); for (const [ref, fbEntry] of Object.entries(res.__fallbacks)) map.set(ref, fbEntry); @@ -411,21 +481,41 @@ export async function handleGetPageText(params) { // Default must match the MCP schema (text.ts: maxLength .default(5000)) — // it drifted 10x here once, so direct-WS callers got 50000 while MCP callers // got 5000 from the same knob. - const { tabId, selector, maxLength = 5000 } = params; + const { tabId, selector, maxLength = 5000, mode = 'all', offset = 0 } = params; await resolveTab(tabId); - const args = selector === undefined ? [null, maxLength] : [selector, maxLength]; - - return safeExec(tabId, (_sel, _max) => { - const root = _sel ? document.querySelector(_sel) : document.body; + const max = Math.min(Math.max(1, Number(maxLength) || 5000), 100_000); + const from = Math.max(0, Number(offset) || 0); + + return execDom(tabId, (_sel, _max, _mode, _from) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; + const article = _mode === 'article'; + let root = document.body; + if (_sel) { + const hit = D.resolve(null, _sel, null); + if (hit.error === 'INVALID_SELECTOR') return { success: false, error: `Invalid CSS selector: ${_sel}` }; + root = hit.el; + } else if (article) { + root = D.articleRoot(); + } if (!root) return { success: false, error: 'Element not found' }; - let text = root.innerText || root.textContent || ''; - text = text.replace(/\t/g, ' ').replace(/\n\s*\n/g, '\n\n').replace(/ +/g, ' ').trim(); + // Composed text: includes open/closed shadow roots and same-origin frames + // (innerText alone misses web-component content such as caniuse's tables). + let text = D.pageText(root, { article, max: _from + _max + 1000 }); + const total = text.length; + if (_from) text = text.slice(_from); const truncated = text.length > _max; if (truncated) text = text.slice(0, _max) + '...'; - return { success: true, url: location.href, title: document.title, text, length: text.length, truncated }; - }, args); + return { + success: true, url: location.href, title: document.title, text, length: text.length, truncated, + ...(_from ? { offset: _from } : {}), + ...(truncated ? { nextOffset: _from + _max } : {}), + ...(article ? { mode: 'article' } : {}), + ...(total && _from >= total ? { note: `offset ${_from} is past the end (${total} chars)` } : {}), + }; + }, [selector ?? null, max, mode, from]); } /** diff --git a/extension/handlers/interaction-advanced.js b/extension/handlers/interaction-advanced.js index 6fe73ef..61c94fa 100644 --- a/extension/handlers/interaction-advanced.js +++ b/extension/handlers/interaction-advanced.js @@ -3,7 +3,7 @@ * orchestration. Kept separate from the common pointer/keyboard handlers so * each module stays focused and reviewable. */ -import { resolveTab, safeExec } from '../lib/page-exec.js'; +import { resolveTab, safeExec, execDom, getFallback } from '../lib/page-exec.js'; import { withCdp } from '../lib/cdp-session.js'; import { openShield, releaseShield } from '../lib/trusted-input.js'; @@ -64,32 +64,19 @@ export async function handleDrag(params) { let sx = startX, sy = startY, ex = endX, ey = endY; if (sx == null || sy == null || ex == null || ey == null) { - const coords = await safeExec(tabId, (_sRef, _sSel, _eRef, _eSel) => { - function deepQuery(sel) { - const query = (doc, depth) => { - try { const el = doc.querySelector(sel); if (el) return el; } catch {} - if (depth >= 3) return null; - for (const frame of doc.querySelectorAll('iframe')) { - try { - const child = frame.contentDocument; - if (child) { const el = query(child, depth + 1); if (el) return el; } - } catch {} - } - return null; - }; - return query(document, 0); - } - - function find(ref, selector) { - let el = ref ? deepQuery(`[data-mcp-ref="${ref}"]`) : null; - if (!el && selector) el = deepQuery(selector); + const coords = await execDom(tabId, (_sRef, _sSel, _eRef, _eSel, _sFb, _eFb) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; + function find(ref, selector, fb) { + if (!ref && !selector) return null; + const el = D.resolve(ref, selector, fb).el; if (!el) return null; el.scrollIntoView({ behavior: 'instant', block: 'center' }); - const rect = el.getBoundingClientRect(); - return { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 }; + const { x, y } = D.centerOf(el); + return { x, y }; } - return { start: find(_sRef, _sSel), end: find(_eRef, _eSel) }; - }, [startRef, startSelector, endRef, endSelector]); + return { start: find(_sRef, _sSel, _sFb), end: find(_eRef, _eSel, _eFb) }; + }, [startRef, startSelector, endRef, endSelector, getFallback(tabId, startRef), getFallback(tabId, endRef)]); if (coords.start) { sx = coords.start.x; sy = coords.start.y; } if (coords.end) { ex = coords.end.x; ey = coords.end.y; } @@ -127,21 +114,12 @@ export async function handleFillForm(params) { } await resolveTab(tabId); - return safeExec(tabId, (_fields, _submit) => { - function deepQuery(sel) { - const query = (doc, depth) => { - try { const el = doc.querySelector(sel); if (el) return el; } catch {} - if (depth >= 3) return null; - for (const frame of doc.querySelectorAll('iframe')) { - try { - const child = frame.contentDocument; - if (child) { const el = query(child, depth + 1); if (el) return el; } - } catch {} - } - return null; - }; - return query(document, 0); - } + // Attach each ref's snapshot descriptor so stale refs re-resolve (verified) in the page. + const withFb = fields.map((f) => (f && f.ref ? { ...f, fb: getFallback(tabId, f.ref) } : f)); + + return execDom(tabId, (_fields, _submit) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; const setNativeValue = (target, nextValue) => { const prototype = target instanceof HTMLTextAreaElement @@ -154,9 +132,8 @@ export async function handleFillForm(params) { const results = []; let containingForm = null; for (const field of _fields) { - const { ref, selector, value, clear } = field; - let el = ref ? deepQuery(`[data-mcp-ref="${ref}"]`) : null; - if (!el && selector) el = deepQuery(selector); + const { ref, selector, value, clear, fb } = field; + const el = D.resolve(ref, selector, fb).el; if (!el) { results.push({ selector: selector || ref, success: false, error: 'Not found' }); continue; @@ -164,20 +141,27 @@ export async function handleFillForm(params) { el.focus(); if (el.form && !containingForm) containingForm = el.form; - if (clear !== false) { + const isChoice = el.tagName === 'SELECT' || el.type === 'checkbox' || el.type === 'radio'; + if (clear !== false && !isChoice) { if (el.isContentEditable) el.textContent = ''; else setNativeValue(el, ''); el.dispatchEvent(new Event('input', { bubbles: true })); } if (el.tagName === 'SELECT') { - const option = Array.from(el.options).find((candidate) => candidate.value === String(value)); + // Match the option's value first, then its visible label. + const want = String(value); + const options = Array.from(el.options); + const option = options.find((candidate) => candidate.value === want) + || options.find((candidate) => candidate.textContent.trim() === want.trim()) + || options.find((candidate) => candidate.textContent.trim().toLowerCase() === want.trim().toLowerCase()); if (!option) { results.push({ selector: selector || ref, success: false, error: `Option "${value}" not found` }); continue; } - setNativeValue(el, String(value)); - el.dispatchEvent(new Event('change', { bubbles: true })); + const setter = Object.getOwnPropertyDescriptor(HTMLSelectElement.prototype, 'value')?.set; + if (setter) setter.call(el, option.value); else el.value = option.value; + el.dispatchEvent(new Event('input', { bubbles: true })); } else if (el.type === 'checkbox' || el.type === 'radio') { const checked = value === true || value === 'true'; if (el.checked !== checked) el.click(); @@ -205,5 +189,5 @@ export async function handleFillForm(params) { return failed === 0 ? { success: true, fields: results } : { success: false, error: `${failed} of ${results.length} fields failed`, fields: results }; - }, [fields, submit]); + }, [withFb, submit]); } diff --git a/extension/handlers/interaction.js b/extension/handlers/interaction.js index cc11fd1..3f81740 100644 --- a/extension/handlers/interaction.js +++ b/extension/handlers/interaction.js @@ -3,9 +3,8 @@ * hover, select, click_text, dialog, drag, fill_form — the write side that * drives the page's event system (synthetic events) or CDP when required. */ -import { resolveTab, requireTarget, safeExec, getFallback } from '../lib/page-exec.js'; +import { resolveTab, requireTarget, execDom, getFallback } from '../lib/page-exec.js'; import { autoReSnapshot } from './inspection.js'; -import { PAGE_FALLBACK_INSTALL } from '../utils/smart-selector.js'; import { trustedSender, locateTarget, releaseShield, cdpClickAt, cdpKeyPress, cdpTypeText, keyDefinition } from '../lib/trusted-input.js'; export { handleDialog, handleDrag, handleFillForm } from './interaction-advanced.js'; @@ -31,10 +30,8 @@ export async function handleClick(params) { const { tabId, ref, selector, button = 'left', doubleClick = false, trusted } = params; await resolveTab(tabId); requireTarget(params); + // Snapshot-time descriptor used by the shared resolver when the ref is stale. const fb = getFallback(tabId, ref); - // Install the fallback page runtime only when a descriptor exists (v2 - // install-once pattern — eval rebuilding is impossible under MV3 CSP). - if (fb) await safeExec(tabId, PAGE_FALLBACK_INSTALL, []); // Trusted path: a real mouse click at the element's centre over CDP, so // focus moves, default actions run and the page sees isTrusted:true. @@ -59,33 +56,19 @@ export async function handleClick(params) { // Zero-size element: no point to hit — fall through to the synthetic path. } - const res = await safeExec(tabId, async (_ref, _sel, _btn, _dbl, _fb) => { - // Same-origin iframe piercing (field report: legacy UIs live inside - // #mainFrame — top-document lookups missed every element). - function deepQuery(sel) { - const q = (doc, depth) => { - try { const el = doc.querySelector(sel); if (el) return el; } catch {} - if (depth >= 3) return null; - for (const f of doc.querySelectorAll('iframe')) { - try { const d = f.contentDocument; if (d) { const el = q(d, depth + 1); if (el) return el; } } catch {} - } - return null; - }; - return q(document, 0); - } + const res = await execDom(tabId, async (_ref, _sel, _btn, _dbl, _fb) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; - let el = _ref ? deepQuery(`[data-mcp-ref="${_ref}"]`) : null; - let via = 'ref'; - if (!el && _sel) { el = deepQuery(_sel); via = 'selector'; } - // Resolver comes from the pre-installed page runtime (no eval). - const resolveFallback = (globalThis.__browserControllerFallbackRuntime || {}).resolveFallback || null; - // Smart-selector fallback (plan task 3): ref broke → try robust selector, - // then text+role+tag scan. The agent doesn't request this; it's automatic. - if (!el && _fb && resolveFallback) { el = resolveFallback(_fb); if (el) via = 'fallback'; } + // ref registry → first visible selector match → verified fallback (lib/page-dom.js). + const found = D.resolve(_ref, _sel, _fb); + if (found.error === 'INVALID_SELECTOR') return { success: false, error: `Invalid CSS selector: ${_sel}` }; + let el = found.el || null; + const via = found.via || 'ref'; if (!el) { // Element is gone (likely virtualized away on scroll). Abort WITHOUT // clicking — the background auto-re-snapshots and embeds fresh refs. - return { success: false, error: 'REF_GONE', _ref }; + return { success: false, error: 'REF_GONE', _ref, url: location.href }; } el.scrollIntoView({ behavior: 'instant', block: 'center' }); @@ -100,7 +83,7 @@ export async function handleClick(params) { if (!visible0) { await new Promise((r) => setTimeout(r, 200)); // re-resolve the element (it may have been re-rendered with a new node) - el = _ref ? deepQuery(`[data-mcp-ref="${_ref}"]`) : el; + el = D.resolve(_ref, _sel, _fb).el || el; if (el) el.scrollIntoView({ behavior: 'instant', block: 'center' }); } if (!el) return { success: false, error: 'REF_GONE', _ref }; @@ -146,10 +129,8 @@ export async function handleType(params) { const { tabId, ref, selector, text, clear = false, trusted } = params; await resolveTab(tabId); requireTarget(params); + // Snapshot-time descriptor used by the shared resolver when the ref is stale. const fb = getFallback(tabId, ref); - // Install the fallback page runtime only when a descriptor exists (v2 - // install-once pattern — eval rebuilding is impossible under MV3 CSP). - if (fb) await safeExec(tabId, PAGE_FALLBACK_INSTALL, []); // Trusted path: focus the field, then real key presses over CDP (keydown / // keypress / input / keyup per character). Like a user, this does NOT fire @@ -180,30 +161,18 @@ export async function handleType(params) { await releaseShield(tabId); } - const res = await safeExec(tabId, (_ref, _sel, _text, _clear, _fb) => { - // Same-origin iframe piercing (field report: legacy UIs live inside - // #mainFrame — top-document lookups missed every element). - function deepQuery(sel) { - const q = (doc, depth) => { - try { const el = doc.querySelector(sel); if (el) return el; } catch {} - if (depth >= 3) return null; - for (const f of doc.querySelectorAll('iframe')) { - try { const d = f.contentDocument; if (d) { const el = q(d, depth + 1); if (el) return el; } } catch {} - } - return null; - }; - return q(document, 0); - } + const res = await execDom(tabId, (_ref, _sel, _text, _clear, _fb) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; - let el = _ref ? deepQuery(`[data-mcp-ref="${_ref}"]`) : null; - let via = 'ref'; - if (!el && _sel) { el = deepQuery(_sel); via = 'selector'; } - const resolveFallback = (globalThis.__browserControllerFallbackRuntime || {}).resolveFallback || null; - if (!el && _fb && resolveFallback) { el = resolveFallback(_fb); if (el) via = 'fallback'; } + const found = D.resolve(_ref, _sel, _fb); + if (found.error === 'INVALID_SELECTOR') return { success: false, error: `Invalid CSS selector: ${_sel}` }; + const el = found.el || null; + const via = found.via || 'ref'; if (!el) { // Element gone (virtualized feed) — abort WITHOUT typing; background // auto-re-snapshots and embeds fresh refs for a one-step retry. - return { success: false, error: 'REF_GONE', _ref }; + return { success: false, error: 'REF_GONE', _ref, url: location.href }; } el.focus(); @@ -263,6 +232,7 @@ export async function handlePressKey(params) { const { tabId, ref, selector, trusted } = params; const { key, mods: modifiers } = parseKeyCombo(params.key, params.modifiers || []); await resolveTab(tabId); + const fb = getFallback(tabId, ref); // Trusted path: a real key press, so default actions run (Tab moves focus // and fires blur/focusout, Enter submits, arrows drive autocomplete menus). @@ -270,7 +240,7 @@ export async function handlePressKey(params) { try { keyDefinition(key); } catch { knownKey = false; } const send = knownKey ? await trustedSender(tabId, trusted) : null; if (send) { - const loc = await locateTarget(tabId, { ref, selector, mode: ref || selector ? 'focus' : 'active' }); + const loc = await locateTarget(tabId, { ref, selector, fb, mode: ref || selector ? 'focus' : 'active' }); if (!loc || loc.success === false) { await releaseShield(tabId); if (ref || selector) return { success: false, error: `Element ${ref ? `with ref ${ref}` : `with selector ${selector}`} not found` }; @@ -284,34 +254,18 @@ export async function handlePressKey(params) { return { success: true, key, ...(modifiers.length ? { modifiers } : {}), input: 'cdp', ...(after?.focusedTag ? { focused: after.focusedTag } : {}) }; } - return safeExec(tabId, (_key, _mods, _ref, _sel) => { - // Same-origin iframe piercing (field report: legacy UIs live inside - // #mainFrame — top-document lookups missed every element). - function deepQuery(sel) { - const q = (doc, depth) => { - try { const el = doc.querySelector(sel); if (el) return el; } catch {} - if (depth >= 3) return null; - for (const f of doc.querySelectorAll('iframe')) { - try { const d = f.contentDocument; if (d) { const el = q(d, depth + 1); if (el) return el; } } catch {} - } - return null; - }; - return q(document, 0); - } + return execDom(tabId, (_key, _mods, _ref, _sel, _fb) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; let target = document.activeElement || document.body; // When the caller names a target, an unresolved ref/selector must FAIL — // silently falling back to activeElement sent Enter to the wrong control // with a success result. (Omitting both is still legitimate: intentional // activeElement targeting.) - if (_ref) { - const el = deepQuery(`[data-mcp-ref="${_ref}"]`); - if (!el) return { success: false, error: `Element with ref ${_ref} not found` }; - el.focus(); - target = el; - } else if (_sel) { - const el = deepQuery(_sel); - if (!el) return { success: false, error: `Element with selector ${_sel} not found` }; + if (_ref || _sel) { + const el = D.resolve(_ref, _sel, _fb).el; + if (!el) return { success: false, error: _ref ? `Element with ref ${_ref} not found` : `Element with selector ${_sel} not found` }; el.focus(); target = el; } @@ -332,17 +286,18 @@ export async function handlePressKey(params) { target.dispatchEvent(new KeyboardEvent('keyup', init)); return { success: true, key: _key }; - }, [key, modifiers, ref, selector]); + }, [key, modifiers, ref, selector, fb]); } export async function handleHover(params) { const { tabId, ref, selector, trusted } = params; await resolveTab(tabId); requireTarget(params); + const fb = getFallback(tabId, ref); const send = await trustedSender(tabId, trusted); if (send) { - const loc = await locateTarget(tabId, { ref, selector }); + const loc = await locateTarget(tabId, { ref, selector, fb }); if (loc?.success && loc.visible) { try { await send('Input.dispatchMouseEvent', { type: 'mouseMoved', x: loc.x, y: loc.y }); @@ -355,23 +310,11 @@ export async function handleHover(params) { if (loc && loc.success === false) return { success: false, error: 'Element not found' }; } - return safeExec(tabId, (_ref, _sel) => { - // Same-origin iframe piercing (field report: legacy UIs live inside - // #mainFrame — top-document lookups missed every element). - function deepQuery(sel) { - const q = (doc, depth) => { - try { const el = doc.querySelector(sel); if (el) return el; } catch {} - if (depth >= 3) return null; - for (const f of doc.querySelectorAll('iframe')) { - try { const d = f.contentDocument; if (d) { const el = q(d, depth + 1); if (el) return el; } } catch {} - } - return null; - }; - return q(document, 0); - } + return execDom(tabId, (_ref, _sel, _fb) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; - let el = _ref ? deepQuery(`[data-mcp-ref="${_ref}"]`) : null; - if (!el && _sel) el = deepQuery(_sel); + const el = D.resolve(_ref, _sel, _fb).el; if (!el) return { success: false, error: 'Element not found' }; el.scrollIntoView({ behavior: 'instant', block: 'center' }); @@ -385,7 +328,7 @@ export async function handleHover(params) { el.dispatchEvent(new MouseEvent('mousemove', init)); return { success: true }; - }, [ref, selector]); + }, [ref, selector, fb]); } export async function handleSelect(params) { @@ -395,24 +338,13 @@ export async function handleSelect(params) { if (value === undefined && label === undefined && index === undefined) { throw new Error('One of value, label, or index is required to pick an option.'); } + const fb = getFallback(tabId, ref); - return safeExec(tabId, (_ref, _sel, _val, _lbl, _idx) => { - // Same-origin iframe piercing (field report: legacy UIs live inside - // #mainFrame — top-document lookups missed every element). - function deepQuery(sel) { - const q = (doc, depth) => { - try { const el = doc.querySelector(sel); if (el) return el; } catch {} - if (depth >= 3) return null; - for (const f of doc.querySelectorAll('iframe')) { - try { const d = f.contentDocument; if (d) { const el = q(d, depth + 1); if (el) return el; } } catch {} - } - return null; - }; - return q(document, 0); - } + return execDom(tabId, (_ref, _sel, _val, _lbl, _idx, _fb) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; - let el = _ref ? deepQuery(`[data-mcp-ref="${_ref}"]`) : null; - if (!el && _sel) el = deepQuery(_sel); + const el = D.resolve(_ref, _sel, _fb).el; if (!el) return { success: false, error: 'Element not found' }; if (el.tagName !== 'SELECT') return { success: false, error: 'Not a select element' }; @@ -429,69 +361,98 @@ export async function handleSelect(params) { el.dispatchEvent(new Event('change', { bubbles: true })); el.dispatchEvent(new Event('input', { bubbles: true })); return { success: true, selected: el.value }; - }, [ref, selector, value, label, index]); + }, [ref, selector, value, label, index, fb]); } export async function handleClickByText(params) { - const { tabId, text, index = 0, exact = false } = params; + const { tabId, text, index = 0, exact = false, trusted } = params; await resolveTab(tabId); - - return safeExec(tabId, (_text, _index, _exact) => { - const textLower = _text.toLowerCase(); - const candidates = []; - // Same-origin iframe piercing — walk every frame body, not just the top. - const roots = [document.body]; - (function collectFrames(doc, depth) { - if (depth >= 3) return; - for (const f of doc.querySelectorAll('iframe')) { - try { const d = f.contentDocument; if (d && d.body) { roots.push(d.body); collectFrames(d, depth + 1); } } catch {} - } - })(document, 0); - let node; - for (const root of roots) { - const walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT); - while ((node = walker.nextNode())) { - const s = getComputedStyle(node); - if (s.display === 'none' || s.visibility === 'hidden') continue; - const r = node.getBoundingClientRect(); - if (r.width === 0 || r.height === 0) continue; - - const nodeText = (node.innerText || node.textContent || '').trim(); - const firstLine = nodeText.split('\n')[0].trim(); - const match = _exact - ? firstLine === _text - : firstLine.toLowerCase().includes(textLower); - - if (match) { - candidates.push({ el: node, text: firstLine, depth: getDepth(node) }); + const tempRef = `t${Date.now().toString(36)}${Math.random().toString(36).slice(2, 5)}`; + + // Page side: find the element by accessible name / composed text (shadow + // roots + same-origin frames), climb to the control that owns it, and park + // it in the ref registry so the click itself goes through the normal path. + const found = await execDom(tabId, (_text, _index, _exact, _ref) => { + const D = globalThis.__bcDom; + if (!D) return { __needDom: true }; + const want = D.clean(_text).toLowerCase(); + if (!want) return { success: false, error: 'text is required' }; + const SKIP = new Set(['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEMPLATE', 'HEAD', 'HTML', 'BODY', 'META', 'LINK']); + const hits = []; + const seen = new Set(); + const matches = (s) => { + const t = D.clean(s).toLowerCase(); + if (!t) return false; + return _exact ? t === want : t.includes(want); + }; + for (const root of D.allRoots(true)) { + let els = []; + try { els = root.querySelectorAll('*'); } catch {} + for (const el of els) { + if (SKIP.has(el.tagName)) continue; + const own = D.isInteractive(el) ? D.nameOf(el) : D.composedText(el, 200); + // aria-label / title / value also count as the element's text. + if (!matches(own) && !matches(D.attr(el, 'aria-label')) && !matches(D.attr(el, 'title')) + && !(el.tagName === 'INPUT' && matches(el.value))) continue; + // Climb to the control that owns this text (MUI: inside