diff --git a/chrome-extension/background.js b/chrome-extension/background.js index 532a10f..4a1063e 100644 --- a/chrome-extension/background.js +++ b/chrome-extension/background.js @@ -277,6 +277,10 @@ function sendHello(sock, id) { type: "hello", installId: id || "", browser: "chrome", + // The worker's own UA (never read from a page). Playwright derives the + // platform from Browser.getVersion and needs it to pick Meta vs Control + // for editing shortcuts such as select-all. + userAgent: navigator.userAgent, version: manifest.version, browserwrightVersion: manifest.version, extensionProtocolVersion: BROWSERWRIGHT_EXTENSION_PROTOCOL_VERSION, diff --git a/docs/human-input.md b/docs/human-input.md new file mode 100644 index 0000000..a4181aa --- /dev/null +++ b/docs/human-input.md @@ -0,0 +1,161 @@ +# Agent input defaults + +The resident executor installs `repl/human_input.py` in +`_Worker._bind_live_surface`, before handing its real Playwright `page` and +`context` to agent scripts or helper modules. Chained locators, `Page`/`Frame` +selector methods (`page.click(sel)`, `frame.fill(sel, v)`), child frames, and +pages created by `context.new_page()` share the defaults. Playwright objects +stay real Playwright objects and return values are unchanged. + +The defaults apply on both backends. The facade-side pieces below exist only +on the extension backend; the cdp backend keeps Chrome's native input path. + +## No focus stealing + +Nothing in this module activates a tab, raises a window or calls +`page.bring_to_front()`. The extension backend drives the user's own Chrome, +and browser UI activation steals the desktop's active window from them. +Through `chrome.debugger`, `Page.bringToFront` really does activate the tab +and focus its window, so the rule is enforced by not issuing it. + +Every step works on a tab that is not the active tab, in a window that is not +focused. The extension already keeps attached background tabs rendering +(`keepTabRendered()`: focus emulation plus an active web lifecycle; see +`test_l2_background_render.py`). Pointer moves, coalesced samples, wheel +scrolling, keyboard input and the post-wheel geometry re-reads therefore run +the same in the background as in the foreground. Nothing is degraded. +`test_human_input_no_focus.py` enforces this. It moves the user to another +active tab and a newly focused window, then runs click, fill and +scroll-into-view on the background session tab. It asserts that the input was +trusted, that every window's active tab and the focused window are unchanged, +and that `tabs.onActivated` / `windows.onFocusChanged` never fired for the +session tab or its window. + +An agent that explicitly calls `page.bring_to_front()` still gets native +activation. That is the agent's choice, never a default. + +## click() + +`click()` approaches a random interior point of the target, away from dead +center, through densely sampled mouse moves. It starts from the last tracked +pointer position, or from a viewport edge on first use. It then calls the +native Playwright click with a short randomized press duration (35–95 ms). +Native click still owns actionability, navigation waiting, modifiers, +button, click count, `force`, `trial` and explicit `position`/`delay`. +Preparation and the final action share the caller's timeout. + +Samples are sent in small groups without waiting for each ACK, so Chrome can +coalesce them like a real device (`PointerEvent.getCoalescedEvents()`). The +extension facade forwards a run of consecutive `mouseMoved` commands +concurrently. It finishes the run before it processes a press or any other +command, and browser-session setup stays sequential. Moves stay sequential +while a mouse button is held, which preserves Playwright's drag handling. + +Generated coordinates follow the device-pixel grid. For generated click +positions, the extension facade leases dispatch-time rounding +(`Browserwright.setPointerGrid`, a facade-only CDP method). This covers a +target that moves between the approach and Playwright's final actionability +pass. The lease belongs to the CDP session and tab. It is released after the +action, including on failure, and dropped when that session detaches or the +tab goes away. Explicit caller positions keep native coordinates. On the cdp +backend the method does not exist and the click keeps native dispatch. + +## Scrolling into view + +Before an ordinary click or text fill, a target outside its clipping area is +revealed with native wheel input instead of programmatic `scrollIntoView`. +Geometry comes from a per-context Playwright selector engine +(`_browserwright_scroll`) that runs in Playwright's utility world. It answers +through a detached transport node that never enters the document. Document +scrolling, nested overflow containers and enclosing frames are handled, and +geometry is re-read after each wheel burst. A page that cancels wheel input +uses up the caller's timeout, as it would for a real user. + +## fill() + +For a visible editable text control (text-like ``, ` +
old rich text
+
+
Existing image
+ + + + + + + + +""" + + +@pytest.fixture +def human_input_site(): + # getCoalescedEvents is restricted to secure contexts. A loopback origin + # is trustworthy; about:blank set_content cannot exercise this API. + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + body = _HTML.encode() + self.send_response(200) + self.send_header('Content-Type', 'text/html; charset=utf-8') + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + server = ThreadingHTTPServer(('127.0.0.1', 0), Handler) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + try: + yield f'http://127.0.0.1:{server.server_port}' + finally: + server.shutdown() + server.server_close() + worker.join(timeout=2) + + +def _assert_typed(record: dict, expected: str) -> None: + """Verify actual event order and final replacement, not wrapper internals.""" + assert record["value"] == expected + assert record["return_none"] is True + events = record["events"] + # Clearing a preexisting value can emit Delete/Backspace before typing. + starts = [i for i, event in enumerate(events) + if event["type"] == "keydown" and len(event["key"] or "") == 1 + and not (event.get('ctrl') or event.get('meta'))] + assert len(starts) == len(expected), events + for index, character in zip(starts, expected): + char_events = events[index:index + 4] + assert [event["type"] for event in char_events] == [ + "keydown", "keypress", "input", "keyup", + ], char_events + assert [event["key"] for event in char_events if event["key"]] == [ + character, character, character, + ], char_events + assert char_events[2]["data"] == character + assert all(event["trusted"] for event in char_events) + if "shift" in char_events[0] and character.isascii() and character.isupper(): + assert char_events[0]["shift"] is True, char_events + assert any(event["type"] == "keydown" and event["key"] == "Shift" + for event in events[:index]), events + + +@pytest.mark.parametrize("extension_only_chrome", ["headless"], indirect=True) +def test_executor_human_input_defaults_and_playwright_semantics( + e2e_daemon, e2e_chrome, ext_ready, e2e_artifacts_dir, human_input_site, +): + """Agent-created locators/pages/frames use normal browser input events.""" + pytest.importorskip("playwright.sync_api") + assert not (e2e_chrome.profile_path / 'DevToolsActivePort').exists() + sid = f"e2e-input-{uuid.uuid4().hex}" + seed_ledger_session(bs_home("extension"), sid, backend="extension", name="input") + artifact = e2e_artifacts_dir / "human-input.json" + artifact.unlink(missing_ok=True) + script = "import json\n" + f"page.goto({human_input_site!r}, wait_until='load')\n" + script += "from pathlib import Path\n" + f"artifact = Path({str(artifact)!r})\n" + script += textwrap.dedent(""" + from playwright.sync_api import Error, TimeoutError + result = {} + result['browser_version'] = {'reported': context.browser.version, + 'actual_user_agent': page.evaluate('navigator.userAgent')} + def take(): + return page.evaluate('events.splice(0)') + field = page.get_by_role('region', name='Form').get_by_role('textbox', name='Name') + returned = field.fill('Alice') + result['locator_fill'] = {'value': field.input_value(), 'return_none': returned is None, + 'events': take()} + returned = field.fill('A!b?') + result['shift_fill'] = {'value': field.input_value(), 'return_none': returned is None, + 'events': take()} + empty = page.get_by_role('textbox', name='Empty', exact=True) + returned = empty.fill('Eve') + result['initially_empty_fill'] = {'value': empty.input_value(), + 'return_none': returned is None, 'events': take()} + notes = page.get_by_role('textbox', name='Notes') + returned = notes.fill('New notes', force=True) + result['textarea_fill'] = {'value': notes.input_value(), + 'return_none': returned is None, 'events': take()} + + returned = page.get_by_role('button', name='Submit').click() + result['click'] = {'return_none': returned is None, 'events': take()} + metrics_session = context.new_cdp_session(page) + metrics_session.send('Emulation.setDeviceMetricsOverride', { + 'width': 1280, 'height': 900, 'deviceScaleFactor': 1.75, 'mobile': False}) + page.get_by_role('button', name='Submit').click() + result['scaled_click'] = {'events': take()} + metrics_session.send('Emulation.clearDeviceMetricsOverride') + metrics_session.detach() + returned = page.click('#submit', position={'x': 9, 'y': 11}, force=True) + result['position_click'] = {'return_none': returned is None, 'events': take()} + + returned = field.fill('native', human=False) + result['native_fill'] = {'value': field.input_value(), 'return_none': returned is None, + 'events': take()} + page.click('#submit', human=False) + result['native_click'] = {'events': take()} + + rich = page.get_by_role('textbox', name='Rich text') + returned = rich.fill('Rich') + result['rich_fill'] = {'value': rich.inner_text(), 'return_none': returned is None, + 'events': take()} + empty_rich = page.get_by_role('textbox', name='Empty editor') + returned = empty_rich.fill('Empty rich') + result['empty_editor_fill'] = {'value': empty_rich.inner_text(), + 'return_none': returned is None, 'events': take()} + media = page.get_by_role('textbox', name='Rich media') + returned = media.fill('Media') + result['media_fill'] = {'value': media.inner_text(), 'return_none': returned is None, + 'children': media.locator('img').count(), 'events': take()} + returned = page.fill('#name', 'Page') + result['page_fill'] = {'value': field.input_value(), 'return_none': returned is None, + 'events': take()} + returned = field.fill('') + result['empty_fill'] = {'value': field.input_value(), 'return_none': returned is None, + 'events': take()} + # A forced click can land on an overlay instead of the intended input. + # Replacement still belongs to the requested field, and must not select + # or erase another editor that was focused before the call. + empty.click(human=False) + page.evaluate('''() => { + const r=document.querySelector('#name').getBoundingClientRect(); + const overlay=document.createElement('div');overlay.id='overlay'; + Object.assign(overlay.style,{position:'fixed',left:r.left+'px',top:r.top+'px', + width:r.width+'px',height:r.height+'px',zIndex:'1000'}); + document.body.append(overlay); + }''') + take() + returned = field.fill('Forced', force=True) + result['forced_overlay_fill'] = {'value': field.input_value(), + 'other_value': empty.input_value(), + 'return_none': returned is None, 'events': take()} + page.evaluate("document.querySelector('#overlay').remove()") + + for selector in ['#readonly', '#disabled']: + try: + page.locator(selector).fill('x', timeout=180) + result[selector] = {'error': None} + except TimeoutError as exc: + result[selector] = {'error': type(exc).__name__, 'message': str(exc)} + try: + page.locator('#select').fill('bad', timeout=180) + result['select'] = {'error': None} + except Error as exc: + result['select'] = {'error': type(exc).__name__, 'message': str(exc)} + for selector, value in [('#number', '42'), ('#date', '2026-10-07')]: + returned = page.locator(selector).fill(value) + result[selector] = {'value': page.locator(selector).input_value(), + 'return_none': returned is None, 'events': take()} + try: + page.locator('#number').fill('not a number') + result['invalid_number'] = {'error': None} + except Error as exc: + result['invalid_number'] = {'error': type(exc).__name__, 'message': str(exc)} + # Keep observations available if later page/frame setup fails. + artifact.write_text(json.dumps(result, indent=2) + '\\n') + + # A held button enters Playwright's native drag handling. Preserve its + # state throughout the approach and let native click release normally. + page.mouse.move(20, 20) + page.mouse.down() + take() + returned = page.get_by_role('button', name='Submit').click(force=True) + result['held_button_click'] = {'return_none': returned is None, 'events': take()} + page.mouse.up() + take() + + other = context.new_page() + other.set_content('') + other.evaluate('''() => { + window.events = []; + for (const type of ['keydown', 'keypress', 'input', 'keyup']) + document.addEventListener(type, e => events.push({type, key:e.key || null, + data:e.data ?? null, trusted:e.isTrusted, + ctrl:e.ctrlKey, meta:e.metaKey})); + }''') + returned = other.get_by_role('textbox', name='Other').fill('New') + result['new_page_fill'] = {'value': other.locator('#other').input_value(), + 'return_none': returned is None, + 'events': other.evaluate('events')} + frame = other.main_frame + frame.evaluate('events.splice(0)') + returned = frame.fill('#other', 'Frame') + result['frame_fill'] = {'value': frame.locator('#other').input_value(), + 'return_none': returned is None, + 'events': frame.evaluate('events')} + other.close() + artifact.write_text(json.dumps(result, indent=2) + '\\n') + print('HUMAN_INPUT=' + ','.join(result)) + """) + try: + execution = run_skill(script, backend="extension", runtime_dir=e2e_daemon.runtime_dir, + extra_env={"BD_SESSION": sid}, timeout=90) + output = {"returncode": execution.returncode, "stdout": execution.stdout, + "stderr": execution.stderr} + if artifact.exists(): + output["observations"] = json.loads(artifact.read_text()) + artifact.write_text(json.dumps(output, indent=2) + "\n") + assert execution.returncode == 0, output + observations = output["observations"] + assert 'Chrome/' + observations['browser_version']['reported'] in \ + observations['browser_version']['actual_user_agent'] + _assert_typed(observations['initially_empty_fill'], 'Eve') + assert not any(e['key'] in ('Delete', 'Backspace') + for e in observations['initially_empty_fill']['events']) + assert not any(e.get('ctrl') or e.get('meta') + for e in observations['initially_empty_fill']['events']) + _assert_typed(observations['empty_editor_fill'], 'Empty rich') + assert not any(e['key'] in ('Delete', 'Backspace') or e.get('ctrl') or e.get('meta') + for e in observations['empty_editor_fill']['events']) + # Replacements must select through an actual trusted keyboard shortcut, + # with a full selection visible before deletion. This also covers a + # multiline textarea and contenteditable containing only an image. + for name, old in [('locator_fill', 'replace me'), ('shift_fill', 'Alice'), + ('textarea_fill', 'old multiline\nnotes'), + ('rich_fill', 'old rich text'), ('media_fill', None), + ('page_fill', 'native'), ('empty_fill', 'Page')]: + events = observations[name]['events'] + shortcut = next(e for e in events if e['type'] == 'keyup' + and (e['key'] or '').lower() == 'a' + and (e['ctrl'] or e['meta'])) + assert shortcut['trusted'], shortcut + assert any(e['type'] == 'keydown' and e['key'] in ('Control', 'Meta') + and e['trusted'] for e in events[:events.index(shortcut)]), events + selection = shortcut['selection'] + if 'start' in selection: + assert selection == {'start': 0, 'end': len(old)}, shortcut + else: + assert selection['collapsed'] is False, shortcut + if old is not None: + assert selection['text'] == old, shortcut + assert any(e['type'] == 'keyup' and e['key'] in ('Control', 'Meta') + for e in events[events.index(shortcut) + 1:]), events + for name in ('locator_fill', 'shift_fill', 'initially_empty_fill'): + pressed = {} + shift_down = None + for event in observations[name]['events']: + if event['type'] == 'keydown': + pressed[event['key']] = event['time'] + if event['key'] == 'Shift': + shift_down = event['time'] + elif event['shift'] and shift_down is not None: + assert event['time'] - shift_down >= 15, event + elif event['type'] == 'keyup' and event['key'] in pressed: + assert event['time'] - pressed.pop(event['key']) >= 20, event + first_moves = [e for e in observations['locator_fill']['events'] + if e['type'] == 'pointermove'] + assert first_moves and (first_moves[0]['x'] > 5 or first_moves[0]['y'] > 5), first_moves + for name in ('locator_fill', 'shift_fill', 'initially_empty_fill', 'click', 'scaled_click'): + for event in observations[name]['events']: + if event['type'] == 'pointermove': + assert abs(event['x'] * event['dpr'] - round(event['x'] * event['dpr'])) < .01, event + assert abs(event['y'] * event['dpr'] - round(event['y'] * event['dpr'])) < .01, event + for sample in event['coalesced'] or []: + assert abs(sample['x'] * event['dpr'] - round(sample['x'] * event['dpr'])) < .01, event + assert abs(sample['y'] * event['dpr'] - round(sample['y'] * event['dpr'])) < .01, event + focus = next(event for event in observations["locator_fill"]["events"] + if event["type"] == "focus" and event["target"] == "name") + assert focus["activated"] is True, focus + assert any(event["type"] == "mousedown" and event["target"] == "name" + for event in observations["locator_fill"]["events"]), observations["locator_fill"] + for key, value in [("locator_fill", "Alice"), ("rich_fill", "Rich"), + ("textarea_fill", "New notes"), + ("page_fill", "Page"), ("new_page_fill", "New"), + ("frame_fill", "Frame")]: + _assert_typed(observations[key], value) + _assert_typed(observations["shift_fill"], "A!b?") + assert all(event["shift"] for event in observations["shift_fill"]["events"] + if event["type"] == "keydown" and event["key"] in ("A", "!", "?")) + _assert_typed(observations['media_fill'], 'Media') + assert observations['media_fill']['children'] == 0 + assert observations["empty_fill"]["value"] == "" + assert observations["empty_fill"]["return_none"] is True + forced = observations['forced_overlay_fill'] + assert forced['value'] == 'Forced' and forced['other_value'] == 'Eve', forced + assert forced['return_none'] is True + assert not any(e.get('ctrl') or e.get('meta') for e in forced['events']), forced + assert observations["native_fill"]["value"] == "native" + assert observations["native_fill"]["return_none"] is True + assert not any(event["type"] in ("keydown", "keypress", "keyup") + for event in observations["native_fill"]["events"]) + assert observations["click"]["return_none"] is True + held = observations['held_button_click'] + assert held['return_none'] is True + assert any(e['type'] == 'mousemove' and e['buttons'] == 1 for e in held['events']), held + assert any(e['type'] == 'click' and e['target'] == 'submit' for e in held['events']), held + assert any(e['type'] == 'mouseup' and e['buttons'] == 0 for e in held['events']), held + pointer = observations["click"]["events"] + coalesced = [event for event in pointer if event['type'] == 'pointermove' + and len(event['coalesced'] or []) > 1] + assert coalesced, pointer + for event in coalesced: + assert all(sample['trusted'] for sample in event['coalesced']), event + assert all(a['time'] <= b['time'] for a, b in + zip(event['coalesced'], event['coalesced'][1:])), event + assert sum(event["type"] == "mousemove" for event in pointer) >= 3, pointer + visible_moves = [event for event in pointer if event['type'] == 'mousemove'] + assert len(visible_moves) >= 15, visible_moves + assert all(((b['x'] - a['x']) ** 2 + (b['y'] - a['y']) ** 2) ** .5 <= 15 + for a, b in zip(visible_moves, visible_moves[1:])), visible_moves + # Chrome may deliver several physical samples in one pointermove. + # Check path density using those original samples, rather than requiring + # the browser to dispatch every sample as a separate DOM event. + moves = [sample for event in pointer if event['type'] == 'pointermove' + for sample in (event['coalesced'] or [event])] + assert len(moves) >= 15, moves + assert all(((b["x"] - a["x"]) ** 2 + (b["y"] - a["y"]) ** 2) ** .5 <= 15 + for a, b in zip(moves, moves[1:])), moves + click = next(event for event in pointer if event["type"] == "click") + rect = click["rect"] + assert rect["x"] < click["x"] < rect["x"] + rect["width"] + assert rect["y"] < click["y"] < rect["y"] + rect["height"] + assert (abs(click["x"] - rect["x"] - rect["width"] / 2) > 0.5 or + abs(click["y"] - rect["y"] - rect["height"] / 2) > 0.5), click + down = next(event for event in pointer if event["type"] == "mousedown") + up = next(event for event in pointer if event["type"] == "mouseup") + assert up["time"] - down["time"] >= 5, pointer + positioned = next(event for event in observations["position_click"]["events"] + if event["type"] == "click") + assert abs(positioned["x"] - positioned["rect"]["x"] - + positioned["rect"]["borderX"] - 9) <= 1 + assert abs(positioned["y"] - positioned["rect"]["y"] - + positioned["rect"]["borderY"] - 11) <= 1 + native = next(event for event in observations["native_click"]["events"] + if event["type"] == "click") + assert abs(native["x"] - native["rect"]["x"] - native["rect"]["width"] / 2) <= 1 + assert abs(native["y"] - native["rect"]["y"] - native["rect"]["height"] / 2) <= 1 + for selector in ("#readonly", "#disabled"): + assert observations[selector]["error"] == "TimeoutError", observations[selector] + assert observations["select"]["error"] == "Error" + assert observations["invalid_number"]["error"] == "Error" + assert observations["#number"]["value"] == "42" + assert observations["#date"]["value"] == "2026-10-07" + finally: + drop_ledger_session(bs_home("extension"), sid) diff --git a/tests/daemon/e2e/test_human_input_no_focus.py b/tests/daemon/e2e/test_human_input_no_focus.py new file mode 100644 index 0000000..a9c4ac7 --- /dev/null +++ b/tests/daemon/e2e/test_human_input_no_focus.py @@ -0,0 +1,252 @@ +"""Human input on a background tab must never activate a tab or focus a window. + +The extension backend drives the user's own Chrome. Activating the session's +tab or raising its window steals the desktop's active window from the user. +This test puts the user somewhere else first — a newly focused window, plus a +newer active tab in the session tab's own window — then runs executor +click + fill + scroll-into-view on the (now background) session tab. + +It asserts (a) the page saw the actions as trusted input and they took effect, +and (b) Chrome's own tab/window state is unchanged: the same active tab in +every window, the same focused window, and no `tabs.onActivated` / +`windows.onFocusChanged` event for the session tab or its window at any point +during the actions (so a transient activate-then-restore is caught too). +Chrome state is read from the extension's service worker over the harness's +CDP port, independently of the daemon. + +Repeat with ``bash tests/daemon/e2e/run.sh +tests/daemon/e2e/test_human_input_no_focus.py -v``. The artifact is +``tests/daemon/e2e/_artifacts/human-input-no-focus.json``. Headful by default +(the meaningful case); ``BW_E2E_HEADLESS=1`` still exercises the contract. +""" +from __future__ import annotations + +import json +import textwrap +import threading +import time +import uuid +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import pytest + +from browserwright.cdp import CDPSession + +from .helpers import bs_home, drop_ledger_session, run_skill, seed_ledger_session +from .test_l2_multisession import _extension_id_from_path, _extension_worker_target_id + +_SESSION_HTML = """Session page + + + +
+ +""" + +_USER_HTML = """User page +

The user is reading this tab.

""" + +_WATCH_JS = """(() => { + globalThis.__bwFocusLog = []; + const log = (kind) => (...args) => __bwFocusLog.push({kind, args, t: Date.now()}); + globalThis.__bwFocusListeners = [ + [chrome.tabs.onActivated, log('tabs.onActivated')], + [chrome.tabs.onHighlighted, log('tabs.onHighlighted')], + [chrome.windows.onFocusChanged, log('windows.onFocusChanged')], + ]; + for (const [event, fn] of __bwFocusListeners) event.addListener(fn); + return true; +})()""" + +_STATE_JS = """(async () => { + const windows = await chrome.windows.getAll({populate: true}); + let lastFocused = null; + try { lastFocused = (await chrome.windows.getLastFocused()).id; } catch (e) {} + return { + lastFocusedWindow: lastFocused, + windows: windows.map(w => ({id: w.id, focused: w.focused, + activeTab: (w.tabs.find(t => t.active) || {}).id ?? null, + tabs: w.tabs.map(t => ({id: t.id, active: t.active, url: t.url}))})), + }; +})()""" + + +class _Worker: + """Evaluate in the extension's service worker over the browser CDP port.""" + + def __init__(self, chrome, extension_id: str): + self._cdp = CDPSession(chrome.ws_url) + self._sid = self._cdp.attach(_extension_worker_target_id(self._cdp, extension_id)) + + def eval(self, expression: str): + result = self._cdp.send("Runtime.evaluate", session=self._sid, + expression=expression, returnByValue=True, + awaitPromise=True) + if "exceptionDetails" in result: + raise AssertionError(f"service-worker evaluation failed: {result!r}") + return result.get("result", {}).get("value") + + def close(self) -> None: + self._cdp.close() + + +@pytest.fixture +def pages_site(): + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + body = (_USER_HTML if self.path.startswith("/user") else _SESSION_HTML).encode() + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + try: + yield f"http://127.0.0.1:{server.server_port}" + finally: + server.shutdown() + server.server_close() + worker.join(timeout=2) + + +def _wait_loaded(worker: _Worker, tab_id: int, timeout: float = 15.0) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if worker.eval(f"chrome.tabs.get({tab_id}).then(t => t.status)") == "complete": + return + time.sleep(0.1) + raise AssertionError(f"tab {tab_id} did not finish loading") + + +def test_background_tab_human_input_never_steals_focus( + e2e_daemon, e2e_chrome, ext_ready, patched_ext_dir, e2e_artifacts_dir, pages_site, +): + pytest.importorskip("playwright.sync_api") + artifact = e2e_artifacts_dir / "human-input-no-focus.json" + artifact.unlink(missing_ok=True) + token = uuid.uuid4().hex + session_url = f"{pages_site}/session?{token}" + sid = f"e2e-nofocus-{token}" + seed_ledger_session(bs_home("extension"), sid, backend="extension", name="nofocus") + env = {"BD_SESSION": sid} + worker = _Worker(e2e_chrome, _extension_id_from_path(patched_ext_dir)) + record: dict = {"session_url": session_url} + try: + # 1. Bind the session tab to the fixture page. + setup = run_skill(f"page.goto({session_url!r}, wait_until='load')\n", + backend="extension", runtime_dir=e2e_daemon.runtime_dir, + extra_env=env, timeout=60) + assert setup.returncode == 0, (setup.stdout, setup.stderr) + tabs = worker.eval(f"chrome.tabs.query({{}}).then(ts => ts.filter(" + f"t => (t.url || '').includes({token!r})).map(t => t.id))") + assert len(tabs) == 1, tabs + session_tab = tabs[0] + session_window = worker.eval(f"chrome.tabs.get({session_tab}).then(t => t.windowId)") + + # 2. The user moves on: a newer active tab in the session tab's own + # window, then a separate window that takes focus. + user_tab = worker.eval( + f"chrome.tabs.create({{windowId: {session_window}, " + f"url: {pages_site + '/user-tab'!r}, active: true}}).then(t => t.id)") + user_window = worker.eval( + f"chrome.windows.create({{url: {pages_site + '/user-window'!r}, " + f"focused: true}}).then(w => w.id)") + user_window_tab = worker.eval( + f"chrome.tabs.query({{windowId: {user_window}}}).then(ts => ts[0].id)") + for tab in (user_tab, user_window_tab): + _wait_loaded(worker, tab) + time.sleep(0.5) # let window-manager focus settle before the baseline + before = worker.eval(_STATE_JS) + record["before"] = before + session_win_state = next(w for w in before["windows"] if w["id"] == session_window) + assert session_win_state["activeTab"] == user_tab, before + assert not any(t["active"] for t in session_win_state["tabs"] + if t["id"] == session_tab), before + assert worker.eval(_WATCH_JS) is True + + # 3. Human input on the background session tab. + script = textwrap.dedent(""" + import json, time + out = {'visibility': page.evaluate('document.visibilityState')} + t0 = time.monotonic() + page.get_by_role('button', name='Near').click() + out['click_s'] = time.monotonic() - t0 + t0 = time.monotonic() + page.get_by_role('textbox', name='Name').fill('New Name!') + out['fill_s'] = time.monotonic() - t0 + t0 = time.monotonic() + page.get_by_role('button', name='Far').click() + out['scroll_click_s'] = time.monotonic() - t0 + out['value'] = page.locator('#name').input_value() + out['clicks'] = page.evaluate('clicks') + out['scrollY'] = page.evaluate('scrollY') + out['events'] = page.evaluate('events') + from pathlib import Path + Path(RESULT_PATH).write_text(json.dumps(out)) + """) + result_path = e2e_artifacts_dir / f"human-input-no-focus-{token}.tmp.json" + script = f"RESULT_PATH = {str(result_path)!r}\n" + script + run = run_skill(script, backend="extension", runtime_dir=e2e_daemon.runtime_dir, + extra_env=env, timeout=90) + record["run"] = {"returncode": run.returncode, "stdout": run.stdout, + "stderr": run.stderr} + focus_log = worker.eval("globalThis.__bwFocusLog ?? null") + after = worker.eval(_STATE_JS) + record.update(focus_log=focus_log, after=after) + assert run.returncode == 0, record["run"] + result = json.loads(result_path.read_text()) + result_path.unlink() + record["result"] = result + + # (a) The actions happened, through trusted browser input. + assert result["clicks"] == ["near", "far"], result["clicks"] + assert result["value"] == "New Name!", result["value"] + assert result["scrollY"] > 0, result["scrollY"] + events = result["events"] + for kind in ("pointermove", "mousedown", "click", "keydown", "input", "wheel"): + seen = [e for e in events if e["type"] == kind] + assert seen and all(e["trusted"] for e in seen), (kind, seen) + typed = "".join(e["key"] for e in events + if e["type"] == "keydown" and len(e["key"] or "") == 1) + assert typed.endswith("New Name!"), typed + + # (b) Chrome's focus state is exactly what the user left it as. + assert focus_log is not None, "service worker restarted; focus log lost" + + def activations(state): + return {w["id"]: w["activeTab"] for w in state["windows"]} + + assert activations(after) == activations(before), (before, after) + assert after["lastFocusedWindow"] == before["lastFocusedWindow"], (before, after) + assert ([w["focused"] for w in after["windows"]] + == [w["focused"] for w in before["windows"]]), (before, after) + assert not any(e["kind"] == "tabs.onActivated" + and e["args"][0].get("tabId") == session_tab + for e in focus_log), focus_log + assert not any(e["kind"] == "windows.onFocusChanged" + and e["args"][0] == session_window + for e in focus_log), focus_log + assert not any(e["kind"] == "tabs.onHighlighted" + and session_tab in (e["args"][0].get("tabIds") or []) + for e in focus_log), focus_log + finally: + artifact.write_text(json.dumps(record, indent=2) + "\n") + try: + worker.eval("""(() => { for (const [e, f] of globalThis.__bwFocusListeners || []) + e.removeListener(f); return true; })()""") + finally: + worker.close() + drop_ledger_session(bs_home("extension"), sid) diff --git a/tests/daemon/e2e/test_human_pointer_layout.py b/tests/daemon/e2e/test_human_pointer_layout.py new file mode 100644 index 0000000..a7ff368 --- /dev/null +++ b/tests/daemon/e2e/test_human_pointer_layout.py @@ -0,0 +1,114 @@ +"""Native pointer coordinates after target geometry changes during an approach. + +Repeat with ``mise exec -- uv run pytest tests/daemon/e2e/test_human_pointer_layout.py -q``. +The artifact records trusted browser events and the final target geometry. +""" +from __future__ import annotations + +import json +import textwrap + +import pytest + +from .helpers import run_skill + + +@pytest.fixture +def e2e_chrome(extension_only_chrome): + return extension_only_chrome + + +@pytest.mark.parametrize('extension_only_chrome', ['headless'], indirect=True) +def test_click_after_layout_shift_stays_on_physical_pixel_grid( + e2e_daemon, e2e_chrome, ext_ready, e2e_artifacts_dir, +): + artifact = e2e_artifacts_dir / 'human-pointer-layout.json' + artifact.unlink(missing_ok=True) + script = 'import json\nfrom pathlib import Path\n' + script += f'artifact=Path({str(artifact)!r})\n' + script += textwrap.dedent(''' + result={} + metrics=context.new_cdp_session(page) + for ratio in [2,1.75]: + page.goto('about:blank') + metrics.send('Emulation.setDeviceMetricsOverride', { + 'width':1000,'height':700,'deviceScaleFactor':ratio,'mobile':False}) + page.set_content(""" + """) + returned=page.get_by_role('button',name='Moving').click() + result[str(ratio)]={'return_none':returned is None, + 'observation':page.evaluate("""() => ({clicked,shifted,events, + box:document.querySelector('button').getBoundingClientRect().toJSON()})""")} + # The framework lease must end with the generated click. Explicit + # padding-relative positions and human=False retain native CDP + # coordinates, even when those are between physical pixels. + result[str(ratio)]['explicit']=[] + for native in [False,True]: + page.evaluate('events=[]') + page.get_by_role('button',name='Moving').click( + position={'x':20.123,'y':20.123},human=not native) + result[str(ratio)]['explicit'].append(page.evaluate("""() => ({events, + box:document.querySelector('button').getBoundingClientRect().toJSON(), + border:parseFloat(getComputedStyle(document.querySelector('button')).borderLeftWidth)})""")) + artifact.write_text(json.dumps(result,indent=2)) + metrics.send('Emulation.clearDeviceMetricsOverride') + # Detaching a CDP handle must retire its temporary input state without + # affecting the live page or native calls made through another handle. + lease=context.new_cdp_session(page) + lease.send('Browserwright.setPointerGrid', {'devicePixelRatio':2}) + lease.detach() + page.evaluate('events=[]') + page.get_by_role('button',name='Moving').click( + position={'x':20.123,'y':20.123},human=False) + result['detached_lease']=page.evaluate("""() => ({events, + box:document.querySelector('button').getBoundingClientRect().toJSON(), + border:parseFloat(getComputedStyle(document.querySelector('button')).borderLeftWidth)})""") + artifact.write_text(json.dumps(result,indent=2)) + metrics.detach() + ''') + execution = run_skill(script, backend='extension', runtime_dir=e2e_daemon.runtime_dir, + timeout=40) + output = {'returncode':execution.returncode,'stdout':execution.stdout,'stderr':execution.stderr} + if artifact.exists(): + output['observations'] = json.loads(artifact.read_text()) + artifact.write_text(json.dumps(output,indent=2)+'\n') + assert execution.returncode == 0, output + detached = output['observations'].pop('detached_lease') + press = next(event for event in detached['events'] if event['type']=='pointerdown') + assert press['trusted'] and press['target']=='target', detached + for axis in ('x','y'): + expected = int((detached['box'][axis]+detached['border']+20.123)*100)/100 + assert abs(press[axis]-expected) < .001, detached + for result in output['observations'].values(): + observed = result['observation'] + assert result['return_none'] and observed['clicked'] and observed['shifted'], observed + box = observed['box'] + assert box['x'] == 360.625, box + pointer = [event for event in observed['events'] if event['type'].startswith('pointer')] + assert any(event['type']=='pointerdown' and event['target']=='target' for event in pointer) + for event in pointer: + assert event['trusted'], event + for sample in [event, *(event['coalesced'] or [])]: + assert sample['trusted'], sample + for axis in ('x','y'): + scaled = sample[axis]*event['dpr'] + assert abs(scaled-round(scaled)) < .01, event + for explicit in result['explicit']: + press = next(event for event in explicit['events'] if event['type']=='pointerdown') + assert press['trusted'] and press['target']=='target', explicit + for axis in ('x','y'): + expected = int((explicit['box'][axis]+explicit['border']+20.123)*100)/100 + assert abs(press[axis]-expected) < .001, explicit diff --git a/tests/daemon/e2e/test_human_scroll.py b/tests/daemon/e2e/test_human_scroll.py new file mode 100644 index 0000000..edfdf41 --- /dev/null +++ b/tests/daemon/e2e/test_human_scroll.py @@ -0,0 +1,137 @@ +"""Trusted wheel preparation through the extension-only executor. + +Repeat with ``mise exec -- uv run pytest tests/daemon/e2e/test_human_scroll.py -q``. +The artifact records real wheel/scroll order for document, nested and frame input. +""" +from __future__ import annotations + +import json +import textwrap +import pytest + +from .helpers import run_skill + + +@pytest.fixture +def e2e_chrome(extension_only_chrome): + return extension_only_chrome + + +_HTML = """
+
+
+
+
+
+""" + + +@pytest.mark.parametrize('extension_only_chrome', ['headless'], indirect=True) +def test_scroll_target_at_document_bottom(e2e_daemon, e2e_chrome, ext_ready, e2e_artifacts_dir): + artifact = e2e_artifacts_dir / 'human-scroll-bottom.json' + script = textwrap.dedent(''' + import json + page.set_content(""" +
+ """) + returned=page.get_by_role('button',name='Edge').click(timeout=8000) + result={'return_none':returned is None,'clicked':page.evaluate('window.clicked'), + 'events':page.evaluate('events')} + ''') + script += f'\nfrom pathlib import Path\nPath({str(artifact)!r}).write_text(json.dumps(result,indent=2))\n' + execution = run_skill(script, backend='extension', runtime_dir=e2e_daemon.runtime_dir,timeout=20) + assert execution.returncode == 0, (execution.stdout,execution.stderr) + result = json.loads(artifact.read_text()) + assert result['return_none'] and result['clicked'],result + assert any(e['type']=='wheel' and e['trusted'] for e in result['events']),result + + +@pytest.mark.parametrize('extension_only_chrome', ['headless'], indirect=True) +def test_executor_scroll_has_trusted_wheel_input( + e2e_daemon, e2e_chrome, ext_ready, e2e_artifacts_dir, +): + artifact = e2e_artifacts_dir / 'human-scroll.json' + artifact.unlink(missing_ok=True) + script = 'import json\nfrom pathlib import Path\n' + script += f'artifact=Path({str(artifact)!r})\npage.set_content({_HTML!r})\n' + script += textwrap.dedent(''' + from playwright.sync_api import TimeoutError + result={} + def save(): artifact.write_text(json.dumps(result,indent=2)) + returned=page.get_by_role('button',name='Top').click() + result['visible']={'return_none':returned is None,'events':page.evaluate('events.splice(0)')} + save() + child_frame=page.frames[1] + child_frame.evaluate("""() => { + window.events=[]; window.clicked=false; + for(const type of ['wheel','scroll','click']) document.addEventListener(type,e=>{ + events.push({type,trusted:e.isTrusted,time:performance.now()}); + if(type==='click' && e.target.id==='child') clicked=true; + },true); + }""") + returned=page.get_by_role('textbox',name='Below').fill('Ada') + result['document']={'return_none':returned is None,'value':page.locator('#below').input_value(), + 'events':page.evaluate('events.splice(0)')} + save() + returned=page.get_by_role('button',name='Nested').click(position={'x':12,'y':11},force=True) + result['nested']={'return_none':returned is None,'clicked':page.evaluate('clicked.slice()'), + 'events':page.evaluate('events.splice(0)')} + save() + child=page.frame_locator('#frame').get_by_role('button',name='Child') + returned=child.click() + result['frame']={'return_none':returned is None,'events':page.evaluate('events.splice(0)'), + 'child_events':child_frame.evaluate('events'), + 'child_clicked':child_frame.evaluate('clicked')} + save() + page.get_by_role('button',name='Top').click(human=False) + page.evaluate("events.splice(0); document.addEventListener('wheel',e=>e.preventDefault(),{passive:false})") + try: + page.get_by_role('textbox',name='Below').click(timeout=3000) + result['blocked']={'timeout':False} + except TimeoutError: + result['blocked']={'timeout':True,'events':page.evaluate('events.splice(0)')} + artifact.write_text(json.dumps(result,indent=2)) + ''') + execution = run_skill(script, backend='extension', runtime_dir=e2e_daemon.runtime_dir, + timeout=90) + output = {'returncode':execution.returncode,'stdout':execution.stdout,'stderr':execution.stderr} + if artifact.exists(): + output['observations'] = json.loads(artifact.read_text()) + artifact.write_text(json.dumps(output,indent=2)+'\n') + assert execution.returncode == 0, output + observations = output['observations'] + assert observations['document']['value'] == 'Ada' + assert 'nested' in observations['nested']['clicked'] + assert observations['visible']['return_none'] + assert not any(e['type'] in ['scroll','wheel'] for e in observations['visible']['events']) + assert observations['frame']['child_clicked'] + for name in ['document','nested','frame']: + events = observations[name]['events'] + assert observations[name]['return_none'], observations[name] + assert any(e['type']=='wheel' and e['trusted'] for e in events), events + for i, event in enumerate(events): + if event['type']=='scroll': + assert any(prior['type']=='wheel' and prior['trusted'] + and event['time']-prior['time']<200 for prior in events[:i]), events + child_events=observations['frame']['child_events'] + assert any(e['type']=='wheel' and e['trusted'] for e in child_events), child_events + for i,event in enumerate(child_events): + if event['type']=='scroll': + assert any(prior['type']=='wheel' and prior['trusted'] + and event['time']-prior['time']<200 for prior in child_events[:i]), child_events + assert observations['blocked']['timeout'], observations['blocked'] + assert any(e['type']=='wheel' and e['trusted'] for e in observations['blocked']['events']) + assert not any(e['type']=='scroll' for e in observations['blocked']['events'])