From f2b1865beb42773001bafc08919d203e056c0011 Mon Sep 17 00:00:00 2001 From: Randy Date: Wed, 7 Oct 2026 22:34:47 +0800 Subject: [PATCH 1/4] feat(extension): report the browser's real user agent via Browser.getVersion chrome.debugger cannot answer Browser.getVersion, so the extension backend returned a fixed "extension-relay" identity and Playwright treated every browser as Linux. ControlOrMeta then resolved to Control on macOS, where Control+A moves the caret instead of selecting. The service worker now sends its own navigator.userAgent in the relay hello (never read from a page); Browser.getVersion returns it with the product version parsed from it. Extensions that don't send the field keep the legacy identity. Co-Authored-By: Claude --- chrome-extension/background.js | 4 ++ .../daemon/server/extension_upstream.py | 45 +++++++++++-------- src/browserwright/daemon/server/relay.py | 7 +++ 3 files changed, 38 insertions(+), 18 deletions(-) diff --git a/chrome-extension/background.js b/chrome-extension/background.js index 532a10f..4a1063e 100644 --- a/chrome-extension/background.js +++ b/chrome-extension/background.js @@ -277,6 +277,10 @@ function sendHello(sock, id) { type: "hello", installId: id || "", browser: "chrome", + // The worker's own UA (never read from a page). Playwright derives the + // platform from Browser.getVersion and needs it to pick Meta vs Control + // for editing shortcuts such as select-all. + userAgent: navigator.userAgent, version: manifest.version, browserwrightVersion: manifest.version, extensionProtocolVersion: BROWSERWRIGHT_EXTENSION_PROTOCOL_VERSION, diff --git a/src/browserwright/daemon/server/extension_upstream.py b/src/browserwright/daemon/server/extension_upstream.py index e5035cd..95a6401 100644 --- a/src/browserwright/daemon/server/extension_upstream.py +++ b/src/browserwright/daemon/server/extension_upstream.py @@ -14,7 +14,7 @@ - `Target.setDiscoverTargets` / `Target.setAutoAttach` → silent ack (we don't need Chrome's discover stream — ghost targets come from the extension via "attached"/"detached" event types instead) - - `Browser.getVersion` → daemon-stamped result, used for heartbeat + - `Browser.getVersion` → extension-reported browser identity (heartbeat shape) - `Browser.crash`, `Browser.close` and other unsupported browser-level methods → -32601 ("method not implemented in extension backend") @@ -29,6 +29,7 @@ import json import logging import os +import re import secrets import time from typing import Any, Awaitable, Callable, TYPE_CHECKING @@ -1070,16 +1071,9 @@ async def send_text(self, frame: str) -> None: return if method == "Browser.getVersion": - # Heartbeat — daemon-internal. Return a stable shape so the - # proxy doesn't choke on the heartbeat loop in UpstreamConnection - # land (not used in extension backend, but symmetric). - await self._respond(req_id, { - "product": f"browserwright-daemon-extension/{__version__}", - "userAgent": "extension-relay", - "protocolVersion": "1.3", - "revision": "0", - "jsVersion": "0", - }) + # Same stable CDP shape heartbeat callers rely on, carrying the + # extension-reported browser identity when there is one. + await self._respond(req_id, self._browser_version()) return if isinstance(method, str) and method in _UNSUPPORTED_BROWSER_METHODS: @@ -1444,15 +1438,30 @@ async def send_command(self, method: str, params: dict | None = None, if method == "Target.setDiscoverTargets": return {} if method == "Browser.getVersion": - return { - "product": f"browserwright-daemon-extension/{__version__}", - "userAgent": "extension-relay", - "protocolVersion": "1.3", - "revision": "0", - "jsVersion": "0", - } + return self._browser_version() return {} + def _browser_version(self) -> dict: + """`Browser.getVersion`, which chrome.debugger cannot answer itself. + + Playwright derives the browser platform from ``userAgent`` + (``Macintosh`` → mac) and uses it to resolve ``ControlOrMeta`` and to + attach macOS editing commands to shortcuts such as Meta+A. The + extension's service worker reports its own user agent in the relay + hello; no page is read. Older extensions that don't report one keep + the legacy identity. + """ + user_agent = self._relay.browser_user_agent() + version = re.search(r"\b(?:Headless)?Chrome/[\d.]+", user_agent) + return { + "product": version.group(0) if version else + f"browserwright-daemon-extension/{__version__}", + "userAgent": user_agent or "extension-relay", + "protocolVersion": "1.3", + "revision": "0", + "jsVersion": "0", + } + # ---- helpers --------------------------------------------------------- # ---- relay lifecycle events (ADR-0013 inputs) ------------------------ diff --git a/src/browserwright/daemon/server/relay.py b/src/browserwright/daemon/server/relay.py index 89d590a..590993c 100644 --- a/src/browserwright/daemon/server/relay.py +++ b/src/browserwright/daemon/server/relay.py @@ -230,6 +230,7 @@ class _ExtensionConn: version: str = "" browserwright_version: str = "" extension_protocol_version: str = "" + user_agent: str = "" version_drift: str = VersionDrift.UNKNOWN.value # Where this extension came from, classified at ws handshake from the # Origin header: "store" (Chrome Web Store item id), "development" @@ -846,6 +847,11 @@ def status_payload(self) -> dict: "tab_count": sum(len(e.tabs) for e in self._extensions.values()), } + def browser_user_agent(self) -> str: + """Browser user agent reported by the active extension's service worker.""" + ext = self._pick_active_extension() + return ext.user_agent if ext is not None else "" + async def reload_extensions( self, *, @@ -1345,6 +1351,7 @@ async def _dispatch_from_extension(self, ext: _ExtensionConn, ext.connection_generation = self._connection_generation ext.install_id = str(msg.get("installId") or "") ext.browser = str(msg.get("browser") or "") + ext.user_agent = str(msg.get("userAgent") or "") ext.version = str(msg.get("version") or "") ext.browserwright_version = str(msg.get("browserwrightVersion") or ext.version) ext.extension_protocol_version = str( From 53b8674e99bae80584acf0a747463729ad76947a Mon Sep 17 00:00:00 2001 From: Randy Date: Wed, 7 Oct 2026 22:34:47 +0800 Subject: [PATCH 2/4] feat(facade): forward pointer moves concurrently and lease a pixel grid Two extension-facade pieces the executor's human input relies on: - Consecutive session-scoped Input.dispatchMouseEvent(mouseMoved) frames are forwarded without waiting for each ACK, so Chrome can coalesce the samples like a real pointer device. The run is drained before any other frame is handled, so presses, keys and stateful handshakes stay sequential. - Browserwright.setPointerGrid, a facade-only method, leases dispatch-time rounding of a tab's mouse coordinates to the device-pixel grid and returns the last dispatched point on release. The lease is owned by the CDP session that took it and dropped on its detach, tab eviction or bridge close. Co-Authored-By: Claude --- .../daemon/server/facade_extension.py | 121 +++++++++++++++++- 1 file changed, 120 insertions(+), 1 deletion(-) diff --git a/src/browserwright/daemon/server/facade_extension.py b/src/browserwright/daemon/server/facade_extension.py index 7c3bbc2..bf22f09 100644 --- a/src/browserwright/daemon/server/facade_extension.py +++ b/src/browserwright/daemon/server/facade_extension.py @@ -57,6 +57,7 @@ import contextlib import json import logging +import math import os import secrets from collections import deque @@ -208,6 +209,14 @@ def __init__( # debugger. Reusing a primary session crashes Playwright's CDP router. self._aux_sessions: dict[str, int] = {} self._closed = False + # In-flight pointer-move forwards (see `_queue_mouse_move`). + self._mouse_moves: set[asyncio.Task] = set() + # Device-pixel grid leases for the executor's generated clicks + # (`Browserwright.setPointerGrid`). Scale, owning CDP session and the + # last dispatched point live here, outside every page execution world. + self._pointer_grids: dict[int, float] = {} + self._pointer_grid_owners: dict[int, str] = {} + self._pointer_grid_points: dict[int, dict[str, float]] = {} # Guard concurrent synthesis (autoAttach replay vs fan-out attach) so we # never announce the same tab twice. self._lock = asyncio.Lock() @@ -305,10 +314,54 @@ async def run(self) -> None: continue text = raw if isinstance(raw, str) else raw.decode( "utf-8", errors="replace") + if self._queue_mouse_move(text): + continue + # Pointer-move ACKs arrive only after renderer dispatch. Let a + # run of consecutive samples reach Chrome together (so it can + # coalesce them like a real device), then finish the run + # before a press or any other protocol operation. + await self._drain_mouse_moves() await self._handle_client_frame(text) finally: + await self._drain_mouse_moves() await self.aclose() + def _queue_mouse_move(self, frame: str) -> bool: + """Forward a session-scoped `mouseMoved` without waiting for its ACK. + + Only plain pointer motion is parallelized; presses, keys and every + stateful handshake stay strictly sequential through + `_handle_client_frame`. Each response keeps its own id/sessionId. + """ + try: + msg = json.loads(frame) + except (ValueError, TypeError): + return False + if not isinstance(msg, dict) or msg.get("method") != "Input.dispatchMouseEvent": + return False + params = msg.get("params") + sid = msg.get("sessionId") + if (not isinstance(params, dict) or params.get("type") != "mouseMoved" + or not isinstance(sid, str) or self._tab_id_for_session(sid) is None): + return False + req_id = msg.get("id") if isinstance(msg.get("id"), int) else None + task = asyncio.create_task(self._forward_session_command( + req_id, sid, "Input.dispatchMouseEvent", params)) + self._mouse_moves.add(task) + task.add_done_callback(self._mouse_moves.discard) + return True + + async def _drain_mouse_moves(self) -> None: + if self._mouse_moves: + moves = self._mouse_moves.copy() + await asyncio.gather(*moves, return_exceptions=True) + self._mouse_moves.difference_update(moves) + + def _drop_pointer_grid(self, tab_id: int) -> None: + self._pointer_grids.pop(tab_id, None) + self._pointer_grid_owners.pop(tab_id, None) + self._pointer_grid_points.pop(tab_id, None) + async def aclose(self) -> None: if self._closed: return @@ -321,6 +374,9 @@ async def aclose(self) -> None: self._context_refresh_tasks.clear() self._context_refresh_pending.clear() self._frame_generations.clear() + self._pointer_grids.clear() + self._pointer_grid_owners.clear() + self._pointer_grid_points.clear() self._pending_console_text.clear() # NOTE: we deliberately do NOT call self._ext.close() — ExtensionUpstream # .close() calls relay.set_event_handler(None), which would clobber the @@ -463,7 +519,9 @@ async def _handle_client_frame(self, frame: str) -> None: if session_id is None: if method == "Target.detachFromTarget" and params.get("sessionId") in self._aux_sessions: detached_sid = params["sessionId"] - self._aux_sessions.pop(detached_sid) + detached_tab = self._aux_sessions.pop(detached_sid) + if self._pointer_grid_owners.get(detached_tab) == detached_sid: + self._drop_pointer_grid(detached_tab) await self._respond(req_id, {}) await self._send_to_client(json.dumps({ "method": "Target.detachedFromTarget", @@ -502,6 +560,41 @@ async def _handle_client_frame(self, frame: str) -> None: # it here because it talks raw flat-session CDP to Playwright. --- await self._forward_session_command(req_id, session_id, method, params) + async def _set_pointer_grid(self, req_id: int | None, session_id: str, + tab_id: int, params: dict) -> None: + """`Browserwright.setPointerGrid` — a facade-only method. + + ``{"devicePixelRatio": r}`` leases dispatch-time rounding of this tab's + `Input.dispatchMouseEvent` coordinates to the physical pixel grid; + ``{"devicePixelRatio": null}`` releases the lease and returns the last + dispatched point. Only the executor's generated clicks use it (see + `repl/human_input.py`): Playwright's final actionability pass may move + the target after the pointer approach, so the snap happens here, at + native dispatch. The lease belongs to the CDP session that took it; + detaching that session or losing the tab drops it. + """ + scale = params.get("devicePixelRatio") + if scale is None: + point = None + if self._pointer_grid_owners.get(tab_id) == session_id: + point = self._pointer_grid_points.get(tab_id) + self._drop_pointer_grid(tab_id) + await self._respond(req_id, {"point": point}, session_id=session_id) + return + value = 0.0 + if isinstance(scale, (int, float)) and not isinstance(scale, bool): + with contextlib.suppress(OverflowError, ValueError): + value = float(scale) + if not (math.isfinite(value) and value > 0): + await self._error(req_id, -32602, + "devicePixelRatio must be positive and finite", + session_id=session_id) + return + self._pointer_grids[tab_id] = value + self._pointer_grid_owners[tab_id] = session_id + self._pointer_grid_points.pop(tab_id, None) + await self._respond(req_id, {}, session_id=session_id) + def _command_wait_s(self) -> float | None: """How long a forwarded command may wait in the relay: derived from the deadline of the agent call currently running on this session @@ -521,6 +614,11 @@ async def _forward_session_command(self, req_id: int | None, f"unknown sessionId {session_id!r}", session_id=session_id) return + if method == "Browserwright.setPointerGrid": + await self._set_pointer_grid(req_id, session_id, tab_id, params) + return + if method == "Input.dispatchMouseEvent" and tab_id in self._pointer_grids: + params = _snap_to_grid(params, self._pointer_grids[tab_id]) # PR3: a command scoped to the synthetic main-frame id (which equals the # targetId we handed Playwright) must target the REAL Chrome frame id. self._rewrite_command_frame_id(tab_id, params) @@ -531,6 +629,9 @@ async def _forward_session_command(self, req_id: int | None, try: result = await self._relay.send_cdp( tab_id, method or "", params, timeout=self._command_wait_s()) + if method == "Input.dispatchMouseEvent" and tab_id in self._pointer_grids: + self._pointer_grid_points[tab_id] = { + axis: params[axis] for axis in ("x", "y") if axis in params} if method == "Runtime.enable": self._native_runtime_tabs.add(tab_id) elif method == "Runtime.disable": @@ -1431,6 +1532,7 @@ def _evict_tab(self, tab_id: int) -> None: """Drop all per-tab state for a closed/detached tab and wake any outstanding Runtime.enable barrier so it doesn't hang on a dead tab.""" self._tab_sessions.pop(tab_id, None) + self._drop_pointer_grid(tab_id) self._quiet_runtime_tabs.discard(tab_id) self._native_runtime_tabs.discard(tab_id) self._execution_contexts.pop(tab_id, None) @@ -1473,3 +1575,20 @@ async def _error(self, req_id: int | None, code: int, msg: str, async def _noop_close(self, reason: str) -> None: return + + +def _snap_to_grid(params: dict, scale: float) -> dict: + """Round a mouse event's x/y to the device-pixel grid of ``scale``. + + Malformed coordinates pass through untouched so Chrome reports its normal + invalid-parameter error. + """ + try: + if all(isinstance(params.get(axis), (int, float)) + and not isinstance(params[axis], bool) + and math.isfinite(params[axis] * scale) for axis in ("x", "y")): + return {**params, **{axis: round(params[axis] * scale) / scale + for axis in ("x", "y")}} + except (OverflowError, ValueError): + pass + return params From 8881f9c731f7d47e6077bbaa9ecc4798f4f6cdee Mon Sep 17 00:00:00 2001 From: Randy Date: Wed, 7 Oct 2026 22:34:47 +0800 Subject: [PATCH 3/4] feat(executor): simulated human input for click and fill, without focus stealing The resident executor installs repl/human_input.py on its live surface next to the screenshot defaults. Locator/Page/Frame click() approaches a random interior point through densely sampled, coalescable pointer moves and presses with a randomized duration; fill() focuses through that click, replaces existing text with a native ControlOrMeta+A chord and Backspace, and types with randomized key timing and physical Shift chords. Targets outside their clip are revealed with native wheel input, planned by a selector engine in Playwright's utility world. Native Playwright still owns actionability, navigation waiting and error semantics; specialized inputs, selects and non-editable targets keep native fill. Ported from the extension-backend-battle exercise with every tab/window activation removed: no page.bring_to_front(), no foreground navigation. All steps run on a background tab (the extension keeps attached tabs rendered). Opt out per call with human=False or globally with BW_HUMAN_INPUT=0. See docs/human-input.md. Co-Authored-By: Claude --- docs/human-input.md | 161 +++++ src/browserwright/_executor/process.py | 2 + src/browserwright/repl/human_input.py | 675 ++++++++++++++++++ tests/daemon/e2e/test_human_input.py | 419 +++++++++++ tests/daemon/e2e/test_human_pointer_layout.py | 114 +++ tests/daemon/e2e/test_human_scroll.py | 137 ++++ 6 files changed, 1508 insertions(+) create mode 100644 docs/human-input.md create mode 100644 src/browserwright/repl/human_input.py create mode 100644 tests/daemon/e2e/test_human_input.py create mode 100644 tests/daemon/e2e/test_human_pointer_layout.py create mode 100644 tests/daemon/e2e/test_human_scroll.py diff --git a/docs/human-input.md b/docs/human-input.md new file mode 100644 index 0000000..a4181aa --- /dev/null +++ b/docs/human-input.md @@ -0,0 +1,161 @@ +# Agent input defaults + +The resident executor installs `repl/human_input.py` in +`_Worker._bind_live_surface`, before handing its real Playwright `page` and +`context` to agent scripts or helper modules. Chained locators, `Page`/`Frame` +selector methods (`page.click(sel)`, `frame.fill(sel, v)`), child frames, and +pages created by `context.new_page()` share the defaults. Playwright objects +stay real Playwright objects and return values are unchanged. + +The defaults apply on both backends. The facade-side pieces below exist only +on the extension backend; the cdp backend keeps Chrome's native input path. + +## No focus stealing + +Nothing in this module activates a tab, raises a window or calls +`page.bring_to_front()`. The extension backend drives the user's own Chrome, +and browser UI activation steals the desktop's active window from them. +Through `chrome.debugger`, `Page.bringToFront` really does activate the tab +and focus its window, so the rule is enforced by not issuing it. + +Every step works on a tab that is not the active tab, in a window that is not +focused. The extension already keeps attached background tabs rendering +(`keepTabRendered()`: focus emulation plus an active web lifecycle; see +`test_l2_background_render.py`). Pointer moves, coalesced samples, wheel +scrolling, keyboard input and the post-wheel geometry re-reads therefore run +the same in the background as in the foreground. Nothing is degraded. +`test_human_input_no_focus.py` enforces this. It moves the user to another +active tab and a newly focused window, then runs click, fill and +scroll-into-view on the background session tab. It asserts that the input was +trusted, that every window's active tab and the focused window are unchanged, +and that `tabs.onActivated` / `windows.onFocusChanged` never fired for the +session tab or its window. + +An agent that explicitly calls `page.bring_to_front()` still gets native +activation. That is the agent's choice, never a default. + +## click() + +`click()` approaches a random interior point of the target, away from dead +center, through densely sampled mouse moves. It starts from the last tracked +pointer position, or from a viewport edge on first use. It then calls the +native Playwright click with a short randomized press duration (35–95 ms). +Native click still owns actionability, navigation waiting, modifiers, +button, click count, `force`, `trial` and explicit `position`/`delay`. +Preparation and the final action share the caller's timeout. + +Samples are sent in small groups without waiting for each ACK, so Chrome can +coalesce them like a real device (`PointerEvent.getCoalescedEvents()`). The +extension facade forwards a run of consecutive `mouseMoved` commands +concurrently. It finishes the run before it processes a press or any other +command, and browser-session setup stays sequential. Moves stay sequential +while a mouse button is held, which preserves Playwright's drag handling. + +Generated coordinates follow the device-pixel grid. For generated click +positions, the extension facade leases dispatch-time rounding +(`Browserwright.setPointerGrid`, a facade-only CDP method). This covers a +target that moves between the approach and Playwright's final actionability +pass. The lease belongs to the CDP session and tab. It is released after the +action, including on failure, and dropped when that session detaches or the +tab goes away. Explicit caller positions keep native coordinates. On the cdp +backend the method does not exist and the click keeps native dispatch. + +## Scrolling into view + +Before an ordinary click or text fill, a target outside its clipping area is +revealed with native wheel input instead of programmatic `scrollIntoView`. +Geometry comes from a per-context Playwright selector engine +(`_browserwright_scroll`) that runs in Playwright's utility world. It answers +through a detached transport node that never enters the document. Document +scrolling, nested overflow containers and enclosing frames are handled, and +geometry is re-read after each wheel burst. A page that cancels wheel input +uses up the caller's timeout, as it would for a real user. + +## fill() + +For a visible editable text control (text-like ``, ` +
old rich text
+
+
Existing image
+ + + + + + + + +""" + + +@pytest.fixture +def human_input_site(): + # getCoalescedEvents is restricted to secure contexts. A loopback origin + # is trustworthy; about:blank set_content cannot exercise this API. + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + body = _HTML.encode() + self.send_response(200) + self.send_header('Content-Type', 'text/html; charset=utf-8') + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + server = ThreadingHTTPServer(('127.0.0.1', 0), Handler) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + try: + yield f'http://127.0.0.1:{server.server_port}' + finally: + server.shutdown() + server.server_close() + worker.join(timeout=2) + + +def _assert_typed(record: dict, expected: str) -> None: + """Verify actual event order and final replacement, not wrapper internals.""" + assert record["value"] == expected + assert record["return_none"] is True + events = record["events"] + # Clearing a preexisting value can emit Delete/Backspace before typing. + starts = [i for i, event in enumerate(events) + if event["type"] == "keydown" and len(event["key"] or "") == 1 + and not (event.get('ctrl') or event.get('meta'))] + assert len(starts) == len(expected), events + for index, character in zip(starts, expected): + char_events = events[index:index + 4] + assert [event["type"] for event in char_events] == [ + "keydown", "keypress", "input", "keyup", + ], char_events + assert [event["key"] for event in char_events if event["key"]] == [ + character, character, character, + ], char_events + assert char_events[2]["data"] == character + assert all(event["trusted"] for event in char_events) + if "shift" in char_events[0] and character.isascii() and character.isupper(): + assert char_events[0]["shift"] is True, char_events + assert any(event["type"] == "keydown" and event["key"] == "Shift" + for event in events[:index]), events + + +@pytest.mark.parametrize("extension_only_chrome", ["headless"], indirect=True) +def test_executor_human_input_defaults_and_playwright_semantics( + e2e_daemon, e2e_chrome, ext_ready, e2e_artifacts_dir, human_input_site, +): + """Agent-created locators/pages/frames use normal browser input events.""" + pytest.importorskip("playwright.sync_api") + assert not (e2e_chrome.profile_path / 'DevToolsActivePort').exists() + sid = f"e2e-input-{uuid.uuid4().hex}" + seed_ledger_session(bs_home("extension"), sid, backend="extension", name="input") + artifact = e2e_artifacts_dir / "human-input.json" + artifact.unlink(missing_ok=True) + script = "import json\n" + f"page.goto({human_input_site!r}, wait_until='load')\n" + script += "from pathlib import Path\n" + f"artifact = Path({str(artifact)!r})\n" + script += textwrap.dedent(""" + from playwright.sync_api import Error, TimeoutError + result = {} + result['browser_version'] = {'reported': context.browser.version, + 'actual_user_agent': page.evaluate('navigator.userAgent')} + def take(): + return page.evaluate('events.splice(0)') + field = page.get_by_role('region', name='Form').get_by_role('textbox', name='Name') + returned = field.fill('Alice') + result['locator_fill'] = {'value': field.input_value(), 'return_none': returned is None, + 'events': take()} + returned = field.fill('A!b?') + result['shift_fill'] = {'value': field.input_value(), 'return_none': returned is None, + 'events': take()} + empty = page.get_by_role('textbox', name='Empty', exact=True) + returned = empty.fill('Eve') + result['initially_empty_fill'] = {'value': empty.input_value(), + 'return_none': returned is None, 'events': take()} + notes = page.get_by_role('textbox', name='Notes') + returned = notes.fill('New notes', force=True) + result['textarea_fill'] = {'value': notes.input_value(), + 'return_none': returned is None, 'events': take()} + + returned = page.get_by_role('button', name='Submit').click() + result['click'] = {'return_none': returned is None, 'events': take()} + metrics_session = context.new_cdp_session(page) + metrics_session.send('Emulation.setDeviceMetricsOverride', { + 'width': 1280, 'height': 900, 'deviceScaleFactor': 1.75, 'mobile': False}) + page.get_by_role('button', name='Submit').click() + result['scaled_click'] = {'events': take()} + metrics_session.send('Emulation.clearDeviceMetricsOverride') + metrics_session.detach() + returned = page.click('#submit', position={'x': 9, 'y': 11}, force=True) + result['position_click'] = {'return_none': returned is None, 'events': take()} + + returned = field.fill('native', human=False) + result['native_fill'] = {'value': field.input_value(), 'return_none': returned is None, + 'events': take()} + page.click('#submit', human=False) + result['native_click'] = {'events': take()} + + rich = page.get_by_role('textbox', name='Rich text') + returned = rich.fill('Rich') + result['rich_fill'] = {'value': rich.inner_text(), 'return_none': returned is None, + 'events': take()} + empty_rich = page.get_by_role('textbox', name='Empty editor') + returned = empty_rich.fill('Empty rich') + result['empty_editor_fill'] = {'value': empty_rich.inner_text(), + 'return_none': returned is None, 'events': take()} + media = page.get_by_role('textbox', name='Rich media') + returned = media.fill('Media') + result['media_fill'] = {'value': media.inner_text(), 'return_none': returned is None, + 'children': media.locator('img').count(), 'events': take()} + returned = page.fill('#name', 'Page') + result['page_fill'] = {'value': field.input_value(), 'return_none': returned is None, + 'events': take()} + returned = field.fill('') + result['empty_fill'] = {'value': field.input_value(), 'return_none': returned is None, + 'events': take()} + # A forced click can land on an overlay instead of the intended input. + # Replacement still belongs to the requested field, and must not select + # or erase another editor that was focused before the call. + empty.click(human=False) + page.evaluate('''() => { + const r=document.querySelector('#name').getBoundingClientRect(); + const overlay=document.createElement('div');overlay.id='overlay'; + Object.assign(overlay.style,{position:'fixed',left:r.left+'px',top:r.top+'px', + width:r.width+'px',height:r.height+'px',zIndex:'1000'}); + document.body.append(overlay); + }''') + take() + returned = field.fill('Forced', force=True) + result['forced_overlay_fill'] = {'value': field.input_value(), + 'other_value': empty.input_value(), + 'return_none': returned is None, 'events': take()} + page.evaluate("document.querySelector('#overlay').remove()") + + for selector in ['#readonly', '#disabled']: + try: + page.locator(selector).fill('x', timeout=180) + result[selector] = {'error': None} + except TimeoutError as exc: + result[selector] = {'error': type(exc).__name__, 'message': str(exc)} + try: + page.locator('#select').fill('bad', timeout=180) + result['select'] = {'error': None} + except Error as exc: + result['select'] = {'error': type(exc).__name__, 'message': str(exc)} + for selector, value in [('#number', '42'), ('#date', '2026-10-07')]: + returned = page.locator(selector).fill(value) + result[selector] = {'value': page.locator(selector).input_value(), + 'return_none': returned is None, 'events': take()} + try: + page.locator('#number').fill('not a number') + result['invalid_number'] = {'error': None} + except Error as exc: + result['invalid_number'] = {'error': type(exc).__name__, 'message': str(exc)} + # Keep observations available if later page/frame setup fails. + artifact.write_text(json.dumps(result, indent=2) + '\\n') + + # A held button enters Playwright's native drag handling. Preserve its + # state throughout the approach and let native click release normally. + page.mouse.move(20, 20) + page.mouse.down() + take() + returned = page.get_by_role('button', name='Submit').click(force=True) + result['held_button_click'] = {'return_none': returned is None, 'events': take()} + page.mouse.up() + take() + + other = context.new_page() + other.set_content('') + other.evaluate('''() => { + window.events = []; + for (const type of ['keydown', 'keypress', 'input', 'keyup']) + document.addEventListener(type, e => events.push({type, key:e.key || null, + data:e.data ?? null, trusted:e.isTrusted, + ctrl:e.ctrlKey, meta:e.metaKey})); + }''') + returned = other.get_by_role('textbox', name='Other').fill('New') + result['new_page_fill'] = {'value': other.locator('#other').input_value(), + 'return_none': returned is None, + 'events': other.evaluate('events')} + frame = other.main_frame + frame.evaluate('events.splice(0)') + returned = frame.fill('#other', 'Frame') + result['frame_fill'] = {'value': frame.locator('#other').input_value(), + 'return_none': returned is None, + 'events': frame.evaluate('events')} + other.close() + artifact.write_text(json.dumps(result, indent=2) + '\\n') + print('HUMAN_INPUT=' + ','.join(result)) + """) + try: + execution = run_skill(script, backend="extension", runtime_dir=e2e_daemon.runtime_dir, + extra_env={"BD_SESSION": sid}, timeout=90) + output = {"returncode": execution.returncode, "stdout": execution.stdout, + "stderr": execution.stderr} + if artifact.exists(): + output["observations"] = json.loads(artifact.read_text()) + artifact.write_text(json.dumps(output, indent=2) + "\n") + assert execution.returncode == 0, output + observations = output["observations"] + assert 'Chrome/' + observations['browser_version']['reported'] in \ + observations['browser_version']['actual_user_agent'] + _assert_typed(observations['initially_empty_fill'], 'Eve') + assert not any(e['key'] in ('Delete', 'Backspace') + for e in observations['initially_empty_fill']['events']) + assert not any(e.get('ctrl') or e.get('meta') + for e in observations['initially_empty_fill']['events']) + _assert_typed(observations['empty_editor_fill'], 'Empty rich') + assert not any(e['key'] in ('Delete', 'Backspace') or e.get('ctrl') or e.get('meta') + for e in observations['empty_editor_fill']['events']) + # Replacements must select through an actual trusted keyboard shortcut, + # with a full selection visible before deletion. This also covers a + # multiline textarea and contenteditable containing only an image. + for name, old in [('locator_fill', 'replace me'), ('shift_fill', 'Alice'), + ('textarea_fill', 'old multiline\nnotes'), + ('rich_fill', 'old rich text'), ('media_fill', None), + ('page_fill', 'native'), ('empty_fill', 'Page')]: + events = observations[name]['events'] + shortcut = next(e for e in events if e['type'] == 'keyup' + and (e['key'] or '').lower() == 'a' + and (e['ctrl'] or e['meta'])) + assert shortcut['trusted'], shortcut + assert any(e['type'] == 'keydown' and e['key'] in ('Control', 'Meta') + and e['trusted'] for e in events[:events.index(shortcut)]), events + selection = shortcut['selection'] + if 'start' in selection: + assert selection == {'start': 0, 'end': len(old)}, shortcut + else: + assert selection['collapsed'] is False, shortcut + if old is not None: + assert selection['text'] == old, shortcut + assert any(e['type'] == 'keyup' and e['key'] in ('Control', 'Meta') + for e in events[events.index(shortcut) + 1:]), events + for name in ('locator_fill', 'shift_fill', 'initially_empty_fill'): + pressed = {} + shift_down = None + for event in observations[name]['events']: + if event['type'] == 'keydown': + pressed[event['key']] = event['time'] + if event['key'] == 'Shift': + shift_down = event['time'] + elif event['shift'] and shift_down is not None: + assert event['time'] - shift_down >= 15, event + elif event['type'] == 'keyup' and event['key'] in pressed: + assert event['time'] - pressed.pop(event['key']) >= 20, event + first_moves = [e for e in observations['locator_fill']['events'] + if e['type'] == 'pointermove'] + assert first_moves and (first_moves[0]['x'] > 5 or first_moves[0]['y'] > 5), first_moves + for name in ('locator_fill', 'shift_fill', 'initially_empty_fill', 'click', 'scaled_click'): + for event in observations[name]['events']: + if event['type'] == 'pointermove': + assert abs(event['x'] * event['dpr'] - round(event['x'] * event['dpr'])) < .01, event + assert abs(event['y'] * event['dpr'] - round(event['y'] * event['dpr'])) < .01, event + for sample in event['coalesced'] or []: + assert abs(sample['x'] * event['dpr'] - round(sample['x'] * event['dpr'])) < .01, event + assert abs(sample['y'] * event['dpr'] - round(sample['y'] * event['dpr'])) < .01, event + focus = next(event for event in observations["locator_fill"]["events"] + if event["type"] == "focus" and event["target"] == "name") + assert focus["activated"] is True, focus + assert any(event["type"] == "mousedown" and event["target"] == "name" + for event in observations["locator_fill"]["events"]), observations["locator_fill"] + for key, value in [("locator_fill", "Alice"), ("rich_fill", "Rich"), + ("textarea_fill", "New notes"), + ("page_fill", "Page"), ("new_page_fill", "New"), + ("frame_fill", "Frame")]: + _assert_typed(observations[key], value) + _assert_typed(observations["shift_fill"], "A!b?") + assert all(event["shift"] for event in observations["shift_fill"]["events"] + if event["type"] == "keydown" and event["key"] in ("A", "!", "?")) + _assert_typed(observations['media_fill'], 'Media') + assert observations['media_fill']['children'] == 0 + assert observations["empty_fill"]["value"] == "" + assert observations["empty_fill"]["return_none"] is True + forced = observations['forced_overlay_fill'] + assert forced['value'] == 'Forced' and forced['other_value'] == 'Eve', forced + assert forced['return_none'] is True + assert not any(e.get('ctrl') or e.get('meta') for e in forced['events']), forced + assert observations["native_fill"]["value"] == "native" + assert observations["native_fill"]["return_none"] is True + assert not any(event["type"] in ("keydown", "keypress", "keyup") + for event in observations["native_fill"]["events"]) + assert observations["click"]["return_none"] is True + held = observations['held_button_click'] + assert held['return_none'] is True + assert any(e['type'] == 'mousemove' and e['buttons'] == 1 for e in held['events']), held + assert any(e['type'] == 'click' and e['target'] == 'submit' for e in held['events']), held + assert any(e['type'] == 'mouseup' and e['buttons'] == 0 for e in held['events']), held + pointer = observations["click"]["events"] + coalesced = [event for event in pointer if event['type'] == 'pointermove' + and len(event['coalesced'] or []) > 1] + assert coalesced, pointer + for event in coalesced: + assert all(sample['trusted'] for sample in event['coalesced']), event + assert all(a['time'] <= b['time'] for a, b in + zip(event['coalesced'], event['coalesced'][1:])), event + assert sum(event["type"] == "mousemove" for event in pointer) >= 3, pointer + visible_moves = [event for event in pointer if event['type'] == 'mousemove'] + assert len(visible_moves) >= 15, visible_moves + assert all(((b['x'] - a['x']) ** 2 + (b['y'] - a['y']) ** 2) ** .5 <= 15 + for a, b in zip(visible_moves, visible_moves[1:])), visible_moves + # Chrome may deliver several physical samples in one pointermove. + # Check path density using those original samples, rather than requiring + # the browser to dispatch every sample as a separate DOM event. + moves = [sample for event in pointer if event['type'] == 'pointermove' + for sample in (event['coalesced'] or [event])] + assert len(moves) >= 15, moves + assert all(((b["x"] - a["x"]) ** 2 + (b["y"] - a["y"]) ** 2) ** .5 <= 15 + for a, b in zip(moves, moves[1:])), moves + click = next(event for event in pointer if event["type"] == "click") + rect = click["rect"] + assert rect["x"] < click["x"] < rect["x"] + rect["width"] + assert rect["y"] < click["y"] < rect["y"] + rect["height"] + assert (abs(click["x"] - rect["x"] - rect["width"] / 2) > 0.5 or + abs(click["y"] - rect["y"] - rect["height"] / 2) > 0.5), click + down = next(event for event in pointer if event["type"] == "mousedown") + up = next(event for event in pointer if event["type"] == "mouseup") + assert up["time"] - down["time"] >= 5, pointer + positioned = next(event for event in observations["position_click"]["events"] + if event["type"] == "click") + assert abs(positioned["x"] - positioned["rect"]["x"] - + positioned["rect"]["borderX"] - 9) <= 1 + assert abs(positioned["y"] - positioned["rect"]["y"] - + positioned["rect"]["borderY"] - 11) <= 1 + native = next(event for event in observations["native_click"]["events"] + if event["type"] == "click") + assert abs(native["x"] - native["rect"]["x"] - native["rect"]["width"] / 2) <= 1 + assert abs(native["y"] - native["rect"]["y"] - native["rect"]["height"] / 2) <= 1 + for selector in ("#readonly", "#disabled"): + assert observations[selector]["error"] == "TimeoutError", observations[selector] + assert observations["select"]["error"] == "Error" + assert observations["invalid_number"]["error"] == "Error" + assert observations["#number"]["value"] == "42" + assert observations["#date"]["value"] == "2026-10-07" + finally: + drop_ledger_session(bs_home("extension"), sid) diff --git a/tests/daemon/e2e/test_human_pointer_layout.py b/tests/daemon/e2e/test_human_pointer_layout.py new file mode 100644 index 0000000..a7ff368 --- /dev/null +++ b/tests/daemon/e2e/test_human_pointer_layout.py @@ -0,0 +1,114 @@ +"""Native pointer coordinates after target geometry changes during an approach. + +Repeat with ``mise exec -- uv run pytest tests/daemon/e2e/test_human_pointer_layout.py -q``. +The artifact records trusted browser events and the final target geometry. +""" +from __future__ import annotations + +import json +import textwrap + +import pytest + +from .helpers import run_skill + + +@pytest.fixture +def e2e_chrome(extension_only_chrome): + return extension_only_chrome + + +@pytest.mark.parametrize('extension_only_chrome', ['headless'], indirect=True) +def test_click_after_layout_shift_stays_on_physical_pixel_grid( + e2e_daemon, e2e_chrome, ext_ready, e2e_artifacts_dir, +): + artifact = e2e_artifacts_dir / 'human-pointer-layout.json' + artifact.unlink(missing_ok=True) + script = 'import json\nfrom pathlib import Path\n' + script += f'artifact=Path({str(artifact)!r})\n' + script += textwrap.dedent(''' + result={} + metrics=context.new_cdp_session(page) + for ratio in [2,1.75]: + page.goto('about:blank') + metrics.send('Emulation.setDeviceMetricsOverride', { + 'width':1000,'height':700,'deviceScaleFactor':ratio,'mobile':False}) + page.set_content(""" + """) + returned=page.get_by_role('button',name='Moving').click() + result[str(ratio)]={'return_none':returned is None, + 'observation':page.evaluate("""() => ({clicked,shifted,events, + box:document.querySelector('button').getBoundingClientRect().toJSON()})""")} + # The framework lease must end with the generated click. Explicit + # padding-relative positions and human=False retain native CDP + # coordinates, even when those are between physical pixels. + result[str(ratio)]['explicit']=[] + for native in [False,True]: + page.evaluate('events=[]') + page.get_by_role('button',name='Moving').click( + position={'x':20.123,'y':20.123},human=not native) + result[str(ratio)]['explicit'].append(page.evaluate("""() => ({events, + box:document.querySelector('button').getBoundingClientRect().toJSON(), + border:parseFloat(getComputedStyle(document.querySelector('button')).borderLeftWidth)})""")) + artifact.write_text(json.dumps(result,indent=2)) + metrics.send('Emulation.clearDeviceMetricsOverride') + # Detaching a CDP handle must retire its temporary input state without + # affecting the live page or native calls made through another handle. + lease=context.new_cdp_session(page) + lease.send('Browserwright.setPointerGrid', {'devicePixelRatio':2}) + lease.detach() + page.evaluate('events=[]') + page.get_by_role('button',name='Moving').click( + position={'x':20.123,'y':20.123},human=False) + result['detached_lease']=page.evaluate("""() => ({events, + box:document.querySelector('button').getBoundingClientRect().toJSON(), + border:parseFloat(getComputedStyle(document.querySelector('button')).borderLeftWidth)})""") + artifact.write_text(json.dumps(result,indent=2)) + metrics.detach() + ''') + execution = run_skill(script, backend='extension', runtime_dir=e2e_daemon.runtime_dir, + timeout=40) + output = {'returncode':execution.returncode,'stdout':execution.stdout,'stderr':execution.stderr} + if artifact.exists(): + output['observations'] = json.loads(artifact.read_text()) + artifact.write_text(json.dumps(output,indent=2)+'\n') + assert execution.returncode == 0, output + detached = output['observations'].pop('detached_lease') + press = next(event for event in detached['events'] if event['type']=='pointerdown') + assert press['trusted'] and press['target']=='target', detached + for axis in ('x','y'): + expected = int((detached['box'][axis]+detached['border']+20.123)*100)/100 + assert abs(press[axis]-expected) < .001, detached + for result in output['observations'].values(): + observed = result['observation'] + assert result['return_none'] and observed['clicked'] and observed['shifted'], observed + box = observed['box'] + assert box['x'] == 360.625, box + pointer = [event for event in observed['events'] if event['type'].startswith('pointer')] + assert any(event['type']=='pointerdown' and event['target']=='target' for event in pointer) + for event in pointer: + assert event['trusted'], event + for sample in [event, *(event['coalesced'] or [])]: + assert sample['trusted'], sample + for axis in ('x','y'): + scaled = sample[axis]*event['dpr'] + assert abs(scaled-round(scaled)) < .01, event + for explicit in result['explicit']: + press = next(event for event in explicit['events'] if event['type']=='pointerdown') + assert press['trusted'] and press['target']=='target', explicit + for axis in ('x','y'): + expected = int((explicit['box'][axis]+explicit['border']+20.123)*100)/100 + assert abs(press[axis]-expected) < .001, explicit diff --git a/tests/daemon/e2e/test_human_scroll.py b/tests/daemon/e2e/test_human_scroll.py new file mode 100644 index 0000000..edfdf41 --- /dev/null +++ b/tests/daemon/e2e/test_human_scroll.py @@ -0,0 +1,137 @@ +"""Trusted wheel preparation through the extension-only executor. + +Repeat with ``mise exec -- uv run pytest tests/daemon/e2e/test_human_scroll.py -q``. +The artifact records real wheel/scroll order for document, nested and frame input. +""" +from __future__ import annotations + +import json +import textwrap +import pytest + +from .helpers import run_skill + + +@pytest.fixture +def e2e_chrome(extension_only_chrome): + return extension_only_chrome + + +_HTML = """
+
+
+
+
+
+""" + + +@pytest.mark.parametrize('extension_only_chrome', ['headless'], indirect=True) +def test_scroll_target_at_document_bottom(e2e_daemon, e2e_chrome, ext_ready, e2e_artifacts_dir): + artifact = e2e_artifacts_dir / 'human-scroll-bottom.json' + script = textwrap.dedent(''' + import json + page.set_content(""" +
+ """) + returned=page.get_by_role('button',name='Edge').click(timeout=8000) + result={'return_none':returned is None,'clicked':page.evaluate('window.clicked'), + 'events':page.evaluate('events')} + ''') + script += f'\nfrom pathlib import Path\nPath({str(artifact)!r}).write_text(json.dumps(result,indent=2))\n' + execution = run_skill(script, backend='extension', runtime_dir=e2e_daemon.runtime_dir,timeout=20) + assert execution.returncode == 0, (execution.stdout,execution.stderr) + result = json.loads(artifact.read_text()) + assert result['return_none'] and result['clicked'],result + assert any(e['type']=='wheel' and e['trusted'] for e in result['events']),result + + +@pytest.mark.parametrize('extension_only_chrome', ['headless'], indirect=True) +def test_executor_scroll_has_trusted_wheel_input( + e2e_daemon, e2e_chrome, ext_ready, e2e_artifacts_dir, +): + artifact = e2e_artifacts_dir / 'human-scroll.json' + artifact.unlink(missing_ok=True) + script = 'import json\nfrom pathlib import Path\n' + script += f'artifact=Path({str(artifact)!r})\npage.set_content({_HTML!r})\n' + script += textwrap.dedent(''' + from playwright.sync_api import TimeoutError + result={} + def save(): artifact.write_text(json.dumps(result,indent=2)) + returned=page.get_by_role('button',name='Top').click() + result['visible']={'return_none':returned is None,'events':page.evaluate('events.splice(0)')} + save() + child_frame=page.frames[1] + child_frame.evaluate("""() => { + window.events=[]; window.clicked=false; + for(const type of ['wheel','scroll','click']) document.addEventListener(type,e=>{ + events.push({type,trusted:e.isTrusted,time:performance.now()}); + if(type==='click' && e.target.id==='child') clicked=true; + },true); + }""") + returned=page.get_by_role('textbox',name='Below').fill('Ada') + result['document']={'return_none':returned is None,'value':page.locator('#below').input_value(), + 'events':page.evaluate('events.splice(0)')} + save() + returned=page.get_by_role('button',name='Nested').click(position={'x':12,'y':11},force=True) + result['nested']={'return_none':returned is None,'clicked':page.evaluate('clicked.slice()'), + 'events':page.evaluate('events.splice(0)')} + save() + child=page.frame_locator('#frame').get_by_role('button',name='Child') + returned=child.click() + result['frame']={'return_none':returned is None,'events':page.evaluate('events.splice(0)'), + 'child_events':child_frame.evaluate('events'), + 'child_clicked':child_frame.evaluate('clicked')} + save() + page.get_by_role('button',name='Top').click(human=False) + page.evaluate("events.splice(0); document.addEventListener('wheel',e=>e.preventDefault(),{passive:false})") + try: + page.get_by_role('textbox',name='Below').click(timeout=3000) + result['blocked']={'timeout':False} + except TimeoutError: + result['blocked']={'timeout':True,'events':page.evaluate('events.splice(0)')} + artifact.write_text(json.dumps(result,indent=2)) + ''') + execution = run_skill(script, backend='extension', runtime_dir=e2e_daemon.runtime_dir, + timeout=90) + output = {'returncode':execution.returncode,'stdout':execution.stdout,'stderr':execution.stderr} + if artifact.exists(): + output['observations'] = json.loads(artifact.read_text()) + artifact.write_text(json.dumps(output,indent=2)+'\n') + assert execution.returncode == 0, output + observations = output['observations'] + assert observations['document']['value'] == 'Ada' + assert 'nested' in observations['nested']['clicked'] + assert observations['visible']['return_none'] + assert not any(e['type'] in ['scroll','wheel'] for e in observations['visible']['events']) + assert observations['frame']['child_clicked'] + for name in ['document','nested','frame']: + events = observations[name]['events'] + assert observations[name]['return_none'], observations[name] + assert any(e['type']=='wheel' and e['trusted'] for e in events), events + for i, event in enumerate(events): + if event['type']=='scroll': + assert any(prior['type']=='wheel' and prior['trusted'] + and event['time']-prior['time']<200 for prior in events[:i]), events + child_events=observations['frame']['child_events'] + assert any(e['type']=='wheel' and e['trusted'] for e in child_events), child_events + for i,event in enumerate(child_events): + if event['type']=='scroll': + assert any(prior['type']=='wheel' and prior['trusted'] + and event['time']-prior['time']<200 for prior in child_events[:i]), child_events + assert observations['blocked']['timeout'], observations['blocked'] + assert any(e['type']=='wheel' and e['trusted'] for e in observations['blocked']['events']) + assert not any(e['type']=='scroll' for e in observations['blocked']['events']) From ca5bd9fea78ab7baaa90b4bf0dadd388b77250e5 Mon Sep 17 00:00:00 2001 From: Randy Date: Wed, 7 Oct 2026 22:34:47 +0800 Subject: [PATCH 4/4] test(e2e): human input on a background tab never activates it or its window Moves the user to another active tab and a newly focused window, then runs executor click, fill and scroll-into-view on the background session tab. Asserts trusted input took effect and that Chrome's active tabs, focused window and tabs.onActivated / windows.onFocusChanged events are untouched. Verified to fail when click() calls page.bring_to_front(). Co-Authored-By: Claude --- tests/daemon/e2e/test_human_input_no_focus.py | 252 ++++++++++++++++++ 1 file changed, 252 insertions(+) create mode 100644 tests/daemon/e2e/test_human_input_no_focus.py diff --git a/tests/daemon/e2e/test_human_input_no_focus.py b/tests/daemon/e2e/test_human_input_no_focus.py new file mode 100644 index 0000000..a9c4ac7 --- /dev/null +++ b/tests/daemon/e2e/test_human_input_no_focus.py @@ -0,0 +1,252 @@ +"""Human input on a background tab must never activate a tab or focus a window. + +The extension backend drives the user's own Chrome. Activating the session's +tab or raising its window steals the desktop's active window from the user. +This test puts the user somewhere else first — a newly focused window, plus a +newer active tab in the session tab's own window — then runs executor +click + fill + scroll-into-view on the (now background) session tab. + +It asserts (a) the page saw the actions as trusted input and they took effect, +and (b) Chrome's own tab/window state is unchanged: the same active tab in +every window, the same focused window, and no `tabs.onActivated` / +`windows.onFocusChanged` event for the session tab or its window at any point +during the actions (so a transient activate-then-restore is caught too). +Chrome state is read from the extension's service worker over the harness's +CDP port, independently of the daemon. + +Repeat with ``bash tests/daemon/e2e/run.sh +tests/daemon/e2e/test_human_input_no_focus.py -v``. The artifact is +``tests/daemon/e2e/_artifacts/human-input-no-focus.json``. Headful by default +(the meaningful case); ``BW_E2E_HEADLESS=1`` still exercises the contract. +""" +from __future__ import annotations + +import json +import textwrap +import threading +import time +import uuid +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import pytest + +from browserwright.cdp import CDPSession + +from .helpers import bs_home, drop_ledger_session, run_skill, seed_ledger_session +from .test_l2_multisession import _extension_id_from_path, _extension_worker_target_id + +_SESSION_HTML = """Session page + + + +
+ +""" + +_USER_HTML = """User page +

The user is reading this tab.

""" + +_WATCH_JS = """(() => { + globalThis.__bwFocusLog = []; + const log = (kind) => (...args) => __bwFocusLog.push({kind, args, t: Date.now()}); + globalThis.__bwFocusListeners = [ + [chrome.tabs.onActivated, log('tabs.onActivated')], + [chrome.tabs.onHighlighted, log('tabs.onHighlighted')], + [chrome.windows.onFocusChanged, log('windows.onFocusChanged')], + ]; + for (const [event, fn] of __bwFocusListeners) event.addListener(fn); + return true; +})()""" + +_STATE_JS = """(async () => { + const windows = await chrome.windows.getAll({populate: true}); + let lastFocused = null; + try { lastFocused = (await chrome.windows.getLastFocused()).id; } catch (e) {} + return { + lastFocusedWindow: lastFocused, + windows: windows.map(w => ({id: w.id, focused: w.focused, + activeTab: (w.tabs.find(t => t.active) || {}).id ?? null, + tabs: w.tabs.map(t => ({id: t.id, active: t.active, url: t.url}))})), + }; +})()""" + + +class _Worker: + """Evaluate in the extension's service worker over the browser CDP port.""" + + def __init__(self, chrome, extension_id: str): + self._cdp = CDPSession(chrome.ws_url) + self._sid = self._cdp.attach(_extension_worker_target_id(self._cdp, extension_id)) + + def eval(self, expression: str): + result = self._cdp.send("Runtime.evaluate", session=self._sid, + expression=expression, returnByValue=True, + awaitPromise=True) + if "exceptionDetails" in result: + raise AssertionError(f"service-worker evaluation failed: {result!r}") + return result.get("result", {}).get("value") + + def close(self) -> None: + self._cdp.close() + + +@pytest.fixture +def pages_site(): + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + body = (_USER_HTML if self.path.startswith("/user") else _SESSION_HTML).encode() + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + try: + yield f"http://127.0.0.1:{server.server_port}" + finally: + server.shutdown() + server.server_close() + worker.join(timeout=2) + + +def _wait_loaded(worker: _Worker, tab_id: int, timeout: float = 15.0) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if worker.eval(f"chrome.tabs.get({tab_id}).then(t => t.status)") == "complete": + return + time.sleep(0.1) + raise AssertionError(f"tab {tab_id} did not finish loading") + + +def test_background_tab_human_input_never_steals_focus( + e2e_daemon, e2e_chrome, ext_ready, patched_ext_dir, e2e_artifacts_dir, pages_site, +): + pytest.importorskip("playwright.sync_api") + artifact = e2e_artifacts_dir / "human-input-no-focus.json" + artifact.unlink(missing_ok=True) + token = uuid.uuid4().hex + session_url = f"{pages_site}/session?{token}" + sid = f"e2e-nofocus-{token}" + seed_ledger_session(bs_home("extension"), sid, backend="extension", name="nofocus") + env = {"BD_SESSION": sid} + worker = _Worker(e2e_chrome, _extension_id_from_path(patched_ext_dir)) + record: dict = {"session_url": session_url} + try: + # 1. Bind the session tab to the fixture page. + setup = run_skill(f"page.goto({session_url!r}, wait_until='load')\n", + backend="extension", runtime_dir=e2e_daemon.runtime_dir, + extra_env=env, timeout=60) + assert setup.returncode == 0, (setup.stdout, setup.stderr) + tabs = worker.eval(f"chrome.tabs.query({{}}).then(ts => ts.filter(" + f"t => (t.url || '').includes({token!r})).map(t => t.id))") + assert len(tabs) == 1, tabs + session_tab = tabs[0] + session_window = worker.eval(f"chrome.tabs.get({session_tab}).then(t => t.windowId)") + + # 2. The user moves on: a newer active tab in the session tab's own + # window, then a separate window that takes focus. + user_tab = worker.eval( + f"chrome.tabs.create({{windowId: {session_window}, " + f"url: {pages_site + '/user-tab'!r}, active: true}}).then(t => t.id)") + user_window = worker.eval( + f"chrome.windows.create({{url: {pages_site + '/user-window'!r}, " + f"focused: true}}).then(w => w.id)") + user_window_tab = worker.eval( + f"chrome.tabs.query({{windowId: {user_window}}}).then(ts => ts[0].id)") + for tab in (user_tab, user_window_tab): + _wait_loaded(worker, tab) + time.sleep(0.5) # let window-manager focus settle before the baseline + before = worker.eval(_STATE_JS) + record["before"] = before + session_win_state = next(w for w in before["windows"] if w["id"] == session_window) + assert session_win_state["activeTab"] == user_tab, before + assert not any(t["active"] for t in session_win_state["tabs"] + if t["id"] == session_tab), before + assert worker.eval(_WATCH_JS) is True + + # 3. Human input on the background session tab. + script = textwrap.dedent(""" + import json, time + out = {'visibility': page.evaluate('document.visibilityState')} + t0 = time.monotonic() + page.get_by_role('button', name='Near').click() + out['click_s'] = time.monotonic() - t0 + t0 = time.monotonic() + page.get_by_role('textbox', name='Name').fill('New Name!') + out['fill_s'] = time.monotonic() - t0 + t0 = time.monotonic() + page.get_by_role('button', name='Far').click() + out['scroll_click_s'] = time.monotonic() - t0 + out['value'] = page.locator('#name').input_value() + out['clicks'] = page.evaluate('clicks') + out['scrollY'] = page.evaluate('scrollY') + out['events'] = page.evaluate('events') + from pathlib import Path + Path(RESULT_PATH).write_text(json.dumps(out)) + """) + result_path = e2e_artifacts_dir / f"human-input-no-focus-{token}.tmp.json" + script = f"RESULT_PATH = {str(result_path)!r}\n" + script + run = run_skill(script, backend="extension", runtime_dir=e2e_daemon.runtime_dir, + extra_env=env, timeout=90) + record["run"] = {"returncode": run.returncode, "stdout": run.stdout, + "stderr": run.stderr} + focus_log = worker.eval("globalThis.__bwFocusLog ?? null") + after = worker.eval(_STATE_JS) + record.update(focus_log=focus_log, after=after) + assert run.returncode == 0, record["run"] + result = json.loads(result_path.read_text()) + result_path.unlink() + record["result"] = result + + # (a) The actions happened, through trusted browser input. + assert result["clicks"] == ["near", "far"], result["clicks"] + assert result["value"] == "New Name!", result["value"] + assert result["scrollY"] > 0, result["scrollY"] + events = result["events"] + for kind in ("pointermove", "mousedown", "click", "keydown", "input", "wheel"): + seen = [e for e in events if e["type"] == kind] + assert seen and all(e["trusted"] for e in seen), (kind, seen) + typed = "".join(e["key"] for e in events + if e["type"] == "keydown" and len(e["key"] or "") == 1) + assert typed.endswith("New Name!"), typed + + # (b) Chrome's focus state is exactly what the user left it as. + assert focus_log is not None, "service worker restarted; focus log lost" + + def activations(state): + return {w["id"]: w["activeTab"] for w in state["windows"]} + + assert activations(after) == activations(before), (before, after) + assert after["lastFocusedWindow"] == before["lastFocusedWindow"], (before, after) + assert ([w["focused"] for w in after["windows"]] + == [w["focused"] for w in before["windows"]]), (before, after) + assert not any(e["kind"] == "tabs.onActivated" + and e["args"][0].get("tabId") == session_tab + for e in focus_log), focus_log + assert not any(e["kind"] == "windows.onFocusChanged" + and e["args"][0] == session_window + for e in focus_log), focus_log + assert not any(e["kind"] == "tabs.onHighlighted" + and session_tab in (e["args"][0].get("tabIds") or []) + for e in focus_log), focus_log + finally: + artifact.write_text(json.dumps(record, indent=2) + "\n") + try: + worker.eval("""(() => { for (const [e, f] of globalThis.__bwFocusListeners || []) + e.removeListener(f); return true; })()""") + finally: + worker.close() + drop_ledger_session(bs_home("extension"), sid)