Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 35 additions & 1 deletion src/browserwright/_executor/process.py
Original file line number Diff line number Diff line change
Expand Up @@ -809,7 +809,8 @@ def _on_facade_dead(self, browser: Any | None = None) -> None:
return
sys.stderr.write(
f"executor {self._session_id}: facade transport dropped "
"(daemon restart?); will reconnect on the next call\n")
"(daemon restart, or the link to the browser); will reconnect on "
"the next call\n")
self._connected = False
self._facade_death_handler = None
self._facade_daemon_pid = None
Expand Down Expand Up @@ -1003,6 +1004,20 @@ def _target_closed_response(
outcomes, no loop.
"""
sid = self._session_id or "<id>"
if self._transport_lost():
# Not the tab: the executor's connection to the browser is gone,
# and with it every Page and the context. Rebinding here would run
# on no context, and forgetting the binding would abandon a tab
# that is still alive. The next call reconnects and re-binds the
# SAME tab (`_ensure_connected`), with `state` intact.
error["fix"] = (
"the executor's connection to the browser dropped mid-call "
"(the tab itself was not closed). It reconnects to the same "
"tab on the next call and `state` is kept — RETRY the call. "
"Do not create a new session; if this repeats, check the link "
"to the browser (`browserwright-daemon` log names why the "
"connection ended).")
return self._finish(buf, error=error, exit_code=exit_code)
failure: str | None = None
try:
if not self._rebind_dead_page():
Expand Down Expand Up @@ -1034,6 +1049,25 @@ def _target_closed_response(
)
return self._finish(buf, error=error, exit_code=exit_code)

def _transport_lost(self) -> bool:
"""Is the Playwright Browser disconnected from the cdp surface?

A dropped connection closes every Page at once, so the call fails with
the same TargetClosedError a dead tab produces. Playwright usually
dispatched ``disconnected`` while the failing call pumped events; if it
has not yet, its ``is_connected()`` already reads False and the armed
handler is run here."""
if not self._connected:
return True
is_connected = getattr(self._browser, "is_connected", None)
try:
alive = bool(is_connected()) if callable(is_connected) else True
except Exception: # noqa: BLE001 - a failed probe is disconnected
alive = False
if not alive:
self._on_facade_dead(self._browser)
return not alive

@staticmethod
def _exec_with_return(code: str, globals_: dict[str, Any]) -> str | None:
"""Exec ``code``; if the LAST statement is a bare expression, return its
Expand Down
37 changes: 31 additions & 6 deletions src/browserwright/daemon/server/facade.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,14 +68,15 @@
from ..config import (DEFAULT_FACADE_PORT, LOOPBACK_HOST, Config,
needs_loopback_cobind)
from .. import __version__
from .._net import proxy_toward
from .._net import proxy_toward, redact_url
from ..errors import Unavailable
from ..resolver import resolve as resolve_upstream
from ..._executor.protocol import _MAX_FRAME
from .daemon import Daemon, UnknownSessionError, UpstreamContext
from .exec_relay import serve_exec_relay
from .facade_extension import ExtensionFacadeBridge
from .relay import RelayServer
from .ws_liveness import connect_kwargs as liveness_kwargs, keep_alive

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -669,31 +670,37 @@ async def _bridge(self, client: ServerConnection, upstream_url: str,
Transparent: no id/sessionId rewriting (unlike the agent Router) — a
Playwright client owns the whole browser-level CDP namespace on its own
dedicated upstream connection, so Target.*/Browser.* responses and
events flow back unmodified."""
events flow back unmodified.

The upstream leg's keepalive is `ws_liveness`, not websockets' own:
on a slow remote link a pong stuck behind one big reply used to close
this bridge mid-transfer, which the Playwright client can only read as
its whole browser closing."""
upstream = await websockets.connect(
upstream_url,
max_size=100 * 1024 * 1024,
compression=None,
ping_interval=20,
ping_timeout=20,
# Exactly the session's proxy, or None for direct — never
# websockets' default of reading the daemon's http_proxy/all_proxy,
# which broke LAN / Tailscale / CloakBrowser upstreams (#20). The
# CLI that opened the session resolved it from its own environment
# (#136); per-page proxying is still Chrome's own business.
proxy=proxy_toward(upstream_url, proxy),
**liveness_kwargs(),
)
c2u = asyncio.create_task(self._pump(client, upstream, "c->u"))
u2c = asyncio.create_task(self._pump(upstream, client, "u->c"))
alive = asyncio.create_task(
keep_alive(upstream, label="facade bridge upstream"))
try:
await asyncio.wait(
{c2u, u2c}, return_when=asyncio.FIRST_COMPLETED)
finally:
# Cancel + await BOTH pumps unconditionally. This runs on the normal
# Cancel + await every task unconditionally. This runs on the normal
# FIRST_COMPLETED path AND when stop() cancels the handler task while
# it's suspended in asyncio.wait above (where the `for pending`
# cleanup would otherwise be skipped, orphaning the pump tasks).
for t in (c2u, u2c):
for t in (c2u, u2c, alive):
t.cancel()
# CancelledError is a BaseException; suppress it explicitly so a
# cancelled pump doesn't escape this cleanup.
Expand All @@ -703,6 +710,12 @@ async def _bridge(self, client: ServerConnection, upstream_url: str,
await upstream.close()
with contextlib.suppress(Exception):
await client.close()
# The client sees this as its browser disconnecting, so say which
# side ended it: nothing else in the daemon log would.
logger.info(
"facade: bridge to %s ended (upstream %s, client %s)",
redact_url(upstream_url), _close_summary(upstream),
_close_summary(client))

@staticmethod
async def _pump(src, dst, label: str) -> None:
Expand All @@ -717,6 +730,18 @@ async def _pump(src, dst, label: str) -> None:
return


def _close_summary(conn: Any) -> str:
"""`sent 1011 'keepalive timeout', received none` for one ws side."""
def frame(close: Any) -> str:
if close is None:
return "none"
return f"{int(close.code)} {close.reason!r}"

protocol = getattr(conn, "protocol", None)
return (f"sent {frame(getattr(protocol, 'close_sent', None))}, "
f"received {frame(getattr(protocol, 'close_rcvd', None))}")


#: ADR-0011 name for what this class now is. `PlaywrightFacade` stays as the
#: class's own name (it is spelled in a lot of call sites and tests), but new
#: code should read `EndpointServer` — the object serves three surfaces, only
Expand Down
13 changes: 10 additions & 3 deletions src/browserwright/daemon/server/upstream.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
from websockets.exceptions import ConnectionClosed

from .._net import proxy_hint, proxy_toward, redact_url
from .ws_liveness import connect_kwargs as liveness_kwargs, keep_alive

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -211,6 +212,7 @@ def __init__(
self._ws: websockets.ClientConnection | None = None # type: ignore[name-defined]
self._reader_task: asyncio.Task | None = None
self._heartbeat_task: asyncio.Task | None = None
self._keepalive_task: asyncio.Task | None = None
self._next_internal_id = _DAEMON_ID_BASE
self._pending_internal: dict[int, asyncio.Future] = {}
self._ws_url: str | None = None
Expand Down Expand Up @@ -358,11 +360,12 @@ async def open(self, ws_url: str | None = None, *,
# The CLI that opened the session resolved it from its own
# environment; the facade bridge uses the same value. (#20, #136)
"proxy": proxy,
# Keep the upstream alive with ws-level pings; CDP-level
# ws-level liveness is `ws_liveness.keep_alive` (started below):
# websockets' own keepalive closes a working connection when a
# pong is stuck behind a big reply on a slow link. CDP-level
# Browser.getVersion heartbeat is layered on top for protocol
# liveness.
"ping_interval": 20,
"ping_timeout": 20,
**liveness_kwargs(),
}
try:
self._ws = await asyncio.wait_for(
Expand All @@ -381,6 +384,8 @@ async def open(self, ws_url: str | None = None, *,
self._ws_url = ws_url
self._reader_task = asyncio.create_task(self._reader_loop())
self._heartbeat_task = asyncio.create_task(self._heartbeat_loop())
self._keepalive_task = asyncio.create_task(keep_alive(
self._ws, label=f"{self.backend_name} upstream"))
if lifecycle:
# Tell Chrome to gossip about all targets so the router can keep
# its target table without the client having to enable it.
Expand Down Expand Up @@ -985,6 +990,8 @@ async def close(self, *, code: int = 1000, reason: str = "") -> None:
self._reader_task.cancel()
if self._heartbeat_task is not None:
self._heartbeat_task.cancel()
if self._keepalive_task is not None:
self._keepalive_task.cancel()
ws = self._ws
self._ws = None
for fut in self._pending_internal.values():
Expand Down
77 changes: 77 additions & 0 deletions src/browserwright/daemon/server/ws_liveness.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
"""Keepalive for the daemon's upstream CDP websockets.

websockets' built-in keepalive declares a connection dead when one pong is late
by ``ping_timeout``. A pong travels in the same TCP stream as the data, behind
it. On a slow link — an attached browser across a tailnet relay at 0.3-1 s RTT
and a few hundred KB/s — one large CDP reply (an aria snapshot of a big page, a
screenshot) can hold the pong back for longer than that, although bytes are
arriving the whole time. The built-in keepalive then closes a working
connection with 1011. On the cdp surface that is the Playwright client's whole
browser: every Page reports "Target page, context or browser has been closed"
while the tab itself is fine.

So the verdict here is "nothing at all has arrived": any inbound byte counts as
proof of life, and the ping is only how a quiet connection gets something to
arrive. Bytes, not messages: that one big reply is a single message that takes
the whole transfer to complete. Pings still go out on idle connections, so NAT
and proxy idle timers stay fed.

Open a connection with :func:`connect_kwargs` and run :func:`keep_alive` on it.
"""
from __future__ import annotations

import asyncio
import logging
import time
from typing import Any

from websockets.asyncio.client import ClientConnection
from websockets.exceptions import ConnectionClosed

logger = logging.getLogger(__name__)

#: Ping a connection that has been quiet this long.
KEEPALIVE_INTERVAL = 20.0
#: Close it once it has been silent — not one byte in — this long after a ping.
KEEPALIVE_TIMEOUT = 20.0


class TrackedConnection(ClientConnection):
"""A client connection that records when bytes last arrived."""

def __init__(self, *args: Any, **kwargs: Any) -> None:
super().__init__(*args, **kwargs)
self.last_inbound = time.monotonic()

def data_received(self, data: bytes) -> None:
self.last_inbound = time.monotonic()
super().data_received(data)


def connect_kwargs() -> dict[str, Any]:
"""`websockets.connect` options that hand liveness to :func:`keep_alive`."""
return {"ping_interval": None, "create_connection": TrackedConnection}


async def keep_alive(ws: TrackedConnection, *, label: str,
interval: float = KEEPALIVE_INTERVAL,
timeout: float = KEEPALIVE_TIMEOUT) -> None:
"""Run until ``ws`` closes, as its only keepalive."""
try:
while True:
await asyncio.sleep(interval)
if time.monotonic() - ws.last_inbound < interval:
continue
sent = time.monotonic()
pong = await ws.ping()
while not pong.done():
remaining = max(sent, ws.last_inbound) + timeout - time.monotonic()
if remaining <= 0:
logger.warning(
"%s: nothing received for %.0fs after a ping; "
"closing the connection", label, timeout)
await ws.close(code=1011, reason="keepalive timeout")
return
await asyncio.wait({pong}, timeout=remaining)
except (ConnectionClosed, asyncio.CancelledError):
return
Loading
Loading