From 280862a73b3fa3ee4dfc3444f4b978c126e3532c Mon Sep 17 00:00:00 2001 From: William Loo Date: Mon, 2 Nov 2026 02:24:10 -0800 Subject: [PATCH 01/20] feat(website): live sites with @session, and browser/v1's cookies (FD-4683) @website(domains=[...], live=True, allow_hosts=[...]) declares a real site: its server serves no pages, no API and no built-in session or /go route, only health, the data plane and the card, which advertises live and allow_hosts. @session returns the cookies a browser adds to start signed in; website/v1's session checks them, gives a cookie without a domain the site's first, and answers no_session (501) for a live site without one. self.setting(name) reads a credential its deploy passes. agentenv_protocol.website.browser defines urn:agentenv:browser/v1 with cookies/set and cookies/clear, the Cookie shape, and redacted() for what a record keeps of one; cookie errors name an index and field, never a value. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/agentenv_protocol/website/__init__.py | 10 +- .../src/agentenv_protocol/website/browser.py | 93 ++++++++++ .../agentenv_protocol/website/environment.py | 112 ++++++++++-- .../tests/test_website_live.py | 170 ++++++++++++++++++ 4 files changed, 369 insertions(+), 16 deletions(-) create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/website/browser.py create mode 100644 packages/agentenv-protocol/tests/test_website_live.py diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/website/__init__.py index 4910eb51..2e664177 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/__init__.py @@ -4,10 +4,13 @@ package, and uvicorn to serve. The class is optional: a server in any language that meets the contract works, which ``python -m agentenv_protocol.website.conformance `` checks. A site's pages can call its API with ``client/site-client.js``, a dependency-free ES module (with its ``.d.ts``) to copy into them: -``api()``, ``universeNow()`` from ``X-Universe-Now``, ``session()``, ``signIn()`` and ``signOut()``.""" +``api()``, ``universeNow()`` from ``X-Universe-Now``, ``session()``, ``signIn()`` and ``signOut()``. A live site, +``@website(domains=[...], live=True, allow_hosts=[...])``, serves none of its pages and signs a browser in with +``@session``; ``agentenv_protocol.website.replay`` captures a real site and serves the capture as ``ReplayWebsite``.""" from __future__ import annotations +from .browser import BROWSER_EXTENSION_URI, CLEAR_COOKIES, SET_COOKIES, Cookie from .clock import CLOCK_EXTENSION_URI, UNIVERSE_NOW_HEADER, UniverseClock from .environment import ( DEFAULT_HEALTH_PATH, @@ -16,7 +19,9 @@ WEBSITE_EXTENSION_URI, WebsiteDeclaration, WebsiteEnvironment, + host_problem, ready_check, + session, website, ) from .errors import ApiError, BadRequest, Conflict, Forbidden, NotFound, Unauthorized @@ -26,7 +31,8 @@ from .store import Collections __all__ = [ - "WebsiteEnvironment", "WebsiteDeclaration", "website", "api", "ApiContext", "ApiRoute", "ready_check", + "WebsiteEnvironment", "WebsiteDeclaration", "website", "api", "ApiContext", "ApiRoute", "ready_check", "session", + "host_problem", "Cookie", "BROWSER_EXTENSION_URI", "SET_COOKIES", "CLEAR_COOKIES", "ApiError", "BadRequest", "Unauthorized", "Forbidden", "NotFound", "Conflict", "Collections", "UniverseClock", "SessionSigner", "reserved_path", "WebsiteSpec", "SpecError", "NameTable", "load_spec", "from_card", "from_environment", "name_table", diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/browser.py b/packages/agentenv-protocol/src/agentenv_protocol/website/browser.py new file mode 100644 index 00000000..51d9928b --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/browser.py @@ -0,0 +1,93 @@ +"""``urn:agentenv:browser/v1``: what a browser env offers agentenv's steps and graders, never its agent. ``cookies/set`` +adds cookies to the browser, so its pages start signed in to a site, and ``cookies/clear`` removes them, all of them or +those of some domains. A cookie is ``{name, value, domain, path, secure, httpOnly, expires, sameSite}``, ``expires`` +in epoch seconds and absent for a cookie that lasts the browser's session, as a site's ``session`` returns it. + +A cookie's value is a credential: ``redacted`` is what a log or a record keeps of one, and the errors here name a +cookie by its position and field, never by what it holds.""" +from __future__ import annotations + +from typing import Any, Iterable, Literal, Optional + +from pydantic import BaseModel, ConfigDict, Field, ValidationError + +BROWSER_EXTENSION_URI = "urn:agentenv:browser/v1" +SET_COOKIES = "cookies/set" +CLEAR_COOKIES = "cookies/clear" + + +class Cookie(BaseModel): + """One cookie as a browser adds it, for ``domain`` and ``path``; a leading dot on ``domain`` covers its + subdomains too.""" + + model_config = ConfigDict(extra="forbid") + + name: str = Field(min_length=1) + value: str + domain: str = Field(min_length=1) + path: str = "/" + secure: bool = False + httpOnly: bool = False + expires: Optional[float] = None + sameSite: Optional[Literal["Strict", "Lax", "None"]] = None + + def key(self) -> tuple[str, str, str]: + """What a browser keeps one cookie under: a later one with the same key replaces it.""" + return self.name, self.domain.lower(), self.path + + def matches(self, domains: Iterable[str]) -> bool: + """Whether the cookie is for one of ``domains`` or a subdomain of one.""" + return domain_matches(self.domain, domains) + + +def domain_matches(domain: str, domains: Iterable[str]) -> bool: + """Whether a cookie's ``domain`` is one of ``domains`` or a subdomain of one, leading dots aside.""" + host = domain.lstrip(".").lower() + return any(host == d or host.endswith("." + d) for d in (d.lstrip(".").lower() for d in domains)) + + +def cookies(raw: Any, default_domain: Optional[str] = None) -> list[Cookie]: + """``raw``, a list of cookie objects, as Cookies; one without a domain takes ``default_domain``. Raises ValueError + naming each problem by the cookie's index and field, never by a value.""" + if not isinstance(raw, list): + raise ValueError("cookies is a list of cookie objects") + parsed, problems = [], [] + for i, item in enumerate(raw): + if not isinstance(item, dict): + problems.append(f"cookies[{i}] is not an object") + continue + if not item.get("domain") and default_domain: + item = {**item, "domain": default_domain} + try: + parsed.append(Cookie.model_validate(item)) + except ValidationError as e: + problems += [f"cookies[{i}].{'.'.join(map(str, err['loc'])) or 'cookie'}: {err['msg']}" for err in e.errors()] + if problems: + raise ValueError("; ".join(problems)) + return parsed + + +def redacted(given: Iterable[Cookie]) -> list[dict]: + """What a record keeps of each cookie: everything but its value.""" + return [c.model_dump(exclude={"value"}, exclude_none=True) for c in given] + + +def storage_state(given: Iterable[Cookie]) -> list[dict]: + """The cookies as a Playwright storage state file lists them: ``expires`` -1 for a session cookie.""" + return [{"name": c.name, "value": c.value, "domain": c.domain, "path": c.path, + "expires": c.expires if c.expires is not None else -1, "httpOnly": c.httpOnly, "secure": c.secure, + "sameSite": c.sameSite or "Lax"} for c in given] + + +def devtools(given: Iterable[Cookie]) -> list[dict]: + """The cookies as Chrome DevTools' ``Storage.setCookies`` takes them.""" + out = [] + for c in given: + param = {"name": c.name, "value": c.value, "domain": c.domain, "path": c.path, "secure": c.secure, + "httpOnly": c.httpOnly} + if c.expires is not None: + param["expires"] = c.expires + if c.sameSite is not None: + param["sameSite"] = c.sameSite + out.append(param) + return out diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py b/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py index 77fe1315..638f92bf 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py @@ -14,7 +14,13 @@ Two routes are built in, each replaced by a site's own ``@api`` route at its path: ``/api/session``, which a frontend calls to learn who is signed in, signing the acting user in when nobody is (GET), or to sign out (DELETE); and, for a store-backed site, ``/go?sid=``, that session's ``{initial_state, current_state, state_diff}`` as the website state -verifier reads it; a session no request has used yet answers the initial state, unchanged.""" +verifier reads it; a session no request has used yet answers the initial state, unchanged. + +A live site, ``@website(domains=[...], live=True, allow_hosts=[...])``, is a real site on the internet: its server serves +no pages and no API, only its health path, the data plane and the card. Its data hooks reach the site through what is +known of it (its API, or scripted steps), its ``@session`` method returns the cookies a browser adds to start signed in, +and its credentials are settings, read with ``self.setting(name)``. ``allow_hosts`` names the hosts beyond its domains +its pages load from (APIs, CDNs, sign-in), which a computer's network allowlist lets through.""" from __future__ import annotations import inspect @@ -45,7 +51,9 @@ get_data, reset_data, ) +from ..errors import AgentEnvError from ..types import DataPart +from .browser import cookies as parse_cookies from .clock import CLOCK_EXTENSION_URI, UNIVERSE_NOW_HEADER, UniverseClock, clock_patch_script, iso from .database import ModelState, Sessions, Tables, is_hooks, metadata_of from .errors import NotFound @@ -59,14 +67,17 @@ _WEBSITE_ATTR = "_aee_website" _READY_ATTR = "_aee_website_ready" _DEFAULT_HOOK_ATTR = "_aee_default_store_hook" +_SESSION_ATTR = "_aee_website_session" _DEFAULT_HOOKS = {"reset_data": "_store_reset", "add_data": "_store_add", "get_data": "_store_get"} _HEAD = re.compile(rb"]*>", re.IGNORECASE) +_HOST = re.compile(r"(\*\.)?[A-Za-z0-9]([A-Za-z0-9-]*[A-Za-z0-9])?(\.[A-Za-z0-9]([A-Za-z0-9-]*[A-Za-z0-9])?)*") class WebsiteDeclaration(NamedTuple): """What ``@website`` declares: the site's domains, its static folder and seed data file (relative to the class's file), its health path, whether its HTML pages read the universe's time, the models whose database keeps its state - instead of the default store, and the spec it was declared from.""" + instead of the default store, the spec it was declared from, and for a live site, ``live`` and the other hosts its + pages reach.""" domains: tuple[str, ...] = () static: Optional[str] = None @@ -75,34 +86,53 @@ class WebsiteDeclaration(NamedTuple): data: Optional[str] = None models: Any = None spec: Optional[WebsiteSpec] = None + live: bool = False + allow_hosts: tuple[str, ...] = () def website(_cls: type | None = None, *, domains: list[str] | tuple[str, ...] = (), static: str | None = None, health: str = DEFAULT_HEALTH_PATH, clock_patch: bool = False, data: str | None = None, - models: Any = None, spec: str | Path | WebsiteSpec | None = None) -> Any: + models: Any = None, spec: str | Path | WebsiteSpec | None = None, live: bool = False, + allow_hosts: list[str] | tuple[str, ...] = ()) -> Any: """Declare a WebsiteEnvironment's site. ``data`` names a JSON file the default store starts with; ``models`` keeps the site's state in a database instead, an MCP server's (see ``website.database``), and the site's data plane and ``/go`` are that database's. With ``spec``, a website spec file (relative to the class's file) or a loaded one, the site is the spec's, and the class must serve what it declares; the other arguments but - ``models`` are then the spec's to give.""" + ``models`` are then the spec's to give. ``live`` declares a real site at ``domains``, which serves its own pages, + and ``allow_hosts`` the hosts beyond them its pages reach.""" if _cls is not None: raise TypeError("use @website(...) with parentheses") if spec is not None and (domains or static is not None or health != DEFAULT_HEALTH_PATH or clock_patch - or data is not None): + or data is not None or live or allow_hosts): raise TypeError("@website(spec=...) takes the site from its spec; set domains, health, assets, seed and " - "clock there") + "clock there, and a spec's site is a synthetic one") if not health.startswith("/") or health == "/" or reserved_path(health): raise ValueError(f"health is a path of its own under '/', not {health!r}") if models is not None and data is not None: raise ValueError("data= seeds the default store; a site of models= is seeded through data/add") if models is not None and not is_hooks(models): metadata_of(models) + for label, hosts in (("domains", domains), ("allow_hosts", allow_hosts)): + if isinstance(hosts, str): + raise ValueError(f"{label} is a list of host names, not the string {hosts!r}") + if problems := [problem for host in hosts if (problem := host_problem(host))]: + raise ValueError(f"{label}: {problems[0]}") + if live and not domains: + raise ValueError("a live site names its domains") + if live and (static is not None or data is not None or clock_patch): + raise ValueError("a live site serves none of its pages, so it takes no static, data or clock_patch") + if allow_hosts and not live: + raise ValueError("allow_hosts is for a live site: a synthetic site's pages reach only the sites of its env") def deco(cls: type) -> type: if not (isinstance(cls, type) and issubclass(cls, WebsiteEnvironment)): raise TypeError(f"@website declares a WebsiteEnvironment subclass, not {cls!r}") + if live and (served := routes_of(cls)): + raise ValueError(f"{cls.__name__} is a live site, whose API is the real site's: it serves no @api " + f"route, not {served[0].name}") if spec is None: - setattr(cls, _WEBSITE_ATTR, WebsiteDeclaration(tuple(domains), static, health, clock_patch, data, models)) + setattr(cls, _WEBSITE_ATTR, WebsiteDeclaration(tuple(domains), static, health, clock_patch, data, models, None, live, + tuple(allow_hosts))) else: declared = spec if isinstance(spec, WebsiteSpec) else load(cls._site_path(str(spec))) if models is not None and declared.data.seed: @@ -121,6 +151,23 @@ def deco(cls: type) -> type: return deco +def host_problem(host: Any) -> str | None: + """Why ``host`` isn't a host name a site or an allowlist names, None when it is: a name like ``github.com``, or + ``*.github.com`` for its subdomains, with no scheme, port or path.""" + if not isinstance(host, str) or not _HOST.fullmatch(host): + return f"{host!r} is not a host name such as example.com or *.example.com" + return None + + +def session(fn: Any) -> Any: + """Mark the method that signs a browser in: called with the ``user`` to sign in as (None for the site's own + choice), it returns the cookies a browser adds, a list of ``{name, value, domain, path, secure, httpOnly, expires, + sameSite}``, or ``{"cookies": [...], "expires_at": ...}``. A cookie without a domain is for the site's first + domain. website/v1's ``session`` operation calls it; a live site needs one to start signed in.""" + setattr(fn, _SESSION_ATTR, True) + return fn + + def ready_check(fn: Any) -> Any: """Mark a method that says whether the site is usable yet: the health path answers 200 once every one returns a truthy value, and 503 while one returns a falsy one or raises.""" @@ -220,6 +267,11 @@ def now(self) -> Any: """The universe's time, an aware UTC datetime: the gateway's virtual clock once synced, else wall time.""" return self._clock().now() + def setting(self, name: str, default: str | None = None) -> str | None: + """The site's setting ``name``, as its deploy passes it: an environment variable of the server's, which a + website env's ``settings`` set, a secret among them resolved from its ``secret:`` reference at deploy.""" + return os.environ.get(name, default) + def acting_user(self) -> str | None: """Who a browser with no session is signed in as: the store's ``current_user``, or the one a models site's data named, unless a site says else.""" @@ -247,9 +299,9 @@ def create_app(self) -> Starlette: asgi.router.routes.append(Route(declaration.health, self._health, methods=["GET"])) asgi.router.routes.extend(Route(r.path, endpoint(getattr(self, r.fn.__name__), r, self.context), methods=[r.method]) for r in api_routes) - if SESSION_PATH not in taken: + if SESSION_PATH not in taken and not declaration.live: asgi.router.routes.append(Route(SESSION_PATH, self._session, methods=["GET", "DELETE"])) - if self.uses_store and STATE_PATH not in taken: + if self.uses_store and STATE_PATH not in taken and not declaration.live: asgi.router.routes.append(Route(STATE_PATH, self._state, methods=["GET"])) asgi.router.default = _Fallback(root, type(self).app) self.asgi = asgi @@ -270,8 +322,13 @@ def serve(self) -> None: def extension_params(self, uri: str) -> dict: if uri == WEBSITE_EXTENSION_URI: - return {**self.site_spec().to_card_extension(), "session_cookie": self.session_cookie, - "session_path": SESSION_PATH} + declaration = self.declaration() + params = {**self.site_spec().to_card_extension(), "session_cookie": self.session_cookie, + "session_path": SESSION_PATH, "signs_in": self._session_hook() is not None or not declaration.live} + if declaration.live: + params.update(live=True, allow_hosts=list(declaration.allow_hosts)) + del params["session_cookie"], params["session_path"] + return params return super().extension_params(uri) def _build_card(self) -> Any: @@ -303,8 +360,23 @@ async def _website_openapi(self) -> dict: @extension(WEBSITE_EXTENSION_URI, op="session") async def _website_session(self, user: Optional[str] = None) -> dict: - cookie = self._session_signer().cookie(user) - return {"user": user, "cookies": [cookie], "expires_at": expires_at(cookie["expires"])} + hook = self._session_hook() + if hook is None: + if self.declaration().live: + raise AgentEnvError("no_session", f"{type(self).__name__} is a live site with no @session method, so " + "it can't sign a browser in", status=501) + cookie = self._session_signer().cookie(user) + return {"user": user, "cookies": [cookie], "expires_at": expires_at(cookie["expires"])} + result = await _maybe_await(hook(user)) + given, until = (result.get("cookies"), result.get("expires_at")) if isinstance(result, dict) else (result, None) + domains = self.declaration().domains + try: + signed = parse_cookies(given, default_domain=domains[0] if domains else None) + except ValueError as e: + raise AgentEnvError("invalid_session", f"{hook.__name__} returned {e}", status=500) from None + if until is None and (ends := [c.expires for c in signed if c.expires is not None]): + until = expires_at(int(min(ends))) + return {"user": user, "cookies": [c.model_dump(exclude_none=True) for c in signed], "expires_at": until} @reset_data async def _store_reset(self) -> None: @@ -341,6 +413,12 @@ async def _session(self, request: Request) -> Response: async def _state(self, request: Request) -> Response: return JSONResponse(await self._models.state() if self._models is not None else self.store.state()) + def _session_hook(self) -> Any: + hooks = [member for member, _marker in _marked_members(self, _SESSION_ATTR)] + if len(hooks) > 1: + raise TypeError(f"{type(self).__name__} marks {len(hooks)} methods @session; a site signs in one way") + return hooks[0] if hooks else None + def _clock(self) -> UniverseClock: if not hasattr(self, "clock"): raise TypeError(f"{type(self).__qualname__}.__init__ must call super().__init__()") @@ -357,8 +435,14 @@ def _site_path(cls, path: str) -> Path: return given if given.is_absolute() else Path(inspect.getfile(cls)).resolve().parent / given +def default_store_hook(fn: Any) -> Any: + """Mark a data hook as one a subclass's own hook of that operation replaces, as it replaces the default store's.""" + setattr(fn, _DEFAULT_HOOK_ATTR, True) + return fn + + for _hook in _DEFAULT_HOOKS.values(): - setattr(getattr(WebsiteEnvironment, _hook), _DEFAULT_HOOK_ATTR, True) + default_store_hook(getattr(WebsiteEnvironment, _hook)) async def _ready(check: Any) -> bool: diff --git a/packages/agentenv-protocol/tests/test_website_live.py b/packages/agentenv-protocol/tests/test_website_live.py new file mode 100644 index 00000000..62461bad --- /dev/null +++ b/packages/agentenv-protocol/tests/test_website_live.py @@ -0,0 +1,170 @@ +"""Live websites and browser/v1's cookies: a live site serves none of its pages, declares live and allow_hosts on its +card, and its @session method's cookies come back from website/v1's session checked and given its domain; cookies are +checked without naming their values, and kept in a record without them.""" + +from __future__ import annotations + +import pytest +from starlette.testclient import TestClient + +from agentenv_protocol import DataPart, add_data, get_data, reset_data +from agentenv_protocol.client import find_extension +from agentenv_protocol.website import (WEBSITE_EXTENSION_URI, Cookie, WebsiteEnvironment, api, ready_check, session, + website) +from agentenv_protocol.website.browser import cookies, devtools, redacted, storage_state + +SESSION = "/agentenv/ext/website/session" + + +@website(domains=["github.com"], live=True, allow_hosts=["*.githubusercontent.com", "api.github.com"]) +class GitHub(WebsiteEnvironment): + """A live site: its data hooks would call GitHub's API with the token setting.""" + + def __init__(self) -> None: + super().__init__() + self.repos: list = [] + + @ready_check + def has_token(self) -> bool: + return bool(self.setting("GITHUB_TOKEN")) + + @session + async def sign_in(self, user: str | None) -> list[dict]: + return [{"name": "user_session", "value": f"{self.setting('GITHUB_TOKEN')}:{user}", "secure": True, + "httpOnly": True, "expires": 1_900_000_000}, + {"name": "logged_in", "value": "yes", "domain": ".github.com"}] + + @reset_data + async def reset(self) -> None: + self.repos = [] + + @add_data + async def load(self, parts: list) -> None: + self.repos += [repo for part in parts for repo in part.data["repos"]] + + @get_data + async def export(self) -> list: + return [DataPart(data={"repos": self.repos})] + + +@pytest.fixture +def client(monkeypatch) -> TestClient: + monkeypatch.setenv("GITHUB_TOKEN", "ghp_secret") + return TestClient(GitHub().create_app()) + + +def test_a_live_site_serves_no_pages_only_its_health_data_plane_and_card(client): + for path in ("/", "/login", "/api/session", "/go"): + assert client.get(path).status_code == 404, path + assert client.get("/api/health").status_code == 200 + card = client.get("/.well-known/agent-env.json").json() + params = find_extension(card, WEBSITE_EXTENSION_URI)["params"] + assert params["live"] is True and params["signs_in"] is True + assert params["domains"] == ["github.com"] + assert params["allow_hosts"] == ["*.githubusercontent.com", "api.github.com"] + assert "session_cookie" not in params + + +def test_a_live_sites_health_says_whether_its_account_works(monkeypatch): + monkeypatch.delenv("GITHUB_TOKEN", raising=False) + assert TestClient(GitHub().create_app()).get("/api/health").status_code == 503 + + +def test_the_session_operation_returns_the_session_methods_cookies_for_the_sites_domain(client): + answer = client.post(SESSION, json={"user": "octocat"}).json() + assert answer["user"] == "octocat" + assert answer["cookies"] == [ + {"name": "user_session", "value": "ghp_secret:octocat", "domain": "github.com", "path": "/", "secure": True, + "httpOnly": True, "expires": 1_900_000_000}, + {"name": "logged_in", "value": "yes", "domain": ".github.com", "path": "/", "secure": False, + "httpOnly": False}] + assert answer["expires_at"] == "2030-03-17T17:46:40Z" + + +def test_a_live_site_without_a_session_method_says_so(): + @website(domains=["example.com"], live=True) + class Plain(WebsiteEnvironment): + pass + + client = TestClient(Plain().create_app()) + params = find_extension(client.get("/.well-known/agent-env.json").json(), WEBSITE_EXTENSION_URI)["params"] + assert params["signs_in"] is False + answer = client.post(SESSION, json={}) + assert answer.status_code == 501 and answer.json()["error"]["code"] == "no_session" + + +def test_a_session_method_returning_a_bad_cookie_is_refused_without_its_value(): + @website(domains=["example.com"], live=True) + class Broken(WebsiteEnvironment): + @session + def sign_in(self, user): + return {"cookies": [{"name": "", "value": "hunter2"}], "expires_at": None} + + answer = TestClient(Broken().create_app()).post(SESSION, json={}) + assert answer.status_code == 500 and "cookies[0].name" in answer.text and "hunter2" not in answer.text + + +def test_a_synthetic_site_keeps_its_signed_session_and_may_override_it(): + @website(domains=["shop.example"]) + class Shop(WebsiteEnvironment): + pass + + @website(domains=["persona.example"]) + class Persona(WebsiteEnvironment): + @session + def as_persona(self, user): + return [{"name": "persona", "value": user or "guest"}] + + signed = TestClient(Shop().create_app()).post(SESSION, json={"user": "ann"}).json() + assert signed["cookies"][0]["name"] == "agentenv_session" + persona = TestClient(Persona().create_app()).post(SESSION, json={}).json() + assert persona["cookies"] == [{"name": "persona", "value": "guest", "domain": "persona.example", "path": "/", + "secure": False, "httpOnly": False}] + + +@pytest.mark.parametrize("declared, problem", [ + (dict(live=True), "names its domains"), + (dict(domains=["a.example"], live=True, static="dist"), "takes no static"), + (dict(domains=["a.example"], allow_hosts=["cdn.example"]), "allow_hosts is for a live site"), + (dict(domains=["https://a.example"]), "not a host name"), + (dict(domains=["a.example"], live=True, allow_hosts=["*"]), "not a host name"), + (dict(domains="a.example"), "not the string"), +]) +def test_a_live_declaration_is_checked(declared, problem): + with pytest.raises(ValueError, match=problem): + website(**declared) + + +def test_a_live_site_serves_no_api(): + with pytest.raises(ValueError, match="serves no @api route"): + @website(domains=["a.example"], live=True) + class WithApi(WebsiteEnvironment): + @api.get("/api/x") + def x(self) -> dict: + return {} + + +def test_cookies_are_checked_by_index_and_field_never_by_value(): + with pytest.raises(ValueError) as e: + cookies([{"name": "a", "value": "s3cret", "domain": "a.example", "sameSite": "sometimes"}, "x", + {"name": "b", "value": "s3cret"}]) + message = str(e.value) + assert "cookies[0].sameSite" in message and "cookies[1] is not an object" in message + assert "cookies[2].domain" in message and "s3cret" not in message + with pytest.raises(ValueError, match="list"): + cookies({"name": "a"}) + + +def test_a_cookie_is_kept_without_its_value_and_given_to_a_browser_in_its_own_shape(): + given = cookies([{"name": "sid", "value": "v", "secure": True}, {"name": "t", "value": "w", "expires": 5, + "sameSite": "Strict"}], + default_domain="a.example") + assert redacted(given) == [{"name": "sid", "domain": "a.example", "path": "/", "secure": True, "httpOnly": False}, + {"name": "t", "domain": "a.example", "path": "/", "secure": False, "httpOnly": False, + "expires": 5.0, "sameSite": "Strict"}] + assert storage_state(given)[0] == {"name": "sid", "value": "v", "domain": "a.example", "path": "/", + "expires": -1, "httpOnly": False, "secure": True, "sameSite": "Lax"} + assert "expires" not in devtools(given)[0] and devtools(given)[1]["sameSite"] == "Strict" + assert Cookie(name="a", value="", domain=".A.example").matches(["a.example"]) + assert Cookie(name="a", value="", domain="sub.a.example").matches(["a.example"]) + assert not Cookie(name="a", value="", domain="nota.example").matches(["a.example"]) From c267f23e4b110beeeef7943006127a92c5406236 Mon Sep 17 00:00:00 2001 From: William Loo Date: Mon, 2 Nov 2026 02:24:11 -0800 Subject: [PATCH 02/20] feat(website): capture a site to WARC and serve it as ReplayWebsite (FD-4683) python -m agentenv_protocol.website.replay capture URL... --out capture.warc.gz writes WARC 1.1 with the standard library and httpx: the start URLs, same-site links up to --depth, each page's assets, and redirects as answered. ReplayWebsite serves a capture for any host, matched by method, path and normalised query, 404 for a miss; @api routes win over the capture, the default store holds their state for the data plane, and data/add of a .warc(.gz) file part serves that capture until data/reset. The example replays a captured docs site with a synthetic sign-up. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../examples/website/replay/Dockerfile | 14 + .../website/replay/Dockerfile.dockerignore | 6 + .../examples/website/replay/README.md | 73 +++ .../examples/website/replay/capture.warc.gz | Bin 0 -> 4147 bytes .../examples/website/replay/env.toml | 3 + .../examples/website/replay/site.py | 38 ++ .../examples/website/replay/source/app.js | 8 + .../website/replay/source/guide/advanced.html | 5 + .../website/replay/source/guide/install.html | 9 + .../website/replay/source/guide/usage.html | 9 + .../examples/website/replay/source/index.html | 23 + .../examples/website/replay/source/style.css | 1 + .../src/agentenv_protocol/website/replay.py | 480 ++++++++++++++++++ .../tests/test_website_replay.py | 276 ++++++++++ 14 files changed, 945 insertions(+) create mode 100644 packages/agentenv-protocol/examples/website/replay/Dockerfile create mode 100644 packages/agentenv-protocol/examples/website/replay/Dockerfile.dockerignore create mode 100644 packages/agentenv-protocol/examples/website/replay/README.md create mode 100644 packages/agentenv-protocol/examples/website/replay/capture.warc.gz create mode 100644 packages/agentenv-protocol/examples/website/replay/env.toml create mode 100644 packages/agentenv-protocol/examples/website/replay/site.py create mode 100644 packages/agentenv-protocol/examples/website/replay/source/app.js create mode 100644 packages/agentenv-protocol/examples/website/replay/source/guide/advanced.html create mode 100644 packages/agentenv-protocol/examples/website/replay/source/guide/install.html create mode 100644 packages/agentenv-protocol/examples/website/replay/source/guide/usage.html create mode 100644 packages/agentenv-protocol/examples/website/replay/source/index.html create mode 100644 packages/agentenv-protocol/examples/website/replay/source/style.css create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/website/replay.py create mode 100644 packages/agentenv-protocol/tests/test_website_replay.py diff --git a/packages/agentenv-protocol/examples/website/replay/Dockerfile b/packages/agentenv-protocol/examples/website/replay/Dockerfile new file mode 100644 index 00000000..c641d704 --- /dev/null +++ b/packages/agentenv-protocol/examples/website/replay/Dockerfile @@ -0,0 +1,14 @@ +ARG REPLAY=packages/agentenv-protocol/examples/website/replay + +FROM mirror.gcr.io/library/python:3.12-slim@sha256:423ed6ab25b1921a477529254bfeeabf5855151dc2c3141699a1bfc852199fbf +ARG REPLAY +WORKDIR /app +RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/* +COPY packages/agentenv-protocol /tmp/agentenv-protocol +RUN pip install --no-cache-dir /tmp/agentenv-protocol "uvicorn>=0.30" && rm -rf /tmp/agentenv-protocol +COPY ${REPLAY}/site.py ${REPLAY}/capture.warc.gz /app/ +ENV PORT=8000 +EXPOSE 8000 +RUN useradd --create-home --uid 10001 appuser +USER appuser +CMD ["python", "site.py"] diff --git a/packages/agentenv-protocol/examples/website/replay/Dockerfile.dockerignore b/packages/agentenv-protocol/examples/website/replay/Dockerfile.dockerignore new file mode 100644 index 00000000..10ae9270 --- /dev/null +++ b/packages/agentenv-protocol/examples/website/replay/Dockerfile.dockerignore @@ -0,0 +1,6 @@ +**/node_modules +**/dist +**/__pycache__ +**/.pytest_cache +**/.venv +.git diff --git a/packages/agentenv-protocol/examples/website/replay/README.md b/packages/agentenv-protocol/examples/website/replay/README.md new file mode 100644 index 00000000..0cd4317e --- /dev/null +++ b/packages/agentenv-protocol/examples/website/replay/README.md @@ -0,0 +1,73 @@ +# Widget Docs: a captured site, replayed + +Widget Docs is a small docs site captured to a WARC file and served by +`agentenv_protocol.website.replay.ReplayWebsite`. The captured pages are served exactly as they were captured, for any +host. The newsletter sign-up, which a capture can't answer, is written as `@api` routes on top. + +| File | What it is | +| --- | --- | +| `source/` | The site that was captured: a home page, two guide pages, a stylesheet and a script. | +| `capture.warc.gz` | The capture of `source/` at `--depth 1`: `/`, `/guide/install.html`, `/guide/usage.html?utm_source=home`, `/style.css` and `/app.js`. `/guide/advanced.html` is two links from home, so it isn't in the capture. | +| `Dockerfile`, `env.toml` | The server image, built from the repo root as the Notes example's is, and the website env as a bundle entry. | +| `site.py` | The server: `WidgetDocs(ReplayWebsite)` with `capture = "capture.warc.gz"`, plus `POST /api/subscribe` and `GET /api/subscriptions`. | + +## Capture a site + +```bash +python -m agentenv_protocol.website.replay capture https://docs.example.com/ --out capture.warc.gz --depth 2 +``` + +The tool needs only this package and `httpx`. It fetches each start URL. It then fetches the same-site pages those URLs +link to, up to `--depth` links away, and the assets (stylesheets, scripts, images) each captured page loads. Every +answer goes into a WARC 1.1 file, gzipped one record at a time, which pywb and warcio also read. Notes: + +- Only GETs are captured. +- Redirects are kept as they were answered, and their targets are captured too. +- Bodies are stored decoded, so the stored headers leave out `Content-Encoding` and `Transfer-Encoding`. +- A host is the same site as a start URL when it is that URL's host (without `www.`) or a subdomain of it. +- Request records name only the method, the target and `Host`, so no credential the capturing client sent ends up in + the file. +- `--max-requests` (500 by default) caps a capture, and `--no-assets` leaves out what pages load. + +To capture `source/` again: + +```bash +python -m http.server -d source 8765 & +python -m agentenv_protocol.website.replay capture http://127.0.0.1:8765/ --out capture.warc.gz --depth 1 +``` + +## How a request is matched + +A request is matched by its method (HEAD as GET), its path and its query, never by its host or headers. The query is +normalised before matching: + +- Its parameters are sorted. +- Blank parameters are kept. +- The `utm_*`, `fbclid`, `gclid`, `msclkid` and `_` parameters are dropped. Set `ignore_query` on the class to change + this list. + +Each path segment is percent-decoded and then encoded again, so `%7E` and `~` match. A request the capture doesn't +hold gets a 404. If a capture holds one request twice, the first answer is served. + +When the capture's answers are served: + +- `Set-Cookie` loses its `Domain` attribute, so the cookie works on the replay's own host. +- A `Location` that points at a captured host is made relative. +- `Strict-Transport-Security` and `Alt-Svc` are dropped. + +## Synthetic routes on top + +`@api` routes win over the capture. Here `POST /api/subscribe` keeps sign-ups in the default store. The data plane +exports that store, `data/reset` empties it, and `/go` shows a run's changes for the website state verifier. The +capture itself isn't task state. A `data/add` file part named `*.warc` or `*.warc.gz` (or typed `application/warc`) +replaces the declared capture until the next `data/reset`. This lets a large capture be loaded at deploy time instead +of being built into the image. + +## Run and test it + +```bash +PYTHONPATH=/packages/agentenv-protocol/src python site.py # serves on PORT, 8000 by default +python -m agentenv_protocol.website.replay serve capture.warc.gz --domain docs.widgets.test # a bare replay +python -m pytest packages/agentenv-protocol/tests/test_website_replay.py +docker build -f packages/agentenv-protocol/examples/website/replay/Dockerfile -t widget-docs . # from the repo root +``` diff --git a/packages/agentenv-protocol/examples/website/replay/capture.warc.gz b/packages/agentenv-protocol/examples/website/replay/capture.warc.gz new file mode 100644 index 0000000000000000000000000000000000000000..4d1172587228c02307ab0f57b7769cc235217415 GIT binary patch literal 4147 zcmY+`oYV*bhomwDN9XA79;GxRrDUXpq;yM6 zLK^(}ym_v#|H8RG_c@03!8#6dL7OBbx-hz$zR7L#8^1{Ixs{WU22buSRLM zWdQE_a@My*tI$B+AMORWn;Phv*?VHkaMVh6$>I{jWm7;lzCGDNy`WrrDcXYVBj>)5 z;UI%tR7Be6q#9oxX)xpUR$o2kr5mjgs8Sf^vcXQ z{_^u(|`F0{h8 zihZtea?g}eXf553)09v9;o8yf$j+jC2DUT2P zIH=pl#|VTN0cp^1cZL zOF}!t7tn{+E_^bo0vIx;Qx%ZkR8%p z1Vq)wSwVjiYeCW!>3T{?-s!)dxnEl^HV|@yT$d6l+_jy_oJq~>Qu&gRtPi&(EQ(<;JClGRHT zg&^3fjc!os!E%Wrp%EbCS6~zgI<# zC=ECbDr0z3N0g>~X^5QMEqE%Wwfz_svT&OU^T3vd!U*TZQRr!~Z=?>Wta*{7G7!Ef zeP9y`(!9s|A|?r3U30Q7)NHPxrgGzx74Q~GObi*WxBf?)N34$P9FRH~PkO~I#qF6U zJ}}IvZb(9X?-QwMGv5Fh_Nd3o>ebE1p4x%-?5kK7HsBUdY5{kf>W4EIO{^LziYJ7#`2gg(3uV8A{bk0Ws0wO-IE!E=J9AkX_ zrbCqOpLXj3I$dqzue9U}Je+n%Kb}hlO9~CPkBxtKL`y8D*0X)H@y94yNPbhpv4i9D zCR&MqJcrAV$wL_L8RgWEiv!Z1+1jRC`u+Yc=;mG8kx0N6TOG107;Y^?y4v}K4#xi! z9WrTVpcpEC&N18`0-i$se67Bb82i%cDmv3f#Y!9u89A#N7h+3hFI89IQR)q~F^N5` zlP>DvIBb}W2%A8f#`ugJDNh$Xuf|;iQxpywNL^=%{+qco`fuin)T*b1f4Z*xckjf+ z`y3MjM09c@7|n?0MOI6l-@KkmV!gi9g%l@1Y8m39_Y-O9EMr@mMa3bivB4iNRg0)4UXQ z8=$Xwkv4gj8DGM3md}jd1`!v9=1}n`ZpzB>k%j~mQ~5S=UbdBTLFVglGdLQ#C_vEU z{K$*Sjv7~OjJRo10R352ec5WA1%O=ll60q$N&@GMP^#?nk@D+L$G$@e22f z>T*>Tl=(BbTa^>e)RCze!5*Zy@^B+$G%HBNv=DLSd{^w4;oFCqrv(MicOhEPT59Fy zls&Vmsbl*PqP?ic%3v$~4_lwlT@0$;PA6eLo4FcBd|N3{ayK_$CldQUM(C!({`R@) z?2X~0#WRvUhJgg}q)D==rbT=<*$eSiw6Bjl7x4v&EWom?sO*(^h>~*Fh_T7>W0+ot z)l_W$pwRa=p;ynJm5mFh4qD;97Ps!Kje|I$d{@e{eTYmGwqp)JSsu~3C0mji?56N; z#cg7b8#RotzaX*w#xEaY-iHeWlyS+*932%?*2`5bk=8-dHj=2C3=7(p%31CcjtSF| zr6y8+q8xQCnx082B3VJy54QK2(bKdF!al>C%HB9TAO}sDu8LizK09tiYc@nC{Ruhf zWgWiqG1t4^g~$kpXtdDi2G7^clj`*CzS6$FxuQ78a`4r5eR<37f?r^+%YiumyWH;W zp1?2Jiwj1Cn)rV(1fd6X|L%O%)5Ht9n*hzLzI++xK$GWUpA`+k0GU1=LAuf~l(pp9 z`yXR`-K0akhl8zeH}<(b%TiQdKbs3-WzOaowL0ipU6c4ACmvM+Yc^V&oKx~yH2j&P zzgCS=6p{{FLu(lWTm3Vv?v*4M_WqvlfmN6T*Aw6!Y+wr@W?Grzz;hnsjH0#hGF3I% zRAc!XxyeA&(^-)4z@-xsd0oH}X)?u)Z{1lGT6GQ^m^}7In55<&Pp&NG0NZ_@40#yA z0R@`STNX)Qlmw*v5)i{Bx9jKw9&OD-eTS7|RzHi??F`aqH@avjNK4cvDNvZ-8hI>L zX(+&YG~;`6Y>fU8iz^bgXscOF!HFDhZp4q4wh>@waAW!Z(m;=Dbpe@!@l?nVd4Gp^ z_nk@5tr^7--LK`;Edyz_RJEE8V@-T*%?^9?BsiV8_@iw6z>P}-HNqe1iuVNWIVvgf zgDMgkR@ZfHu*+{+^OmsA%+97b2VFtFVOmbzP4Qr+IPw%uH)?+6B1G@1P4MU*+b65Z zLdNG-q4%E)^n|WFiOM@2FrH-Y>};4%`aqv+z#~w(FCIo$)1738N*y_zMe};acKC7B~m(G2r2C zqqG4t${Sb8v_Az`fb8YFS2;dp&y@DVf$&6eIZA4MQdz>(W^Ne#jMrxos1#LlkzR!E zPiJ5&EJ#L3H*+_5ENB6@t*f@Pj59J*KB5%^_Je(>f9HnS>8;Isd?C6hv zJj&CA4mg zvO}*ym$&XG)}xl@j^v;+2ie~kx9i8<5gl9?wV6jR=-P`LiK^ah{m{#y4pCxjk=H)n z4EBVyU7NjqWFqvf{oZY>Fjw{E3oeg;(hGvjxy&lkBOBH}utAXo6qXd-k99CI$w@L4 zdT&@tUqD-k#e1BbyaG)0ZdzAqr1T0gvt(79ZW|d-E`J@|$VZ~4J??I8Z0bba5uWe> zNbjRgjSa#tS0ktwToVRmIKI^Qu@_R`KfF^N@IflnT0tkS@9UHFRW#i+f+2g4P$GqQ zLO?6{S)!`uU3F-N-StgOmvPZ;`z!qEDs@ut1eC28hoBz~D|M4%4=S%jC3hpg@6<;Lfqw?&PHdM2t9+I24}#hOBlV{cnUX$!Q=&o^44}5AXtg@k#4iR3 zzuBE-DTCikr2ukIQI9EB+IHTEZ4|{l4P|Vss5fHc6!@~4{=*~u8e|P+zq0LQnR{R^ ztj940edv5uy<#PS{=+1$W!Z6a`%MD131N$<44{YDWPY7}Zl~C8P%dj>a8~KzF_v)k z%1t}fDVKscaVV+N`zZ_?d&~Wkj@R-rbrvVVCF?}LpJdMAkaH_3pA+6-NtF4@$jCO5*oYfIxM9ak;x?s%Kgaa1nw84YA@@Z?AX< z-BG}UXf61!wbr`D{O|@kSfSYV(^Htyfgq}C9Os-9#puSnp|gp5@NWOazO!ci_n&Ko z32bNH&t7{0LypP*MR>RP!_TGpyEhD>`jzZ3D!~G;mD+rh8b9J7CvG2m;{QH2=4jks zJM_VS?Vv1S9Wu?$8io+**mou9GL2T9LB$f292hO~GA+SCPnvb$CVAwoxA3D}@<2S- z4~pYc#qg+O>2aH><~Wdo(WgaGPq~=Pw9R@*hIh1}QfL!TXqhY-2&~Lo5H@t6Au@K& z{z-L58IUYOPH5)%C9%(txz*eR@w%_msp;je&0bC?xKDh^B~bsiD#Hqp1V4^hJkCu0 z8Y@eMZ7&LUQ&;UKq+kkVdYP<^3vetKsbCz8d}A=|Ict#!g#)&7~7n~^ dict: + """Sign an email address up for the newsletter.""" + if "@" not in body.email: + raise BadRequest("that is not an email address") + if self.store.find("subscriptions", email=body.email): + raise Conflict(f"{body.email} is subscribed already") + return self.store.insert("subscriptions", {"email": body.email, "at": self.now().isoformat()}) + + @api.get("/api/subscriptions") + def subscriptions(self) -> list[dict]: + """Everyone signed up.""" + return self.store["subscriptions"] + + +if __name__ == "__main__": + WidgetDocs().serve() diff --git a/packages/agentenv-protocol/examples/website/replay/source/app.js b/packages/agentenv-protocol/examples/website/replay/source/app.js new file mode 100644 index 00000000..654537e8 --- /dev/null +++ b/packages/agentenv-protocol/examples/website/replay/source/app.js @@ -0,0 +1,8 @@ +document.getElementById("subscribe").addEventListener("submit", async (event) => { + event.preventDefault(); + const email = new FormData(event.target).get("email"); + const answer = await fetch("api/subscribe", { + method: "POST", headers: {"Content-Type": "application/json"}, body: JSON.stringify({email}), + }); + document.getElementById("subscribed").textContent = answer.ok ? `Subscribed ${email}` : "Try again"; +}); diff --git a/packages/agentenv-protocol/examples/website/replay/source/guide/advanced.html b/packages/agentenv-protocol/examples/website/replay/source/guide/advanced.html new file mode 100644 index 00000000..9646ac17 --- /dev/null +++ b/packages/agentenv-protocol/examples/website/replay/source/guide/advanced.html @@ -0,0 +1,5 @@ + + +Advanced +

Advanced

Two links from the home page, so a capture at --depth 1 leaves it out.

+ diff --git a/packages/agentenv-protocol/examples/website/replay/source/guide/install.html b/packages/agentenv-protocol/examples/website/replay/source/guide/install.html new file mode 100644 index 00000000..a67cb046 --- /dev/null +++ b/packages/agentenv-protocol/examples/website/replay/source/guide/install.html @@ -0,0 +1,9 @@ + + +Install + +

Install

+ 
pip install widgets
+

Next: usage · Home

+ + diff --git a/packages/agentenv-protocol/examples/website/replay/source/guide/usage.html b/packages/agentenv-protocol/examples/website/replay/source/guide/usage.html new file mode 100644 index 00000000..4c8d47b6 --- /dev/null +++ b/packages/agentenv-protocol/examples/website/replay/source/guide/usage.html @@ -0,0 +1,9 @@ + + +Usage + +

Usage

+ 
import widgets; widgets.spin()
+

Advanced · Home

+ + diff --git a/packages/agentenv-protocol/examples/website/replay/source/index.html b/packages/agentenv-protocol/examples/website/replay/source/index.html new file mode 100644 index 00000000..0d3a8bdf --- /dev/null +++ b/packages/agentenv-protocol/examples/website/replay/source/index.html @@ -0,0 +1,23 @@ + + + + + Widget Docs + + + +

Widget Docs

+

Everything about widgets.

+ +
+ + +
+

+ + + diff --git a/packages/agentenv-protocol/examples/website/replay/source/style.css b/packages/agentenv-protocol/examples/website/replay/source/style.css new file mode 100644 index 00000000..8f7eb819 --- /dev/null +++ b/packages/agentenv-protocol/examples/website/replay/source/style.css @@ -0,0 +1 @@ +body { font-family: sans-serif; max-width: 40rem; margin: 2rem auto; } diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/replay.py b/packages/agentenv-protocol/src/agentenv_protocol/website/replay.py new file mode 100644 index 00000000..f2c54e07 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/replay.py @@ -0,0 +1,480 @@ +"""Capture a real site and serve the capture as a website env. + +``python -m agentenv_protocol.website.replay capture URL... --out capture.warc.gz`` fetches each start URL, then the +same-site pages it links to up to ``--depth`` links away and the assets each captured page loads, and writes every +answer to a WARC 1.1 file, gzipped a record at a time as other WARC tools write them. Only GETs are captured, redirects +are kept as they were answered (and followed), and bodies are stored decoded: an answer's ``Content-Encoding`` and +``Transfer-Encoding`` are dropped and its ``Content-Length`` is the decoded body's. A host is the same site as a start +URL when it is that URL's host, without any ``www.``, or a subdomain of it. The request records name no header but +``Host``, so a capture holds no credential the capturing client sent. + +``ReplayWebsite`` serves a capture for any host name: each request is matched by its method (HEAD as GET), its path and +its query as ``request_key`` normalises them, and a request the capture doesn't hold answers 404. ``@api`` routes of a +subclass are layered on top and win over the capture, and the default store holds their state, which the data plane +exports. ``data/add`` of a file part named ``*.warc`` or ``*.warc.gz`` (or typed ``application/warc``) serves that +capture instead of the declared one until ``data/reset``. ``python -m agentenv_protocol.website.replay serve +capture.warc.gz --domain example.com`` serves one on ``PORT``. + +The normalisation, for the capture and for each request alike: the method is upper-cased; the path is ``/`` when +empty and each of its segments is percent-decoded and encoded again the one way, so ``%7E`` and ``~`` match, while an +encoded ``/`` stays inside its segment; the query's parameters are sorted by name and then value, blank ones kept, +and those ``ignore_query`` names (the ``utm_*`` tracking parameters, ``fbclid``, ``gclid``, ``msclkid`` and the +cache-busting ``_`` by default) are dropped. The host, the fragment and every header are not part of the match. When a +capture holds one request twice, its first answer is served.""" +from __future__ import annotations + +import argparse +import base64 +import dataclasses +import datetime +import gzip +import io +import sys +import uuid +from collections import deque +from html.parser import HTMLParser +from pathlib import Path +from typing import Any, BinaryIO, ClassVar, Iterable, Iterator, Optional +from urllib.parse import parse_qsl, quote, unquote, urlencode, urljoin, urlsplit, urlunsplit + +from starlette.applications import Starlette +from starlette.responses import Response +from starlette.routing import Route + +from ..agent_env_environment import add_data, reset_data +from ..types import FilePart +from .environment import SESSION_PATH, STATE_PATH, WebsiteEnvironment, default_store_hook, website +from .errors import NotFound +from .routes import error_response, reserved_path + +DEFAULT_IGNORED_QUERY = ("utm_*", "fbclid", "gclid", "msclkid", "_") +WARC_VERSION = "WARC/1.1" +WARC_TYPES = ("application/warc", "application/warc+gzip") +_HOP_BY_HOP = {"connection", "keep-alive", "proxy-authenticate", "proxy-authorization", "te", "trailer", + "transfer-encoding", "upgrade"} +_NOT_REPLAYED = _HOP_BY_HOP | {"content-length", "strict-transport-security", "alt-svc"} +_NOT_STORED = _HOP_BY_HOP | {"content-encoding", "content-length"} +_SEGMENT_SAFE = "!$&'()*+,;=:@-._~" + + +def request_key(method: str, path: str, query: str = "", ignore_query: Iterable[str] = DEFAULT_IGNORED_QUERY, + ) -> tuple[str, str, str]: + """``(method, path, query)`` as a capture and a request are matched by them.""" + method = method.upper() + if method == "HEAD": + method = "GET" + segments = (path or "/").split("/") + path = "/".join(quote(unquote(segment), safe=_SEGMENT_SAFE) for segment in segments) or "/" + if not path.startswith("/"): + path = "/" + path + ignored = tuple(ignore_query) + params = sorted((k, v) for k, v in parse_qsl(query, keep_blank_values=True) if not _ignored(k, ignored)) + return method, path, urlencode(params, quote_via=quote) + + +def _ignored(name: str, ignored: tuple[str, ...]) -> bool: + return any(name.startswith(rule[:-1]) if rule.endswith("*") else name == rule for rule in ignored) + + +@dataclasses.dataclass(frozen=True) +class Entry: + """One captured answer: the URL it answered, its status, its headers in order, and its body.""" + + method: str + url: str + status: int + headers: tuple[tuple[str, str], ...] + body: bytes + + +class Archive: + """A capture's answers by ``request_key``, and the hosts they came from.""" + + def __init__(self, entries: Iterable[Entry] = (), ignore_query: Iterable[str] = DEFAULT_IGNORED_QUERY) -> None: + self.ignore_query = tuple(ignore_query) + self.entries: dict[tuple[str, str, str], Entry] = {} + self.hosts: set[str] = set() + for entry in entries: + self.add(entry) + + def __len__(self) -> int: + return len(self.entries) + + def add(self, entry: Entry) -> bool: + """Keep ``entry`` unless the archive holds an answer to its request already; whether it was kept.""" + url = urlsplit(entry.url) + key = request_key(entry.method, url.path, url.query, self.ignore_query) + if key in self.entries: + return False + self.entries[key] = entry + if url.hostname: + self.hosts.add(url.hostname.lower()) + return True + + def find(self, method: str, path: str, query: str = "") -> Optional[Entry]: + return self.entries.get(request_key(method, path, query, self.ignore_query)) + + def holds(self, path: str) -> bool: + """Whether any captured answer is for ``path``, whatever its method and query.""" + target = request_key("GET", path)[1] + return any(key[1] == target for key in self.entries) + + @classmethod + def read(cls, source: str | Path | bytes, ignore_query: Iterable[str] = DEFAULT_IGNORED_QUERY) -> Archive: + """The answers of a WARC file, gzipped or not, given as its path or its bytes.""" + raw = source if isinstance(source, bytes) else Path(source).read_bytes() + if raw[:2] == b"\x1f\x8b": + raw = gzip.decompress(raw) + return cls(_entries(io.BytesIO(raw)), ignore_query) + + +def _records(stream: BinaryIO) -> Iterator[tuple[dict[str, str], bytes]]: + """Each WARC record's headers, by lower-cased name, and its content block.""" + while True: + line = stream.readline() + if not line: + return + if not line.strip(): + continue + if not line.startswith(b"WARC/"): + raise ValueError(f"not a WARC record: {line[:40]!r}") + headers: dict[str, str] = {} + while (line := stream.readline()).strip(): + name, _, value = line.decode("utf-8").partition(":") + headers[name.strip().lower()] = value.strip() + block = stream.read(int(headers.get("content-length", "0"))) + yield headers, block + + +def _entries(stream: BinaryIO) -> Iterator[Entry]: + methods: dict[str, str] = {} + responses = [] + for headers, block in _records(stream): + kind = headers.get("warc-type") + if kind == "request": + method = block.split(b" ", 1)[0].decode("latin-1").upper() + if headers.get("warc-concurrent-to"): + methods[headers["warc-concurrent-to"]] = method + elif kind == "response" and "application/http" in headers.get("content-type", ""): + responses.append((headers, block)) + for headers, block in responses: + status, entry_headers, body = _http_response(block) + yield Entry(methods.get(headers.get("warc-record-id", ""), "GET"), headers["warc-target-uri"], status, + entry_headers, body) + + +def _http_response(block: bytes) -> tuple[int, tuple[tuple[str, str], ...], bytes]: + head, _, body = block.partition(b"\r\n\r\n") + lines = head.decode("latin-1").split("\r\n") + status = int(lines[0].split(" ", 2)[1]) + headers = tuple((name.strip(), value.strip()) for name, _, value in (line.partition(":") for line in lines[1:]) + if name.strip()) + if any(n.lower() == "transfer-encoding" and "chunked" in v.lower() for n, v in headers): + body = _dechunked(body) + headers = tuple((n, v) for n, v in headers if n.lower() != "transfer-encoding") + return status, headers, body + + +def _dechunked(body: bytes) -> bytes: + out, stream = bytearray(), io.BytesIO(body) + while (size_line := stream.readline()) and (size := int(size_line.split(b";")[0].strip() or b"0", 16)): + out += stream.read(size) + stream.readline() + return bytes(out) + + +class WarcWriter: + """Writes WARC 1.1 records to a binary file, each its own gzip member when ``gzipped``.""" + + def __init__(self, out: BinaryIO, gzipped: bool = True) -> None: + self.out, self.gzipped = out, gzipped + + def warcinfo(self, fields: dict[str, str]) -> None: + body = "".join(f"{k}: {v}\r\n" for k, v in fields.items()).encode() + self._record("warcinfo", None, "application/warc-fields", body) + + def exchange(self, url: str, method: str, status: int, reason: str, headers: Iterable[tuple[str, str]], + body: bytes) -> None: + """A response record of the answer, and a request record naming only its method, target and host.""" + response_id = f"" + lines = [f"HTTP/1.1 {status} {reason or 'Unknown'}"] + lines += [f"{name}: {value}" for name, value in headers if name.lower() not in _NOT_STORED] + lines.append(f"Content-Length: {len(body)}") + payload = ("\r\n".join(lines) + "\r\n\r\n").encode("latin-1", "replace") + body + self._record("response", url, "application/http; msgtype=response", payload, record_id=response_id) + split = urlsplit(url) + target = urlunsplit(("", "", split.path or "/", split.query, "")) + request = f"{method} {target} HTTP/1.1\r\nHost: {split.netloc}\r\n\r\n".encode("latin-1", "replace") + self._record("request", url, "application/http; msgtype=request", request, + extra={"WARC-Concurrent-To": response_id}) + + def _record(self, kind: str, url: Optional[str], content_type: str, block: bytes, *, + record_id: Optional[str] = None, extra: Optional[dict] = None) -> None: + headers = {"WARC-Type": kind, "WARC-Record-ID": record_id or f"", + "WARC-Date": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")} + if url is not None: + headers["WARC-Target-URI"] = url + headers.update(extra or {}) + headers.update({"Content-Type": content_type, "Content-Length": str(len(block))}) + record = (WARC_VERSION + "\r\n" + "".join(f"{k}: {v}\r\n" for k, v in headers.items()) + "\r\n").encode() + record += block + b"\r\n\r\n" + self.out.write(gzip.compress(record) if self.gzipped else record) + + +class _Links(HTMLParser): + """The links of an HTML page: ``pages`` its anchors and frames lead to, ``assets`` what it loads to show.""" + + _ASSET_RELS = {"stylesheet", "icon", "shortcut", "preload", "modulepreload", "manifest", "apple-touch-icon"} + + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.pages: list[str] = [] + self.assets: list[str] = [] + self.base: Optional[str] = None + + def handle_starttag(self, tag: str, attrs: list[tuple[str, Optional[str]]]) -> None: + a = {k: v for k, v in attrs if v} + if tag == "base" and "href" in a and self.base is None: + self.base = a["href"] + elif tag in ("a", "area") and "href" in a: + self.pages.append(a["href"]) + elif tag in ("iframe", "frame") and "src" in a: + self.pages.append(a["src"]) + elif tag == "link" and "href" in a: + rels = set((a.get("rel") or "").lower().split()) + (self.assets if rels & self._ASSET_RELS else self.pages).append(a["href"]) + elif tag in ("script", "img", "source", "video", "audio", "track", "embed", "input") and "src" in a: + self.assets.append(a["src"]) + if "srcset" in a: + self.assets += [item.strip().split(" ")[0] for item in a["srcset"].split(",") if item.strip()] + + +def site_of(url: str) -> str: + host = (urlsplit(url).hostname or "").lower() + return host[4:] if host.startswith("www.") else host + + +def same_site(url: str, sites: Iterable[str]) -> bool: + host = (urlsplit(url).hostname or "").lower() + return any(host == site or host.endswith("." + site) for site in sites) + + +@dataclasses.dataclass +class CaptureResult: + captured: int = 0 + failed: list[tuple[str, str]] = dataclasses.field(default_factory=list) + skipped: int = 0 + + +def capture(urls: list[str], out: BinaryIO, *, depth: int = 1, max_requests: int = 500, assets: bool = True, + timeout: float = 20.0, client: Any = None, user_agent: str = "agentenv-replay-capture/1", + gzipped: bool = True) -> CaptureResult: + """Capture ``urls`` and what they lead to into ``out``, a WARC file; ``client`` is an ``httpx.Client`` to fetch + with instead of one of its own, which must not follow redirects.""" + import httpx + + for url in urls: + if urlsplit(url).scheme not in ("http", "https"): + raise ValueError(f"a start URL is http or https, not {url!r}") + sites = {site_of(url) for url in urls} + writer = WarcWriter(out, gzipped) + writer.warcinfo({"software": "agentenv_protocol.website.replay", "format": "WARC File Format 1.1", + "start-urls": " ".join(urls), "depth": str(depth)}) + own = client is None + client = client or httpx.Client(follow_redirects=False, timeout=timeout, headers={"User-Agent": user_agent}) + queue: deque[tuple[str, int, bool]] = deque((url, 0, False) for url in urls) + seen: set = set() + result = CaptureResult() + try: + while queue and result.captured + len(result.failed) < max_requests: + url, level, is_asset = queue.popleft() + if (key := _fetched_as(url)) in seen: + continue + seen.add(key) + try: + response = client.get(url) + except httpx.HTTPError as e: + result.failed.append((url, f"{type(e).__name__}: {e}")) + continue + writer.exchange(url, "GET", response.status_code, response.reason_phrase, response.headers.multi_items(), + response.content) + result.captured += 1 + for link, next_level, link_is_asset in _next(url, response, level, is_asset, assets): + if next_level > depth or not same_site(link, sites): + result.skipped += next_level > depth and _fetched_as(link) not in seen + continue + queue.append((link, next_level, link_is_asset)) + finally: + if own: + client.close() + return result + + +def _fetched_as(url: str) -> tuple: + split = urlsplit(url) + return split.scheme, (split.hostname or "").lower(), split.port, *request_key("GET", split.path, split.query, ()) + + +def _next(url: str, response: Any, level: int, is_asset: bool, assets: bool) -> Iterator[tuple[str, int, bool]]: + """What a captured answer leads to: a redirect's target at its level, and an HTML page's links, its pages a level + further and its assets at its level.""" + if 300 <= response.status_code < 400 and response.headers.get("location"): + yield _absolute(url, response.headers["location"]), level, is_asset + return + if is_asset or "html" not in response.headers.get("content-type", ""): + return + links = _Links() + try: + links.feed(response.text) + except Exception: + return + base = urljoin(url, links.base) if links.base else url + for href in links.pages: + if (target := _absolute(base, href)) is not None: + yield target, level + 1, False + for href in links.assets if assets else (): + if (target := _absolute(base, href)) is not None: + yield target, level, True + + +def _absolute(base: str, href: str) -> Optional[str]: + target = urljoin(base, href.strip()) + split = urlsplit(target) + if split.scheme not in ("http", "https"): + return None + return urlunsplit((split.scheme, split.netloc, split.path or "/", split.query, "")) + + +class ReplayWebsite(WebsiteEnvironment): + """A site served from a capture: set ``capture`` to its WARC file (relative to the class's file) or pass one, and + add ``@api`` routes to answer what the capture can't, which win over it. Declare it with ``@website(domains=...)`` + as any site; a static folder or ``live`` has no place on it.""" + + capture: ClassVar[Optional[str]] = None + ignore_query: ClassVar[tuple[str, ...]] = DEFAULT_IGNORED_QUERY + + def __init__(self, capture: str | Path | bytes | None = None) -> None: + super().__init__() + given = capture if capture is not None else type(self).capture + source = self._site_path(given) if isinstance(given, (str, Path)) else given + self.declared_archive = Archive.read(source, self.ignore_query) if source is not None else Archive() + self.archive = self.declared_archive + + def create_app(self) -> Starlette: + declaration = self.declaration() + if declaration.static is not None or declaration.live: + raise ValueError(f"{type(self).__name__} serves its capture, so it takes no static folder and isn't live") + asgi = super().create_app() + built_in = {SESSION_PATH: self._session, STATE_PATH: self._state} + asgi.router.routes = [r for r in asgi.router.routes if not ( + isinstance(r, Route) and built_in.get(r.path) == r.endpoint and self.archive.holds(r.path))] + asgi.router.default = _Replay(self, type(self).app) + return asgi + + @default_store_hook + @add_data + async def _store_add(self, parts: list) -> None: + """Serve a capture among ``parts`` instead of the declared one; the rest load the store.""" + captures = [part for part in parts if _is_capture(part)] + for part in captures: + self.archive = Archive.read(_capture_source(part), self.ignore_query) + self.store.load_parts([part for part in parts if not _is_capture(part)]) + self.store.mark_initial() + + @default_store_hook + @reset_data + async def _store_reset(self) -> None: + self.store.reset() + self.archive = self.declared_archive + + +def _is_capture(part: Any) -> bool: + if not isinstance(part, FilePart): + return False + file = part.file + name = (getattr(file, "name", None) or getattr(file, "uri", None) or "").lower() + return getattr(file, "mimeType", None) in WARC_TYPES or name.endswith((".warc", ".warc.gz")) + + +def _capture_source(part: FilePart) -> bytes | Path: + file = part.file + if getattr(file, "bytes", None) is not None: + return base64.b64decode(file.bytes) + uri = urlsplit(file.uri) + if uri.scheme != "file": + raise ValueError(f"a capture is loaded by its bytes or a file:// URI, not {file.uri!r}") + return Path(unquote(uri.path)) + + +class _Replay: + """What no route answers: the capture's answer to the request, then the mounted app, then 404.""" + + def __init__(self, env: ReplayWebsite, app: Any) -> None: + self.env, self.app = env, app + + async def __call__(self, scope: dict, receive: Any, send: Any) -> None: + if scope["type"] != "http": + if self.app is not None: + await self.app(scope, receive, send) + return + raw = (scope.get("raw_path") or b"").decode("latin-1").split("?", 1)[0] + path = raw if raw and unquote(raw) == scope["path"] else quote(scope["path"]) + if reserved_path(scope["path"]): + return await error_response(NotFound("not found"))(scope, receive, send) + entry = self.env.archive.find(scope["method"], path, scope.get("query_string", b"").decode("latin-1")) + if entry is None: + if self.app is not None: + return await self.app(scope, receive, send) + return await error_response(NotFound("not in the capture"))(scope, receive, send) + await self._answer(entry, scope["method"] == "HEAD")(scope, receive, send) + + def _answer(self, entry: Entry, head: bool) -> Response: + response = Response(b"" if head else entry.body, status_code=entry.status) + headers = [(name.lower().encode("latin-1"), self._header(name.lower(), value).encode("latin-1", "replace")) + for name, value in entry.headers if name.lower() not in _NOT_REPLAYED] + response.raw_headers = headers + [(b"content-length", str(len(entry.body)).encode())] + return response + + def _header(self, name: str, value: str) -> str: + """``value`` as the replay answers it: a Set-Cookie without its Domain, so it holds at the replay's host, and + a redirect to a captured host made relative, so it stays on the replay.""" + if name == "set-cookie": + return ";".join(p for p in value.split(";") if not p.strip().lower().startswith("domain=")) + if name == "location": + split = urlsplit(value) + if split.hostname and split.hostname.lower() in self.env.archive.hosts: + return urlunsplit(("", "", split.path or "/", split.query, split.fragment)) + return value + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="python -m agentenv_protocol.website.replay", + description="Capture a site to a WARC file, or serve a capture.") + commands = parser.add_subparsers(dest="command", required=True) + grab = commands.add_parser("capture", help="capture start URLs and the same-site links they lead to") + grab.add_argument("urls", nargs="+", metavar="URL") + grab.add_argument("--out", required=True, help="the WARC file to write; gzipped unless it ends in .warc") + grab.add_argument("--depth", type=int, default=1, help="how many links away from a start URL to follow (1)") + grab.add_argument("--max-requests", type=int, default=500, help="stop after this many requests (500)") + grab.add_argument("--no-assets", action="store_true", help="capture pages only, not what they load") + grab.add_argument("--timeout", type=float, default=20.0, help="seconds one request may take (20)") + serve = commands.add_parser("serve", help="serve a capture on PORT as a website/v1 site") + serve.add_argument("capture") + serve.add_argument("--domain", action="append", default=[], help="a domain the site is declared at") + args = parser.parse_args(argv) + if args.command == "capture": + with open(args.out, "wb") as out: + result = capture(args.urls, out, depth=args.depth, max_requests=args.max_requests, + assets=not args.no_assets, timeout=args.timeout, gzipped=not args.out.endswith(".warc")) + for url, why in result.failed: + print(f"failed {url}: {why}", file=sys.stderr) + print(f"captured {result.captured} answers to {args.out}; {len(result.failed)} failed, " + f"{result.skipped} links beyond the depth") + return 1 if result.captured == 0 else 0 + site = website(domains=args.domain)(type("Replay", (ReplayWebsite,), {})) + site(Path(args.capture).resolve()).serve() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/agentenv-protocol/tests/test_website_replay.py b/packages/agentenv-protocol/tests/test_website_replay.py new file mode 100644 index 00000000..96b643de --- /dev/null +++ b/packages/agentenv-protocol/tests/test_website_replay.py @@ -0,0 +1,276 @@ +"""Capture and replay: the WARC a capture writes reads back as it was answered, a capture follows same-site links to +its depth and loads each page's assets, and ReplayWebsite serves a capture for any host, matched by method, path and +normalised query, with @api routes winning over it, its store exported by the data plane, and a capture loaded +through data/add until data/reset; the replay example serves its capture and its synthetic sign-up.""" + +from __future__ import annotations + +import base64 +import gzip +import importlib.util +import io +import sys +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import pytest +from starlette.testclient import TestClient + +from agentenv_protocol.website import WebsiteEnvironment, api, conformance, website +from agentenv_protocol.website.replay import Archive, Entry, ReplayWebsite, WarcWriter, capture, main, request_key + +RPC = "/agentenv" +EXAMPLE_DIR = Path(__file__).resolve().parents[1] / "examples" / "website" / "replay" + + +def rpc(client: TestClient, method: str, params: dict | None = None) -> dict: + body = client.post(RPC, json={"jsonrpc": "2.0", "id": 1, "method": method, "params": params or {}}).json() + assert "error" not in body, body + return body["result"] + + +def warc(*exchanges: tuple, gzipped: bool = True) -> bytes: + """A WARC of ``(url, status, headers, body)`` answers to GETs, or ``(url, status, headers, body, method)``.""" + out = io.BytesIO() + writer = WarcWriter(out, gzipped) + writer.warcinfo({"software": "test"}) + for url, status, headers, body, *method in exchanges: + writer.exchange(url, method[0] if method else "GET", status, "OK", headers, body) + return out.getvalue() + + +CAPTURE = warc( + ("https://shop.example/", 200, [("Content-Type", "text/html")], b"home"), + ("https://shop.example/search?q=lamp&page=2", 200, [("Content-Type", "application/json")], b'{"hits": 2}'), + ("https://shop.example/old", 301, [("Location", "https://shop.example/new?x=1")], b""), + ("https://shop.example/login", 200, [("Set-Cookie", "sid=abc; Domain=shop.example; Path=/; HttpOnly"), + ("Strict-Transport-Security", "max-age=1")], b"ok"), + ("https://shop.example/api/price", 200, [("Content-Type", "application/json")], b'{"price": 10}'), + ("https://shop.example/api/session", 200, [("Content-Type", "application/json")], b'{"captured": true}'), + ("https://shop.example/", 200, [("Content-Type", "text/html")], b"a second capture of home"), +) + + +@website(domains=["shop.example"]) +class Shop(ReplayWebsite): + @api.get("/api/price") + def price(self) -> dict: + return {"price": 7, "overridden": True} + + @api.post("/api/cart") + def add_to_cart(self, body: dict) -> dict: + return self.store.insert("cart", body) + + +@pytest.fixture +def client() -> TestClient: + return TestClient(Shop(CAPTURE).create_app()) + + +def test_a_request_key_normalises_method_path_and_query(): + assert request_key("head", "", "") == ("GET", "/", "") + assert request_key("GET", "/a/%7e b", "b=2&a=1&a=0") == ("GET", "/a/~%20b", "a=0&a=1&b=2") + assert request_key("GET", "/x", "utm_source=mail&fbclid=1&_=123&keep=&q=1") == ("GET", "/x", "keep=&q=1") + assert request_key("GET", "/a%2Fb", "")[1] != request_key("GET", "/a/b", "")[1] + assert request_key("GET", "/x", "_=1", ignore_query=())[2] == "_=1" + + +@pytest.mark.parametrize("gzipped", [True, False]) +def test_a_warc_reads_back_each_answer_and_the_method_its_request_record_names(gzipped): + raw = warc(("https://a.example/x", 200, [("Content-Type", "text/plain"), ("Content-Encoding", "gzip"), + ("Transfer-Encoding", "chunked")], b"hello"), + ("https://a.example/form", 201, [], b"made", "POST"), gzipped=gzipped) + assert (raw[:2] == b"\x1f\x8b") is gzipped + archive = Archive.read(raw) + get = archive.find("GET", "/x") + assert (get.status, get.body) == (200, b"hello") + assert ("Content-Length", "5") in get.headers + assert not any(name.lower() in ("content-encoding", "transfer-encoding") for name, _ in get.headers) + assert archive.find("POST", "/form").body == b"made" and archive.find("GET", "/form") is None + assert b"WARC/1.1" in (gzip.decompress(raw) if gzipped else raw) + + +def test_a_warc_another_tool_wrote_with_a_chunked_body_reads_dechunked(tmp_path): + http = b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nhello\r\n6\r\n world\r\n0\r\n\r\n" + record = (f"WARC/1.0\r\nWARC-Type: response\r\nWARC-Target-URI: http://b.example/\r\n" + f"Content-Type: application/http;msgtype=response\r\nContent-Length: {len(http)}\r\n\r\n").encode() + (tmp_path / "other.warc").write_bytes(record + http + b"\r\n\r\n") + entry = Archive.read(tmp_path / "other.warc").find("GET", "/") + assert entry.body == b"hello world" and entry.headers == () + + +class _Site: + """A small site to capture: pages, a redirect, an asset, a link to another site and one too deep to follow.""" + + PAGES = { + "/": (200, "text/html", b'' + b'amoved' + b'away'), + "/a": (200, "text/html", b'deepself'), + "/moved": (302, "text/html", b""), + "/b": (200, "text/html", b"b"), + "/deep": (200, "text/html", b"too deep at depth 1"), + "/s.css": (200, "text/css", b"body{}"), + "/i.png": (200, "image/png", b"\x89PNG"), + } + + def __init__(self) -> None: + self.seen: list[tuple[str, dict]] = [] + pages, seen = self.PAGES, self.seen + + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + seen.append((self.path, dict(self.headers))) + status, kind, body = pages.get(self.path.split("?")[0], (404, "text/plain", b"no")) + self.send_response(status) + self.send_header("Content-Type", kind) + if status == 302: + self.send_header("Location", "/b") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args) -> None: + pass + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.url = f"http://127.0.0.1:{self.server.server_address[1]}" + threading.Thread(target=self.server.serve_forever, daemon=True).start() + + def close(self) -> None: + self.server.shutdown() + + +@pytest.fixture +def live_site(): + site = _Site() + yield site + site.close() + + +def test_a_capture_follows_same_site_links_to_its_depth_with_each_pages_assets_and_redirects(live_site, tmp_path): + out = io.BytesIO() + result = capture([live_site.url + "/"], out, depth=1) + archive = Archive.read(out.getvalue()) + assert {key[1] for key in archive.entries} == {"/", "/a", "/moved", "/b", "/s.css", "/i.png"} + assert archive.find("GET", "/a").body.startswith(b'') + assert archive.find("GET", "/moved").status == 302 + assert result.captured == 6 and result.skipped == 1 and not result.failed + assert all(path != "/deep" for path, _ in live_site.seen) + + path = tmp_path / "site.warc.gz" + assert main(["capture", live_site.url + "/", "--out", str(path), "--depth", "0", "--no-assets"]) == 0 + assert {key[1] for key in Archive.read(path).entries} == {"/"} + + +def test_the_request_records_of_a_capture_name_no_header_but_host(live_site): + out = io.BytesIO() + capture([live_site.url + "/b"], out, depth=0) + requests = [block for block in gzip.decompress(out.getvalue()).split(b"WARC/1.1\r\n") + if b"msgtype=request" in block] + assert len(requests) == 1 + request = requests[0].split(b"\r\n\r\n")[1] + assert request == f"GET /b HTTP/1.1\r\nHost: {live_site.url.split('//')[1]}".encode() + + +def test_a_start_url_that_isnt_http_is_refused(): + with pytest.raises(ValueError, match="http or https"): + capture(["file:///etc/passwd"], io.BytesIO()) + + +def test_a_replay_serves_its_capture_for_any_host_and_404s_what_it_doesnt_hold(client): + for host in ("shop.example", "anything.test"): + home = client.get("/", headers={"Host": host}) + assert (home.status_code, home.text) == (200, "home") + assert client.get("/search?page=2&q=lamp&utm_campaign=x").json() == {"hits": 2} + assert client.get("/search?q=lamp").status_code == 404 + assert client.get("/nowhere").status_code == 404 + assert client.post("/").status_code == 404 + head = client.head("/") + assert head.status_code == 200 and head.content == b"" and head.headers["content-length"] == "30" + + +def test_a_replay_keeps_cookies_and_redirects_on_its_own_host(client): + login = client.get("/login") + assert login.headers["set-cookie"] == "sid=abc; Path=/; HttpOnly" + assert "strict-transport-security" not in login.headers + moved = client.get("/old", follow_redirects=False) + assert (moved.status_code, moved.headers["location"]) == (301, "/new?x=1") + + +def test_api_routes_win_over_the_capture_and_the_capture_over_built_in_routes(client): + assert client.get("/api/price").json() == {"price": 7, "overridden": True} + assert client.get("/api/session").json() == {"captured": True} + assert client.get("/agentenv/ext/website/openapi").status_code == 200 + assert client.get("/.well-known/agent-env.json").json()["name"] + assert client.get("/agentenv/../agentenv").status_code in (404, 405) + + +def test_the_data_plane_exports_the_overrides_state(client): + assert client.post("/api/cart", json={"sku": "lamp"}).json() == {"sku": "lamp", "id": 1} + assert rpc(client, "data/get")["parts"][0]["data"] == {"cart": [{"sku": "lamp", "id": 1}]} + assert client.get("/go").json()["state_diff"]["cart"]["added"] == [{"sku": "lamp", "id": 1}] + rpc(client, "data/reset") + assert rpc(client, "data/get")["parts"][0]["data"] == {} + + +def test_a_capture_loaded_through_the_data_plane_is_served_until_a_reset(client, tmp_path): + other = warc(("https://shop.example/", 200, [("Content-Type", "text/plain")], b"the other capture")) + by_bytes = {"kind": "file", "file": {"bytes": base64.b64encode(other).decode(), "name": "other.warc.gz"}} + rpc(client, "data/add", {"parts": [by_bytes, {"kind": "data", "data": {"cart": [{"id": 9}]}}]}) + assert client.get("/").text == "the other capture" + assert client.get("/search?q=lamp&page=2").status_code == 404 + assert rpc(client, "data/get")["parts"][0]["data"] == {"cart": [{"id": 9}]} + rpc(client, "data/reset") + assert client.get("/").text == "home" + + (tmp_path / "c.warc").write_bytes(warc(("https://x.example/", 200, [], b"from a file"), gzipped=False)) + rpc(client, "data/add", {"parts": [{"kind": "file", "file": {"uri": (tmp_path / "c.warc").as_uri(), + "mimeType": "application/warc"}}]}) + assert client.get("/").text == "from a file" + + +def test_a_replay_with_a_static_folder_is_refused(tmp_path): + @website(domains=["x.example"], static=str(tmp_path)) + class Static(ReplayWebsite): + pass + + with pytest.raises(ValueError, match="no static folder"): + Static(CAPTURE).create_app() + + +def test_a_replay_passes_the_conformance_checks(client): + results = {r.name: r.status for r in conformance.check("http://testserver", client=client)} + assert {name for name, status in results.items() if status == "failed"} == set(), results + + +def _example(): + spec = importlib.util.spec_from_file_location("replay_example_site", EXAMPLE_DIR / "site.py") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def test_the_example_serves_its_capture_and_its_sign_up(): + example = _example() + assert issubclass(example.WidgetDocs, WebsiteEnvironment) + client = TestClient(example.WidgetDocs().create_app()) + assert "Widget Docs" in client.get("/", headers={"Host": "docs.widgets.test"}).text + assert client.get("/guide/usage.html").status_code == 200 + assert client.get("/guide/usage.html?utm_source=home").status_code == 200 + assert client.get("/guide/advanced.html").status_code == 404 + assert client.get("/style.css").headers["content-type"].startswith("text/css") + made = client.post("/api/subscribe", json={"email": "ann@example.com"}) + assert made.status_code == 201 + assert client.post("/api/subscribe", json={"email": "ann@example.com"}).status_code == 409 + assert client.post("/api/subscribe", json={"email": "nobody"}).status_code == 400 + exported = rpc(client, "data/get")["parts"][0]["data"] + assert [s["email"] for s in exported["subscriptions"]] == ["ann@example.com"] + + +def test_an_entry_is_kept_once_per_request(): + archive = Archive([Entry("GET", "https://a.example/x?b=1&a=2", 200, (), b"first"), + Entry("GET", "https://a.example/x?a=2&b=1", 200, (), b"second")]) + assert len(archive) == 1 and archive.find("GET", "/x", "a=2&b=1").body == b"first" From 14e895df9af1ff9526132bb3da338a85caa82d94 Mon Sep 17 00:00:00 2001 From: William Loo Date: Mon, 2 Nov 2026 02:26:07 -0800 Subject: [PATCH 03/20] feat(website): the Playwright MCP browser serves browser/v1's cookies/set and cookies/clear (FD-4683) The recorder's card, the browser's child env, now serves browser/v1. Playwright MCP launches its browser at the agent's first tool call and starts each context from --storage-state, so cookies/set writes the cookies to that file (0600, merged by name, domain and path) and, through DevTools, into every context of a browser already running; cookies/clear removes those of some domains or all. Answers and errors never repeat a cookie's value. The recorder's protocol floor rises to 0.1.405, the first release with agentenv_protocol.website.browser. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../env/envs/playwright_mcp/entrypoint.sh | 3 + .../playwright_mcp/recorder-requirements.txt | 4 +- .../env/envs/playwright_mcp/recorder.py | 111 ++++++++++++- .../env/envs/test_playwright_mcp_cookies.py | 147 ++++++++++++++++++ .../env/envs/test_playwright_mcp_recorder.py | 5 +- 5 files changed, 263 insertions(+), 7 deletions(-) create mode 100644 tst/unit/env/envs/test_playwright_mcp_cookies.py diff --git a/src/agent_env/env/envs/playwright_mcp/entrypoint.sh b/src/agent_env/env/envs/playwright_mcp/entrypoint.sh index 33788584..68a60210 100644 --- a/src/agent_env/env/envs/playwright_mcp/entrypoint.sh +++ b/src/agent_env/env/envs/playwright_mcp/entrypoint.sh @@ -7,6 +7,8 @@ set -euo pipefail # The recorder serves the browser's card on RECORDER_PORT, restarted whenever it exits; the MCP server starts once # it listens, so the card is there by the time the server is healthy, or after RECORDER_WAIT_S without it. export RECORDER_PORT="${RECORDER_PORT:-18766}" +export PLAYWRIGHT_STORAGE_STATE="${PLAYWRIGHT_STORAGE_STATE:-/app/storage-state.json}" +[ -f "${PLAYWRIGHT_STORAGE_STATE}" ] || (umask 077 && echo '{"cookies": [], "origins": []}' > "${PLAYWRIGHT_STORAGE_STATE}") (while true; do /opt/recorder/bin/python /app/recorder.py || true; sleep 1; done) & for _ in $(seq 1 "$(( ${RECORDER_WAIT_S:-20} * 10 ))"); do (exec 3<>"/dev/tcp/127.0.0.1/${RECORDER_PORT}") 2>/dev/null && break @@ -17,6 +19,7 @@ exec npx @playwright/mcp@${PLAYWRIGHT_MCP_VERSION} \ --browser chromium \ --headless \ --isolated \ + --storage-state "${PLAYWRIGHT_STORAGE_STATE}" \ --no-sandbox \ --host "${MCP_HOST:-0.0.0.0}" \ --port "${MCP_PORT:-18765}" \ diff --git a/src/agent_env/env/envs/playwright_mcp/recorder-requirements.txt b/src/agent_env/env/envs/playwright_mcp/recorder-requirements.txt index 57ae4a2e..4dc5eef9 100644 --- a/src/agent_env/env/envs/playwright_mcp/recorder-requirements.txt +++ b/src/agent_env/env/envs/playwright_mcp/recorder-requirements.txt @@ -1,4 +1,4 @@ -# The recorder installs the newest agentenv-framework-protocol at or above this floor, the first with computer-recording/v1; +# The recorder installs the newest agentenv-framework-protocol at or above this floor, the first with browser/v1's cookies; # raise it when the recorder needs a newer protocol. -agentenv-framework-protocol[computer]>=0.1.382 +agentenv-framework-protocol[computer]>=0.1.405 websockets==15.0.1 diff --git a/src/agent_env/env/envs/playwright_mcp/recorder.py b/src/agent_env/env/envs/playwright_mcp/recorder.py index c82dbedf..2bb7e9cd 100644 --- a/src/agent_env/env/envs/playwright_mcp/recorder.py +++ b/src/agent_env/env/envs/playwright_mcp/recorder.py @@ -2,7 +2,12 @@ screencast frames, on a card and port of its own beside the MCP server. It is idle until a recording starts, which finds the browser through the remote debugging port Playwright opens for an ``--isolated`` Chromium and follows the page the agent is on. It runs standalone in the browser's image, so it imports only -``agentenv-framework-protocol[computer]`` and, to talk to the browser, ``websockets``.""" +``agentenv-framework-protocol[computer]`` and, to talk to the browser, ``websockets``. + +The same card serves browser/v1's ``cookies/set`` and ``cookies/clear``, so a step can start the browser signed in to +a site. Playwright MCP starts each browser context it opens from the storage state file ``--storage-state`` names +(``PLAYWRIGHT_STORAGE_STATE``), and launches its browser only at the agent's first tool call, so the cookies go into +that file, and through DevTools into every context of a browser already running.""" from __future__ import annotations import asyncio @@ -30,6 +35,10 @@ ) from agentenv_protocol.computer.card import COMPUTER_RECORDING from agentenv_protocol.computer.recording import RecordedFrame, RecordHints, Recording, Recordings, recording_params +from agentenv_protocol.errors import AgentEnvError +from agentenv_protocol.website.browser import (BROWSER_EXTENSION_URI, CLEAR_COOKIES, SET_COOKIES, Cookie, + devtools, domain_matches, storage_state) +from agentenv_protocol.website.browser import cookies as parse_cookies logger = logging.getLogger("playwright-mcp-recorder") @@ -43,6 +52,7 @@ #: How long one DevTools command may take. CALL_S = 10.0 JPEG_QUALITY = 80 +DEFAULT_STORAGE_STATE = "/app/storage-state.json" class CdpError(Exception): @@ -222,16 +232,96 @@ async def _show(self, cdp: Cdp, target: Optional[str]) -> None: self._session = session +class BrowserCookies: + """browser/v1's cookies: kept in the storage state file each new browser context starts from, and set in, or + cleared from, every context of the browser that is running, if one is.""" + + def __init__(self, path: Path, find: Callable[[], Optional[int]], + connect: Callable[[int], AsyncContextManager[Cdp]]) -> None: + self.path, self._find, self._connect = path, find, connect + + async def set(self, given: list[Cookie]) -> dict: + kept = {c.key(): c for c in self._read()} + kept.update({c.key(): c for c in given}) + self._write(list(kept.values())) + + async def add(cdp: Cdp, context: dict) -> None: + await cdp.call("Storage.setCookies", {"cookies": devtools(given), **context}) + + return {"set": len(given), "open_browser": await self._each_context(add)} + + async def clear(self, domains: Optional[list[str]]) -> dict: + before = self._read() + kept = [c for c in before if domains is not None and not c.matches(domains)] + self._write(kept) + + async def remove(cdp: Cdp, context: dict) -> None: + current = [] if domains is None else (await cdp.call("Storage.getCookies", context)).get("cookies") or [] + await cdp.call("Storage.clearCookies", context) + others = [_settable(c) for c in current if not domain_matches(c.get("domain", ""), domains or [])] + if others: + await cdp.call("Storage.setCookies", {"cookies": others, **context}) + + return {"cleared": len(before) - len(kept), "open_browser": await self._each_context(remove)} + + async def _each_context(self, act: Callable[[Cdp, dict], Any]) -> bool: + """``act`` on the default context and each other one of the running browser; False when none runs. An error + names the DevTools call's kind only, since its message may repeat a cookie.""" + port = await asyncio.to_thread(self._find) + if port is None: + return False + try: + async with self._connect(port) as cdp: + contexts = (await cdp.call("Target.getBrowserContexts")).get("browserContextIds") or [] + for context in [{}, *({"browserContextId": c} for c in contexts)]: + await act(cdp, context) + except (OSError, CdpError, httpx.HTTPError, KeyError, ValueError, asyncio.TimeoutError) as e: + raise AgentEnvError("browser_unreachable", f"the running browser didn't take the change " + f"({type(e).__name__}); the next context it opens starts with it", applied="unknown", + status=502) from None + return True + + def _read(self) -> list[Cookie]: + try: + state = json.loads(self.path.read_text(encoding="utf-8")) + except FileNotFoundError: + return [] + return [Cookie.model_validate({**c, "expires": None if c.get("expires", -1) in (-1, None) else c["expires"]}) + for c in state.get("cookies") or []] + + def _write(self, kept: list[Cookie]) -> None: + try: + origins = json.loads(self.path.read_text(encoding="utf-8")).get("origins") or [] + except (FileNotFoundError, ValueError): + origins = [] + staged = self.path.with_name(self.path.name + ".tmp") + staged.write_text(json.dumps({"cookies": storage_state(kept), "origins": origins}), encoding="utf-8") + os.chmod(staged, 0o600) + os.replace(staged, self.path) + + +def _settable(cookie: dict) -> dict: + """A cookie DevTools listed, as ``Storage.setCookies`` takes it back.""" + param = {k: cookie[k] for k in ("name", "value", "domain", "path", "secure", "httpOnly", "sameSite") if k in cookie} + if not cookie.get("session", False) and cookie.get("expires", -1) not in (-1, None): + param["expires"] = cookie["expires"] + return param + + @environment_card(name="playwright-mcp") class BrowserRecorder(AgentEnvEnvironment): - """computer-recording/v1 of the Playwright MCP browser's page; the browser's tools stay Playwright MCP's.""" + """computer-recording/v1 of the Playwright MCP browser's page, and browser/v1's cookies; the browser's tools stay + Playwright MCP's.""" recording = Recording() def __init__(self, find: Callable[[], Optional[int]] = find_browser, - connect: Callable[[int], AsyncContextManager[Cdp]] = connect_cdp) -> None: + connect: Callable[[int], AsyncContextManager[Cdp]] = connect_cdp, + storage_state_path: Optional[Path] = None) -> None: self._find, self._connect = find, connect self.recordings = Recordings(_Host(self)) + path = storage_state_path or Path(os.environ.get("PLAYWRIGHT_STORAGE_STATE", DEFAULT_STORAGE_STATE)) + self.cookies = BrowserCookies(path, find, connect) def extension_params(self, uri: str) -> dict: if uri == COMPUTER_RECORDING: @@ -250,6 +340,21 @@ async def _recording_start(self, request_id: str, scope: str = "display", window async def _recording_end(self, recording_id: str, upload: Optional[dict] = None, reason: str = "end") -> dict: return await self.recordings.end(recording_id, upload, reason) + @extension(BROWSER_EXTENSION_URI, op=SET_COOKIES, + description="The browser's cookies, for agentenv's steps, never its agent: cookies/set adds cookies " + "(name, value, domain, path, secure, httpOnly, expires, sameSite) to every context, open " + "and to come, and cookies/clear removes those of some domains, or all of them.") + async def _cookies_set(self, cookies: list[dict]) -> dict: + try: + given = parse_cookies(cookies) + except ValueError as e: + raise AgentEnvError("invalid_params", str(e)) from None + return await self.cookies.set(given) + + @extension(BROWSER_EXTENSION_URI, op=CLEAR_COOKIES) + async def _cookies_clear(self, domains: Optional[list[str]] = None) -> dict: + return await self.cookies.clear(domains) + async def frames(self, hints: RecordHints) -> AsyncIterator[RecordedFrame]: """The page at up to ``fps``: a frame each time it changes, and the last one again after ``HOLD_S``.""" screencast = Screencast(hints.max_edge, self._find, self._connect) diff --git a/tst/unit/env/envs/test_playwright_mcp_cookies.py b/tst/unit/env/envs/test_playwright_mcp_cookies.py new file mode 100644 index 00000000..7bce89a5 --- /dev/null +++ b/tst/unit/env/envs/test_playwright_mcp_cookies.py @@ -0,0 +1,147 @@ +"""browser/v1's cookies in the Playwright MCP browser: cookies/set keeps the cookies in the storage state file every +context Playwright MCP opens starts from and sets them in each context of a running browser, cookies/clear removes them +by domain or all of them, and neither answer, nor an error, repeats a cookie's value.""" + +from __future__ import annotations + +import json +import os +import re +import stat +from pathlib import Path + +import httpx +import pytest + +pytest.importorskip("av") + +from agentenv_protocol import client as protocol_client # noqa: E402 +from agentenv_protocol.website.browser import BROWSER_EXTENSION_URI # noqa: E402 + +from agent_env.env.envs import playwright_mcp # noqa: E402 +from agent_env.env.envs.playwright_mcp import recorder # noqa: E402 +from agent_env.env.gateway.constants import GATEWAY_EXTENSIONS # noqa: E402 +from agent_env.env.gateway.rest_proxy import rewrite_child_card # noqa: E402 + +SET, CLEAR = "/agentenv/ext/browser/cookies/set", "/agentenv/ext/browser/cookies/clear" +SID = {"name": "sid", "value": "s3cret", "domain": "github.com", "secure": True, "httpOnly": True, + "expires": 1_900_000_000} +OTHER = {"name": "pref", "value": "dark", "domain": ".example.com"} + + +class _DevTools: + """A browser's DevTools with a default context and two of Playwright's, each holding its cookies.""" + + def __init__(self, refuse: bool = False) -> None: + self.calls: list[tuple[str, dict]] = [] + self.jars: dict = {None: [], "ctx-1": [], "ctx-2": []} + self.refuse = refuse + + async def call(self, method, params=None, session=None): + params = dict(params or {}) + self.calls.append((method, params)) + jar = self.jars[params.pop("browserContextId", None)] + if method == "Target.getBrowserContexts": + return {"browserContextIds": ["ctx-1", "ctx-2"]} + if self.refuse: + raise recorder.CdpError("Invalid cookie fields: s3cret") + if method == "Storage.setCookies": + jar += [{**c, "session": "expires" not in c, "size": 1} for c in params["cookies"]] + elif method == "Storage.clearCookies": + jar.clear() + elif method == "Storage.getCookies": + return {"cookies": list(jar)} + return {} + + +def _recorder(tmp_path: Path, devtools: _DevTools | None) -> recorder.BrowserRecorder: + class _Connection: + async def __aenter__(self): + return devtools + + async def __aexit__(self, *exc): + return False + + return recorder.BrowserRecorder(find=lambda: None if devtools is None else 9222, + connect=lambda port: _Connection(), + storage_state_path=tmp_path / "storage-state.json") + + +async def _post(app, path: str, body: dict) -> httpx.Response: + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://recorder") as http: + return await http.post(path, json=body) + + +@pytest.mark.asyncio +async def test_cookies_go_into_the_storage_state_file_before_the_browser_runs(tmp_path): + app = recorder.create_app(_recorder(tmp_path, None)) + answer = await _post(app, SET, {"cookies": [SID, OTHER]}) + assert answer.json() == {"set": 2, "open_browser": False} and "s3cret" not in answer.text + path = tmp_path / "storage-state.json" + state = json.loads(path.read_text()) + assert state["origins"] == [] + assert state["cookies"] == [ + {"name": "sid", "value": "s3cret", "domain": "github.com", "path": "/", "expires": 1_900_000_000, + "httpOnly": True, "secure": True, "sameSite": "Lax"}, + {"name": "pref", "value": "dark", "domain": ".example.com", "path": "/", "expires": -1, "httpOnly": False, + "secure": False, "sameSite": "Lax"}] + assert stat.S_IMODE(os.stat(path).st_mode) == 0o600 + + await _post(app, SET, {"cookies": [{**SID, "value": "newer"}]}) + assert [c["value"] for c in json.loads(path.read_text())["cookies"]] == ["newer", "dark"] + + +@pytest.mark.asyncio +async def test_cookies_are_set_in_every_context_of_a_running_browser(tmp_path): + devtools = _DevTools() + app = recorder.create_app(_recorder(tmp_path, devtools)) + assert (await _post(app, SET, {"cookies": [SID]})).json() == {"set": 1, "open_browser": True} + for jar in devtools.jars.values(): + assert jar == [{"name": "sid", "value": "s3cret", "domain": "github.com", "path": "/", "secure": True, + "httpOnly": True, "expires": 1_900_000_000, "session": False, "size": 1}] + + +@pytest.mark.asyncio +async def test_clearing_a_domain_keeps_the_other_cookies_and_clearing_all_keeps_none(tmp_path): + devtools = _DevTools() + app = recorder.create_app(_recorder(tmp_path, devtools)) + await _post(app, SET, {"cookies": [SID, OTHER]}) + assert (await _post(app, CLEAR, {"domains": ["github.com"]})).json() == {"cleared": 1, "open_browser": True} + assert [c["name"] for c in json.loads((tmp_path / "storage-state.json").read_text())["cookies"]] == ["pref"] + for jar in devtools.jars.values(): + assert [(c["name"], c.get("expires")) for c in jar] == [("pref", None)] + assert (await _post(app, CLEAR, {})).json() == {"cleared": 1, "open_browser": True} + assert json.loads((tmp_path / "storage-state.json").read_text())["cookies"] == [] + assert all(jar == [] for jar in devtools.jars.values()) + + +@pytest.mark.asyncio +async def test_a_bad_cookie_or_a_browser_that_refuses_answers_without_the_value(tmp_path): + app = recorder.create_app(_recorder(tmp_path, _DevTools(refuse=True))) + bad = await _post(app, SET, {"cookies": [{"name": "sid", "value": "s3cret"}]}) + assert bad.status_code == 400 and "cookies[0].domain" in bad.text and "s3cret" not in bad.text + refused = await _post(app, SET, {"cookies": [SID]}) + assert refused.status_code == 502 and refused.json()["error"]["code"] == "browser_unreachable" + assert "s3cret" not in refused.text + assert json.loads((tmp_path / "storage-state.json").read_text())["cookies"][0]["name"] == "sid" + + +@pytest.mark.asyncio +async def test_the_gateway_nests_browser_v1_under_the_browsers_child_env(tmp_path): + app = recorder.create_app(_recorder(tmp_path, None)) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://recorder") as http: + served = (await http.get("/.well-known/agent-env.json")).json() + composed = {"name": "env", "capabilities": {"extensions": GATEWAY_EXTENSIONS}, + "children_environments": [rewrite_child_card("mcp-playwright-mcp", served, GATEWAY_EXTENSIONS)]} + child = protocol_client.find_child(composed, playwright_mcp.PLAYWRIGHT_MCP_ENVIRONMENT_NAME) + methods = protocol_client.find_extension(child, BROWSER_EXTENSION_URI)["params"]["methods"] + assert {op: m["endpoint"] for op, m in methods.items()} == { + "cookies/set": "/svc/mcp-playwright-mcp/agentenv/ext/browser/cookies/set", + "cookies/clear": "/svc/mcp-playwright-mcp/agentenv/ext/browser/cookies/clear"} + + +def test_playwright_mcp_starts_each_context_from_the_file_the_recorder_writes(): + entrypoint = (Path(playwright_mcp.__file__).parent / "entrypoint.sh").read_text() + default = re.search(r'PLAYWRIGHT_STORAGE_STATE="\$\{PLAYWRIGHT_STORAGE_STATE:-([^}]+)\}"', entrypoint).group(1) + assert default == recorder.DEFAULT_STORAGE_STATE + assert '--storage-state "${PLAYWRIGHT_STORAGE_STATE}"' in entrypoint diff --git a/tst/unit/env/envs/test_playwright_mcp_recorder.py b/tst/unit/env/envs/test_playwright_mcp_recorder.py index 207c0d45..7c70c9d9 100644 --- a/tst/unit/env/envs/test_playwright_mcp_recorder.py +++ b/tst/unit/env/envs/test_playwright_mcp_recorder.py @@ -196,8 +196,9 @@ async def test_a_recording_through_the_card_is_an_mp4_of_the_page(home): manifest = (await http.post(END, json={"recording_id": started["recording_id"]})).json() assert card["name"] == "playwright-mcp" and card["capabilities"]["operations"] == [] - assert [e["uri"] for e in card["capabilities"]["extensions"]] == [COMPUTER_RECORDING] - params = card["capabilities"]["extensions"][0]["params"] + extensions = {e["uri"]: e for e in card["capabilities"]["extensions"]} + assert set(extensions) == {COMPUTER_RECORDING, "urn:agentenv:browser/v1"} + params = extensions[COMPUTER_RECORDING]["params"] assert params["cursor"] is False and params["source"] == "stream" and set(params["methods"]) == {"start", "end"} assert (manifest["ended_by"], manifest["cursor"], manifest["source"]) == ("end", False, "stream") (segment,) = manifest["segments"] From 72293be1bd07bfbb1467a60b9e19e03839d6f515 Mon Sep 17 00:00:00 2001 From: William Loo Date: Mon, 2 Nov 2026 02:32:29 -0800 Subject: [PATCH 04/20] feat(website): sign the browser in to live sites, mark live runs not reproducible, website env settings (FD-4683) deploy_env marks a deploy with a live website child: context.metadata reproducible=False and live_websites (each site's domains and allow_hosts, checked against the run's computer allowlists, which can't change once a sandbox runs, so a host one leaves out is recorded and warned about), and the deploy record live. The sign_in_browser step calls each live site's website/v1 session and passes the cookies straight to the browser child's browser/v1 cookies/set; the context, logs and errors keep cookies by name, domain, path and expiry only. reset_env signs in again after a reset. A website env's settings are its container's environment variables: a secret is stored as a secret:/env: reference and resolved only into the compose file. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/agent_env/env/envs/website.py | 46 +++- .../env_providers/env_gateway_provider.py | 16 +- src/agent_env/task/episode/rules.py | 2 +- src/agent_env/task_step/registry.py | 2 + .../task_step/task_steps/deploy_env.py | 4 +- .../task_step/task_steps/reset_env.py | 6 +- .../task_step/task_steps/sign_in_browser.py | 256 ++++++++++++++++++ .../env/envs/test_website_env_settings.py | 58 ++++ tst/unit/task_step/sign_in_browser_test.py | 204 ++++++++++++++ 9 files changed, 587 insertions(+), 7 deletions(-) create mode 100644 src/agent_env/task_step/task_steps/sign_in_browser.py create mode 100644 tst/unit/env/envs/test_website_env_settings.py create mode 100644 tst/unit/task_step/sign_in_browser_test.py diff --git a/src/agent_env/env/envs/website.py b/src/agent_env/env/envs/website.py index a47b4e7e..22eb8c54 100644 --- a/src/agent_env/env/envs/website.py +++ b/src/agent_env/env/envs/website.py @@ -3,6 +3,11 @@ env.toml may carry the site's website spec, ``spec = "website.toml"`` or an inline ``[spec]`` table, whose site name, health path and domains the env takes.""" + +Its ``settings`` are environment variables of the server's container, a live site's credentials among them: a setting +whose name reads as a secret's (a token, a password, ...) is stored as a ``secret:KEY`` or ``env:NAME`` reference, +never its value, and resolved from the deploying user's config only when the container is written.""" + from __future__ import annotations import asyncio @@ -42,6 +47,9 @@ DEFAULT_HEALTH_PATH = AGENT_ENV_WEBSITE_HEALTH_PATH _HEALTH_PATH = re.compile(r"/[A-Za-z0-9._~/-]*") +_SETTING_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") +RESERVED_SETTINGS = frozenset({"DATABASE_URL", "SERVICE_NAME", "ENVIRONMENT_NAME", "PORT", "PYTHONUNBUFFERED", + "WEBSITE_ROLE_KEY"}) class WebsiteEnv(Env): @@ -51,7 +59,8 @@ class WebsiteEnv(Env): EntityRef.artifact("frontend_image", artifact_type="docker_image", optional=True)) toml_keys: ClassVar[dict[str, type]] = {"backend_image": object, "frontend_image": object, "environment_name": str, - "health": str, "env_provider_type": str, "spec": object} + "health": str, "env_provider_type": str, "spec": object, + "settings": dict} def __init__( self, @@ -66,6 +75,7 @@ def __init__( metadata: Optional[dict[str, str]] = None, env_provider_type: str = "gateway", domains: tuple[str, ...] | list[str] = (), + settings: Optional[dict[str, str]] = None, ): super().__init__(id, version, metadata=metadata) if not environment_name: @@ -81,6 +91,9 @@ def __init__( raise ValueError(f"website env {id!r}: {problem}") self.health_path = health_path self.domains = tuple(domains) + if problems := settings_problems(settings or {}): + raise ValueError(f"website env {id!r}: {'; '.join(problems)}") + self.settings = dict(settings or {}) self.environment_name = environment_name self.env_provider_type = env_provider_type self._sandbox = None @@ -113,6 +126,8 @@ def to_dict(self) -> dict: base["service_name"] = self.environment_name base["environment_name"] = self.environment_name # dual-write with service_name during the rename base["env_provider_type"] = self.env_provider_type + if self.settings: + base["settings"] = dict(self.settings) return base @classmethod @@ -130,6 +145,7 @@ def from_dict(cls, data: dict) -> WebsiteEnv: metadata=data.get("metadata", {}), env_provider_type=data.get("env_provider_type", "gateway"), domains=data.get("domains", ()), + settings=data.get("settings"), ) @classmethod @@ -153,7 +169,7 @@ def from_toml(cls, data: dict, ctx: AuthoringContext) -> WebsiteEnv: environment_name=fields["environment_name"], health_path=fields.get("health", DEFAULT_HEALTH_PATH), env_provider_type=fields["env_provider_type"], - domains=fields.get("domains", ())) + domains=fields.get("domains", ()), settings=fields.get("settings")) def to_toml(self, out: ExportContext) -> dict: """The env.toml of a folder this env's images are built from, which ``from_toml`` reads back as it.""" @@ -165,6 +181,8 @@ def to_toml(self, out: ExportContext) -> dict: toml["health"] = self.health_path if self.domains: toml["spec"] = {"site": {"domains": list(self.domains)}} + if self.settings: + toml["settings"] = dict(self.settings) return {**toml, **self.provider_toml(out, self.env_provider_type)} @classmethod @@ -181,6 +199,9 @@ def accept_toml(cls, data: dict, ctx: AuthoringContext) -> dict: health = data.get("health", DEFAULT_HEALTH_PATH) if isinstance(health, str) and (problem := health_path_problem(health)): ctx.refuse([ctx.config_problem(problem)]) + settings = data.get("settings") + if isinstance(settings, dict) and (problems := settings_problems(settings)): + ctx.refuse([ctx.config_problem(problem) for problem in problems]) fields = ctx.accept_env(data, cls, named_by="backend_image") if spec is not None and spec.site.domains: fields["domains"] = spec.site.domains @@ -431,6 +452,27 @@ def website_spec_of(data: dict, folder: Path) -> tuple[Optional[WebsiteSpec], li return spec, problems +def settings_problems(settings: dict) -> list[str]: + """Why ``settings`` can't be a website env's, each problem naming the setting and never its value: a name is an + environment variable's the gateway doesn't set itself, a value a string, and a secret's a bare ``secret:KEY`` or + ``env:NAME`` reference.""" + from agent_env.config.describe import is_secret_key + from agent_env.config.loader import ENV_REF_PREFIX, SECRET_REF_PREFIX + + problems = [] + for name, value in settings.items(): + if not isinstance(name, str) or not _SETTING_NAME.fullmatch(name) or name in RESERVED_SETTINGS: + problems.append(f"settings.{name}: a setting is an environment variable's name the gateway doesn't set " + f"itself ({', '.join(sorted(RESERVED_SETTINGS))} are its own)") + elif not isinstance(value, str): + problems.append(f"settings.{name} is a string, not a {type(value).__name__}") + elif is_secret_key(name) and not (value.startswith((SECRET_REF_PREFIX, ENV_REF_PREFIX)) and "?" not in value + and value not in (SECRET_REF_PREFIX, ENV_REF_PREFIX)): + problems.append(f"settings.{name} is a secret, so it takes a {SECRET_REF_PREFIX}KEY or {ENV_REF_PREFIX}NAME " + "reference with no ?default, not its value") + return problems + + def health_path_problem(health_path: str) -> str | None: """Why ``health_path`` can't be where a site answers once it's usable, or None: it's a path of plain characters, since the gateway writes it into a health check.""" diff --git a/src/agent_env/providers/env_providers/env_gateway_provider.py b/src/agent_env/providers/env_providers/env_gateway_provider.py index cdcc6ae4..68292f60 100644 --- a/src/agent_env/providers/env_providers/env_gateway_provider.py +++ b/src/agent_env/providers/env_providers/env_gateway_provider.py @@ -122,12 +122,22 @@ class DeployedGateway: @dataclass class WebsiteConfig: """Configuration for a website in a gateway deployment: its server image, which serves the site's pages and API and - answers at ``health_path`` once usable, and a deprecated separate frontend image for a site built on one.""" + answers at ``health_path`` once usable, a deprecated separate frontend image for a site built on one, and the + env's ``settings`` as stored, secrets as their references, which only the compose file holds resolved.""" backend_image: str frontend_image: str | None environment_name: str health_path: str = AGENT_ENV_WEBSITE_HEALTH_PATH + settings: dict[str, str] | None = None + + def env_vars(self) -> dict[str, str]: + """The settings with each ``secret:`` / ``env:`` reference resolved from the deploying user's config.""" + if not self.settings: + return {} + from agent_env.config import get_config + resolve = get_config().resolve_reference + return {name: resolve(value) for name, value in self.settings.items()} @dataclass @@ -539,6 +549,7 @@ def create_docker_compose( f" - PORT={AGENT_ENV_WEBSITE_BACKEND_PORT}", " - PYTHONUNBUFFERED=1", *([f" - {WEBSITE_ROLE_KEY_ENV}={role_key}"] if role_key else []), + *(f" - {name}={value}" for name, value in wc.env_vars().items()), " healthcheck:", f' test: ["CMD", "curl", "-f", "http://localhost:{AGENT_ENV_WEBSITE_BACKEND_PORT}{wc.health_path}"]', " interval: 10s", @@ -1515,7 +1526,8 @@ def website_config(env: WebsiteEnv) -> WebsiteConfig: frontend = env.frontend_docker_image_artifact return WebsiteConfig(backend_image=env.server_docker_image_artifact.image_name, frontend_image=frontend.image_name if frontend else None, - environment_name=env.environment_name, health_path=env.health_path) + environment_name=env.environment_name, health_path=env.health_path, + settings=dict(env.settings) or None) def _gateway_topology(env: Env) -> _GatewayTopology: diff --git a/src/agent_env/task/episode/rules.py b/src/agent_env/task/episode/rules.py index 10f3d78c..4ba76560 100644 --- a/src/agent_env/task/episode/rules.py +++ b/src/agent_env/task/episode/rules.py @@ -151,7 +151,7 @@ def _rubrics_verifier(step: RubricsVerifierTaskStep, outside: Outside) -> list[s "aggregate_verifiers", "apply_server_config", "build_mcp_cli", "combine_universe_verdicts", "configure_scheduled_checks", "deploy_env", "deploy_sandbox", "end_computer_recording", "start_computer_recording", "env_outcome_verifier", "modify_env_tool_access", "reset_env", "review", - "run_container_unit_tests_verifier", "run_docker_container", "snapshot_env", "snapshot_computer_env", "sync_env_clock", + "run_container_unit_tests_verifier", "run_docker_container", "sign_in_browser", "snapshot_env", "snapshot_computer_env", "sync_env_clock", "validation_gate_aggregator", "verify_env_card", "verify_env_core_protocol", "verify_mcp_tool_schema", "verify_spec_conformance", "verify_universe_load_export_roundtrip", "website_state_verifier", # They read only what an outside turn leaves: its reply. diff --git a/src/agent_env/task_step/registry.py b/src/agent_env/task_step/registry.py index a4457a32..d7809369 100644 --- a/src/agent_env/task_step/registry.py +++ b/src/agent_env/task_step/registry.py @@ -47,6 +47,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: from agent_env.task_step.task_steps.register_agent_triggers import RegisterAgentTriggersStep from agent_env.task_step.task_steps.sync_env_clock import SyncEnvClockTaskStep from agent_env.task_step.task_steps.configure_scheduled_checks import ConfigureScheduledChecksTaskStep + from agent_env.task_step.task_steps.sign_in_browser import SignInBrowserTaskStep from agent_env.task_step.task_steps.install_agent import InstallAgentTaskStep from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep from agent_env.task_step.task_steps.verifiers.run_container_unit_tests_verifier import RunContainerUnitTestsVerifierTaskStep @@ -105,6 +106,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: _get_type(RegisterAgentTriggersStep): RegisterAgentTriggersStep, _get_type(SyncEnvClockTaskStep): SyncEnvClockTaskStep, _get_type(ConfigureScheduledChecksTaskStep): ConfigureScheduledChecksTaskStep, + _get_type(SignInBrowserTaskStep): SignInBrowserTaskStep, _get_type(InstallAgentTaskStep): InstallAgentTaskStep, _get_type(RubricsVerifierTaskStep): RubricsVerifierTaskStep, _get_type(RunContainerUnitTestsVerifierTaskStep): RunContainerUnitTestsVerifierTaskStep, diff --git a/src/agent_env/task_step/task_steps/deploy_env.py b/src/agent_env/task_step/task_steps/deploy_env.py index 375a4c51..a353f1b4 100644 --- a/src/agent_env/task_step/task_steps/deploy_env.py +++ b/src/agent_env/task_step/task_steps/deploy_env.py @@ -191,7 +191,9 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: deployed_env = await env.deploy(**deploy_kwargs) # create_instance runs inside env.deploy(), so an annotation set after it # returns has to be persisted explicitly to reach the instance record. - annotations = {"deploy_step_id": self.id} + from agent_env.task_step.task_steps.sign_in_browser import mark_live + + annotations = {"deploy_step_id": self.id, **mark_live(context, deployed_env)} deployed_env.metadata = {**(deployed_env.metadata or {}), **annotations} if deployed_env.instance_id: update_env_instance_metadata(deployed_env.instance_id, annotations) diff --git a/src/agent_env/task_step/task_steps/reset_env.py b/src/agent_env/task_step/task_steps/reset_env.py index 88059fdd..7ff94195 100644 --- a/src/agent_env/task_step/task_steps/reset_env.py +++ b/src/agent_env/task_step/task_steps/reset_env.py @@ -3,7 +3,8 @@ Env-agnostic: looks up the deployed env in context and calls its ``reset()``. Best-effort by default — a reset failure (including an env that doesn't support it) is logged and the run continues. Place right before prompt_agent in a -warm-reuse task. +warm-reuse task. After a reset, the browser is signed in again to the live websites +a sign_in_browser step of the run signed it in to. """ from __future__ import annotations @@ -61,6 +62,9 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: env = Env.get(deployed.env_id, deployed.env_version) await env.reset(deployed) logger.info("reset_env: reset env '%s'", self.env_id) + from agent_env.task_step.task_steps.sign_in_browser import sign_in_again + + await sign_in_again(context, self.env_id) except NotImplementedError: # Env doesn't support reset — not an error. logger.info("reset_env: env '%s' does not support reset; skipping", self.env_id) diff --git a/src/agent_env/task_step/task_steps/sign_in_browser.py b/src/agent_env/task_step/task_steps/sign_in_browser.py new file mode 100644 index 00000000..57ac5f48 --- /dev/null +++ b/src/agent_env/task_step/task_steps/sign_in_browser.py @@ -0,0 +1,256 @@ +"""Live websites in a run: marking a run that uses one, and signing the browser in to them. + +A website child env whose card's website/v1 says ``live`` is a real site. ``mark_live``, which deploy_env runs on every +env it deploys, records each one under ``context.metadata["live_websites"]``, sets ``context.metadata["reproducible"]`` +to False, and marks the deploy record ``live``. It also checks the site's domains and ``allow_hosts`` against the network +allowlist of each computer deployed in the run. An allowlist is fixed when its sandbox starts, so a host one leaves out +is recorded under ``unreachable`` and warned about, not added; a run whose sandboxes restrict no egress records +``network: "unrestricted"``, since nothing there enforces ``allow_hosts`` either. + +``sign_in_browser`` calls each live site's website/v1 ``session`` and hands the cookies straight to the browser's +browser/v1 ``cookies/set``. The cookies never enter the context, a log or the step's record, which keeps only each +cookie's name, domain, path and expiry. reset_env signs the browser in again after a reset, as the last sign-in did. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import Any, ClassVar, Optional + +import httpx +from agentenv_protocol import client as protocol_v1 +from agentenv_protocol.website import BROWSER_EXTENSION_URI, CLEAR_COOKIES, SET_COOKIES, WEBSITE_EXTENSION_URI +from agentenv_protocol.website.browser import cookies as parse_cookies +from agentenv_protocol.website.browser import redacted + +from agent_env.entity.refs import EntityRef +from agent_env.env.env import EnvCapabilityUnsupported +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_step import TaskStep, TaskStepDependency + +logger = logging.getLogger(__name__) + +LIVE_KEY = "live_websites" +REPRODUCIBLE_KEY = "reproducible" +SIGN_INS_KEY = "browser_sign_ins" + + +@dataclass(frozen=True) +class LiveSite: + """A live website child env, as its card declares it.""" + + name: str + domains: tuple[str, ...] + allow_hosts: tuple[str, ...] + signs_in: bool + + def hosts(self) -> list[str]: + return [*self.domains, *(h for h in self.allow_hosts if h not in self.domains)] + + def record(self) -> dict: + return {"name": self.name, "domains": list(self.domains), "allow_hosts": list(self.allow_hosts), + "signs_in": self.signs_in} + + +def _children(card: dict) -> list[dict]: + children = card.get("children_environments") + return [c for c in children if isinstance(c, dict)] if isinstance(children, list) else [card] + + +def live_sites(card: Optional[dict]) -> list[LiveSite]: + """The live websites among ``card``'s child envs, or ``card`` itself for a leaf card.""" + sites = [] + for child in _children(card or {}): + params = (protocol_v1.find_extension(child, WEBSITE_EXTENSION_URI) or {}).get("params") or {} + if params.get("live") is True and isinstance(child.get("name"), str): + sites.append(LiveSite(child["name"], tuple(params.get("domains") or ()), + tuple(params.get("allow_hosts") or ()), + params.get("signs_in") is not False + and protocol_v1.find_extension_method(child, WEBSITE_EXTENSION_URI, "session") is not None)) + return sites + + +def browsers(card: Optional[dict]) -> list[str]: + """The names of ``card``'s child envs that serve browser/v1's cookies/set.""" + return [child["name"] for child in _children(card or {}) if isinstance(child.get("name"), str) + and protocol_v1.find_extension_method(child, BROWSER_EXTENSION_URI, SET_COOKIES) is not None] + + +def _allows(entries: list[str], host: str) -> bool: + host = host.lower().rstrip(".") + for entry in (e.lower().rstrip(".") for e in entries): + if entry == host or (entry.startswith("*.") and host.endswith(entry[1:])): + return True + return False + + +def network_check(context: TaskStepContext, sites: list[LiveSite]) -> dict: + """How the run's computers' network allowlists treat the live ``sites``' hosts: each restricting computer's + hosts it leaves out, or ``unrestricted`` when no computer in the run restricts egress.""" + from agent_env.providers.sandbox_providers.sandbox import NetworkPolicy + + hosts = list(dict.fromkeys(h for site in sites for h in site.hosts())) + unreachable: dict[str, list[str]] = {} + restricted = False + for deployed in context.deployed_envs: + network = ((deployed.metadata or {}).get("computer") or {}).get("network") + if not network: + continue + policy = NetworkPolicy.from_dict(network) + if not policy.restricts_egress: + continue + restricted = True + if missing := [h for h in hosts if not _allows(list(policy.allow_hosts), h)]: + unreachable[deployed.env_id] = missing + if not restricted: + return {"network": "unrestricted"} + return {"network": "allowlist", "unreachable": unreachable} + + +def mark_live(context: TaskStepContext, deployed: Any) -> dict: + """Record ``deployed``'s live websites in ``context`` and return what its deploy record gains: ``live`` and + ``reproducible`` when it has one, nothing when it has none.""" + sites = live_sites(deployed.environment_card) + if not sites: + return {} + network = network_check(context, sites) + context.metadata.setdefault(LIVE_KEY, {})[deployed.env_id] = {"sites": [s.record() for s in sites], **network} + context.metadata[REPRODUCIBLE_KEY] = False + for env_id, missing in (network.get("unreachable") or {}).items(): + logger.warning("live websites of env %s: env %s's network allowlist leaves out %s, which it can't add once its " + "sandbox runs, so those pages won't load there", deployed.env_id, env_id, missing) + if network["network"] == "unrestricted": + logger.info("live websites of env %s (%s): no sandbox in this run restricts egress, so allow_hosts is " + "recorded, not enforced", deployed.env_id, [s.name for s in sites]) + return {"live": True, REPRODUCIBLE_KEY: False} + + +async def sign_in(context: TaskStepContext, deployed: Any, *, users: Optional[dict[str, str]] = None, + sites: Optional[list[str]] = None, browser: Optional[str] = None, clear: bool = True, + timeout: int = 30) -> dict: + """Sign ``deployed``'s browser in to its live websites (those named in ``sites``, else every one with a session), + each as ``users[name]`` or the site's own choice; with ``clear``, a site's old cookies go first. Returns the record + of what was set, cookies by everything but their values.""" + card = deployed.environment_card or {} + live = {site.name: site for site in live_sites(card)} + unknown = sorted(set(sites or []) - set(live)) + if unknown: + raise RuntimeError(f"env {deployed.env_id!r} has no live website named {unknown}; its live ones are " + f"{sorted(live) or 'none'}") + chosen = [live[name] for name in sites] if sites else [s for s in live.values() if s.signs_in] + offered = browsers(card) + if chosen and browser is None: + if not offered: + raise RuntimeError(f"env {deployed.env_id!r} has live websites with a session ({[s.name for s in chosen]}) " + f"but no child env serves {BROWSER_EXTENSION_URI} {SET_COOKIES} to sign a browser in") + browser = offered[0] + record: dict = {"env_id": deployed.env_id, "browser": browser, "sites": []} + for site in chosen: + user = (users or {}).get(site.name) + try: + session = await deployed.invoke(WEBSITE_EXTENSION_URI, "session", {"user": user}, + environment_name=site.name, timeout=timeout) + except Exception as e: + raise RuntimeError(f"live website {site.name!r}: its session failed ({_why(e)})") from None + try: + given = parse_cookies(session.get("cookies") if isinstance(session, dict) else None, + default_domain=site.domains[0] if site.domains else None) + except ValueError as e: + raise RuntimeError(f"live website {site.name!r}: its session returned {e}") from None + try: + if clear and site.domains: + await deployed.invoke(BROWSER_EXTENSION_URI, CLEAR_COOKIES, {"domains": list(site.domains)}, + environment_name=browser, timeout=timeout) + await deployed.invoke(BROWSER_EXTENSION_URI, SET_COOKIES, {"cookies": [c.model_dump(exclude_none=True) + for c in given]}, + environment_name=browser, timeout=timeout) + except Exception as e: + raise RuntimeError(f"browser {browser!r}: setting {site.name!r}'s cookies failed ({_why(e)})") from None + record["sites"].append({"name": site.name, "user": user, "expires_at": session.get("expires_at"), + "cookies": redacted(given)}) + logger.info("sign_in_browser: %s signed in to %s as %s (%s)", browser, site.name, user or "its own choice", + [c.name for c in given]) + return record + + +def _why(error: Exception) -> str: + """An error as a record keeps it: a call's status and error code, or the error's kind, never a message that may + repeat a cookie.""" + if isinstance(error, httpx.HTTPStatusError): + code = getattr(error, "code", None) + return f"HTTP {error.response.status_code}{f' {code}' if code else ''}" + if isinstance(error, EnvCapabilityUnsupported): + return str(error) + return type(error).__name__ + + +async def sign_in_again(context: TaskStepContext, env_id: str) -> None: + """After a reset of ``env_id``, sign its browser in again as each sign-in of this run did; never raises.""" + deployed = next((d for d in context.deployed_envs if d.env_id == env_id), None) + for step_id, entry in list((context.metadata.get(SIGN_INS_KEY) or {}).items()): + if deployed is None or entry.get("env_id") != env_id or not entry.get("sites"): + continue + users = {s["name"]: s["user"] for s in entry["sites"] if s.get("user")} + try: + context.metadata[SIGN_INS_KEY][step_id] = await sign_in( + context, deployed, users=users, sites=[s["name"] for s in entry["sites"]], browser=entry.get("browser")) + except Exception as e: + logger.warning("reset_env: signing env %s's browser in again failed: %s", env_id, e) + entry["error"] = str(e)[:500] + + +class SignInBrowserTaskStep(TaskStep): + """Sign a deployed env's browser in to its live websites before the agent starts. ``users`` maps a site's name to + the user to sign in as; ``sites`` limits it to some live sites; ``browser`` names the child env serving browser/v1 + when there are several.""" + + type: ClassVar[str] = "sign_in_browser" + entity_refs = (EntityRef.env("env_id"),) + + def __init__(self, id: str, version: Optional[int], env_id: str, users: Optional[dict[str, str]] = None, + sites: Optional[list[str]] = None, browser: Optional[str] = None, + depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = True, + timeout_seconds: int = 30): + super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + if not env_id: + raise ValueError("sign_in_browser needs the env_id of the env whose browser it signs in") + if users is not None and not (isinstance(users, dict) and all( + isinstance(k, str) and isinstance(v, str) for k, v in users.items())): + raise ValueError("sign_in_browser's users maps a site's name to a user's") + self.env_id, self.users, self.sites, self.browser = env_id, users, sites, browser + self.timeout_seconds = timeout_seconds + + def to_dict(self) -> dict: + data = {**super().to_dict(), "env_id": self.env_id, "timeout_seconds": self.timeout_seconds} + for key in ("users", "sites", "browser"): + if getattr(self, key) is not None: + data[key] = getattr(self, key) + return data + + @classmethod + def from_dict(cls, data: dict) -> SignInBrowserTaskStep: + return cls(**cls._base_from_dict(data), env_id=data["env_id"], users=data.get("users"), + sites=data.get("sites"), browser=data.get("browser"), + timeout_seconds=data.get("timeout_seconds", 30)) + + async def execute(self, context: TaskStepContext) -> TaskStepContext: + deployed = next((d for d in context.deployed_envs if d.env_id == self.env_id), None) + if deployed is None: + raise RuntimeError(f"Env '{self.env_id}' not found in context.deployed_envs") + if live_sites(deployed.environment_card) and self.env_id not in (context.metadata.get(LIVE_KEY) or {}): + mark_live(context, deployed) + try: + record = await sign_in(context, deployed, users=self.users, sites=self.sites, browser=self.browser, + timeout=self.timeout_seconds) + except Exception as e: + context.metadata.setdefault(SIGN_INS_KEY, {})[self.id] = {"env_id": self.env_id, "error": str(e)[:500]} + logger.warning("sign_in_browser %s: %s", self.id, e) + if self.fail_task_on_error: + raise + return context + context.metadata.setdefault(SIGN_INS_KEY, {})[self.id] = record + if not record["sites"]: + logger.info("sign_in_browser %s: env %s has no live website with a session; nothing to sign in to", + self.id, self.env_id) + return context diff --git a/tst/unit/env/envs/test_website_env_settings.py b/tst/unit/env/envs/test_website_env_settings.py new file mode 100644 index 00000000..d20cd77d --- /dev/null +++ b/tst/unit/env/envs/test_website_env_settings.py @@ -0,0 +1,58 @@ +"""A website env's settings: environment variables of its server's container, a live site's credentials among them. +A secret is stored as its reference, never its value, and resolved from the deploying user's config only into the +compose file the gateway runs.""" + +import pytest + +from agent_env.artifact.artifacts.docker_image import DockerImageArtifact +from agent_env.artifact.store import get_artifact_store +from agent_env.env import Env +from agent_env.env.envs.service_db import ServiceDBConfig +from agent_env.env.envs.website import WebsiteEnv, settings_problems +from agent_env.providers.env_providers.env_gateway_provider import EnvironmentGatewayProvider, website_config +from agent_env.providers.env_state import LocalPostgresStateProvider + + +def _image(id): + return get_artifact_store().put_document(DockerImageArtifact( + id=id, description="", image_name=f"registry.example/{id}@sha256:" + "a" * 64)) + + +def test_a_secret_setting_is_stored_as_its_reference(local_stores): + WebsiteEnv.put(id="github-site", server_docker_image_artifact=_image("github-server"), environment_name="github", + settings={"GITHUB_TOKEN": "secret:github-token", "GITHUB_ORG": "acme"}) + stored = Env.get("github-site") + assert stored.settings == {"GITHUB_TOKEN": "secret:github-token", "GITHUB_ORG": "acme"} + assert stored.to_dict()["settings"] == stored.settings + assert "settings" not in WebsiteEnv.put(id="plain", server_docker_image_artifact=_image("plain-server"), + environment_name="plain").to_dict() + + +@pytest.mark.parametrize("settings, problem", [ + ({"GITHUB_TOKEN": "ghp_the_value_itself"}, "settings.GITHUB_TOKEN is a secret"), + ({"API_PASSWORD": "secret:pw?fallback"}, "with no ?default"), + ({"DATABASE_URL": "postgres://elsewhere"}, "the gateway doesn't set itself"), + ({"not a name": "x"}, "an environment variable's name"), + ({"RETRIES": 3}, "settings.RETRIES is a string, not a int"), +]) +def test_a_setting_that_isnt_one_is_refused_without_its_value(settings, problem): + (found,) = settings_problems(settings) + assert problem in found + assert "ghp_the_value_itself" not in found and "fallback" not in found + with pytest.raises(ValueError, match="website env 'x'"): + WebsiteEnv(id="x", version=None, environment_name="x", settings=settings) + + +def test_the_compose_file_holds_the_resolved_settings_and_the_gateway_config_the_references(local_stores, + monkeypatch): + env = WebsiteEnv(id="github-site", version=None, server_docker_image_artifact=_image("github-server"), + environment_name="github", settings={"GITHUB_TOKEN": "env:TEST_GITHUB_TOKEN", "ORG": "acme"}) + monkeypatch.setenv("TEST_GITHUB_TOKEN", "ghp_resolved") + config = website_config(env) + assert config.settings == {"GITHUB_TOKEN": "env:TEST_GITHUB_TOKEN", "ORG": "acme"} + compose = EnvironmentGatewayProvider().create_docker_compose( + mcp_servers=[], website_configs=[config], + state_provider=LocalPostgresStateProvider(service_db_config=ServiceDBConfig()), + state_instance=LocalPostgresStateProvider.default_instance()) + assert " - GITHUB_TOKEN=ghp_resolved" in compose.splitlines() + assert " - ORG=acme" in compose.splitlines() diff --git a/tst/unit/task_step/sign_in_browser_test.py b/tst/unit/task_step/sign_in_browser_test.py new file mode 100644 index 00000000..7e5a54b5 --- /dev/null +++ b/tst/unit/task_step/sign_in_browser_test.py @@ -0,0 +1,204 @@ +"""sign_in_browser and the live-site marks: a deploy with a live website is marked not reproducible with its hosts +checked against the run's network allowlists, and the step hands each live site's session cookies to the browser's +browser/v1 cookies/set, through the gateway's child routes, keeping no cookie value in the context or its logs.""" + +from __future__ import annotations + +import json +import logging +from typing import Optional + +import pytest +from agentenv_protocol import client as protocol_client +from agentenv_protocol.agent_env_environment import (AgentEnvEnvironment, AgentEnvStarletteApplication, + environment_card, extension) +from agentenv_protocol.website import BROWSER_EXTENSION_URI, WebsiteEnvironment, session, website +from starlette.applications import Starlette +from starlette.routing import Mount +from starlette.testclient import TestClient + +from agent_env.env import env as env_module +from agent_env.env.env import DeployedEnv +from agent_env.env.gateway.constants import GATEWAY_EXTENSIONS +from agent_env.env.gateway.rest_proxy import rewrite_child_card +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.registry import get_task_step_registry +from agent_env.task_step.task_steps.reset_env import ResetEnvTaskStep +from agent_env.task_step.task_steps.sign_in_browser import (LIVE_KEY, SIGN_INS_KEY, SignInBrowserTaskStep, live_sites, + mark_live) + +TOKEN = "ghp_live_secret_value" + + +@environment_card(name="github") +@website(domains=["github.com"], live=True, allow_hosts=["*.githubusercontent.com"]) +class GitHub(WebsiteEnvironment): + @session + def sign_in(self, user: Optional[str]) -> list[dict]: + return [{"name": "user_session", "value": f"{TOKEN}-{user}", "secure": True, "httpOnly": True, + "expires": 1_900_000_000}] + + +@environment_card(name="notes") +@website(domains=["notes.test"]) +class Notes(WebsiteEnvironment): + pass + + +@environment_card(name="playwright-mcp") +class Browser(AgentEnvEnvironment): + def __init__(self, refuse: bool = False) -> None: + self.calls: list[tuple[str, dict]] = [] + self.refuse = refuse + + @extension(BROWSER_EXTENSION_URI, op="cookies/set") + async def _set(self, cookies: list[dict]) -> dict: + self.calls.append(("set", {"cookies": cookies})) + if self.refuse: + from agentenv_protocol.errors import AgentEnvError + raise AgentEnvError("browser_unreachable", f"refused {cookies[0]['value']}", status=502) + return {"set": len(cookies), "open_browser": False} + + @extension(BROWSER_EXTENSION_URI, op="cookies/clear") + async def _clear(self, domains: Optional[list[str]] = None) -> dict: + self.calls.append(("clear", {"domains": domains})) + return {"cleared": 0, "open_browser": False} + + +def _card_app(env: AgentEnvEnvironment) -> Starlette: + app = Starlette() + AgentEnvStarletteApplication(env._build_card(), env).add_routes_to_app(app) + return app + + +class _Gateway: + """The gateway's child routes and composed card, over the children's own apps in process.""" + + def __init__(self, browser: Browser, *, live: bool = True) -> None: + children = {"github": GitHub().create_app() if live else None, "notes": Notes().create_app(), + "mcp-playwright-mcp": _card_app(browser)} + children = {key: app for key, app in children.items() if app is not None} + self.client = TestClient(Starlette(routes=[Mount(f"/svc/{key}", app) for key, app in children.items()])) + cards = [rewrite_child_card(key, TestClient(app).get("/.well-known/agent-env.json").json(), GATEWAY_EXTENSIONS) + for key, app in children.items()] + self.card = {"name": "env", "capabilities": {"extensions": GATEWAY_EXTENSIONS}, + "children_environments": cards} + + def deployed(self, metadata: Optional[dict] = None) -> DeployedEnv: + return DeployedEnv(env_id="e1", env_version=1, environment_card_url="http://gw/.well-known/agent-env.json", + environment_card=self.card, metadata=metadata) + + async def invoke(self, base_url, card, uri, params=None, timeout=30, verify=True, *, method=None): + op = protocol_client.find_extension_method(card, uri, method) + response = self.client.request(op["method"], op["endpoint"], json=params or {}) + if response.status_code >= 400: + raise protocol_client.ExtensionCallError(response) + return response.json() + + +@pytest.fixture +def browser() -> Browser: + return Browser() + + +@pytest.fixture +def gateway(browser, monkeypatch) -> _Gateway: + gateway = _Gateway(browser) + monkeypatch.setattr(env_module.protocol_v1, "invoke_extension", gateway.invoke) + return gateway + + +def test_the_step_is_registered_and_round_trips(): + step = SignInBrowserTaskStep(id="s", version=1, env_id="e1", users={"github": "octocat"}, sites=["github"]) + data = step.to_dict() + assert data["type"] == "sign_in_browser" and get_task_step_registry()["sign_in_browser"] is SignInBrowserTaskStep + again = SignInBrowserTaskStep.from_dict(data) + assert (again.users, again.sites, again.browser, again.fail_task_on_error) == ({"github": "octocat"}, ["github"], + None, True) + with pytest.raises(ValueError, match="maps a site"): + SignInBrowserTaskStep(id="s", version=1, env_id="e1", users={"github": 1}) + + +def test_only_a_live_child_is_a_live_site(gateway): + (site,) = live_sites(gateway.card) + assert (site.name, site.domains, site.allow_hosts, site.signs_in) == ("github", ("github.com",), + ("*.githubusercontent.com",), True) + + +def test_a_deploy_with_a_live_site_is_marked_not_reproducible(gateway): + context = TaskStepContext() + assert mark_live(context, gateway.deployed()) == {"live": True, "reproducible": False} + assert context.metadata["reproducible"] is False + assert context.metadata[LIVE_KEY]["e1"] == { + "sites": [{"name": "github", "domains": ["github.com"], "allow_hosts": ["*.githubusercontent.com"], + "signs_in": True}], "network": "unrestricted"} + assert mark_live(TaskStepContext(), _Gateway(Browser(), live=False).deployed()) == {} + + +def test_a_computers_allowlist_that_leaves_out_a_live_host_is_recorded_and_warned(gateway, caplog): + computer = DeployedEnv(env_id="pc", env_version=1, metadata={"computer": {"network": { + "mode": "allowlist", "allow_hosts": ["github.com"]}}}) + context = TaskStepContext(deployed_envs=[computer]) + with caplog.at_level(logging.WARNING): + mark_live(context, gateway.deployed()) + assert context.metadata[LIVE_KEY]["e1"]["unreachable"] == {"pc": ["*.githubusercontent.com"]} + assert "leaves out ['*.githubusercontent.com']" in caplog.text + + +@pytest.mark.asyncio +async def test_the_browser_is_signed_in_with_the_live_sites_session_and_no_value_is_kept(gateway, browser, caplog): + context = TaskStepContext(deployed_envs=[gateway.deployed()]) + step = SignInBrowserTaskStep(id="sign-in", version=1, env_id="e1", users={"github": "octocat"}) + with caplog.at_level(logging.DEBUG): + await step.execute(context) + assert browser.calls == [ + ("clear", {"domains": ["github.com"]}), + ("set", {"cookies": [{"name": "user_session", "value": f"{TOKEN}-octocat", "domain": "github.com", "path": "/", + "secure": True, "httpOnly": True, "expires": 1_900_000_000}]})] + record = context.metadata[SIGN_INS_KEY]["sign-in"] + assert record["browser"] == "playwright-mcp" and record["sites"][0]["user"] == "octocat" + assert record["sites"][0]["cookies"] == [{"name": "user_session", "domain": "github.com", "path": "/", + "secure": True, "httpOnly": True, "expires": 1_900_000_000.0}] + assert context.metadata["reproducible"] is False + assert TOKEN not in json.dumps(context.to_safe_dict(), default=str) and TOKEN not in caplog.text + + +@pytest.mark.asyncio +async def test_a_browser_that_refuses_fails_the_step_without_the_value(monkeypatch, caplog): + browser = Browser(refuse=True) + gateway = _Gateway(browser) + monkeypatch.setattr(env_module.protocol_v1, "invoke_extension", gateway.invoke) + context = TaskStepContext(deployed_envs=[gateway.deployed()]) + with pytest.raises(RuntimeError, match="HTTP 502 browser_unreachable") as e: + await SignInBrowserTaskStep(id="s", version=1, env_id="e1").execute(context) + assert TOKEN not in str(e.value) and TOKEN not in caplog.text + assert TOKEN not in json.dumps(context.metadata[SIGN_INS_KEY]) + + +@pytest.mark.asyncio +async def test_live_sites_without_a_browser_are_refused(monkeypatch, gateway): + card = {**gateway.card, "children_environments": [c for c in gateway.card["children_environments"] + if c["name"] != "playwright-mcp"]} + deployed = DeployedEnv(env_id="e1", env_version=1, environment_card_url="http://gw/.well-known/agent-env.json", + environment_card=card) + with pytest.raises(RuntimeError, match="no child env serves urn:agentenv:browser/v1"): + await SignInBrowserTaskStep(id="s", version=1, env_id="e1").execute(TaskStepContext(deployed_envs=[deployed])) + step = SignInBrowserTaskStep(id="s", version=1, env_id="e1", sites=["notes"], fail_task_on_error=False) + context = await step.execute(TaskStepContext(deployed_envs=[deployed])) + assert "no live website named ['notes']" in context.metadata[SIGN_INS_KEY]["s"]["error"] + + +@pytest.mark.asyncio +async def test_a_reset_signs_the_browser_in_again(gateway, browser, monkeypatch): + context = TaskStepContext(deployed_envs=[gateway.deployed()]) + await SignInBrowserTaskStep(id="sign-in", version=1, env_id="e1", users={"github": "octocat"}).execute(context) + browser.calls.clear() + + class _Env: + async def reset(self, deployed): + return None + + monkeypatch.setattr("agent_env.env.env.Env.get", lambda *a, **k: _Env()) + await ResetEnvTaskStep(id="reset", version=1, env_id="e1").execute(context) + assert [call for call, _ in browser.calls] == ["clear", "set"] + assert browser.calls[1][1]["cookies"][0]["value"] == f"{TOKEN}-octocat" From 30e2ac2aeb4f2511f62f78b8cb386fe23778c328 Mon Sep 17 00:00:00 2001 From: William Loo Date: Mon, 2 Nov 2026 10:40:07 -0800 Subject: [PATCH 05/20] fix(website): the merged website env docstring, and @website's declaration by keyword (FD-4683) Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/agentenv_protocol/website/environment.py | 4 ++-- src/agent_env/env/envs/website.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py b/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py index 638f92bf..de624e8d 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py @@ -131,8 +131,8 @@ def deco(cls: type) -> type: raise ValueError(f"{cls.__name__} is a live site, whose API is the real site's: it serves no @api " f"route, not {served[0].name}") if spec is None: - setattr(cls, _WEBSITE_ATTR, WebsiteDeclaration(tuple(domains), static, health, clock_patch, data, models, None, live, - tuple(allow_hosts))) + setattr(cls, _WEBSITE_ATTR, WebsiteDeclaration(tuple(domains), static, health, clock_patch, data, models, + live=live, allow_hosts=tuple(allow_hosts))) else: declared = spec if isinstance(spec, WebsiteSpec) else load(cls._site_path(str(spec))) if models is not None and declared.data.seed: diff --git a/src/agent_env/env/envs/website.py b/src/agent_env/env/envs/website.py index 22eb8c54..b6857300 100644 --- a/src/agent_env/env/envs/website.py +++ b/src/agent_env/env/envs/website.py @@ -1,8 +1,7 @@ """Website environment: one site, served by one server image that holds its pages and API. A site built before the server image held its pages may still name a separate frontend image, which is deprecated. Its env.toml may carry the site's website spec, ``spec = "website.toml"`` or an inline ``[spec]`` table, whose site name, -health path and domains the env takes.""" - +health path and domains the env takes. Its ``settings`` are environment variables of the server's container, a live site's credentials among them: a setting whose name reads as a secret's (a token, a password, ...) is stored as a ``secret:KEY`` or ``env:NAME`` reference, From 9f0d2757a83e3aa78e7dd04902aed7e996de2a55 Mon Sep 17 00:00:00 2001 From: William Loo Date: Mon, 2 Nov 2026 10:48:41 -0800 Subject: [PATCH 06/20] fix(website): a card says signs_in only for a live site, so a synthetic site's card is its spec's; tests follow the session cookie and invoke's headers (FD-4683) Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/agentenv_protocol/website/environment.py | 5 +++-- packages/agentenv-protocol/tests/test_website_live.py | 2 +- tst/unit/task_step/sign_in_browser_test.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py b/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py index de624e8d..14dc760d 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py @@ -324,9 +324,10 @@ def extension_params(self, uri: str) -> dict: if uri == WEBSITE_EXTENSION_URI: declaration = self.declaration() params = {**self.site_spec().to_card_extension(), "session_cookie": self.session_cookie, - "session_path": SESSION_PATH, "signs_in": self._session_hook() is not None or not declaration.live} + "session_path": SESSION_PATH} if declaration.live: - params.update(live=True, allow_hosts=list(declaration.allow_hosts)) + params.update(live=True, allow_hosts=list(declaration.allow_hosts), + signs_in=self._session_hook() is not None) del params["session_cookie"], params["session_path"] return params return super().extension_params(uri) diff --git a/packages/agentenv-protocol/tests/test_website_live.py b/packages/agentenv-protocol/tests/test_website_live.py index 62461bad..8d6a8c41 100644 --- a/packages/agentenv-protocol/tests/test_website_live.py +++ b/packages/agentenv-protocol/tests/test_website_live.py @@ -116,7 +116,7 @@ def as_persona(self, user): return [{"name": "persona", "value": user or "guest"}] signed = TestClient(Shop().create_app()).post(SESSION, json={"user": "ann"}).json() - assert signed["cookies"][0]["name"] == "agentenv_session" + assert signed["cookies"][0]["name"] == "agentenv_session_Shop" persona = TestClient(Persona().create_app()).post(SESSION, json={}).json() assert persona["cookies"] == [{"name": "persona", "value": "guest", "domain": "persona.example", "path": "/", "secure": False, "httpOnly": False}] diff --git a/tst/unit/task_step/sign_in_browser_test.py b/tst/unit/task_step/sign_in_browser_test.py index 7e5a54b5..ba523d6e 100644 --- a/tst/unit/task_step/sign_in_browser_test.py +++ b/tst/unit/task_step/sign_in_browser_test.py @@ -88,7 +88,7 @@ def deployed(self, metadata: Optional[dict] = None) -> DeployedEnv: return DeployedEnv(env_id="e1", env_version=1, environment_card_url="http://gw/.well-known/agent-env.json", environment_card=self.card, metadata=metadata) - async def invoke(self, base_url, card, uri, params=None, timeout=30, verify=True, *, method=None): + async def invoke(self, base_url, card, uri, params=None, timeout=30, verify=True, *, method=None, headers=None): op = protocol_client.find_extension_method(card, uri, method) response = self.client.request(op["method"], op["endpoint"], json=params or {}) if response.status_code >= 400: From 0c0d164d9b3cab2fe40f1d54d11a8e08a2c1582a Mon Sep 17 00:00:00 2001 From: William Loo Date: Mon, 2 Nov 2026 14:50:30 -0800 Subject: [PATCH 07/20] test(website): a website browser's browser/v1 is the one whose cookies sign_in_browser sets (FD-4683) main's WebsiteBrowserEnvironment (#449) declares urn:agentenv:browser/v1 with its build, clock and window; this branch's cookies/set and cookies/clear are operations of the same extension. A test keeps the two constants one URI. Co-Authored-By: Claude Opus 5.5 (1M context) --- packages/agentenv-protocol/tests/test_website_live.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/agentenv-protocol/tests/test_website_live.py b/packages/agentenv-protocol/tests/test_website_live.py index 8d6a8c41..48f7dd30 100644 --- a/packages/agentenv-protocol/tests/test_website_live.py +++ b/packages/agentenv-protocol/tests/test_website_live.py @@ -168,3 +168,12 @@ def test_a_cookie_is_kept_without_its_value_and_given_to_a_browser_in_its_own_sh assert Cookie(name="a", value="", domain=".A.example").matches(["a.example"]) assert Cookie(name="a", value="", domain="sub.a.example").matches(["a.example"]) assert not Cookie(name="a", value="", domain="nota.example").matches(["a.example"]) + + +def test_a_website_browser_declares_the_browser_v1_whose_cookies_sign_in_browser_sets(): + """A WebsiteBrowserEnvironment's card and the cookie operations name one extension, so a browser that adds them + is the one sign_in_browser finds.""" + from agentenv_protocol.website import BROWSER_EXTENSION_URI + from agentenv_protocol.website_browser import BROWSER + + assert BROWSER == BROWSER_EXTENSION_URI From 7d091de79207a42c932e2fd38e5a99c229504e7e Mon Sep 17 00:00:00 2001 From: William Loo Date: Mon, 2 Nov 2026 16:43:58 -0800 Subject: [PATCH 08/20] fix(website): a website setting is one quoted environment entry of its container, whatever its value holds (FD-4683) Each setting is written as one double-quoted YAML scalar, its quotes, backslashes and non-printable characters escaped and each $ doubled, so a newline can't add service keys like privileged or volumes and compose doesn't interpolate it. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../env_providers/env_gateway_provider.py | 19 +++++++++++++++++- .../env/envs/test_website_env_settings.py | 20 +++++++++++++++++-- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/src/agent_env/providers/env_providers/env_gateway_provider.py b/src/agent_env/providers/env_providers/env_gateway_provider.py index e0b26833..66710fcc 100644 --- a/src/agent_env/providers/env_providers/env_gateway_provider.py +++ b/src/agent_env/providers/env_providers/env_gateway_provider.py @@ -557,7 +557,7 @@ def create_docker_compose( f" - PORT={AGENT_ENV_WEBSITE_BACKEND_PORT}", " - PYTHONUNBUFFERED=1", *([f" - {WEBSITE_ROLE_KEY_ENV}={role_key}"] if role_key else []), - *(f" - {name}={value}" for name, value in wc.env_vars().items()), + *(f" - {compose_env_entry(name, value)}" for name, value in wc.env_vars().items()), " healthcheck:", # Up once it answers its card, or for a site serving none its health path; the deploy then waits # for it to be ready through the card (_wait_for_websites). @@ -1542,6 +1542,23 @@ class _GatewayTopology: names_gateway_on_vm: bool # a MultiEnv's record names its gateway sandbox even on a VM +def compose_env_entry(name: str, value: str) -> str: + """``name=value`` as one quoted YAML scalar of a compose ``environment:`` list, so the value stays that variable's + whatever it holds: a newline, quote or ``: `` can't start more of the service, and each ``$`` is ``$$``, which + compose doesn't interpolate. The scalar is plain ASCII, any other character written as its escape.""" + escaped = [] + for char in f"{name}={value}".replace("$", "$$"): + if char in '"\\': + escaped.append("\\" + char) + elif " " <= char <= "~": + escaped.append(char) + elif ord(char) <= 0xFFFF: + escaped.append(f"\\u{ord(char):04x}") + else: + escaped.append(f"\\U{ord(char):08x}") + return '"' + "".join(escaped) + '"' + + def website_config(env: WebsiteEnv) -> WebsiteConfig: """A website env as the gateway runs it: its server image, any deprecated frontend image, and its health path.""" frontend = env.frontend_docker_image_artifact diff --git a/tst/unit/env/envs/test_website_env_settings.py b/tst/unit/env/envs/test_website_env_settings.py index d20cd77d..7a738dbe 100644 --- a/tst/unit/env/envs/test_website_env_settings.py +++ b/tst/unit/env/envs/test_website_env_settings.py @@ -3,6 +3,7 @@ compose file the gateway runs.""" import pytest +import yaml from agent_env.artifact.artifacts.docker_image import DockerImageArtifact from agent_env.artifact.store import get_artifact_store @@ -54,5 +55,20 @@ def test_the_compose_file_holds_the_resolved_settings_and_the_gateway_config_the mcp_servers=[], website_configs=[config], state_provider=LocalPostgresStateProvider(service_db_config=ServiceDBConfig()), state_instance=LocalPostgresStateProvider.default_instance()) - assert " - GITHUB_TOKEN=ghp_resolved" in compose.splitlines() - assert " - ORG=acme" in compose.splitlines() + environment = yaml.safe_load(compose)["services"]["github-website-backend"]["environment"] + assert "GITHUB_TOKEN=ghp_resolved" in environment + assert "ORG=acme" in environment + + +def test_a_setting_holding_compose_syntax_stays_one_variable(local_stores): + hostile = 'ok\n privileged: true\n volumes:\n - /:/host\n# "q" \\ $HOME ${X} \u2028 é 😀 \x85' + env = WebsiteEnv(id="site", version=None, server_docker_image_artifact=_image("site-server"), + environment_name="site", settings={"EXTRA": hostile}) + compose = EnvironmentGatewayProvider().create_docker_compose( + mcp_servers=[], website_configs=[website_config(env)], + state_provider=LocalPostgresStateProvider(service_db_config=ServiceDBConfig()), + state_instance=LocalPostgresStateProvider.default_instance()) + service = yaml.safe_load(compose)["services"]["site-website-backend"] + assert "privileged" not in service and "volumes" not in service + (extra,) = [entry for entry in service["environment"] if entry.startswith("EXTRA=")] + assert extra == "EXTRA=" + hostile.replace("$", "$$") From f77fd85c0ecfd4243e85f2b2bac0e2537f94f5fb Mon Sep 17 00:00:00 2001 From: William Loo Date: Mon, 2 Nov 2026 16:43:58 -0800 Subject: [PATCH 09/20] fix(website): a replay's built-in routes and data follow the capture and models it serves now (FD-4683) Whether /api/session and /go answer from the capture is decided per request against the archive being served, so a capture loaded or reset through the data plane is matched as the declared one. data/add and data/reset hand what isn't a capture to the base hooks, so a models replay loads and resets its database. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/agentenv_protocol/website/replay.py | 42 +++++++++++---- .../tests/test_website_replay.py | 54 +++++++++++++++++++ 2 files changed, 85 insertions(+), 11 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/replay.py b/packages/agentenv-protocol/src/agentenv_protocol/website/replay.py index f2c54e07..1fd23246 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/replay.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/replay.py @@ -39,7 +39,7 @@ from starlette.applications import Starlette from starlette.responses import Response -from starlette.routing import Route +from starlette.routing import BaseRoute, Match, NoMatchFound, Route from ..agent_env_environment import add_data, reset_data from ..types import FilePart @@ -366,25 +366,26 @@ def create_app(self) -> Starlette: raise ValueError(f"{type(self).__name__} serves its capture, so it takes no static folder and isn't live") asgi = super().create_app() built_in = {SESSION_PATH: self._session, STATE_PATH: self._state} - asgi.router.routes = [r for r in asgi.router.routes if not ( - isinstance(r, Route) and built_in.get(r.path) == r.endpoint and self.archive.holds(r.path))] - asgi.router.default = _Replay(self, type(self).app) + paths = {r.path for r in asgi.router.routes if isinstance(r, Route) and built_in.get(r.path) == r.endpoint} + replay = _Replay(self, type(self).app) + asgi.router.routes.insert(0, _Captured(self, paths, replay)) + asgi.router.default = replay return asgi @default_store_hook @add_data async def _store_add(self, parts: list) -> None: - """Serve a capture among ``parts`` instead of the declared one; the rest load the store.""" - captures = [part for part in parts if _is_capture(part)] - for part in captures: - self.archive = Archive.read(_capture_source(part), self.ignore_query) - self.store.load_parts([part for part in parts if not _is_capture(part)]) - self.store.mark_initial() + """Serve a capture among ``parts`` instead of the declared one; the rest load the site's data as any site's.""" + for part in parts: + if _is_capture(part): + self.archive = Archive.read(_capture_source(part), self.ignore_query) + await super()._store_add([part for part in parts if not _is_capture(part)]) @default_store_hook @reset_data async def _store_reset(self) -> None: - self.store.reset() + """Serve the declared capture again, and reset the site's data as any site's.""" + await super()._store_reset() self.archive = self.declared_archive @@ -406,6 +407,25 @@ def _capture_source(part: FilePart) -> bytes | Path: return Path(unquote(uri.path)) +class _Captured(BaseRoute): + """The built-in routes at ``paths``, answered from the capture instead for a path the archive serving now holds, + so a capture loaded or reset later is matched as the one declared.""" + + def __init__(self, env: ReplayWebsite, paths: set[str], replay: _Replay) -> None: + self.env, self.paths, self.replay = env, paths, replay + + def matches(self, scope: dict) -> tuple[Match, dict]: + if scope["type"] == "http" and scope["path"] in self.paths and self.env.archive.holds(scope["path"]): + return Match.FULL, {} + return Match.NONE, {} + + def url_path_for(self, name: str, /, **path_params: Any) -> Any: + raise NoMatchFound(name, path_params) + + async def handle(self, scope: dict, receive: Any, send: Any) -> None: + await self.replay(scope, receive, send) + + class _Replay: """What no route answers: the capture's answer to the request, then the mounted app, then 404.""" diff --git a/packages/agentenv-protocol/tests/test_website_replay.py b/packages/agentenv-protocol/tests/test_website_replay.py index 96b643de..46f7cdf3 100644 --- a/packages/agentenv-protocol/tests/test_website_replay.py +++ b/packages/agentenv-protocol/tests/test_website_replay.py @@ -231,6 +231,60 @@ def test_a_capture_loaded_through_the_data_plane_is_served_until_a_reset(client, assert client.get("/").text == "from a file" +def test_the_built_in_routes_follow_the_capture_being_served(client): + def load(capture: bytes) -> None: + rpc(client, "data/add", {"parts": [{"kind": "file", "file": {"bytes": base64.b64encode(capture).decode(), + "name": "c.warc.gz"}}]}) + + assert client.get("/api/session").json() == {"captured": True} + load(warc(("https://shop.example/", 200, [], b"no session here"))) + assert client.get("/api/session").json()["signed_in"] is False + rpc(client, "data/reset") + assert client.get("/api/session").json() == {"captured": True} + + empty = TestClient(Shop(warc()).create_app()) + assert empty.get("/go").json()["state_diff"] == {} + rpc(empty, "data/add", {"parts": [{"kind": "file", "file": {"bytes": base64.b64encode(warc( + ("https://shop.example/go", 200, [("Content-Type", "application/json")], b'{"captured go": 1}'))).decode(), + "name": "go.warc"}}]}) + assert empty.get("/go").json() == {"captured go": 1} + + +class Rows: + """Data hooks of a models site, a list of rows.""" + + def __init__(self) -> None: + self.rows: list = [] + + def load(self, data: dict) -> None: + self.rows += data.get("rows", []) + + def export(self) -> dict: + return {"rows": list(self.rows)} + + def reset(self) -> None: + self.rows = [] + + +def test_a_models_replay_loads_and_resets_its_models_beside_the_capture(): + rows = Rows() + + @website(domains=["shop.example"], models=rows) + class Desk(ReplayWebsite): + pass + + desk = TestClient(Desk(CAPTURE).create_app()) + other = warc(("https://shop.example/", 200, [], b"the other capture")) + rpc(desk, "data/add", {"parts": [{"kind": "file", "file": {"bytes": base64.b64encode(other).decode(), + "name": "o.warc.gz"}}, + {"kind": "data", "data": {"rows": [{"id": 1}]}}]}) + assert desk.get("/").text == "the other capture" + assert rpc(desk, "data/get")["parts"][0]["data"] == {"rows": [{"id": 1}]} + rpc(desk, "data/reset") + assert rows.rows == [] and rpc(desk, "data/get")["parts"][0]["data"] == {"rows": []} + assert desk.get("/").text == "home" + + def test_a_replay_with_a_static_folder_is_refused(tmp_path): @website(domains=["x.example"], static=str(tmp_path)) class Static(ReplayWebsite): From 1f2fbe67ca2acd31a98ec31929c446deaa7c9f17 Mon Sep 17 00:00:00 2001 From: William Loo Date: Mon, 2 Nov 2026 16:44:54 -0800 Subject: [PATCH 10/20] fix(website): a computer deployed after a live site is checked against its hosts too (FD-4683) mark_live makes the network check of every live site recorded so far again on each deploy, with the env being deployed among the run's computers, so a website-first run records the allowlist that leaves a site's hosts out instead of unrestricted. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../task_step/task_steps/sign_in_browser.py | 46 +++++++++++-------- tst/unit/task_step/sign_in_browser_test.py | 15 ++++++ 2 files changed, 43 insertions(+), 18 deletions(-) diff --git a/src/agent_env/task_step/task_steps/sign_in_browser.py b/src/agent_env/task_step/task_steps/sign_in_browser.py index 57ac5f48..cf1e1b3d 100644 --- a/src/agent_env/task_step/task_steps/sign_in_browser.py +++ b/src/agent_env/task_step/task_steps/sign_in_browser.py @@ -85,16 +85,19 @@ def _allows(entries: list[str], host: str) -> bool: return False -def network_check(context: TaskStepContext, sites: list[LiveSite]) -> dict: +def network_check(context: TaskStepContext, sites: list[LiveSite], deployed: Any = None) -> dict: """How the run's computers' network allowlists treat the live ``sites``' hosts: each restricting computer's - hosts it leaves out, or ``unrestricted`` when no computer in the run restricts egress.""" + hosts it leaves out, or ``unrestricted`` when no computer in the run restricts egress. ``deployed``, an env being + deployed that the context doesn't hold yet, is one of the run's.""" from agent_env.providers.sandbox_providers.sandbox import NetworkPolicy hosts = list(dict.fromkeys(h for site in sites for h in site.hosts())) unreachable: dict[str, list[str]] = {} restricted = False - for deployed in context.deployed_envs: - network = ((deployed.metadata or {}).get("computer") or {}).get("network") + envs = [*context.deployed_envs, *([deployed] if deployed is not None and deployed not in context.deployed_envs + else [])] + for env in envs: + network = ((env.metadata or {}).get("computer") or {}).get("network") if not network: continue policy = NetworkPolicy.from_dict(network) @@ -102,7 +105,7 @@ def network_check(context: TaskStepContext, sites: list[LiveSite]) -> dict: continue restricted = True if missing := [h for h in hosts if not _allows(list(policy.allow_hosts), h)]: - unreachable[deployed.env_id] = missing + unreachable[env.env_id] = missing if not restricted: return {"network": "unrestricted"} return {"network": "allowlist", "unreachable": unreachable} @@ -110,20 +113,27 @@ def network_check(context: TaskStepContext, sites: list[LiveSite]) -> dict: def mark_live(context: TaskStepContext, deployed: Any) -> dict: """Record ``deployed``'s live websites in ``context`` and return what its deploy record gains: ``live`` and - ``reproducible`` when it has one, nothing when it has none.""" + ``reproducible`` when it has one, nothing when it has none. The network check of every live website recorded so + far is made again with ``deployed`` among the run's computers, so a computer deployed after a site is checked + too.""" sites = live_sites(deployed.environment_card) - if not sites: - return {} - network = network_check(context, sites) - context.metadata.setdefault(LIVE_KEY, {})[deployed.env_id] = {"sites": [s.record() for s in sites], **network} - context.metadata[REPRODUCIBLE_KEY] = False - for env_id, missing in (network.get("unreachable") or {}).items(): - logger.warning("live websites of env %s: env %s's network allowlist leaves out %s, which it can't add once its " - "sandbox runs, so those pages won't load there", deployed.env_id, env_id, missing) - if network["network"] == "unrestricted": - logger.info("live websites of env %s (%s): no sandbox in this run restricts egress, so allow_hosts is " - "recorded, not enforced", deployed.env_id, [s.name for s in sites]) - return {"live": True, REPRODUCIBLE_KEY: False} + if sites: + context.metadata.setdefault(LIVE_KEY, {})[deployed.env_id] = {"sites": [s.record() for s in sites]} + context.metadata[REPRODUCIBLE_KEY] = False + for env_id, entry in (context.metadata.get(LIVE_KEY) or {}).items(): + before = entry.pop("unreachable", None) or {} + entry_sites = sites if env_id == deployed.env_id else [ + LiveSite(r["name"], tuple(r["domains"]), tuple(r["allow_hosts"]), r["signs_in"]) for r in entry["sites"]] + network = network_check(context, entry_sites, deployed) + entry.update(network) + for computer, missing in (network.get("unreachable") or {}).items(): + if before.get(computer) != missing: + logger.warning("live websites of env %s: env %s's network allowlist leaves out %s, which it can't add " + "once its sandbox runs, so those pages won't load there", env_id, computer, missing) + if env_id == deployed.env_id and network["network"] == "unrestricted": + logger.info("live websites of env %s (%s): no sandbox in this run restricts egress, so allow_hosts is " + "recorded, not enforced", env_id, [s.name for s in sites]) + return {"live": True, REPRODUCIBLE_KEY: False} if sites else {} async def sign_in(context: TaskStepContext, deployed: Any, *, users: Optional[dict[str, str]] = None, diff --git a/tst/unit/task_step/sign_in_browser_test.py b/tst/unit/task_step/sign_in_browser_test.py index ba523d6e..33828c1e 100644 --- a/tst/unit/task_step/sign_in_browser_test.py +++ b/tst/unit/task_step/sign_in_browser_test.py @@ -145,6 +145,21 @@ def test_a_computers_allowlist_that_leaves_out_a_live_host_is_recorded_and_warne assert "leaves out ['*.githubusercontent.com']" in caplog.text +def test_a_computer_deployed_after_a_live_site_is_checked_too(gateway, caplog): + context = TaskStepContext() + site = gateway.deployed() + mark_live(context, site) + context.deployed_envs.append(site) + assert context.metadata[LIVE_KEY]["e1"]["network"] == "unrestricted" + computer = DeployedEnv(env_id="pc", env_version=1, metadata={"computer": {"network": { + "mode": "allowlist", "allow_hosts": ["github.com"]}}}) + with caplog.at_level(logging.WARNING): + assert mark_live(context, computer) == {} + assert context.metadata[LIVE_KEY]["e1"]["network"] == "allowlist" + assert context.metadata[LIVE_KEY]["e1"]["unreachable"] == {"pc": ["*.githubusercontent.com"]} + assert "env pc's network allowlist leaves out ['*.githubusercontent.com']" in caplog.text + + @pytest.mark.asyncio async def test_the_browser_is_signed_in_with_the_live_sites_session_and_no_value_is_kept(gateway, browser, caplog): context = TaskStepContext(deployed_envs=[gateway.deployed()]) From 21209ea0351bcf25408f3051facb664f9eaa3e2c Mon Sep 17 00:00:00 2001 From: William Loo Date: Mon, 2 Nov 2026 17:52:45 -0800 Subject: [PATCH 11/20] fix(website): a replay behind the gateway's prefix serves its capture by the site's own path (FD-4683) The gateway's X-Forwarded-Prefix leads the scope's path and raw_path, so the replay searched the capture by /website//... and answered 404. The replay and the captured built-in routes now match the path without the root_path, keeping its escaped segments. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/agentenv_protocol/website/replay.py | 21 +++++++++++++++---- .../tests/test_website_replay.py | 13 ++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/replay.py b/packages/agentenv-protocol/src/agentenv_protocol/website/replay.py index 1fd23246..f171fef0 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/replay.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/replay.py @@ -37,6 +37,7 @@ from typing import Any, BinaryIO, ClassVar, Iterable, Iterator, Optional from urllib.parse import parse_qsl, quote, unquote, urlencode, urljoin, urlsplit, urlunsplit +from starlette._utils import get_route_path from starlette.applications import Starlette from starlette.responses import Response from starlette.routing import BaseRoute, Match, NoMatchFound, Route @@ -415,7 +416,8 @@ def __init__(self, env: ReplayWebsite, paths: set[str], replay: _Replay) -> None self.env, self.paths, self.replay = env, paths, replay def matches(self, scope: dict) -> tuple[Match, dict]: - if scope["type"] == "http" and scope["path"] in self.paths and self.env.archive.holds(scope["path"]): + path = get_route_path(scope) + if scope["type"] == "http" and path in self.paths and self.env.archive.holds(path): return Match.FULL, {} return Match.NONE, {} @@ -426,6 +428,18 @@ async def handle(self, scope: dict, receive: Any, send: Any) -> None: await self.replay(scope, receive, send) +def _site_path(scope: dict) -> str: + """The site's own path a request names, escaped as it was sent: its ``raw_path`` without the ``root_path`` a + forwarded prefix leads it with, or the decoded path quoted again when the raw one doesn't spell it.""" + path = get_route_path(scope) or "/" + raw = (scope.get("raw_path") or b"").decode("latin-1").split("?", 1)[0] + root = scope.get("root_path", "") + if root and raw: + cut = next((i for i, char in enumerate(raw) if char == "/" and unquote(raw[:i]) == root), None) + raw = raw[cut:] if cut is not None else "" + return raw if raw and unquote(raw) == path else quote(path) + + class _Replay: """What no route answers: the capture's answer to the request, then the mounted app, then 404.""" @@ -437,9 +451,8 @@ async def __call__(self, scope: dict, receive: Any, send: Any) -> None: if self.app is not None: await self.app(scope, receive, send) return - raw = (scope.get("raw_path") or b"").decode("latin-1").split("?", 1)[0] - path = raw if raw and unquote(raw) == scope["path"] else quote(scope["path"]) - if reserved_path(scope["path"]): + path = _site_path(scope) + if reserved_path(unquote(path)): return await error_response(NotFound("not found"))(scope, receive, send) entry = self.env.archive.find(scope["method"], path, scope.get("query_string", b"").decode("latin-1")) if entry is None: diff --git a/packages/agentenv-protocol/tests/test_website_replay.py b/packages/agentenv-protocol/tests/test_website_replay.py index 46f7cdf3..c8f76ffa 100644 --- a/packages/agentenv-protocol/tests/test_website_replay.py +++ b/packages/agentenv-protocol/tests/test_website_replay.py @@ -215,6 +215,19 @@ def test_the_data_plane_exports_the_overrides_state(client): assert rpc(client, "data/get")["parts"][0]["data"] == {} +def test_a_replay_behind_the_gateways_prefix_matches_the_sites_own_path(client): + behind = {"X-Forwarded-Prefix": "/website/shop"} + assert client.get("/", headers=behind).text == "home" + assert client.get("/search?q=lamp&page=2", headers=behind).json() == {"hits": 2} + assert client.get("/api/session", headers=behind).json() == {"captured": True} + assert client.get("/api/price", headers=behind).json() == {"price": 7, "overridden": True} + assert client.get("/nowhere", headers=behind).status_code == 404 + + spaced = TestClient(Shop(warc(("https://shop.example/a%2Fb/x%20y", 200, [], b"escaped"))).create_app()) + assert spaced.get("/a%2Fb/x%20y", headers=behind).text == "escaped" + assert spaced.get("/a/b/x%20y", headers=behind).status_code == 404 + + def test_a_capture_loaded_through_the_data_plane_is_served_until_a_reset(client, tmp_path): other = warc(("https://shop.example/", 200, [("Content-Type", "text/plain")], b"the other capture")) by_bytes = {"kind": "file", "file": {"bytes": base64.b64encode(other).decode(), "name": "other.warc.gz"}} From cd70e0717c04e9ab89535379d020f414d8888110 Mon Sep 17 00:00:00 2001 From: William Loo Date: Mon, 2 Nov 2026 17:54:03 -0800 Subject: [PATCH 12/20] fix(website): the logged compose masks every website setting (FD-4683) compose_env_entry writes a setting as a quoted "NAME=VALUE" entry, which the secret-key mask didn't match, so a resolved token reached the deploy log. Every quoted setting line is now masked whatever its name, since a secret: or env: reference may resolve to a credential under any name. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../env_providers/env_gateway_provider.py | 8 +++++-- .../env/envs/test_website_env_settings.py | 22 ++++++++++++++++++- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/agent_env/providers/env_providers/env_gateway_provider.py b/src/agent_env/providers/env_providers/env_gateway_provider.py index b220b7cc..dd9a3c77 100644 --- a/src/agent_env/providers/env_providers/env_gateway_provider.py +++ b/src/agent_env/providers/env_providers/env_gateway_provider.py @@ -1625,6 +1625,7 @@ def _compose_name_problems(mcp_servers: list[MCPServerConfig], website_configs: # rendered compose is logged. Matches the key to the left of the first '=' on an # ` - KEY=VALUE` env line. _SECRET_ENV_KEY_RE = re.compile(r"^(\s*-\s*)([A-Za-z0-9_]*(?:SECRET|PASSWORD|TOKEN|API_KEY|ROLE_KEY)[A-Za-z0-9_]*)=.*$") +_SETTING_ENV_RE = re.compile(r'^(\s*-\s*")([A-Za-z_][A-Za-z0-9_]*)=.*"$') def _compose_ps_rows(text: str) -> list[dict]: @@ -1655,9 +1656,12 @@ def _why_down(row: dict) -> str | None: def _redact_compose_secrets(compose_content: str) -> str: - """Mask secret-valued env lines so the logged compose never carries a live secret.""" + """Mask secret-valued env lines so the logged compose never carries a live secret, and every website setting's + quoted ``"NAME=VALUE"`` line as ``compose_env_entry`` writes it, whatever its name: its value may be a credential + a ``secret:`` or ``env:`` reference resolved.""" return "\n".join( - _SECRET_ENV_KEY_RE.sub(r"\1\2=***", line) for line in compose_content.splitlines() + _SETTING_ENV_RE.sub(r'\1\2=***"', _SECRET_ENV_KEY_RE.sub(r"\1\2=***", line)) + for line in compose_content.splitlines() ) diff --git a/tst/unit/env/envs/test_website_env_settings.py b/tst/unit/env/envs/test_website_env_settings.py index 7a738dbe..18e7c658 100644 --- a/tst/unit/env/envs/test_website_env_settings.py +++ b/tst/unit/env/envs/test_website_env_settings.py @@ -10,7 +10,11 @@ from agent_env.env import Env from agent_env.env.envs.service_db import ServiceDBConfig from agent_env.env.envs.website import WebsiteEnv, settings_problems -from agent_env.providers.env_providers.env_gateway_provider import EnvironmentGatewayProvider, website_config +from agent_env.providers.env_providers.env_gateway_provider import ( + EnvironmentGatewayProvider, + _redact_compose_secrets, + website_config, +) from agent_env.providers.env_state import LocalPostgresStateProvider @@ -72,3 +76,19 @@ def test_a_setting_holding_compose_syntax_stays_one_variable(local_stores): assert "privileged" not in service and "volumes" not in service (extra,) = [entry for entry in service["environment"] if entry.startswith("EXTRA=")] assert extra == "EXTRA=" + hostile.replace("$", "$$") + + +def test_the_logged_compose_masks_every_resolved_setting(local_stores, monkeypatch): + env = WebsiteEnv(id="github-site", version=None, server_docker_image_artifact=_image("github-server"), + environment_name="github", settings={"GITHUB_TOKEN": "env:TEST_GITHUB_TOKEN", + "WEBHOOK": "env:TEST_WEBHOOK", "ORG": 'ac"me'}) + monkeypatch.setenv("TEST_GITHUB_TOKEN", "ghp_resolved") + monkeypatch.setenv("TEST_WEBHOOK", "https://hooks.example/s3cret") + compose = EnvironmentGatewayProvider().create_docker_compose( + mcp_servers=[], website_configs=[website_config(env)], + state_provider=LocalPostgresStateProvider(service_db_config=ServiceDBConfig()), + state_instance=LocalPostgresStateProvider.default_instance()) + logged = _redact_compose_secrets(compose) + assert "ghp_resolved" not in logged and "s3cret" not in logged + environment = yaml.safe_load(logged)["services"]["github-website-backend"]["environment"] + assert {"GITHUB_TOKEN=***", "WEBHOOK=***", "ORG=***", "SERVICE_NAME=github"} <= set(environment) From a3f20446f741190cc401db70bae877c44a19ac95 Mon Sep 17 00:00:00 2001 From: William Loo Date: Mon, 2 Nov 2026 17:55:44 -0800 Subject: [PATCH 13/20] fix(website): a replay loads a capture from the signed https URL a plugin's deployment sends (FD-4683) data/add of a capture took only its bytes or a file:// URI, so a load through a plugin's provider, which sends each file as a signed URL, failed after its data/reset. An https capture is now fetched, up to MAX_CAPTURE_URL_BYTES, and an error names its status, never the signed URL. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/agentenv_protocol/website/replay.py | 33 +++++++++++++++---- .../tests/test_website_replay.py | 30 +++++++++++++++++ 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/replay.py b/packages/agentenv-protocol/src/agentenv_protocol/website/replay.py index f171fef0..b2f9b093 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/replay.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/replay.py @@ -12,7 +12,8 @@ its query as ``request_key`` normalises them, and a request the capture doesn't hold answers 404. ``@api`` routes of a subclass are layered on top and win over the capture, and the default store holds their state, which the data plane exports. ``data/add`` of a file part named ``*.warc`` or ``*.warc.gz`` (or typed ``application/warc``) serves that -capture instead of the declared one until ``data/reset``. ``python -m agentenv_protocol.website.replay serve +capture instead of the declared one until ``data/reset``, the part's bytes, its ``file://`` file or what its https URL +(the signed URL a plugin's deployment sends) answers, up to ``MAX_CAPTURE_URL_BYTES``. ``python -m agentenv_protocol.website.replay serve capture.warc.gz --domain example.com`` serves one on ``PORT``. The normalisation, for the capture and for each request alike: the method is upper-cased; the path is ``/`` when @@ -48,6 +49,7 @@ from .errors import NotFound from .routes import error_response, reserved_path +MAX_CAPTURE_URL_BYTES = 1 << 30 DEFAULT_IGNORED_QUERY = ("utm_*", "fbclid", "gclid", "msclkid", "_") WARC_VERSION = "WARC/1.1" WARC_TYPES = ("application/warc", "application/warc+gzip") @@ -379,7 +381,7 @@ async def _store_add(self, parts: list) -> None: """Serve a capture among ``parts`` instead of the declared one; the rest load the site's data as any site's.""" for part in parts: if _is_capture(part): - self.archive = Archive.read(_capture_source(part), self.ignore_query) + self.archive = Archive.read(await _capture_source(part), self.ignore_query) await super()._store_add([part for part in parts if not _is_capture(part)]) @default_store_hook @@ -398,14 +400,33 @@ def _is_capture(part: Any) -> bool: return getattr(file, "mimeType", None) in WARC_TYPES or name.endswith((".warc", ".warc.gz")) -def _capture_source(part: FilePart) -> bytes | Path: +async def _capture_source(part: FilePart) -> bytes | Path: file = part.file if getattr(file, "bytes", None) is not None: return base64.b64decode(file.bytes) uri = urlsplit(file.uri) - if uri.scheme != "file": - raise ValueError(f"a capture is loaded by its bytes or a file:// URI, not {file.uri!r}") - return Path(unquote(uri.path)) + if uri.scheme == "file": + return Path(unquote(uri.path)) + if uri.scheme != "https": + raise ValueError(f"a capture is loaded by its bytes, a file:// URI or an https URL, not a {uri.scheme}: URI") + return await _fetched(file.uri) + + +async def _fetched(url: str) -> bytes: + """What ``url`` answers, refused past ``MAX_CAPTURE_URL_BYTES``; an error names its status, never the URL, which + may be signed.""" + import httpx + + data = bytearray() + async with httpx.AsyncClient(follow_redirects=True, timeout=60) as client: + async with client.stream("GET", url) as response: + if response.status_code != 200: + raise ValueError(f"the capture's URL answered {response.status_code}") + async for chunk in response.aiter_bytes(): + data += chunk + if len(data) > MAX_CAPTURE_URL_BYTES: + raise ValueError(f"the capture at the part's URL is over {MAX_CAPTURE_URL_BYTES} bytes") + return bytes(data) class _Captured(BaseRoute): diff --git a/packages/agentenv-protocol/tests/test_website_replay.py b/packages/agentenv-protocol/tests/test_website_replay.py index c8f76ffa..24898322 100644 --- a/packages/agentenv-protocol/tests/test_website_replay.py +++ b/packages/agentenv-protocol/tests/test_website_replay.py @@ -244,6 +244,36 @@ def test_a_capture_loaded_through_the_data_plane_is_served_until_a_reset(client, assert client.get("/").text == "from a file" +def test_a_capture_loaded_by_a_signed_https_url_is_fetched_within_its_limit(client, monkeypatch): + import httpx + + from agentenv_protocol.website import replay + + other = warc(("https://shop.example/", 200, [], b"from a signed url")) + asked = [] + + def answer(request: httpx.Request) -> httpx.Response: + asked.append(str(request.url)) + return httpx.Response(200, content=other) if request.url.path == "/c.warc.gz" else httpx.Response(403) + + real = httpx.AsyncClient + monkeypatch.setattr(httpx, "AsyncClient", lambda **kw: real(transport=httpx.MockTransport(answer), **kw)) + + def load(url: str) -> dict: + part = {"kind": "file", "file": {"uri": url, "name": "c.warc.gz"}} + return client.post(RPC, json={"jsonrpc": "2.0", "id": 1, "method": "data/add", + "params": {"parts": [part]}}).json() + + assert "error" not in load("https://bucket.example/c.warc.gz?X-Amz-Signature=abc") + assert asked == ["https://bucket.example/c.warc.gz?X-Amz-Signature=abc"] + assert client.get("/").text == "from a signed url" + refused = load("https://bucket.example/gone.warc.gz?X-Amz-Signature=abc") + assert "error" in refused and "X-Amz-Signature" not in str(refused) + monkeypatch.setattr(replay, "MAX_CAPTURE_URL_BYTES", 10) + assert "error" in load("https://bucket.example/c.warc.gz?X-Amz-Signature=abc") + assert "error" in load("http://bucket.example/c.warc.gz") + + def test_the_built_in_routes_follow_the_capture_being_served(client): def load(capture: bytes) -> None: rpc(client, "data/add", {"parts": [{"kind": "file", "file": {"bytes": base64.b64encode(capture).decode(), From 5e9c264c3f9d1ef0536fe3511915640990741fa5 Mon Sep 17 00:00:00 2001 From: William Loo Date: Mon, 2 Nov 2026 18:28:49 -0800 Subject: [PATCH 14/20] fix(website): a replay behind the gateway's prefix keeps its redirects under it (FD-4683) A redirect to a captured host, and a root-relative one, are led by the request's root_path, so a replay opened at /website// redirects to /website//new rather than the gateway's /new. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/agentenv_protocol/website/replay.py | 16 ++++++++++------ .../tests/test_website_replay.py | 8 ++++++++ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/replay.py b/packages/agentenv-protocol/src/agentenv_protocol/website/replay.py index b2f9b093..66aea1c4 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/replay.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/replay.py @@ -480,24 +480,28 @@ async def __call__(self, scope: dict, receive: Any, send: Any) -> None: if self.app is not None: return await self.app(scope, receive, send) return await error_response(NotFound("not in the capture"))(scope, receive, send) - await self._answer(entry, scope["method"] == "HEAD")(scope, receive, send) + await self._answer(entry, scope["method"] == "HEAD", scope.get("root_path", ""))(scope, receive, send) - def _answer(self, entry: Entry, head: bool) -> Response: + def _answer(self, entry: Entry, head: bool, root: str = "") -> Response: response = Response(b"" if head else entry.body, status_code=entry.status) - headers = [(name.lower().encode("latin-1"), self._header(name.lower(), value).encode("latin-1", "replace")) + headers = [(name.lower().encode("latin-1"), + self._header(name.lower(), value, root).encode("latin-1", "replace")) for name, value in entry.headers if name.lower() not in _NOT_REPLAYED] response.raw_headers = headers + [(b"content-length", str(len(entry.body)).encode())] return response - def _header(self, name: str, value: str) -> str: + def _header(self, name: str, value: str, root: str = "") -> str: """``value`` as the replay answers it: a Set-Cookie without its Domain, so it holds at the replay's host, and - a redirect to a captured host made relative, so it stays on the replay.""" + a redirect to a captured host or a root-relative one led by the ``root`` path the replay is served under, so + it stays on the replay behind a gateway's prefix too.""" if name == "set-cookie": return ";".join(p for p in value.split(";") if not p.strip().lower().startswith("domain=")) if name == "location": split = urlsplit(value) if split.hostname and split.hostname.lower() in self.env.archive.hosts: - return urlunsplit(("", "", split.path or "/", split.query, split.fragment)) + return urlunsplit(("", "", root.rstrip("/") + (split.path or "/"), split.query, split.fragment)) + if not split.scheme and not split.netloc and split.path.startswith("/"): + return root.rstrip("/") + value return value diff --git a/packages/agentenv-protocol/tests/test_website_replay.py b/packages/agentenv-protocol/tests/test_website_replay.py index 24898322..6abb75af 100644 --- a/packages/agentenv-protocol/tests/test_website_replay.py +++ b/packages/agentenv-protocol/tests/test_website_replay.py @@ -222,6 +222,14 @@ def test_a_replay_behind_the_gateways_prefix_matches_the_sites_own_path(client): assert client.get("/api/session", headers=behind).json() == {"captured": True} assert client.get("/api/price", headers=behind).json() == {"price": 7, "overridden": True} assert client.get("/nowhere", headers=behind).status_code == 404 + moved = client.get("/old", headers=behind, follow_redirects=False) + assert moved.headers["location"] == "/website/shop/new?x=1" + + relative = TestClient(Shop(warc(("https://shop.example/a", 302, [("Location", "/b")], b""), + ("https://shop.example/c", 302, [("Location", "d")], b""))).create_app()) + assert relative.get("/a", headers=behind, follow_redirects=False).headers["location"] == "/website/shop/b" + assert relative.get("/c", headers=behind, follow_redirects=False).headers["location"] == "d" + assert relative.get("/a", follow_redirects=False).headers["location"] == "/b" spaced = TestClient(Shop(warc(("https://shop.example/a%2Fb/x%20y", 200, [], b"escaped"))).create_app()) assert spaced.get("/a%2Fb/x%20y", headers=behind).text == "escaped" From 8ecf20d9191d897a6af491dd75fdc665cbacb8e4 Mon Sep 17 00:00:00 2001 From: William Loo Date: Mon, 2 Nov 2026 18:28:49 -0800 Subject: [PATCH 15/20] fix(website): browser/v1 cookie changes run one at a time (FD-4683) BrowserCookies holds a lock across each set or clear, its file write and DevTools calls together, so concurrent clears for different sites, or a set during a clear, never restore or erase what the other changed. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/agent_env/env/envs/playwright_mcp/recorder.py | 14 ++++++++++++-- tst/unit/env/envs/test_playwright_mcp_cookies.py | 12 ++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/agent_env/env/envs/playwright_mcp/recorder.py b/src/agent_env/env/envs/playwright_mcp/recorder.py index 2bb7e9cd..e289d8a1 100644 --- a/src/agent_env/env/envs/playwright_mcp/recorder.py +++ b/src/agent_env/env/envs/playwright_mcp/recorder.py @@ -234,13 +234,23 @@ async def _show(self, cdp: Cdp, target: Optional[str]) -> None: class BrowserCookies: """browser/v1's cookies: kept in the storage state file each new browser context starts from, and set in, or - cleared from, every context of the browser that is running, if one is.""" + cleared from, every context of the browser that is running, if one is. One change runs at a time, its file + write and DevTools calls together, so a concurrent set or clear never restores what another removed.""" def __init__(self, path: Path, find: Callable[[], Optional[int]], connect: Callable[[int], AsyncContextManager[Cdp]]) -> None: self.path, self._find, self._connect = path, find, connect + self._lock = asyncio.Lock() async def set(self, given: list[Cookie]) -> dict: + async with self._lock: + return await self._set(given) + + async def clear(self, domains: Optional[list[str]]) -> dict: + async with self._lock: + return await self._clear(domains) + + async def _set(self, given: list[Cookie]) -> dict: kept = {c.key(): c for c in self._read()} kept.update({c.key(): c for c in given}) self._write(list(kept.values())) @@ -250,7 +260,7 @@ async def add(cdp: Cdp, context: dict) -> None: return {"set": len(given), "open_browser": await self._each_context(add)} - async def clear(self, domains: Optional[list[str]]) -> dict: + async def _clear(self, domains: Optional[list[str]]) -> dict: before = self._read() kept = [c for c in before if domains is not None and not c.matches(domains)] self._write(kept) diff --git a/tst/unit/env/envs/test_playwright_mcp_cookies.py b/tst/unit/env/envs/test_playwright_mcp_cookies.py index 7bce89a5..22de103b 100644 --- a/tst/unit/env/envs/test_playwright_mcp_cookies.py +++ b/tst/unit/env/envs/test_playwright_mcp_cookies.py @@ -4,6 +4,7 @@ from __future__ import annotations +import asyncio import json import os import re @@ -38,6 +39,7 @@ def __init__(self, refuse: bool = False) -> None: self.refuse = refuse async def call(self, method, params=None, session=None): + await asyncio.sleep(0) params = dict(params or {}) self.calls.append((method, params)) jar = self.jars[params.pop("browserContextId", None)] @@ -115,6 +117,16 @@ async def test_clearing_a_domain_keeps_the_other_cookies_and_clearing_all_keeps_ assert all(jar == [] for jar in devtools.jars.values()) +@pytest.mark.asyncio +async def test_concurrent_changes_never_restore_what_another_removed(tmp_path): + devtools = _DevTools() + app = recorder.create_app(_recorder(tmp_path, devtools)) + await _post(app, SET, {"cookies": [SID, OTHER]}) + await asyncio.gather(_post(app, CLEAR, {"domains": ["github.com"]}), _post(app, CLEAR, {"domains": ["example.com"]})) + assert json.loads((tmp_path / "storage-state.json").read_text())["cookies"] == [] + assert all(jar == [] for jar in devtools.jars.values()) + + @pytest.mark.asyncio async def test_a_bad_cookie_or_a_browser_that_refuses_answers_without_the_value(tmp_path): app = recorder.create_app(_recorder(tmp_path, _DevTools(refuse=True))) From 88b480beec3059f3bf34e5b6b91e9defed28934b Mon Sep 17 00:00:00 2001 From: William Loo Date: Tue, 3 Nov 2026 08:29:54 -0800 Subject: [PATCH 16/20] fix(website): a capture is refused once it expands past MAX_CAPTURE_BYTES (FD-4683) Archive.read gunzipped a whole capture before looking at it, so a small, highly compressed file could pass the download limit and fill the website container's memory during data/add. It now reads a chunk at a time and stops past MAX_CAPTURE_BYTES, for inline bytes, file:// captures and fetched URLs alike. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/agentenv_protocol/website/replay.py | 29 +++++++++++++++---- .../tests/test_website_replay.py | 24 +++++++++++++++ 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/replay.py b/packages/agentenv-protocol/src/agentenv_protocol/website/replay.py index 66aea1c4..214c31cd 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/replay.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/replay.py @@ -13,7 +13,8 @@ subclass are layered on top and win over the capture, and the default store holds their state, which the data plane exports. ``data/add`` of a file part named ``*.warc`` or ``*.warc.gz`` (or typed ``application/warc``) serves that capture instead of the declared one until ``data/reset``, the part's bytes, its ``file://`` file or what its https URL -(the signed URL a plugin's deployment sends) answers, up to ``MAX_CAPTURE_URL_BYTES``. ``python -m agentenv_protocol.website.replay serve +(the signed URL a plugin's deployment sends) answers, up to ``MAX_CAPTURE_URL_BYTES``. Any capture read, declared or +added, is refused once it expands past ``MAX_CAPTURE_BYTES``. ``python -m agentenv_protocol.website.replay serve capture.warc.gz --domain example.com`` serves one on ``PORT``. The normalisation, for the capture and for each request alike: the method is upper-cased; the path is ``/`` when @@ -50,6 +51,7 @@ from .routes import error_response, reserved_path MAX_CAPTURE_URL_BYTES = 1 << 30 +MAX_CAPTURE_BYTES = 1 << 30 DEFAULT_IGNORED_QUERY = ("utm_*", "fbclid", "gclid", "msclkid", "_") WARC_VERSION = "WARC/1.1" WARC_TYPES = ("application/warc", "application/warc+gzip") @@ -124,13 +126,30 @@ def holds(self, path: str) -> bool: @classmethod def read(cls, source: str | Path | bytes, ignore_query: Iterable[str] = DEFAULT_IGNORED_QUERY) -> Archive: - """The answers of a WARC file, gzipped or not, given as its path or its bytes.""" - raw = source if isinstance(source, bytes) else Path(source).read_bytes() - if raw[:2] == b"\x1f\x8b": - raw = gzip.decompress(raw) + """The answers of a WARC file, gzipped or not, given as its path or its bytes, refused once it expands past + ``MAX_CAPTURE_BYTES``.""" + if isinstance(source, bytes): + raw = _expanded(io.BytesIO(source)) + else: + with open(source, "rb") as file: + raw = _expanded(file) return cls(_entries(io.BytesIO(raw)), ignore_query) +def _expanded(stream: BinaryIO) -> bytearray: + """A WARC file's bytes, gunzipped when it's gzipped, read a chunk at a time and refused past + ``MAX_CAPTURE_BYTES``, so a small file that expands a great deal never fills the memory.""" + gzipped = stream.read(2) == b"\x1f\x8b" + stream.seek(0) + reader: BinaryIO = gzip.GzipFile(fileobj=stream) if gzipped else stream + data = bytearray() + while chunk := reader.read(1 << 20): + data += chunk + if len(data) > MAX_CAPTURE_BYTES: + raise ValueError(f"the capture is over {MAX_CAPTURE_BYTES} bytes expanded") + return data + + def _records(stream: BinaryIO) -> Iterator[tuple[dict[str, str], bytes]]: """Each WARC record's headers, by lower-cased name, and its content block.""" while True: diff --git a/packages/agentenv-protocol/tests/test_website_replay.py b/packages/agentenv-protocol/tests/test_website_replay.py index 6abb75af..60cb8729 100644 --- a/packages/agentenv-protocol/tests/test_website_replay.py +++ b/packages/agentenv-protocol/tests/test_website_replay.py @@ -379,3 +379,27 @@ def test_an_entry_is_kept_once_per_request(): archive = Archive([Entry("GET", "https://a.example/x?b=1&a=2", 200, (), b"first"), Entry("GET", "https://a.example/x?a=2&b=1", 200, (), b"second")]) assert len(archive) == 1 and archive.find("GET", "/x", "a=2&b=1").body == b"first" + + +def test_a_capture_that_expands_past_its_limit_is_refused_while_it_is_read(client, tmp_path, monkeypatch): + from agentenv_protocol.website import replay + + big = warc(("https://shop.example/", 200, [], b"x" * 100_000)) + assert len(big) < 1_000 + monkeypatch.setattr(replay, "MAX_CAPTURE_BYTES", 50_000) + with pytest.raises(ValueError, match="over 50000 bytes expanded"): + Archive.read(big) + (tmp_path / "big.warc.gz").write_bytes(big) + with pytest.raises(ValueError, match="expanded"): + Archive.read(tmp_path / "big.warc.gz") + (tmp_path / "big.warc").write_bytes(gzip.decompress(big)) + with pytest.raises(ValueError, match="expanded"): + Archive.read(tmp_path / "big.warc") + inline = {"kind": "file", "file": {"bytes": base64.b64encode(big).decode(), "name": "big.warc.gz"}} + by_file = {"kind": "file", "file": {"uri": (tmp_path / "big.warc.gz").as_uri(), "name": "big.warc.gz"}} + for part in (inline, by_file): + answer = client.post(RPC, json={"jsonrpc": "2.0", "id": 1, "method": "data/add", + "params": {"parts": [part]}}).json() + assert "error" in answer + assert client.get("/").text == "home" + assert Archive.read(warc(("https://shop.example/", 200, [], b"x" * 40_000))).find("GET", "/").body == b"x" * 40_000 From 8e343b841ffde44c5c8498e41eb2b21ad87297ec Mon Sep 17 00:00:00 2001 From: William Loo Date: Tue, 3 Nov 2026 08:30:44 -0800 Subject: [PATCH 17/20] fix(website): a capture saves what its stylesheets import and load (FD-4683) A capture stopped at each stylesheet, so a background image or font a stylesheet named answered 404 in the replay. A captured stylesheet's @import and url() targets are now captured as assets, within the same host allowlist and request limit, and not at all with --no-assets. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../examples/website/replay/README.md | 3 +- .../src/agentenv_protocol/website/replay.py | 16 ++++++++-- .../tests/test_website_replay.py | 31 +++++++++++++++++++ 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/packages/agentenv-protocol/examples/website/replay/README.md b/packages/agentenv-protocol/examples/website/replay/README.md index 0cd4317e..1a406280 100644 --- a/packages/agentenv-protocol/examples/website/replay/README.md +++ b/packages/agentenv-protocol/examples/website/replay/README.md @@ -18,7 +18,8 @@ python -m agentenv_protocol.website.replay capture https://docs.example.com/ --o ``` The tool needs only this package and `httpx`. It fetches each start URL. It then fetches the same-site pages those URLs -link to, up to `--depth` links away, and the assets (stylesheets, scripts, images) each captured page loads. Every +link to, up to `--depth` links away, and the assets (stylesheets, scripts, images) each captured page loads, with the +stylesheets, images and fonts a captured stylesheet's `@import` and `url()` name. Every answer goes into a WARC 1.1 file, gzipped one record at a time, which pywb and warcio also read. Notes: - Only GETs are captured. diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/replay.py b/packages/agentenv-protocol/src/agentenv_protocol/website/replay.py index 214c31cd..27b44926 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/replay.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/replay.py @@ -1,7 +1,8 @@ """Capture a real site and serve the capture as a website env. ``python -m agentenv_protocol.website.replay capture URL... --out capture.warc.gz`` fetches each start URL, then the -same-site pages it links to up to ``--depth`` links away and the assets each captured page loads, and writes every +same-site pages it links to up to ``--depth`` links away and the assets each captured page loads (with what a +captured stylesheet's ``@import`` and ``url()`` name), and writes every answer to a WARC 1.1 file, gzipped a record at a time as other WARC tools write them. Only GETs are captured, redirects are kept as they were answered (and followed), and bodies are stored decoded: an answer's ``Content-Encoding`` and ``Transfer-Encoding`` are dropped and its ``Content-Length`` is the decoded body's. A host is the same site as a start @@ -31,6 +32,7 @@ import datetime import gzip import io +import re import sys import uuid from collections import deque @@ -60,6 +62,8 @@ _NOT_REPLAYED = _HOP_BY_HOP | {"content-length", "strict-transport-security", "alt-svc"} _NOT_STORED = _HOP_BY_HOP | {"content-encoding", "content-length"} _SEGMENT_SAFE = "!$&'()*+,;=:@-._~" +_CSS_URL = re.compile(r"""url\(\s*(["']?)(?P[^"')\s]*)\1\s*\)|@import\s+(["'])(?P[^"']*)\3""", + re.IGNORECASE) def request_key(method: str, path: str, query: str = "", ignore_query: Iterable[str] = DEFAULT_IGNORED_QUERY, @@ -338,11 +342,17 @@ def _fetched_as(url: str) -> tuple: def _next(url: str, response: Any, level: int, is_asset: bool, assets: bool) -> Iterator[tuple[str, int, bool]]: - """What a captured answer leads to: a redirect's target at its level, and an HTML page's links, its pages a level - further and its assets at its level.""" + """What a captured answer leads to: a redirect's target at its level, a stylesheet's ``@import`` and ``url()`` + targets as assets at its level, and an HTML page's links, its pages a level further and its assets at its level.""" if 300 <= response.status_code < 400 and response.headers.get("location"): yield _absolute(url, response.headers["location"]), level, is_asset return + if "css" in response.headers.get("content-type", ""): + for match in _CSS_URL.finditer(response.text) if assets else (): + href = match["url"] or match["imported"] + if href and (target := _absolute(url, href)) is not None: + yield target, level, True + return if is_asset or "html" not in response.headers.get("content-type", ""): return links = _Links() diff --git a/packages/agentenv-protocol/tests/test_website_replay.py b/packages/agentenv-protocol/tests/test_website_replay.py index 60cb8729..536d99d2 100644 --- a/packages/agentenv-protocol/tests/test_website_replay.py +++ b/packages/agentenv-protocol/tests/test_website_replay.py @@ -403,3 +403,34 @@ def test_a_capture_that_expands_past_its_limit_is_refused_while_it_is_read(clien assert "error" in answer assert client.get("/").text == "home" assert Archive.read(warc(("https://shop.example/", 200, [], b"x" * 40_000))).find("GET", "/").body == b"x" * 40_000 + + +class _StyledSite(_Site): + """A page whose stylesheet imports another and loads an image, a font, a data URI and another site's image.""" + + PAGES = { + "/": (200, "text/html", b'

styled

'), + "/s.css": (200, "text/css", b"@import '/t.css'; body{background:url( \"bg.png\" )}" + b"i{background:url(https://elsewhere.example/x.png)}" + b"b{background:url(data:image/png;base64,AA)}"), + "/t.css": (200, "text/css; charset=utf-8", b"@font-face{src:URL(/f.woff2) format('woff2')}"), + "/bg.png": (200, "image/png", b"\x89PNG"), + "/f.woff2": (200, "font/woff2", b"wOF2"), + } + + +def test_a_capture_follows_a_stylesheets_imports_and_assets_on_the_site(): + site = _StyledSite() + try: + out = io.BytesIO() + result = capture([site.url + "/"], out, depth=0) + assert {key[1] for key in Archive.read(out.getvalue()).entries} == {"/", "/s.css", "/t.css", "/bg.png", + "/f.woff2"} + assert result.captured == 5 and not result.failed + bare = io.BytesIO() + capture([site.url + "/"], bare, depth=0, assets=False) + assert {key[1] for key in Archive.read(bare.getvalue()).entries} == {"/"} + limited = io.BytesIO() + assert capture([site.url + "/"], limited, depth=0, max_requests=3).captured == 3 + finally: + site.close() From a74194ca829da723bf28b3a19931043bc8875f72 Mon Sep 17 00:00:00 2001 From: William Loo Date: Tue, 3 Nov 2026 08:33:03 -0800 Subject: [PATCH 18/20] fix(website): a replayed page's same-site links stay on the replay (FD-4683) Captured HTML was served unchanged, so behind /website// a link to /style.css reached the gateway root and https://shop.example/... went back to the real site. A captured page's href, src, srcset, action, formaction, poster and style values, its ' + "" + "ahome" + 'awayctop' + '/not-a-link' + '
' + '
é
').encode() + css = b"@import '/t.css'; a{background:url(https://shop.example/bg.png)} b{src:url(f.woff2)}" + site = TestClient(Shop(warc(("https://shop.example/", 200, [("Content-Type", "text/html; charset=utf-8")], page), + ("https://shop.example/style.css", 200, [("Content-Type", "text/css")], css), + ("https://shop.example/app.js", 200, [("Content-Type", "text/javascript")], + b"fetch('/api/x')"))).create_app()) + behind = {"X-Forwarded-Prefix": "/website/shop"} + served = site.get("/", headers=behind) + assert served.content == page.replace(b'"/style.css"', b'"/website/shop/style.css"').replace( + b'url("/b.png")', b'url("/website/shop/b.png")').replace( + b"'https://shop.example/a?x=1&y=2'", b"'/website/shop/a?x=1&y=2'").replace( + b"href=//shop.example>", b"href=/website/shop/>").replace(b"src=/i.png", b"src=/website/shop/i.png").replace( + b'"/i.png 1x, https://shop.example/i2.png 2x"', b'"/website/shop/i.png 1x, /website/shop/i2.png 2x"').replace( + b'"/search"', b'"/website/shop/search"').replace(b'"/go2"', b'"/website/shop/go2"').replace( + b"url('/d.png')", b"url('/website/shop/d.png')") + assert b"var u = '/api/x'; location.href = \"https://shop.example/js\"" in served.content + assert served.headers["content-length"] == str(len(served.content)) + assert site.head("/", headers=behind).headers["content-length"] == str(len(served.content)) + assert site.get("/style.css", headers=behind).content == ( + b"@import '/website/shop/t.css'; a{background:url(/website/shop/bg.png)} b{src:url(f.woff2)}") + assert site.get("/app.js", headers=behind).content == b"fetch('/api/x')" + + direct = site.get("/").content + assert b"'/a?x=1&y=2'" in direct and b'"/style.css"' in direct and b"https://elsewhere.example/z" in direct From aea02974549b7aac9550a6abb12d218023406334 Mon Sep 17 00:00:00 2001 From: William Loo Date: Tue, 3 Nov 2026 09:27:49 -0800 Subject: [PATCH 19/20] fix(website): a replayed stylesheet loads though its link named an integrity hash (FD-4683) A replay rewrites a captured stylesheet's same-site URLs, which changes its bytes, so the page's tags now lose their integrity attribute rather than have the browser block the rewritten sheet. The capture reader's chunk size and the capture download's timeout are named constants. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/agentenv_protocol/website/replay.py | 15 +++++++++++---- .../tests/test_website_replay.py | 8 ++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/replay.py b/packages/agentenv-protocol/src/agentenv_protocol/website/replay.py index f0ff85a5..e3c0c581 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/replay.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/replay.py @@ -57,6 +57,8 @@ MAX_CAPTURE_URL_BYTES = 1 << 30 MAX_CAPTURE_BYTES = 1 << 30 +CAPTURE_READ_CHUNK_BYTES = 1 << 20 +CAPTURE_URL_TIMEOUT_SECONDS = 60 DEFAULT_IGNORED_QUERY = ("utm_*", "fbclid", "gclid", "msclkid", "_") WARC_VERSION = "WARC/1.1" WARC_TYPES = ("application/warc", "application/warc+gzip") @@ -150,7 +152,7 @@ def _expanded(stream: BinaryIO) -> bytearray: stream.seek(0) reader: BinaryIO = gzip.GzipFile(fileobj=stream) if gzipped else stream data = bytearray() - while chunk := reader.read(1 << 20): + while chunk := reader.read(CAPTURE_READ_CHUNK_BYTES): data += chunk if len(data) > MAX_CAPTURE_BYTES: raise ValueError(f"the capture is over {MAX_CAPTURE_BYTES} bytes expanded") @@ -450,7 +452,7 @@ async def _fetched(url: str) -> bytes: import httpx data = bytearray() - async with httpx.AsyncClient(follow_redirects=True, timeout=60) as client: + async with httpx.AsyncClient(follow_redirects=True, timeout=CAPTURE_URL_TIMEOUT_SECONDS) as client: async with client.stream("GET", url) as response: if response.status_code != 200: raise ValueError(f"the capture's URL answered {response.status_code}") @@ -577,7 +579,9 @@ def _rewritten_srcset(srcset: str, hosts: set[str], root: str) -> str: class _Rewriter(HTMLParser): """Finds what to rewrite in an HTML page so its links stay on the replay: the ``href``, ``src``, ``srcset``, ``action``, ``formaction`` and ``poster`` values of each tag, its ``style`` value's and each ``' "" From ed065a0921ac600e6abd519b6119cccaf089c8f0 Mon Sep 17 00:00:00 2001 From: William Loo Date: Tue, 3 Nov 2026 10:25:59 -0800 Subject: [PATCH 20/20] fix(website): a replay serves no captured Content-Security-Policy, so its rewritten pages still load (FD-4683) The captured policy's hashes and origins name the content as captured; once the replay rewrites a page's or a stylesheet's URLs behind /website/, a hashed ') + headers = [("Content-Type", "text/html"), ("Content-Security-Policy", "style-src 'sha256-abc'"), + ("Content-Security-Policy-Report-Only", "default-src https://shop.example"), ("X-Kept", "1")] + site = TestClient(Shop(warc(("https://shop.example/", 200, headers, page))).create_app()) + served = site.get("/", headers={"X-Forwarded-Prefix": "/website/shop"}) + assert "content-security-policy" not in served.headers + assert "content-security-policy-report-only" not in served.headers + assert served.headers["x-kept"] == "1" + assert served.content == b'' + + def test_a_replay_keeps_a_pages_and_a_stylesheets_same_site_links_on_the_replay(): page = ('' ""