From e5082eb62bbe9d1431a4ec1800039d7f07c9f7fb Mon Sep 17 00:00:00 2001 From: William Loo Date: Tue, 3 Nov 2026 12:52:03 -0800 Subject: [PATCH 01/12] feat(website): sync the clock of, snapshot, and configure a multi env's websites as its MCP servers (FD-4683) A website env, and a multi env's websites, get the harness treatment MCP servers get: sync_env_clock syncs them (and tells them main's timezone), snapshot_env exports each through its data plane and fails on a site whose card can't be read, and an apply_server_config broadcast reaches them; a site without the clock, a data plane or the config extension is skipped. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/agent_env/env/legacy_protocol.py | 53 +++++- .../task_steps/apply_server_config.py | 75 ++++---- .../task_step/task_steps/snapshot_env.py | 90 ++++++--- .../task_step/task_steps/sync_env_clock.py | 57 ++++-- tst/unit/env/test_legacy_protocol.py | 19 ++ .../env_gateway_provider_test.py | 12 ++ tst/unit/task_step/sync_env_clock_test.py | 153 ++++++++++++++- .../task_step/test_apply_server_config.py | 91 +++++++++ tst/unit/task_step/test_snapshot_env.py | 174 ++++++++++++++++++ 9 files changed, 635 insertions(+), 89 deletions(-) diff --git a/src/agent_env/env/legacy_protocol.py b/src/agent_env/env/legacy_protocol.py index 16899025b..8790157ac 100644 --- a/src/agent_env/env/legacy_protocol.py +++ b/src/agent_env/env/legacy_protocol.py @@ -9,7 +9,8 @@ import asyncio import logging -from typing import TYPE_CHECKING, Optional +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Optional import httpx from agentenv_protocol import RPC_PATH, DataPart, client as protocol_v1 @@ -33,20 +34,62 @@ def environment_base_url(gateway_url: Optional[str], environment_name: str, mcp: return f"{gateway_url}/svc/{prefix}{environment_name}" +@dataclass(frozen=True) +class BackingServer: + """A server behind an env's gateway, by its environment_name: an MCP server, at ``/svc/mcp-{name}``, or a website, + at ``/svc/{name}``.""" + + environment_name: str + mcp: bool = True + + +def backing_servers(env: Any) -> list[BackingServer]: + """The servers behind env's gateway: a MultiEnv's MCP servers then its websites, a WebsiteEnv's website, or else + the env's one MCP server; empty for an env with none.""" + from agent_env.env.envs.website import WebsiteEnv + + if isinstance(env, WebsiteEnv): + return [BackingServer(env.environment_name, mcp=False)] + servers = [BackingServer(e.environment_name) for e in getattr(env, "mcp_server_envs", None) or []] + servers += [BackingServer(e.environment_name, mcp=False) for e in getattr(env, "website_envs", None) or []] + if not servers and getattr(env, "environment_name", None): + servers = [BackingServer(env.environment_name)] + return servers + + +def shared_names(servers: list[BackingServer]) -> set[str]: + """The environment_names an MCP server and a website in ``servers`` share, which a stored env card can't tell apart.""" + return {s.environment_name for s in servers if s.mcp} & {s.environment_name for s in servers if not s.mcp} + + +def mcp_wins(servers: list[BackingServer]) -> list[BackingServer]: + """``servers`` without each website that shares its environment_name with an MCP server, as a MultiEnv routes an + artifact of that name to the MCP server.""" + shared = shared_names(servers) + return [s for s in servers if s.mcp or s.environment_name not in shared] + + async def v1_base_url(deployed: Optional[DeployedEnv], gateway_url: Optional[str], environment_name: str, mcp: bool = True) -> Optional[str]: - """A child env's v1 data-plane base from the stored env card, or None for legacy; without a stored card, the live probe decides.""" + """A child env's v1 data-plane base from the stored env card, or None for legacy; without a stored card, the live probe + decides. A website's probe is a card read that is None only on a 404 and raises on any other failure, so a website whose + card could not be read is never taken for one serving no data plane.""" if deployed is None or not deployed.environment_card: base_url = environment_base_url(gateway_url, environment_name, mcp=mcp) + if not mcp: + _, card = await child_env_card(None, gateway_url, environment_name, mcp=False) + return base_url if card is not None else None return base_url if await protocol_v1.supports_v1(base_url) else None child = deployed.get_child_env_card(environment_name) return deployed.environment_url + child.get("url", RPC_PATH).removesuffix(RPC_PATH) if child else None -async def child_env_card(deployed: Optional[DeployedEnv], gateway_url: Optional[str], environment_name: str, timeout: int = 10) -> tuple[str, Optional[dict]]: - """An MCP child env's card and the base its endpoints resolve against: from the stored env card, or without one a live read (None on 404).""" +async def child_env_card(deployed: Optional[DeployedEnv], gateway_url: Optional[str], environment_name: str, timeout: int = 10, + mcp: bool = True) -> tuple[str, Optional[dict]]: + """A child env's card and the base its endpoints resolve against: from the stored env card, or without one a live read + at the MCP server's gateway path, or with ``mcp=False`` the website's (None on 404).""" if deployed is None or not deployed.environment_card: - base_url = environment_base_url((gateway_url or "").rstrip("/"), environment_name) + base_url = environment_base_url((gateway_url or "").rstrip("/"), environment_name, mcp=mcp) try: return base_url, await protocol_v1.get_card(base_url, timeout=timeout) except httpx.HTTPStatusError as e: diff --git a/src/agent_env/task_step/task_steps/apply_server_config.py b/src/agent_env/task_step/task_steps/apply_server_config.py index 41f82ffd0..c8c8e1f08 100644 --- a/src/agent_env/task_step/task_steps/apply_server_config.py +++ b/src/agent_env/task_step/task_steps/apply_server_config.py @@ -1,15 +1,15 @@ """Task step that applies a task's behavior config to deployed servers. At setup (after the env is deployed + universe loaded, before the agent runs) -this step invokes each backing MCP server's ``set_*`` config extensions over the -gateway. Extensions are advertised on the server's ``EnvironmentCard`` (served at +this step invokes each backing MCP server's or website's ``set_*`` config extensions +over the gateway. Extensions are advertised on the server's ``EnvironmentCard`` (served at ``/.well-known/agent-env.json``) under ``capabilities.extensions[]`` and invoked at their advertised REST endpoint (e.g. ``/agentenv/ext/set_errors``). Config lives on the backing server, not on the gateway itself. Each server's card comes from the env card stored on the deployed record, whose extension endpoints the -gateway has already rewritten to its ``/svc/mcp-{service}`` proxy; a record without a -stored card reads it live there. The agent never sees the card or these endpoints; +gateway has already rewritten to its ``/svc/mcp-{service}`` (or a website's ``/svc/{service}``) +proxy; a record without a stored card reads it live there. The agent never sees the card or these endpoints; only this harness step invokes them. """ @@ -76,7 +76,7 @@ class ApplyServerConfigStep(TaskStep): entity_refs = (EntityRef.env("env_id"),) # A directive whose ``service`` is this sentinel is a broadcast: at execution - # it expands to one directive per MCP server in the env (same uri + args). + # it expands to one directive per MCP server and website in the env (same uri + args). # Lets a caller apply an env-wide config (e.g. set_acting_user across every # server) without enumerating services itself. Pairs naturally with # ``tolerate_unadvertised`` so servers that don't opt in are simply skipped. @@ -131,44 +131,51 @@ def from_dict(cls, data: dict) -> "ApplyServerConfigStep": tolerate_unadvertised=data.get("tolerate_unadvertised", False), ) - def _expand_directives(self, deployed) -> list[ConfigDirective]: + def _expand_directives(self, deployed) -> tuple[list[ConfigDirective], set[str], set[int]]: """Expand any broadcast directive (service == ``*``) into one directive - per MCP server in the env, resolved from the env definition. The env's - child service names aren't otherwise enumerable here (the deployed card - list carries no roster), so we read them from ``Env.get`` — matching how - deploy_env / load_artifact resolve the env. Non-broadcast directives - pass through untouched.""" + per MCP server and website in the env, resolved from the env definition, + and return them with the names that are websites, whose cards a live read + finds at ``/svc/{name}``, and the positions of the directives a broadcast + made for a website. Those are skipped, as under ``tolerate_unadvertised``, + when the website has no card or doesn't advertise the URI, so a website + the task didn't name never fails it; a directive naming a website, and + every MCP server, stays strict. The env's child service names aren't otherwise + enumerable here (the deployed card list carries no roster), so we read + them from ``Env.get`` — matching how deploy_env / load_artifact resolve + the env. A website sharing its name with an MCP server is left to the MCP + server, as the stored env card can't tell the two apart. Non-broadcast + directives pass through untouched.""" if not any(d.service == self._BROADCAST for d in self.directives): - return self.directives + return self.directives, set(), set() + from agent_env.env import legacy_protocol from agent_env.env.env import Env env = Env.get(deployed.env_id, deployed.env_version) - environment_names = [ - e.environment_name for e in (getattr(env, "mcp_server_envs", None) or []) - ] # A single-server env (a bare MCPServerEnv) exposes `environment_name` and no - # `mcp_server_envs`; mirror snapshot_env / exporter enumeration and fall - # back to it, else a broadcast silently matches nothing on single-server envs. - if not environment_names and getattr(env, "environment_name", None): - environment_names = [env.environment_name] + # `mcp_server_envs`; backing_servers falls back to it, as snapshot_env does, + # else a broadcast silently matches nothing on single-server envs. + servers = legacy_protocol.mcp_wins(legacy_protocol.backing_servers(env)) # A broadcast that resolves to zero services would apply nothing while the # step reports success: the exact silent misconfig this step prevents. Fail # loud (tolerate_unadvertised governs advertisement, not missing servers). - if not environment_names: + if not servers: raise ApplyServerConfigError( - f"apply_server_config: broadcast directive matched no MCP servers " - f"in env {self.env_id!r} (env exposes neither mcp_server_envs nor a " + f"apply_server_config: broadcast directive matched no MCP servers or websites " + f"in env {self.env_id!r} (env exposes neither mcp_server_envs, website_envs nor a " f"environment_name); nothing to apply, likely a task authoring error" ) expanded: list[ConfigDirective] = [] + lenient: set[int] = set() for d in self.directives: if d.service != self._BROADCAST: expanded.append(d) continue - for svc in environment_names: - expanded.append(ConfigDirective(service=svc, uri=d.uri, args=d.args)) - return expanded + for server in servers: + if not server.mcp: + lenient.add(len(expanded)) + expanded.append(ConfigDirective(service=server.environment_name, uri=d.uri, args=d.args)) + return expanded, {server.environment_name for server in servers if not server.mcp}, lenient async def execute(self, context: TaskStepContext) -> TaskStepContext: deployed = next((d for d in context.deployed_envs if d.env_id == self.env_id), None) @@ -180,7 +187,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: # actually armed with — a partial apply must not look like a no-op. changes = context.metadata.setdefault("server_config_changes", []) # Directives skipped because the target server doesn't advertise the - # extension (only when tolerate_unadvertised). Kept separate from + # extension (when tolerate_unadvertised, or for a website a broadcast reached). Kept separate from # `changes` so the audit trail never shows a skip as an applied change. skipped = context.metadata.setdefault("server_config_skipped", []) # Expand broadcast (service="*") directives to one-per-env-service. @@ -188,7 +195,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: # network), so wrap it to keep the broadcast path's failure contract the # same ApplyServerConfigError as the per-directive HTTP calls below. try: - directives = self._expand_directives(deployed) + directives, websites, lenient = self._expand_directives(deployed) except ApplyServerConfigError: raise except Exception as e: @@ -204,15 +211,17 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: # card: a server predating env-card support, one whose card name differs, or one # whose card the deploy couldn't read. cards: dict[str, tuple[str, Optional[dict]]] = {} - for directive in directives: + for position, directive in enumerate(directives): + tolerate = self.tolerate_unadvertised or position in lenient try: if directive.service not in cards: cards[directive.service] = await legacy_protocol.child_env_card( - deployed, gateway_url_of(deployed), directive.service, timeout=self.timeout_seconds + deployed, gateway_url_of(deployed), directive.service, timeout=self.timeout_seconds, + mcp=directive.service not in websites, ) base_url, card = cards[directive.service] if card is None: - if not self.tolerate_unadvertised: + if not tolerate: raise ApplyServerConfigError( f"Service {directive.service!r} has no env card (env={self.env_id}), so {directive.uri!r} " f"can't be applied: it serves none, its card isn't named {directive.service!r}, or the deploy couldn't read it" @@ -220,7 +229,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: logger.warning( f"apply_server_config: service {directive.service!r} serves no " f"environment card (env={self.env_id}); skipping {directive.uri!r} " - f"(tolerate_unadvertised)" + f"({'tolerate_unadvertised' if self.tolerate_unadvertised else 'broadcast to a website'})" ) skipped.append({ "step_id": self.id, @@ -233,11 +242,11 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: continue ext = protocol_v1.find_extension(card, directive.uri) if ext is None: - if self.tolerate_unadvertised: + if tolerate: logger.warning( f"apply_server_config: {directive.uri!r} not advertised on " f"service {directive.service!r} (env={self.env_id}); skipping " - f"(tolerate_unadvertised)" + f"({'tolerate_unadvertised' if self.tolerate_unadvertised else 'broadcast to a website'})" ) skipped.append({ "step_id": self.id, diff --git a/src/agent_env/task_step/task_steps/snapshot_env.py b/src/agent_env/task_step/task_steps/snapshot_env.py index 48fc9d6dc..9415dac98 100644 --- a/src/agent_env/task_step/task_steps/snapshot_env.py +++ b/src/agent_env/task_step/task_steps/snapshot_env.py @@ -13,7 +13,7 @@ from dataclasses import dataclass from datetime import datetime, timezone from pathlib import PurePosixPath -from typing import ClassVar, Optional +from typing import TYPE_CHECKING, ClassVar, Optional from urllib.parse import urlparse import httpx @@ -37,6 +37,9 @@ from agent_env.entity.scopes import FieldScope from agent_env.task_step.task_step import TaskStep, TaskStepDependency +if TYPE_CHECKING: + from agent_env.env.legacy_protocol import BackingServer + logger = logging.getLogger(__name__) DEFAULT_EXPORT_TIMEOUT_SECONDS = 600 @@ -167,6 +170,9 @@ class SnapshotEnvTaskStep(TaskStep): Output: ``context.metadata["env_snapshotted_universes"][self.id]`` = ``{"id", "version"}``. Fails the step if any service fails to export. + Websites are snapshotted through their v1 data plane; a website serving + none is left out, and one sharing its name with an MCP server is left to + the MCP server, as a MultiEnv loads that name's artifact into it. ``include_env_changes`` also stores who changed what (urn:agentenv:changes/v1) under ``metadata["env_changes"]``. @@ -469,28 +475,36 @@ def _resolve_loaded_universe_ref(deployed) -> Optional[tuple[str, Optional[int]] return None return (su["id"], su.get("version")) if su and su.get("id") else None + @classmethod + def _enumerate_environments(cls, env) -> list[str]: + """environment names for a multi env, a website env or a single MCP server env.""" + return [server.environment_name for server in cls._enumerate_servers(env)] + @staticmethod - def _enumerate_environments(env) -> list[str]: - """environment names for a multi env or a single MCP server env.""" - if getattr(env, "mcp_server_envs", None): - return [e.environment_name for e in env.mcp_server_envs] - if getattr(env, "environment_name", None): - return [env.environment_name] - raise RuntimeError( - f"Env '{env.id}' v{env.version} has no MCP services to snapshot " - "(expected mcp_server_envs or environment_name)" - ) + def _enumerate_servers(env) -> list[BackingServer]: + """The servers to snapshot: the env's MCP servers and websites, a website sharing its name with an MCP + server left to it.""" + from agent_env.env import legacy_protocol + + servers = legacy_protocol.mcp_wins(legacy_protocol.backing_servers(env)) + if not servers: + raise RuntimeError( + f"Env '{env.id}' v{env.version} has no MCP services or websites to snapshot " + "(expected mcp_server_envs, website_envs or environment_name)" + ) + return servers @staticmethod async def _export_environment_to_file( gateway_url: Optional[str], environment_name: str, tmp_path: str, timeout_seconds: float, deployed=None, - data_get_params: Optional[dict] = None, - ) -> str | tuple[GetDataResponse, Optional[_SnapshotUpload]]: - """Export one service; return the suffix written (".json"/".zip") to - ``tmp_path``, or the object url a FilePart names when the service uploaded the - bundle itself (nothing written to ``tmp_path`` then — export_one registers it), or, for - an environment whose card declares a file tree, its data/get answer and the upload it - answered under, which export_one keeps file by file. ``data_get_params`` go to its data/get. + data_get_params: Optional[dict] = None, mcp: bool = True, + ) -> Optional[str | tuple[GetDataResponse, Optional[_SnapshotUpload]]]: + """Export one service, an MCP server or with ``mcp=False`` a website; return the + suffix written (".json"/".zip") to ``tmp_path``, or the object url a FilePart names + when the service uploaded the bundle itself (nothing written to ``tmp_path`` then — + export_one registers it), or, for an environment whose card declares a file tree, its + data/get answer and the upload it answered under, which export_one keeps file by file, + or None for a website serving no v1 data plane. ``data_get_params`` go to its data/get. v1 ``get_data``, handed a namespace grant when the service takes the data-objects form: an upload through it → its object url; ``DataPart`` → json; ``FilePart`` → @@ -499,10 +513,12 @@ async def _export_environment_to_file( guard. Errors are caught by the caller (fails just this service).""" from agent_env.env import legacy_protocol - base_url = await legacy_protocol.v1_base_url(deployed, gateway_url, environment_name, mcp=True) + base_url = await legacy_protocol.v1_base_url(deployed, gateway_url, environment_name, mcp=mcp) + if base_url is None and not mcp: + return None if base_url is not None: headers = deployed.control_headers_for(base_url) if deployed is not None else None - card_base, card = await legacy_protocol.child_env_card(deployed, gateway_url, environment_name) + card_base, card = await legacy_protocol.child_env_card(deployed, gateway_url, environment_name, mcp=mcp) if returns_file_tree(card or {}): upload = await asyncio.to_thread(_snapshot_upload, card or {}, timeout_seconds, getattr(deployed, "sandbox_type", None), FILE_TREE_LIMITS) @@ -611,7 +627,10 @@ async def snapshot_env_state( from agent_env.artifact import FileArtifact, EnvironmentArtifact, EnvironmentUniverseArtifact from agent_env.env.env import gateway_url_of - services = cls._enumerate_environments(env) if gateway_url else _card_environments(deployed) + from agent_env.env.legacy_protocol import BackingServer + + servers = cls._enumerate_servers(env) if gateway_url else [BackingServer(name) for name in _card_environments(deployed)] + services = [server.environment_name for server in servers] gateway = gateway_url.rstrip("/") if gateway_url else None # `gateway` identifies which deployment was exported: env= alone doesn't when one env_id has several. logger.info( @@ -643,9 +662,12 @@ async def snapshot_env_state( # The stored card describes the deployment's own gateway; an override pointing elsewhere keeps the live probe. card_record = deployed if deployed is not None and (gateway is None or (gateway_url_of(deployed) or "").rstrip("/") == gateway) else None sem = asyncio.Semaphore(concurrency) + no_data_plane: set[str] = set() - async def export_one(environment_name: str): - """Returns EnvironmentArtifact on success, (public_err, log_err) on failure.""" + async def export_one(server: BackingServer): + """Returns EnvironmentArtifact on success, (public_err, log_err) on failure, None for a website serving no data plane.""" + environment_name = server.environment_name + website = {} if server.mcp else {"mcp": False} async with sem: # Suffix (.json/.zip) is reported by _export_environment_to_file. fd, tmp_path = tempfile.mkstemp(prefix=f"{environment_name}-") @@ -656,7 +678,7 @@ async def export_one(environment_name: str): try: result = await cls._export_environment_to_file( gateway, environment_name, tmp_path, export_timeout_seconds, - deployed=card_record, data_get_params=(data_get_params or {}).get(environment_name), + deployed=card_record, data_get_params=(data_get_params or {}).get(environment_name), **website, ) except httpx.HTTPStatusError as e: body_snippet = "" @@ -671,6 +693,9 @@ async def export_one(environment_name: str): return public, f"{public}: {e}" except ValueError as e: return str(e), str(e) + if result is None: + no_data_plane.add(environment_name) + return None if isinstance(result, tuple): try: @@ -744,7 +769,7 @@ def _register_uploaded_bundle(): except OSError: pass - results = await asyncio.gather(*(export_one(name) for name in services)) + results = await asyncio.gather(*(export_one(server) for server in servers)) environment_artifacts = [] skipped: dict[str, tuple[str, str]] = {} @@ -767,9 +792,18 @@ def _register_uploaded_bundle(): f"{len(skipped)}/{len(services)} services were not exported " f"(missing={sorted(skipped)}): {public_summary}" ) - if empty := [name for name, result in zip(services, results) if result is None]: - if len(empty) == len(services): - raise RuntimeError(f"{', '.join(map(repr, empty))} returned no files to snapshot") + without_data_plane = [name for name in services if name in no_data_plane] + if without_data_plane: + logger.warning( + f"snapshot_env: websites {without_data_plane} of {env.id} serve no v1 data plane; " + "their state is not in the snapshot" + ) + empty = [name for name, result in zip(services, results) if result is None and name not in no_data_plane] + if not environment_artifacts: + reasons = [f"its websites {without_data_plane} serve no v1 data plane"] if without_data_plane else [] + reasons += [f"{', '.join(map(repr, empty))} returned no files to snapshot"] if empty else [] + raise RuntimeError(f"Env '{env.id}' v{env.version} exported no state: {'; '.join(reasons)}") + if empty: logger.info(f"snapshot_env: {empty} returned no files to keep, so {snapshot_id} leaves them out") universe = await asyncio.to_thread( diff --git a/src/agent_env/task_step/task_steps/sync_env_clock.py b/src/agent_env/task_step/task_steps/sync_env_clock.py index 76ff196e8..c80cef8a8 100644 --- a/src/agent_env/task_step/task_steps/sync_env_clock.py +++ b/src/agent_env/task_step/task_steps/sync_env_clock.py @@ -1,16 +1,16 @@ """Arm a deployed env gateway's virtual clock, then sync its backing servers to it. Arms the clock (set_time) and reads the gateway's server-reachable env_get_time_url (state), both as the -stored env card declares clock/v1, then invokes sync_time on each backing server whose child env card -advertises it. Non-advertisers are skipped when tolerate_missing_sync_time (default True). Place right -before prompt_agent so the virtual timeline starts at agent-start. +stored env card declares clock/v1, then invokes sync_time on each backing server, MCP server or website, +whose child env card advertises it. Non-advertisers are skipped when tolerate_missing_sync_time (default +True), and websites always. Place right before prompt_agent so the virtual timeline starts at agent-start. """ from __future__ import annotations import logging from datetime import datetime, timezone as _utc_tz -from typing import ClassVar, Optional +from typing import TYPE_CHECKING, ClassVar, Optional from zoneinfo import ZoneInfo, ZoneInfoNotFoundError import httpx @@ -22,6 +22,9 @@ from agent_env.entity.refs import EntityRef from agent_env.task_step.task_step import TaskStep, TaskStepDependency +if TYPE_CHECKING: + from agent_env.env.legacy_protocol import BackingServer + logger = logging.getLogger(__name__) @@ -107,16 +110,15 @@ def from_dict(cls, data: dict) -> SyncEnvClockTaskStep: timezone=data.get("timezone"), ) - def _service_names(self, deployed) -> list[str]: - """Backing MCP server names for the env (mirrors apply_server_config's enumeration).""" + def _backing_servers(self, deployed) -> list[BackingServer]: + """The env's backing MCP servers and websites (``legacy_protocol.backing_servers``). A website sharing its + name with an MCP server has its card read live, since the stored env card can't tell the two apart.""" + from agent_env.env import legacy_protocol from agent_env.env.env import Env - env = Env.get(deployed.env_id, deployed.env_version) - names = [e.environment_name for e in (getattr(env, "mcp_server_envs", None) or [])] - if not names and getattr(env, "environment_name", None): - names = [env.environment_name] - return names + return legacy_protocol.backing_servers(Env.get(deployed.env_id, deployed.env_version)) async def execute(self, context: TaskStepContext) -> TaskStepContext: + from agent_env.env import legacy_protocol from agent_env.env.env import gateway_url_of deployed = next((d for d in context.deployed_envs if d.env_id == self.env_id), None) @@ -131,10 +133,17 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: state = await deployed.invoke(EXT_CLOCK_URI, "state", timeout=self.timeout_seconds) env_get_time_url = state.get("env_get_time_url") + servers = self._backing_servers(deployed) + shared = legacy_protocol.shared_names(servers) synced, skipped = [], [] - for service in self._service_names(deployed): - outcome = await self._sync_one(deployed, gateway_url_of(deployed), service, env_get_time_url) - (synced if outcome.get("synced") else skipped).append({"service": service, "environment": service, **outcome}) + for server in servers: + service = server.environment_name + record = None if service in shared and not server.mcp else deployed + outcome = await self._sync_one(record, gateway_url_of(deployed), service, env_get_time_url, mcp=server.mcp, + tolerate=self.tolerate_missing_sync_time or not server.mcp, + headers=deployed.control_headers) + entry = {"service": service, "environment": service, **({} if server.mcp else {"website": True}), **outcome} + (synced if outcome.get("synced") else skipped).append(entry) configuration = { "step_id": self.id, "env_id": self.env_id, @@ -151,17 +160,24 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: f"({[(s['service'], s.get('reason')) for s in skipped]}) — every tool still answers wall time") return context - async def _sync_one(self, deployed, gateway_url: str, service: str, env_get_time_url: Optional[str]) -> dict: - """Invoke sync_time on one server if its card advertises clock/v1; skip (or raise) per the flag.""" + async def _sync_one(self, deployed, gateway_url: str, service: str, env_get_time_url: Optional[str], mcp: bool = True, + tolerate: Optional[bool] = None, headers: Optional[dict] = None) -> dict: + """Invoke sync_time on one server, an MCP server or with ``mcp=False`` a website, if its card advertises + clock/v1; skip a server with no card or no clock/v1 when ``tolerate`` (tolerate_missing_sync_time by default), + else raise. ``execute`` tolerates every website, so one that doesn't follow the clock never fails the step; one + that follows it but can't take the task's timezone does, as an MCP server does. ``headers`` are the env's control + headers, ``deployed``'s by default; ``execute`` passes them because it hands a website that shares its name with + an MCP server no record, so its card is read live.""" + tolerate = self.tolerate_missing_sync_time if tolerate is None else tolerate from agent_env.env import legacy_protocol - base_url, card = await legacy_protocol.child_env_card(deployed, gateway_url, service, timeout=self.timeout_seconds) + base_url, card = await legacy_protocol.child_env_card(deployed, gateway_url, service, timeout=self.timeout_seconds, mcp=mcp) if card is None: - if self.tolerate_missing_sync_time: + if tolerate: return {"synced": False, "reason": "no_env_card"} raise RuntimeError(f"{service} has no env card: it serves none, its card isn't named {service!r}, or the deploy couldn't read it") if protocol_v1.find_extension(card, EXT_CLOCK_URI) is None: - if self.tolerate_missing_sync_time: + if tolerate: logger.info(f"sync_env_clock: {service} does not advertise {EXT_CLOCK_URI}; skipping") return {"synced": False, "reason": "not_advertised"} raise RuntimeError(f"{service} does not advertise {EXT_CLOCK_URI}") @@ -176,5 +192,6 @@ async def _sync_one(self, deployed, gateway_url: str, service: str, env_get_time params["timezone"] = self.timezone result = await protocol_v1.invoke_extension(base_url, card, EXT_CLOCK_URI, params=params, - timeout=self.timeout_seconds, headers=deployed.control_headers) + timeout=self.timeout_seconds, + headers=deployed.control_headers if headers is None else headers) return {"synced": True, "result": result} diff --git a/tst/unit/env/test_legacy_protocol.py b/tst/unit/env/test_legacy_protocol.py index 9508e8d2e..7d107ad34 100644 --- a/tst/unit/env/test_legacy_protocol.py +++ b/tst/unit/env/test_legacy_protocol.py @@ -177,3 +177,22 @@ async def test_a_v1_services_state_is_the_data_it_answers_with_else_its_export_s assert await legacy_protocol.service_state(None, "http://gw", "slack") == state assert ("GET /svc/mcp-slack/export-state" in asked) is reads_export_state + + +def test_backing_servers_are_an_envs_mcp_servers_then_its_websites(): + from types import SimpleNamespace + + from agent_env.env.envs.website import WebsiteEnv + from agent_env.env.legacy_protocol import BackingServer + + named = lambda *names: [SimpleNamespace(environment_name=n) for n in names] + multi = SimpleNamespace(mcp_server_envs=named("slack", "email"), website_envs=named("shop", "slack")) + + assert legacy_protocol.backing_servers(multi) == [BackingServer("slack"), BackingServer("email"), + BackingServer("shop", mcp=False), BackingServer("slack", mcp=False)] + assert legacy_protocol.shared_names(legacy_protocol.backing_servers(multi)) == {"slack"} + assert legacy_protocol.mcp_wins(legacy_protocol.backing_servers(multi)) == [ + BackingServer("slack"), BackingServer("email"), BackingServer("shop", mcp=False)] + assert legacy_protocol.backing_servers(WebsiteEnv(id="w", version=1, environment_name="shop")) == [BackingServer("shop", mcp=False)] + assert legacy_protocol.backing_servers(SimpleNamespace(environment_name="notes")) == [BackingServer("notes")] + assert legacy_protocol.backing_servers(SimpleNamespace()) == [] diff --git a/tst/unit/providers/env_providers/env_gateway_provider_test.py b/tst/unit/providers/env_providers/env_gateway_provider_test.py index a2f9958bc..f604e42cf 100644 --- a/tst/unit/providers/env_providers/env_gateway_provider_test.py +++ b/tst/unit/providers/env_providers/env_gateway_provider_test.py @@ -2466,6 +2466,18 @@ def test_a_one_image_website_env_is_fronted_with_only_its_server_image(): assert topology.website_images == [server] +def test_a_websites_server_reaches_the_gateways_clock_at_the_url_the_gateway_hands_it(): + """sync_env_clock hands each server http://gateway:/clock/time: the gateway's compose service is that host, + listening on that port, on the one network the website's server is on.""" + from agent_env.env.gateway.constants import GATEWAY_COMPOSE_HOST + + services = yaml.safe_load(_render([], [WebsiteConfig(backend_image="shop-be", frontend_image=None, environment_name="shop")]))["services"] + gateway, site = services[GATEWAY_COMPOSE_HOST], services["shop-website-backend"] + + assert "MCP_PORT=18765" in gateway["environment"] + assert gateway["networks"] == site["networks"] == ["env-network"] + + def _sites(handler): """Route the deploy's site checks to ``handler(request) -> httpx.Response``.""" real = httpx.AsyncClient diff --git a/tst/unit/task_step/sync_env_clock_test.py b/tst/unit/task_step/sync_env_clock_test.py index 5ab7bee8e..5b83c51d7 100644 --- a/tst/unit/task_step/sync_env_clock_test.py +++ b/tst/unit/task_step/sync_env_clock_test.py @@ -5,6 +5,7 @@ from agent_env.env.env import DeployedEnv, DeployedGatewayEnv, EnvCapabilityUnsupported from agent_env.env.gateway.constants import EXT_CLOCK_URI +from agent_env.env.legacy_protocol import BackingServer from agent_env.task_step.context import TaskStepContext from agent_env.task_step.task_steps import sync_env_clock as mod from agent_env.task_step.task_steps.sync_env_clock import SyncEnvClockTaskStep, resolve_virtual_time @@ -94,7 +95,7 @@ async def test_sync_one_skips_or_raises_for_a_child_env_missing_from_the_stored_ @pytest.mark.asyncio async def test_execute_syncs_each_child_env_through_the_stored_card(monkeypatch): sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) - monkeypatch.setattr(SyncEnvClockTaskStep, "_service_names", lambda self, deployed: ["email"]) + monkeypatch.setattr(SyncEnvClockTaskStep, "_backing_servers", lambda self, deployed: [BackingServer("email")]) context = TaskStepContext(deployed_envs=[_carded(await _composed_card({"mcp-email": [_CLOCK_EXT]}))]) await _step().execute(context) @@ -124,6 +125,96 @@ async def test_execute_keeps_the_arm_failure_message(monkeypatch): await _step().execute(context) +@pytest.mark.asyncio +async def test_execute_syncs_a_multi_envs_websites_alongside_its_mcp_servers(monkeypatch): + from types import SimpleNamespace + from unittest.mock import patch + + sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) + env = SimpleNamespace(mcp_server_envs=[SimpleNamespace(environment_name="email")], + website_envs=[SimpleNamespace(environment_name="shop"), SimpleNamespace(environment_name="blog")]) + context = TaskStepContext(deployed_envs=[_carded(await _composed_card({"mcp-email": [_CLOCK_EXT], "shop": [_CLOCK_EXT], "blog": []}))]) + + with patch("agent_env.env.env.Env.get", return_value=env): + await _step().execute(context) + + assert [(r.method, str(r.url)) for r in sent][2:] == [ + ("POST", f"{_CARD}/svc/mcp-email/agentenv/ext/sync_time"), ("POST", f"{_CARD}/svc/shop/agentenv/ext/sync_time"), + ] + record = context.metadata["clock_configurations"][0] + assert [(s["service"], s.get("website")) for s in record["synced"]] == [("email", None), ("shop", True)] + assert [(s["service"], s["website"], s["reason"]) for s in record["skipped"]] == [("blog", True, "not_advertised")] + + +@pytest.mark.asyncio +async def test_execute_syncs_a_single_website_env(monkeypatch): + from unittest.mock import patch + + from agent_env.env.envs.website import WebsiteEnv + + sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) + context = TaskStepContext(deployed_envs=[_carded(await _composed_card({"shop": [_CLOCK_EXT]}))]) + + with patch("agent_env.env.env.Env.get", return_value=WebsiteEnv(id="e1", version=1, environment_name="shop")): + await _step().execute(context) + + assert [(r.method, str(r.url)) for r in sent][2:] == [("POST", f"{_CARD}/svc/shop/agentenv/ext/sync_time")] + assert json.loads(sent[2].content) == {"env_get_time_url": CLOCK_URL} + + +@pytest.mark.asyncio +async def test_sync_one_reads_a_websites_card_live_at_its_own_gateway_path(monkeypatch): + read = [] + + async def fake_card(base_url, **kw): + read.append(base_url) + return {"capabilities": {"extensions": []}} + + monkeypatch.setattr(mod.protocol_v1, "get_card", fake_card) + assert await _step()._sync_one(None, "http://gw", "shop", CLOCK_URL, mcp=False) == {"synced": False, "reason": "not_advertised"} + assert read == ["http://gw/svc/shop"] + + +@pytest.mark.asyncio +async def test_execute_reads_a_website_sharing_an_mcp_servers_name_live(monkeypatch): + from types import SimpleNamespace + from unittest.mock import patch + + sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) + env = SimpleNamespace(mcp_server_envs=[SimpleNamespace(environment_name="email")], + website_envs=[SimpleNamespace(environment_name="email")]) + context = TaskStepContext(deployed_envs=[_carded(await _composed_card({"mcp-email": [_CLOCK_EXT]}))]) + + with patch("agent_env.env.env.Env.get", return_value=env): + await _step().execute(context) + + assert [(r.method, str(r.url)) for r in sent][2:] == [ + ("POST", f"{_CARD}/svc/mcp-email/agentenv/ext/sync_time"), ("GET", "http://gw/svc/email/.well-known/agent-env.json"), + ] + + +@pytest.mark.asyncio +async def test_strict_execute_skips_websites_but_not_mcp_servers_that_do_not_follow_the_clock(monkeypatch): + from types import SimpleNamespace + from unittest.mock import patch + + _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) + env = SimpleNamespace(mcp_server_envs=[SimpleNamespace(environment_name="email")], + website_envs=[SimpleNamespace(environment_name="shop"), SimpleNamespace(environment_name="legacy")]) + context = TaskStepContext(deployed_envs=[_carded(await _composed_card({"mcp-email": [_CLOCK_EXT], "shop": []}))]) + + with patch("agent_env.env.env.Env.get", return_value=env): + await _step(tolerate_missing_sync_time=False).execute(context) + + record = context.metadata["clock_configurations"][0] + assert [s["service"] for s in record["synced"]] == ["email"] + assert [(s["service"], s["reason"]) for s in record["skipped"]] == [("shop", "not_advertised"), ("legacy", "no_env_card")] + + unsynced = TaskStepContext(deployed_envs=[_carded(await _composed_card({"mcp-email": [], "shop": [_CLOCK_EXT]}))]) + with patch("agent_env.env.env.Env.get", return_value=env), pytest.raises(RuntimeError, match="email does not advertise"): + await _step(tolerate_missing_sync_time=False).execute(unsynced) + + _CLOCK_EXT = {"uri": EXT_CLOCK_URI, "params": {"endpoint": "/agentenv/ext/sync_time", "methods": {"sync_time": {"method": "POST"}}}} _ZONE_EXT = {"uri": EXT_CLOCK_URI, "params": {"endpoint": "/agentenv/ext/sync_time", "methods": {"sync_time": {"method": "POST", "request": { "type": "object", "properties": {"env_get_time_url": {"type": "string"}, "timezone": {"type": "string"}}, "required": ["env_get_time_url"]}}}}} @@ -226,7 +317,7 @@ def test_without_a_zone_virtual_time_is_passed_through_untouched(): @pytest.mark.asyncio async def test_execute_arms_the_gateway_at_the_local_instant_and_tells_every_server_the_zone(monkeypatch): sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) - monkeypatch.setattr(SyncEnvClockTaskStep, "_service_names", lambda self, deployed: ["email"]) + monkeypatch.setattr(SyncEnvClockTaskStep, "_backing_servers", lambda self, deployed: [BackingServer("email")]) context = TaskStepContext(deployed_envs=[_carded(await _composed_card({"mcp-email": [_ZONE_EXT]}))]) await _zoned().execute(context) @@ -241,7 +332,7 @@ async def test_execute_arms_the_gateway_at_the_local_instant_and_tells_every_ser @pytest.mark.parametrize("tolerate", [True, False]) async def test_a_server_that_cannot_take_the_zone_fails_the_step_before_any_sync_is_sent(monkeypatch, tolerate): sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) - monkeypatch.setattr(SyncEnvClockTaskStep, "_service_names", lambda self, deployed: ["email"]) + monkeypatch.setattr(SyncEnvClockTaskStep, "_backing_servers", lambda self, deployed: [BackingServer("email")]) context = TaskStepContext(deployed_envs=[_carded(await _composed_card({"mcp-email": [_OLD_EXT]}))]) with pytest.raises(RuntimeError, match="takes no timezone"): @@ -264,6 +355,62 @@ async def test_a_server_that_does_not_advertise_clock_is_still_skipped_when_a_zo assert out == {"synced": False, "reason": "not_advertised"} +@pytest.mark.asyncio +async def test_a_zoned_step_tells_a_website_that_takes_the_zone_and_fails_on_one_that_cannot(monkeypatch): + """A website is tolerated for not following the clock, but one that follows it without taking the zone fails the + step, as an MCP server does.""" + from types import SimpleNamespace + from unittest.mock import patch + + env = SimpleNamespace(mcp_server_envs=[], website_envs=[SimpleNamespace(environment_name="shop")]) + sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) + context = TaskStepContext(deployed_envs=[_carded(await _composed_card({"shop": [_ZONE_EXT]}))]) + with patch("agent_env.env.env.Env.get", return_value=env): + await _zoned().execute(context) + assert json.loads(sent[2].content) == {"env_get_time_url": CLOCK_URL, "timezone": "America/New_York"} + assert context.metadata["clock_configurations"][0]["synced"][0]["website"] is True + + _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) + older = TaskStepContext(deployed_envs=[_carded(await _composed_card({"shop": [_OLD_EXT]}))]) + with patch("agent_env.env.env.Env.get", return_value=env), pytest.raises(RuntimeError, match="shop advertises .* takes no timezone"): + await _zoned().execute(older) + + +@pytest.mark.asyncio +async def test_a_website_sharing_an_mcp_servers_name_is_synced_with_the_envs_control_headers(monkeypatch): + """Its card is read live, with no record, and its sync_time still carries the env's control token.""" + import dataclasses + from types import SimpleNamespace + from unittest.mock import patch + + import httpx + + sent, real = [], httpx.AsyncClient + website_card = {"name": "email", "url": "/agentenv", "capabilities": {"extensions": [_CLOCK_EXT]}} + + def handle(request: httpx.Request) -> httpx.Response: + sent.append(request) + if request.url.path.endswith("/clock/state"): + return httpx.Response(200, json={"env_get_time_url": CLOCK_URL}) + if request.url.path == "/svc/email/.well-known/agent-env.json": + return httpx.Response(200, json=website_card) + return httpx.Response(200, json={}) + + monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **k: real(transport=httpx.MockTransport(handle))) + env = SimpleNamespace(mcp_server_envs=[SimpleNamespace(environment_name="email")], + website_envs=[SimpleNamespace(environment_name="email")]) + record = dataclasses.replace(_carded(await _composed_card({"mcp-email": [_CLOCK_EXT]})), control_token="ctl") + context = TaskStepContext(deployed_envs=[record]) + + with patch("agent_env.env.env.Env.get", return_value=env): + await _step().execute(context) + + website_sync = [r for r in sent if str(r.url) == "http://gw/svc/email/agentenv/ext/sync_time"] + assert len(website_sync) == 1 + assert {k: website_sync[0].headers.get(k) for k in record.control_headers} == record.control_headers + assert [s["service"] for s in context.metadata["clock_configurations"][0]["synced"]] == ["email", "email"] + + def test_a_padded_zone_is_stored_and_sent_stripped(): s = _zoned(timezone=" America/New_York ") assert s.timezone == "America/New_York" and s.to_dict()["timezone"] == "America/New_York" diff --git a/tst/unit/task_step/test_apply_server_config.py b/tst/unit/task_step/test_apply_server_config.py index f9badb26c..ddeb96806 100644 --- a/tst/unit/task_step/test_apply_server_config.py +++ b/tst/unit/task_step/test_apply_server_config.py @@ -589,6 +589,97 @@ async def test_live_read_strips_a_trailing_slash_from_the_gateway_url(): assert [url for url, _ in gw.post_calls] == [f"{_GATEWAY}/svc/mcp-slack/agentenv/ext/set_errors"] +# --- websites --- + + +def _fake_multi_env(mcp_names, website_names): + return SimpleNamespace(mcp_server_envs=[SimpleNamespace(environment_name=s) for s in mcp_names], + website_envs=[SimpleNamespace(environment_name=s) for s in website_names]) + + +class _KeyedGateway(_MockGateway): + """A gateway serving each card at /svc//.well-known/agent-env.json, where a website's key is its bare name.""" + + async def get(self, url, *, timeout=None, params=None): + self.get_calls.append(url) + for key, card in self.cards.items(): + if url.endswith(f"/svc/{key}/.well-known/agent-env.json"): + return _resp(200, url, card) + return _resp(404, url, {"error": "no card"}) + + +@pytest.mark.asyncio +async def test_stored_card_broadcast_reaches_websites_and_leaves_a_shared_name_to_the_mcp_server(): + gw = _MockGateway({}) + ext = _ext(_SET_ACTING_USER_URI, "/agentenv/ext/set_acting_user") + ctx = _carded_context(await _composed_card({"mcp-slack": [ext], "shop": [ext]})) + step = _step(directives=[{"service": "*", "uri": _SET_ACTING_USER_URI, "args": {"user_email": "a@x.com"}}]) + with _patch_httpx(gw), patch("agent_env.env.env.Env.get", return_value=_fake_multi_env(["slack"], ["shop", "slack"])): + out = await step.execute(ctx) + + assert gw.get_calls == [] + assert [url for url, _ in gw.post_calls] == [f"{_CARD}/svc/mcp-slack/agentenv/ext/set_acting_user", + f"{_CARD}/svc/shop/agentenv/ext/set_acting_user"] + assert [c["service"] for c in out.metadata["server_config_changes"]] == ["slack", "shop"] + + +@pytest.mark.asyncio +async def test_live_read_broadcast_reads_a_websites_card_at_its_own_gateway_path(): + ext = _ext(_SET_ACTING_USER_URI, "/agentenv/ext/set_acting_user") + gw = _KeyedGateway({"mcp-slack": _card(ext), "shop": _card(ext)}) + step = _step(directives=[{"service": "*", "uri": _SET_ACTING_USER_URI, "args": {"user_email": "a@x.com"}}]) + with _patch_httpx(gw), patch("agent_env.env.env.Env.get", return_value=_fake_multi_env(["slack"], ["shop"])): + await step.execute(_make_context()) + + assert gw.get_calls == [f"{_GATEWAY}/svc/mcp-slack/.well-known/agent-env.json", f"{_GATEWAY}/svc/shop/.well-known/agent-env.json"] + assert [url for url, _ in gw.post_calls] == [f"{_GATEWAY}/svc/mcp-slack/agentenv/ext/set_acting_user", + f"{_GATEWAY}/svc/shop/agentenv/ext/set_acting_user"] + + +@pytest.mark.asyncio +async def test_broadcast_reaches_a_single_website_env(): + from agent_env.env.envs.website import WebsiteEnv + + gw = _MockGateway({}) + ctx = _carded_context(await _composed_card({"shop": [_ext(_SET_ACTING_USER_URI, "/agentenv/ext/set_acting_user")]})) + step = _step(directives=[{"service": "*", "uri": _SET_ACTING_USER_URI, "args": {"user_email": "a@x.com"}}]) + with _patch_httpx(gw), patch("agent_env.env.env.Env.get", return_value=WebsiteEnv(id="env-x", version=1, environment_name="shop")): + await step.execute(ctx) + + assert [url for url, _ in gw.post_calls] == [f"{_CARD}/svc/shop/agentenv/ext/set_acting_user"] + + +@pytest.mark.asyncio +async def test_strict_broadcast_skips_websites_without_a_card_or_the_extension(): + gw = _MockGateway({}) + ext = _ext(_SET_ERRORS_URI, "/agentenv/ext/set_errors") + ctx = _carded_context(await _composed_card({"mcp-slack": [ext], "shop": []})) + step = _step(directives=[{"service": "*", "uri": _SET_ERRORS_URI, "args": {"error_rate": 1.0}}]) + with _patch_httpx(gw), patch("agent_env.env.env.Env.get", return_value=_fake_multi_env(["slack"], ["shop", "legacy"])): + out = await step.execute(ctx) + + assert [url for url, _ in gw.post_calls] == [f"{_CARD}/svc/mcp-slack/agentenv/ext/set_errors"] + assert [c["service"] for c in out.metadata["server_config_changes"]] == ["slack"] + assert [(s["service"], s["reason"]) for s in out.metadata["server_config_skipped"]] == [ + ("shop", "extension_not_advertised"), ("legacy", "no_env_card")] + + +@pytest.mark.asyncio +async def test_strict_broadcast_still_fails_for_an_mcp_server_or_a_named_website_lacking_the_extension(): + ext = _ext(_SET_ERRORS_URI, "/agentenv/ext/set_errors") + broadcast = {"service": "*", "uri": _SET_ERRORS_URI, "args": {"error_rate": 1.0}} + unadvertising_mcp = _carded_context(await _composed_card({"mcp-slack": [], "shop": [ext]})) + with _patch_httpx(_MockGateway({})), patch("agent_env.env.env.Env.get", return_value=_fake_multi_env(["slack"], ["shop"])): + with pytest.raises(ApplyServerConfigError, match="not advertised on card for service 'slack'"): + await _step(directives=[broadcast]).execute(unadvertising_mcp) + + unadvertising_shop = _carded_context(await _composed_card({"mcp-slack": [ext], "shop": []})) + named = {"service": "shop", "uri": _SET_ERRORS_URI, "args": {"error_rate": 1.0}} + with _patch_httpx(_MockGateway({})), patch("agent_env.env.env.Env.get", return_value=_fake_multi_env(["slack"], ["shop"])): + with pytest.raises(ApplyServerConfigError, match="not advertised on card for service 'shop'"): + await _step(directives=[broadcast, named]).execute(unadvertising_shop) + + # --- schema / validation (pure, no I/O) --- diff --git a/tst/unit/task_step/test_snapshot_env.py b/tst/unit/task_step/test_snapshot_env.py index 877244c37..380690fc2 100644 --- a/tst/unit/task_step/test_snapshot_env.py +++ b/tst/unit/task_step/test_snapshot_env.py @@ -1367,3 +1367,177 @@ def shared_credentials_env(self) -> dict[str, str]: store = _SharingS3Store(client=object(), bucket="bucket") monkeypatch.setattr(snapshot_mod, "get_config", lambda: type("Cfg", (), {"get_object_store": lambda self: store})()) + + +# ── websites ───────────────────────────────────────────────────────────────── + +def _multi_env(mcp_names, website_names): + return SimpleNamespace(id="env-1", version=3, mcp_server_envs=[_FakeServiceEnv(n) for n in mcp_names], + website_envs=[_FakeServiceEnv(n) for n in website_names]) + + +def test_enumerate_includes_websites_and_leaves_a_shared_name_to_the_mcp_server(): + servers = SnapshotEnvTaskStep._enumerate_servers(_multi_env(["slack", "gmail"], ["shop", "slack"])) + assert [(s.environment_name, s.mcp) for s in servers] == [("slack", True), ("gmail", True), ("shop", False)] + + +def test_enumerate_a_single_website_env_and_a_multi_env_of_websites_only(): + from agent_env.env.envs.website import WebsiteEnv + + site = WebsiteEnv(id="site-env", version=1, environment_name="shop") + assert [(s.environment_name, s.mcp) for s in SnapshotEnvTaskStep._enumerate_servers(site)] == [("shop", False)] + assert SnapshotEnvTaskStep._enumerate_environments(_multi_env([], ["shop"])) == ["shop"] + + +@pytest.mark.asyncio +async def test_export_of_a_website_reads_its_data_plane_from_the_stored_card(monkeypatch, tmp_path): + record = _carded(await _card({"mcp-slack": [], "shop": []})) + sent = _mock_http(monkeypatch, rpc_result={"parts": [{"kind": "data", "data": {"orders": 2}}]}) + + out = str(tmp_path / "export-tmp") + suffix = await SnapshotEnvTaskStep._export_environment_to_file("https://gw-1", "shop", out, 30, record, mcp=False) + + assert suffix == ".json" + assert [(r.method, str(r.url)) for r in sent] == [("POST", f"{_CARD}/svc/shop/agentenv")] + with open(out) as f: + assert json.load(f) == {"orders": 2} + + +@pytest.mark.asyncio +async def test_export_of_a_website_without_a_stored_card_probes_its_own_gateway_path(monkeypatch, tmp_path): + import dataclasses + + record = dataclasses.replace(_carded({}), environment_card=None) + sent = _mock_http(monkeypatch, rpc_result={"parts": [{"kind": "data", "data": {"orders": 2}}]}, + body=b'{"name": "shop", "url": "/agentenv"}') + + suffix = await SnapshotEnvTaskStep._export_environment_to_file("https://gw-1", "shop", str(tmp_path / "out"), 30, record, mcp=False) + + assert suffix == ".json" + assert [(r.method, str(r.url)) for r in sent] == [ + ("GET", "https://gw-1/svc/shop/.well-known/agent-env.json"), + ("GET", "https://gw-1/svc/shop/.well-known/agent-env.json"), + ("POST", "https://gw-1/svc/shop/agentenv"), + ] + + +def _serve_website_card(monkeypatch, card_response): + """A gateway whose website ``shop`` answers its card read with ``card_response`` (a response or an exception to + raise) and whose MCP server ``slack`` serves a v1 data plane; returns the requests sent.""" + import httpx + + sent, real = [], httpx.AsyncClient + + def handle(request: httpx.Request) -> httpx.Response: + sent.append(request) + if request.url.path == f"/svc/shop{WELL_KNOWN_PATH}": + if isinstance(card_response, Exception): + raise card_response + return card_response + if request.url.path.endswith("/agentenv"): + return httpx.Response(200, json={"jsonrpc": "2.0", "id": 1, "result": {"parts": [{"kind": "data", "data": {}}]}}) + return httpx.Response(200, json={}) + + monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **k: real(transport=httpx.MockTransport(handle))) + return sent + + +def _card_read_failures(): + import httpx + + return [httpx.Response(500, json={}), httpx.Response(503, json={}), httpx.ReadTimeout("card read timed out"), + httpx.ConnectError("connection refused")] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("card_response", _card_read_failures(), ids=["500", "503", "timeout", "connect-error"]) +async def test_export_of_a_website_whose_card_read_fails_raises_rather_than_skipping_it(monkeypatch, tmp_path, card_response): + """Without a stored card, a website card read that fails is no proof the website serves no data plane.""" + import httpx + + sent = _serve_website_card(monkeypatch, card_response) + + with pytest.raises(httpx.HTTPError): + await SnapshotEnvTaskStep._export_environment_to_file("https://gw-1", "shop", str(tmp_path / "out"), 30, mcp=False) + assert [str(r.url) for r in sent] == [f"https://gw-1/svc/shop{WELL_KNOWN_PATH}"] + + +@pytest.mark.asyncio +async def test_export_of_a_website_without_a_card_is_skipped(monkeypatch, tmp_path): + import httpx + + sent = _serve_website_card(monkeypatch, httpx.Response(404, json={})) + + assert await SnapshotEnvTaskStep._export_environment_to_file("https://gw-1", "shop", str(tmp_path / "out"), 30, mcp=False) is None + assert [str(r.url) for r in sent] == [f"https://gw-1/svc/shop{WELL_KNOWN_PATH}"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("card_response", _card_read_failures(), ids=["500", "503", "timeout", "connect-error"]) +async def test_core_fails_a_snapshot_whose_website_card_cannot_be_read(monkeypatch, card_response): + """Another server exporting does not let the snapshot through without a website it could not check.""" + real_export = SnapshotEnvTaskStep.__dict__["_export_environment_to_file"] + _install_core_artifact_fakes(monkeypatch, EnvironmentUniverseArtifact) + monkeypatch.setattr(SnapshotEnvTaskStep, "_export_environment_to_file", real_export) + _serve_website_card(monkeypatch, card_response) + + with pytest.raises(RuntimeError, match=r"1/2 services were not exported \(missing=\['shop'\]\)"): + await SnapshotEnvTaskStep.snapshot_env_state( + env=_multi_env(["slack"], ["shop"]), gateway_url="https://gw-1", snapshot_id="snap-x", + ) + + +@pytest.mark.asyncio +async def test_export_of_a_website_serving_no_data_plane_exports_nothing_and_sends_nothing(monkeypatch, tmp_path): + record = _carded(await _card({"mcp-slack": []})) + sent = _mock_http(monkeypatch) + + assert await SnapshotEnvTaskStep._export_environment_to_file("https://gw-1", "shop", str(tmp_path / "out"), 30, record, mcp=False) is None + assert sent == [] + + +@pytest.mark.asyncio +async def test_core_snapshots_websites_through_their_data_plane_and_leaves_out_those_without_one(monkeypatch): + calls = [] + + async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None, **website): + calls.append((name, website)) + if name == "legacy-site": + return None + with open(tmp_path, "w") as f: + json.dump({"ok": name}, f) + return ".json" + + universe_puts = {} + + class _FakeUniverse: + id = "snap-x" + version = 1 + + @staticmethod + def put(id, *, environment_artifacts, metadata=None): + universe_puts["names"] = [sa.environment_name for sa in environment_artifacts] + return _FakeUniverse() + + _install_core_artifact_fakes(monkeypatch, _FakeUniverse) + monkeypatch.setattr(SnapshotEnvTaskStep, "_export_environment_to_file", staticmethod(fake_export)) + + result = await SnapshotEnvTaskStep.snapshot_env_state( + env=_multi_env(["slack"], ["shop", "slack", "legacy-site"]), gateway_url="https://gw", snapshot_id="snap-x", + ) + + assert sorted(calls) == [("legacy-site", {"mcp": False}), ("shop", {"mcp": False}), ("slack", {})] + assert universe_puts["names"] == ["slack", "shop"] + assert result.environments_snapshotted == ["slack", "shop"] and result.total == 3 + + +@pytest.mark.asyncio +async def test_core_raises_when_no_service_exported_state(monkeypatch): + async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, **website): + return None + + _install_core_artifact_fakes(monkeypatch, EnvironmentUniverseArtifact) + monkeypatch.setattr(SnapshotEnvTaskStep, "_export_environment_to_file", staticmethod(fake_export)) + + with pytest.raises(RuntimeError, match=r"exported no state: its websites \['shop'\] serve no v1 data plane"): + await SnapshotEnvTaskStep.snapshot_env_state(env=_multi_env([], ["shop"]), gateway_url="https://gw", snapshot_id="snap-x") From b18049d5eeaed225444da4c5da146f8850b4e1ef Mon Sep 17 00:00:00 2001 From: William Loo Date: Tue, 3 Nov 2026 12:52:03 -0800 Subject: [PATCH 02/12] fix(website): sync_env_clock, snapshot_env and broadcasts reach each server the env card lists, at its own route (FD-4683) The servers behind an env's gateway come from its card's children_environments, each at its own /svc route, through new protocol helpers (ChildEnv, children, child_at, child_named, serves), not from the env's definition: a website sharing an MCP server's name is its own server, and a card without data/get in its operations serves no v1 data plane. The definition is read only for servers that serve no card, and for a record whose card can't be read, logged. Co-Authored-By: Claude Opus 5.5 (1M context) --- packages/agentenv-protocol/README.md | 10 ++ .../src/agentenv_protocol/client.py | 80 +++++++++ .../agentenv-protocol/tests/test_client.py | 52 ++++++ src/agent_env/env/legacy_protocol.py | 105 ++++++++++-- .../task_steps/apply_server_config.py | 81 +++++---- .../task_step/task_steps/snapshot_env.py | 86 +++++----- .../task_step/task_steps/sync_env_clock.py | 44 ++--- tst/unit/env/test_legacy_protocol.py | 100 ++++++++++- tst/unit/task/local_run_write_fence_test.py | 2 +- tst/unit/task_step/sync_env_clock_test.py | 80 ++++++--- .../task_step/test_apply_server_config.py | 30 +++- tst/unit/task_step/test_snapshot_env.py | 156 ++++++++++++++---- 12 files changed, 631 insertions(+), 195 deletions(-) diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index b6cc8c649..4e3b0de48 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -198,6 +198,16 @@ with any other transport are kept beside the SDK's entry. Card URLs are paths, r address the card was fetched from. On the client side, `client.mcp_path(card)` returns the declared path, or `/mcp` for a card without one. +A composed card, such as a gateway's, nests each backing environment's card under +`children_environments`, its `url` and endpoints rooted at the route the composer serves it at. +`client.children(card)` returns each as a `ChildEnv` (its `name`, `url` and `card`, with its +`extensions`, `operations` and `serves(method)`, and `base_url(address)`, its data plane resolved +against the composed card's address). Two children may share a name, as a website and an MCP +server can, so `client.child_at(card, url)` finds one by its url or route; `client.child_named(card, +name)` raises `AmbiguousChildError` for a shared name, and `client.find_child` returns the first. +`client.serves(card, method)` says whether a card serves a data-plane method by its declared +`operations`, None for a card that declares none. + Extensions are invoked from what the card advertises. `client.find_extension_method(card, uri, method)` returns one advertised method, whose `endpoint` is the method's own or else the extension's, and `client.invoke_extension(base_url, card, uri, params, method=...)` calls it with diff --git a/packages/agentenv-protocol/src/agentenv_protocol/client.py b/packages/agentenv-protocol/src/agentenv_protocol/client.py index dcddaf541..a1e352879 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/client.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/client.py @@ -7,6 +7,7 @@ import logging from collections.abc import Iterator from contextvars import ContextVar +from dataclasses import dataclass, field import httpx @@ -210,6 +211,83 @@ def find_tool(card: dict, name: str) -> dict | None: return None +def serves(card: dict, method: str) -> bool | None: + """Whether a card serves the data-plane ``method`` (``data/get`` say): True or False by its declared + ``capabilities.operations``, or None when it declares none, as an older card doesn't, so the caller decides as it + did before cards declared them (a probe, say).""" + declared = operations(card) + return None if declared is None else method in declared + + +@dataclass(frozen=True) +class ChildEnv: + """One child env a composed card nests under ``children_environments``: its ``name`` and ``url`` as its card gives + them, and that ``card``, which says what it offers. A composer roots each child's ``url`` (and its endpoints) at + the route it serves the child at, so two children sharing a name have distinct urls; ``url`` is None for a child + whose card names none.""" + + name: str | None + url: str | None + card: dict = field(hash=False, repr=False) + + @property + def extensions(self) -> list[dict]: + """The extensions the child's card advertises (``capabilities.extensions``).""" + return list((self.card.get("capabilities") or {}).get("extensions") or []) + + @property + def operations(self) -> list[str] | None: + """The data-plane methods the child's card declares (``operations``), or None when it declares none.""" + return operations(self.card) + + def find_extension(self, uri: str) -> dict | None: + """The extension the child's card advertises under ``uri``, or None (``find_extension``).""" + return find_extension(self.card, uri) + + def serves(self, method: str) -> bool | None: + """Whether the child's card serves the data-plane ``method``, None when it declares no operations (``serves``).""" + return serves(self.card, method) + + def base_url(self, address: str) -> str: + """Where the child's data plane is, its ``url`` without the RPC path, resolved against ``address``, the composed + card's: a path joins onto it, a full URL stands alone, and a child naming no url is at ``address`` itself.""" + url = (self.url or RPC_PATH).removesuffix(RPC_PATH) + return url if "://" in url else f"{address.rstrip('/')}{url}" + + +def children(card: dict) -> list[ChildEnv]: + """Each child env a composed card nests under ``children_environments``, in its order; none for a leaf card, or + one whose ``children_environments`` is missing or null. An entry that isn't an object is skipped.""" + return [ChildEnv(name=child.get("name"), url=child.get("url"), card=child) + for child in card.get("children_environments") or [] if isinstance(child, dict)] + + +def child_at(card: dict, url: str) -> ChildEnv | None: + """The child env of a composed card at ``url``: the one whose ``url`` is ``url``, or whose data plane is there + (``url`` without the RPC path, ``/svc/mcp-slack`` for ``/svc/mcp-slack/agentenv`` say), a trailing slash aside; + None when no child is. A composer serves each child at its own route, so this is the lookup two children sharing a + name can't confuse.""" + wanted = url.rstrip("/") + for child in children(card): + if child.url is not None and wanted in (child.url.rstrip("/"), child.url.removesuffix(RPC_PATH).rstrip("/")): + return child + return None + + +class AmbiguousChildError(LookupError): + """More than one child env of a composed card has the name asked for, so the name can't say which is meant.""" + + +def child_named(card: dict, name: str) -> ChildEnv | None: + """The one child env of a composed card named ``name``, or None when none is; raises ``AmbiguousChildError`` when + several are, as a website and an MCP server sharing a name are, which ``child_at`` tells apart by their urls.""" + found = [child for child in children(card) if child.name == name] + if len(found) > 1: + raise AmbiguousChildError(f"{len(found)} child envs are named {name!r} ({', '.join(str(c.url) for c in found)}); " + "look one up by its url") + return found[0] if found else None + + def find_child(card: dict, name: str) -> dict | None: """Return the nested child EnvironmentCard named `name`, or None. @@ -218,6 +296,8 @@ def find_child(card: dict, name: str) -> dict | None: rewritten to be reachable from the composed card's base_url. Callers resolve a service by name here, then use the child card with `find_extension`/`invoke_extension`. Tolerates a missing/null `children_environments` so single/leaf cards return None rather than raising. + Two children sharing `name` return the first; `child_named` refuses that, and `child_at` + finds a child by its url, which tells them apart. """ for child in card.get("children_environments") or []: if child.get("name") == name: diff --git a/packages/agentenv-protocol/tests/test_client.py b/packages/agentenv-protocol/tests/test_client.py index edcd38798..bfc65e9a7 100644 --- a/packages/agentenv-protocol/tests/test_client.py +++ b/packages/agentenv-protocol/tests/test_client.py @@ -190,6 +190,58 @@ def test_find_child_tolerates_missing_children(): assert protocol_v1.find_child({"children_environments": None}, "slack") is None +_SHARED_NAME = {"name": "gw", "children_environments": [ + {"name": "mail", "url": "/svc/mcp-mail/agentenv", "capabilities": {"operations": ["data/get"], "extensions": [ + {"uri": "urn:agentenv:clock/v1"}]}}, + {"name": "mail", "url": "/svc/mail/agentenv", "capabilities": {"operations": []}}, + {"name": "docs", "capabilities": {}}, + "not a card", +]} + + +def test_children_are_each_nested_card_by_its_name_url_and_card(): + found = protocol_v1.children(_SHARED_NAME) + + assert [(c.name, c.url) for c in found] == [("mail", "/svc/mcp-mail/agentenv"), ("mail", "/svc/mail/agentenv"), ("docs", None)] + assert found[0].card is _SHARED_NAME["children_environments"][0] + assert [c.extensions for c in found] == [[{"uri": "urn:agentenv:clock/v1"}], [], []] + assert [c.operations for c in found] == [["data/get"], [], None] + assert [c.serves("data/get") for c in found] == [True, False, None] + assert found[0].find_extension("urn:agentenv:clock/v1") is not None and found[1].find_extension("urn:agentenv:clock/v1") is None + assert protocol_v1.children({"name": "leaf"}) == [] and protocol_v1.children({"children_environments": None}) == [] + + +def test_a_childs_base_url_is_its_url_without_the_rpc_path_against_the_composed_cards_address(): + mcp, website, docs = protocol_v1.children(_SHARED_NAME) + + assert mcp.base_url("https://env.example/") == "https://env.example/svc/mcp-mail" + assert website.base_url("https://env.example") == "https://env.example/svc/mail" + assert docs.base_url("https://env.example") == "https://env.example" + assert protocol_v1.ChildEnv("x", "https://x.example/agentenv", {}).base_url("https://env.example") == "https://x.example" + + +def test_child_at_finds_a_child_by_its_url_or_data_plane_route_where_two_share_a_name(): + assert protocol_v1.child_at(_SHARED_NAME, "/svc/mail").url == "/svc/mail/agentenv" + assert protocol_v1.child_at(_SHARED_NAME, "/svc/mcp-mail/").url == "/svc/mcp-mail/agentenv" + assert protocol_v1.child_at(_SHARED_NAME, "/svc/mail/agentenv").url == "/svc/mail/agentenv" + assert protocol_v1.child_at(_SHARED_NAME, "/svc/docs") is None + assert protocol_v1.child_at({"name": "leaf"}, "/svc/mail") is None + + +def test_child_named_refuses_a_name_two_children_share(): + assert protocol_v1.child_named(_SHARED_NAME, "docs").name == "docs" + assert protocol_v1.child_named(_SHARED_NAME, "gmail") is None + with pytest.raises(protocol_v1.AmbiguousChildError, match="2 child envs are named 'mail'"): + protocol_v1.child_named(_SHARED_NAME, "mail") + assert protocol_v1.find_child(_SHARED_NAME, "mail")["url"] == "/svc/mcp-mail/agentenv" + + +def test_serves_is_what_a_card_declares_of_a_method_and_none_when_it_declares_no_operations(): + assert protocol_v1.serves({"capabilities": {"operations": ["data/get"]}}, "data/get") is True + assert protocol_v1.serves({"capabilities": {"operations": ["data/add"]}}, "data/get") is False + assert protocol_v1.serves({"capabilities": {}}, "data/get") is None + + def test_find_extension_ignores_children(): # set-errors/v1 is advertised only on the children, never at the top level assert protocol_v1.find_extension(_CARD_WITH_CHILDREN, "urn:agentenv:set-errors/v1") is None diff --git a/src/agent_env/env/legacy_protocol.py b/src/agent_env/env/legacy_protocol.py index 8790157ac..f5ca7a002 100644 --- a/src/agent_env/env/legacy_protocol.py +++ b/src/agent_env/env/legacy_protocol.py @@ -8,12 +8,15 @@ from __future__ import annotations import asyncio +import dataclasses import logging from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Optional import httpx -from agentenv_protocol import RPC_PATH, DataPart, client as protocol_v1 +from agentenv_protocol import METHOD_GET, RPC_PATH, DataPart, client as protocol_v1 + +from agent_env.env.gateway.constants import EXT_WEBSITE_URI if TYPE_CHECKING: from agent_env.env.env import DeployedEnv @@ -36,16 +39,28 @@ def environment_base_url(gateway_url: Optional[str], environment_name: str, mcp: @dataclass(frozen=True) class BackingServer: - """A server behind an env's gateway, by its environment_name: an MCP server, at ``/svc/mcp-{name}``, or a website, - at ``/svc/{name}``.""" + """A server behind an env's address, by its environment_name: an MCP server or, with ``mcp=False``, a website. + + One the env's card lists (``card_servers``) carries that ``child`` and the ``address`` its paths resolve against. + One the card lists none for, a server serving no card, carries the ``address`` alone: it has no card, and none is + read for it. One named by the env definition of a record whose card can't be read carries neither, so its card is + read live at its gateway path, ``/svc/mcp-{name}`` or a website's ``/svc/{name}`` (``server_card``).""" environment_name: str mcp: bool = True + child: Optional[protocol_v1.ChildEnv] = None + address: Optional[str] = None + + @property + def card(self) -> Optional[dict]: + """The server's card as the env's card lists it, or None.""" + return self.child.card if self.child is not None else None -def backing_servers(env: Any) -> list[BackingServer]: - """The servers behind env's gateway: a MultiEnv's MCP servers then its websites, a WebsiteEnv's website, or else - the env's one MCP server; empty for an env with none.""" +def definition_servers(env: Any) -> list[BackingServer]: + """The servers an env definition names: a MultiEnv's MCP servers then its websites, a WebsiteEnv's website, or + else the env's one MCP server; empty for an env with none. Read only for what the env's card can't say + (``backing_servers``).""" from agent_env.env.envs.website import WebsiteEnv if isinstance(env, WebsiteEnv): @@ -57,16 +72,74 @@ def backing_servers(env: Any) -> list[BackingServer]: return servers -def shared_names(servers: list[BackingServer]) -> set[str]: - """The environment_names an MCP server and a website in ``servers`` share, which a stored env card can't tell apart.""" - return {s.environment_name for s in servers if s.mcp} & {s.environment_name for s in servers if not s.mcp} - - -def mcp_wins(servers: list[BackingServer]) -> list[BackingServer]: - """``servers`` without each website that shares its environment_name with an MCP server, as a MultiEnv routes an - artifact of that name to the MCP server.""" - shared = shared_names(servers) - return [s for s in servers if s.mcp or s.environment_name not in shared] +def card_servers(card: dict, address: str) -> list[BackingServer]: + """The servers ``card``, read at ``address``, lists, each with its own card: a leaf card's env itself; else each + child the composed card nests, after the env itself when its own card serves ``data/get``, as a computer's driver + fronting its apps does. A child is a website when its card carries ``website/v1`` or its url is at a website's + gateway route, ``/svc/{name}`` (an MCP server's is ``/svc/mcp-{name}``), so a website and an MCP server sharing a + name are two servers.""" + own = protocol_v1.ChildEnv(name=card.get("name"), url=card.get("url"), card=card) + if not isinstance(card.get("children_environments"), list): + return [_listed(own, address)] + servers = [_listed(own, address)] if protocol_v1.serves(card, METHOD_GET) else [] + return servers + [_listed(child, address) for child in protocol_v1.children(card)] + + +def _listed(child: protocol_v1.ChildEnv, address: str) -> BackingServer: + route, url = f"/svc/{child.name}", (child.url or "").rstrip("/") + website = child.find_extension(EXT_WEBSITE_URI) is not None or url == route or url.startswith(f"{route}/") + return BackingServer(child.name, mcp=not website, child=child, address=address) + + +async def backing_servers(deployed: Optional[DeployedEnv], gateway_url: Optional[str], env: Any, + timeout: int = 10) -> list[BackingServer]: + """The servers behind an env as its card lists them (``card_servers``), then, behind a gateway, each server its + definition ``env`` names that the card lists none of the same name and kind for: one serving no card, which the + gateway leaves off its card, so it has none. + + The card is ``deployed``'s stored one when ``gateway_url`` is its own gateway or None, else read live at + ``gateway_url`` (an override, or a record that kept no card). A record with no card URL, an older one, or no + record, or a live read that fails, leaves the definition alone: its servers, each read live at its gateway path, + logged once.""" + from agent_env.env.env import gateway_url_of + + gateway = gateway_url.rstrip("/") if gateway_url else None + card, address, why = None, None, "it has no record" + if deployed is not None and deployed.environment_card and (gateway is None or (gateway_url_of(deployed) or "").rstrip("/") == gateway): + card, address = deployed.environment_card, deployed.environment_url + elif deployed is not None and deployed.environment_card_url and gateway: + try: + card, address = await protocol_v1.get_card(gateway, timeout=timeout), gateway + except httpx.HTTPError as e: + why = f"its card at {gateway} could not be read ({type(e).__name__})" + elif deployed is not None: + why = "its record has no card URL" + named = definition_servers(env) + if not isinstance(card, dict): + logger.info(f"env {getattr(env, 'id', None)!r}: {why}, so its servers are the ones its definition names, " + f"each read live at its gateway path: {[(s.environment_name, s.mcp) for s in named]}") + return named + servers = card_servers(card, address) + if not gateway: + return servers + listed = {(s.environment_name, s.mcp) for s in servers} + return servers + [dataclasses.replace(s, address=address) for s in named if (s.environment_name, s.mcp) not in listed] + + +async def server_card(server: BackingServer, gateway_url: Optional[str], timeout: int = 10) -> tuple[Optional[str], Optional[dict]]: + """A server's card and the base its endpoints resolve against: as the env's card lists it, None for a server the + card lists none for, else read live at its gateway path (``child_env_card``).""" + if server.address is not None: + return server.address, server.card + return await child_env_card(None, gateway_url, server.environment_name, timeout=timeout, mcp=server.mcp) + + +async def server_v1_base_url(server: BackingServer, gateway_url: Optional[str]) -> Optional[str]: + """A server's v1 data-plane base: its card's ``url`` resolved against the env's address, None for a server the + env's card lists none for, else as the live probe at its gateway path finds it (``v1_base_url``).""" + if server.address is not None: + return server.child.base_url(server.address) if server.child is not None else None + return await v1_base_url(None, gateway_url, server.environment_name, mcp=server.mcp) async def v1_base_url(deployed: Optional[DeployedEnv], gateway_url: Optional[str], environment_name: str, mcp: bool = True) -> Optional[str]: diff --git a/src/agent_env/task_step/task_steps/apply_server_config.py b/src/agent_env/task_step/task_steps/apply_server_config.py index c8c8e1f08..91cc24167 100644 --- a/src/agent_env/task_step/task_steps/apply_server_config.py +++ b/src/agent_env/task_step/task_steps/apply_server_config.py @@ -9,14 +9,16 @@ Config lives on the backing server, not on the gateway itself. Each server's card comes from the env card stored on the deployed record, whose extension endpoints the gateway has already rewritten to its ``/svc/mcp-{service}`` (or a website's ``/svc/{service}``) -proxy; a record without a stored card reads it live there. The agent never sees the card or these endpoints; -only this harness step invokes them. +proxy; a record without a stored card reads it live there. A broadcast reaches each server +the env card lists, by that server's own card and route, so a website sharing its name with +an MCP server is reached too (``legacy_protocol.backing_servers``). The agent never sees the +card or these endpoints; only this harness step invokes them. """ from __future__ import annotations import logging -from typing import Any, ClassVar, Optional +from typing import TYPE_CHECKING, Any, ClassVar, Optional import httpx from agentenv_protocol import client as protocol_v1 @@ -25,6 +27,9 @@ from agent_env.entity.refs import EntityRef from agent_env.task_step.task_step import TaskStep, TaskStepDependency +if TYPE_CHECKING: + from agent_env.env.legacy_protocol import BackingServer + logger = logging.getLogger(__name__) @@ -131,51 +136,40 @@ def from_dict(cls, data: dict) -> "ApplyServerConfigStep": tolerate_unadvertised=data.get("tolerate_unadvertised", False), ) - def _expand_directives(self, deployed) -> tuple[list[ConfigDirective], set[str], set[int]]: - """Expand any broadcast directive (service == ``*``) into one directive - per MCP server and website in the env, resolved from the env definition, - and return them with the names that are websites, whose cards a live read - finds at ``/svc/{name}``, and the positions of the directives a broadcast - made for a website. Those are skipped, as under ``tolerate_unadvertised``, - when the website has no card or doesn't advertise the URI, so a website - the task didn't name never fails it; a directive naming a website, and - every MCP server, stays strict. The env's child service names aren't otherwise - enumerable here (the deployed card list carries no roster), so we read - them from ``Env.get`` — matching how deploy_env / load_artifact resolve - the env. A website sharing its name with an MCP server is left to the MCP - server, as the stored env card can't tell the two apart. Non-broadcast - directives pass through untouched.""" + async def _expand_directives(self, deployed) -> list[tuple[ConfigDirective, Optional[BackingServer], bool]]: + """Each directive with the server it targets and whether it is lenient: a broadcast (service ``*``) expanded + into one directive per MCP server and website the env card lists (``legacy_protocol.backing_servers``), each + targeting that server by its own card and route. A directive a broadcast made for a website is lenient: it is + skipped, as under ``tolerate_unadvertised``, when the website has no card or doesn't advertise the URI, so a + website the task didn't name never fails it. A named directive, and every MCP server, stays strict; a named one + targets the server of that name, the MCP server when a website shares it, as a MultiEnv's load does, or, without + a broadcast to list them, is looked up by name (None).""" if not any(d.service == self._BROADCAST for d in self.directives): - return self.directives, set(), set() + return [(d, None, False) for d in self.directives] from agent_env.env import legacy_protocol - from agent_env.env.env import Env + from agent_env.env.env import Env, gateway_url_of - env = Env.get(deployed.env_id, deployed.env_version) - # A single-server env (a bare MCPServerEnv) exposes `environment_name` and no - # `mcp_server_envs`; backing_servers falls back to it, as snapshot_env does, - # else a broadcast silently matches nothing on single-server envs. - servers = legacy_protocol.mcp_wins(legacy_protocol.backing_servers(env)) + servers = await legacy_protocol.backing_servers(deployed, gateway_url_of(deployed), Env.get(deployed.env_id, deployed.env_version), + timeout=self.timeout_seconds) # A broadcast that resolves to zero services would apply nothing while the # step reports success: the exact silent misconfig this step prevents. Fail # loud (tolerate_unadvertised governs advertisement, not missing servers). if not servers: raise ApplyServerConfigError( f"apply_server_config: broadcast directive matched no MCP servers or websites " - f"in env {self.env_id!r} (env exposes neither mcp_server_envs, website_envs nor a " - f"environment_name); nothing to apply, likely a task authoring error" + f"in env {self.env_id!r} (its card lists no child envs, and its definition names neither " + f"mcp_server_envs, website_envs nor an environment_name); nothing to apply, likely a task authoring error" ) - expanded: list[ConfigDirective] = [] - lenient: set[int] = set() + expanded: list[tuple[ConfigDirective, Optional[BackingServer], bool]] = [] for d in self.directives: if d.service != self._BROADCAST: - expanded.append(d) + named = [s for s in servers if s.environment_name == d.service] + expanded.append((d, next((s for s in named if s.mcp), named[0] if named else None), False)) continue for server in servers: - if not server.mcp: - lenient.add(len(expanded)) - expanded.append(ConfigDirective(service=server.environment_name, uri=d.uri, args=d.args)) - return expanded, {server.environment_name for server in servers if not server.mcp}, lenient + expanded.append((ConfigDirective(service=server.environment_name, uri=d.uri, args=d.args), server, not server.mcp)) + return expanded async def execute(self, context: TaskStepContext) -> TaskStepContext: deployed = next((d for d in context.deployed_envs if d.env_id == self.env_id), None) @@ -195,7 +189,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: # network), so wrap it to keep the broadcast path's failure contract the # same ApplyServerConfigError as the per-directive HTTP calls below. try: - directives, websites, lenient = self._expand_directives(deployed) + directives = await self._expand_directives(deployed) except ApplyServerConfigError: raise except Exception as e: @@ -210,16 +204,17 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: # multiple tools) against the same server. A None card marks a child env with no # card: a server predating env-card support, one whose card name differs, or one # whose card the deploy couldn't read. - cards: dict[str, tuple[str, Optional[dict]]] = {} - for position, directive in enumerate(directives): - tolerate = self.tolerate_unadvertised or position in lenient + cards: dict[Any, tuple[Optional[str], Optional[dict]]] = {} + for directive, server, lenient in directives: + tolerate = self.tolerate_unadvertised or lenient + key = directive.service if server is None else server try: - if directive.service not in cards: - cards[directive.service] = await legacy_protocol.child_env_card( - deployed, gateway_url_of(deployed), directive.service, timeout=self.timeout_seconds, - mcp=directive.service not in websites, - ) - base_url, card = cards[directive.service] + if key not in cards: + cards[key] = (await legacy_protocol.child_env_card(deployed, gateway_url_of(deployed), directive.service, + timeout=self.timeout_seconds) + if server is None else + await legacy_protocol.server_card(server, gateway_url_of(deployed), timeout=self.timeout_seconds)) + base_url, card = cards[key] if card is None: if not tolerate: raise ApplyServerConfigError( diff --git a/src/agent_env/task_step/task_steps/snapshot_env.py b/src/agent_env/task_step/task_steps/snapshot_env.py index 9415dac98..abb5573f3 100644 --- a/src/agent_env/task_step/task_steps/snapshot_env.py +++ b/src/agent_env/task_step/task_steps/snapshot_env.py @@ -475,36 +475,43 @@ def _resolve_loaded_universe_ref(deployed) -> Optional[tuple[str, Optional[int]] return None return (su["id"], su.get("version")) if su and su.get("id") else None - @classmethod - def _enumerate_environments(cls, env) -> list[str]: - """environment names for a multi env, a website env or a single MCP server env.""" - return [server.environment_name for server in cls._enumerate_servers(env)] - @staticmethod - def _enumerate_servers(env) -> list[BackingServer]: - """The servers to snapshot: the env's MCP servers and websites, a website sharing its name with an MCP - server left to it.""" - from agent_env.env import legacy_protocol - - servers = legacy_protocol.mcp_wins(legacy_protocol.backing_servers(env)) + def _snapshot_servers(servers: list[BackingServer], env) -> list[BackingServer]: + """The servers to snapshot of ``servers`` (``legacy_protocol.backing_servers``), one per environment_name: a + universe names each environment by it, and a load hands an environment of a name an MCP server and a website + share to the MCP server, so a server whose name one kept earlier holds (the env itself, else an MCP server) is + left out, and logged.""" if not servers: raise RuntimeError( f"Env '{env.id}' v{env.version} has no MCP services or websites to snapshot " - "(expected mcp_server_envs, website_envs or environment_name)" + "(its card lists no child envs, and its definition names no mcp_server_envs, website_envs or environment_name)" ) - return servers + if not all(server.environment_name for server in servers): + raise RuntimeError(f"Env '{env.id}' v{env.version} has a card naming no environment to snapshot") + kept: dict[str, BackingServer] = {} + for server in sorted(servers, key=lambda s: not s.mcp): + if (holder := kept.get(server.environment_name)) is not None: + logger.warning(f"snapshot_env: {'website' if not server.mcp else 'server'} {server.environment_name!r} " + f"of {env.id} is not snapshotted: it shares its name with the {'MCP server' if holder.mcp else 'website'} " + "kept, and a universe names an environment by its name") + continue + kept[server.environment_name] = server + return list(kept.values()) @staticmethod async def _export_environment_to_file( gateway_url: Optional[str], environment_name: str, tmp_path: str, timeout_seconds: float, deployed=None, - data_get_params: Optional[dict] = None, mcp: bool = True, + data_get_params: Optional[dict] = None, mcp: bool = True, server: Optional[BackingServer] = None, ) -> Optional[str | tuple[GetDataResponse, Optional[_SnapshotUpload]]]: """Export one service, an MCP server or with ``mcp=False`` a website; return the suffix written (".json"/".zip") to ``tmp_path``, or the object url a FilePart names when the service uploaded the bundle itself (nothing written to ``tmp_path`` then — export_one registers it), or, for an environment whose card declares a file tree, its data/get answer and the upload it answered under, which export_one keeps file by file, - or None for a website serving no v1 data plane. ``data_get_params`` go to its data/get. + or None for a service serving no v1 data plane: a website with no card, or a service whose card declares + operations without data/get. ``data_get_params`` go to its data/get. ``server`` is the service as the env's + card lists it (``legacy_protocol.backing_servers``), its card and route; without it, the service is looked up + by name on ``deployed``'s stored card, or probed live at its gateway path. v1 ``get_data``, handed a namespace grant when the service takes the data-objects form: an upload through it → its object url; ``DataPart`` → json; ``FilePart`` → @@ -513,12 +520,22 @@ async def _export_environment_to_file( guard. Errors are caught by the caller (fails just this service).""" from agent_env.env import legacy_protocol - base_url = await legacy_protocol.v1_base_url(deployed, gateway_url, environment_name, mcp=mcp) + if server is not None: + mcp = server.mcp + base_url = await legacy_protocol.server_v1_base_url(server, gateway_url) + else: + base_url = await legacy_protocol.v1_base_url(deployed, gateway_url, environment_name, mcp=mcp) + card_base, card = None, None + if base_url is not None: + card_base, card = (await legacy_protocol.server_card(server, gateway_url) if server is not None else + await legacy_protocol.child_env_card(deployed, gateway_url, environment_name, mcp=mcp)) + if card is not None and protocol_v1.serves(card, METHOD_GET) is False: + logger.info(f"snapshot_env: {environment_name}'s card declares {protocol_v1.operations(card)}, no {METHOD_GET}") + return None if base_url is None and not mcp: return None if base_url is not None: headers = deployed.control_headers_for(base_url) if deployed is not None else None - card_base, card = await legacy_protocol.child_env_card(deployed, gateway_url, environment_name, mcp=mcp) if returns_file_tree(card or {}): upload = await asyncio.to_thread(_snapshot_upload, card or {}, timeout_seconds, getattr(deployed, "sandbox_type", None), FILE_TREE_LIMITS) @@ -625,13 +642,12 @@ async def snapshot_env_state( A classmethod so callers that have no step instance can reuse it.""" from agent_env.artifact import FileArtifact, EnvironmentArtifact, EnvironmentUniverseArtifact + from agent_env.env import legacy_protocol from agent_env.env.env import gateway_url_of - from agent_env.env.legacy_protocol import BackingServer - - servers = cls._enumerate_servers(env) if gateway_url else [BackingServer(name) for name in _card_environments(deployed)] - services = [server.environment_name for server in servers] gateway = gateway_url.rstrip("/") if gateway_url else None + servers = cls._snapshot_servers(await legacy_protocol.backing_servers(deployed, gateway, env), env) + services = [server.environment_name for server in servers] # `gateway` identifies which deployment was exported: env= alone doesn't when one env_id has several. logger.info( f"snapshot_env: env={env.id} v{env.version} snapshot_id={snapshot_id} " @@ -665,7 +681,7 @@ async def snapshot_env_state( no_data_plane: set[str] = set() async def export_one(server: BackingServer): - """Returns EnvironmentArtifact on success, (public_err, log_err) on failure, None for a website serving no data plane.""" + """Returns EnvironmentArtifact on success, (public_err, log_err) on failure, None for a service serving no data plane.""" environment_name = server.environment_name website = {} if server.mcp else {"mcp": False} async with sem: @@ -678,7 +694,8 @@ async def export_one(server: BackingServer): try: result = await cls._export_environment_to_file( gateway, environment_name, tmp_path, export_timeout_seconds, - deployed=card_record, data_get_params=(data_get_params or {}).get(environment_name), **website, + deployed=card_record, data_get_params=(data_get_params or {}).get(environment_name), + server=server, **website, ) except httpx.HTTPStatusError as e: body_snippet = "" @@ -795,12 +812,12 @@ def _register_uploaded_bundle(): without_data_plane = [name for name in services if name in no_data_plane] if without_data_plane: logger.warning( - f"snapshot_env: websites {without_data_plane} of {env.id} serve no v1 data plane; " + f"snapshot_env: {without_data_plane} of {env.id} serve no v1 data plane; " "their state is not in the snapshot" ) empty = [name for name, result in zip(services, results) if result is None and name not in no_data_plane] if not environment_artifacts: - reasons = [f"its websites {without_data_plane} serve no v1 data plane"] if without_data_plane else [] + reasons = [f"{without_data_plane} serve no v1 data plane"] if without_data_plane else [] reasons += [f"{', '.join(map(repr, empty))} returned no files to snapshot"] if empty else [] raise RuntimeError(f"Env '{env.id}' v{env.version} exported no state: {'; '.join(reasons)}") if empty: @@ -883,25 +900,6 @@ def registered_upload(object_url: str, file_id: str, filename: str, content_type id=file_id, description=description, filename=filename, content_type=content_type, object_url=object_url)) -def _card_environments(deployed) -> list[str]: - """The environments a deployment without a gateway serves, as the card it recorded names them: the env itself when - it has no children; else each child its name reaches that exports, and the env too, first, when it exports beside - them, as a computer fronting its apps does.""" - card = deployed.environment_card or {} - children = card.get("children_environments") - if not isinstance(children, list): - names = [card.get("name")] - else: - names = [child.get("name") for child in children if deployed.get_child_env_card(child.get("name")) is child - and ((operations := protocol_v1.operations(child)) is None - or METHOD_GET in operations)] - if METHOD_GET in (protocol_v1.operations(card) or []): - names.insert(0, card.get("name")) - if not all(names): - raise RuntimeError(f"Deployed env '{deployed.env_id}' recorded a card naming no environment to snapshot") - return names - - def _changes_route(deployed, override_url: Optional[str]) -> tuple[str, str]: """The verb and URL of the env's changes: an override keeps `{override}/changes`, else the stored card names them.""" if override_url: diff --git a/src/agent_env/task_step/task_steps/sync_env_clock.py b/src/agent_env/task_step/task_steps/sync_env_clock.py index c80cef8a8..8ae463f56 100644 --- a/src/agent_env/task_step/task_steps/sync_env_clock.py +++ b/src/agent_env/task_step/task_steps/sync_env_clock.py @@ -1,9 +1,10 @@ """Arm a deployed env gateway's virtual clock, then sync its backing servers to it. Arms the clock (set_time) and reads the gateway's server-reachable env_get_time_url (state), both as the -stored env card declares clock/v1, then invokes sync_time on each backing server, MCP server or website, -whose child env card advertises it. Non-advertisers are skipped when tolerate_missing_sync_time (default -True), and websites always. Place right before prompt_agent so the virtual timeline starts at agent-start. +stored env card declares clock/v1, then invokes sync_time on each backing server, MCP server or website, the +env card lists (``legacy_protocol.backing_servers``) whose own card advertises it, at the route that card gives. +Non-advertisers are skipped when tolerate_missing_sync_time (default True), and websites always. Place right +before prompt_agent so the virtual timeline starts at agent-start. """ from __future__ import annotations @@ -110,15 +111,15 @@ def from_dict(cls, data: dict) -> SyncEnvClockTaskStep: timezone=data.get("timezone"), ) - def _backing_servers(self, deployed) -> list[BackingServer]: - """The env's backing MCP servers and websites (``legacy_protocol.backing_servers``). A website sharing its - name with an MCP server has its card read live, since the stored env card can't tell the two apart.""" + async def _backing_servers(self, deployed) -> list[BackingServer]: + """The env's backing MCP servers and websites, as its card lists them (``legacy_protocol.backing_servers``): + a website sharing its name with an MCP server is a server of its own, at its own route.""" from agent_env.env import legacy_protocol - from agent_env.env.env import Env - return legacy_protocol.backing_servers(Env.get(deployed.env_id, deployed.env_version)) + from agent_env.env.env import Env, gateway_url_of + return await legacy_protocol.backing_servers(deployed, gateway_url_of(deployed), Env.get(deployed.env_id, deployed.env_version), + timeout=self.timeout_seconds) async def execute(self, context: TaskStepContext) -> TaskStepContext: - from agent_env.env import legacy_protocol from agent_env.env.env import gateway_url_of deployed = next((d for d in context.deployed_envs if d.env_id == self.env_id), None) @@ -133,15 +134,11 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: state = await deployed.invoke(EXT_CLOCK_URI, "state", timeout=self.timeout_seconds) env_get_time_url = state.get("env_get_time_url") - servers = self._backing_servers(deployed) - shared = legacy_protocol.shared_names(servers) synced, skipped = [], [] - for server in servers: + for server in await self._backing_servers(deployed): service = server.environment_name - record = None if service in shared and not server.mcp else deployed - outcome = await self._sync_one(record, gateway_url_of(deployed), service, env_get_time_url, mcp=server.mcp, - tolerate=self.tolerate_missing_sync_time or not server.mcp, - headers=deployed.control_headers) + outcome = await self._sync_one(deployed, gateway_url_of(deployed), service, env_get_time_url, mcp=server.mcp, + tolerate=self.tolerate_missing_sync_time or not server.mcp, server=server) entry = {"service": service, "environment": service, **({} if server.mcp else {"website": True}), **outcome} (synced if outcome.get("synced") else skipped).append(entry) @@ -161,17 +158,20 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: return context async def _sync_one(self, deployed, gateway_url: str, service: str, env_get_time_url: Optional[str], mcp: bool = True, - tolerate: Optional[bool] = None, headers: Optional[dict] = None) -> dict: + tolerate: Optional[bool] = None, server: Optional[BackingServer] = None) -> dict: """Invoke sync_time on one server, an MCP server or with ``mcp=False`` a website, if its card advertises clock/v1; skip a server with no card or no clock/v1 when ``tolerate`` (tolerate_missing_sync_time by default), else raise. ``execute`` tolerates every website, so one that doesn't follow the clock never fails the step; one - that follows it but can't take the task's timezone does, as an MCP server does. ``headers`` are the env's control - headers, ``deployed``'s by default; ``execute`` passes them because it hands a website that shares its name with - an MCP server no record, so its card is read live.""" + that follows it but can't take the task's timezone does, as an MCP server does. ``server`` is the server as the + env's card lists it (``legacy_protocol.server_card``); without it, the card is ``service``'s on ``deployed``'s + stored card, or read live at its gateway path.""" tolerate = self.tolerate_missing_sync_time if tolerate is None else tolerate from agent_env.env import legacy_protocol - base_url, card = await legacy_protocol.child_env_card(deployed, gateway_url, service, timeout=self.timeout_seconds, mcp=mcp) + if server is not None: + base_url, card = await legacy_protocol.server_card(server, gateway_url, timeout=self.timeout_seconds) + else: + base_url, card = await legacy_protocol.child_env_card(deployed, gateway_url, service, timeout=self.timeout_seconds, mcp=mcp) if card is None: if tolerate: return {"synced": False, "reason": "no_env_card"} @@ -193,5 +193,5 @@ async def _sync_one(self, deployed, gateway_url: str, service: str, env_get_time result = await protocol_v1.invoke_extension(base_url, card, EXT_CLOCK_URI, params=params, timeout=self.timeout_seconds, - headers=deployed.control_headers if headers is None else headers) + headers=deployed.control_headers) return {"synced": True, "result": result} diff --git a/tst/unit/env/test_legacy_protocol.py b/tst/unit/env/test_legacy_protocol.py index 7d107ad34..ef8fee3f8 100644 --- a/tst/unit/env/test_legacy_protocol.py +++ b/tst/unit/env/test_legacy_protocol.py @@ -179,7 +179,7 @@ async def test_a_v1_services_state_is_the_data_it_answers_with_else_its_export_s assert ("GET /svc/mcp-slack/export-state" in asked) is reads_export_state -def test_backing_servers_are_an_envs_mcp_servers_then_its_websites(): +def test_definition_servers_are_an_envs_mcp_servers_then_its_websites(): from types import SimpleNamespace from agent_env.env.envs.website import WebsiteEnv @@ -188,11 +188,93 @@ def test_backing_servers_are_an_envs_mcp_servers_then_its_websites(): named = lambda *names: [SimpleNamespace(environment_name=n) for n in names] multi = SimpleNamespace(mcp_server_envs=named("slack", "email"), website_envs=named("shop", "slack")) - assert legacy_protocol.backing_servers(multi) == [BackingServer("slack"), BackingServer("email"), - BackingServer("shop", mcp=False), BackingServer("slack", mcp=False)] - assert legacy_protocol.shared_names(legacy_protocol.backing_servers(multi)) == {"slack"} - assert legacy_protocol.mcp_wins(legacy_protocol.backing_servers(multi)) == [ - BackingServer("slack"), BackingServer("email"), BackingServer("shop", mcp=False)] - assert legacy_protocol.backing_servers(WebsiteEnv(id="w", version=1, environment_name="shop")) == [BackingServer("shop", mcp=False)] - assert legacy_protocol.backing_servers(SimpleNamespace(environment_name="notes")) == [BackingServer("notes")] - assert legacy_protocol.backing_servers(SimpleNamespace()) == [] + assert legacy_protocol.definition_servers(multi) == [BackingServer("slack"), BackingServer("email"), + BackingServer("shop", mcp=False), BackingServer("slack", mcp=False)] + assert legacy_protocol.definition_servers(WebsiteEnv(id="w", version=1, environment_name="shop")) == [BackingServer("shop", mcp=False)] + assert legacy_protocol.definition_servers(SimpleNamespace(environment_name="notes")) == [BackingServer("notes")] + assert legacy_protocol.definition_servers(SimpleNamespace()) == [] + + +_COMPOSED = {"name": "gw", "url": RPC_PATH, "capabilities": {"extensions": []}, "children_environments": [ + {"name": "slack", "url": "/svc/mcp-slack/agentenv", "capabilities": {"extensions": []}}, + {"name": "slack", "url": "/svc/slack/agentenv", "capabilities": {"extensions": []}}, + {"name": "docs", "url": "/svc/mcp-docs/agentenv", "capabilities": {"extensions": [{"uri": "urn:agentenv:website/v1"}]}}, +]} + + +def test_card_servers_are_the_cards_children_a_website_told_apart_by_its_route_or_its_website_extension(): + servers = legacy_protocol.card_servers(_COMPOSED, "https://env.example") + + assert [(s.environment_name, s.mcp, s.address) for s in servers] == [ + ("slack", True, "https://env.example"), ("slack", False, "https://env.example"), ("docs", False, "https://env.example")] + assert [s.child.base_url(s.address) for s in servers] == [ + "https://env.example/svc/mcp-slack", "https://env.example/svc/slack", "https://env.example/svc/mcp-docs"] + + +def test_card_servers_of_a_leaf_card_is_the_env_itself_and_a_composed_card_serving_data_get_comes_first(): + leaf = {"name": "notes", "url": RPC_PATH} + assert [(s.environment_name, s.card) for s in legacy_protocol.card_servers(leaf, "https://n")] == [("notes", leaf)] + driver = {**_COMPOSED, "name": "desktop", "capabilities": {"operations": ["data/get"]}} + assert [s.environment_name for s in legacy_protocol.card_servers(driver, "https://d")] == ["desktop", "slack", "slack", "docs"] + + +def _record(card: dict | None, card_url: str | None = "https://env.example/.well-known/agent-env.json"): + from agent_env.env.env import DeployedGatewayEnv + + return DeployedGatewayEnv(env_id="e", env_version=1, gateway_url="http://gw", mcp_url="http://gw/mcp", db_web_url=None, + sandbox_id="sb", environment_card_url=card_url, environment_card=card) + + +@pytest.mark.asyncio +async def test_backing_servers_are_the_stored_cards_then_those_the_definition_names_serving_no_card(): + from types import SimpleNamespace + + env = SimpleNamespace(mcp_server_envs=[SimpleNamespace(environment_name=n) for n in ("slack", "legacy")], + website_envs=[SimpleNamespace(environment_name="slack")]) + + servers = await legacy_protocol.backing_servers(_record(_COMPOSED), "http://gw", env) + + assert [(s.environment_name, s.mcp, s.card is not None, s.address) for s in servers] == [ + ("slack", True, True, "https://env.example"), ("slack", False, True, "https://env.example"), + ("docs", False, True, "https://env.example"), ("legacy", True, False, "https://env.example")] + assert await legacy_protocol.server_card(servers[-1], "http://gw") == ("https://env.example", None) + assert await legacy_protocol.server_v1_base_url(servers[-1], "http://gw") is None + + +@pytest.mark.asyncio +async def test_backing_servers_read_the_card_live_without_a_stored_one_or_for_another_gateway(monkeypatch): + from types import SimpleNamespace + + read = [] + + async def get_card(base_url, **kw): + read.append(base_url) + return _COMPOSED + + monkeypatch.setattr(legacy_protocol.protocol_v1, "get_card", get_card) + env = SimpleNamespace(environment_name="slack") + for record, gateway in ((_record(None), "http://gw/"), (_record(_COMPOSED), "http://elsewhere")): + servers = await legacy_protocol.backing_servers(record, gateway, env) + assert [(s.environment_name, s.address) for s in servers] == [ + ("slack", gateway.rstrip("/")), ("slack", gateway.rstrip("/")), ("docs", gateway.rstrip("/"))] + assert read == ["http://gw", "http://elsewhere"] + + +@pytest.mark.asyncio +async def test_a_record_whose_card_cant_be_read_falls_back_to_its_definition_logged_once(monkeypatch, caplog): + from types import SimpleNamespace + + async def get_card(base_url, **kw): + raise httpx.ConnectError("refused") + + monkeypatch.setattr(legacy_protocol.protocol_v1, "get_card", get_card) + env = SimpleNamespace(id="e", mcp_server_envs=[SimpleNamespace(environment_name="slack")], + website_envs=[SimpleNamespace(environment_name="slack")]) + with caplog.at_level("INFO", logger="agent_env.env.legacy_protocol"): + for record in (_record(None, card_url=None), _record(None), None): + servers = await legacy_protocol.backing_servers(record, "http://gw", env) + assert [(s.environment_name, s.mcp, s.address) for s in servers] == [("slack", True, None), ("slack", False, None)] + + assert [r.getMessage().split(",")[0] for r in caplog.records] == [ + "env 'e': its record has no card URL", "env 'e': its card at http://gw could not be read (ConnectError)", + "env 'e': it has no record"] diff --git a/tst/unit/task/local_run_write_fence_test.py b/tst/unit/task/local_run_write_fence_test.py index d3f60293e..108709d94 100644 --- a/tst/unit/task/local_run_write_fence_test.py +++ b/tst/unit/task/local_run_write_fence_test.py @@ -264,7 +264,7 @@ def __init__(self, id, version=None): self.mcp_server_envs = [SimpleNamespace(environment_name="gmail")] -async def _export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None): +async def _export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None, **_): Path(tmp_path).write_text(json.dumps({"service": name})) return ".json" diff --git a/tst/unit/task_step/sync_env_clock_test.py b/tst/unit/task_step/sync_env_clock_test.py index 5b83c51d7..a9a109e2a 100644 --- a/tst/unit/task_step/sync_env_clock_test.py +++ b/tst/unit/task_step/sync_env_clock_test.py @@ -1,11 +1,11 @@ """Unit tests for SyncEnvClockTaskStep — round-trip + the per-server sync_time fan-out (mocked).""" import json +from types import SimpleNamespace import pytest from agent_env.env.env import DeployedEnv, DeployedGatewayEnv, EnvCapabilityUnsupported from agent_env.env.gateway.constants import EXT_CLOCK_URI -from agent_env.env.legacy_protocol import BackingServer from agent_env.task_step.context import TaskStepContext from agent_env.task_step.task_steps import sync_env_clock as mod from agent_env.task_step.task_steps.sync_env_clock import SyncEnvClockTaskStep, resolve_virtual_time @@ -95,7 +95,7 @@ async def test_sync_one_skips_or_raises_for_a_child_env_missing_from_the_stored_ @pytest.mark.asyncio async def test_execute_syncs_each_child_env_through_the_stored_card(monkeypatch): sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) - monkeypatch.setattr(SyncEnvClockTaskStep, "_backing_servers", lambda self, deployed: [BackingServer("email")]) + monkeypatch.setattr("agent_env.env.env.Env.get", lambda *a, **k: SimpleNamespace(environment_name="email")) context = TaskStepContext(deployed_envs=[_carded(await _composed_card({"mcp-email": [_CLOCK_EXT]}))]) await _step().execute(context) @@ -176,21 +176,64 @@ async def fake_card(base_url, **kw): @pytest.mark.asyncio -async def test_execute_reads_a_website_sharing_an_mcp_servers_name_live(monkeypatch): - from types import SimpleNamespace +async def test_execute_syncs_a_website_sharing_an_mcp_servers_name_at_its_own_route(monkeypatch): + """The card lists the two as children at distinct urls, so each is synced by its own card, with no card read.""" from unittest.mock import patch sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) env = SimpleNamespace(mcp_server_envs=[SimpleNamespace(environment_name="email")], website_envs=[SimpleNamespace(environment_name="email")]) - context = TaskStepContext(deployed_envs=[_carded(await _composed_card({"mcp-email": [_CLOCK_EXT]}))]) + context = TaskStepContext(deployed_envs=[_carded(await _composed_card({"mcp-email": [_CLOCK_EXT], "email": [_CLOCK_EXT]}))]) with patch("agent_env.env.env.Env.get", return_value=env): await _step().execute(context) assert [(r.method, str(r.url)) for r in sent][2:] == [ - ("POST", f"{_CARD}/svc/mcp-email/agentenv/ext/sync_time"), ("GET", "http://gw/svc/email/.well-known/agent-env.json"), + ("POST", f"{_CARD}/svc/mcp-email/agentenv/ext/sync_time"), ("POST", f"{_CARD}/svc/email/agentenv/ext/sync_time"), ] + record = context.metadata["clock_configurations"][0] + assert [(s["service"], s.get("website")) for s in record["synced"]] == [("email", None), ("email", True)] + + +@pytest.mark.asyncio +async def test_execute_without_a_card_url_syncs_the_servers_the_definition_names_read_live(monkeypatch): + """An older record kept no card URL, so its servers come from its env definition and each card is read live at + its gateway path.""" + from unittest.mock import patch + + import httpx + + sent, real = [], httpx.AsyncClient + cards = {"/svc/mcp-email/.well-known/agent-env.json": {"name": "email", "capabilities": {"extensions": [_CLOCK_EXT]}}, + "/svc/shop/.well-known/agent-env.json": {"name": "shop", "capabilities": {"extensions": []}}} + + def handle(request: httpx.Request) -> httpx.Response: + sent.append(request) + if request.url.path == "/clock/state": + return httpx.Response(200, json={"env_get_time_url": CLOCK_URL}) + return httpx.Response(200, json=cards.get(request.url.path, {})) + + monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **k: real(transport=httpx.MockTransport(handle))) + record = _cardless() + + async def invoke(uri, method, params=None, **kw): + return {"env_get_time_url": CLOCK_URL} if method == "state" else {} + + record.invoke = invoke + env = SimpleNamespace(mcp_server_envs=[SimpleNamespace(environment_name="email")], + website_envs=[SimpleNamespace(environment_name="shop")]) + context = TaskStepContext(deployed_envs=[record]) + + with patch("agent_env.env.env.Env.get", return_value=env): + await _step().execute(context) + + assert [(r.method, str(r.url)) for r in sent] == [ + ("GET", "http://gw/svc/mcp-email/.well-known/agent-env.json"), ("POST", "http://gw/svc/mcp-email/agentenv/ext/sync_time"), + ("GET", "http://gw/svc/shop/.well-known/agent-env.json"), + ] + cfg = context.metadata["clock_configurations"][0] + assert [s["service"] for s in cfg["synced"]] == ["email"] + assert [(s["service"], s["reason"]) for s in cfg["skipped"]] == [("shop", "not_advertised")] @pytest.mark.asyncio @@ -317,7 +360,7 @@ def test_without_a_zone_virtual_time_is_passed_through_untouched(): @pytest.mark.asyncio async def test_execute_arms_the_gateway_at_the_local_instant_and_tells_every_server_the_zone(monkeypatch): sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) - monkeypatch.setattr(SyncEnvClockTaskStep, "_backing_servers", lambda self, deployed: [BackingServer("email")]) + monkeypatch.setattr("agent_env.env.env.Env.get", lambda *a, **k: SimpleNamespace(environment_name="email")) context = TaskStepContext(deployed_envs=[_carded(await _composed_card({"mcp-email": [_ZONE_EXT]}))]) await _zoned().execute(context) @@ -332,7 +375,7 @@ async def test_execute_arms_the_gateway_at_the_local_instant_and_tells_every_ser @pytest.mark.parametrize("tolerate", [True, False]) async def test_a_server_that_cannot_take_the_zone_fails_the_step_before_any_sync_is_sent(monkeypatch, tolerate): sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) - monkeypatch.setattr(SyncEnvClockTaskStep, "_backing_servers", lambda self, deployed: [BackingServer("email")]) + monkeypatch.setattr("agent_env.env.env.Env.get", lambda *a, **k: SimpleNamespace(environment_name="email")) context = TaskStepContext(deployed_envs=[_carded(await _composed_card({"mcp-email": [_OLD_EXT]}))]) with pytest.raises(RuntimeError, match="takes no timezone"): @@ -378,34 +421,19 @@ async def test_a_zoned_step_tells_a_website_that_takes_the_zone_and_fails_on_one @pytest.mark.asyncio async def test_a_website_sharing_an_mcp_servers_name_is_synced_with_the_envs_control_headers(monkeypatch): - """Its card is read live, with no record, and its sync_time still carries the env's control token.""" import dataclasses - from types import SimpleNamespace from unittest.mock import patch - import httpx - - sent, real = [], httpx.AsyncClient - website_card = {"name": "email", "url": "/agentenv", "capabilities": {"extensions": [_CLOCK_EXT]}} - - def handle(request: httpx.Request) -> httpx.Response: - sent.append(request) - if request.url.path.endswith("/clock/state"): - return httpx.Response(200, json={"env_get_time_url": CLOCK_URL}) - if request.url.path == "/svc/email/.well-known/agent-env.json": - return httpx.Response(200, json=website_card) - return httpx.Response(200, json={}) - - monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **k: real(transport=httpx.MockTransport(handle))) + sent = _mock_http(monkeypatch, state={"env_get_time_url": CLOCK_URL}) env = SimpleNamespace(mcp_server_envs=[SimpleNamespace(environment_name="email")], website_envs=[SimpleNamespace(environment_name="email")]) - record = dataclasses.replace(_carded(await _composed_card({"mcp-email": [_CLOCK_EXT]})), control_token="ctl") + record = dataclasses.replace(_carded(await _composed_card({"mcp-email": [_CLOCK_EXT], "email": [_CLOCK_EXT]})), control_token="ctl") context = TaskStepContext(deployed_envs=[record]) with patch("agent_env.env.env.Env.get", return_value=env): await _step().execute(context) - website_sync = [r for r in sent if str(r.url) == "http://gw/svc/email/agentenv/ext/sync_time"] + website_sync = [r for r in sent if str(r.url) == f"{_CARD}/svc/email/agentenv/ext/sync_time"] assert len(website_sync) == 1 assert {k: website_sync[0].headers.get(k) for k in record.control_headers} == record.control_headers assert [s["service"] for s in context.metadata["clock_configurations"][0]["synced"]] == ["email", "email"] diff --git a/tst/unit/task_step/test_apply_server_config.py b/tst/unit/task_step/test_apply_server_config.py index ddeb96806..94ed35986 100644 --- a/tst/unit/task_step/test_apply_server_config.py +++ b/tst/unit/task_step/test_apply_server_config.py @@ -609,18 +609,38 @@ async def get(self, url, *, timeout=None, params=None): @pytest.mark.asyncio -async def test_stored_card_broadcast_reaches_websites_and_leaves_a_shared_name_to_the_mcp_server(): +async def test_stored_card_broadcast_reaches_websites_and_a_website_sharing_an_mcp_servers_name(): + """The card lists a website and an MCP server sharing a name as two children at distinct urls, so a broadcast + reaches both, each at its own route; a directive naming the name means the MCP server.""" gw = _MockGateway({}) ext = _ext(_SET_ACTING_USER_URI, "/agentenv/ext/set_acting_user") - ctx = _carded_context(await _composed_card({"mcp-slack": [ext], "shop": [ext]})) - step = _step(directives=[{"service": "*", "uri": _SET_ACTING_USER_URI, "args": {"user_email": "a@x.com"}}]) + ctx = _carded_context(await _composed_card({"mcp-slack": [ext], "shop": [ext], "slack": [ext]})) + step = _step(directives=[{"service": "*", "uri": _SET_ACTING_USER_URI, "args": {"user_email": "a@x.com"}}, + {"service": "slack", "uri": _SET_ACTING_USER_URI, "args": {"user_email": "b@x.com"}}]) with _patch_httpx(gw), patch("agent_env.env.env.Env.get", return_value=_fake_multi_env(["slack"], ["shop", "slack"])): out = await step.execute(ctx) + assert gw.get_calls == [] + assert gw.post_calls == [(f"{_CARD}/svc/mcp-slack/agentenv/ext/set_acting_user", {"user_email": "a@x.com"}), + (f"{_CARD}/svc/shop/agentenv/ext/set_acting_user", {"user_email": "a@x.com"}), + (f"{_CARD}/svc/slack/agentenv/ext/set_acting_user", {"user_email": "a@x.com"}), + (f"{_CARD}/svc/mcp-slack/agentenv/ext/set_acting_user", {"user_email": "b@x.com"})] + assert [c["service"] for c in out.metadata["server_config_changes"]] == ["slack", "shop", "slack", "slack"] + + +@pytest.mark.asyncio +async def test_stored_card_broadcast_to_an_mcp_only_env_is_unchanged(): + gw = _MockGateway({}) + ext = _ext(_SET_ACTING_USER_URI, "/agentenv/ext/set_acting_user") + ctx = _carded_context(await _composed_card({"mcp-slack": [ext], "mcp-email": [ext]})) + step = _step(directives=[{"service": "*", "uri": _SET_ACTING_USER_URI, "args": {"user_email": "a@x.com"}}]) + with _patch_httpx(gw), patch("agent_env.env.env.Env.get", return_value=_fake_env("slack", "email")): + out = await step.execute(ctx) + assert gw.get_calls == [] assert [url for url, _ in gw.post_calls] == [f"{_CARD}/svc/mcp-slack/agentenv/ext/set_acting_user", - f"{_CARD}/svc/shop/agentenv/ext/set_acting_user"] - assert [c["service"] for c in out.metadata["server_config_changes"]] == ["slack", "shop"] + f"{_CARD}/svc/mcp-email/agentenv/ext/set_acting_user"] + assert out.metadata["server_config_skipped"] == [] @pytest.mark.asyncio diff --git a/tst/unit/task_step/test_snapshot_env.py b/tst/unit/task_step/test_snapshot_env.py index 380690fc2..cf75fb7ca 100644 --- a/tst/unit/task_step/test_snapshot_env.py +++ b/tst/unit/task_step/test_snapshot_env.py @@ -14,6 +14,7 @@ from agent_env.artifact import EnvironmentUniverseArtifact from agent_env.artifact.store import reset_artifact_store from agent_env.config import set_object_store +from agent_env.env import legacy_protocol from agent_env.env.env import DeployedEnv, DeployedGatewayEnv, DeployedSandboxEnv from agent_env.env.gateway.constants import EXT_STATE_URI, GATEWAY_EXTENSIONS, WELL_KNOWN_PATH from agent_env.task_step.context import TaskStepContext @@ -33,7 +34,7 @@ def _deployed(env_id="env-1", instance_id="env-1-abc123", gateway="https://gw-1" sandbox_id="sb-1", instance_id=instance_id, environment_card_url=f"{gateway}{WELL_KNOWN_PATH}", - environment_card={"capabilities": {"extensions": GATEWAY_EXTENSIONS}}, + environment_card={"capabilities": {"extensions": GATEWAY_EXTENSIONS}, "children_environments": []}, ) @@ -565,7 +566,7 @@ def __init__(self, names): @pytest.mark.asyncio async def test_core_all_or_nothing_on_partial_failure(monkeypatch): - async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None): + async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None, **_): if name == "slack": raise ValueError("export-state returned non-JSON body (starts with b'<')") with open(tmp_path, "w") as f: @@ -609,7 +610,7 @@ def put(id, *, environment_name, file_artifact): @pytest.mark.asyncio async def test_core_success_creates_universe_with_metadata(monkeypatch): - async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None): + async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None, **_): with open(tmp_path, "w") as f: json.dump({"ok": name}, f) return ".json" @@ -673,7 +674,7 @@ async def test_core_zip_export_creates_zip_filename_artifact(monkeypatch): """A FilePart(zip) export flows through as a .zip-suffixed FileArtifact path, so FileArtifact.filename (= basename) ends in .zip — the load side routes a .zip file:// artifact to reset_data() for a lossless round-trip.""" - async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None): + async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None, **_): # Simulate the lossless-zip case: write zip bytes and report ".zip". with open(tmp_path, "wb") as f: f.write(b"PK\x03\x04zip-bytes") @@ -911,7 +912,7 @@ async def test_core_registers_a_service_upload_in_place_and_it_loads_back(local_ bundle_file.write_bytes(b"PK\x03\x04bundle") uploaded = store.put_file_at(upload_url, str(bundle_file)) - async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None): + async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None, **_): return uploaded monkeypatch.setattr(SnapshotEnvTaskStep, "_export_environment_to_file", staticmethod(fake_export)) @@ -933,7 +934,7 @@ async def test_core_fails_a_service_whose_upload_the_store_cannot_find(local_sto set_object_store(FakeObjectStore()) reset_artifact_store() - async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None): + async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None, **_): return "fake://shared/snapshots/missing.zip" monkeypatch.setattr(SnapshotEnvTaskStep, "_export_environment_to_file", staticmethod(fake_export)) @@ -950,7 +951,7 @@ async def test_core_snapshots_under_an_id_longer_than_a_filename_can_be(local_st local store may be the whole id.""" snapshot_id = f"task-{'x' * 250}__snapshot-snap-1" - async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None): + async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None, **_): with open(tmp_path, "w") as f: json.dump({"service": name}, f) return ".json" @@ -1044,14 +1045,16 @@ async def test_export_of_a_child_env_missing_from_the_card_streams_legacy_export @pytest.mark.asyncio async def test_snapshot_uses_the_stored_card_only_for_the_deployments_own_gateway(monkeypatch): + """An override's gateway has its card read live, and the record's stored card describes its own gateway only.""" seen = [] - async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None): - seen.append((gateway_url, deployed)) + async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None, server=None, **_): + seen.append((gateway_url, deployed, server.address, server.card and server.card["url"])) raise ValueError("stop after the export call") monkeypatch.setattr(SnapshotEnvTaskStep, "_export_environment_to_file", staticmethod(fake_export)) - record = _carded({"name": "gw", "children_environments": []}) + sent = _mock_http(monkeypatch, body=json.dumps(await _card({"mcp-slack": []})).encode()) + record = _carded(await _card({"mcp-slack": []})) record.gateway_url = "https://gw-1/" for gateway_url in ("https://gw-1/", "https://elsewhere"): with pytest.raises(RuntimeError, match="not exported"): @@ -1059,7 +1062,9 @@ async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_ env=_FakeMultiEnv(["slack"]), gateway_url=gateway_url, snapshot_id="snap-x", deployed=record, ) - assert seen == [("https://gw-1", record), ("https://elsewhere", None)] + assert seen == [("https://gw-1", record, _CARD, "/svc/mcp-slack/agentenv"), + ("https://elsewhere", None, "https://elsewhere", "/svc/mcp-slack/agentenv")] + assert [str(r.url) for r in sent] == [f"https://elsewhere{WELL_KNOWN_PATH}"] @pytest.mark.asyncio @@ -1081,19 +1086,26 @@ async def test_export_with_a_record_that_has_no_stored_card_keeps_the_live_probe ] -def test_enumerate_services_single_mcp_server_env(): +@pytest.mark.asyncio +async def test_a_record_without_a_card_snapshots_the_servers_its_definition_names(): env = _FakeServiceEnv("calendar") env.id, env.version = "cal-env", 1 - assert SnapshotEnvTaskStep._enumerate_environments(env) == ["calendar"] + servers = await legacy_protocol.backing_servers(_deployed_without_a_card(), "https://gw-1", env) + assert [(s.environment_name, s.mcp, s.address) for s in SnapshotEnvTaskStep._snapshot_servers(servers, env)] == [ + ("calendar", True, None)] -def test_enumerate_services_no_services_raises(): +def test_snapshot_servers_refuses_an_env_with_none(): env = type("E", (), {"id": "x", "version": 1})() with pytest.raises(RuntimeError, match="no MCP services"): - SnapshotEnvTaskStep._enumerate_environments(env) + SnapshotEnvTaskStep._snapshot_servers(legacy_protocol.definition_servers(env), env) -# ── loaded-universe metadata carry-over ────────────────────────────────────── +def _deployed_without_a_card() -> DeployedEnv: + """An older record: no card URL, so its servers are the ones its definition names.""" + return DeployedGatewayEnv(env_id="env-1", env_version=3, gateway_url="https://gw-1", mcp_url="https://gw-1/mcp", + db_web_url=None, sandbox_id="sb-1") + def test_resolve_loaded_universe_ref_reads_store(monkeypatch): import agent_env.env.store as store_mod @@ -1139,7 +1151,7 @@ def get_environment_universe(self, instance_id): def _install_core_artifact_fakes(monkeypatch, universe_cls): """Wire the export + FileArtifact/EnvironmentArtifact fakes shared by the core carry-over tests; the caller supplies the EnvironmentUniverseArtifact fake.""" - async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None): + async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None, **_): with open(tmp_path, "w") as f: json.dump({"ok": name}, f) return ".json" @@ -1199,7 +1211,7 @@ def get_environment_universe(self, instance_id): await SnapshotEnvTaskStep.snapshot_env_state( env=_FakeMultiEnv(["slack"]), - gateway_url="https://gw", + gateway_url="https://gw-1", snapshot_id="snap-x", deployed=_deployed(instance_id="env-1-abc123"), ) @@ -1240,7 +1252,7 @@ def get_environment_universe(self, instance_id): await SnapshotEnvTaskStep.snapshot_env_state( original_universe_artifact_id="explicit-base", env=_FakeMultiEnv(["slack"]), - gateway_url="https://gw", + gateway_url="https://gw-1", snapshot_id="snap-x", deployed=_deployed(), ) @@ -1276,7 +1288,7 @@ def get_environment_universe(self, instance_id): result = await SnapshotEnvTaskStep.snapshot_env_state( env=_FakeMultiEnv(["slack"]), - gateway_url="https://gw", + gateway_url="https://gw-1", snapshot_id="snap-x", deployed=_deployed(), ) @@ -1302,7 +1314,7 @@ def get(artifact_id, version=None): await SnapshotEnvTaskStep.snapshot_env_state( original_universe_artifact_id="bad", env=_FakeMultiEnv(["slack"]), - gateway_url="https://gw", + gateway_url="https://gw-1", snapshot_id="snap-x", deployed=_deployed(), ) @@ -1376,17 +1388,103 @@ def _multi_env(mcp_names, website_names): website_envs=[_FakeServiceEnv(n) for n in website_names]) -def test_enumerate_includes_websites_and_leaves_a_shared_name_to_the_mcp_server(): - servers = SnapshotEnvTaskStep._enumerate_servers(_multi_env(["slack", "gmail"], ["shop", "slack"])) - assert [(s.environment_name, s.mcp) for s in servers] == [("slack", True), ("gmail", True), ("shop", False)] +@pytest.mark.asyncio +async def test_the_servers_snapshotted_are_the_cards_children_one_per_name_and_those_serving_no_card(caplog): + """A website sharing an MCP server's name is its own server on the card, left out of the snapshot (logged) since a + universe names an environment by its name; a server serving no card, which the card leaves off, keeps its legacy + export.""" + record = _carded(await _card({"mcp-slack": [], "mcp-gmail": [], "shop": [], "slack": []})) + env = _multi_env(["slack", "gmail", "legacy"], ["shop", "slack"]) + + listed = await legacy_protocol.backing_servers(record, "https://gw-1", env) + with caplog.at_level("WARNING"): + servers = SnapshotEnvTaskStep._snapshot_servers(listed, env) + + assert [(s.environment_name, s.mcp, s.card and s.card["url"]) for s in listed] == [ + ("slack", True, "/svc/mcp-slack/agentenv"), ("gmail", True, "/svc/mcp-gmail/agentenv"), + ("shop", False, "/svc/shop/agentenv"), ("slack", False, "/svc/slack/agentenv"), ("legacy", True, None)] + assert [(s.environment_name, s.mcp) for s in servers] == [("slack", True), ("gmail", True), ("legacy", True), ("shop", False)] + assert "website 'slack' of env-1 is not snapshotted: it shares its name with the MCP server kept" in caplog.text -def test_enumerate_a_single_website_env_and_a_multi_env_of_websites_only(): +@pytest.mark.asyncio +async def test_a_single_website_env_and_a_multi_env_of_websites_only_are_snapshotted_as_websites(): from agent_env.env.envs.website import WebsiteEnv site = WebsiteEnv(id="site-env", version=1, environment_name="shop") - assert [(s.environment_name, s.mcp) for s in SnapshotEnvTaskStep._enumerate_servers(site)] == [("shop", False)] - assert SnapshotEnvTaskStep._enumerate_environments(_multi_env([], ["shop"])) == ["shop"] + assert [(s.environment_name, s.mcp) for s in legacy_protocol.definition_servers(site)] == [("shop", False)] + record = _carded(await _card({"shop": []})) + assert [(s.environment_name, s.mcp) for s in await legacy_protocol.backing_servers(record, "https://gw-1", _multi_env([], ["shop"]))] == [ + ("shop", False)] + + +@pytest.mark.asyncio +async def test_a_card_only_website_is_left_out_as_serving_no_v1_data_plane(monkeypatch, tmp_path): + """Its card declares no data/get, so it is skipped with no request, not failed on a data/get it doesn't serve.""" + record = _carded(await _card({"mcp-slack": [], "shop": []})) + record.environment_card["children_environments"][1]["capabilities"]["operations"] = [] + (shop,) = [s for s in await legacy_protocol.backing_servers(record, "https://gw-1", _multi_env(["slack"], ["shop"])) + if s.environment_name == "shop"] + sent = _mock_http(monkeypatch) + + assert await SnapshotEnvTaskStep._export_environment_to_file("https://gw-1", "shop", str(tmp_path / "out"), 30, record, + server=shop) is None + assert sent == [] + + +@pytest.mark.asyncio +async def test_a_card_only_website_is_reported_as_serving_no_v1_data_plane(monkeypatch): + real_export = SnapshotEnvTaskStep.__dict__["_export_environment_to_file"] + _install_core_artifact_fakes(monkeypatch, EnvironmentUniverseArtifact) + monkeypatch.setattr(SnapshotEnvTaskStep, "_export_environment_to_file", real_export) + record = _carded(await _card({"shop": []})) + record.environment_card["children_environments"][0]["capabilities"]["operations"] = ["data/reset"] + sent = _mock_http(monkeypatch) + + with pytest.raises(RuntimeError, match=r"exported no state: \['shop'\] serve no v1 data plane"): + await SnapshotEnvTaskStep.snapshot_env_state(env=_multi_env([], ["shop"]), gateway_url="https://gw-1", + snapshot_id="snap-x", deployed=record) + assert sent == [] + + +@pytest.mark.asyncio +async def test_an_mcp_only_env_snapshots_each_server_at_its_route_and_one_serving_no_card_by_export_state(monkeypatch): + real_export = SnapshotEnvTaskStep.__dict__["_export_environment_to_file"] + universe_puts = {} + + class _FakeUniverse: + id, version = "snap-x", 1 + + @staticmethod + def put(id, *, environment_artifacts, metadata=None): + universe_puts["names"] = [sa.environment_name for sa in environment_artifacts] + return _FakeUniverse() + + _install_core_artifact_fakes(monkeypatch, _FakeUniverse) + monkeypatch.setattr(SnapshotEnvTaskStep, "_export_environment_to_file", real_export) + sent = _mock_http(monkeypatch, rpc_result={"parts": [{"kind": "data", "data": {"rows": 1}}]}, body=b'{"rows": 2}') + record = _carded(await _card({"mcp-slack": [], "mcp-gmail": []})) + + result = await SnapshotEnvTaskStep.snapshot_env_state(env=_FakeMultiEnv(["slack", "gmail", "legacy"]), gateway_url="https://gw-1", + snapshot_id="snap-x", deployed=record) + + assert sorted((r.method, str(r.url)) for r in sent) == [ + ("GET", "https://gw-1/svc/mcp-legacy/export-state"), ("POST", f"{_CARD}/svc/mcp-gmail/agentenv"), + ("POST", f"{_CARD}/svc/mcp-slack/agentenv")] + assert universe_puts["names"] == ["slack", "gmail", "legacy"] and result.total == 3 + + +@pytest.mark.asyncio +async def test_a_website_sharing_an_mcp_servers_name_is_exported_at_its_own_route(monkeypatch, tmp_path): + """The card tells the two apart by their urls, so the website's data/get goes to /svc/slack, not the MCP server.""" + record = _carded(await _card({"mcp-slack": [], "slack": []})) + servers = await legacy_protocol.backing_servers(record, "https://gw-1", _multi_env(["slack"], ["slack"])) + sent = _mock_http(monkeypatch, rpc_result={"parts": [{"kind": "data", "data": {"orders": 2}}]}) + + for server in servers: + assert await SnapshotEnvTaskStep._export_environment_to_file("https://gw-1", "slack", str(tmp_path / "out"), 30, record, + server=server) == ".json" + assert [str(r.url) for r in sent] == [f"{_CARD}/svc/mcp-slack/agentenv", f"{_CARD}/svc/slack/agentenv"] @pytest.mark.asyncio @@ -1500,7 +1598,7 @@ async def test_export_of_a_website_serving_no_data_plane_exports_nothing_and_sen async def test_core_snapshots_websites_through_their_data_plane_and_leaves_out_those_without_one(monkeypatch): calls = [] - async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None, **website): + async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, data_get_params=None, server=None, **website): calls.append((name, website)) if name == "legacy-site": return None @@ -1539,5 +1637,5 @@ async def fake_export(gateway_url, name, tmp_path, timeout, deployed=None, **web _install_core_artifact_fakes(monkeypatch, EnvironmentUniverseArtifact) monkeypatch.setattr(SnapshotEnvTaskStep, "_export_environment_to_file", staticmethod(fake_export)) - with pytest.raises(RuntimeError, match=r"exported no state: its websites \['shop'\] serve no v1 data plane"): + with pytest.raises(RuntimeError, match=r"exported no state: \['shop'\] serve no v1 data plane"): await SnapshotEnvTaskStep.snapshot_env_state(env=_multi_env([], ["shop"]), gateway_url="https://gw", snapshot_id="snap-x") From 76145d25ad039e3509f6841220b8594234346e1e Mon Sep 17 00:00:00 2001 From: William Loo Date: Tue, 3 Nov 2026 13:34:07 -0800 Subject: [PATCH 03/12] feat(website): WebsiteEnvironment serves a site on the website/v1 contract, and a conformance check tests any server (FD-4683) This PR's change, rebased as one commit onto #427. Co-Authored-By: Claude Opus 5.5 (1M context) --- AGENTS.md | 2 +- packages/agentenv-protocol/README.md | 27 + .../src/agentenv_protocol/website/__init__.py | 36 ++ .../src/agentenv_protocol/website/clock.py | 114 ++++ .../agentenv_protocol/website/conformance.py | 351 +++++++++++ .../agentenv_protocol/website/environment.py | 450 ++++++++++++++ .../src/agentenv_protocol/website/errors.py | 39 ++ .../agentenv_protocol/website/readiness.py | 73 +++ .../src/agentenv_protocol/website/routes.py | 369 ++++++++++++ .../src/agentenv_protocol/website/session.py | 60 ++ .../src/agentenv_protocol/website/state.py | 56 ++ .../src/agentenv_protocol/website/store.py | 134 +++++ .../tests/test_website_api.py | 413 +++++++++++++ .../tests/test_website_conformance.py | 265 +++++++++ .../tests/test_website_environment.py | 552 ++++++++++++++++++ .../tests/test_website_readiness.py | 209 +++++++ .../tests/test_website_state.py | 121 ++++ src/agent_env/env/env.py | 59 +- src/agent_env/env/gateway/gateway.py | 85 ++- src/agent_env/env/legacy_protocol.py | 22 +- .../env_providers/env_gateway_provider.py | 21 +- .../task_step/task_steps/add_skills.py | 16 +- .../mcp_cli_builder/build_mcp_cli.py | 11 +- .../verifiers/website_state_verifier.py | 98 +++- tst/unit/env/gateway/changes_test.py | 15 +- .../env/gateway/rest_proxy_stream_test.py | 15 +- tst/unit/env/gateway/triggers_website_test.py | 2 +- .../gateway/website_reserved_paths_test.py | 96 ++- tst/unit/env/test_env.py | 111 +++- tst/unit/env/test_legacy_protocol.py | 15 + .../env_gateway_provider_test.py | 55 ++ tst/unit/task_step/add_skills_env_test.py | 11 + .../task_step/test_manifest_cli_builder.py | 11 + .../test_website_state_verifier_step.py | 74 ++- 34 files changed, 3893 insertions(+), 95 deletions(-) create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/website/__init__.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/website/clock.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/website/conformance.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/website/environment.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/website/errors.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/website/readiness.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/website/routes.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/website/session.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/website/state.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/website/store.py create mode 100644 packages/agentenv-protocol/tests/test_website_api.py create mode 100644 packages/agentenv-protocol/tests/test_website_conformance.py create mode 100644 packages/agentenv-protocol/tests/test_website_environment.py create mode 100644 packages/agentenv-protocol/tests/test_website_readiness.py create mode 100644 packages/agentenv-protocol/tests/test_website_state.py diff --git a/AGENTS.md b/AGENTS.md index aa980e2e8..17a4fa4e5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -117,7 +117,7 @@ and deserialized through a registry. | Package | Contents | |---|---| -| `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`, one site: a server image, `backend_image` in its env.toml, serving the pages and API, checked at its `health` path; a separate frontend image is deprecated, warned of at put and optional in a bundle, and a stored `backend_docker_image_artifact` reads as the server image), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]` (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`, `bundle register` and `run`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register that reuses a computer env composes what it has left (`pending`), a run only the forms its sandbox's deploys boot (`booted_forms`: on each link, the first form it boots natively, by the deploy's own rule, `boot.first_native`; a compose of a disk adds its container), and a deploy composes first what it boots that no compose started (`ComputerEnv.compose_unstarted`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 (`utils/probing.py`) then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference, over its env.toml's `[settings]` and a computer env's `settings` beside it in `apps`, each by top-level field (`params.app_settings`; checked at put, a required one left to the deploy; the deploy record keeps the merged values as `app_settings`, secrets withheld); its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. Its site route, `/website//`, never forwards a site's data plane or card (`/agentenv/…`, `/.well-known/agent-env.json`), which agentenv reaches through `/svc//`; a site with no frontend image is served by its server image there. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | +| `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`, one site: a server image, `backend_image` in its env.toml, serving the pages and API, ready when its card's `website/v1` `ready` answers a `WebsiteReadiness` that says so, as `agentenv_protocol.website` (`website_readiness`, `env_readiness`) defines a website env's health for the deploy and `DeployedEnv.readiness()` alike, or, for a card without `ready`, when its `health` path answers 200, which is deprecated; the gateway container's `env/gateway/constants.py` mirrors the URI and op, a test keeping them equal; a separate frontend image is deprecated, warned of at put and optional in a bundle, and a stored `backend_docker_image_artifact` reads as the server image), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]` (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`, `bundle register` and `run`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register that reuses a computer env composes what it has left (`pending`), a run only the forms its sandbox's deploys boot (`booted_forms`: on each link, the first form it boots natively, by the deploy's own rule, `boot.first_native`; a compose of a disk adds its container), and a deploy composes first what it boots that no compose started (`ComputerEnv.compose_unstarted`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 (`utils/probing.py`) then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference, over its env.toml's `[settings]` and a computer env's `settings` beside it in `apps`, each by top-level field (`params.app_settings`; checked at put, a required one left to the deploy; the deploy record keeps the merged values as `app_settings`, secrets withheld); its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. Its site route, `/website//`, never forwards a site's data plane or card (`/agentenv/…`, `/.well-known/agent-env.json`), which agentenv reaches through `/svc//`; a site with no frontend image is served by its server image there. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Every artifact but an environment universe, whose `metadata` key holds its legacy metadata file ids, carries a `metadata` dict outside its versioned fields, changed by compare-and-swap (`merge_metadata`). Built-ins: `CliArtifact` (`cli`), `ComputerArtifact` (`computer`, a base computer: its architectures, OS, session user, settings and container form, whose image `put` reads from its registry into `metadata.validated_images`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`: its environment's files by path, stored as the one wrapped `file_artifact_ref` every environment artifact had when it has one file at its filename, else as `file_artifact_refs`; `get_file_artifacts()` reads either, and a load hands each file to the env's `data/add` with its path, which a pre-v1 load, taking one file, refuses), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | | `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `website_state_verifier`, which reads each checked session of a deployed website env's sites, or a multi env's, at the site's `/go?sid=` through `/svc//` and checks one dot path of its `current_state`, `initial_state` or `state_diff` each, optionally reduced by `len`, `keys` or `bool`, with `eq`, `ne`, `ge`, `gt`, `le`, `lt`, `in` or `contains`, its sites checked before a run against the `env_version` it pins, `start_computer_recording` and `end_computer_recording`, and more). On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the local namespaces, `@local` and `@hf` (`is_local_id`), each with a documents file of its own: while the CLI or a `build_app` server runs, a local id and what a local task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 4e3b0de48..847455739 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -430,6 +430,33 @@ async with ToolSession(mcp_url=mcp_url, role="alice@example.com") as session: - **Tools:** `list_tools()` returns the tools the role may use as `EnvironmentTool`, the type a card's `capabilities.tools` holds. Each `inputSchema` is the JSON Schema a model API takes as the tool's parameters. +## Website readiness + +`agentenv_protocol.website` is the one definition of whether a website env is healthy. A site's card advertises +`urn:agentenv:website/v1`'s `ready` (`WEBSITE_READY_OP`), a GET answering a `WebsiteReadiness`: `ready`, a boolean, +and `problems`, the names of what keeps it from being ready (a `WebsiteEnvironment`'s are its `@ready_check` methods +not yet true). The model is strict about those two (`"yes"` is no `ready`) and ignores fields it doesn't know, so a +reader keeps reading a newer site. Everything that judges a site reads it through these helpers: agent-env's deploy, +`DeployedEnv.readiness()`, and the conformance checker's `health`, which fails an answer that isn't a +`WebsiteReadiness`. + +```python +from agentenv_protocol.website import env_readiness, website_readiness + +site = await website_readiness(base_url, card) # WebsiteReadiness, or None for a card without ready +env = await env_readiness(gateway_url, env_card) # WebsiteReadiness of every site of the env +print(env.ready, env.problems) # False ['shop: seeded'] +``` + +- **One site:** `website_readiness(base_url, card, *, timeout=READY_TIMEOUT_S, headers=None)` asks the site as its card + advertises `ready`. It raises httpx's error when the site can't be reached, `ExtensionCallError` for a non-2xx + answer and `ValueError` (pydantic's `ValidationError`) for an answer that isn't a `WebsiteReadiness`. +- **An env:** `env_readiness(base_url, card, ...)` asks, at once, the card itself when it advertises `ready` (a single + site's card), and every child under a composed card's `children_environments` that does, its endpoints rooted at + `base_url` as a gateway's are. A child's problems are prefixed with its name (`shop: seeded`); one that can't be + asked, or answers no `WebsiteReadiness`, is a problem itself. A card or child without `ready`, an MCP server's or an + older site's, isn't counted, so an env with no site serving it is ready with no problems. + ## Computer drivers A computer driver is what runs inside a computer an agent uses through screenshots and input. Write diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/website/__init__.py new file mode 100644 index 000000000..cfdc1a39e --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/__init__.py @@ -0,0 +1,36 @@ +"""The website SDK: a site's server, meeting ``urn:agentenv:website/v1``. Subclass WebsiteEnvironment, declare the +site with ``@website(domains=[...], static="dist/")``, mark its API with ``@api.get/post/put/patch/delete(path)``, and +``serve()`` it; it needs only the plain package, and uvicorn to serve. The class is optional: a server in any language +that meets the contract works, which ``python -m agentenv_protocol.website.conformance `` checks. Whether a site +is ready is its ``WebsiteReadiness`` answer to ``ready``, which ``website_readiness`` reads of one site and +``env_readiness`` of every site of an env. What a task left a site in is its ``WebsiteState`` answer to ``state``, +which ``website_state`` reads of one site of those ``website_sites`` names.""" + +from __future__ import annotations + +from .clock import CLOCK_EXTENSION_URI, UNIVERSE_NOW_HEADER, UniverseClock +from .environment import ( + DEFAULT_HEALTH_PATH, + SESSION_PATH, + STATE_PATH, + WebsiteDeclaration, + WebsiteEnvironment, + ready_check, + website, +) +from .errors import ApiError, BadRequest, Conflict, Forbidden, NotFound, Unauthorized +from .readiness import READY_TIMEOUT_S, WEBSITE_EXTENSION_URI, WEBSITE_READY_OP, WebsiteReadiness, env_readiness, website_readiness +from .routes import ROLE_KEY_ENV, ROLE_KEY_HEADER, ApiContext, ApiRoute, api, reserved_path +from .session import SESSION_COOKIE, SessionSigner +from .state import STATE_TIMEOUT_S, WEBSITE_STATE_OP, WebsiteState, advertises_state, website_sites, website_state +from .store import Collections + +__all__ = [ + "WebsiteEnvironment", "WebsiteDeclaration", "website", "api", "ApiContext", "ApiRoute", "ready_check", + "ApiError", "BadRequest", "Unauthorized", "Forbidden", "NotFound", "Conflict", + "Collections", "UniverseClock", "SessionSigner", "reserved_path", + "WEBSITE_EXTENSION_URI", "CLOCK_EXTENSION_URI", "UNIVERSE_NOW_HEADER", "DEFAULT_HEALTH_PATH", "SESSION_PATH", + "STATE_PATH", "SESSION_COOKIE", "ROLE_KEY_ENV", "ROLE_KEY_HEADER", + "WEBSITE_READY_OP", "READY_TIMEOUT_S", "WebsiteReadiness", "website_readiness", "env_readiness", + "WEBSITE_STATE_OP", "STATE_TIMEOUT_S", "WebsiteState", "advertises_state", "website_sites", "website_state", +] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/clock.py b/packages/agentenv-protocol/src/agentenv_protocol/website/clock.py new file mode 100644 index 000000000..0d27d86a9 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/clock.py @@ -0,0 +1,114 @@ +"""The universe's clock as a site reads it: ``clock/v1``'s ``sync_time`` hands the server the gateway's ``/clock/time`` +URL, as it hands it to MCP servers, and the site's time then follows that clock's ``{"virtual_time": }``, +read at most once a second and extrapolated between reads. The rate it extrapolates at is one two successive intervals +between reads agreed on, so a clock set or advanced between two reads is taken for a jump, not a speed; until two agree +it is real time's. An unsynced site reads wall time. Every answer of a site carries the time it answered at as ``X-Universe-Now``, +RFC3339 in UTC. ``sync_time`` may also name the universe's time zone, an IANA name, which the clock keeps for the site +to show local times in; the time itself stays UTC.""" +from __future__ import annotations + +import asyncio +import datetime +import json +import logging +import time + +import httpx + +logger = logging.getLogger(__name__) + +CLOCK_EXTENSION_URI = "urn:agentenv:clock/v1" +UNIVERSE_NOW_HEADER = "X-Universe-Now" +REFRESH_S = 1.0 +READ_TIMEOUT_S = 5.0 +RATE_AGREEMENT = 0.1 + + +def iso(moment: datetime.datetime) -> str: + return moment.astimezone(datetime.timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z") + + +def parse_time(value: str) -> datetime.datetime: + moment = datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) + return moment if moment.tzinfo else moment.replace(tzinfo=datetime.timezone.utc) + + +class UniverseClock: + """The time a site answers with: wall time until ``sync`` names a gateway clock, that clock's after.""" + + def __init__(self) -> None: + self.url: str | None = None + self.timezone: str | None = None + self._virtual: datetime.datetime | None = None + self._read_at = 0.0 + self._tried_at = 0.0 + self._rate = 1.0 + self._measured: float | None = None + self._lock: asyncio.Lock | None = None + + @property + def synced(self) -> bool: + return self._virtual is not None + + def now(self) -> datetime.datetime: + if self._virtual is None: + return datetime.datetime.now(datetime.timezone.utc) + return self._virtual + datetime.timedelta(seconds=(time.monotonic() - self._read_at) * self._rate) + + async def sync(self, url: str, timezone: str | None = None) -> datetime.datetime: + """Follow the clock at ``url`` from now on, in ``timezone`` (none when the sync names none); raises when it + can't be read, keeping the clock and zone it followed. It waits for a refresh reading the clock it followed, + so that read never lands on the new clock.""" + async with self._reading(): + virtual = await _read(url) + self.url, self._virtual, self._rate, self._measured = url, virtual, 1.0, None + self.timezone = timezone + self._read_at = self._tried_at = time.monotonic() + return virtual + + async def refresh(self) -> None: + """Read the synced clock again when the last read is a second old; a failed read keeps extrapolating.""" + if self.url is None or time.monotonic() - self._tried_at < REFRESH_S: + return + async with self._reading(): + started = time.monotonic() + if started - self._tried_at < REFRESH_S: + return + self._tried_at = started + try: + virtual = await _read(self.url) + except Exception as e: + logger.debug("reading the universe's clock at %s failed: %r", self.url, e) + return + read_at = time.monotonic() + if self._virtual is not None and read_at > self._read_at: + measured = max(0.0, (virtual - self._virtual).total_seconds() / (read_at - self._read_at)) + if self._measured is not None and abs(measured - self._measured) <= RATE_AGREEMENT * max(measured, self._measured, 1.0): + self._rate = measured + self._measured = measured + self._virtual, self._read_at = virtual, read_at + + def _reading(self) -> asyncio.Lock: + """The lock a sync and a refresh each read the clock under.""" + self._lock = self._lock or asyncio.Lock() + return self._lock + + +async def _read(url: str) -> datetime.datetime: + async with httpx.AsyncClient() as client: + response = await client.get(url, timeout=READ_TIMEOUT_S) + response.raise_for_status() + return parse_time(response.json()["virtual_time"]) + + +def clock_patch_script(now: datetime.datetime) -> bytes: + """An inline script that shifts the page's ``Date`` to the universe's time ``now``, and follows the + ``X-Universe-Now`` header of every ``fetch`` the page makes after. The ``Date`` it puts in place answers called, + with the time's string, as constructed, and keeps ``Date.parse``, ``Date.UTC`` and ``instanceof Date``.""" + return ("" % (int(now.timestamp() * 1000), json.dumps(UNIVERSE_NOW_HEADER))).encode() diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/conformance.py b/packages/agentenv-protocol/src/agentenv_protocol/website/conformance.py new file mode 100644 index 000000000..8df978f4b --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/conformance.py @@ -0,0 +1,351 @@ +"""Check a running site's server against ``urn:agentenv:website/v1``, whatever it's written in: +``python -m agentenv_protocol.website.conformance `` prints each check and exits 1 when one failed, 2 when the +server couldn't be reached. Everything goes over HTTP: the card, ``/`` under a made-up host, readiness by the card's +``website/v1`` ``ready``, whose answer must be a ``WebsiteReadiness`` (by the declared health path, deprecated, for a +card without it), ``X-Universe-Now`` on every answer the other checks got, a data-plane round trip at the card's ``url`` +(data/get, data/reset, data/get, which no longer returns the exported data, data/add of what the first data/get +returned, data/get again), ``clock/v1``'s +``sync_time`` against a clock the checker serves, and the forwarded headers wherever an answer shows them, on ``/`` and +on a GET route with its trailing slash toggled, which a server may redirect. + +The round trip leaves the server's data as it found it, adding the export back when a step after the reset fails too. +data/add sets the state a task starts from, so a server whose state shows changes since its initial state, as an SDK +site's does, would have them folded into it: the round trip is skipped for such a server. Its state is its card's +``website/v1`` ``state``, else, for a card without it, what the path its ``params.state.path`` names answers, else +``/go``'s, deprecated. The clock check leaves the +server following a clock that is gone once the check ends, so a server it ran against keeps extrapolating from that +clock's time until it's synced again. The checker's clock starts at ``CLOCK_START``, far enough from wall time that a +synced server shows it, and a synced server's ``X-Universe-Now`` may be ``CLOCK_SLACK_S`` seconds from it.""" +from __future__ import annotations + +import argparse +import contextlib +import dataclasses +import datetime +import json +import sys +import threading +import time +import uuid +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Callable, Iterator, Literal + +import httpx +from pydantic import ValidationError + +from ..client import find_extension, find_extension_method +from ..types import METHOD_ADD, METHOD_GET, METHOD_RESET, RPC_PATH, WELL_KNOWN_PATH +from .clock import CLOCK_EXTENSION_URI, UNIVERSE_NOW_HEADER, iso, parse_time +from .environment import DEFAULT_HEALTH_PATH, STATE_PATH +from .readiness import WEBSITE_EXTENSION_URI, WEBSITE_READY_OP, WebsiteReadiness +from .state import WEBSITE_STATE_OP + +Status = Literal["passed", "failed", "skipped"] +CLOCK_START = datetime.datetime(2031, 5, 6, 7, 8, 9, tzinfo=datetime.timezone.utc) +CLOCK_SLACK_S = 120 + + +@dataclasses.dataclass(frozen=True) +class Result: + name: str + status: Status + detail: str = "" + + @property + def passed(self) -> bool: + return self.status != "failed" + + +def check(base_url: str, *, client: httpx.Client | None = None, clock_host: str = "127.0.0.1", + timeout: float = 30) -> list[Result]: + """Each check's result against the server at ``base_url``. ``clock_host`` is the address the server reaches the + checker's clock at. Raises httpx's error when the server can't be reached at all.""" + own = client is None + client = client or httpx.Client(timeout=timeout) + try: + return _Checker(base_url.rstrip("/"), client, clock_host).run() + finally: + if own: + client.close() + + +class _Checker: + def __init__(self, base_url: str, client: httpx.Client, clock_host: str) -> None: + self.base_url, self.client, self.clock_host = base_url, client, clock_host + self.rpc_url = base_url + RPC_PATH + self.responses: list[httpx.Response] = [] + + def get(self, path: str, **kwargs) -> httpx.Response: + return self._seen(self.client.get(self.base_url + path, **kwargs)) + + def _seen(self, response: httpx.Response) -> httpx.Response: + self.responses.append(response) + return response + + def run(self) -> list[Result]: + response = self.get(WELL_KNOWN_PATH) + try: + card = response.json() if response.status_code == 200 else {} + except ValueError: + card = {} + self.rpc_url = _rpc_url(self.base_url, card) + site = (find_extension(card, WEBSITE_EXTENSION_URI) or {}).get("params") or {} + health = site.get("health") if isinstance(site.get("health"), str) else DEFAULT_HEALTH_PATH + steps = (("any_host", self._any_host), ("health", lambda: self._health(card, health)), + ("universe_now", self._universe_now), ("data_plane", lambda: self._data_plane(card)), + ("clock", lambda: self._clock(card, health)), ("forwarded", lambda: self._forwarded(site)), + ("cookies", lambda: self._cookies(site))) + outcomes = {name: _outcome(step) for name, step in steps if name != "universe_now"} + outcomes["universe_now"] = _outcome(self._universe_now) + return [self._card(response, site), *(Result(name, *outcomes[name]) for name, _step in steps)] + + def _card(self, response: httpx.Response, site: dict) -> Result: + if response.status_code != 200: + return Result("card", "failed", f"{WELL_KNOWN_PATH} answered {response.status_code}") + problems = [] + if not site: + problems.append(f"declares no {WEBSITE_EXTENSION_URI} with params") + if not isinstance(site.get("domains"), list): + problems.append("its params have no domains list") + if not (isinstance(site.get("health"), str) and site["health"].startswith("/")): + problems.append("its params have no health path") + routes = site.get("routes") + if not isinstance(routes, list) or not all(isinstance(r, dict) and {"method", "path", "name"} <= set(r) + for r in routes): + problems.append("its params' routes aren't a list of {method, path, name}") + return Result("card", "failed" if problems else "passed", "; ".join(problems)) + + def _any_host(self) -> tuple[Status, str]: + host = f"conformance-{uuid.uuid4().hex[:8]}.invalid" + response = self.get("/", headers={"Host": host}) + kind = response.headers.get("content-type", "") + if response.status_code == 200 and kind.startswith("text/html"): + return "passed", f"/ served HTML for Host {host}" + return "failed", f"/ for Host {host} answered {response.status_code} {kind or 'without a content type'}" + + def _health(self, card: dict, health: str) -> tuple[Status, str]: + ready = find_extension_method(card, WEBSITE_EXTENSION_URI, WEBSITE_READY_OP) + if ready is None: + response = self.get(health) + return (("passed" if response.status_code == 200 else "failed"), + f"{health} answered {response.status_code}; the card has no website/v1 ready, so the health path " + "was read, which is deprecated") + answer = self._seen(self.client.request(ready.get("method", "GET"), self.base_url + ready["endpoint"])).json() + try: + readiness = WebsiteReadiness.model_validate(answer) + except ValidationError as e: + return "failed", (f"website/v1 ready answered {answer!r}, not a WebsiteReadiness: " + f"{'; '.join(_field_error(error) for error in e.errors())}") + if readiness.ready: + return "passed", "website/v1 ready says it is ready" + return "failed", f"website/v1 ready says it is not ready: {', '.join(readiness.problems) or 'no problem named'}" + + def _universe_now(self) -> tuple[Status, str]: + missing = [str(r.url.path) for r in self.responses if UNIVERSE_NOW_HEADER not in r.headers] + if missing: + return "failed", f"no {UNIVERSE_NOW_HEADER} on {', '.join(missing)}" + for response in self.responses: + parse_time(response.headers[UNIVERSE_NOW_HEADER]) + return "passed", f"on all {len(self.responses)} answers" + + def _rpc(self, method: str, params: dict | None = None) -> dict: + response = self._seen(self.client.post(self.rpc_url, json={"jsonrpc": "2.0", "id": method, + "method": method, "params": params or {}})) + body = response.json() + if isinstance(body, dict) and "error" in body: + raise RuntimeError(f"{method} answered {body['error']}") + if not response.is_success or not isinstance(body, dict) or "result" not in body: + raise RuntimeError(f"{method} answered {response.status_code} with no JSON-RPC result: {response.text[:200]}") + return body["result"] or {} + + def _data_plane(self, card: dict) -> tuple[Status, str]: + operations = (card.get("capabilities") or {}).get("operations") + missing = [m for m in (METHOD_GET, METHOD_RESET, METHOD_ADD) if operations is not None and m not in operations] + if missing: + return "failed", f"the card's operations lack {', '.join(missing)}" + exported = self._rpc(METHOD_GET).get("parts") or [] + if where := self._changed_since_initial(card): + return "skipped", (f"data/get answered; {where} shows changes since the initial state, which data/reset " + "and data/add would fold into it, so the round trip was not run") + self._rpc(METHOD_RESET) + added_back = False + try: + cleared = self._rpc(METHOD_GET).get("parts") or [] + if _holds_data(exported) and cleared == exported: + added_back = True + return "failed", "data/get after data/reset returned the data it returned before" + if not exported: + added_back = True + return "passed", "data/get, data/reset and data/get answered; data/get returned nothing to add back" + added_back = True + self._rpc(METHOD_ADD, {"parts": exported}) + again = self._rpc(METHOD_GET).get("parts") or [] + finally: + if not added_back: + with contextlib.suppress(Exception): + self._rpc(METHOD_ADD, {"parts": exported}) + if again != exported: + return "failed", "data/get after data/reset and data/add of its own export returned other data" + return "passed", f"{len(exported)} part(s) round-tripped through reset and add" + + def _changed_since_initial(self, card: dict) -> str | None: + """Where the server's state was read when its ``state_diff`` names changes, None when it names none or can't be + read: the card's ``website/v1`` ``state``, else the path its ``params.state.path`` names, else ``/go``. Read + aside, as no check's answer, since a server without one needn't stamp its 404.""" + op = find_extension_method(card, WEBSITE_EXTENSION_URI, WEBSITE_STATE_OP) + if op is not None and op.get("endpoint"): + where, request = f"website/v1 {WEBSITE_STATE_OP}", (op["method"], self.base_url + op["endpoint"]) + else: + declared = ((find_extension(card, WEBSITE_EXTENSION_URI) or {}).get("params") or {}).get("state") + path = declared.get("path") if isinstance(declared, dict) else None + where = path if isinstance(path, str) and path.startswith("/") else STATE_PATH + request = ("GET", self.base_url + where) + try: + state = self.client.request(*request).json() + except Exception: + return None + return where if isinstance(state, dict) and state.get("state_diff") else None + + def _clock(self, card: dict, health: str) -> tuple[Status, str]: + if find_extension(card, CLOCK_EXTENSION_URI) is None: + return "failed", f"the card declares no {CLOCK_EXTENSION_URI}" + params = find_extension(card, CLOCK_EXTENSION_URI).get("params") or {} + method = find_extension_method(card, CLOCK_EXTENSION_URI, next(iter(params.get("methods") or {}), "sync_time")) + if method is None or not method.get("endpoint"): + return "failed", f"{CLOCK_EXTENSION_URI} advertises no sync_time endpoint" + with _clock_server(self.clock_host) as (url, virtual_now): + response = self._seen(self.client.request(method["method"], self.base_url + method["endpoint"], + json={"env_get_time_url": url})) + if response.status_code >= 400: + return "failed", f"sync_time answered {response.status_code}: {response.text[:200]}" + seen = self.get(health).headers.get(UNIVERSE_NOW_HEADER) + expected = virtual_now() + if seen is None: + return "failed", f"no {UNIVERSE_NOW_HEADER} after sync_time" + off = abs((parse_time(seen) - expected).total_seconds()) + if off > CLOCK_SLACK_S: + return "failed", f"after sync_time {UNIVERSE_NOW_HEADER} is {seen}, {off:.0f}s from the clock's {iso(expected)}" + return "passed", f"{UNIVERSE_NOW_HEADER} followed the clock to {seen}" + + def _forwarded(self, site: dict) -> tuple[Status, str]: + internal = f"internal-{uuid.uuid4().hex[:8]}.invalid" + forwarded = f"site-{uuid.uuid4().hex[:8]}.example" + headers = {"Host": internal, "X-Forwarded-Host": forwarded, "X-Forwarded-Proto": "https"} + paths = ["/", *_slash_toggled(site.get("routes"))] + for path in paths: + response = self.get(path, headers=headers, follow_redirects=False) + location = response.headers.get("location", "") + shown = [where for where, text in (("body", response.text), ("Location", location)) if internal in text] + if shown: + return "failed", f"the {' and '.join(shown)} of {path} named the internal Host {internal}, not {forwarded}" + if location.startswith(f"http://{forwarded}"): + return "failed", f"the Location of {path} is {location}, not https as X-Forwarded-Proto said" + return "passed", f"no answer to {', '.join(paths)} named the internal Host" + + def _cookies(self, site: dict) -> tuple[Status, str]: + if isinstance(site.get("session_path"), str): + self.get(site["session_path"]) + cookies = [c for r in self.responses for c in r.headers.get_list("set-cookie")] + if not cookies: + return "skipped", "no answer set a cookie" + scoped = [c.split("=", 1)[0] for c in cookies if any(a.strip().lower().startswith("domain=") for a in c.split(";"))] + if scoped: + return "failed", f"cookies tied to a domain: {', '.join(sorted(set(scoped)))}" + return "passed", f"{len(cookies)} cookie(s), none tied to a domain" + + +def _rpc_url(base_url: str, card: dict) -> str: + """Where the card says its data plane is: its ``url``, a path joined onto ``base_url`` or a full URL, else + ``RPC_PATH``.""" + url = card.get("url") if isinstance(card.get("url"), str) and card.get("url") else RPC_PATH + return url if "://" in url else base_url + url + + +def _outcome(step: Callable[[], tuple[Status, str]]) -> tuple[Status, str]: + try: + return step() + except Exception as e: + return "failed", f"{type(e).__name__}: {e}" + + +def _field_error(error: dict) -> str: + """One of a ``ValidationError``'s errors as a person reads it: where, then what.""" + where = ".".join(map(str, error.get("loc") or ())) or "the answer" + return f"{where}: {error.get('msg')}" + + +def _holds_data(parts: list) -> bool: + """Whether exported parts carry task data: a part that isn't a data part, or a data part with a value in it.""" + return any(not (isinstance(p, dict) and p.get("kind") == "data") or _filled(p.get("data")) for p in parts) + + +def _filled(value: object) -> bool: + if isinstance(value, (dict, list)): + return any(_filled(v) for v in (value.values() if isinstance(value, dict) else value)) + return value not in (None, "") + + +def _slash_toggled(routes: object) -> list[str]: + """The first GET route without path parameters, with its trailing slash added or taken off: a path a server + may redirect to the route's own, which shows the address the redirect names.""" + for route in routes if isinstance(routes, list) else []: + path = route.get("path") if isinstance(route, dict) and route.get("method") == "GET" else None + if isinstance(path, str) and path.startswith("/") and path != "/" and "{" not in path: + return [path[:-1] if path.endswith("/") else path + "/"] + return [] + + +@contextlib.contextmanager +def _clock_server(host: str) -> Iterator[tuple[str, Callable[[], datetime.datetime]]]: + """A gateway-shaped ``/clock/time`` running from ``CLOCK_START`` at real speed, and its time now.""" + started = time.monotonic() + + def virtual_now() -> datetime.datetime: + return CLOCK_START + datetime.timedelta(seconds=time.monotonic() - started) + + class Clock(BaseHTTPRequestHandler): + def do_GET(self) -> None: + body = json.dumps({"virtual_time": iso(virtual_now())}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args) -> None: + return None + + server = ThreadingHTTPServer(("127.0.0.1" if host in ("127.0.0.1", "localhost") else "0.0.0.0", 0), Clock) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + address = f"[{host}]" if ":" in host else host + yield f"http://{address}:{server.server_port}/clock/time", virtual_now + finally: + server.shutdown() + server.server_close() + + +_MARK = {"passed": "PASS", "failed": "FAIL", "skipped": "----"} + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="python -m agentenv_protocol.website.conformance", + description="Check a running site's server against urn:agentenv:website/v1.") + parser.add_argument("url", help="where the server answers, such as http://localhost:8000") + parser.add_argument("--clock-host", default="127.0.0.1", help="the address the server reaches this checker at") + parser.add_argument("--timeout", type=float, default=30, help="seconds for each request") + args = parser.parse_args(argv) + try: + results = check(args.url, clock_host=args.clock_host, timeout=args.timeout) + except httpx.TransportError as e: + print(f"couldn't reach {args.url}: {type(e).__name__}: {e}", file=sys.stderr) + return 2 + for result in results: + print(f"{_MARK[result.status]} {result.name}{': ' + result.detail if result.detail else ''}") + failed = sum(not r.passed for r in results) + print(f"{args.url}: {len(results) - failed} passed or skipped, {failed} failed") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py b/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py new file mode 100644 index 000000000..31663b55a --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py @@ -0,0 +1,450 @@ +"""WebsiteEnvironment: an AgentEnvEnvironment that is a site's server, meeting ``urn:agentenv:website/v1``. It serves +plain HTTP on ``PORT`` for any host name: its ``@api`` routes, its static folder at ``/`` with ``index.html`` for the +pages a single-page app routes itself, its health path, and the data plane and card the base class serves. Every answer +carries ``X-Universe-Now``, the universe's time, which ``self.now()`` reads and ``clock/v1``'s ``sync_time`` ties to +the gateway's virtual clock. Without data hooks of its own a site keeps its state in ``self.store``. + +Two routes are built in, each replaced by a site's own ``@api`` route at its very path and answering before one whose +path parameters would also match it: ``/api/session``, which a frontend calls to learn who is signed in, signing the +acting user in when nobody is (GET), or to sign out (DELETE); and, for a store-backed site, ``/go``, its +``{initial_state, current_state, state_diff}`` at the path ``website/v1``'s ``params.state.path`` names, which a reader +of a card without ``state`` falls back to. A store-backed site answers ``website/v1``'s ``state`` with the same +``WebsiteState``, among agentenv's own routes, which is how the website state verifier reads it.""" +from __future__ import annotations + +import inspect +import json +import os +import re +from pathlib import Path +from typing import Any, ClassVar, NamedTuple, Optional + +from starlette.applications import Starlette +from starlette.datastructures import MutableHeaders +from starlette.middleware import Middleware +from starlette.requests import Request +from starlette.responses import FileResponse, JSONResponse, Response +from starlette.routing import Route + +from ..agent_env_environment import ( + _OP_ATTR, + AgentEnvEnvironment, + AgentEnvStarletteApplication, + _marked_members, + _maybe_await, + add_data, + extension, + get_data, + reset_data, +) +from ..types import DataPart, EnvironmentExtension +from .clock import CLOCK_EXTENSION_URI, UNIVERSE_NOW_HEADER, UniverseClock, clock_patch_script, iso +from .errors import NotFound +from .readiness import WEBSITE_EXTENSION_URI, WEBSITE_READY_OP, WebsiteReadiness +from .routes import ApiContext, ApiRoute, endpoint, error_response, openapi, openapi_path, request_role, reserved_path, routes_of +from .session import SESSION_COOKIE, SessionSigner, expires_at +from .state import WEBSITE_STATE_OP, WebsiteState +from .store import Collections + +DEFAULT_HEALTH_PATH = "/api/health" +SESSION_PATH = "/api/session" +STATE_PATH = "/go" +_WEBSITE_ATTR = "_aee_website" +_READY_ATTR = "_aee_website_ready" +_DEFAULT_HOOK_ATTR = "_aee_default_store_hook" +_DEFAULT_HOOKS = {"reset_data": "_store_reset", "add_data": "_store_add", "get_data": "_store_get"} +_HEAD = re.compile(rb"]*>", re.IGNORECASE) + + +class WebsiteDeclaration(NamedTuple): + """What ``@website`` declares: the site's domains, its static folder and seed data file (relative to the class's + file), its health path, and whether its HTML pages read the universe's time.""" + + domains: tuple[str, ...] = () + static: Optional[str] = None + health: str = DEFAULT_HEALTH_PATH + clock_patch: bool = False + data: Optional[str] = None + + +def website(_cls: type | None = None, *, domains: list[str] | tuple[str, ...] = (), static: str | None = None, + health: str = DEFAULT_HEALTH_PATH, clock_patch: bool = False, data: str | None = None) -> Any: + """Declare a WebsiteEnvironment's site. ``data`` names a JSON file the default store starts with.""" + if _cls is not None: + raise TypeError("use @website(...) with parentheses") + if not health.startswith("/") or health == "/" or reserved_path(health): + raise ValueError(f"health is a path of its own under '/', not {health!r}") + + def deco(cls: type) -> type: + if not (isinstance(cls, type) and issubclass(cls, WebsiteEnvironment)): + raise TypeError(f"@website declares a WebsiteEnvironment subclass, not {cls!r}") + setattr(cls, _WEBSITE_ATTR, WebsiteDeclaration(tuple(domains), static, health, clock_patch, data)) + _check_health(cls) + return cls + + return deco + + +def ready_check(fn: Any) -> Any: + """Mark a method that says whether the site is usable yet: the health path answers 200 once every one returns a + truthy value, and 503 while one returns a falsy one or raises.""" + setattr(fn, _READY_ATTR, True) + return fn + + +class WebsiteEnvironment(AgentEnvEnvironment): + """Subclass it, declare the site with ``@website(...)``, mark its API with ``@api``, and ``serve()`` it. A + subclass's ``__init__`` calls ``super().__init__()``. ``app`` may be a Starlette or FastAPI app answering what + ``@api`` and the static folder don't (streaming, uploads); its lifespan is not run.""" + + app: ClassVar[Any] = None + asgi: Any = None + + def __init_subclass__(cls, **kwargs: Any) -> None: + super().__init_subclass__(**kwargs) + routes_of(cls) + own = {getattr(member, _OP_ATTR) for klass in cls.__mro__ for member in vars(klass).values() + if getattr(member, _OP_ATTR, None) is not None and not getattr(member, _DEFAULT_HOOK_ATTR, False)} + for op, name in _DEFAULT_HOOKS.items(): + if op in own: + setattr(cls, name, None) + + def __init__(self) -> None: + declaration = self.declaration() + self.store = Collections() + if declaration.data is not None: + self.store.load(json.loads(self._site_path(declaration.data).read_text(encoding="utf-8"))) + self.store.mark_initial() + self.clock = UniverseClock() + self._signer = SessionSigner() + + @classmethod + def declaration(cls) -> WebsiteDeclaration: + return getattr(cls, _WEBSITE_ATTR, None) or WebsiteDeclaration() + + @classmethod + def routes(cls) -> list[ApiRoute]: + return routes_of(cls) + + @property + def database_url(self) -> str | None: + """Where the site keeps its task state: its own schema of the env's database.""" + return os.environ.get("DATABASE_URL") + + @property + def uses_store(self) -> bool: + """Whether the site's data plane exports ``self.store``: it declares no ``@get_data`` of its own.""" + return type(self)._store_get is not None + + def now(self) -> Any: + """The universe's time, an aware UTC datetime: the gateway's virtual clock once synced, else wall time.""" + return self._clock().now() + + @property + def timezone(self) -> str | None: + """The universe's time zone, an IANA name such as ``America/Chicago``, as the last ``sync_time`` named it; + None when it named none. ``now()`` is UTC either way.""" + return self._clock().timezone + + def acting_user(self) -> str | None: + """Who a browser with no session is signed in as: the store's ``current_user`` unless a site says else.""" + return self.store.current_user if hasattr(self, "store") else None + + def context(self, request: Request) -> ApiContext: + """The ``ApiContext`` of ``request``, for a mounted ``app``'s routes as for ``@api`` ones.""" + session = self._session_signer().decode(request.cookies.get(SESSION_COOKIE)) + user = session["user"] if session is not None else self.acting_user() + return ApiContext(request, user, request_role(request), session, self._session_signer()) + + def create_app(self) -> Starlette: + if self._mounted: + raise RuntimeError(f"{type(self).__name__} is already mounted") + declaration = self.declaration() + _check_health(type(self)) + api_routes = self.routes() + taken = {openapi_path(r.path) for r in api_routes} + root = self._site_path(declaration.static) if declaration.static is not None else None + asgi = Starlette(middleware=[Middleware(_Forwarded), Middleware(_Universe, env=self)]) + AgentEnvStarletteApplication(self._build_card(), self).add_routes_to_app(asgi) + asgi.router.routes.append(Route(declaration.health, self._health, methods=["GET"])) + if SESSION_PATH not in taken: + asgi.router.routes.append(Route(SESSION_PATH, self._session, methods=["GET", "DELETE"])) + if self._serves_state_path(): + asgi.router.routes.append(Route(STATE_PATH, self._state, methods=["GET"])) + asgi.router.routes.extend(Route(r.path, _unreserved(endpoint(getattr(self, r.fn.__name__), r, self.context)), + methods=[r.method]) for r in api_routes) + asgi.router.default = _Fallback(root, type(self).app) + self.asgi = asgi + self._mounted = True + return asgi + + def mount(self, app: Any) -> Any: + raise TypeError(f"{type(self).__name__} builds its own app with create_app(); set app = ... to serve more") + + def serve(self) -> None: + """Serve the site on ``PORT`` (8000 by default) with uvicorn, of the ``agent`` or ``app`` extra.""" + try: + import uvicorn + except ImportError as e: + raise ImportError("WebsiteEnvironment.serve needs uvicorn: pip install 'uvicorn>=0.30'") from e + uvicorn.run(self.asgi or self.create_app(), host=os.environ.get("HOST", "0.0.0.0"), + port=int(os.environ.get("PORT", "8000"))) + + def extension_params(self, uri: str) -> dict: + if uri == WEBSITE_EXTENSION_URI: + declaration = self.declaration() + return {"domains": list(declaration.domains), "health": declaration.health, + "routes": [r.advertised() for r in self.routes()], "session_cookie": SESSION_COOKIE, + "session_path": SESSION_PATH, **({"state": {"path": STATE_PATH}} if self._serves_state_path() else {})} + return super().extension_params(uri) + + def served_extensions(self, extensions: list[EnvironmentExtension]) -> list[EnvironmentExtension]: + """The extensions the base class serves, ``website/v1``'s ``state`` only for a store-backed site: one with data + hooks of its own keeps its state where ``self.store`` can't see it.""" + served = super().served_extensions(extensions) + if self.uses_store: + return served + return [_without_method(e, WEBSITE_STATE_OP) if e.uri == WEBSITE_EXTENSION_URI else e for e in served] + + @extension(CLOCK_EXTENSION_URI, description="Sync this site to the gateway's virtual clock.") + async def sync_time(self, env_get_time_url: str, timezone: Optional[str] = None) -> dict: + """Follow the gateway's virtual time at ``env_get_time_url`` from now on, in ``timezone`` when the task names + one.""" + virtual = await self._clock().sync(env_get_time_url, timezone.strip() if timezone else None) + return {"env_get_time_url": env_get_time_url, "gateway_virtual_time": iso(virtual), + **({"timezone": self.timezone} if self.timezone else {})} + + @extension(WEBSITE_EXTENSION_URI, op="openapi", method="GET", description=( + "A site's server: domains, its health path and its @api routes by name; openapi returns the OpenAPI document " + "of its routes, ready whether the site is usable now and why not, state the task state it is in (its " + "initial and current state and their diff), and session the cookies a browser adds to start signed in as user, " + "or signed out without one.")) + async def _website_openapi(self) -> dict: + name = self._application.environment_card.name if self._application is not None else type(self).__name__ + return openapi(name, self.routes()) + + @extension(WEBSITE_EXTENSION_URI, op=WEBSITE_READY_OP, method="GET") + async def _website_ready(self) -> dict: + """Whether the site is usable now, a ``WebsiteReadiness`` by its ``@ready_check`` methods: ``problems`` names + each not yet true. A deploy reads it through the card, as ``computer-app/v1``'s ``ready`` of an app.""" + problems = await self._not_ready() + return WebsiteReadiness(ready=not problems, problems=problems).model_dump() + + @extension(WEBSITE_EXTENSION_URI, op=WEBSITE_STATE_OP, method="GET") + async def _website_state(self, sid: Optional[str] = None) -> dict: + """The site's task state, a ``WebsiteState`` of ``self.store``: its state when the task started, now, and what + changed between them. A store-backed site keeps one session, so ``sid`` names no other.""" + return WebsiteState.model_validate(self.store.state()).model_dump() + + @extension(WEBSITE_EXTENSION_URI, op="session") + async def _website_session(self, user: Optional[str] = None) -> dict: + cookie = self._session_signer().cookie(user) + return {"user": user, "cookies": [cookie], "expires_at": expires_at(cookie["expires"])} + + @reset_data + async def _store_reset(self) -> None: + self.store.reset() + + @add_data + async def _store_add(self, parts: list) -> None: + self.store.load_parts(parts) + self.store.mark_initial() + + @get_data + async def _store_get(self) -> list: + return [DataPart(data=self.store.export())] + + async def _not_ready(self) -> list[str]: + return [check.__name__ for check, _marker in _marked_members(self, _READY_ATTR) if not await _ready(check)] + + async def _health(self, request: Request) -> Response: + waiting = await self._not_ready() + if waiting: + return JSONResponse({"status": "starting", "waiting_for": waiting}, status_code=503) + return JSONResponse({"status": "ok"}) + + async def _session(self, request: Request) -> Response: + context = self.context(request) + if request.method == "DELETE": + context.sign_out() + elif context.session is None and context.user is not None: + context.sign_in(context.user) + return context.apply(JSONResponse({"user": context.user, "role": context.role, + "signed_in": context.user is not None})) + + def _serves_state_path(self) -> bool: + """Whether the site serves its state at ``STATE_PATH``: it is store-backed and no ``@api`` route takes the path.""" + return self.uses_store and STATE_PATH not in {openapi_path(r.path) for r in self.routes()} + + async def _state(self, request: Request) -> Response: + return JSONResponse(self.store.state()) + + def _clock(self) -> UniverseClock: + if not hasattr(self, "clock"): + raise TypeError(f"{type(self).__qualname__}.__init__ must call super().__init__()") + return self.clock + + def _session_signer(self) -> SessionSigner: + if not hasattr(self, "_signer"): + raise TypeError(f"{type(self).__qualname__}.__init__ must call super().__init__()") + return self._signer + + @classmethod + def _site_path(cls, path: str) -> Path: + given = Path(path) + return given if given.is_absolute() else Path(inspect.getfile(cls)).resolve().parent / given + + +for _hook in _DEFAULT_HOOKS.values(): + setattr(getattr(WebsiteEnvironment, _hook), _DEFAULT_HOOK_ATTR, True) + + +def _without_method(extension: EnvironmentExtension, method: str) -> EnvironmentExtension: + params = dict(extension.params or {}) + params["methods"] = {name: entry for name, entry in (params.get("methods") or {}).items() if name != method} + return extension.model_copy(update={"params": params}) + + +async def _ready(check: Any) -> bool: + try: + return bool(await _maybe_await(check())) + except Exception: + return False + + +def _check_health(cls: type) -> None: + health = cls.declaration().health + clash = [r.name for r in routes_of(cls) if r.method == "GET" and openapi_path(r.path) == health] + if clash: + raise ValueError(f"{cls.__name__}'s health path {health} is served by @api {clash[0]}; use @ready_check") + + +class _Forwarded: + """Puts the address the browser reached the site at in the scope, from ``X-Forwarded-Host`` and + ``X-Forwarded-Proto``, before anything routes it, so redirects, ``request.url`` and a mounted app's ``url_for`` + name that address and not the internal one the proxy reached the server at.""" + + def __init__(self, app: Any) -> None: + self.app = app + + async def __call__(self, scope: dict, receive: Any, send: Any) -> None: + if scope["type"] in ("http", "websocket"): + headers = dict(scope["headers"]) + host = _first(headers.get(b"x-forwarded-host")) + proto = _first(headers.get(b"x-forwarded-proto")).lower() + scope = dict(scope) + if host: + scope["headers"] = [(k, v) for k, v in scope["headers"] if k != b"host"] + [(b"host", host.encode())] + if proto in ("http", "https"): + scope["scheme"] = proto if scope["type"] == "http" else {"http": "ws", "https": "wss"}[proto] + await self.app(scope, receive, send) + + +def _first(value: bytes | None) -> str: + return value.decode("latin-1").split(",")[0].strip() if value else "" + + +class _Universe: + """Stamps ``X-Universe-Now`` on every answer, and with ``clock_patch`` puts the clock script in its HTML.""" + + def __init__(self, app: Any, env: WebsiteEnvironment) -> None: + self.app, self.env = app, env + + async def __call__(self, scope: dict, receive: Any, send: Any) -> None: + if scope["type"] != "http": + return await self.app(scope, receive, send) + clock = self.env._clock() + await clock.refresh() + patch = self.env.declaration().clock_patch and scope["method"] != "HEAD" + held: dict = {} + + async def stamped(message: dict) -> None: + if message["type"] == "http.response.start": + headers = MutableHeaders(scope=message) + now = clock.now() + headers[UNIVERSE_NOW_HEADER] = iso(now) + if patch and headers.get("content-type", "").startswith("text/html") and "content-encoding" not in headers: + held.update(start=message, body=[], now=now) + return + elif held and message["type"] == "http.response.body": + held["body"].append(message.get("body", b"")) + if message.get("more_body"): + return + body = _with_script(b"".join(held["body"]), clock_patch_script(held["now"])) + MutableHeaders(scope=held["start"])["content-length"] = str(len(body)) + await send(held["start"]) + message = {"type": "http.response.body", "body": body} + await send(message) + + await self.app(scope, receive, stamped) + + +def _with_script(html: bytes, script: bytes) -> bytes: + head = _HEAD.search(html) + return html[:head.end()] + script + html[head.end():] if head else script + html + + +def _unreserved(handler: Any) -> Any: + """``handler``, an ``@api`` route's endpoint, answering 404 at the data plane's and card's paths, so a broad route + (``/{rest:path}``) never serves a page there; the routes agentenv registers at those paths answer before it.""" + async def answer(request: Request) -> Response: + if reserved_path(request.scope["path"]): + return error_response(NotFound("not found")) + return await handler(request) + return answer + + +class _Fallback: + """What no route answers: a file of the static folder, then the mounted app, then for a page's path the folder's + ``index.html``; the data plane's and card's paths answer 404 however they're spelled.""" + + def __init__(self, root: Path | None, app: Any) -> None: + self.root = root.resolve() if root is not None else None + self.app = app + + async def __call__(self, scope: dict, receive: Any, send: Any) -> None: + if scope["type"] != "http": + if self.app is not None: + await self.app(scope, receive, send) + return + path, reading = scope["path"], scope["method"] in ("GET", "HEAD") + if reserved_path(path): + return await error_response(NotFound("not found"))(scope, receive, send) + if reading and (file := self._file(path)) is not None: + return await FileResponse(file)(scope, receive, send) + index = self._file("/") if reading and _page(path) else None + if self.app is not None: + return await self._app_or(index, scope, receive, send) + if index is not None: + return await FileResponse(index)(scope, receive, send) + await error_response(NotFound("not found"))(scope, receive, send) + + def _file(self, path: str) -> Path | None: + if self.root is None: + return None + candidate = (self.root / path.lstrip("/")).resolve() + if candidate.is_dir(): + candidate = (candidate / "index.html").resolve() + if not candidate.is_relative_to(self.root): + return None + return candidate if candidate.is_file() else None + + async def _app_or(self, index: Path | None, scope: dict, receive: Any, send: Any) -> None: + """The mounted app's answer, or ``index`` instead of the app's 404 when there is one.""" + missed = False + + async def unless_missed(message: dict) -> None: + nonlocal missed + if message["type"] == "http.response.start" and message["status"] == 404 and index is not None: + missed = True + if not missed: + await send(message) + + await self.app(scope, receive, unless_missed) + if missed: + await FileResponse(index)(scope, receive, send) + + +def _page(path: str) -> bool: + """Whether ``path`` is a page a single-page app routes itself: not the API's, and not a file's.""" + return not (path == "/api" or path.startswith("/api/")) and "." not in path.rsplit("/", 1)[-1] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/errors.py b/packages/agentenv-protocol/src/agentenv_protocol/website/errors.py new file mode 100644 index 000000000..13e349b88 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/errors.py @@ -0,0 +1,39 @@ +"""Errors an ``@api`` method raises to answer with an HTTP status: the body is ``{"error": message, "code": code}``.""" +from __future__ import annotations + + +class ApiError(Exception): + """An answer other than success, ``status`` with ``{"error": message, "code": code}``; a subclass sets both + defaults, and either may be given per raise.""" + + status: int = 500 + code: str = "error" + + def __init__(self, message: str, *, code: str | None = None, status: int | None = None) -> None: + super().__init__(message) + self.message = message + self.code = code or type(self).code + self.status = status or type(self).status + + def body(self) -> dict: + return {"error": self.message, "code": self.code} + + +class BadRequest(ApiError): + status, code = 400, "bad_request" + + +class Unauthorized(ApiError): + status, code = 401, "unauthorized" + + +class Forbidden(ApiError): + status, code = 403, "forbidden" + + +class NotFound(ApiError): + status, code = 404, "not_found" + + +class Conflict(ApiError): + status, code = 409, "conflict" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/readiness.py b/packages/agentenv-protocol/src/agentenv_protocol/website/readiness.py new file mode 100644 index 000000000..958f828e7 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/readiness.py @@ -0,0 +1,73 @@ +"""How a website env's health is determined, the one definition everything that judges it reads: a site says whether +it is usable now through its card's ``urn:agentenv:website/v1`` ``ready`` (``WEBSITE_READY_OP``), answering a +``WebsiteReadiness``. ``website_readiness`` asks one site as its card advertises it; ``env_readiness`` asks every site +of an env, a composed card's children or a single site's own card, and answers for the env as a whole; asking one +site may take ``READY_TIMEOUT_S`` by default.""" +from __future__ import annotations + +import asyncio + +import httpx +from pydantic import BaseModel, ConfigDict + +from ..client import find_extension_method, invoke_extension + +WEBSITE_EXTENSION_URI = "urn:agentenv:website/v1" +WEBSITE_READY_OP = "ready" +READY_TIMEOUT_S = 10.0 + + +class WebsiteReadiness(BaseModel): + """What a site says of its readiness, ``website/v1``'s ``ready``: whether it is ``ready``, and the ``problems`` + that keep it from being so, each a name a person reads, such as a ``@ready_check`` method's. Fields it doesn't + name are ignored rather than refused, so a reader keeps reading a newer site that says more, as ``ready`` and + ``problems`` keep their meaning. Its types are strict, as JSON's are: ``"yes"`` or ``1`` is no ``ready``.""" + + model_config = ConfigDict(extra="ignore", strict=True) + + ready: bool + problems: list[str] = [] + + +def _advertises_ready(card: dict) -> bool: + return find_extension_method(card, WEBSITE_EXTENSION_URI, WEBSITE_READY_OP) is not None + + +async def website_readiness(base_url: str, card: dict, *, timeout: float = READY_TIMEOUT_S, + headers: dict[str, str] | None = None) -> WebsiteReadiness | None: + """What the site whose ``card`` is read from ``base_url`` says of its readiness, None when the card doesn't + advertise ``website/v1``'s ``ready``. Raises httpx's error when it can't be asked, ``ExtensionCallError`` for a + non-2xx answer, and ``ValueError`` (pydantic's ``ValidationError`` among them) for an answer that isn't a + ``WebsiteReadiness``.""" + if not _advertises_ready(card): + return None + answer = await invoke_extension(base_url, card, WEBSITE_EXTENSION_URI, timeout=timeout, method=WEBSITE_READY_OP, + headers=headers) + return WebsiteReadiness.model_validate(answer) + + +async def env_readiness(base_url: str, card: dict, *, timeout: float = READY_TIMEOUT_S, + headers: dict[str, str] | None = None) -> WebsiteReadiness: + """What the env whose ``card`` is read from ``base_url`` says of its readiness, from every site it has, asked at + once: the card itself when it advertises ``website/v1``'s ``ready``, as a single site's does, its problems as the + site names them, and each child under a composed card's ``children_environments`` that advertises it, its endpoints + rooted at ``base_url`` as a gateway's are, its problems prefixed with its name. A site that can't be asked, or + answers something that isn't a ``WebsiteReadiness``, is a problem of its own, and one not ready that names no + problem is one too. A child, or a card, that doesn't advertise ``ready`` isn't counted, so an env with no site + serving it, as one of MCP servers alone, is ready with no problems.""" + sites = [("", card)] if _advertises_ready(card) else [] + sites += [(f"{child.get('name') or 'child'}: ", child) for child in card.get("children_environments") or [] + if isinstance(child, dict) and _advertises_ready(child)] + answers = await asyncio.gather(*(_problems(base_url, site, timeout, headers) for _prefix, site in sites)) + problems = [prefix + problem for (prefix, _site), found in zip(sites, answers) for problem in found] + return WebsiteReadiness(ready=not problems, problems=problems) + + +async def _problems(base_url: str, card: dict, timeout: float, headers: dict[str, str] | None) -> list[str]: + try: + readiness = await website_readiness(base_url, card, timeout=timeout, headers=headers) + except (httpx.HTTPError, ValueError) as e: + return [f"{WEBSITE_READY_OP} could not be read: {type(e).__name__}: {e}"] + if readiness.ready: + return [] + return readiness.problems or [f"{WEBSITE_READY_OP} says not ready"] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/routes.py b/packages/agentenv-protocol/src/agentenv_protocol/website/routes.py new file mode 100644 index 000000000..221f5e1b2 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/routes.py @@ -0,0 +1,369 @@ +"""A site's API: ``@api.get/post/put/patch/delete(path)`` methods, each bound from its type hints. A parameter named in +the path is a path parameter, one annotated with a pydantic model or ``dict`` is the JSON body, one annotated +``ApiContext`` receives the request's context, and every other one is a query parameter; path and query values are +coerced to their annotated types (int, float, bool, datetime, UUID, enum, Literal). What the method returns is the +JSON answer, a Starlette ``Response`` as it is; an ``ApiError`` it raises answers with its status, and a request its +parameters can't be bound from answers 400.""" +from __future__ import annotations + +import dataclasses +import datetime +import hmac +import inspect +import json +import logging +import os +import posixpath +import re +from typing import Any, Callable, Literal, NamedTuple, Optional, get_args, get_origin, get_type_hints + +from pydantic import BaseModel, TypeAdapter, ValidationError +from pydantic_core import to_jsonable_python +from starlette.requests import Request +from starlette.responses import JSONResponse, Response + +from ..agent_env_environment import _coerce_scalar, _maybe_await, _schema_for_annotation, _unwrap_annotation +from ..tool_context import DEFAULT_ROLE, caller_from_request +from ..types import RPC_PATH, WELL_KNOWN_PATH +from .errors import ApiError, BadRequest, Forbidden +from .session import SESSION_COOKIE, SessionSigner + +logger = logging.getLogger(__name__) + +_ROUTE_ATTR = "_aee_api_route" +_PATH_PARAM = re.compile(r"{(\w+)(?::\w+)?}") +Kind = Literal["path", "query", "body", "context"] + + +def reserved_path(path: str) -> bool: + """Whether ``path`` is the data plane's or the card's, however it's spelled, as the gateway decides it.""" + normalized = posixpath.normpath("/" + path.lstrip("/")) + return normalized == RPC_PATH or normalized.startswith(RPC_PATH + "/") or normalized == WELL_KNOWN_PATH + + +class ApiRoute(NamedTuple): + """One ``@api`` method: its verb, its Starlette path, its name on the card and in the OpenAPI document, the roles + it answers (any when empty), the status it succeeds with, and its function.""" + + method: str + path: str + name: str + roles: tuple[str, ...] + status: int + fn: Callable + + def advertised(self) -> dict: + return {"method": self.method, "path": openapi_path(self.path), "name": self.name, + "roles": list(self.roles) or None} + + +class _Api: + """``api.get(path, name=..., roles=[...])`` and its siblings mark a method as one route of the site's API.""" + + def get(self, path: str, **options: Any) -> Callable: + return self.route("GET", path, **options) + + def post(self, path: str, **options: Any) -> Callable: + return self.route("POST", path, **options) + + def put(self, path: str, **options: Any) -> Callable: + return self.route("PUT", path, **options) + + def patch(self, path: str, **options: Any) -> Callable: + return self.route("PATCH", path, **options) + + def delete(self, path: str, **options: Any) -> Callable: + return self.route("DELETE", path, **options) + + def route(self, method: str, path: str, *, name: str | None = None, roles: list[str] | None = None, + status: int = 200) -> Callable: + if not path.startswith("/"): + raise ValueError(f"an @api path starts with '/', not {path!r}") + if reserved_path(path): + raise ValueError(f"@api path {path!r} is under the data plane's {RPC_PATH} or the card's {WELL_KNOWN_PATH}") + + def mark(fn: Callable) -> Callable: + missing = sorted(set(_PATH_PARAM.findall(path)) - set(inspect.signature(fn).parameters)) + if missing: + raise TypeError(f"{fn.__qualname__} takes no parameter for the path's {missing}") + setattr(fn, _ROUTE_ATTR, ApiRoute(method.upper(), path, name or fn.__name__, tuple(roles or ()), status, fn)) + return fn + + return mark + + +api = _Api() + + +def routes_of(cls: type) -> list[ApiRoute]: + """The class's ``@api`` routes in the order they were first defined, each as the class resolves its name; + a name or a verb and path served twice is refused.""" + names = dict.fromkeys(name for klass in reversed(cls.__mro__) for name in vars(klass)) + routes = [route for name in names if (route := getattr(getattr(cls, name, None), _ROUTE_ATTR, None)) is not None] + for label, keys in (("name", [r.name for r in routes]), ("route", [f"{r.method} {r.path}" for r in routes])): + if twice := sorted({k for k in keys if keys.count(k) > 1}): + raise ValueError(f"{cls.__name__} serves the @api {label} {twice} twice") + return routes + + +def openapi_path(path: str) -> str: + return _PATH_PARAM.sub(r"{\1}", path) + + +@dataclasses.dataclass +class ApiContext: + """The request an ``@api`` method answers: its headers, cookies and query, the user it's signed in as (the + session cookie's, else the site's acting user), the caller's role (``request_role``), and the site's own address as the browser reached it. ``sign_in`` and ``sign_out`` set the + session cookie on the answer.""" + + request: Request + user: Optional[str] + role: str + session: Optional[dict] + _signer: SessionSigner = dataclasses.field(repr=False, default_factory=SessionSigner) + _set_cookie: list = dataclasses.field(repr=False, default_factory=list) + + @property + def headers(self) -> Any: + return self.request.headers + + @property + def cookies(self) -> dict[str, str]: + return self.request.cookies + + @property + def query(self) -> Any: + return self.request.query_params + + @property + def base_url(self) -> str: + return base_url(self.request) + + def sign_in(self, user: str) -> None: + self._set_session(user) + + def sign_out(self) -> None: + self._set_session(None) + + def _set_session(self, user: str | None) -> None: + value, expires = self._signer.encode(user) + self._set_cookie.append((value, expires)) + self.user, self.session = user, {"user": user} + + def apply(self, response: Response) -> Response: + for value, expires in self._set_cookie: + at = datetime.datetime.fromtimestamp(expires, datetime.timezone.utc) + response.set_cookie(SESSION_COOKIE, value, expires=at, path="/", httponly=True, samesite="lax") + return response + + +def base_url(request: Request) -> str: + """The site's address as the browser reached it: ``X-Forwarded-Proto`` and ``X-Forwarded-Host`` when a proxy + sent them, the request's own scheme and ``Host`` otherwise.""" + proto = request.headers.get("x-forwarded-proto", "").split(",")[0].strip() or request.url.scheme + host = (request.headers.get("x-forwarded-host", "").split(",")[0].strip() or request.headers.get("host") + or request.url.netloc) + return f"{proto}://{host}" + + +ROLE_KEY_ENV = "WEBSITE_ROLE_KEY" +ROLE_KEY_HEADER = "AgentEnv-Role-Key" + + +def request_role(request: Request) -> str: + """The role ``AgentEnv-Role`` names, believed only from a caller that also sends ``AgentEnv-Role-Key`` equal to + ``WEBSITE_ROLE_KEY``; every other caller, and every caller of a site with no key set, is ``default``. The gateway + forwards whatever role a browser sends, so without the key a page could claim any role.""" + claimed = caller_from_request(None, request.headers).role or DEFAULT_ROLE + key = os.environ.get(ROLE_KEY_ENV, "") + if claimed == DEFAULT_ROLE or not key: + return DEFAULT_ROLE + given = request.headers.get(ROLE_KEY_HEADER, "") + return claimed if hmac.compare_digest(given.encode(), key.encode()) else DEFAULT_ROLE + + +class _Param(NamedTuple): + name: str + kind: Kind + annotation: Any + default: Any + + +def plan(route: ApiRoute) -> list[_Param]: + """How each of the route's parameters is bound; at most one is the JSON body, and never on a GET.""" + hints = _hints(route.fn) + path_params = set(_PATH_PARAM.findall(route.path)) + params = [] + for name, param in list(inspect.signature(route.fn).parameters.items())[1:]: + if param.kind in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD): + continue + annotation = hints.get(name, param.annotation) + kind: Kind = ("context" if _is_context(annotation) else "path" if name in path_params + else "body" if _body_type(annotation) is not None else "query") + params.append(_Param(name, kind, annotation, param.default)) + bodies = [p.name for p in params if p.kind == "body"] + if len(bodies) > 1 or (bodies and route.method == "GET"): + raise TypeError(f"{route.fn.__qualname__}: a route takes at most one JSON body and a GET route none, " + f"not {bodies}") + return params + + +def _is_context(annotation: Any) -> bool: + if isinstance(annotation, str): + return annotation.strip("'\"").rsplit(".", 1)[-1] in ("ApiContext", "Optional[ApiContext]") + return any(t is ApiContext for t in _unwrap_annotation(annotation)) + + +def _body_type(annotation: Any) -> type | None: + for t in _unwrap_annotation(annotation): + base = get_origin(t) or t + if base is dict or (isinstance(base, type) and issubclass(base, BaseModel)): + return t + return None + + +async def bind(params: list[_Param], request: Request, context: ApiContext) -> dict: + """The method's arguments from ``request``; raises BadRequest for one that is missing or doesn't fit.""" + kwargs: dict = {} + for p in params: + if p.kind == "context": + kwargs[p.name] = context + elif p.kind == "body": + kwargs[p.name] = await _body(p, request) + else: + source = request.path_params if p.kind == "path" else request.query_params + if p.name not in source: + if p.default is inspect.Parameter.empty: + raise BadRequest(f"missing {p.kind} parameter {p.name!r}", code="invalid_params") + kwargs[p.name] = p.default + continue + many = _list_type(p.annotation) if p.kind == "query" else None + try: + kwargs[p.name] = ([_coerce_scalar(v, get_args(many)[0] if get_args(many) else None) + for v in source.getlist(p.name)] if many is not None + else _coerce_scalar(source[p.name], p.annotation)) + except (TypeError, ValueError) as e: + raise BadRequest(f"{p.kind} parameter {p.name!r}: {e}", code="invalid_params") from None + return kwargs + + +def _list_type(annotation: Any) -> Any: + """The ``list[T]`` an annotation names, Optional or Annotated or not; None when it names no list.""" + inner = _unwrap_annotation(annotation) + return inner[0] if len(inner) == 1 and (inner[0] is list or get_origin(inner[0]) is list) else None + + +async def _body(p: _Param, request: Request) -> Any: + raw = await request.body() + if not raw: + if p.default is not inspect.Parameter.empty: + return p.default + raise BadRequest("this route takes a JSON body", code="invalid_params") + try: + value = json.loads(raw) + except ValueError: + raise BadRequest("the request body is not JSON", code="invalid_params") from None + try: + return TypeAdapter(p.annotation).validate_python(value) + except ValidationError as e: + problems = "; ".join(f"{'.'.join(map(str, err['loc'])) or 'body'}: {err['msg']}" for err in e.errors()) + raise BadRequest(f"invalid body: {problems}", code="invalid_params") from None + + +def endpoint(fn: Callable, route: ApiRoute, context_for: Callable[[Request], ApiContext]) -> Callable: + """The Starlette endpoint of one route, calling the bound method ``fn``.""" + params = plan(route) + + async def handle(request: Request) -> Response: + context = None + try: + context = context_for(request) + if route.roles and context.role not in route.roles: + raise Forbidden(f"role {context.role!r} may not call {route.name}") + result = await _maybe_await(fn(**await bind(params, request, context))) + if isinstance(result, Response): + response = result + elif not _has_body(route.status): + response = Response(status_code=route.status) + else: + response = JSONResponse(to_jsonable_python(result), status_code=route.status) + except ApiError as e: + response = error_response(e) + except Exception: + logger.exception("@api %s failed", route.name) + response = error_response(ApiError("internal error", code="internal")) + return context.apply(response) if context is not None else response + + return handle + + +def _has_body(status: int) -> bool: + """Whether an answer with ``status`` may carry a body: not a 1xx, 204 or 304.""" + return status >= 200 and status not in (204, 304) + + +def error_response(error: ApiError) -> Response: + return JSONResponse(error.body(), status_code=error.status) + + +def openapi(title: str, routes: list[ApiRoute]) -> dict: + """The OpenAPI 3.1 document of ``routes``, each operation's id its route's name; the models their bodies and + answers name, nested ones too, are its ``components.schemas``, which their schemas reference. Bodies are + described as they are validated and answers as they are serialized; a model the two modes see differently is in + ``components.schemas`` twice, as ``-Input`` and ``-Output``.""" + paths: dict = {} + schemas: list = [] + for route in routes: + doc = inspect.getdoc(route.fn) or "" + operation: dict = {"operationId": route.name, "parameters": [], + "responses": {str(route.status): {"description": "Success"}}} + if doc: + operation["summary"], operation["description"] = doc.splitlines()[0], doc + if route.roles: + operation["x-agentenv-roles"] = list(route.roles) + for p in plan(route): + if p.kind in ("path", "query"): + schema = _schema_for_annotation(p.annotation) + if p.default is not inspect.Parameter.empty: + schema["default"] = to_jsonable_python(p.default) + operation["parameters"].append({"name": p.name, "in": p.kind, "schema": schema, + "required": p.kind == "path" or p.default is inspect.Parameter.empty}) + elif p.kind == "body": + media: dict = {} + operation["requestBody"] = {"required": p.default is inspect.Parameter.empty, + "content": {"application/json": media}} + schemas.append((media, p.annotation, "validation")) + returns = _hints(route.fn).get("return") + if _has_body(route.status) and _adapter(returns) is not None: + media = {} + operation["responses"][str(route.status)]["content"] = {"application/json": media} + schemas.append((media, returns, "serialization")) + paths.setdefault(openapi_path(route.path), {})[route.method.lower()] = operation + document: dict = {"openapi": "3.1.0", "info": {"title": title, "version": "1.0"}, "paths": paths} + adapters = [(i, mode, adapter) for i, (_media, annotation, mode) in enumerate(schemas) + if (adapter := _adapter(annotation)) is not None] + found, definitions = TypeAdapter.json_schemas(adapters, ref_template="#/components/schemas/{model}") + for i, (media, _annotation, mode) in enumerate(schemas): + media["schema"] = found.get((i, mode), {}) + if definitions.get("$defs"): + document["components"] = {"schemas": definitions["$defs"]} + return document + + +def _hints(fn: Callable) -> dict: + try: + return get_type_hints(fn) + except Exception: + return {} + + +def _adapter(annotation: Any) -> TypeAdapter | None: + """The adapter of a type a JSON schema can describe; None for no type, None itself, or one it can't describe.""" + if annotation is None or annotation is type(None) or annotation is inspect.Parameter.empty: + return None + try: + adapter = TypeAdapter(annotation) + adapter.json_schema() + except Exception: + return None + return adapter diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/session.py b/packages/agentenv-protocol/src/agentenv_protocol/website/session.py new file mode 100644 index 000000000..20e4ad36f --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/session.py @@ -0,0 +1,60 @@ +"""A site's signed session cookie: who the browser is signed in as, or that it signed out, signed with HMAC-SHA256 +under ``WEBSITE_SESSION_SECRET``, or a secret of the process's own when that isn't set. The cookie names no domain, +so it holds under whatever host the site is reached at.""" +from __future__ import annotations + +import base64 +import binascii +import datetime +import hashlib +import hmac +import json +import os +import secrets +import time + +SESSION_COOKIE = "agentenv_session" +SECRET_ENV = "WEBSITE_SESSION_SECRET" +SESSION_TTL_S = 7 * 24 * 3600 +SECRET_BYTES = 32 + + +class SessionSigner: + def __init__(self, secret: str | bytes | None = None) -> None: + secret = secret if secret is not None else os.environ.get(SECRET_ENV) + self._key = (secret.encode() if isinstance(secret, str) else secret) or secrets.token_bytes(SECRET_BYTES) + + def encode(self, user: str | None, ttl_s: int = SESSION_TTL_S) -> tuple[str, int]: + """The cookie value for ``user`` signed in, or signed out when None, and when it expires, in epoch + seconds.""" + expires = int(time.time()) + ttl_s + payload = base64.urlsafe_b64encode(json.dumps({"u": user, "exp": expires}).encode()).decode().rstrip("=") + return f"{payload}.{self._sign(payload)}", expires + + def decode(self, value: str | None) -> dict | None: + """The session a cookie value carries, ``{"user": ...}``; None for one that is absent, forged or expired.""" + payload, _, signature = (value or "").partition(".") + if not payload or not hmac.compare_digest(signature.encode("utf-8", "surrogateescape"), + self._sign(payload).encode()): + return None + try: + data = json.loads(base64.urlsafe_b64decode(payload + "=" * (-len(payload) % 4))) + except (ValueError, binascii.Error): + return None + if not isinstance(data, dict) or not isinstance(data.get("exp"), int) or data["exp"] < time.time(): + return None + return {"user": data.get("u")} + + def cookie(self, user: str | None, ttl_s: int = SESSION_TTL_S) -> dict: + """The cookie a browser sets to be signed in as ``user``, as a browser automation adds one; its domain is + left to whoever adds it.""" + value, expires = self.encode(user, ttl_s) + return {"name": SESSION_COOKIE, "value": value, "path": "/", "expires": expires, "httpOnly": True, + "sameSite": "Lax"} + + def _sign(self, payload: str) -> str: + return hmac.new(self._key, payload.encode("utf-8", "surrogateescape"), hashlib.sha256).hexdigest() + + +def expires_at(epoch: int) -> str: + return datetime.datetime.fromtimestamp(epoch, datetime.timezone.utc).isoformat().replace("+00:00", "Z") diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/state.py b/packages/agentenv-protocol/src/agentenv_protocol/website/state.py new file mode 100644 index 000000000..77e0b5558 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/state.py @@ -0,0 +1,56 @@ +"""What a website env's task state is, the one definition everything that grades a site reads: a site gives the state a +task left it in through its card's ``urn:agentenv:website/v1`` ``state`` (``WEBSITE_STATE_OP``), answering a +``WebsiteState``, its state when the task started, its state now and what changed between them. ``website_state`` +asks one site as its card advertises it, of one session when the site keeps several; ``website_sites`` names every +site of an env, a single site's own card or a composed card's website children. The op is one of agentenv's own +routes, under the data plane's path, so a browser on the site's route never reads what grades it.""" +from __future__ import annotations + +from typing import Any, Optional + +from pydantic import BaseModel, ConfigDict + +from ..client import ChildEnv, children, find_extension, find_extension_method, invoke_extension +from .readiness import WEBSITE_EXTENSION_URI + +WEBSITE_STATE_OP = "state" +STATE_TIMEOUT_S = 30.0 + + +class WebsiteState(BaseModel): + """What a site says of its task state, ``website/v1``'s ``state``: ``initial_state``, the state the task started + from, ``current_state``, the state now, and ``state_diff``, what changed between them, each a JSON object or None + when the site gives none. Fields it doesn't name are ignored rather than refused, so a reader keeps reading a newer + site that says more.""" + + model_config = ConfigDict(extra="ignore") + + initial_state: Optional[dict[str, Any]] = None + current_state: Optional[dict[str, Any]] = None + state_diff: Optional[dict[str, Any]] = None + + +def advertises_state(card: dict) -> bool: + """Whether ``card`` advertises ``website/v1``'s ``state``.""" + return find_extension_method(card, WEBSITE_EXTENSION_URI, WEBSITE_STATE_OP) is not None + + +def website_sites(card: dict) -> list[ChildEnv]: + """Every site of the env whose card is ``card``: the card itself when it carries ``website/v1``, as a single site's + does, else each child of a composed card that carries it, in its order; none for an env with no site.""" + if find_extension(card, WEBSITE_EXTENSION_URI) is not None: + return [ChildEnv(name=card.get("name"), url=card.get("url"), card=card)] + return [child for child in children(card) if child.find_extension(WEBSITE_EXTENSION_URI) is not None] + + +async def website_state(base_url: str, card: dict, *, sid: str | None = None, timeout: float = STATE_TIMEOUT_S, + headers: dict[str, str] | None = None) -> WebsiteState | None: + """What the site whose ``card`` is read from ``base_url`` says of its task state, of session ``sid`` when it's + given, for a site that keeps several; None when the card doesn't advertise ``website/v1``'s ``state``. Raises + httpx's error when it can't be asked, ``ExtensionCallError`` for a non-2xx answer, and ``ValueError`` (pydantic's + ``ValidationError`` among them) for an answer that isn't a ``WebsiteState``.""" + if not advertises_state(card): + return None + answer = await invoke_extension(base_url, card, WEBSITE_EXTENSION_URI, {"sid": sid} if sid is not None else None, + timeout=timeout, method=WEBSITE_STATE_OP, headers=headers) + return WebsiteState.model_validate(answer) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/store.py b/packages/agentenv-protocol/src/agentenv_protocol/website/store.py new file mode 100644 index 000000000..ce21ad84c --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/store.py @@ -0,0 +1,134 @@ +"""The default state store of a simple site: named collections of JSON objects, each with an ``id``. Its data is a JSON +object of collections by name, beside which a string ``current_user`` names the user a browser starts signed in as. +It remembers the state it was last loaded with, so a verifier can read the initial state, the current one and the +difference between them.""" +from __future__ import annotations + +import base64 +import copy +import json +from pathlib import Path +from typing import Any, Iterable +from urllib.parse import unquote, urlparse + +from ..types import DataPart, FilePart, Part, TextPart + +CURRENT_USER = "current_user" + + +class Collections: + """Named lists of JSON objects; ``store["orders"]`` is the list itself, created empty on first use.""" + + def __init__(self, data: dict | None = None) -> None: + self._collections: dict[str, list[dict]] = {} + self.current_user: str | None = None + self.initial: dict = {} + if data: + self.load(data) + self.mark_initial() + + def __getitem__(self, name: str) -> list[dict]: + return self._collections.setdefault(name, []) + + def names(self) -> list[str]: + return list(self._collections) + + def get(self, name: str, id: Any) -> dict | None: + return next((doc for doc in self[name] if doc.get("id") == id), None) + + def find(self, name: str, **match: Any) -> list[dict]: + return [doc for doc in self[name] if all(doc.get(k) == v for k, v in match.items())] + + def insert(self, name: str, doc: dict) -> dict: + """Add ``doc``, giving it the next integer ``id`` of its collection when it has none.""" + doc = dict(doc) + if doc.get("id") is None: + doc["id"] = 1 + max((d["id"] for d in self[name] if isinstance(d.get("id"), int)), default=0) + self[name].append(doc) + return doc + + def update(self, name: str, id: Any, **fields: Any) -> dict | None: + doc = self.get(name, id) + if doc is not None: + doc.update(fields) + return doc + + def delete(self, name: str, id: Any) -> bool: + doc = self.get(name, id) + if doc is not None: + self[name].remove(doc) + return doc is not None + + def load(self, data: dict) -> None: + """Add ``data``'s collections to the store's, and take its ``current_user`` when it names one.""" + if not isinstance(data, dict): + raise ValueError("a store's data is a JSON object of collections by name") + for name, docs in data.items(): + if name == CURRENT_USER: + self.current_user = docs + elif isinstance(docs, list) and all(isinstance(d, dict) for d in docs): + self[name].extend(copy.deepcopy(docs)) + else: + raise ValueError(f"collection {name!r} is not a list of JSON objects") + + def load_parts(self, parts: Iterable[Part]) -> None: + for part in parts: + self.load(_part_data(part)) + + def reset(self) -> None: + self._collections.clear() + self.current_user = None + self.initial = {} + + def export(self) -> dict: + data: dict = copy.deepcopy(self._collections) + if self.current_user is not None: + data[CURRENT_USER] = self.current_user + return data + + def mark_initial(self) -> None: + self.initial = self.export() + + def diff(self) -> dict: + """Each collection that changed since the initial state, as its ``added``, ``removed`` and ``changed`` + objects, matched by ``id``; a changed one as its ``id``, ``before`` and ``after``.""" + current = self.export() + changes: dict = {} + for name in dict.fromkeys([*self.initial, *current]): + if name == CURRENT_USER: + continue + before = {_key(d): d for d in self.initial.get(name, [])} + after = {_key(d): d for d in current.get(name, [])} + entry = { + "added": [d for k, d in after.items() if k not in before], + "removed": [d for k, d in before.items() if k not in after], + "changed": [{"id": d.get("id"), "before": before[k], "after": d} + for k, d in after.items() if k in before and before[k] != d], + } + if any(entry.values()): + changes[name] = entry + return changes + + def state(self) -> dict: + return {"initial_state": copy.deepcopy(self.initial), "current_state": self.export(), "state_diff": self.diff()} + + +def _key(doc: dict) -> str: + return json.dumps(doc.get("id")) if doc.get("id") is not None else json.dumps(doc, sort_keys=True) + + +def _part_data(part: Part) -> dict: + """A part's JSON: a data part's data, a text part's text, or a file part's bytes or ``file://`` file.""" + if isinstance(part, DataPart): + return part.data + if isinstance(part, TextPart): + return json.loads(part.text) + if isinstance(part, FilePart): + file = part.file + if getattr(file, "bytes", None) is not None: + return json.loads(base64.b64decode(file.bytes)) + uri = urlparse(file.uri) + if uri.scheme == "file": + return json.loads(Path(unquote(uri.path)).read_text(encoding="utf-8")) + raise ValueError(f"the store reads file parts by their bytes or a file:// URI, not {file.uri!r}") + raise ValueError(f"the store can't read a {type(part).__name__}") diff --git a/packages/agentenv-protocol/tests/test_website_api.py b/packages/agentenv-protocol/tests/test_website_api.py new file mode 100644 index 000000000..21b73530c --- /dev/null +++ b/packages/agentenv-protocol/tests/test_website_api.py @@ -0,0 +1,413 @@ +"""The website SDK's @api routes: binding from type hints, JSON answers, errors, roles, and what a class may not +declare.""" + +from __future__ import annotations + +import datetime +import enum +from typing import Optional, Union + +import pytest +from pydantic import BaseModel, computed_field +from starlette.responses import PlainTextResponse +from starlette.testclient import TestClient + +from agentenv_protocol.website import ( + ROLE_KEY_ENV, + ROLE_KEY_HEADER, + ApiContext, + BadRequest, + Conflict, + NotFound, + WebsiteEnvironment, + api, + reserved_path, + website, +) + + +class Color(enum.Enum): + RED = "red" + BLUE = "blue" + + +class NewItem(BaseModel): + title: str + price: float = 0 + + +@website(domains=["shop.example"]) +class Shop(WebsiteEnvironment): + def __init__(self) -> None: + super().__init__() + self.items: dict[int, dict] = {1: {"id": 1, "title": "mug", "price": 3.5}} + + @api.get("/api/items/{item_id}", name="get_item") + async def get_item(self, item_id: int) -> dict: + """Read one item.""" + if item_id not in self.items: + raise NotFound(f"no item {item_id}") + return self.items[item_id] + + @api.get("/api/items") + def list_items(self, limit: int = 10, cheap: bool = False, color: Optional[Color] = None, + since: Optional[datetime.date] = None, tags: list[str] = []) -> dict: + return {"limit": limit, "cheap": cheap, "color": color.value if color else None, + "since": since.isoformat() if since else None, "tags": tags} + + @api.post("/api/items", name="create_item", status=201) + async def create_item(self, item: NewItem) -> NewItem: + if any(i["title"] == item.title for i in self.items.values()): + raise Conflict(f"{item.title} exists") + self.items[len(self.items) + 1] = {"id": len(self.items) + 1, **item.model_dump()} + return item + + @api.patch("/api/items/{item_id}") + async def patch_item(self, item_id: int, changes: dict) -> dict: + self.items[item_id].update(changes) + return self.items[item_id] + + @api.delete("/api/items/{item_id}", roles=["admin"]) + async def delete_item(self, item_id: int) -> None: + self.items.pop(item_id, None) + + @api.put("/api/teapot") + async def teapot(self) -> None: + raise BadRequest("short and stout", code="teapot", status=418) + + @api.get("/api/whoami") + async def whoami(self, ctx: ApiContext) -> dict: + return {"user": ctx.user, "role": ctx.role, "base_url": ctx.base_url, "q": ctx.query.get("q"), + "header": ctx.headers.get("x-probe"), "cookie": ctx.cookies.get("c")} + + @api.get("/api/plain") + async def plain(self) -> PlainTextResponse: + return PlainTextResponse("as is") + + @api.get("/api/boom") + async def boom(self) -> dict: + raise RuntimeError("secret detail") + + @api.get("/api/picked") + def picked(self, ids: Optional[list[int]] = None) -> dict: + return {"ids": ids} + + @api.delete("/api/cart", status=204) + async def empty_cart(self) -> None: + return None + + +@pytest.fixture +def client() -> TestClient: + return TestClient(Shop().create_app()) + + +def test_path_query_and_body_parameters_are_bound_from_type_hints(client: TestClient) -> None: + assert client.get("/api/items/1").json() == {"id": 1, "title": "mug", "price": 3.5} + assert client.get("/api/items").json() == {"limit": 10, "cheap": False, "color": None, "since": None, "tags": []} + listed = client.get("/api/items?limit=3&cheap=yes&color=blue&since=2030-01-02&tags=a&tags=b").json() + assert listed == {"limit": 3, "cheap": True, "color": "blue", "since": "2030-01-02", "tags": ["a", "b"]} + created = client.post("/api/items", json={"title": "pen", "price": 1}) + assert created.status_code == 201 and created.json() == {"title": "pen", "price": 1.0} + assert client.patch("/api/items/1", json={"price": 4}).json()["price"] == 4 + + +def test_a_request_its_parameters_cannot_be_bound_from_answers_400(client: TestClient) -> None: + for response in (client.get("/api/items/abc"), client.get("/api/items?limit=many"), + client.get("/api/items?color=green"), client.post("/api/items", json={"price": 2}), + client.post("/api/items", content=b"not json"), client.post("/api/items"), + client.patch("/api/items/1", json=[1, 2])): + assert response.status_code == 400, response.text + assert response.json()["code"] == "invalid_params" + assert "title" in client.post("/api/items", json={"price": 2}).json()["error"] + + +class Pin(BaseModel): + lat: float + + +class Place(BaseModel): + name: str + pin: Pin + + +class Note(BaseModel): + text: str + + +@website(domains=["map.example"]) +class Map(WebsiteEnvironment): + @api.post("/api/places") + def add_place(self, place: Place) -> list[Place]: + return [place] + + @api.post("/api/marks") + def mark(self, mark: Union[Note, Pin]) -> dict: + return {"kind": type(mark).__name__} + + @api.put("/api/settings") + def settings(self, settings: Optional[dict]) -> dict: + return {"settings": settings} + + +def test_a_body_is_validated_against_its_whole_annotation() -> None: + client = TestClient(Map().create_app()) + assert client.put("/api/settings", content=b"null", headers={"content-type": "application/json"}).json() == { + "settings": None} + assert client.post("/api/marks", json={"lat": 1.5}).json() == {"kind": "Pin"} + assert client.post("/api/marks", json={"text": "hi"}).json() == {"kind": "Note"} + assert client.post("/api/marks", json={"other": 1}).status_code == 400 + + +def test_nested_models_are_the_openapi_document_s_components_its_schemas_reference() -> None: + document = TestClient(Map().create_app()).get("/agentenv/ext/website/openapi").json() + schemas = document["components"]["schemas"] + assert set(schemas) == {"Pin", "Place", "Note"} + assert schemas["Place"]["properties"]["pin"] == {"$ref": "#/components/schemas/Pin"} + body = document["paths"]["/api/places"]["post"]["requestBody"]["content"]["application/json"]["schema"] + answer = document["paths"]["/api/places"]["post"]["responses"]["200"]["content"]["application/json"]["schema"] + assert body == {"$ref": "#/components/schemas/Place"} + assert answer == {"type": "array", "items": {"$ref": "#/components/schemas/Place"}} + refs = [v for v in _walk(document) if isinstance(v, str) and v.startswith("#/")] + assert refs and all(ref.split("/")[-1] in schemas and ref.startswith("#/components/schemas/") for ref in refs) + + +class Cart(BaseModel): + prices: list[float] + + @computed_field + @property + def total(self) -> float: + return sum(self.prices) + + +@website(domains=["till.example"]) +class Till(WebsiteEnvironment): + @api.post("/api/carts") + def price(self, cart: Cart) -> Cart: + return cart + + +def test_an_answer_s_schema_describes_it_as_serialized_and_a_body_s_as_validated() -> None: + client = TestClient(Till().create_app()) + assert client.post("/api/carts", json={"prices": [1, 2]}).json() == {"prices": [1, 2], "total": 3} + document = client.get("/agentenv/ext/website/openapi").json() + schemas = document["components"]["schemas"] + operation = document["paths"]["/api/carts"]["post"] + body = operation["requestBody"]["content"]["application/json"]["schema"] + answer = operation["responses"]["200"]["content"]["application/json"]["schema"] + assert body == {"$ref": "#/components/schemas/Cart-Input"} + assert answer == {"$ref": "#/components/schemas/Cart-Output"} + assert "total" not in schemas["Cart-Input"]["properties"] and "total" in schemas["Cart-Output"]["properties"] + + +def _walk(value: object): + if isinstance(value, dict): + for v in value.values(): + yield from _walk(v) + elif isinstance(value, list): + for v in value: + yield from _walk(v) + else: + yield value + + +def test_api_errors_answer_with_their_status_and_code(client: TestClient) -> None: + missing = client.get("/api/items/9") + assert (missing.status_code, missing.json()) == (404, {"error": "no item 9", "code": "not_found"}) + assert client.post("/api/items", json={"title": "mug"}).json() == {"error": "mug exists", "code": "conflict"} + teapot = client.put("/api/teapot") + assert (teapot.status_code, teapot.json()["code"]) == (418, "teapot") + boom = client.get("/api/boom") + assert (boom.status_code, boom.json()) == (500, {"error": "internal error", "code": "internal"}) + + +def test_responses_pass_through_and_none_is_json_null(client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(ROLE_KEY_ENV, "k3y") + assert client.get("/api/plain").text == "as is" + deleted = client.delete("/api/items/1", headers={"AgentEnv-Role": "admin", ROLE_KEY_HEADER: "k3y"}) + assert (deleted.status_code, deleted.json()) == (200, None) + + +def test_an_optional_list_query_parameter_is_a_list_of_its_coerced_values(client: TestClient) -> None: + assert client.get("/api/picked?ids=1&ids=2").json() == {"ids": [1, 2]} + assert client.get("/api/picked").json() == {"ids": None} + assert client.get("/api/picked?ids=x").status_code == 400 + + +def test_a_route_declared_204_answers_with_no_body(client: TestClient) -> None: + emptied = client.delete("/api/cart") + assert (emptied.status_code, emptied.content) == (204, b"") + assert "content-length" not in emptied.headers and "content-type" not in emptied.headers + + +def test_a_204_answer_keeps_a_real_server_connection_usable() -> None: + import socket + import threading + import time + + uvicorn = pytest.importorskip("uvicorn") + server = uvicorn.Server(uvicorn.Config(Shop().create_app(), host="127.0.0.1", port=0, log_level="error", + ws="none")) + thread = threading.Thread(target=server.run, daemon=True) + thread.start() + while not server.started: + time.sleep(0.01) + port = server.servers[0].sockets[0].getsockname()[1] + try: + with socket.create_connection(("127.0.0.1", port), timeout=5) as conn: + conn.sendall(b"DELETE /api/cart HTTP/1.1\r\nHost: x\r\n\r\nGET /api/items/1 HTTP/1.1\r\nHost: x\r\n\r\n") + received = b"" + while b"mug" not in received: + chunk = conn.recv(65536) + if not chunk: + break + received += chunk + assert received.startswith(b"HTTP/1.1 204") and b"HTTP/1.1 200" in received and b"mug" in received + finally: + server.should_exit = True + thread.join(5) + + +def test_a_session_cookie_that_is_not_ascii_is_no_session(client: TestClient) -> None: + for path in ("/api/whoami", "/api/session"): + response = client.get(path, headers={"Cookie": "agentenv_session=abc.\xe9".encode("latin-1")}) + assert response.status_code == 200, response.text + assert response.json()["user"] is None + + +def test_roles_refuse_other_callers_with_403(client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(ROLE_KEY_ENV, "k3y") + for headers in ({}, {"AgentEnv-Role": "default"}, {"AgentEnv-Role": "agent", ROLE_KEY_HEADER: "k3y"}): + refused = client.delete("/api/items/1", headers=headers) + assert refused.status_code == 403 and refused.json()["code"] == "forbidden" + assert client.get("/api/items/1").status_code == 200 + assert client.delete("/api/items/1", headers={"AgentEnv-Role": "admin", ROLE_KEY_HEADER: "k3y"}).status_code == 200 + + +def test_a_role_is_believed_only_with_the_site_s_role_key(client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv(ROLE_KEY_ENV, raising=False) + claim = {"AgentEnv-Role": "admin"} + assert client.delete("/api/items/1", headers=claim).status_code == 403 + assert client.delete("/api/items/1", headers={**claim, ROLE_KEY_HEADER: ""}).status_code == 403 + monkeypatch.setenv(ROLE_KEY_ENV, "k3y") + for key in (None, "", "wrong", "k3y "): + headers = claim if key is None else {**claim, ROLE_KEY_HEADER: key} + assert client.delete("/api/items/1", headers=headers).status_code == 403 + assert client.get("/api/whoami", headers=headers).json()["role"] == "default" + assert client.get("/api/whoami", headers={**claim, ROLE_KEY_HEADER: "k3y"}).json()["role"] == "admin" + + +def test_the_context_carries_the_request_role_and_forwarded_address(client: TestClient, + monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(ROLE_KEY_ENV, "k3y") + client.cookies.set("c", "crumb") + seen = client.get("/api/whoami?q=1", headers={"x-probe": "p", "AgentEnv-Role": "agent", ROLE_KEY_HEADER: "k3y"}).json() + assert seen == {"user": None, "role": "agent", "base_url": "http://testserver", "q": "1", "header": "p", + "cookie": "crumb"} + forwarded = client.get("/api/whoami", headers={"X-Forwarded-Host": "shop.example, proxy", + "X-Forwarded-Proto": "https"}).json() + assert (forwarded["base_url"], forwarded["role"]) == ("https://shop.example", "default") + + +def test_a_redirect_names_the_forwarded_address(client: TestClient) -> None: + redirected = client.get("/api/items/", follow_redirects=False, + headers={"Host": "internal-x.invalid", "X-Forwarded-Host": "shop.example", + "X-Forwarded-Proto": "https"}) + assert (redirected.status_code, redirected.headers["location"]) == (307, "https://shop.example/api/items") + assert client.get("/api/items/", follow_redirects=False).headers["location"] == "http://testserver/api/items" + + +def test_parameterized_routes_never_take_the_data_plane_or_card() -> None: + @website(domains=["wiki.example"]) + class Wiki(WebsiteEnvironment): + @api.get("/{slug}") + def page(self, slug: str) -> dict: + return {"slug": slug} + + @api.post("/{collection}") + def add(self, collection: str) -> dict: + return {"captured": collection} + + @api.get("/.well-known/{name}") + def well_known(self, name: str) -> dict: + return {"captured": name} + + @api.get("/files/{rest:path}") + def files(self, rest: str) -> dict: + return {"rest": rest} + + client = TestClient(Wiki().create_app()) + assert client.get("/hello").json() == {"slug": "hello"} + assert client.post("/notes").json() == {"captured": "notes"} + assert client.get("/.well-known/other.json").json() == {"captured": "other.json"} + assert client.get("/files/a/b.txt").json() == {"rest": "a/b.txt"} + reset = client.post("/agentenv", json={"jsonrpc": "2.0", "id": 1, "method": "data/get", "params": {}}).json() + assert reset["id"] == 1 and "result" in reset + assert client.get("/.well-known/agent-env.json").json()["name"] == "Wiki" + assert client.get("/api/health").json() == {"status": "ok"} + + +@pytest.mark.parametrize("path", ["/agentenv", "/agentenv/ext/x", "/.well-known/agent-env.json", + "/api/../agentenv/x", "//agentenv"]) +def test_api_paths_under_the_reserved_ones_are_refused(path: str) -> None: + assert reserved_path(path) + with pytest.raises(ValueError, match="data plane"): + api.get(path) + + +def test_a_class_serving_a_name_or_route_twice_is_refused() -> None: + with pytest.raises(ValueError, match="name"): + class Twice(WebsiteEnvironment): + @api.get("/a", name="same") + def a(self) -> None: ... + + @api.get("/b", name="same") + def b(self) -> None: ... + + with pytest.raises(ValueError, match="route"): + class Again(WebsiteEnvironment): + @api.post("/a") + def a(self) -> None: ... + + @api.post("/a") + def b(self) -> None: ... + + +def test_a_route_must_take_its_path_parameters_and_at_most_one_body() -> None: + with pytest.raises(TypeError, match="item_id"): + api.get("/api/{item_id}")(lambda self: None) + + class Bodies(WebsiteEnvironment): + @api.post("/api/x") + def x(self, a: dict, b: NewItem) -> None: ... + + with pytest.raises(TypeError, match="one JSON body"): + Bodies().create_app() + + +def test_the_health_path_is_not_an_api_route() -> None: + with pytest.raises(ValueError, match="ready_check"): + @website(health="/api/up") + class Clash(WebsiteEnvironment): + @api.get("/api/up") + def up(self) -> dict: + return {} + + with pytest.raises(ValueError, match="health"): + website(health="/agentenv/health") + + +@website(domains=["shelf.example"]) +class Shelf(WebsiteEnvironment): + @api.delete("/api/books/{book_id}", status=204) + def drop(self, book_id: int) -> Pin: + return Pin(lat=1.0) + + +def test_a_bodyless_status_publishes_no_body_and_answers_none() -> None: + client = TestClient(Shelf().create_app()) + answered = client.delete("/api/books/1") + assert answered.status_code == 204 and answered.content == b"" + responses = client.get("/agentenv/ext/website/openapi").json()["paths"]["/api/books/{book_id}"]["delete"]["responses"] + assert "content" not in responses["204"] diff --git a/packages/agentenv-protocol/tests/test_website_conformance.py b/packages/agentenv-protocol/tests/test_website_conformance.py new file mode 100644 index 000000000..50f981b54 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_website_conformance.py @@ -0,0 +1,265 @@ +"""The website/v1 conformance checker against an SDK site, a hand-written server that meets the contract, and servers +that don't.""" + +from __future__ import annotations + +import datetime +from pathlib import Path + +import pytest +from starlette.applications import Starlette +from starlette.requests import Request +from starlette.responses import HTMLResponse, JSONResponse, Response +from starlette.routing import Route +from starlette.testclient import TestClient + +from agentenv_protocol.website import WebsiteEnvironment, api, website +from agentenv_protocol.website import conformance +from agentenv_protocol.website.clock import UNIVERSE_NOW_HEADER, iso + +CHECKS = ["card", "any_host", "health", "universe_now", "data_plane", "clock", "forwarded", "cookies"] + + +def statuses(results: list[conformance.Result]) -> dict[str, str]: + return {r.name: r.status for r in results} + + +@pytest.fixture +def notes(tmp_path: Path) -> WebsiteEnvironment: + (tmp_path / "index.html").write_text("hi") + + @website(domains=["notes.example"], static=str(tmp_path)) + class Notes(WebsiteEnvironment): + @api.get("/api/notes") + def notes(self) -> list: + return self.store["notes"] + + env = Notes() + env.store.load({"current_user": "ann", "notes": [{"id": 1, "text": "hello"}]}) + env.store.mark_initial() + return env + + +@pytest.fixture +def sdk_site(notes: WebsiteEnvironment) -> TestClient: + return TestClient(notes.create_app()) + + +def test_an_sdk_site_passes_every_check(sdk_site: TestClient) -> None: + results = conformance.check("http://testserver", client=sdk_site) + assert [r.name for r in results] == CHECKS + assert all(r.status == "passed" for r in results), results + assert sdk_site.post("/agentenv", json={"jsonrpc": "2.0", "id": 1, "method": "data/get"}).json()["result"][ + "parts"][0]["data"] == {"current_user": "ann", "notes": [{"id": 1, "text": "hello"}]} + + +def _handwritten(*, leak_host: bool = False, domain_cookie: bool = False, clock: bool = True, + leak_redirect: bool = False, stamp_rpc: bool = True, reset_keeps: bool = False, + get_fails_after_reset: bool = False, rpc_reply: tuple[int, dict] | None = None, + ready_answer: object = None, rpc_path: str | None = None, state_path: str | None = None, + changed: bool = False) -> Starlette: + """A server written without the SDK: the contract by hand, optionally broken. Its state is its ``state.data``; with + ``ready_answer`` its card advertises ``website/v1``'s ``ready``, which answers it. With ``rpc_path`` its card's + ``url`` names its data plane there, not at ``/agentenv``, and with ``state_path`` its ``params.state.path`` names + where it serves its state, which shows changes since the initial state when ``changed``.""" + state = {"items": ["a"], "url": None, "at": None, "reset": False} + + def now() -> datetime.datetime: + return state["at"] or datetime.datetime.now(datetime.timezone.utc) + + def stamp(response: Response) -> Response: + response.headers[UNIVERSE_NOW_HEADER] = iso(now()) + return response + + async def page(request: Request) -> Response: + host = request.headers["host"] if leak_host else request.headers.get("x-forwarded-host", request.headers["host"]) + response = stamp(HTMLResponse(f'x')) + response.set_cookie("s", "1", domain="example.com" if domain_cookie else None) + return response + + async def health(request: Request) -> Response: + return stamp(JSONResponse({"ok": True})) + + async def ready(request: Request) -> Response: + return stamp(JSONResponse(ready_answer)) + + async def items(request: Request) -> Response: + return stamp(JSONResponse(state["items"])) + + async def items_slash(request: Request) -> Response: + host = request.headers["host"] if leak_redirect else request.headers.get("x-forwarded-host", request.headers["host"]) + return stamp(Response(status_code=307, headers={"Location": f"https://{host}/items"})) + + async def card(request: Request) -> Response: + extensions = [{"uri": "urn:agentenv:website/v1", + "params": {"domains": ["h.example"], "health": "/healthz", + "routes": [{"method": "GET", "path": "/items", "name": "items"}]}}] + if ready_answer is not None: + extensions[0]["params"]["methods"] = {"ready": {"method": "GET", "endpoint": "/ready"}} + if state_path is not None: + extensions[0]["params"]["state"] = {"path": state_path} + if clock: + extensions.append({"uri": "urn:agentenv:clock/v1", "params": { + "endpoint": "/agentenv/ext/sync_time", "methods": {"sync_time": {"method": "POST"}}}}) + return stamp(JSONResponse({"name": "hand", **({"url": rpc_path} if rpc_path else {}), + "capabilities": {"extensions": extensions, + "operations": ["data/reset", "data/add", "data/get"]}})) + + async def site_state(request: Request) -> Response: + return JSONResponse({"state_diff": {"items": {"added": ["b"]}} if changed else {}}) + + async def rpc(request: Request) -> Response: + body = await request.json() + method = body["method"] + if rpc_reply is not None: + return stamp(JSONResponse(rpc_reply[1], status_code=rpc_reply[0])) + if method == "data/reset": + state["reset"] = True + if not reset_keeps: + state["items"] = [] + elif method == "data/add": + state["reset"] = False + added = [item for part in body["params"]["parts"] for item in part["data"]["items"]] + state["items"] = added if reset_keeps else state["items"] + added + if method == "data/get" and get_fails_after_reset and state["reset"]: + answer = {"jsonrpc": "2.0", "id": body["id"], "error": {"code": -32603, "message": "down"}} + else: + result = {"parts": [{"kind": "data", "data": {"items": state["items"]}}]} if method == "data/get" else {} + answer = {"jsonrpc": "2.0", "id": body["id"], "result": result} + return stamp(JSONResponse(answer)) if stamp_rpc else JSONResponse(answer) + + async def sync_time(request: Request) -> Response: + import httpx + + url = (await request.json())["env_get_time_url"] + async with httpx.AsyncClient() as client: + virtual = (await client.get(url)).json()["virtual_time"] + state["at"] = datetime.datetime.fromisoformat(virtual.replace("Z", "+00:00")) + return stamp(JSONResponse({"gateway_virtual_time": virtual})) + + app = Starlette(routes=[Route("/", page), Route("/healthz", health), Route("/ready", ready), Route("/items", items), + Route("/items/", items_slash), + Route("/.well-known/agent-env.json", card), Route(rpc_path or "/agentenv", rpc, methods=["POST"]), + Route("/agentenv/ext/sync_time", sync_time, methods=["POST"]), + *([Route(state_path, site_state)] if state_path else [])]) + app.state.data = state + return app + + +def test_a_handwritten_server_meeting_the_contract_passes() -> None: + results = conformance.check("http://testserver", client=TestClient(_handwritten())) + assert statuses(results) == dict.fromkeys(CHECKS, "passed"), results + + +@pytest.mark.parametrize(("answer", "status", "detail"), [ + ({"ready": True, "problems": [], "since": "boot"}, "passed", "website/v1 ready says it is ready"), + ({"ready": False, "problems": ["seeded", "indexed"]}, "failed", "website/v1 ready says it is not ready: seeded, indexed"), + ({"ready": False}, "failed", "website/v1 ready says it is not ready: no problem named"), +]) +def test_health_reads_the_cards_ready_as_a_website_readiness(answer: dict, status: str, detail: str) -> None: + results = {r.name: r for r in conformance.check("http://testserver", client=TestClient(_handwritten(ready_answer=answer)))} + assert (results["health"].status, results["health"].detail) == (status, detail) + + +@pytest.mark.parametrize(("answer", "why"), [ + ({"ok": True}, "ready: Field required"), + ({"ready": "soon", "problems": []}, "ready: Input should be a valid boolean"), + ({"ready": False, "problems": "seeded"}, "problems: Input should be a valid list"), + (["ready"], "the answer: Input should be a valid dictionary or instance of WebsiteReadiness"), +]) +def test_an_answer_to_ready_that_isnt_a_website_readiness_fails_health_saying_why(answer: object, why: str) -> None: + health = next(r for r in conformance.check("http://testserver", client=TestClient(_handwritten(ready_answer=answer))) + if r.name == "health") + assert health.status == "failed" + assert health.detail.startswith(f"website/v1 ready answered {answer!r}, not a WebsiteReadiness: ") and why in health.detail + + +def test_a_server_breaking_the_contract_fails_those_checks() -> None: + results = statuses(conformance.check("http://testserver", + client=TestClient(_handwritten(leak_host=True, domain_cookie=True, clock=False)))) + assert results["forwarded"] == results["cookies"] == results["clock"] == "failed" + assert results["card"] == results["health"] == results["data_plane"] == "passed" + + +def test_the_forwarded_check_follows_no_redirect_and_reads_its_location(sdk_site: TestClient) -> None: + forwarded = {r.name: r for r in conformance.check("http://testserver", client=sdk_site)}["forwarded"] + assert forwarded.status == "passed" and "/api/notes/" in forwarded.detail + leaked = {r.name: r for r in conformance.check("http://testserver", + client=TestClient(_handwritten(leak_redirect=True)))}["forwarded"] + assert leaked.status == "failed" and "Location of /items/" in leaked.detail + + +def test_universe_now_is_checked_on_the_answers_of_every_check() -> None: + result = {r.name: r for r in conformance.check("http://testserver", + client=TestClient(_handwritten(stamp_rpc=False)))}["universe_now"] + assert result.status == "failed" and "/agentenv" in result.detail + + +def test_a_reset_that_clears_nothing_fails_the_data_plane_and_leaves_the_data() -> None: + app = _handwritten(reset_keeps=True) + result = {r.name: r for r in conformance.check("http://testserver", client=TestClient(app))}["data_plane"] + assert result.status == "failed" and "data/reset" in result.detail + assert app.state.data["items"] == ["a"] + + +def test_the_data_is_added_back_when_a_step_after_the_reset_fails() -> None: + app = _handwritten(get_fails_after_reset=True) + result = {r.name: r for r in conformance.check("http://testserver", client=TestClient(app))}["data_plane"] + assert result.status == "failed" and "down" in result.detail + assert app.state.data["items"] == ["a"] + + +@pytest.mark.parametrize("reply", [(500, {}), (200, {}), (500, {"jsonrpc": "2.0", "id": "x", "result": {}})]) +def test_a_data_request_without_a_2xx_json_rpc_result_fails_the_data_plane(reply: tuple[int, dict]) -> None: + app = _handwritten(rpc_reply=reply) + result = {r.name: r for r in conformance.check("http://testserver", client=TestClient(app))}["data_plane"] + assert result.status == "failed" and "no JSON-RPC result" in result.detail + + +def test_an_sdk_site_keeps_its_initial_state_and_one_changed_since_skips_the_round_trip( + notes: WebsiteEnvironment, sdk_site: TestClient) -> None: + before = sdk_site.get("/go").json() + conformance.check("http://testserver", client=sdk_site) + assert sdk_site.get("/go").json() == before + notes.store.insert("notes", {"text": "new"}) + changed = sdk_site.get("/go").json() + assert changed["state_diff"] + result = {r.name: r for r in conformance.check("http://testserver", client=sdk_site)}["data_plane"] + assert result.status == "skipped" and "website/v1 state shows changes" in result.detail + assert sdk_site.get("/go").json() == changed + + +def test_the_data_plane_is_reached_at_the_cards_url() -> None: + app = _handwritten(rpc_path="/rpc") + result = {r.name: r for r in conformance.check("http://testserver", client=TestClient(app))}["data_plane"] + assert result.status == "passed", result + assert app.state.data["items"] == ["a"] + + +@pytest.mark.parametrize(("changed", "status"), [(True, "skipped"), (False, "passed")]) +def test_a_card_without_state_is_read_at_the_state_path_it_names(changed: bool, status: str) -> None: + app = _handwritten(state_path="/grading/state", changed=changed) + result = {r.name: r for r in conformance.check("http://testserver", client=TestClient(app))}["data_plane"] + assert result.status == status, result + assert ("/grading/state shows changes" in result.detail) is changed + + +def test_a_server_with_no_card_health_or_header_fails_them() -> None: + bare = Starlette(routes=[Route("/", lambda request: JSONResponse({"not": "html"}))]) + results = statuses(conformance.check("http://testserver", client=TestClient(bare))) + assert results["card"] == results["any_host"] == results["health"] == results["universe_now"] == "failed" + assert results["data_plane"] == results["clock"] == "failed" + assert results["cookies"] == "skipped" + + +def test_main_prints_each_check_and_exits_by_outcome(monkeypatch, capsys) -> None: + monkeypatch.setattr(conformance, "check", lambda url, **options: [conformance.Result("card", "passed"), + conformance.Result("clock", "failed", "late")]) + assert conformance.main(["http://site"]) == 1 + printed = capsys.readouterr().out + assert "PASS card" in printed and "FAIL clock: late" in printed + + +def test_main_exits_2_when_the_server_cannot_be_reached(capsys) -> None: + assert conformance.main(["http://127.0.0.1:9", "--timeout", "2"]) == 2 + assert "couldn't reach" in capsys.readouterr().err diff --git a/packages/agentenv-protocol/tests/test_website_environment.py b/packages/agentenv-protocol/tests/test_website_environment.py new file mode 100644 index 000000000..6f28c4ba1 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_website_environment.py @@ -0,0 +1,552 @@ +"""WebsiteEnvironment as a site's server: static pages, health, the card and its OpenAPI document, the default store +and its data plane, sessions, the universe's clock, and a mounted app.""" + +from __future__ import annotations + +import asyncio +import base64 +import datetime +import json +import shutil +import subprocess +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import pytest +from starlette.applications import Starlette +from starlette.responses import PlainTextResponse +from starlette.routing import Route +from starlette.testclient import TestClient + +from agentenv_protocol import DataPart, get_data +from agentenv_protocol.website import ( + SESSION_COOKIE, + UNIVERSE_NOW_HEADER, + WEBSITE_EXTENSION_URI, + WEBSITE_READY_OP, + ApiContext, + Collections, + WebsiteEnvironment, + WebsiteReadiness, + api, + ready_check, + website, +) +from agentenv_protocol.website import clock as clock_module +from agentenv_protocol.website.clock import parse_time + +RPC = "/agentenv" + + +def rpc(client: TestClient, method: str, params: dict | None = None) -> dict: + body = client.post(RPC, json={"jsonrpc": "2.0", "id": 1, "method": method, "params": params or {}}).json() + assert "error" not in body, body + return body["result"] + + +@pytest.fixture +def dist(tmp_path: Path) -> Path: + root = tmp_path / "dist" + (root / "assets").mkdir(parents=True) + (root / "index.html").write_text("Shophome") + (root / "assets" / "app.js").write_text("console.log(1)") + (tmp_path / "secret.txt").write_text("outside") + return root + + +def site(dist: Path, **declared) -> type[WebsiteEnvironment]: + @website(domains=["shop.example", "www.shop.example"], static=str(dist), **declared) + class Shop(WebsiteEnvironment): + @api.get("/api/orders", name="list_orders") + def list_orders(self, ctx: ApiContext) -> list: + """List the signed-in user's orders.""" + return self.store.find("orders", user=ctx.user) + + @api.post("/api/orders", name="place_order", roles=["default", "agent"]) + def place_order(self, order: dict, ctx: ApiContext) -> dict: + return self.store.insert("orders", {**order, "user": ctx.user, "at": self.now().isoformat()}) + + @api.post("/api/sign-in") + def sign_in(self, body: dict, ctx: ApiContext) -> dict: + ctx.sign_in(body["user"]) + return {"user": ctx.user} + + return Shop + + +SEED = {"current_user": "alice", "users": [{"id": "alice"}, {"id": "bob"}], "orders": [{"id": 1, "user": "bob"}]} + + +@pytest.fixture +def shop(dist: Path) -> WebsiteEnvironment: + env = site(dist)() + env.store.load(SEED) + env.store.mark_initial() + return env + + +@pytest.fixture +def client(shop: WebsiteEnvironment) -> TestClient: + return TestClient(shop.create_app()) + + +def test_static_files_and_index_fallback_answer_for_any_host(client: TestClient) -> None: + for host in ("testserver", "shop.example", "10.0.0.7:8000", "anything.invalid"): + page = client.get("/", headers={"Host": host}) + assert page.status_code == 200 and "home" in page.text and page.headers["content-type"].startswith("text/html") + assert client.get("/assets/app.js").text == "console.log(1)" + assert "home" in client.get("/orders/42/edit").text + assert client.get("/assets/missing.js").status_code == 404 + assert client.get("/api/nothing-here").json() == {"error": "not found", "code": "not_found"} + assert client.get("/../secret.txt").status_code == 404 + assert client.post("/orders").status_code == 404 + + +def test_a_link_out_of_the_static_folder_is_never_served(dist: Path) -> None: + secret = dist.parent / "secret.txt" + (dist / "leak.txt").symlink_to(secret) + (dist / "linked").mkdir() + (dist / "linked" / "index.html").symlink_to(secret) + client = TestClient(site(dist)().create_app()) + assert "outside" not in client.get("/leak.txt").text + assert "outside" not in client.get("/linked").text + assert "outside" not in client.get("/linked/").text + + +def test_the_fallback_never_shadows_api_health_or_reserved_paths(client: TestClient) -> None: + assert client.get("/api/orders").json() == [] + assert client.put("/api/orders").status_code == 405 + assert client.get("/api/health").json() == {"status": "ok"} + for path in ("/agentenv", "/agentenv/anything", "/agentenv/ext/website/nope", "/agentenv/../agentenv/x"): + assert client.get(path).status_code in (404, 405), path + assert client.get("/.well-known/agent-env.json").json()["name"] == "Shop" + + +def test_health_waits_for_every_ready_check(dist: Path) -> None: + class Slow(site(dist)): + ready = False + + @ready_check + def database_up(self) -> bool: + return self.ready + + @ready_check + async def cache_warm(self) -> bool: + return True + + env = Slow() + client = TestClient(env.create_app()) + waiting = client.get("/api/health") + assert waiting.status_code == 503 and waiting.json() == {"status": "starting", "waiting_for": ["database_up"]} + env.ready = True + assert client.get("/api/health").status_code == 200 + + +def test_the_cards_website_v1_ready_says_whether_the_site_is_usable_by_its_ready_checks(dist: Path) -> None: + class Slow(site(dist)): + ready = False + + @ready_check + def database_up(self) -> bool: + return self.ready + + env = Slow() + client = TestClient(env.create_app()) + card = client.get("/.well-known/agent-env.json").json() + ready = _extension(card, WEBSITE_EXTENSION_URI)["params"]["methods"][WEBSITE_READY_OP] + assert (ready["method"], ready["endpoint"]) == ("GET", "/agentenv/ext/website/ready") + assert client.get(ready["endpoint"]).json() == {"ready": False, "problems": ["database_up"]} + assert WebsiteReadiness.model_validate(client.get(ready["endpoint"]).json()) == WebsiteReadiness( + ready=False, problems=["database_up"]) + env.ready = True + assert client.get(ready["endpoint"]).json() == {"ready": True, "problems": []} + + + +def test_a_declared_health_path_is_served_there(dist: Path) -> None: + client = TestClient(site(dist, health="/healthz")().create_app()) + assert client.get("/healthz").json() == {"status": "ok"} + card = client.get("/.well-known/agent-env.json").json() + assert _extension(card, WEBSITE_EXTENSION_URI)["params"]["health"] == "/healthz" + + +def _extension(card: dict, uri: str) -> dict: + return next(e for e in card["capabilities"]["extensions"] if e["uri"] == uri) + + +def test_the_card_declares_website_v1_its_routes_clock_and_data_plane(client: TestClient) -> None: + card = client.get("/.well-known/agent-env.json").json() + assert card["capabilities"]["operations"] == ["data/reset", "data/add", "data/get"] + params = _extension(card, WEBSITE_EXTENSION_URI)["params"] + assert params["domains"] == ["shop.example", "www.shop.example"] + assert params["health"] == "/api/health" + assert params["routes"] == [ + {"method": "GET", "path": "/api/orders", "name": "list_orders", "roles": None}, + {"method": "POST", "path": "/api/orders", "name": "place_order", "roles": ["default", "agent"]}, + {"method": "POST", "path": "/api/sign-in", "name": "sign_in", "roles": None}, + ] + assert params["methods"]["openapi"] == {"method": "GET", "endpoint": "/agentenv/ext/website/openapi", + "request": {"type": "object", "properties": {}}} + assert params["methods"]["session"]["endpoint"] == "/agentenv/ext/website/session" + clock = _extension(card, "urn:agentenv:clock/v1")["params"] + assert clock["endpoint"] == "/agentenv/ext/sync_time" + + +def test_the_openapi_document_names_each_route(client: TestClient) -> None: + document = client.get("/agentenv/ext/website/openapi").json() + assert document["openapi"].startswith("3.") and document["info"]["title"] == "Shop" + listed = document["paths"]["/api/orders"]["get"] + assert listed["operationId"] == "list_orders" and listed["summary"] == "List the signed-in user's orders." + placed = document["paths"]["/api/orders"]["post"] + assert placed["x-agentenv-roles"] == ["default", "agent"] + assert placed["requestBody"]["content"]["application/json"]["schema"]["type"] == "object" + assert listed["parameters"] == [] + + +def test_the_default_store_round_trips_through_the_data_plane(client: TestClient, shop: WebsiteEnvironment) -> None: + assert rpc(client, "data/get")["parts"][0]["data"] == SEED + rpc(client, "data/reset") + assert rpc(client, "data/get")["parts"][0]["data"] == {} + as_file = base64.b64encode(json.dumps({"orders": [{"id": 7, "user": "carol"}]}).encode()).decode() + rpc(client, "data/add", {"parts": [{"kind": "data", "data": {"current_user": "carol", "users": [{"id": "carol"}]}}, + {"kind": "file", "file": {"bytes": as_file, "name": "orders.json"}}, + {"kind": "text", "text": json.dumps({"notes": [{"id": "n1"}]})}]}) + assert rpc(client, "data/get")["parts"][0]["data"] == { + "current_user": "carol", "users": [{"id": "carol"}], "orders": [{"id": 7, "user": "carol"}], + "notes": [{"id": "n1"}]} + assert shop.acting_user() == "carol" + + +def test_go_reports_initial_current_and_diff_state(client: TestClient) -> None: + client.post("/api/orders", json={"sku": "mug"}) + state = client.get("/go", params={"sid": "default"}).json() + assert state["initial_state"] == SEED + assert state["current_state"]["orders"][-1]["sku"] == "mug" + assert state["state_diff"] == {"orders": {"added": [state["current_state"]["orders"][-1]], "removed": [], + "changed": []}} + + +def test_broad_routes_leave_the_built_in_ones_answering_their_paths(dist: Path) -> None: + class Broad(site(dist)): + @api.get("/{slug}") + def page(self, slug: str) -> dict: + return {"page": slug} + + @api.get("/api/{name}") + def named(self, name: str) -> dict: + return {"named": name} + + env = Broad() + env.store.load(SEED) + env.store.mark_initial() + client = TestClient(env.create_app()) + assert client.get("/go").json()["initial_state"] == SEED + assert client.get("/api/session").json()["user"] == "alice" + assert client.get("/api/health").json() == {"status": "ok"} + assert client.get("/about").json() == {"page": "about"} and client.get("/api/other").json() == {"named": "other"} + + +def test_a_broad_route_never_serves_a_page_at_the_data_planes_or_cards_paths(dist: Path) -> None: + class Catchall(site(dist)): + @api.get("/{rest:path}") + def page(self, rest: str) -> dict: + return {"page": rest} + + client = TestClient(Catchall().create_app()) + for path in ("/agentenv/anything", "/agentenv/", "/x/../agentenv/y", "/.well-known/agent-env.json/"): + assert client.get(path).status_code == 404, path + assert client.get("/.well-known/agent-env.json").status_code == 200 + assert client.get("/docs/a").json() == {"page": "docs/a"} + + +def test_a_site_with_its_own_hooks_keeps_them_and_serves_no_go(dist: Path) -> None: + class Own(site(dist)): + @get_data + def export(self) -> list: + return [DataPart(data={"mine": True})] + + client = TestClient(Own().create_app()) + assert rpc(client, "data/get")["parts"][0]["data"] == {"mine": True} + assert client.get("/.well-known/agent-env.json").json()["capabilities"]["operations"] == ["data/reset", "data/add", "data/get"] + assert "home" in client.get("/go").text + + +def test_a_seed_data_file_is_the_initial_state(tmp_path: Path, dist: Path) -> None: + (tmp_path / "seed.json").write_text(json.dumps(SEED)) + env = site(dist, data=str(tmp_path / "seed.json"))() + assert env.store.export() == SEED and env.store.initial == SEED and env.acting_user() == "alice" + + +def test_the_acting_user_is_signed_in_until_a_session_says_else(client: TestClient) -> None: + assert client.get("/api/orders").json() == [] + first = client.get("/api/session") + assert first.json() == {"user": "alice", "role": "default", "signed_in": True} + cookie = first.headers["set-cookie"] + assert cookie.startswith(f"{SESSION_COOKIE}=") and "domain" not in cookie.lower() and "httponly" in cookie.lower() + client.post("/api/sign-in", json={"user": "bob"}) + assert client.get("/api/orders").json() == [{"id": 1, "user": "bob"}] + assert client.get("/api/session").json()["user"] == "bob" + assert client.delete("/api/session").json() == {"user": None, "role": "default", "signed_in": False} + assert client.get("/api/session").json()["user"] is None + client.cookies.set(SESSION_COOKIE, "forged.signature") + assert client.get("/api/session").json()["user"] == "alice" + + +def test_the_session_cookie_expires_when_the_session_does(client: TestClient) -> None: + from email.utils import parsedate_to_datetime + + cookie = client.get("/api/session").headers["set-cookie"] + expires = next(a.split("=", 1)[1] for a in cookie.split(";") if a.strip().lower().startswith("expires=")) + ahead = parsedate_to_datetime(expires) - datetime.datetime.now(datetime.timezone.utc) + assert datetime.timedelta(days=7) - datetime.timedelta(minutes=1) < ahead <= datetime.timedelta(days=7) + + +def test_the_session_operation_mints_cookies_for_a_user_or_signed_out(shop: WebsiteEnvironment) -> None: + client = TestClient(shop.create_app()) + minted = client.post("/agentenv/ext/website/session", json={"user": "bob"}).json() + assert minted["user"] == "bob" and minted["expires_at"].endswith("Z") + (cookie,) = minted["cookies"] + assert cookie["name"] == SESSION_COOKIE and "domain" not in cookie and cookie["path"] == "/" + browser = TestClient(shop.asgi, cookies={SESSION_COOKIE: cookie["value"]}) + assert browser.get("/api/session").json()["user"] == "bob" + signed_out = client.post("/agentenv/ext/website/session", json={}).json()["cookies"][0]["value"] + assert TestClient(shop.asgi, cookies={SESSION_COOKIE: signed_out}).get("/api/session").json()["user"] is None + + +def test_every_answer_carries_the_universe_time(client: TestClient) -> None: + for response in (client.get("/"), client.get("/api/orders"), client.get("/nope.js"), + client.post(RPC, json={"jsonrpc": "2.0", "id": 1, "method": "data/get"}), + client.get("/.well-known/agent-env.json")): + now = parse_time(response.headers[UNIVERSE_NOW_HEADER]) + assert abs((now - datetime.datetime.now(datetime.timezone.utc)).total_seconds()) < 60 + + +class _Clock: + """A gateway-shaped /clock/time whose time a test sets.""" + + def __init__(self, virtual: str) -> None: + self.virtual = virtual + outer = self + + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + body = json.dumps({"virtual_time": outer.virtual}).encode() + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args) -> None: + return None + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=self.server.serve_forever, daemon=True).start() + self.url = f"http://127.0.0.1:{self.server.server_port}/clock/time" + + def close(self) -> None: + self.server.shutdown() + self.server.server_close() + + +@pytest.fixture +def gateway_clock(): + clock = _Clock("2031-01-01T00:00:00Z") + yield clock + clock.close() + + +def test_sync_time_ties_now_and_the_header_to_the_gateway_clock(client: TestClient, shop: WebsiteEnvironment, + gateway_clock: _Clock, monkeypatch) -> None: + synced = client.post("/agentenv/ext/sync_time", json={"env_get_time_url": gateway_clock.url}).json() + assert synced == {"env_get_time_url": gateway_clock.url, "gateway_virtual_time": "2031-01-01T00:00:00.000Z"} + assert shop.now().year == 2031 + assert client.get("/").headers[UNIVERSE_NOW_HEADER].startswith("2031-01-01T00:00:0") + order = client.post("/api/orders", json={"sku": "pen"}).json() + assert order["at"].startswith("2031-01-01") + gateway_clock.virtual = "2031-06-01T12:00:00Z" + monkeypatch.setattr(clock_module, "REFRESH_S", 0.0) + assert client.get("/").headers[UNIVERSE_NOW_HEADER].startswith("2031-06-01T12:00") + + +def test_sync_time_takes_the_universe_s_time_zone_and_a_later_sync_without_one_drops_it( + client: TestClient, shop: WebsiteEnvironment, gateway_clock: _Clock) -> None: + method = _extension(client.get("/.well-known/agent-env.json").json(), "urn:agentenv:clock/v1")["params"]["methods"]["sync_time"] + assert "timezone" in method["request"]["properties"] and method["request"]["required"] == ["env_get_time_url"] + zoned = client.post("/agentenv/ext/sync_time", json={"env_get_time_url": gateway_clock.url, "timezone": " America/Chicago "}) + assert zoned.json()["timezone"] == "America/Chicago" == shop.timezone + assert shop.now().tzinfo == datetime.timezone.utc + unzoned = client.post("/agentenv/ext/sync_time", json={"env_get_time_url": gateway_clock.url}).json() + assert "timezone" not in unzoned and shop.timezone is None + + +def test_a_clock_that_cannot_be_read_is_refused_and_a_lost_one_extrapolates(client: TestClient, + shop: WebsiteEnvironment, + gateway_clock: _Clock, monkeypatch) -> None: + refused = client.post("/agentenv/ext/sync_time", json={"env_get_time_url": "http://127.0.0.1:9/clock/time"}) + assert refused.status_code == 500 and shop.now().year == datetime.datetime.now().year + client.post("/agentenv/ext/sync_time", json={"env_get_time_url": gateway_clock.url}) + gateway_clock.close() + monkeypatch.setattr(clock_module, "REFRESH_S", 0.0) + assert client.get("/").headers[UNIVERSE_NOW_HEADER].startswith("2031-01-01") + + +def test_clock_patch_puts_the_clock_script_in_html_only(dist: Path, gateway_clock: _Clock) -> None: + client = TestClient(site(dist, clock_patch=True)().create_app()) + client.post("/agentenv/ext/sync_time", json={"env_get_time_url": gateway_clock.url}) + page = client.get("/") + head = page.text.index("") + len("") + assert page.text[head:].startswith(" + + diff --git a/packages/agentenv-protocol/examples/website/notes/web/package-lock.json b/packages/agentenv-protocol/examples/website/notes/web/package-lock.json new file mode 100644 index 000000000..938f9c2cd --- /dev/null +++ b/packages/agentenv-protocol/examples/website/notes/web/package-lock.json @@ -0,0 +1,1020 @@ +{ + "name": "notes-site-web", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "notes-site-web", + "version": "0.0.0", + "devDependencies": { + "vite": "5.4.10" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", + "integrity": "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.21.5.tgz", + "integrity": "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.21.5.tgz", + "integrity": "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.21.5.tgz", + "integrity": "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.21.5.tgz", + "integrity": "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.21.5.tgz", + "integrity": "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.21.5.tgz", + "integrity": "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.21.5.tgz", + "integrity": "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.21.5.tgz", + "integrity": "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.21.5.tgz", + "integrity": "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.21.5.tgz", + "integrity": "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.21.5.tgz", + "integrity": "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.21.5.tgz", + "integrity": "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.21.5.tgz", + "integrity": "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.21.5.tgz", + "integrity": "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.21.5.tgz", + "integrity": "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.21.5.tgz", + "integrity": "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.21.5.tgz", + "integrity": "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.21.5.tgz", + "integrity": "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.21.5.tgz", + "integrity": "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.21.5.tgz", + "integrity": "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.21.5.tgz", + "integrity": "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.21.5.tgz", + "integrity": "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@napi-rs/lzma-linux-x64-gnu": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/@napi-rs/lzma-linux-x64-gnu/-/lzma-linux-x64-gnu-1.5.1.tgz", + "integrity": "sha512-oTXEIha4SsuXdTA4Iyskj0kpdx2yVXdhd75c2v3xGrHFfVMsbhTPZU/nMPL4sWKo4pBHm3aucLaqGlF696dTyQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^22.20 || ^24.12 || >=25" + } + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.67.2.tgz", + "integrity": "sha512-qB7Gzl1QI45zP1zXgqLKzjFmOBGQw2s2MLtdw0eB4xdFGPNE8LhmOnItFgyYJXXNSW8Go9c2ovvXrUUzw5sAMQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.67.2.tgz", + "integrity": "sha512-8+hXNO1J+GsYCGHwyJuBlnhyozivEvIEm/P6NkJeuoVUN+60TDvYYXk38UTm5bKRTLhIpqEQYx4d9CeN6kjh0Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-x64": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-x64/-/rollup-android-x64-4.67.2.tgz", + "integrity": "sha512-/uEZY+tcWDE8DXa/rajU3Bwv6IfvQavvzjmbkn39GgQenT+UYtAxfEMdB7VKr74y8aonncm0eQa1EbiNfg+Cjg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.67.2.tgz", + "integrity": "sha512-3tCLEgakEvb5M3/KyYdhPjoteyWYziMTeznoXfcs/31eeFu+Vgs0VNRlb6Zc+YC07/sLa84OwTDF5zHOMFk92Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.67.2.tgz", + "integrity": "sha512-z31ODzkdkX7OOJCssvpz++hhtkQKWKw2L0eO2HAxFP7RMSXXedzkrTjesqCUxs4XLckRShjNfxsNxjGY5DQCGg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.67.2.tgz", + "integrity": "sha512-CTvrb0KTAQqcrpcj7rg8FgAI0+jRWnCl+En6exqXDZztCo7vspxlRUX8z9Hp/J/ja2XRnQ1JDbexkBtRyD9jSw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.67.2.tgz", + "integrity": "sha512-SJdblZpBkbOh1e4JwMM5pJJ2PdVmD3WWbMTXvncMSRLA3DdZuwU+P93RaSChOAXN4Mu53tBoFp1ADPzi3dIVAA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.67.2.tgz", + "integrity": "sha512-pUBEd7uI9dAu3TpbPcvi9eU2aJslsjIomormeTaQHmzWNhyqNx3mmq+ekC6blGNZu8F3uTHngTibKp7tbblF0A==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.67.2.tgz", + "integrity": "sha512-wW6ARw2Ah0/teOBTgTeSJIpVdPKaxQmEgZ/irkUfr3F9sgimqpApCv/NbQzjm0Nfi5EFilHh1H8kz01uQLaq5Q==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.67.2.tgz", + "integrity": "sha512-xcFYMHNL42fRu2MFipvxznr+zd+DRqwL98bhNvDWbMPRcFtMYf34NAz4xVOlnEhzgpNRE5htSemcqAsAxf2eQg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.67.2.tgz", + "integrity": "sha512-U9HrnJ9PFXqA//P/oEGaXBIJraEqzYMS9yuQOdOcPCvHZIC1FFOlnd8ocAaeDDPDpY8xOrhfxMRtr6KqKi3gyg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.67.2.tgz", + "integrity": "sha512-2hZN8CFGfsbDw+DNtImkPRitpLNjVM2XEbi4ZQ5igLxMFD4gI/P6Zo9czGxuvQXtRyZtZyZp8Kr4ERsAipplBw==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.67.2.tgz", + "integrity": "sha512-HZO7UfWEsReAnkdVByyQaFsNXJCtvdfun/LFb2Xd4qnhgF1ZfhikpbgqrTRtNMmB5OgNnSwFLiPobOT8oNT9iw==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.67.2.tgz", + "integrity": "sha512-AdYIihuLkJ2TScy9BpSPiDex3zOMjkNJLsFlk9NeJ0U3WEtOx4W8z0nOkPEhDS2S4gwsRWBa6ZZexJ9e6OxyzQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.67.2.tgz", + "integrity": "sha512-iZE0NXcAGqLoCWeHRPkBxJe93JT4Q981z1ZqR+y7qAiIksdRlWhw8hmqHFo1B4ZnU66He31NUb52Iyv+vwMEBQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.67.2.tgz", + "integrity": "sha512-vC7EAtJpSqSrrbg8huMfgxtwlR8fUsFCe5rStDk1WA8POky/6EEp52dxENMmqLKjSfdM3TT1WWPcZ01hiYZcdw==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.67.2.tgz", + "integrity": "sha512-DdqG6x72MHpQOlkK/cqrru9b2QMJHLkNojCB3bteB5fmkE/7XEAkReJMANp8wgef8eiEC99SvdP/BH7Ho/BpLA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.67.2.tgz", + "integrity": "sha512-F10mjdnRmGg9K+LGdqQVeUYFr08f3fVBOh7YNE5p1KIpTzhtKKOLqcQEsqZG6yEM2eouXaMo2AZzeDAcseVTrQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.67.2.tgz", + "integrity": "sha512-ChzK9f3iE4N5GInBquiZtzWOqh+CYS5nlLlXGJxeG3d2T9rk8L1Hxg0s+v2Eh6vmAc/1T6IhOVkQU0ORP2eV7g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.67.2.tgz", + "integrity": "sha512-95WwzC+KH9hPHtbgwvwwW6MkKME6vMAdAQLS2diCvoi/iUiYdC0Y1RIPYVESx0iWqkN5ZaZq6qD1BflLcyTMEQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.67.2.tgz", + "integrity": "sha512-eKbpwlolQmDKd/dbFjvqB1+2qdYQLrgxi3Mtbijp7/3yBBvO9OoHtQ3ecYIaCdW46ijSjy5/yydyrrxHI416/g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.67.2.tgz", + "integrity": "sha512-is9lnlv6khapl+H9Hjp0gIVsUVtDxQga2nwMHpQbhDXMCMGcNmlel4/q9ihf/3FvvbL4GdGOPOGvWoei2RCggg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.67.2.tgz", + "integrity": "sha512-yTq+7YjsgbBcMdlk7Wq+VoJilGQtdtRc4PyTvN7vhO958atCO2nQeIjCF/YANSD/Z/fnrCr27fYsGgC6PFTndQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.67.2.tgz", + "integrity": "sha512-nAvZib4Xz1zx8OKhtRuyCsSmCmbfegv1AGbfpG+CowQKxCBW1pLlfYWjcr1+GI3BzRKQbBl7l9640GwWAwOF7g==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.67.2.tgz", + "integrity": "sha512-R3CVbEbNa1OXjpldD1tOLarb5rB08xjQiRMpe7PZKtT8eHCZhrV4ClSPfLLbSgRMM1RfLtmAfouBRf14f47lMA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.67.2.tgz", + "integrity": "sha512-DeuVFilvaGtoS9TFIbTIQ0Kt4CBqP3xnjJ8SFqWWbXlFbrpmCdivEW7Rm38trRA8rZzQQ8lhs73nEetWotJpTg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@types/estree": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.10.tgz", + "integrity": "sha512-lTyOMnOcmRBzdCv6/4zUZCrEhV0xrqQiUZBZtVcbmLHBXKLnN8q89v/wKRF/Q0HbwVsqbIG3o5HDoFl9sduM+g==", + "dev": true, + "license": "MIT" + }, + "node_modules/esbuild": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz", + "integrity": "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.21.5", + "@esbuild/android-arm": "0.21.5", + "@esbuild/android-arm64": "0.21.5", + "@esbuild/android-x64": "0.21.5", + "@esbuild/darwin-arm64": "0.21.5", + "@esbuild/darwin-x64": "0.21.5", + "@esbuild/freebsd-arm64": "0.21.5", + "@esbuild/freebsd-x64": "0.21.5", + "@esbuild/linux-arm": "0.21.5", + "@esbuild/linux-arm64": "0.21.5", + "@esbuild/linux-ia32": "0.21.5", + "@esbuild/linux-loong64": "0.21.5", + "@esbuild/linux-mips64el": "0.21.5", + "@esbuild/linux-ppc64": "0.21.5", + "@esbuild/linux-riscv64": "0.21.5", + "@esbuild/linux-s390x": "0.21.5", + "@esbuild/linux-x64": "0.21.5", + "@esbuild/netbsd-x64": "0.21.5", + "@esbuild/openbsd-x64": "0.21.5", + "@esbuild/sunos-x64": "0.21.5", + "@esbuild/win32-arm64": "0.21.5", + "@esbuild/win32-ia32": "0.21.5", + "@esbuild/win32-x64": "0.21.5" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/nanoid": { + "version": "3.3.20", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.20.tgz", + "integrity": "sha512-uKdg2G3GNCKQn9byYOpxbGqrT2fGO5KRt5J/8b3pok8rT6qxGWF6hxMyJiEYtAf+FVyYuD9hRaDqX5uPFYJ4ZQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/postcss": { + "version": "8.5.32", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.32.tgz", + "integrity": "sha512-LJlKCCAUFKOuZxipUdYw1NzIITEsPcgyOG9LkvRJvuKj/bRzbuRY0ARXvL0pWGpsZvn8jku+LMep2eU0seHp8g==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.20", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.2" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/rollup": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.67.2.tgz", + "integrity": "sha512-sFZk5BeRUQzkjmuClrJI41BowIP4yv/40d5gM9vtjxVLEt+8JG9FGlBps6cliTP2KhFiw22oqNGo0G9pVpPZlw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.10" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@napi-rs/lzma-linux-x64-gnu": "1.5.1", + "@rollup/rollup-android-arm-eabi": "4.67.2", + "@rollup/rollup-android-arm64": "4.67.2", + "@rollup/rollup-android-x64": "4.67.2", + "@rollup/rollup-darwin-arm64": "4.67.2", + "@rollup/rollup-darwin-x64": "4.67.2", + "@rollup/rollup-freebsd-arm64": "4.67.2", + "@rollup/rollup-freebsd-x64": "4.67.2", + "@rollup/rollup-linux-arm-gnueabihf": "4.67.2", + "@rollup/rollup-linux-arm-musleabihf": "4.67.2", + "@rollup/rollup-linux-arm64-gnu": "4.67.2", + "@rollup/rollup-linux-arm64-musl": "4.67.2", + "@rollup/rollup-linux-loong64-gnu": "4.67.2", + "@rollup/rollup-linux-loong64-musl": "4.67.2", + "@rollup/rollup-linux-ppc64-gnu": "4.67.2", + "@rollup/rollup-linux-ppc64-musl": "4.67.2", + "@rollup/rollup-linux-riscv64-gnu": "4.67.2", + "@rollup/rollup-linux-riscv64-musl": "4.67.2", + "@rollup/rollup-linux-s390x-gnu": "4.67.2", + "@rollup/rollup-linux-x64-gnu": "4.67.2", + "@rollup/rollup-linux-x64-musl": "4.67.2", + "@rollup/rollup-openbsd-x64": "4.67.2", + "@rollup/rollup-openharmony-arm64": "4.67.2", + "@rollup/rollup-win32-arm64-msvc": "4.67.2", + "@rollup/rollup-win32-ia32-msvc": "4.67.2", + "@rollup/rollup-win32-x64-gnu": "4.67.2", + "@rollup/rollup-win32-x64-msvc": "4.67.2", + "fsevents": "~2.3.2" + } + }, + "node_modules/source-map-js": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.2.tgz", + "integrity": "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/vite": { + "version": "5.4.10", + "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.10.tgz", + "integrity": "sha512-1hvaPshuPUtxeQ0hsVH3Mud0ZanOLwVTneA1EgbAM5LhaZEqyPWGRQ7BtaMvUrTDeEaC8pxtj6a6jku3x4z6SQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.21.3", + "postcss": "^8.4.43", + "rollup": "^4.20.0" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^18.0.0 || >=20.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^18.0.0 || >=20.0.0", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "sass-embedded": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.4.0" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + } + } + } + } +} diff --git a/packages/agentenv-protocol/examples/website/notes/web/package.json b/packages/agentenv-protocol/examples/website/notes/web/package.json new file mode 100644 index 000000000..c741095dd --- /dev/null +++ b/packages/agentenv-protocol/examples/website/notes/web/package.json @@ -0,0 +1,14 @@ +{ + "name": "notes-site-web", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview" + }, + "devDependencies": { + "vite": "5.4.10" + } +} diff --git a/packages/agentenv-protocol/examples/website/notes/web/src/main.js b/packages/agentenv-protocol/examples/website/notes/web/src/main.js new file mode 100644 index 000000000..f4318fabf --- /dev/null +++ b/packages/agentenv-protocol/examples/website/notes/web/src/main.js @@ -0,0 +1,210 @@ +import "./style.css"; + +let offsetMs = 0; +let filter = "all"; +let renders = 0; +const saving = new Map(); +const lastSent = new Map(); +const landed = new Map(); +const CLOCK_REFRESH_MS = 5000; + +export function universeNow() { + return new Date(Date.now() + offsetMs); +} + +function followUniverse(response) { + const at = Date.parse(response.headers.get("X-Universe-Now") || ""); + if (!Number.isNaN(at)) offsetMs = at - Date.now(); +} + +async function api(path, { method = "GET", body } = {}) { + const response = await fetch(path, { + method, + headers: body === undefined ? {} : { "Content-Type": "application/json" }, + body: body === undefined ? undefined : JSON.stringify(body), + }); + followUniverse(response); + const data = await response.json().catch(() => null); + if (!response.ok) throw new Error(data?.error || `${method} ${path}: ${response.status}`); + return data; +} + +const $ = (id) => document.getElementById(id); + +function queueSave(id, fields) { + lastSent.set(id, fields); + const queued = saving.get(id); + if (queued) { + queued.next = fields; + return; + } + const state = { next: fields }; + saving.set(id, state); + run(async () => { + try { + while (state.next) { + const next = state.next; + state.next = null; + await api(`api/notes/${id}`, { method: "PATCH", body: next }); + } + landed.set(id, renders); + } catch (error) { + lastSent.set(id, {}); + landed.delete(id); + throw error; + } finally { + saving.delete(id); + } + }); +} + +function showError(error) { + $("error").textContent = error ? error.message : ""; + $("error").hidden = !error; +} + +function formatTime(value) { + return new Date(value).toLocaleString(undefined, { dateStyle: "medium", timeStyle: "short", timeZone: "UTC" }); +} + +function tick() { + $("universe-now").textContent = `${formatTime(universeNow())} UTC`; + $("universe-now").dateTime = universeNow().toISOString(); +} + +function noteItem(note) { + const item = document.createElement("li"); + item.className = `note card${note.done ? " done" : ""}`; + item.dataset.id = note.id; + + const done = document.createElement("input"); + done.type = "checkbox"; + done.checked = note.done; + done.setAttribute("aria-label", `Mark "${note.title}" ${note.done ? "not done" : "done"}`); + done.addEventListener("change", () => run(() => api(`api/notes/${note.id}/toggle`, { method: "POST" }))); + + const title = document.createElement("input"); + title.className = "title"; + title.defaultValue = note.title; + title.setAttribute("aria-label", "Title"); + + const body = document.createElement("textarea"); + body.className = "body"; + body.rows = 2; + body.defaultValue = note.body; + body.placeholder = "Details"; + body.setAttribute("aria-label", "Details"); + + const save = () => { + const sent = lastSent.get(note.id) ?? { title: note.title, body: note.body }; + if (title.value === sent.title && body.value === sent.body) return; + queueSave(note.id, { title: title.value, body: body.value }); + }; + for (const field of [title, body]) { + field.addEventListener("change", save); + field.addEventListener("blur", save); + } + + const meta = document.createElement("small"); + meta.textContent = `Created ${formatTime(note.created_at)} · updated ${formatTime(note.updated_at)}`; + + const remove = document.createElement("button"); + remove.className = "delete"; + remove.textContent = "Delete"; + remove.addEventListener("click", () => run(() => api(`api/notes/${note.id}`, { method: "DELETE" }))); + + const text = document.createElement("div"); + text.className = "text"; + text.append(title, body, meta); + item.append(done, text, remove); + return item; +} + +function editing() { + const kept = []; + for (const field of document.querySelectorAll("#notes .title, #notes .body")) { + const focused = field === document.activeElement; + if (focused || field.value !== field.defaultValue) { + kept.push({ id: field.closest("li").dataset.id, kind: field.className, value: field.value, focused, + start: field.selectionStart, end: field.selectionEnd }); + } + } + return kept; +} + +function restore(kept) { + for (const { id, kind, value, focused, start, end } of kept) { + const field = document.querySelector(`#notes li[data-id="${id}"] .${kind}`); + if (!field) continue; + field.value = value; + if (focused) { + field.focus(); + field.setSelectionRange(start, end); + } + } +} + +async function render() { + const query = filter === "all" ? "" : `?done=${filter === "done"}`; + const request = ++renders; + const notes = await api(`api/notes${query}`); + if (request !== renders) return; + for (const [id, after] of landed) { + if (request > after && !saving.has(id)) { + lastSent.delete(id); + landed.delete(id); + } + } + const kept = editing(); + $("notes").replaceChildren(...notes.map(noteItem)); + restore(kept); + if (!notes.length) { + const empty = document.createElement("li"); + empty.className = "empty"; + empty.textContent = "No notes here yet."; + $("notes").append(empty); + } +} + +async function run(action) { + try { + await action(); + showError(null); + await render(); + } catch (error) { + showError(error); + } +} + +async function start() { + const [session, users] = await Promise.all([api("api/session"), api("api/users")]); + $("user").replaceChildren(...users.map((user) => new Option(user.name || user.id, user.id))); + $("user").value = session.user ?? ""; + $("user").addEventListener("change", (event) => + run(() => api("api/sign-in", { method: "POST", body: { user: event.target.value } }))); + + $("new-note").addEventListener("submit", (event) => { + event.preventDefault(); + const body = { title: $("new-title").value, body: $("new-body").value }; + run(async () => { + await api("api/notes", { method: "POST", body }); + if ($("new-title").value === body.title) $("new-title").value = ""; + if ($("new-body").value === body.body) $("new-body").value = ""; + }); + }); + + for (const button of document.querySelectorAll("[data-filter]")) { + button.addEventListener("click", () => { + filter = button.dataset.filter; + document.querySelectorAll("[data-filter]").forEach((b) => b.classList.toggle("active", b === button)); + run(async () => {}); + }); + } + + tick(); + setInterval(tick, 1000); + setInterval(() => fetch("healthz").then(followUniverse).catch(() => {}), CLOCK_REFRESH_MS); + await render(); +} + +start().catch(showError); diff --git a/packages/agentenv-protocol/examples/website/notes/web/src/style.css b/packages/agentenv-protocol/examples/website/notes/web/src/style.css new file mode 100644 index 000000000..8ecd48b6a --- /dev/null +++ b/packages/agentenv-protocol/examples/website/notes/web/src/style.css @@ -0,0 +1,57 @@ +:root { + --bg: #f6f6f3; + --card: #ffffff; + --text: #1d1d1b; + --muted: #6b6b66; + --line: #deded8; + --accent: #2f5bd3; + font-family: system-ui, -apple-system, "Segoe UI", sans-serif; + color: var(--text); + background: var(--bg); +} + +* { box-sizing: border-box; } +body { margin: 0; } + +.bar { + display: flex; + flex-wrap: wrap; + gap: 12px; + align-items: center; + justify-content: space-between; + padding: 12px 16px; + border-bottom: 1px solid var(--line); + background: var(--card); +} +.bar h1 { margin: 0; font-size: 1.25rem; } +.who { display: flex; flex-wrap: wrap; gap: 12px; align-items: center; color: var(--muted); font-size: 0.9rem; } +.who time { font-variant-numeric: tabular-nums; } + +main { max-width: 720px; margin: 0 auto; padding: 16px; } + +.card { background: var(--card); border: 1px solid var(--line); border-radius: 8px; padding: 12px; } + +input, textarea, select, button { font: inherit; color: inherit; } +input, textarea { width: 100%; border: 1px solid var(--line); border-radius: 6px; padding: 6px 8px; background: transparent; } +textarea { resize: vertical; } +button { border: 1px solid var(--line); border-radius: 6px; padding: 6px 12px; background: var(--card); cursor: pointer; } +button[type="submit"], .filters .active { background: var(--accent); border-color: var(--accent); color: #fff; } + +#new-note { display: grid; gap: 8px; } +#new-note button { justify-self: end; } + +.filters { display: flex; gap: 8px; margin: 16px 0; } + +#error { color: #b3261e; margin: 0 0 12px; } + +#notes { list-style: none; margin: 0; padding: 0; display: grid; gap: 8px; } +.note { display: grid; grid-template-columns: auto 1fr auto; gap: 12px; align-items: start; } +.note .text { display: grid; gap: 6px; min-width: 0; } +.note .title { font-weight: 600; border-color: transparent; } +.note .body { border-color: transparent; color: var(--muted); } +.note .title:focus, .note .body:focus { border-color: var(--line); } +.note small { color: var(--muted); } +.note.done .title { text-decoration: line-through; color: var(--muted); } +.note input[type="checkbox"] { width: auto; margin-top: 10px; } +.delete { color: #b3261e; } +.empty { color: var(--muted); text-align: center; padding: 24px; } diff --git a/packages/agentenv-protocol/examples/website/notes/web/vite.config.js b/packages/agentenv-protocol/examples/website/notes/web/vite.config.js new file mode 100644 index 000000000..d07ecd2c2 --- /dev/null +++ b/packages/agentenv-protocol/examples/website/notes/web/vite.config.js @@ -0,0 +1,11 @@ +import { defineConfig } from "vite"; + +export default defineConfig({ + base: "./", + server: { + proxy: { + "/api": "http://127.0.0.1:8000", + "/healthz": "http://127.0.0.1:8000", + }, + }, +}); diff --git a/packages/agentenv-protocol/tests/test_website_notes_example.py b/packages/agentenv-protocol/tests/test_website_notes_example.py new file mode 100644 index 000000000..e02147b10 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_website_notes_example.py @@ -0,0 +1,237 @@ +"""The Notes example site (examples/website/notes) as website/v1's reference site: it passes the conformance checker, +and its own behavior holds: the acting user's auto sign-in and signing in as another user, notes kept per user, the +admin-only route, timestamps from the universe's clock, and the store's seed loaded, exported and reset through the +data plane.""" + +from __future__ import annotations + +import importlib.util +import json +import re +import sys +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.parse import urljoin, urlsplit + +import pytest +from starlette.responses import PlainTextResponse +from starlette.testclient import TestClient + +from agentenv_protocol.website import (ROLE_KEY_ENV, ROLE_KEY_HEADER, SESSION_COOKIE, UNIVERSE_NOW_HEADER, WebsiteEnvironment, + conformance, website) + +NOTES_DIR = Path(__file__).resolve().parents[1] / "examples" / "website" / "notes" +SEED = json.loads((NOTES_DIR / "data" / "seed.json").read_text(encoding="utf-8")) +RPC = "/agentenv" + + +def _load_site(): + """The example's ``site.py``, imported under a name of its own: ``site`` is the standard library's.""" + spec = importlib.util.spec_from_file_location("notes_example_site", NOTES_DIR / "site.py") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +notes_site = _load_site() + + +def rpc(client: TestClient, method: str, params: dict | None = None) -> dict: + body = client.post(RPC, json={"jsonrpc": "2.0", "id": 1, "method": method, "params": params or {}}).json() + assert "error" not in body, body + return body["result"] + + +@pytest.fixture +def notes_class(tmp_path: Path) -> type[WebsiteEnvironment]: + """The example's class, serving a stand-in page instead of ``web/dist``, which only a Vite build makes.""" + (tmp_path / "index.html").write_text("Notes") + declared = notes_site.Notes.declaration()._replace(static=str(tmp_path), data=str(NOTES_DIR / "data" / "seed.json")) + return website(**declared._asdict())(type("Notes", (notes_site.Notes,), {})) + + +@pytest.fixture +def env(notes_class: type[WebsiteEnvironment]) -> WebsiteEnvironment: + return notes_class() + + +@pytest.fixture +def app(env: WebsiteEnvironment): + return env.create_app() + + +@pytest.fixture +def client(app) -> TestClient: + return TestClient(app) + + +class _Clock: + """A gateway-shaped /clock/time whose time a test sets.""" + + def __init__(self, virtual: str) -> None: + self.virtual = virtual + outer = self + + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + body = json.dumps({"virtual_time": outer.virtual}).encode() + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args) -> None: + return None + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=self.server.serve_forever, daemon=True).start() + self.url = f"http://127.0.0.1:{self.server.server_port}/clock/time" + + def close(self) -> None: + self.server.shutdown() + self.server.server_close() + + +@pytest.fixture +def gateway_clock(): + clock = _Clock("2040-02-03T04:05:06Z") + yield clock + clock.close() + + +def test_the_example_passes_every_conformance_check(client: TestClient) -> None: + results = conformance.check("http://testserver", client=client) + assert {r.name: r.status for r in results} == dict.fromkeys( + ["card", "any_host", "health", "universe_now", "data_plane", "clock", "forwarded", "cookies"], "passed"), results + + +def test_the_card_declares_the_site_its_health_path_and_its_routes_by_name(client: TestClient) -> None: + card = client.get("/.well-known/agent-env.json").json() + site = next(e for e in card["capabilities"]["extensions"] if e["uri"] == "urn:agentenv:website/v1")["params"] + assert card["name"] == "notes" + assert (site["domains"], site["health"]) == (["notes.test"], "/healthz") + assert [r["name"] for r in site["routes"]] == ["list_users", "sign_in", "list_notes", "create_note", "get_note", + "edit_note", "toggle_note", "delete_note", "all_notes"] + assert next(r for r in site["routes"] if r["name"] == "all_notes")["roles"] == ["admin"] + assert client.get("/healthz").json() == {"status": "ok"} + + +def test_a_browser_is_signed_in_as_the_acting_user_and_can_sign_in_as_another(client: TestClient) -> None: + session = client.get("/api/session") + assert session.json() == {"user": "ann", "role": "default", "signed_in": True} + assert SESSION_COOKIE in session.cookies + assert [n["title"] for n in client.get("/api/notes").json()] == ["Buy milk", "Book dentist"] + assert client.post("/api/sign-in", json={"user": "ben"}).json() == {"user": "ben"} + assert client.get("/api/session").json()["user"] == "ben" + assert [n["title"] for n in client.get("/api/notes").json()] == ["Draft the quarterly report"] + refused = client.post("/api/sign-in", json={"user": "mallory"}) + assert refused.status_code == 404 and refused.json()["code"] == "not_found" + + +def test_notes_are_created_edited_toggled_and_deleted_by_their_owner_only(app, client: TestClient) -> None: + created = client.post("/api/notes", json={"title": " Call Sam ", "body": "about the trip"}) + assert created.status_code == 201 + note = created.json() + assert (note["owner"], note["title"], note["done"]) == ("ann", "Call Sam", False) + assert note["created_at"] == note["updated_at"] + + ben = TestClient(app) + ben.post("/api/sign-in", json={"user": "ben"}) + assert "Call Sam" not in [n["title"] for n in ben.get("/api/notes").json()] + for response in (ben.get(f"/api/notes/{note['id']}"), ben.patch(f"/api/notes/{note['id']}", json={"done": True}), + ben.post(f"/api/notes/{note['id']}/toggle"), ben.delete(f"/api/notes/{note['id']}")): + assert response.status_code == 404, response.text + + edited = client.patch(f"/api/notes/{note['id']}", json={"body": "about the trip, before Friday"}).json() + assert (edited["title"], edited["body"]) == ("Call Sam", "about the trip, before Friday") + assert client.post(f"/api/notes/{note['id']}/toggle").json()["done"] is True + assert [n["title"] for n in client.get("/api/notes", params={"done": "true"}).json()] == ["Call Sam", "Book dentist"] + assert client.patch(f"/api/notes/{note['id']}", json={"title": " "}).json()["code"] == "empty_title" + assert client.post("/api/notes", json={"body": "no title"}).status_code == 400 + assert client.delete(f"/api/notes/{note['id']}").json() == {"deleted": note["id"]} + assert client.get(f"/api/notes/{note['id']}").status_code == 404 + + +def test_every_users_notes_are_listed_for_an_admin_caller_only(client: TestClient, + monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(ROLE_KEY_ENV, "k3y") + for headers in ({}, {"AgentEnv-Role": "agent", ROLE_KEY_HEADER: "k3y"}, {"AgentEnv-Role": "admin"}, + {"AgentEnv-Role": "admin", ROLE_KEY_HEADER: "guess"}): + refused = client.get("/api/admin/notes", headers=headers) + assert refused.status_code == 403, headers + every = client.get("/api/admin/notes", headers={"AgentEnv-Role": "admin", ROLE_KEY_HEADER: "k3y"}).json() + assert {n["owner"] for n in every} == {"ann", "ben"} + + +def test_timestamps_follow_the_universes_clock_once_synced(client: TestClient, gateway_clock: _Clock) -> None: + before = client.post("/api/notes", json={"title": "before the sync"}).json() + assert not before["created_at"].startswith("2040") + client.post("/agentenv/ext/sync_time", json={"env_get_time_url": gateway_clock.url}) + after = client.post("/api/notes", json={"title": "after the sync"}) + assert after.json()["created_at"].startswith("2040-02-03T04:05") + assert after.headers[UNIVERSE_NOW_HEADER].startswith("2040-02-03T04:05") + toggled = client.post(f"/api/notes/{before['id']}/toggle").json() + assert toggled["created_at"] == before["created_at"] and toggled["updated_at"].startswith("2040-02-03T04:05") + + +def test_the_seed_loads_exports_and_resets_through_the_data_plane(client: TestClient) -> None: + assert rpc(client, "data/get")["parts"][0]["data"] == SEED + rpc(client, "data/reset") + assert rpc(client, "data/get")["parts"][0]["data"] == {} + assert client.get("/api/notes").status_code == 401 + assert client.get("/healthz").status_code == 503 + + seed_file = {"kind": "file", "file": {"uri": (NOTES_DIR / "data" / "seed.json").as_uri(), + "mimeType": "application/json"}} + rpc(client, "data/add", {"parts": [seed_file]}) + assert client.get("/healthz").status_code == 200 + assert rpc(client, "data/get")["parts"][0]["data"] == SEED + assert client.get("/api/session").json()["user"] == "ann" + + client.post("/api/notes/1/toggle") + state = client.get("/go").json() + assert state["initial_state"] == SEED + assert state["state_diff"]["notes"]["changed"][0]["after"]["done"] is True + + +GATEWAY_PREFIX = "/website/notes" + + +def _behind_gateway_prefix(app): + """The gateway's ``/website//`` route: a request under the prefix reaches the site with the prefix taken off, + and any other one answers the gateway's own 404.""" + + async def gateway(scope, receive, send): + if scope["type"] == "http" and scope["path"].startswith(GATEWAY_PREFIX + "/"): + path = scope["path"][len(GATEWAY_PREFIX):] + return await app({**scope, "path": path, "raw_path": path.encode()}, receive, send) + if scope["type"] != "http": + return await app(scope, receive, send) + await PlainTextResponse("no gateway route", status_code=404)(scope, receive, send) + + return gateway + + +@pytest.mark.skipif(not (NOTES_DIR / "web" / "dist" / "index.html").is_file(), reason="web/dist needs a Vite build") +def test_the_built_page_loads_its_assets_and_api_under_the_gateways_prefix() -> None: + page_url = f"http://gateway.test{GATEWAY_PREFIX}/" + browser = TestClient(_behind_gateway_prefix(notes_site.Notes().create_app()), base_url="http://gateway.test") + page = browser.get(page_url) + assert page.status_code == 200 and page.headers["content-type"].startswith("text/html") + + assets = [urljoin(page_url, ref) for ref in re.findall(r'(?:src|href)="([^"]+)"', page.text)] + assert assets and all(urlsplit(url).path.startswith(GATEWAY_PREFIX + "/") for url in assets), assets + scripts = [] + for url in assets: + fetched = browser.get(url) + assert fetched.status_code == 200, url + if url.endswith(".js"): + scripts.append(fetched.text) + + called = {path for script in scripts for path in re.findall(r'["`](/?api/[\w-]+)', script)} + assert {"api/session", "api/users", "api/notes", "api/sign-in"} <= called + assert not [path for path in called if path.startswith("/")], called + assert browser.get(urljoin(page_url, "api/session")).json()["user"] == "ann" + assert [u["id"] for u in browser.get(urljoin(page_url, "api/users")).json()] == ["ann", "ben"] From 4ee1be2eda14b9e81161edadb85f902efb737ca1 Mon Sep 17 00:00:00 2001 From: William Loo Date: Tue, 3 Nov 2026 14:22:37 -0800 Subject: [PATCH 09/12] fix(website): the Notes page saves queued edits before switching users, and adds a note once per submit (FD-4683) Co-Authored-By: Claude Opus 5.5 (1M context) --- .../examples/website/notes/web/src/main.js | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/packages/agentenv-protocol/examples/website/notes/web/src/main.js b/packages/agentenv-protocol/examples/website/notes/web/src/main.js index f4318fabf..d061a6cea 100644 --- a/packages/agentenv-protocol/examples/website/notes/web/src/main.js +++ b/packages/agentenv-protocol/examples/website/notes/web/src/main.js @@ -40,7 +40,7 @@ function queueSave(id, fields) { } const state = { next: fields }; saving.set(id, state); - run(async () => { + state.done = run(async () => { try { while (state.next) { const next = state.next; @@ -180,14 +180,31 @@ async function start() { const [session, users] = await Promise.all([api("api/session"), api("api/users")]); $("user").replaceChildren(...users.map((user) => new Option(user.name || user.id, user.id))); $("user").value = session.user ?? ""; - $("user").addEventListener("change", (event) => - run(() => api("api/sign-in", { method: "POST", body: { user: event.target.value } }))); + $("user").addEventListener("change", (event) => { + const user = event.target.value; + $("user").disabled = true; + run(async () => { + try { + while (saving.size) await Promise.all([...saving.values()].map((state) => state.done)); + await api("api/sign-in", { method: "POST", body: { user } }); + } finally { + $("user").disabled = false; + } + }); + }); + const add = $("new-note").querySelector("button[type=submit]"); $("new-note").addEventListener("submit", (event) => { event.preventDefault(); + if (add.disabled) return; const body = { title: $("new-title").value, body: $("new-body").value }; + add.disabled = true; run(async () => { - await api("api/notes", { method: "POST", body }); + try { + await api("api/notes", { method: "POST", body }); + } finally { + add.disabled = false; + } if ($("new-title").value === body.title) $("new-title").value = ""; if ($("new-body").value === body.body) $("new-body").value = ""; }); From 687c4a98852671eff1ab3589ac4fde5bb069ba7d Mon Sep 17 00:00:00 2001 From: William Loo Date: Tue, 3 Nov 2026 15:01:02 -0800 Subject: [PATCH 10/12] fix(website): the Notes page keeps the signed-in user when a save or sign-in fails, and reloads itself at its slash-ending address (FD-4683) Co-Authored-By: Claude Opus 5.5 (1M context) --- .../examples/website/notes/README.md | 2 +- .../examples/website/notes/web/index.html | 3 +++ .../examples/website/notes/web/src/main.js | 13 +++++++++++-- .../tests/test_website_notes_example.py | 1 + 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/packages/agentenv-protocol/examples/website/notes/README.md b/packages/agentenv-protocol/examples/website/notes/README.md index 3311112b0..67bac1ae3 100644 --- a/packages/agentenv-protocol/examples/website/notes/README.md +++ b/packages/agentenv-protocol/examples/website/notes/README.md @@ -8,7 +8,7 @@ them. The page shows the universe's time. | --- | --- | | `site.py` | The server: the `Notes` WebsiteEnvironment, its `@api` routes, and `serve()` on `PORT` (8000). | | `data/seed.json` | The seed data: two users (`ann`, `ben`), their notes, and `current_user`, the acting user. | -| `web/` | A minimal Vite app (`index.html`, `src/main.js`, `src/style.css`). Its assets (`base: "./"`) and API calls are relative to the page, so it works at `/` and under the gateway's `/website/notes/` alike. | +| `web/` | A minimal Vite app (`index.html`, `src/main.js`, `src/style.css`). Its assets (`base: "./"`) and API calls are relative to the page, so it works at `/` and under the gateway's `/website/notes/` alike; opened at a path without its final slash, such as `/website/notes`, the page first reloads itself with it. | | `Dockerfile` | The server image: Node builds `web/`, then Python installs this repo's `packages/agentenv-protocol` with uvicorn. It installs `curl`, which the gateway's compose health check runs. | | `env.toml` | The website env as a bundle entry. | diff --git a/packages/agentenv-protocol/examples/website/notes/web/index.html b/packages/agentenv-protocol/examples/website/notes/web/index.html index 1e498d3c1..468ef6385 100644 --- a/packages/agentenv-protocol/examples/website/notes/web/index.html +++ b/packages/agentenv-protocol/examples/website/notes/web/index.html @@ -4,6 +4,9 @@ Notes +
diff --git a/packages/agentenv-protocol/examples/website/notes/web/src/main.js b/packages/agentenv-protocol/examples/website/notes/web/src/main.js index d061a6cea..fafffe38f 100644 --- a/packages/agentenv-protocol/examples/website/notes/web/src/main.js +++ b/packages/agentenv-protocol/examples/website/notes/web/src/main.js @@ -49,6 +49,7 @@ function queueSave(id, fields) { } landed.set(id, renders); } catch (error) { + state.error = error; lastSent.set(id, {}); landed.delete(id); throw error; @@ -179,15 +180,23 @@ async function run(action) { async function start() { const [session, users] = await Promise.all([api("api/session"), api("api/users")]); $("user").replaceChildren(...users.map((user) => new Option(user.name || user.id, user.id))); - $("user").value = session.user ?? ""; + let signedInAs = session.user ?? ""; + $("user").value = signedInAs; $("user").addEventListener("change", (event) => { const user = event.target.value; $("user").disabled = true; run(async () => { try { - while (saving.size) await Promise.all([...saving.values()].map((state) => state.done)); + while (saving.size) { + const pending = [...saving.values()]; + await Promise.all(pending.map((state) => state.done)); + const failed = pending.find((state) => state.error); + if (failed) throw failed.error; + } await api("api/sign-in", { method: "POST", body: { user } }); + signedInAs = user; } finally { + $("user").value = signedInAs; $("user").disabled = false; } }); diff --git a/packages/agentenv-protocol/tests/test_website_notes_example.py b/packages/agentenv-protocol/tests/test_website_notes_example.py index e02147b10..c7ae6212d 100644 --- a/packages/agentenv-protocol/tests/test_website_notes_example.py +++ b/packages/agentenv-protocol/tests/test_website_notes_example.py @@ -220,6 +220,7 @@ def test_the_built_page_loads_its_assets_and_api_under_the_gateways_prefix() -> browser = TestClient(_behind_gateway_prefix(notes_site.Notes().create_app()), base_url="http://gateway.test") page = browser.get(page_url) assert page.status_code == 200 and page.headers["content-type"].startswith("text/html") + assert page.text.index('location.pathname.endsWith("/")') < page.text.index(" diff --git a/packages/agentenv-protocol/examples/website/notes/web/src/main.js b/packages/agentenv-protocol/examples/website/notes/web/src/main.js index fafffe38f..61ead9d31 100644 --- a/packages/agentenv-protocol/examples/website/notes/web/src/main.js +++ b/packages/agentenv-protocol/examples/website/notes/web/src/main.js @@ -6,6 +6,7 @@ let renders = 0; const saving = new Map(); const lastSent = new Map(); const landed = new Map(); +const drafts = new Map(); const CLOCK_REFRESH_MS = 5000; export function universeNow() { @@ -31,6 +32,21 @@ async function api(path, { method = "GET", body } = {}) { const $ = (id) => document.getElementById(id); +function setDraft(id, kind, value, saved) { + const draft = { ...drafts.get(id) }; + if (value === saved) delete draft[kind]; + else draft[kind] = value; + if (Object.keys(draft).length) drafts.set(id, draft); + else drafts.delete(id); +} + +function settleDraft(id, sent) { + const draft = drafts.get(id); + if (!draft) return; + for (const kind of Object.keys(sent)) if (draft[kind] === sent[kind]) delete draft[kind]; + if (!Object.keys(draft).length) drafts.delete(id); +} + function queueSave(id, fields) { lastSent.set(id, fields); const queued = saving.get(id); @@ -46,6 +62,7 @@ function queueSave(id, fields) { const next = state.next; state.next = null; await api(`api/notes/${id}`, { method: "PATCH", body: next }); + settleDraft(id, next); } landed.set(id, renders); } catch (error) { @@ -84,15 +101,18 @@ function noteItem(note) { done.setAttribute("aria-label", `Mark "${note.title}" ${note.done ? "not done" : "done"}`); done.addEventListener("change", () => run(() => api(`api/notes/${note.id}/toggle`, { method: "POST" }))); + const draft = drafts.get(note.id) ?? {}; const title = document.createElement("input"); title.className = "title"; title.defaultValue = note.title; + title.value = draft.title ?? note.title; title.setAttribute("aria-label", "Title"); const body = document.createElement("textarea"); body.className = "body"; body.rows = 2; body.defaultValue = note.body; + body.value = draft.body ?? note.body; body.placeholder = "Details"; body.setAttribute("aria-label", "Details"); @@ -101,7 +121,8 @@ function noteItem(note) { if (title.value === sent.title && body.value === sent.body) return; queueSave(note.id, { title: title.value, body: body.value }); }; - for (const field of [title, body]) { + for (const [kind, field] of [["title", title], ["body", body]]) { + field.addEventListener("input", () => setDraft(note.id, kind, field.value, note[kind])); field.addEventListener("change", save); field.addEventListener("blur", save); } @@ -112,7 +133,10 @@ function noteItem(note) { const remove = document.createElement("button"); remove.className = "delete"; remove.textContent = "Delete"; - remove.addEventListener("click", () => run(() => api(`api/notes/${note.id}`, { method: "DELETE" }))); + remove.addEventListener("click", () => run(async () => { + await api(`api/notes/${note.id}`, { method: "DELETE" }); + drafts.delete(note.id); + })); const text = document.createElement("div"); text.className = "text"; @@ -121,28 +145,19 @@ function noteItem(note) { return item; } -function editing() { - const kept = []; - for (const field of document.querySelectorAll("#notes .title, #notes .body")) { - const focused = field === document.activeElement; - if (focused || field.value !== field.defaultValue) { - kept.push({ id: field.closest("li").dataset.id, kind: field.className, value: field.value, focused, - start: field.selectionStart, end: field.selectionEnd }); - } - } - return kept; +function focused() { + const field = document.activeElement; + if (!field?.matches?.("#notes .title, #notes .body")) return null; + return { id: field.closest("li").dataset.id, kind: field.className, start: field.selectionStart, + end: field.selectionEnd }; } -function restore(kept) { - for (const { id, kind, value, focused, start, end } of kept) { - const field = document.querySelector(`#notes li[data-id="${id}"] .${kind}`); - if (!field) continue; - field.value = value; - if (focused) { - field.focus(); - field.setSelectionRange(start, end); - } - } +function refocus(kept) { + if (!kept) return; + const field = document.querySelector(`#notes li[data-id="${kept.id}"] .${kept.kind}`); + if (!field) return; + field.focus(); + field.setSelectionRange(kept.start, kept.end); } async function render() { @@ -156,9 +171,9 @@ async function render() { landed.delete(id); } } - const kept = editing(); + const kept = focused(); $("notes").replaceChildren(...notes.map(noteItem)); - restore(kept); + refocus(kept); if (!notes.length) { const empty = document.createElement("li"); empty.className = "empty"; @@ -195,6 +210,7 @@ async function start() { } await api("api/sign-in", { method: "POST", body: { user } }); signedInAs = user; + drafts.clear(); } finally { $("user").value = signedInAs; $("user").disabled = false;