Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 18 additions & 4 deletions packages/agentenv-protocol/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -757,8 +757,11 @@ are the profile's template. The base class adds:
- **Its settings**, `BrowserSettings`, which a subclass's `Settings` extends: `start_urls`, opened in
tabs at launch, the first active (`["about:blank"]` by default; none may start with `-`), headless
the first from its command line and the rest over its pipe, since headless Chromium takes one; `window`,
1280 × 800, alone only; `locale`; `time_zone`, set as `TZ` in its launch's `env`; and `geolocation` and
`notifications`, `deny` (the default), which answers a page's prompt for it, or `ask`.
1280 × 800, alone only; `locale`; `time_zone`, set as `TZ` in its launch's `env`; `geolocation` and
`notifications`, `deny` (the default), which answers a page's prompt for it, or `ask`; and `dialogs`, who
answers a page's JavaScript dialogs: `agent` (the default), or agentenv, accepting (`accept`, a `prompt()`
with its default text) or dismissing (`dismiss`) each. Either way agentenv leaves a page that asks before
it's left (`beforeunload`), so none of its answers blocks a navigation.
- **The flags agentenv owns**, after the subclass's command and before the start tabs: its profile, its
HTTP cache under `~/.cache` (`--disk-cache-dir`), `--remote-debugging-pipe`, `--no-first-run`,
`--no-default-browser-check`, `--lang` and, when `/dev/shm` holds under 1 GiB (Docker gives 64 MiB),
Expand Down Expand Up @@ -786,8 +789,19 @@ are the profile's template. The base class adds:
launches it again, and a launch while it runs, from its menu launcher say, opens another window over
the pipe rather than a second copy, which would hand its window to the first and exit. The ready check
`devtools_answers` asks `Browser.getVersion` over the pipe.
- **`browser/v1`** on its card, params only: `chromium` (its `version`), `clock: "real"` and, alone,
`window`.
- **Its control connection**, `self.control`, a `BrowserControl` over the pipe. At each launch it attaches
to every page with flattened sessions and keeps a record, `since` its server started: open tabs in
opened order, the one whose page has focus `active`; `closed_tabs`, each with `closed_at` and
`closed_by` (`agentenv`, over this connection; `unknown`, anyone else; `exit` or `stop`, when the browser
ended on its own or its server stopped it); `downloads`, into `~/Downloads`, each with its state
and, completed, its size and sha256; and `dialogs`, each with its type, message, answer and who gave it.
`hidden_page()` gives agentenv a page of its own for reading state, in no window and never in the record.
- **`browser/v1`** on its card: `chromium` (its `version`), `clock: "real"` and, alone, `window`; and
`record` (GET), framework-only, which returns that record with `launches`, the app server's log of each
launch (with its cause: `boot`, `server_restart`, `launch`, `data/add` or `data/reset`), each stop it made
and each exit it didn't, with its exit `code` or the `signal` that ended it, since the computer booted.
Nothing relaunches a browser that exits on its own: in a computer the agent can, and alone it's an infra
failure, its ready check false.
- **Its skill**: sites by their real names, files to upload in `~/Uploads` and downloads in
`~/Downloads`, folders it makes.

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,16 @@

from ..agent_env_environment import OP_ADD_DATA, OP_GET_DATA, OP_RESET_DATA
from ..transfers import WriteNamespaceGrant
from ..types import FilePart, uploaded_object_path
from ..types import METHOD_ADD, METHOD_RESET, FilePart, uploaded_object_path
from .dataplane import file_parts, part_bytes

if TYPE_CHECKING:
from .app_runtime import AppRuntime

logger = logging.getLogger(__name__)

#: The cause a launch after each data call logs.
_CAUSES = {OP_ADD_DATA: METHOD_ADD, OP_RESET_DATA: METHOD_RESET}
#: Where an app's server keeps its paths as installed, in the app's folder.
INITIAL = Path(".agentenv/initial")

Expand Down Expand Up @@ -80,7 +82,7 @@ async def _change(self, op: str, served: Callable, args: tuple, answer: asyncio.
raise
finally:
if running:
await self.runtime.launch_app()
await self.runtime.launch_app(_CAUSES[op])
except Exception as e:
if not answer.done():
answer.set_exception(e)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,13 @@

import asyncio
import inspect
import json
import os
import shlex
import signal
import time
from collections.abc import Awaitable, Callable
from datetime import datetime, timezone
from pathlib import Path
from typing import TYPE_CHECKING, Any, Optional

Expand All @@ -26,6 +29,11 @@
#: Where an app's server keeps the pid of the app it launched last, in the app's folder, so a server that restarts can
#: still stop it.
PID = Path(".agentenv/app.pid")
#: Where an app's server logs each launch, stop and exit of the app, beside its pid; a boot starts the app's folder as
#: its image has it, so the log is the boot's.
LAUNCHES = Path(".agentenv/launches")
#: How long an app that ended on its own may take to be reaped, for its exit code.
EXIT_WAIT_S = 5.0
#: How long a stopped app may take to exit before it's killed.
STOP_TIMEOUT_S = 10.0

Expand Down Expand Up @@ -59,6 +67,7 @@ def __init__(self, launch: Optional[Launch], ready_checks: dict[str, Callable[[]
self.launch_fds = launch_fds
self.relaunch_on_start = relaunch_on_start
self.launch_running = launch_running
self.exits: set[asyncio.Future] = set()
self.started = time.time()
self.launch_failed: Optional[str] = None
self.checking: dict[str, asyncio.Future] = {}
Expand All @@ -80,14 +89,14 @@ async def start(self) -> None:
try:
if relaunch:
await self.stop_app()
await self.launch_app()
await self.launch_app("server_restart" if relaunch else "boot")
except Exception as e:
self.launch_failed = f"{type(e).__name__}: {e}"

async def launch_app(self) -> int:
async def launch_app(self, cause: str = "launch") -> int:
"""Start the app from its folder, marked as launched, with the fds the app gives it, whose copies here are
closed once it has them; its pid. With ``launch_running``, an app this server launched that still runs does
that instead of starting a second copy, as a browser opens another window."""
closed once it has them, and logged with ``cause``; its pid. With ``launch_running``, an app this server
launched that still runs does that instead of starting a second copy, as a browser opens another window."""
if self.launch_running is not None and (running := _running()) is not None:
await self.launch_running()
return running
Expand All @@ -104,6 +113,7 @@ async def launch_app(self) -> int:
PID.write_text(str(pid))
_mark().parent.mkdir(parents=True, exist_ok=True)
_mark().touch()
self.log("launch", pid, cause=cause)
return pid

async def stop_app(self) -> bool:
Expand All @@ -115,9 +125,39 @@ async def stop_app(self) -> bool:
PID.unlink()
if not session.alive(pid):
return False
self.log("stop", pid)
await session.stop(pid, STOP_TIMEOUT_S)
return True

def exited(self) -> bool:
"""Log that the app this server launched last ended on its own, with its exit code or the signal that ended
it, unless the server stopped it; whether it did."""
if (pid := _recorded_pid()) is None:
return False
noting = asyncio.ensure_future(self._log_exit(pid))
self.exits.add(noting)
noting.add_done_callback(self.exits.discard)
return True

async def _log_exit(self, pid: int) -> None:
code = await session.returncode(pid, EXIT_WAIT_S)
self.log("exit", pid, **({} if code is None else
{"signal": signal.Signals(-code).name} if code < 0 else {"code": code}))

def log(self, event: str, pid: int, **fields: Any) -> None:
"""Append ``event`` for the app's ``pid`` to its launch log, which outlives its server for the boot."""
entry = {"at": datetime.now(timezone.utc).isoformat(), "event": event, "pid": pid, **fields}
_log().parent.mkdir(parents=True, exist_ok=True)
with _log().open("a", encoding="utf-8") as log:
log.write(json.dumps(entry) + "\n")

def launches(self) -> list[dict]:
"""Every launch, stop and exit of the app since the computer booted, in order."""
try:
return [json.loads(line) for line in _log().read_text(encoding="utf-8").splitlines() if line]
except FileNotFoundError:
return []

async def readiness(self) -> AppReadiness:
ready = self.launch.ready if self.launch is not None else None
checks: list[tuple[str, Awaitable[Optional[str]]]] = []
Expand Down Expand Up @@ -154,6 +194,17 @@ def _mark() -> Path:
return LAUNCHED / Path.cwd().name


def _log() -> Path:
return LAUNCHES


def _recorded_pid() -> Optional[int]:
try:
return int(PID.read_text())
except (FileNotFoundError, ValueError):
return None


def _launched_at() -> Optional[float]:
"""When the app was last launched since the computer booted, by its mark."""
try:
Expand All @@ -164,11 +215,8 @@ def _launched_at() -> Optional[float]:

def _running() -> Optional[int]:
"""The pid of the app this server launched last, while it runs."""
try:
pid = int(PID.read_text())
except (FileNotFoundError, ValueError):
return None
return pid if session.alive(pid) else None
pid = _recorded_pid()
return pid if pid is not None and session.alive(pid) else None


async def _bounded(name: str, check: Awaitable[Optional[str]]) -> Optional[str]:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
from pydantic import TypeAdapter, ValidationError

from ..agent_env_environment import add_data, get_data
from ..transfers import NamespaceUploader, ReadObject, RelativePath, WriteNamespaceGrant, download
from ..transfers import NamespaceUploader, ReadObject, RelativePath, TransferError, WriteNamespaceGrant, download
from ..types import DATA_OBJECTS_EXTENSION_URI, FilePart, FileWithBytes, FileWithUri, uploaded_file_part, uploaded_object_path

if TYPE_CHECKING:
Expand Down Expand Up @@ -64,22 +64,31 @@ async def _computer_data_get(self, write_namespace: Optional[WriteNamespaceGrant
async def file_parts(home: Path, paths: list[str], write_namespace: Optional[WriteNamespaceGrant],
leaving_out: list[str] = ()) -> list:
"""A part for each file under ``paths`` of ``home`` but under none of ``leaving_out``, naming its path relative to
``home``: uploaded under ``write_namespace`` when there is one, else inline."""
``home``: uploaded under ``write_namespace`` when there is one, else inline. A file a running app removes before it's
read is left out."""
files = [(relative, source) for relative, source in _data_files(home, paths)
if not any(PurePosixPath(relative).is_relative_to(owned) for owned in leaving_out)]
parts = []
if write_namespace is not None:
uploader = NamespaceUploader(write_namespace)
parts = []
for relative, source in files:
await uploader.upload(relative, source)
try:
await uploader.upload(relative, source)
except (TransferError, OSError):
if source.exists():
raise
continue
parts.append(uploaded_file_part(relative, name=source.name, mime_type=_mime(source)))
return parts
return [
FilePart(file=FileWithBytes(bytes=base64.b64encode(source.read_bytes()).decode("ascii"),
name=source.name, mimeType=_mime(source)),
metadata={DATA_OBJECTS_EXTENSION_URI: {"path": relative}})
for relative, source in files
]
for relative, source in files:
try:
data = source.read_bytes()
except FileNotFoundError:
continue
parts.append(FilePart(file=FileWithBytes(bytes=base64.b64encode(data).decode("ascii"), name=source.name,
mimeType=_mime(source)),
metadata={DATA_OBJECTS_EXTENSION_URI: {"path": relative}}))
return parts


async def part_bytes(part: FilePart) -> bytes:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@
WITH_FDS = Path(__file__).with_name("_with_fds.py")
#: The processes launched into the session, kept until they exit so each is reaped.
_running: set[asyncio.Task] = set()
#: Each launched process's reaping, by pid, until its exit is asked for: its return code, negative for a signal.
_exits: dict[int, asyncio.Future] = {}
#: How many exits no one has asked for are kept, the latest.
KEPT_EXITS = 16


async def wait_until_up() -> None:
Expand All @@ -38,12 +42,35 @@ async def launch(argv: list[str], cwd: Path, env: dict[str, str], fds: Optional[
reaped = asyncio.ensure_future(process.wait())
_running.add(reaped)
reaped.add_done_callback(_running.discard)
_exits[process.pid] = reaped
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
reaped.add_done_callback(lambda _: _forget_old_exits())
return process.pid


def _forget_old_exits() -> None:
"""Drop all but the latest ``KEPT_EXITS`` exits no one has asked for."""
for pid in [pid for pid, reaped in _exits.items() if reaped.done()][:-KEPT_EXITS]:
del _exits[pid]


async def returncode(pid: int, timeout_s: float) -> Optional[int]:
"""The return code the process ``launch`` started as ``pid`` exited with, negative for the signal that ended it,
once it's reaped within ``timeout_s``; None for one ``launch`` didn't start or that hasn't exited."""
reaped = _exits.get(pid)
if reaped is None:
return None
try:
code = await asyncio.wait_for(asyncio.shield(reaped), timeout_s)
except TimeoutError:
return None
_exits.pop(pid, None)
return code


async def stop(pid: int, timeout_s: float) -> None:
"""End the session ``launch`` started as ``pid``, every process in it: asked to exit, then killed after
``timeout_s``."""
``timeout_s``. No one asks how it exited once it's stopped."""
_exits.pop(pid, None)
try:
os.killpg(pid, signal.SIGTERM)
except ProcessLookupError:
Expand Down
Loading
Loading