From c07358888ba75e4a360f2b2f2239dde53326e953 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 16:20:02 -0800 Subject: [PATCH 01/21] feat(computer): the ubuntu-24.04 disk form and Linux disk composition The ubuntu-24.04 base gets a disk form: a GPT disk of an ESP holding one UKI (Ubuntu's kernel and a busybox initramfs) and an ext4 root that is the container form's root filesystem plus /etc/agentenv/vm/image.env. Its init reads a NoCloud seed and switch_roots into the image's entrypoint; UEFI only. Composing builds each arch's disk after its container with the disk tools (mkfs.ext4 -d, unprivileged, on the builder's own platform), keeping the base disk's ESP, keyed by the container's key, the base disk and the recipe. Gate 3 probes a disk where a provider boots disks natively; elsewhere it stays unverified and a deploy's first boot with KVM probes it. A deploy naming form = "disk" boots it through create_vm(image=..., env=...): on local Docker, the QEMU launcher with /dev/kvm. The release publishes the disk tools, the launcher and the base's disk by a digest over its sources, and boots the disk before pushing it. FD-4363 (M4.1). Co-Authored-By: Claude Opus 5.5 --- .github/dependabot.yml | 3 + .github/scripts/infra_build.py | 78 +++++- .github/scripts/infra_images.py | 23 +- .github/workflows/local-backends.yml | 6 + .github/workflows/publish-pypi.yml | 132 +++++++++- AGENTS.md | 9 +- src/agent_env/artifact/artifacts/computer.py | 104 +++++++- src/agent_env/computers/README.md | 12 + .../artifacts/ubuntu-24.04/artifact.toml | 6 + .../computers/disk/linux/boot/Dockerfile | 39 +++ src/agent_env/computers/disk/linux/boot/init | 65 +++++ .../computers/disk/linux/boot/udhcpc.script | 11 + .../computers/disk/linux/boot/uki.sh | 39 +++ .../computers/disk/linux/launcher/Dockerfile | 9 + .../computers/disk/linux/launcher/launch | 46 ++++ .../computers/disk/linux/launcher/seed | 12 + .../computers/disk/linux/tools/Dockerfile | 7 + src/agent_env/computers/disk/linux/tools/disk | 47 ++++ .../computers/disk/linux/tools/image-env | 23 ++ src/agent_env/env/bootstrap.py | 15 +- src/agent_env/env/envs/computer/env.py | 20 +- src/agent_env/env/envs/computer/utils/boot.py | 37 ++- .../env/envs/computer/utils/builder.py | 42 +++- .../env/envs/computer/utils/composer.py | 215 +++++++++++++--- src/agent_env/env/published_images.py | 45 +++- .../env_providers/env_computer_provider.py | 28 ++- .../providers/sandbox_providers/AGENTS.md | 4 +- .../sandbox_providers/local_docker/disk.py | 57 +++++ .../local_docker/provider.py | 38 +++ .../sandbox_providers/sandbox_provider.py | 13 + .../store/image_store/image_contents.py | 54 +++- .../store/image_store/registry_api.py | 20 ++ .../computer/disk/rootfs/Dockerfile | 6 + .../computer/disk/tasks/probe.json | 6 + tst/integration/computer/disk_boot_test.py | 36 +++ tst/integration/computer/disk_test.py | 133 ++++++++++ tst/unit/artifact/test_computer_artifact.py | 2 +- tst/unit/bundle/computer_env_folder_test.py | 16 +- tst/unit/env/envs/_computer.py | 34 ++- tst/unit/env/envs/conftest.py | 10 +- tst/unit/env/envs/test_computer_disk.py | 238 ++++++++++++++++++ tst/unit/env/envs/test_disk_recipe.py | 106 ++++++++ tst/unit/env/published_images_test.py | 37 ++- tst/unit/test_infra_build_script.py | 43 +++- tst/unit/test_infra_images_script.py | 24 +- tst/unit/test_release_infra_images.py | 27 +- tst/util/capabilities.py | 16 ++ tst/util/oci_registry.py | 6 +- 48 files changed, 1846 insertions(+), 153 deletions(-) create mode 100644 src/agent_env/computers/disk/linux/boot/Dockerfile create mode 100755 src/agent_env/computers/disk/linux/boot/init create mode 100755 src/agent_env/computers/disk/linux/boot/udhcpc.script create mode 100755 src/agent_env/computers/disk/linux/boot/uki.sh create mode 100644 src/agent_env/computers/disk/linux/launcher/Dockerfile create mode 100755 src/agent_env/computers/disk/linux/launcher/launch create mode 100755 src/agent_env/computers/disk/linux/launcher/seed create mode 100644 src/agent_env/computers/disk/linux/tools/Dockerfile create mode 100755 src/agent_env/computers/disk/linux/tools/disk create mode 100755 src/agent_env/computers/disk/linux/tools/image-env create mode 100644 src/agent_env/providers/sandbox_providers/local_docker/disk.py create mode 100644 tst/integration/computer/disk/rootfs/Dockerfile create mode 100644 tst/integration/computer/disk/tasks/probe.json create mode 100644 tst/integration/computer/disk_boot_test.py create mode 100644 tst/integration/computer/disk_test.py create mode 100644 tst/unit/env/envs/test_computer_disk.py create mode 100644 tst/unit/env/envs/test_disk_recipe.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 6b1884986..49d3f28dd 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -56,6 +56,9 @@ updates: - "/src/agent_env/agents/agents/pydantic-ai" - "/src/agent_env/examples/mail-mcp/envs/mail" - "/src/agent_env/computers/artifacts/ubuntu-24.04" + - "/src/agent_env/computers/disk/linux/boot" + - "/src/agent_env/computers/disk/linux/tools" + - "/src/agent_env/computers/disk/linux/launcher" schedule: interval: "weekly" open-pull-requests-limit: 5 diff --git a/.github/scripts/infra_build.py b/.github/scripts/infra_build.py index c4ba84fa8..155117641 100644 --- a/.github/scripts/infra_build.py +++ b/.github/scripts/infra_build.py @@ -4,8 +4,11 @@ images agent-env ships before it's built, pushed or recorded. ``check ENTRY`` fails unless ENTRY, a JSON object as ``infra_images.py plan`` lists an image, names an image agent-env -ships, with its Dockerfile, context, repository and a source tag. ``exists REF`` exits 0 when the registry has REF, 1 -when it says it doesn't, and 2 when it can't tell. ``smoke NAME IMAGE`` checks that IMAGE, a local build of NAME, runs. +ships, with its Dockerfile, context, repository and a source tag, and for a disk form, the build args naming its base +computer's image and the disk tools by their repositories and source tags. ``exists REF`` exits 0 when the registry has +REF, 1 when it says it doesn't, and 2 when it can't tell. ``config IMAGE`` prints the config object of IMAGE's OCI +config, base64'd, as a disk form's build takes its base's. ``smoke NAME IMAGE`` checks that IMAGE, a local build of +NAME, runs: a disk form boots to its desktop in the QEMU launcher built here as ``agentenv-qemu-launcher``. ``verify-record PATH`` fails unless PATH records each image agent-env ships once, with the digest the registry serves under its tag now. ``retry COMMAND...`` runs COMMAND, again while a registry rate-limits it or PyPI doesn't serve yet the agentenv-framework-protocol an agent image's build installs. @@ -14,6 +17,7 @@ from __future__ import annotations import argparse +import base64 import json import posixpath import re @@ -43,12 +47,20 @@ "agentenv-pydantic-ai": "src/agent_env/agents/agents/pydantic-ai/Dockerfile", "agentenv-mail-mcp": "src/agent_env/examples/mail-mcp/envs/mail/Dockerfile", "agentenv-ubuntu-24.04": "src/agent_env/computers/artifacts/ubuntu-24.04/Dockerfile", + "agentenv-disk-tools": "src/agent_env/computers/disk/linux/tools/Dockerfile", + "agentenv-qemu-launcher": "src/agent_env/computers/disk/linux/launcher/Dockerfile", + "agentenv-ubuntu-24.04-disk": "src/agent_env/computers/disk/linux/boot/Dockerfile", } +# Each disk form's build args, by the images agent-env ships that they name. +DISK_ARGS = {"agentenv-ubuntu-24.04-disk": {"ROOTFS": "agentenv-ubuntu-24.04", "TOOLS": "agentenv-disk-tools"}} _TAG = re.compile(r"^src-[0-9a-f]{64}$") _DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$") _HEALTHY_WITHIN_SECONDS = 120 _HEALTH_POLL_SECONDS = 2 _LOG_TAIL_LINES = 30 # of a container that never turned healthy, in the error +# How long a disk form may take to boot to its desktop: seconds with KVM, minutes under TCG. +_BOOTED_WITHIN_SECONDS = 900 +_BOOTED = re.compile(r"success: (xvfb|dbus|desktop) entered RUNNING") _PRINTED_CHARS = 200 # of what docker printed in place of a manifest, in the error # What a registry answers when it rate-limits a request: ECR Public's 429, and the toomanyrequests error code GHCR and # Docker Hub send. @@ -94,6 +106,11 @@ " s=$(supervisorctl -c /etc/supervisor/supervisord.conf status);" " echo \"$s\" | grep -q ' RUNNING ' && ! echo \"$s\" | grep -qv ' RUNNING ' && exit 0;" " done; supervisorctl -c /etc/supervisor/supervisord.conf status; exit 1"], + "agentenv-disk-tools": ["docker", "run", "--rm", "--entrypoint", "sh", "{image}", "-c", + "command -v disk image-env mkfs.ext4 mkfs.vfat mcopy sfdisk qemu-img"], + "agentenv-qemu-launcher": ["docker", "run", "--rm", "--entrypoint", "sh", "{image}", "-c", + "qemu-system-x86_64 --version && qemu-system-aarch64 --version" + " && test -f /usr/share/OVMF/OVMF_CODE_4M.fd && test -f /usr/share/AAVMF/AAVMF_CODE.fd"], } @@ -102,6 +119,7 @@ def main() -> int: commands = parser.add_subparsers(dest="command", required=True) commands.add_parser("check").add_argument("entry") commands.add_parser("exists").add_argument("ref") + commands.add_parser("config").add_argument("image") smoke = commands.add_parser("smoke") smoke.add_argument("name", choices=sorted(IMAGES)) smoke.add_argument("image") @@ -112,6 +130,9 @@ def main() -> int: return _tried_again(lambda: _streamed(args.argv)).returncode if args.command == "exists": return _exists(args.ref) + if args.command == "config": + print(_config(args.image)) + return 0 if args.command == "smoke": return _smoke(args.name, args.image) problems = _check(args.entry) if args.command == "check" else _verify_record(args.path) @@ -134,11 +155,26 @@ def _check(entry: str) -> list[str]: for key, value in expected.items() if image.get(key) != value] if not _TAG.match(str(image.get("tag"))): problems.append(f"{name}: {image.get('tag')!r} isn't a source tag") - if extra := sorted(set(image) - {*expected, "tag"}): + if name in DISK_ARGS: + problems += _check_args(name, image.get("args")) + if extra := sorted(set(image) - {*expected, "tag", *(("args",) if name in DISK_ARGS else ())}): problems.append(f"{name}: unexpected {', '.join(extra)}") return problems +def _check_args(name: str, args: object) -> list[str]: + """The problems with a disk form's build args: each must name the image agent-env ships it's for, by a source tag.""" + wanted = DISK_ARGS[name] + if not isinstance(args, dict) or set(args) != set(wanted): + return [f"{name}: args is {args!r}, not {', '.join(sorted(wanted))}"] + problems = [] + for arg, image in wanted.items(): + repository, _, tag = str(args[arg]).rpartition(":") + if repository != f"{REGISTRY}/{image}" or not _TAG.match(tag): + problems.append(f"{name}: {arg} is {args[arg]!r}, not {REGISTRY}/{image} under a source tag") + return problems + + def _tried_again(attempt: Callable[[], subprocess.CompletedProcess]) -> subprocess.CompletedProcess: """``attempt()``, and again after each of ``_RETRY_WAITS_SECONDS`` while what it printed says a later try can work (``_why_again``).""" @@ -232,9 +268,18 @@ def _verify_record(path: Path) -> list[str]: return problems +def _config(image: str) -> str: + """The config object of ``image``'s OCI config, a local image's, as base64'd JSON.""" + inspected = subprocess.run(["docker", "image", "inspect", "--format", "{{json .Config}}", image], + capture_output=True, text=True, check=True) + return base64.b64encode(inspected.stdout.strip().encode()).decode() + + def _smoke(name: str, image: str) -> int: if name == "agentenv-service-db": return _healthy(image) + if name in DISK_ARGS: + return _boots(image) command = [part.replace("{image}", image) for part in _SMOKES[name]] result = subprocess.run(command, capture_output=True, text=True) if result.returncode != 0: @@ -242,6 +287,33 @@ def _smoke(name: str, image: str) -> int: return result.returncode +def _boots(image: str) -> int: + """Boot ``image``, a disk form, in the launcher built here, with KVM where it can, and wait for its desktop's + programs to run, which its console shows.""" + arch = subprocess.run(["docker", "image", "inspect", "--format", "{{.Architecture}}", image], capture_output=True, + text=True, check=True).stdout.strip() + machine = f"{image}-machine" + subprocess.run(["docker", "build", "-q", "-t", machine, "-"], text=True, check=True, capture_output=True, + input=f"FROM agentenv-qemu-launcher\nCOPY --from={image} /disk/ /disk/\n") + kvm = ["--device", "/dev/kvm"] if Path("/dev/kvm").exists() else [] + container = subprocess.run(["docker", "run", "-d", *kvm, "-e", f"VM_ARCH={arch}", machine], capture_output=True, + text=True, check=True).stdout.strip() + try: + deadline, logs = time.monotonic() + _BOOTED_WITHIN_SECONDS, "" + while time.monotonic() < deadline: + logs = subprocess.run(["docker", "logs", container], capture_output=True, text=True, errors="replace") + logs = logs.stdout + logs.stderr + if len(set(_BOOTED.findall(logs))) == 3: + print(f"{image} booted to its desktop{' with KVM' if kvm else ' under TCG'}") + return 0 + time.sleep(_HEALTH_POLL_SECONDS) + tail = "\n".join(logs.splitlines()[-_LOG_TAIL_LINES:]) + print(f"::error::{image} didn't boot to its desktop within {_BOOTED_WITHIN_SECONDS}s:\n{tail}") + return 1 + finally: + subprocess.run(["docker", "rm", "-f", container], capture_output=True) + + def _healthy(image: str) -> int: """Start ``image`` and wait for its health check to pass, as compose does before starting what depends on it.""" container = subprocess.run(["docker", "run", "-d", image], capture_output=True, text=True, check=True).stdout.strip() diff --git a/.github/scripts/infra_images.py b/.github/scripts/infra_images.py index 6c520cade..ea6bb22d4 100644 --- a/.github/scripts/infra_images.py +++ b/.github/scripts/infra_images.py @@ -2,11 +2,13 @@ the release workflow's jobs that only read: this imports agent-env, so it runs where agent-env and its dependencies are installed, and no job that holds a token to push images or publish to PyPI runs it (those run ``infra_build.py``). -``plan`` prints each image's name, repository, tag, Dockerfile and context as JSON, ``--missing`` only those the +``plan`` prints each image's name, repository, tag, Dockerfile and context as JSON, and for an image built over other +published images, the build args naming each by its repository and tag, ``--missing`` only those the registry hasn't published: it has no image under the tag, or refuses to show one, as GHCR does for a package that -doesn't exist yet, so the first publish can start. ``--only infra`` lists only the infra images, and ``--only agents`` +doesn't exist yet, so the first publish can start. ``--only infra`` lists only the infra images, ``--only agents`` only the images that install the protocol, the agents' and the website browser's, which a release builds after it -publishes the protocol, as their protocol floor may name it. ``write PATH`` records the digest the registry serves under every +publishes the protocol, as their protocol floor may name it, and ``--only disks`` the base computers' disk forms, built +over their published images. ``write PATH`` records the digest the registry serves under every image's tag as a release wheel carries them, and fails if any is missing. ``public`` warns about each image a user, who pulls anonymously, can't. The registry is read with ``GHCR_TOKEN`` (as ``GITHUB_ACTOR``) when it's set, anonymously otherwise, and always anonymously by ``public``. @@ -21,7 +23,7 @@ from pathlib import Path from agent_env.env.bootstrap import PLAYWRIGHT_MCP_DOCKERFILE, PUBLISHED_IMAGES -from agent_env.env.published_images import BUNDLE_IMAGES, REGISTRY, PublishedImage, write_release_digests +from agent_env.env.published_images import BUNDLE_IMAGES, DISK_IMAGES, REGISTRY, PublishedImage, write_release_digests from agent_env.store.image_store.oci_registry_credentials import RegistryAuth from agent_env.store.image_store.registry_api import NoSuchImageError, RegistryRefusedError, pin_digest @@ -30,7 +32,8 @@ # example env's, and the Playwright MCP browser's, whose recorder installs it, matched by its Dockerfile. _AFTER_PROTOCOL = BUNDLE_IMAGES + tuple(image for image in PUBLISHED_IMAGES if image.dockerfile == PLAYWRIGHT_MCP_DOCKERFILE) -_GROUPS = {"infra": tuple(image for image in PUBLISHED_IMAGES if image not in _AFTER_PROTOCOL), "agents": _AFTER_PROTOCOL} +_GROUPS = {"infra": tuple(image for image in PUBLISHED_IMAGES if image not in _AFTER_PROTOCOL + DISK_IMAGES), + "agents": _AFTER_PROTOCOL, "disks": DISK_IMAGES} def main() -> int: @@ -59,9 +62,13 @@ def main() -> int: return 0 -def _described(image: PublishedImage) -> dict[str, str]: - return {"name": image.name, "repository": image.repository, "tag": image.tag(), - "dockerfile": str(image.dockerfile.relative_to(REPO)), "context": str(image.dockerfile.parent.relative_to(REPO))} +def _described(image: PublishedImage) -> dict: + described = {"name": image.name, "repository": image.repository, "tag": image.tag(), + "dockerfile": str(image.dockerfile.relative_to(REPO)), + "context": str(image.dockerfile.parent.relative_to(REPO))} + if image.args: + described["args"] = {arg: built.ref() for arg, built in image.args} + return described def _digest(image: PublishedImage) -> str | None: diff --git a/.github/workflows/local-backends.yml b/.github/workflows/local-backends.yml index 0b68f8a6a..84c77616f 100644 --- a/.github/workflows/local-backends.yml +++ b/.github/workflows/local-backends.yml @@ -138,6 +138,12 @@ jobs: run: | docker run --detach --name registry --publish 5000:5000 mirror.gcr.io/library/registry:2 timeout 30 sh -c 'until curl -sf http://localhost:5000/v2/ >/dev/null; do sleep 1; done' + # x64 hosted runners have /dev/kvm; the disk form's tests boot under it, and skip without it, which fails below. + - name: Let the local provider boot disks with KVM + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm.rules + sudo udevadm control --reload-rules && sudo udevadm trigger --name-match=kvm + test -w /dev/kvm - name: Slow integration tier on the local defaults env: AGENTENV_TEST_SHARD: ${{ matrix.shard }}/8 diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index 752c5c71b..ec971b824 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -9,11 +9,12 @@ name: Publish to PyPI # rerun is safe. # # It also publishes the images agent-env ships (agent_env.env.published_images), its infra images and its bundles' -# (its agents' and the mail example's env), +# (its agents' and the mail example's env), and its base computers' disk forms, # whose tags the registry lacks, and records their digests in the wheel and the release notes, once the repository # variable PUBLISH_INFRA_IMAGES is "true". The infra images are published before agentenv-framework-protocol. The # bundles' images keep a protocol floor, so each is built only when its folder changes, once the protocol is on PyPI, -# as a raised floor may name the protocol this release ships. agentenv-framework waits for every image. Every job that +# as a raised floor may name the protocol this release ships. A disk form is built over its base computer's published +# image with the published disk tools, so after both. agentenv-framework waits for every image. Every job that # reads the shipped bundles' folders first pins the protocol in their envs and artifacts as the wheel does # (.github/scripts/shipped_bundles.py pin), so an image's tag, its build and the wheel all see the same sources. Run # by hand on main, the workflow publishes the images alone, with nothing sent to PyPI; an agent's image then installs @@ -71,6 +72,7 @@ jobs: outputs: missing_infra: ${{ steps.plan.outputs.missing_infra }} missing_agents: ${{ steps.plan.outputs.missing_agents }} + missing_disks: ${{ steps.plan.outputs.missing_disks }} steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: @@ -90,8 +92,10 @@ jobs: uv sync --locked --no-dev missing_infra=$(uv run --no-sync python .github/scripts/infra_images.py plan --missing --only infra) missing_agents=$(uv run --no-sync python .github/scripts/infra_images.py plan --missing --only agents) + missing_disks=$(uv run --no-sync python .github/scripts/infra_images.py plan --missing --only disks) echo "missing_infra=$missing_infra" >> "$GITHUB_OUTPUT" echo "missing_agents=$missing_agents" >> "$GITHUB_OUTPUT" + echo "missing_disks=$missing_disks" >> "$GITHUB_OUTPUT" # Once main's CI has passed, so nothing is pushed for a commit that fails it: builds each missing infra image on a # runner of each platform, checks that it runs, and pushes it under its tag and platform (src--amd64), which @@ -343,17 +347,139 @@ jobs: "${ref}-amd64" "${ref}-arm64" done + # Each base computer's disk form whose tag the registry lacks, built on a runner of each platform from the base's + # published image of that platform and the published disk tools, booted with the QEMU launcher built from this + # checkout (with KVM on x64 runners, under TCG where a runner has none, arm64's), and pushed under its tag and + # platform as infra-build does; disk-publish then tags the two as one multi-platform image. + disk-build: + needs: [main-ci, infra-plan, infra-publish, agent-publish] + if: >- + !cancelled() && needs.main-ci.result == 'success' && needs.infra-plan.result == 'success' + && needs.infra-plan.outputs.missing_disks != '[]' + && contains(fromJSON('["success", "skipped"]'), needs.infra-publish.result) + && contains(fromJSON('["success", "skipped"]'), needs.agent-publish.result) + strategy: + fail-fast: true + matrix: + image: ${{ fromJSON(needs.infra-plan.outputs.missing_disks) }} + arch: [amd64, arm64] + runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + timeout-minutes: 60 + permissions: + contents: read + packages: write + env: + ENTRY: ${{ toJSON(matrix.image) }} + NAME: ${{ matrix.image.name }} + DOCKERFILE: ${{ matrix.image.dockerfile }} + CONTEXT: ${{ matrix.image.context }} + TAG: ${{ matrix.image.tag }} + REF: ${{ matrix.image.repository }}:${{ matrix.image.tag }} + BUILT: ${{ matrix.image.repository }}:${{ matrix.image.tag }}-${{ matrix.arch }} + ROOTFS: ${{ matrix.image.args.ROOTFS }}-${{ matrix.arch }} + TOOLS: ${{ matrix.image.args.TOOLS }} + PLATFORM: linux/${{ matrix.arch }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + - name: Pin agentenv-framework-protocol in the shipped bundles, as the release does + run: python3 .github/scripts/shipped_bundles.py pin + - name: Check that it's an image agent-env ships, and that no run has published it + id: wanted + env: + GHCR_TOKEN: ${{ github.token }} + run: | + python3 .github/scripts/infra_build.py check "$ENTRY" + echo "$GHCR_TOKEN" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin + code=0 + python3 .github/scripts/infra_build.py exists "$REF" || code=$? + case "$code" in + 0) echo "build=false" >> "$GITHUB_OUTPUT"; echo "$REF is published already, so it isn't built again" ;; + 1) echo "build=true" >> "$GITHUB_OUTPUT" ;; + *) exit 1 ;; + esac + - name: Pull its base images, and the launcher's, from mirror.gcr.io, as public.ecr.aws can rate-limit a runner + if: steps.wanted.outputs.build == 'true' + run: python3 .github/scripts/mirror_base_images.py "$DOCKERFILE" src/agent_env/computers/disk/linux/launcher/Dockerfile + - name: Let the launcher use KVM, where the runner has it + if: steps.wanted.outputs.build == 'true' + run: | + if [ -e /dev/kvm ]; then + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm.rules + sudo udevadm control --reload-rules && sudo udevadm trigger --name-match=kvm + fi + - name: Build + if: steps.wanted.outputs.build == 'true' + run: | + python3 .github/scripts/infra_build.py retry docker pull --platform "$PLATFORM" "$ROOTFS" + config=$(python3 .github/scripts/infra_build.py config "$ROOTFS") + # No provenance attestation, which would show in the multi-platform image as an unknown platform. + python3 .github/scripts/infra_build.py retry \ + docker build --platform "$PLATFORM" --provenance=false -f "$DOCKERFILE" -t "$BUILT" \ + --build-arg "ROOTFS=$ROOTFS" --build-arg "TOOLS=$TOOLS" --build-arg "ROOTFS_CONFIG=$config" \ + --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \ + --label "org.opencontainers.image.revision=${GITHUB_SHA}" \ + --label "io.agentenv.source-tag=${TAG}" "$CONTEXT" + - name: Check that it runs + if: steps.wanted.outputs.build == 'true' + run: | + docker build -q -t agentenv-qemu-launcher src/agent_env/computers/disk/linux/launcher + python3 .github/scripts/infra_build.py smoke "$NAME" "$BUILT" + - name: Push it under its tag and platform + if: steps.wanted.outputs.build == 'true' + run: python3 .github/scripts/infra_build.py retry docker push "$BUILT" + + # Each new disk form's platform builds tagged as one multi-platform image, as infra-publish does the infra images. + disk-publish: + needs: [main-ci, infra-plan, disk-build] + if: >- + !cancelled() && needs.main-ci.result == 'success' && needs.disk-build.result == 'success' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + packages: write + env: + MISSING: ${{ needs.infra-plan.outputs.missing_disks }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + - name: Tag each image's platform builds as one multi-platform image under its source tag + env: + GHCR_TOKEN: ${{ github.token }} + run: | + echo "$GHCR_TOKEN" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin + jq -c '.[]' <<< "$MISSING" | while read -r entry; do + python3 .github/scripts/infra_build.py check "$entry" + ref=$(jq -r '"\(.repository):\(.tag)"' <<< "$entry") + code=0 + python3 .github/scripts/infra_build.py exists "$ref" || code=$? + if [ "$code" = 0 ]; then + echo "$ref is published already, so it's left as it is" + continue + fi + [ "$code" = 1 ] || exit 1 + python3 .github/scripts/infra_build.py retry docker buildx imagetools create -t "$ref" \ + --annotation "index:org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \ + --annotation "index:org.opencontainers.image.revision=${GITHUB_SHA}" \ + "${ref}-amd64" "${ref}-arm64" + done + # The digest the registry serves under every image's tag, for the wheel and the release notes, which the framework # job checks against the registry before the wheel carries it; and a warning for each image an anonymous pull, as an # install makes, doesn't get: GHCR keeps a new package private until its admin makes it public. infra-record: - needs: [infra-plan, infra-build, infra-publish, agent-build, agent-publish] + needs: [infra-plan, infra-build, infra-publish, agent-build, agent-publish, disk-build, disk-publish] if: >- !cancelled() && needs.infra-plan.result == 'success' && contains(fromJSON('["success", "skipped"]'), needs.infra-build.result) && contains(fromJSON('["success", "skipped"]'), needs.infra-publish.result) && contains(fromJSON('["success", "skipped"]'), needs.agent-build.result) && contains(fromJSON('["success", "skipped"]'), needs.agent-publish.result) + && contains(fromJSON('["success", "skipped"]'), needs.disk-build.result) + && contains(fromJSON('["success", "skipped"]'), needs.disk-publish.result) runs-on: ubuntu-latest timeout-minutes: 10 permissions: diff --git a/AGENTS.md b/AGENTS.md index bf0168d1a..db1040ea2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -132,7 +132,7 @@ and deserialized through a registry. | `explorer/` | Optional local web UI: `agent-env up`, binds loopback `:8234`. | | `cli/` | Click CLI with the groups `a2a-agent`, `artifact`, `bundle`, `config`, `env`, `eval`, `plugin`, `run`, `task`, `up`. `task run`, `task run-batch` and `eval run` hand their runs to the configured `[runner]` and follow them. | | `agents/` | The `agents` bundle: the agents agent-env ships, one folder each under `agents/agents/`, with a smoke task each under `agents/tasks/`. Its Claude Code, `@local/agentenv-framework/agents/claude-code`, is the default agent. An agent's Python packages are its folder's `requirements.txt`, installed in its image, never in agent-env's own environment. A run that pulls one of its agents reads the whole bundle, so `tst/unit/bundle/shipped_bundles_test.py` parses and checks every shipped bundle. | -| `computers/` | The `computers` bundle: the base computers agent-env ships as `computer` artifacts, `ubuntu-24.04` (XFCE on Xvfb, supervisord as the session user, settings as `AGENTENV_SETTING_`), aliased through `[artifacts]`. It has no tasks either. | +| `computers/` | The `computers` bundle: the base computers agent-env ships as `computer` artifacts, `ubuntu-24.04` (XFCE on Xvfb, supervisord as the session user, settings as `AGENTENV_SETTING_`), aliased through `[artifacts]`. It has no tasks either. `disk/linux/` is the Linux disk form's recipe: `boot/`, whose Dockerfile makes a disk form (a GPT disk of an EFI system partition holding one UKI, Ubuntu's kernel with a busybox initramfs whose init reads a NoCloud seed and `switch_root`s into the image's entrypoint, and an ext4 root filesystem that is a container image's, plus `/etc/agentenv/vm/image.env` from its OCI config), `tools/`, the disk tools that make it unprivileged on the builder's own platform (`mkfs.ext4 -d`), and `launcher/`, the QEMU launcher that boots one under KVM. A base's `[forms.disk]` with no image is made from its container form; the composer makes each arch's disk from its composed container, keeping the base disk's EFI system partition, keyed by the container's key, the base disk and the recipe; gate 3 probes it where a provider boots disks natively (`disk_image_platforms()`), and a deploy's first boot with KVM probes one left `unverified`; a deploy naming `form = "disk"` boots it through `create_vm(image=..., env=...)`, its settings in the seed. | | `browsers/` | The `browsers` bundle: the website browsers agent-env ships, today `chromium`, a `computer_app` env (Chrome for Testing at the build Playwright pins, for amd64 and arm64, set up for unattended use by managed policies in `/etc/opt/chrome_for_testing/policies/managed`, opened at boot on its `start_urls`, its profile its data). All of it, its pin, install step, policies and command, is in `envs/chromium/chromium.py`; `tst/integration/computer/chromium_app_test.py` runs it. Like `drivers`, it has no tasks. | | `drivers/` | The `drivers` bundle: the computer drivers agent-env ships as `computer_driver` envs, the reference driver (X11 on Linux, recording through `x11grab`) aliased `reference`, and trycua's cua-driver aliased `cua-driver` (its daemon from the pinned PyPI wheel, its own instructions, tools and skill resources served to agents through `@server_instructions`, `forwarded_tools` and `@resource`, its tools beside `computer_*`, `computer/v1` on its desktop-scope calls, its skill pack leading the computer skill; `tst/integration/computer/cua_driver_test.py` runs it). It has no tasks: a bundle with neither tasks nor evals is refused unless its `aliases.toml` names something for other bundles to use. The shipped `desktop` example composes it with the `ubuntu-24.04` base into a computer env, which `tst/integration/computer/` deploys, probes, records and has an agent use. | | `examples/` | The example bundles agent-env ships (`hello`, and `echo-mcp`, which needs Docker, and `desktop`, a computer env of `ubuntu-24.04` and `reference` with a probe task and an agent task). Tests run them, but they are examples for users, not fixtures. Every shipped bundle is registered under `agent_env.bundles` in `pyproject.toml`, ` = ""` shipping `src///`. | @@ -302,7 +302,7 @@ orchestrators call it the same way. What callers rely on: - Skips are declared capability gaps, decided at collection time: the gates in `tst/util/capabilities.py` produce `skipif` marks with the reason `agentenv-capability-missing: ` (`model_endpoint_configured`, `remote_sandbox`, - `default_a2a_agent`, `mcp_server_sources`, `gnu_stat`, `trl_installed`). CI rejects any other skip reason; a broken configuration must fail loudly, not skip. + `default_a2a_agent`, `mcp_server_sources`, `gnu_stat`, `trl_installed`, `kvm`). CI rejects any other skip reason; a broken configuration must fail loudly, not skip. - The suite is backend-agnostic: a test about one particular backend lives with that backend, not here. A custom store must pass `tst/store/{conformance,object_conformance,image_conformance,secret_conformance}.py`, the same kits the built-ins pass, and a custom `[runner]` must pass `tst/runner/conformance.py`, which @@ -372,7 +372,10 @@ and creates the GitHub release, whose notes are the titles of the pull requests merged since the previous release. Once the repository variable `PUBLISH_INFRA_IMAGES` is `true`, it also publishes the infra images agent-env ships and the shipped bundles', one per agent folder, one for the env `a2a-agent validate` deploys (the `mail-mcp` -example's) and one per shipped base computer (`agentenv-ubuntu-24.04`; `agent_env.env.published_images`), whose `src-` tags GHCR lacks: after `main`'s CI passes, each is +example's) and one per shipped base computer (`agentenv-ubuntu-24.04`; `agent_env.env.published_images`), the disk tools +and QEMU launcher, and each base computer's disk form (`agentenv-ubuntu-24.04-disk`, tagged by the boot recipe's sources +and its base's and the tools' tags, built over them once they're published and booted in the launcher, with KVM on +x64 runners and under TCG on arm64's), whose `src-` tags GHCR lacks: after `main`'s CI passes, each is built on a runner of each platform, checked, and tagged as one multi-platform image, and a tag that exists is never moved. The infra images are published before the protocol, and a failed infra image job sends neither package to PyPI. An agent's `requirements.txt`, and that env's, keeps its protocol floor, unpinned, so its image is built again diff --git a/src/agent_env/artifact/artifacts/computer.py b/src/agent_env/artifact/artifacts/computer.py index 98858bfc9..93185964e 100644 --- a/src/agent_env/artifact/artifacts/computer.py +++ b/src/agent_env/artifact/artifacts/computer.py @@ -9,7 +9,16 @@ from datetime import datetime, timezone from typing import TYPE_CHECKING, Any, ClassVar, Literal, Self -from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator, model_validator +from pydantic import ( + BaseModel, + ConfigDict, + Field, + SerializerFunctionWrapHandler, + ValidationError, + field_validator, + model_serializer, + model_validator, +) from agent_env.artifact.artifact import Artifact from agent_env.artifact.artifacts.docker_image import DockerImageArtifact @@ -17,13 +26,15 @@ from agent_env.entity.refs import EntityRef from agent_env.store.base import NotFoundError from agent_env.store.image_store.oci_registry_credentials import names_registry -from agent_env.store.image_store.image_contents import UnreadableLayerError, os_release, platform_manifests +from agent_env.store.image_store.image_contents import UnreadableLayerError, disk_layer, os_release, platform_manifests if TYPE_CHECKING: from agent_env.bundle.authoring import AuthoringContext OS_TYPES = ("linux",) -FORMS = ("container",) +FORMS = ("container", "disk") +#: The file a disk form's image holds, as KubeVirt's containerDisk has it, owned by the uid its QEMU runs as. +DISK_FILE, DISK_OWNER = "disk/disk.img", 107 #: What a setting holds: its default's type, which a value an env or a deploy gives must share. SettingValue = str | int | float | bool @@ -81,10 +92,30 @@ class ContainerForm(BaseModel): input_channel: Literal["env"] +class DiskForm(BaseModel): + """The computer as a disk a VM boots from UEFI: an EFI system partition holding one UKI, whose init reads deploy + settings from a NoCloud seed (``input_channel``) and starts the image's own ``init``, and its root filesystem. With + no ``image``, it's the container form's root filesystem on agent-env's Linux boot recipe, made when composed.""" + + model_config = ConfigDict(extra="forbid") + + image: ArtifactRef | None = None + init: Literal["supervisord"] + input_channel: Literal["nocloud"] + firmware: list[Literal["efi"]] = Field(default=["efi"], min_length=1) + min_disk_gb: int = Field(ge=1, description="Its virtual size, which a deploy's disk is at least") + + class Forms(BaseModel): model_config = ConfigDict(extra="forbid") container: ContainerForm + disk: DiskForm | None = None + + @model_serializer(mode="wrap") + def _without_absent_forms(self, handler: SerializerFunctionWrapHandler) -> dict[str, Any]: + """Only the forms the computer has, so a computer of one form is stored as it was before others existed.""" + return {form: value for form, value in handler(self).items() if value is not None} class ImageCheck(BaseModel): @@ -115,8 +146,9 @@ class ComputerArtifact(Artifact): forms: Forms config: dict[str, Setting] = Field(default_factory=dict, description="Its settings, by name") - toml_refs: ClassVar[tuple[EntityRef, ...]] = (EntityRef.artifact("forms.container.image", - artifact_type="docker_image"),) + toml_refs: ClassVar[tuple[EntityRef, ...]] = ( + EntityRef.artifact("forms.container.image", artifact_type="docker_image"), + EntityRef.artifact("forms.disk.image", artifact_type="docker_image")) toml_keys: ClassVar[dict[str, type]] = {"description": str, "arch": list, "os": dict, "session": dict, "forms": dict, "config": dict} @@ -158,9 +190,14 @@ def put(cls, *, inspect: bool = True, **fields: Any) -> Self: from agent_env.artifact.store import get_artifact_store computer = cls(**fields) - image = computer.image() + image, disk = computer.image(), computer.forms.disk checks = {"container": _inspect_container(image, computer.arch, computer.os) if inspect else _unread(computer.arch, "inspection was skipped")} + if disk is not None and disk.image is None: + checks["disk"] = _unread(computer.arch, "it's made from the container form when composed") + elif disk is not None: + checks["disk"] = (_inspect_disk(computer.disk_image(), computer.arch, disk.min_disk_gb) if inspect + else _unread(computer.arch, "inspection was skipped")) problems = [f"forms.{form} ({arch}): {check.problem}" for form, by_arch in checks.items() for arch, check in by_arch.items() if check.status == "failed"] if problems: @@ -175,10 +212,10 @@ def from_toml(cls, data: dict[str, Any], ctx: AuthoringContext) -> Self: """Write the computer an artifact.toml declares, its form's image as the bundle wrote it or the artifact the key names.""" fields = {key: value for key, value in data.items() if key not in ("id", "type", "version")} - container = (fields.get("forms") or {}).get("container") - if isinstance(container, dict) and "image" in container: - image = ctx.artifact(container["image"]) - container["image"] = {"id": image.id, "version": image.version} + for form in (fields.get("forms") or {}).values(): + if isinstance(form, dict) and "image" in form: + image = ctx.artifact(form["image"]) + form["image"] = {"id": image.id, "version": image.version} try: return cls.put(id=ctx.id, **fields) except ValidationError as e: @@ -189,11 +226,18 @@ def from_toml(cls, data: dict[str, Any], ctx: AuthoringContext) -> Self: def image(self) -> DockerImageArtifact: """The container form's image artifact, refused when the store has no docker_image artifact of that id and version.""" - ref = self.forms.container.image + return self._image("container", self.forms.container.image) + + def disk_image(self) -> DockerImageArtifact | None: + """The disk form's image artifact, None when it has none, refused as ``image`` is.""" + disk = self.forms.disk + return None if disk is None or disk.image is None else self._image("disk", disk.image) + + def _image(self, form: str, ref: ArtifactRef) -> DockerImageArtifact: try: return DockerImageArtifact.get(ref.id, ref.version) except (NotFoundError, TypeError) as e: - raise ValueError(f"computer {self.id!r}: forms.container.image: {e}") from e + raise ValueError(f"computer {self.id!r}: forms.{form}.image: {e}") from e def _inspect_container(image: DockerImageArtifact, arches: list[str], os: LinuxOS) -> dict[str, ImageCheck]: @@ -233,5 +277,41 @@ def _inspect_container(image: DockerImageArtifact, arches: list[str], os: LinuxO return checks +def _inspect_disk(image: DockerImageArtifact, arches: list[str], min_disk_gb: int) -> dict[str, ImageCheck]: + """Each arch's disk image read from the start of its one layer: its file, its owner and its virtual size.""" + from agent_env.config import get_config + + if not names_registry(image.image_name): + return _unread(arches, "it's stored under a tag no registry serves, so its first boot checks it") + ref = image.pinned_name + auth = get_config().get_image_store().auth(ref) + try: + ref, manifests = platform_manifests(ref, auth) + except ValueError as e: + return {arch: ImageCheck(status="failed", image=ref, problem=str(e)) for arch in arches} + checks = {} + for arch in arches: + if (digest := manifests.get(arch)) is None: + checks[arch] = ImageCheck(status="failed", image=ref, problem=f"it has no linux/{arch} image") + continue + try: + found = disk_layer(ref, digest, auth) + except UnreadableLayerError as e: + checks[arch] = ImageCheck(status="uninspected", image=ref, manifest=digest, + reason=f"{e}, so its first boot checks it") + continue + except ValueError as e: + checks[arch] = ImageCheck(status="failed", image=ref, manifest=digest, problem=str(e)) + continue + problems = [problem for problem, wrong in ( + (f"its first file is {found.path}, not {DISK_FILE}", found.path != DISK_FILE), + (f"{found.path} is owned by uid {found.uid}, not {DISK_OWNER}", found.uid != DISK_OWNER), + (f"its virtual size is {found.virtual_size / 2**30:.1f} GiB, under min_disk_gb ({min_disk_gb})", + found.virtual_size < min_disk_gb * 2**30)) if wrong] + checks[arch] = ImageCheck(status="failed" if problems else "passed", image=ref, manifest=digest, + problem="; ".join(problems) or None) + return checks + + def _unread(arches: list[str], why: str) -> dict[str, ImageCheck]: return {arch: ImageCheck(status="uninspected", reason=why) for arch in arches} diff --git a/src/agent_env/computers/README.md b/src/agent_env/computers/README.md index 9c260dae0..6934bfd96 100644 --- a/src/agent_env/computers/README.md +++ b/src/agent_env/computers/README.md @@ -8,6 +8,18 @@ container form's image is built from the folder's `Dockerfile`, and a release pu building it. Name it from another bundle as `ubuntu-24.04`, the alias `aliases.toml` gives it, or by its id, `@local/agentenv-framework/computers/ubuntu-24.04`. +It also has a disk form, a disk a VM boots from UEFI, which is the container form's root filesystem on the Linux boot +recipe in `disk/linux/`: a GPT disk of an EFI system partition holding one UKI, Ubuntu's kernel with a busybox initramfs, +and an ext4 root filesystem that is the image's own, with `/etc/agentenv/vm/image.env`, its entrypoint, command, +user, working folder and environment, from its OCI config. The init mounts it, brings up the network, reads the deploy's +settings from a NoCloud seed (`CIDATA`) and becomes the image's entrypoint, as its user. A release publishes it as +`ghcr.io/scaleapi/agentenv-ubuntu-24.04-disk:src-`, tagged by both its sources; without it, composing makes it. +Composing a computer env makes each arch's disk from its composed container, the root filesystem rebuilt with +`mkfs.ext4 -d` by the disk tools (`disk/linux/tools/`) on the builder's own platform and the base disk's EFI system +partition kept, so nothing is emulated or privileged. A disk boots where `/dev/kvm` is, in the QEMU launcher +(`disk/linux/launcher/`): gate 3 probes it there, and elsewhere it stays `unverified` until a deploy's first boot with +KVM probes it. A deploy boots it with `env_params = { form = "disk" }`. + Its settings reach the computer as environment variables, `AGENTENV_SETTING_`, read as it starts: | Setting | Default | What it does | diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04/artifact.toml b/src/agent_env/computers/artifacts/ubuntu-24.04/artifact.toml index 654f2f5e4..07f3f05ee 100644 --- a/src/agent_env/computers/artifacts/ubuntu-24.04/artifact.toml +++ b/src/agent_env/computers/artifacts/ubuntu-24.04/artifact.toml @@ -9,6 +9,12 @@ image = { dockerfile = "Dockerfile" } init = "supervisord" input_channel = "env" +[forms.disk] +init = "supervisord" +input_channel = "nocloud" +firmware = ["efi"] +min_disk_gb = 16 + [config.resolution] default = "1280x800" phase = "deploy" diff --git a/src/agent_env/computers/disk/linux/boot/Dockerfile b/src/agent_env/computers/disk/linux/boot/Dockerfile new file mode 100644 index 000000000..06453d9c7 --- /dev/null +++ b/src/agent_env/computers/disk/linux/boot/Dockerfile @@ -0,0 +1,39 @@ +# check=skip=InvalidDefaultArgInFrom +# A Linux computer's disk form for one arch: a GPT disk (qcow2) of an EFI system partition holding one UKI, Ubuntu's +# kernel with a busybox initramfs whose init starts the computer, and an ext4 root filesystem that is ROOTFS, a container +# image's root filesystem, plus /etc/agentenv/vm/image.env, its start, from ROOTFS_CONFIG (its OCI config's config +# object, as base64'd JSON). The EFI system partition comes from ESP: the uki stage, on a disk of SIZE_GB, or a disk +# form's image, whose partition and size the disk keeps. Only the uki stage runs the target's programs: the disk itself +# is made on the build's own platform by TOOLS, the disk tools image, so an existing ESP needs no emulation. +ARG ROOTFS +ARG TOOLS +ARG ESP=uki + +FROM public.ecr.aws/docker/library/ubuntu:24.04@sha256:24737cef17f4e22b1977df7d9a05e77c4e28b7da5b8911f8d8ccda136bf3573c AS kernel +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + linux-image-virtual busybox-static kmod zstd cpio systemd-boot-efi systemd-ukify python3-pefile \ + && rm -rf /var/lib/apt/lists/* +COPY init udhcpc.script uki.sh /build/ +RUN sh /build/uki.sh /uki + +FROM scratch AS uki +COPY --from=kernel /uki/ / + +FROM --platform=$BUILDPLATFORM ${TOOLS} AS start +ARG ROOTFS_CONFIG +RUN printf %s "$ROOTFS_CONFIG" | base64 -d > /config.json && image-env /config.json > /image.env + +FROM ${ROOTFS} AS root +COPY --from=start /image.env /etc/agentenv/vm/image.env + +FROM ${ESP} AS esp + +FROM --platform=$BUILDPLATFORM ${TOOLS} AS disk +ARG TARGETARCH +ARG SIZE_GB=16 +RUN --mount=type=bind,from=root,target=/rootfs --mount=type=bind,from=esp,target=/esp \ + disk "$TARGETARCH" /rootfs /esp /out/disk.img "$SIZE_GB" + +FROM scratch +COPY --from=disk --chown=107:107 /out/disk.img /disk/disk.img diff --git a/src/agent_env/computers/disk/linux/boot/init b/src/agent_env/computers/disk/linux/boot/init new file mode 100755 index 000000000..7b8fc8c50 --- /dev/null +++ b/src/agent_env/computers/disk/linux/boot/init @@ -0,0 +1,65 @@ +#!/bin/busybox sh +# The disk form's first process: mounts the root filesystem the computer's image became, brings up the network, reads +# the image's start (/etc/agentenv/vm/image.env, from its OCI config) and the deploy's environment (the NoCloud seed's +# user-data), then becomes that start, as its user, as the container runtime would: PID 1 is the image's entrypoint. +/bin/busybox --install -s /bin +export PATH=/bin +mount -t proc proc /proc +mount -t sysfs sysfs /sys +mount -t devtmpfs devtmpfs /dev +log() { echo "agentenv-init: $*" > /dev/console; } +fail() { log "$*"; sleep 2; poweroff -f; } +while read -r module; do insmod "$module" 2>/dev/null; done < /modules.order +root="" +for _ in $(seq 200); do root=$(findfs LABEL=agentenv-root 2>/dev/null) && break; sleep 0.05; done +[ -n "$root" ] || fail "no filesystem labelled agentenv-root" +mount -t ext4 -o rw,noatime "$root" /newroot || fail "can't mount $root" +ifconfig lo 127.0.0.1 up +iface=$(ls /sys/class/net | grep -vx lo | head -n 1) +[ -n "$iface" ] && ifconfig "$iface" up && timeout 30 udhcpc -i "$iface" -s /udhcpc.script -q -n -t 10 -T 2 -f >/dev/null 2>&1 \ + || log "no DHCP lease on ${iface:-any interface}" +seed=$(findfs LABEL=CIDATA 2>/dev/null || true) +[ -n "$seed" ] && mount -o ro "$seed" /seed +config=/newroot/etc/agentenv/vm/image.env +[ -f "$config" ] || fail "the root filesystem has no $config" +for name in $(env | cut -d= -f1); do case "$name" in PATH|PWD|SHLVL) ;; *) unset "$name" ;; esac; done +user=root workdir=/ +set -- +while read -r line; do + key=${line%%=*} value=${line#*=} + case "$key" in + env) export "$(echo "$value" | base64 -d)" ;; + argv) set -- "$@" "$(echo "$value" | base64 -d)" ;; + user) user=$(echo "$value" | base64 -d) ;; + workdir) workdir=$(echo "$value" | base64 -d) ;; + esac +done < "$config" +if [ -f /seed/user-data ]; then + while read -r line; do + key=${line%%=*} value=${line#*=} + case "$key" in ''|\#*) continue ;; esac + export "$key=$(echo "$value" | base64 -d)" + done < /seed/user-data +fi +hostname=$( [ -f /seed/meta-data ] && sed -n 's/^local-hostname: *//p' /seed/meta-data) +hostname "${hostname:-computer}" +hostname > /newroot/etc/hostname +printf '127.0.0.1\tlocalhost\n::1\tlocalhost ip6-localhost ip6-loopback\n127.0.1.1\t%s\n' "$(hostname)" > /newroot/etc/hosts +[ -s /resolv.conf ] && cp /resolv.conf /newroot/etc/resolv.conf +entry=$(awk -F: -v u="${user%%:*}" '$1 == u || $3 == u' /newroot/etc/passwd | head -n 1) +[ -n "$entry" ] || fail "the image's user ${user%%:*} isn't in /etc/passwd" +uid=$(echo "$entry" | cut -d: -f3) gid=$(echo "$entry" | cut -d: -f4) +export HOME="${HOME:-$(echo "$entry" | cut -d: -f6)}" HOSTNAME="$(hostname)" +[ "$PATH" = /bin ] && export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +umount /seed 2>/dev/null +[ "$#" -gt 0 ] || fail "the image names no command" +mount --move /sys /newroot/sys +mount --move /dev /newroot/dev +mkdir -p /newroot/dev/pts /newroot/dev/shm +mount -t devpts -o gid=5,mode=620,ptmxmode=666 devpts /newroot/dev/pts +mount -t tmpfs -o mode=1777,size=512m shm /newroot/dev/shm +log "starting $* as $user ($uid:$gid) in $workdir" +# Last, as busybox runs its applets through /proc/self/exe. +mount --move /proc /newroot/proc +exec /bin/busybox switch_root /newroot /usr/bin/setpriv --reuid="$uid" --regid="$gid" --init-groups \ + /usr/bin/env -C "$workdir" "$@" diff --git a/src/agent_env/computers/disk/linux/boot/udhcpc.script b/src/agent_env/computers/disk/linux/boot/udhcpc.script new file mode 100755 index 000000000..13f4bf682 --- /dev/null +++ b/src/agent_env/computers/disk/linux/boot/udhcpc.script @@ -0,0 +1,11 @@ +#!/bin/busybox sh +case "$1" in + bound|renew) + ifconfig "$interface" "$ip" netmask "${subnet:-255.255.255.0}" up + for router in ${router:-}; do route add default gw "$router" dev "$interface" && break; done + : > /resolv.conf + for server in ${dns:-}; do echo "nameserver $server" >> /resolv.conf; done + [ -n "${domain:-}" ] && echo "search $domain" >> /resolv.conf + ;; +esac +exit 0 diff --git a/src/agent_env/computers/disk/linux/boot/uki.sh b/src/agent_env/computers/disk/linux/boot/uki.sh new file mode 100755 index 000000000..c7451cd2b --- /dev/null +++ b/src/agent_env/computers/disk/linux/boot/uki.sh @@ -0,0 +1,39 @@ +#!/bin/sh +# uki.sh OUT: the one file the disk form's EFI system partition holds, written to OUT under the name UEFI firmware boots +# from removable media: a UKI of Ubuntu's kernel and an initramfs of busybox, init, and the modules a virtio guest needs +# that the kernel doesn't build in, decompressed, since busybox's insmod reads no zstd. +set -eu +out=$1 +kver=$(ls /lib/modules | sort -V | tail -1) +case "$(dpkg --print-architecture)" in + amd64) efi=BOOTX64.EFI stub=linuxx64.efi.stub console=ttyS0 ;; + arm64) efi=BOOTAA64.EFI stub=linuxaa64.efi.stub console=ttyAMA0 ;; +esac +root=/tmp/initramfs +mkdir -p "$root/bin" "$root/dev" "$root/proc" "$root/sys" "$root/newroot" "$root/seed" "$root/lib/modules" "$out" +cp /bin/busybox "$root/bin/busybox" +cp /build/init /build/udhcpc.script "$root/" +chmod 755 "$root/init" "$root/udhcpc.script" +: > "$root/modules.order" +for module in virtio_pci virtio_mmio virtio_blk virtio_scsi virtio_net virtio_rng virtio_console ext4 vfat \ + nls_cp437 nls_iso8859-1 nls_utf8 af_packet; do + modprobe --show-depends -S "$kver" "$module" 2>/dev/null | while read -r kind path _; do + [ "$kind" = insmod ] || continue + name=$(basename "$path" .zst) + grep -qx "/lib/modules/$name" "$root/modules.order" && continue + case "$path" in + *.zst) zstd -qdc "$path" > "$root/lib/modules/$name" ;; + *) cp "$path" "$root/lib/modules/$name" ;; + esac + echo "/lib/modules/$name" >> "$root/modules.order" + done +done +(cd "$root" && find . | sort | cpio -o -H newc --quiet | gzip -9n) > /tmp/initrd.img +# arm64's kernel is a gzipped Image, which the stub can't start; the Image itself is a PE the firmware can. +if [ "$(head -c2 "/boot/vmlinuz-$kver" | od -An -tx1 | tr -d ' ')" = 1f8b ]; then + zcat "/boot/vmlinuz-$kver" > /tmp/vmlinuz +else + cp "/boot/vmlinuz-$kver" /tmp/vmlinuz +fi +ukify build --uname "$kver" --linux /tmp/vmlinuz --initrd /tmp/initrd.img \ + --cmdline "console=$console panic=1 quiet loglevel=4" --stub "/usr/lib/systemd/boot/efi/$stub" --output "$out/$efi" diff --git a/src/agent_env/computers/disk/linux/launcher/Dockerfile b/src/agent_env/computers/disk/linux/launcher/Dockerfile new file mode 100644 index 000000000..9d4f5a1f5 --- /dev/null +++ b/src/agent_env/computers/disk/linux/launcher/Dockerfile @@ -0,0 +1,9 @@ +# The QEMU launcher: boots a Linux computer's disk form, mounted or copied at /disk/disk.img, with its settings in a +# NoCloud seed (see launch). It boots either arch's disk, with KVM where /dev/kvm is passed in and the arch is its own. +FROM public.ecr.aws/docker/library/ubuntu:24.04@sha256:24737cef17f4e22b1977df7d9a05e77c4e28b7da5b8911f8d8ccda136bf3573c +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + qemu-system-x86 qemu-system-arm qemu-utils ovmf qemu-efi-aarch64 dosfstools mtools \ + && rm -rf /var/lib/apt/lists/* +COPY launch seed /usr/local/bin/ +ENTRYPOINT ["/usr/local/bin/launch"] diff --git a/src/agent_env/computers/disk/linux/launcher/launch b/src/agent_env/computers/disk/linux/launcher/launch new file mode 100755 index 000000000..5a88b3a88 --- /dev/null +++ b/src/agent_env/computers/disk/linux/launcher/launch @@ -0,0 +1,46 @@ +#!/bin/sh +# launch: boot /disk/disk.img, a Linux computer's disk form, under QEMU until it powers off, on a copy-on-write overlay +# so the disk stays as it is: with KVM when /dev/kvm is here and the disk is this machine's arch, else with TCG, from +# UEFI firmware. The guest's console is this container's output. +# VM_ENV_ the computer's environment, which goes in its NoCloud seed (CIDATA; see seed) as NAME +# VM_PORTS guest TCP ports, each forwarded from the same port here +# VM_ARCH the disk's arch, amd64 or arm64; this machine's by default +# VM_CPUS, VM_MEMORY_MB its size: 2 and 4096 by default +set -eu +here=$(dpkg --print-architecture) +arch=${VM_ARCH:-$here} +work=/run/vm +rm -rf "$work" +mkdir -p "$work" +qemu-img create -q -f qcow2 -b /disk/disk.img -F qcow2 "$work/disk.qcow2" +seed "$work" +mkfs.vfat -C -n CIDATA "$work/seed.img" 2048 >/dev/null +mcopy -i "$work/seed.img" "$work/user-data" "$work/meta-data" ::/ +forwards="" +for port in ${VM_PORTS:-}; do + forwards="$forwards,hostfwd=tcp::$port-:$port" +done +accel=tcg cpu=max +if [ -w /dev/kvm ] && [ "$arch" = "$here" ]; then + accel=kvm cpu=host +fi +case "$arch" in + amd64) + cp /usr/share/OVMF/OVMF_VARS_4M.fd "$work/vars.fd" + set -- qemu-system-x86_64 -machine "q35,accel=$accel" \ + -drive if=pflash,format=raw,readonly=on,file=/usr/share/OVMF/OVMF_CODE_4M.fd ;; + arm64) + cp /usr/share/AAVMF/AAVMF_CODE.fd "$work/code.fd" + truncate -s 64M "$work/code.fd" "$work/vars.fd" + set -- qemu-system-aarch64 -machine "virt,accel=$accel" \ + -drive if=pflash,format=raw,readonly=on,file="$work/code.fd" ;; + *) + echo "launch: no architecture $arch" >&2 + exit 2 ;; +esac +exec "$@" -cpu "$cpu" -smp "${VM_CPUS:-2}" -m "${VM_MEMORY_MB:-4096}" \ + -drive if=pflash,format=raw,file="$work/vars.fd" \ + -drive file="$work/disk.qcow2",format=qcow2,if=virtio \ + -drive file="$work/seed.img",format=raw,if=virtio,readonly=on \ + -netdev "user,id=net$forwards" -device virtio-net-pci,netdev=net -device virtio-rng-pci \ + -display none -serial stdio -no-reboot diff --git a/src/agent_env/computers/disk/linux/launcher/seed b/src/agent_env/computers/disk/linux/launcher/seed new file mode 100755 index 000000000..98b8e4de8 --- /dev/null +++ b/src/agent_env/computers/disk/linux/launcher/seed @@ -0,0 +1,12 @@ +#!/bin/sh +# seed DIR: the NoCloud seed's files for the computer launch boots, in DIR: user-data holding each VM_ENV_ variable +# as NAME and its value base64'd, a line each, as the disk's init reads them, and meta-data naming the instance. +set -eu +{ + echo "#agentenv-environment" + for name in $(env | sed -n 's/^VM_ENV_\([A-Za-z_][A-Za-z0-9_]*\)=.*/\1/p' | sort -u); do + eval "value=\${VM_ENV_$name}" + printf '%s=%s\n' "$name" "$(printf %s "$value" | base64 | tr -d '\n')" + done +} > "$1/user-data" +printf 'instance-id: %s\nlocal-hostname: computer\n' "$(od -An -N8 -tx1 /dev/urandom | tr -d ' \n')" > "$1/meta-data" diff --git a/src/agent_env/computers/disk/linux/tools/Dockerfile b/src/agent_env/computers/disk/linux/tools/Dockerfile new file mode 100644 index 000000000..ca10b1e3e --- /dev/null +++ b/src/agent_env/computers/disk/linux/tools/Dockerfile @@ -0,0 +1,7 @@ +# The disk tools: what makes a Linux computer's disk form from a root filesystem without mounting or booting anything, +# so a build runs them unprivileged on its own platform, for either arch. +FROM public.ecr.aws/docker/library/ubuntu:24.04@sha256:24737cef17f4e22b1977df7d9a05e77c4e28b7da5b8911f8d8ccda136bf3573c +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends e2fsprogs dosfstools mtools fdisk qemu-utils python3 \ + && rm -rf /var/lib/apt/lists/* +COPY disk image-env /usr/local/bin/ diff --git a/src/agent_env/computers/disk/linux/tools/disk b/src/agent_env/computers/disk/linux/tools/disk new file mode 100755 index 000000000..08b770fc3 --- /dev/null +++ b/src/agent_env/computers/disk/linux/tools/disk @@ -0,0 +1,47 @@ +#!/bin/sh +# disk ARCH ROOTFS ESP OUT [SIZE_GB]: a Linux computer's disk form, written to OUT as qcow2: a GPT disk of an EFI system +# partition and an ext4 root filesystem made from the folder ROOTFS, ownership kept. ESP is a folder holding either a +# disk form's image, disk/disk.img, whose EFI system partition and size the disk keeps byte for byte, or a UKI, which a +# new EFI system partition holds on a disk of SIZE_GB. Nothing is mounted or booted, so it runs unprivileged, on any +# architecture, for either. +set -eu +arch=$1 rootfs=$2 esp=$3 out=$4 size_gb=${5:-16} +case "$arch" in + amd64) uki=BOOTX64.EFI roottype=4F68BCE3-E8CD-4DB1-96E7-FBCAF984B709 ;; + arm64) uki=BOOTAA64.EFI roottype=B921B045-1DF0-41C3-AF44-4C6F280D3FAE ;; + *) echo "disk: no architecture $arch" >&2; exit 2 ;; +esac +esptype=C12A7328-F81F-11D2-BA4B-00A0C93EC93B +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +mkdir -p "$(dirname "$out")" +if [ -f "$esp/disk/disk.img" ]; then + qemu-img convert -O raw "$esp/disk/disk.img" "$work/base.raw" + size=$(stat -c %s "$work/base.raw") + partition=$(sfdisk -d "$work/base.raw" | grep -i "type=$esptype") + esp_start=$(echo "$partition" | sed -n 's/.*start= *\([0-9]*\).*/\1/p') + esp_sectors=$(echo "$partition" | sed -n 's/.*size= *\([0-9]*\).*/\1/p') + [ -n "$esp_start" ] && [ -n "$esp_sectors" ] || { echo "disk: $esp/disk/disk.img has no EFI system partition" >&2; exit 1; } + dd if="$work/base.raw" of="$work/esp.img" bs=512 skip="$esp_start" count="$esp_sectors" status=none + rm "$work/base.raw" +elif [ -f "$esp/$uki" ]; then + size=$((size_gb * 1024 * 1024 * 1024)) esp_start=2048 esp_sectors=$((64 * 2048)) + mkfs.vfat -C -n ESP "$work/esp.img" $((esp_sectors / 2)) >/dev/null + mmd -i "$work/esp.img" ::/EFI ::/EFI/BOOT + mcopy -i "$work/esp.img" "$esp/$uki" "::/EFI/BOOT/$uki" +else + echo "disk: $esp holds neither disk/disk.img nor $uki" >&2 + exit 1 +fi +root_start=$(((esp_start + esp_sectors + 2047) / 2048 * 2048)) +root_sectors=$((size / 512 - root_start - 2048)) +mkfs.ext4 -q -F -L agentenv-root -E root_owner=0:0 -d "$rootfs" "$work/root.img" "$((root_sectors / 2))k" +truncate -s "$size" "$work/disk.raw" +sfdisk -q "$work/disk.raw" < list[str]: + return [*(f"env={_encoded(variable)}" for variable in config.get("Env") or ()), + *(f"argv={_encoded(arg)}" for arg in [*(config.get("Entrypoint") or ()), *(config.get("Cmd") or ())]), + f"user={_encoded(config.get('User') or 'root')}", f"workdir={_encoded(config.get('WorkingDir') or '/')}"] + + +def _encoded(value: str) -> str: + return base64.b64encode(value.encode()).decode() + + +if __name__ == "__main__": + with open(sys.argv[1]) as file: + print("\n".join(lines(json.load(file)))) diff --git a/src/agent_env/env/bootstrap.py b/src/agent_env/env/bootstrap.py index 1e34ba927..b041dee91 100644 --- a/src/agent_env/env/bootstrap.py +++ b/src/agent_env/env/bootstrap.py @@ -23,7 +23,14 @@ from agent_env.env.envs import GatewayEnv, MCPServerEnv from agent_env.env.envs.service_db import ServiceDBEnv from agent_env.env.envs.playwright_mcp import PLAYWRIGHT_MCP_ENVIRONMENT_NAME, PLAYWRIGHT_MCP_IMAGE_TAG -from agent_env.env.published_images import BUNDLE_IMAGES, PublishedImage, published_digest +from agent_env.env.published_images import ( + BUNDLE_IMAGES, + DISK_IMAGES, + DISK_TOOLS_IMAGE, + LAUNCHER_IMAGE, + PublishedImage, + published_digest, +) from agent_env.store.base import NotFoundError from agent_env.store.document_store import LocalSqliteDocumentStore from agent_env.store.ids import derive_id @@ -155,14 +162,18 @@ def _metadata(kind: str, extra: Mapping[str, Any] | None) -> dict[str, Any]: SERVICE_DB: [SERVICE_DB_DOCKERFILE, DB_WEB_DOCKERFILE, DB_MCP_DOCKERFILE], WEBSITE_BROWSER: [PLAYWRIGHT_MCP_DOCKERFILE], } -# The image each shipped Dockerfile is published as: the infra envs', the shipped agents' and an example env's. +# The image each shipped Dockerfile is published as: the infra envs', the disk tools and QEMU launcher, the shipped +# agents', an example env's, the base computers' and their disk forms. PUBLISHED_IMAGES = ( PublishedImage("agentenv-gateway", GATEWAY_DOCKERFILE), PublishedImage("agentenv-service-db", SERVICE_DB_DOCKERFILE), PublishedImage("agentenv-db-web", DB_WEB_DOCKERFILE), PublishedImage("agentenv-db-mcp", DB_MCP_DOCKERFILE), PublishedImage("agentenv-playwright-mcp", PLAYWRIGHT_MCP_DOCKERFILE), + DISK_TOOLS_IMAGE, + LAUNCHER_IMAGE, *BUNDLE_IMAGES, + *DISK_IMAGES, ) _PUBLISHED = {image.dockerfile: image for image in PUBLISHED_IMAGES} diff --git a/src/agent_env/env/envs/computer/env.py b/src/agent_env/env/envs/computer/env.py index a92ccd9e7..fdd0daff5 100644 --- a/src/agent_env/env/envs/computer/env.py +++ b/src/agent_env/env/envs/computer/env.py @@ -23,7 +23,7 @@ deploy_through_provider, provider_or_class, ) -from agent_env.env.envs.computer.utils import composer, gate, inside +from agent_env.env.envs.computer.utils import boot, composer, gate, inside from agent_env.env.envs.computer.utils.composer import Composing from agent_env.env.envs.computer.utils import params as computer_params from agent_env.env.envs.computer.driver_env import ComputerDriverEnv @@ -45,6 +45,8 @@ DEFAULT_DESTINATION = "Downloads" #: The computer app env's type: an app composed into a computer env, and a computer env of its own. APP_TYPE = "computer_app" +#: The statuses of a build a deploy boots, by form: a disk may also boot unprobed, as KVM is where a disk is probed. +_DEPLOYABLE = {"container": ("ready",), "disk": ("ready", "unverified")} class ComputerEnv(Env): @@ -144,12 +146,13 @@ def computer_artifact(self) -> ComputerArtifact: """The env's computer: its base composed with its driver, over the images built.""" return ComputerArtifact.get(self.metadata["computer"]["id"], self.metadata["computer"]["version"]) - def ready_builds(self, form: Optional[str] = None) -> list[tuple[str, str, dict]]: - """Each ready build, as (form, arch, record), in the base's order of forms and of arches; only - ``form``'s when it's given.""" + def deployable_builds(self, form: Optional[str] = None) -> list[tuple[str, str, dict]]: + """Each build a deploy may boot, as (form, arch, record), in the base's order of forms and of arches; only + ``form``'s when it's given: a ready one, or a disk no compose could probe (unverified), which its first boot on + a provider that boots it natively probes before it's used.""" composed, arches = self.metadata.get("composed", {}), self.computer.arch return [(name, arch, composed[name][arch]) for name in gate.forms(self.computer) if form in (None, name) - for arch in arches if composed.get(name, {}).get(arch, {}).get("status") == "ready"] + for arch in arches if composed.get(name, {}).get(arch, {}).get("status") in _DEPLOYABLE[name]] def build_statuses(self) -> dict[str, str]: return {f"{form}/{arch}": result.get("status") for form, by_arch in self.metadata.get("composed", {}).items() @@ -176,12 +179,13 @@ def deploy_refusal(self, *, sandbox_type: str | None = None, env_params: Optiona if refusal := deploy_refusal(self, provider_or_class(self), {**options, "env_params": env_params}): return refusal - _, _, problems = computer_params.checked(self, env_params) + params, _, problems = computer_params.checked(self, env_params) sandbox_provider = build_sandbox_provider(sandbox_type) if sandbox_type else get_env_sandbox_provider() links, arches = sandbox_provider.links, gate.arches(self.computer, self.driver, self.composed_apps()) - if not any(f"linux/{arch}" in link.native_platforms() for link in links for arch in arches): + forms = [params.form] if params.form else gate.forms(self.computer) + if not any(boot.native_links(sandbox_provider, arch, form) for form in forms for arch in arches): problems.append(f"{', '.join(type(link).__name__ for link in links)} runs none of its arches " - f"({', '.join(arches)}) natively") + f"({', '.join(arches)}) natively as {' or '.join(forms)}") policy = NetworkPolicy.from_dict(self.network) if self.network else None if policy is not None and not any(link.supports_network_policy(policy) for link in links): problems.append(f"{', '.join(type(link).__name__ for link in links)} can't enforce its network") diff --git a/src/agent_env/env/envs/computer/utils/boot.py b/src/agent_env/env/envs/computer/utils/boot.py index daa8fd269..0f9f095b8 100644 --- a/src/agent_env/env/envs/computer/utils/boot.py +++ b/src/agent_env/env/envs/computer/utils/boot.py @@ -1,6 +1,6 @@ -"""Booting a composed computer: one build of it in a container, on a sandbox provider that runs its arch natively, with -its settings in the environment the base and the driver read them from, until its driver says it's ready. A deploy and -gate 3 boot computers this way.""" +"""Booting a composed computer: one build of it, its container in a container or its disk in a VM, on a sandbox provider +that runs its arch natively, with its settings in the environment the base and the driver read them from (a disk's in +its NoCloud seed), until its driver says it's ready. A deploy and gate 3 boot computers this way.""" from __future__ import annotations @@ -38,6 +38,8 @@ WITHHELD = "" #: How many random bytes a computer's control token carries: 256 bits, past guessing. CONTROL_TOKEN_BYTES = 32 +#: The size of a disk's VM when a deploy sets none: what the desktop was measured booting and passing the probe in. +DISK_CPU, DISK_MEMORY_MB = 2.0, 4096 @dataclass @@ -47,9 +49,11 @@ class Booted: card: dict -def native_links(sandbox_provider: SandboxProvider, arch: str) -> list[SandboxProvider]: - """The providers a boot of ``arch`` may run on, in the order a chain tries them: those that run it natively.""" - return [link for link in sandbox_provider.links if f"linux/{arch}" in link.native_platforms()] +def native_links(sandbox_provider: SandboxProvider, arch: str, form: str = "container") -> list[SandboxProvider]: + """The providers a boot of ``form`` for ``arch`` may run on, in the order a chain tries them: those that run its + container, or boot its disk, natively.""" + return [link for link in sandbox_provider.links if f"linux/{arch}" in ( + link.disk_image_platforms() if form == "disk" else link.native_platforms())] def settings_environment(computer: ComputerArtifact, values: dict[str, Any], *, environment_name: str, @@ -105,13 +109,20 @@ def control_environment(token: Optional[str]) -> dict[str, str]: async def boot(sandbox_provider: SandboxProvider, *, image: str, port: int, environment: dict[str, str], token: str, network_policy: Optional[NetworkPolicy], timeout: int, attribution: Optional[Attribution], disk_size_gb: float = 10, cpu: Optional[float] = None, memory_mb: Optional[int] = None, - ready_s: float = READY_TIMEOUT_S) -> Booted: - """``image`` booted with ``environment`` and its control ``token`` on ``sandbox_provider``, which the caller chose - to run its arch natively, its driver's ``port`` reached on the sandbox, once the driver, asked with that token, says - the computer is ready; a boot that fails is torn down.""" - sandbox = await sandbox_provider.create_container( - image_name=image, port=port, env={**environment, **control_environment(token)}, network_policy=network_policy, - disk_size_gb=disk_size_gb, timeout=timeout, attribution=dict(attribution or {}), **_size_kwargs(cpu, memory_mb)) + ready_s: float = READY_TIMEOUT_S, form: str = "container") -> Booted: + """``image``, of ``form``, booted with ``environment`` and its control ``token`` on ``sandbox_provider``, which the + caller chose to run its arch natively, its driver's ``port`` reached on the sandbox, once the driver, asked with that + token, says the computer is ready; a boot that fails is torn down.""" + environment = {**environment, **control_environment(token)} + if form == "disk": + sandbox = await sandbox_provider.create_vm( + image=image, env=environment, exposed_ports=[port], network_policy=network_policy, + disk_size_gb=disk_size_gb, timeout=timeout, attribution=dict(attribution or {}), + cpu=cpu or DISK_CPU, memory=memory_mb or DISK_MEMORY_MB) + else: + sandbox = await sandbox_provider.create_container( + image_name=image, port=port, env=environment, network_policy=network_policy, disk_size_gb=disk_size_gb, + timeout=timeout, attribution=dict(attribution or {}), **_size_kwargs(cpu, memory_mb)) try: url = sandbox.tunnel_urls[port] return Booted(sandbox, url, await ready(url, ready_s, token)) diff --git a/src/agent_env/env/envs/computer/utils/builder.py b/src/agent_env/env/envs/computer/utils/builder.py index 4904407ed..7371f6c68 100644 --- a/src/agent_env/env/envs/computer/utils/builder.py +++ b/src/agent_env/env/envs/computer/utils/builder.py @@ -125,9 +125,18 @@ def mark_runnable(self, paths: list[str]) -> None: """Make each of ``paths`` executable that starts as a script or a Linux binary does.""" @abstractmethod - def build(self, dockerfile: str, tag: str, platform: str, labels: dict[str, str], context: str = CONTEXT) -> None: - """Build ``context`` with ``dockerfile``, written into it, as the image ``tag`` for ``platform``. Raises - DockerBuildError.""" + def build(self, dockerfile: str, tag: str, platform: str, labels: dict[str, str], context: str = CONTEXT, + build_args: dict[str, str] | None = None) -> None: + """Build ``context`` with ``dockerfile``, written into it, and ``build_args``, as the image ``tag`` for + ``platform``. Raises DockerBuildError.""" + + @abstractmethod + def image_config(self, tag: str) -> dict: + """The ``config`` object of the image config of ``tag``, an image its Docker holds.""" + + @abstractmethod + def login(self, auth) -> None: + """Sign its Docker in to the registry ``auth``, image store credentials or None, names.""" @abstractmethod def installed(self, tag: str, prefix: str) -> dict[str, str]: @@ -310,10 +319,21 @@ def mark_runnable(self, paths: list[str]) -> None: if file.read(4).startswith(RUN_ONLY): (self.work / path).chmod(EXECUTABLE) - def build(self, dockerfile: str, tag: str, platform: str, labels: dict[str, str], context: str = CONTEXT) -> None: + def build(self, dockerfile: str, tag: str, platform: str, labels: dict[str, str], context: str = CONTEXT, + build_args: dict[str, str] | None = None) -> None: (self.work / context).mkdir(parents=True, exist_ok=True) (self.work / context / "Dockerfile").write_text(dockerfile) - build_image(self.work / context / "Dockerfile", self.work / context, tag, platform=platform, labels=labels) + build_image(self.work / context / "Dockerfile", self.work / context, tag, platform=platform, labels=labels, + build_args=build_args) + + def image_config(self, tag: str) -> dict: + inspected = subprocess.run(["docker", "image", "inspect", "--format", "{{json .Config}}", tag], + capture_output=True, text=True, check=True) + return json.loads(inspected.stdout) + + def login(self, auth) -> None: + if auth is not None: + docker_login(auth) def installed(self, tag: str, prefix: str) -> dict[str, str]: created = subprocess.run(["docker", "create", tag], capture_output=True, text=True, check=True).stdout.strip() @@ -423,15 +443,23 @@ def place(self, copies: dict[str, str], fetches: dict[str, tuple[str, str]]) -> def mark_runnable(self, paths: list[str]) -> None: self._each([f"{self.helper} runnable {self._at(path)}" for path in paths]) - def build(self, dockerfile: str, tag: str, platform: str, labels: dict[str, str], context: str = CONTEXT) -> None: + def build(self, dockerfile: str, tag: str, platform: str, labels: dict[str, str], context: str = CONTEXT, + build_args: dict[str, str] | None = None) -> None: folder = f"{self.work}/{context}" self._write(f"{folder}/Dockerfile", dockerfile.encode()) - options = " ".join(f"--label {shlex.quote(f'{name}={value}')}" for name, value in labels.items()) + options = " ".join([*(f"--label {shlex.quote(f'{name}={value}')}" for name, value in labels.items()), + *(f"--build-arg {shlex.quote(f'{name}={value}')}" for name, value in (build_args or {}).items())]) code, stdout, stderr = self._output(f"cd {folder} && docker build --platform {shlex.quote(platform)} {options} " f"-f Dockerfile -t {shlex.quote(tag)} . 2>&1") if code != 0: raise DockerBuildError(f"docker build of {tag} failed (exit {code}):\n{(stdout + stderr).rstrip()}") + def image_config(self, tag: str) -> dict: + return json.loads(self._exec(f"docker image inspect --format '{{{{json .Config}}}}' {shlex.quote(tag)}")) + + def login(self, auth) -> None: + self._login(auth) + def installed(self, tag: str, prefix: str) -> dict[str, str]: listed = self._exec(f"{self.helper} installed {shlex.quote(tag)} {shlex.quote(f'{prefix}/installed.json')}") return _packages(listed.encode()) if listed.strip() else {} diff --git a/src/agent_env/env/envs/computer/utils/composer.py b/src/agent_env/env/envs/computer/utils/composer.py index 195e2c9d6..4fbf4cec6 100644 --- a/src/agent_env/env/envs/computer/utils/composer.py +++ b/src/agent_env/env/envs/computer/utils/composer.py @@ -1,12 +1,15 @@ """The composer: a computer env's base, driver and apps built into one image for each form and arch, on a builder the put's sandbox provider gives (``builder``), tagged in the env's image store by a key of everything that goes into it, so -an image of that key, built for any env and on any builder, is reused. +an image of that key, built for any env and on any builder, is reused. A disk form is made from its arch's container +once that has built: its root filesystem as the disk's, on the base disk's EFI system partition, by a build that runs +only on the builder's own platform. Each result is recorded in the env's metadata as it finishes (``composed``), and the arches that built are written as the env's computer, a ``computer`` artifact over an index of their images.""" from __future__ import annotations import asyncio +import base64 import contextlib import copy import hashlib @@ -20,7 +23,7 @@ import threading import time import urllib.parse -from concurrent.futures import ThreadPoolExecutor, as_completed +from concurrent.futures import Future, ThreadPoolExecutor, as_completed from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path, PurePosixPath @@ -40,10 +43,11 @@ from agent_env.env.env import Env from agent_env.env.envs.computer.utils import boot, builder, clashes, gate, packages from agent_env.env.envs.computer.utils.install import Install +from agent_env.env.published_images import DISK, DISK_TOOLS_IMAGE, published_disk, published_pinned from agent_env.providers.sandbox_providers.host_images import platform_tag from agent_env.store.base import ConcurrentModificationError from agent_env.store.ids import derive_id -from agent_env.store.image_store.image_contents import labels +from agent_env.store.image_store.image_contents import image_config, labels from agent_env.store.image_store.oci_registry_credentials import names_registry from agent_env.store.image_store.registry_api import NoSuchImageError, pin_digest, write_index from agent_env.utils.docker.build import DockerBuildError @@ -115,20 +119,25 @@ def compose(env: ComputerEnv, pairs: list[tuple[str, str]] | None = None, compos with _Progress(env, composing.on_progress) as progress, ThreadPoolExecutor(max_workers=max(len(pairs), 1)) as pool: progress.say(f"composing {', '.join(f'{form}/{arch}' for form, arch in pairs)} on " f"{' or '.join(type(link).__name__ for link in provider.links)}") - futures = {pool.submit(_composed, env, base, driver, form, arch, builds, progress): (form, arch) - for form, arch in pairs} + containers = {arch: pool.submit(_composed, env, base, driver, form, arch, builds, progress) + for form, arch in pairs if form == "container"} + futures = {future: ("container", arch) for arch, future in containers.items()} + futures |= {pool.submit(_composed_disk, env, base, driver, arch, containers.get(arch), builds, progress): + ("disk", arch) for form, arch in pairs if form == "disk"} for future in as_completed(futures): form, arch = futures[future] result = future.result() progress.ended(f"{form}/{arch}", result) record.finished(form, arch, result) - record.computer(_computer(env, base, record.built())) + record.computer(_computer(env, base, record.built(), record.built("disk"))) def current(env: ComputerEnv) -> bool: """Whether the env's computer was written over every pair built, as a put or env build that failed writing it leaves it not.""" - return "computer" in env.metadata and env.metadata["computer"].get("images") == _Record(env).built() + computer, record = env.metadata.get("computer"), _Record(env) + return computer is not None and computer.get("images") == record.built() and computer.get( + "disks", {}) == record.built("disk") def pending(env: ComputerEnv, sandbox_provider: SandboxProvider | None = None) -> list[tuple[str, str]]: @@ -141,7 +150,7 @@ def pending(env: ComputerEnv, sandbox_provider: SandboxProvider | None = None) - for form in gate.forms(env.computer) for arch in gate.arches(env.computer, env.driver, env.composed_apps())} return [(form, arch) for (form, arch), status in statuses.items() - if status not in _BUILT or (status == "unverified" and boot.native_links(provider, arch))] + if status not in _BUILT or (status == "unverified" and boot.native_links(provider, arch, form))] def _composed(env: ComputerEnv, base: ComputerArtifact, driver: ComputerDriverEnv, form: str, arch: str, @@ -153,16 +162,46 @@ def _composed(env: ComputerEnv, base: ComputerArtifact, driver: ComputerDriverEn return _probed(env, driver, form, arch, result, builds.provider, progress) +def _composed_disk(env: ComputerEnv, base: ComputerArtifact, driver: ComputerDriverEnv, arch: str, + container: Future | None, builds: builder.Builds, progress: _Progress) -> dict: + """The disk form of ``arch``, once its container, composed now or before, has built: built or reused, then probed + when it built (gate 3).""" + built = container.result() if container is not None else env.metadata.get("composed", {}).get( + "container", {}).get(arch, {}) + if built.get("status") not in _BUILT: + return {"status": "skipped", "reason": f"its container form didn't build ({built.get('status', 'never')})", + "skipped_at": _now()} + result = _built_disk(env, base, arch, built, builds, progress) + if result["status"] != "unverified": + return result + return _probed(env, driver, "disk", arch, result, builds.provider, progress) + + +async def probe_first_boot(env: ComputerEnv, form: str, arch: str, sandbox_provider: SandboxProvider) -> dict: + """Gate 3 for a build its compose couldn't probe, on its first boot on ``sandbox_provider``, which boots it + natively: the result, recorded.""" + result = await _checked([sandbox_provider], env, env.driver, form, arch, env.metadata["composed"][form][arch]) + await asyncio.to_thread(_Record(env).finished, form, arch, result) + return result + + def _probed(env: ComputerEnv, driver: ComputerDriverEnv, form: str, arch: str, result: dict, sandbox_provider: SandboxProvider, progress: _Progress) -> dict: - """Gate 3: the build of ``result`` booted on the first link of ``sandbox_provider`` that runs ``arch`` natively, and - probed at ``full``: ready, with the driver's card, or failed at the probe; each with a summary of the probe and its - report stored. Still unverified, saying why, when no link runs ``arch`` natively.""" - if not (links := boot.native_links(sandbox_provider, arch)): - return {**result, "reason": f"no link of the sandbox provider runs linux/{arch} natively to probe it"} + """Gate 3: the build of ``result`` booted on the first link of ``sandbox_provider`` that runs ``arch``'s ``form`` + natively, and probed at ``full`` (``_checked``). Still unverified, saying why, when no link does.""" + if not (links := boot.native_links(sandbox_provider, arch, form)): + how = f"boots linux/{arch} disks" if form == "disk" else f"runs linux/{arch}" + return {**result, "reason": f"no link of the sandbox provider {how} natively to probe it"} progress.at(f"{form}/{arch}", "probing", f"probing on {type(links[0]).__name__}") + return asyncio.run(_checked(links, env, driver, form, arch, result)) + + +async def _checked(links: list[SandboxProvider], env: ComputerEnv, driver: ComputerDriverEnv, form: str, arch: str, + result: dict) -> dict: + """The build of ``result`` booted on the first of ``links`` that boots it and probed at ``full``: ready, with the + driver's card, or failed at the probe; each with a summary of the probe and its report stored.""" try: - report, card, provider = asyncio.run(_probe(links, env, driver, result["image"])) + report, card, provider = await _probe(links, env, driver, result["image"], form) except TimeoutError: return _failed(env, form, arch, "probe", result["key"], f"the probe didn't finish within {_PROBE_TIMEOUT_S} s") except Exception as e: @@ -180,11 +219,11 @@ def _probed(env: ComputerEnv, driver: ComputerDriverEnv, form: str, arch: str, r return {**result, "status": "ready", "card": card, "probe": probe} -async def _probe(links: list[SandboxProvider], env: ComputerEnv, driver: ComputerDriverEnv, - image: str) -> tuple[ProbeReport, dict, str]: - """``image`` booted at the env's settings with a control token of its own, as a deploy boots it, on the first of - ``links`` that boots it, probed at ``full`` with that token within ``_PROBE_TIMEOUT_S`` and torn down; the report, - the driver's card and the provider's name.""" +async def _probe(links: list[SandboxProvider], env: ComputerEnv, driver: ComputerDriverEnv, image: str, + form: str = "container") -> tuple[ProbeReport, dict, str]: + """``image``, of ``form``, booted at the env's settings with a control token of its own, as a deploy boots it, on + the first of ``links`` that boots it, probed at ``full`` with that token within ``_PROBE_TIMEOUT_S`` and torn down; + the report, the driver's card and the provider's name.""" token = boot.control_token() environment = boot.settings_environment(env.computer, _values(env), environment_name=env.environment_name) problems = [] @@ -192,7 +231,7 @@ async def _probe(links: list[SandboxProvider], env: ComputerEnv, driver: Compute try: booted = await boot.boot(link, image=image, port=driver.port, environment=environment, token=token, network_policy=None, timeout=_PROBE_TTL_S, attribution=None, - ready_s=boot.ready_timeout(env.composed_apps(), env.computer.os.type)) + ready_s=boot.ready_timeout(env.composed_apps(), env.computer.os.type), form=form) except Exception as e: problems.append(f"{type(link).__name__}: {e}") continue @@ -289,6 +328,94 @@ def on() -> builder.Builder: return _failed(env, form, arch, stage, key, f"{type(e).__name__}: {e}") +def _built_disk(env: ComputerEnv, base: ComputerArtifact, arch: str, container: dict, builds: builder.Builds, + progress: _Progress) -> dict: + """The status of the disk form of ``arch`` once built from ``container``, its container's result, and pushed, or + reused, or the stage that failed and why. Its key is of the container's key, the base disk and the disk recipe; + the base disk is the base's own, the one published for it, or else one made on the builder from the base's + container and the boot recipe. Its build runs no program of ``arch``'s but that recipe's.""" + stage, key, pair, platform = "build", None, f"disk/{arch}", f"linux/{arch}" + try: + with tempfile.TemporaryDirectory(prefix="agentenv-compose-") as work, contextlib.ExitStack() as opened: + context = Path(work) / builder.CONTEXT + shutil.copytree(DISK / "boot", context) + shutil.copytree(DISK / "tools", context / "tools") + base_disk, base_identity, recipe = *_base_disk(base, arch), _digest(context) + key = "sha256:" + hashlib.sha256(json.dumps({ + "container": container["key"], "base_disk": base_identity, "recipe": recipe, "form": "disk", + "arch": arch, "format": FORMAT, + }, sort_keys=True).encode()).hexdigest() + store = get_config().get_image_store_for(env.stored_id) + store.ensure_repository(REPOSITORY) + ref = store.image_ref(REPOSITORY, key.removeprefix("sha256:")) + if (pinned := _served(ref, store)) is not None: + progress.at(pair, "reusing", "reusing the disk of the same key, made earlier") + return {"status": "unverified", "key": key, "image": pinned, "reused": True, "built_at": _now()} + link = builds.link(platform) + progress.at(pair, "starting its builder", f"making the disk {builder.where(link)}") + on = opened.enter_context(builds.open(link, platform, Path(work), base.image(), {})) + on.login(store.auth(container["image"])) + if (tools := published_pinned(DISK_TOOLS_IMAGE)) is None: + tools = f"agentenv-disk-tools:{DISK_TOOLS_IMAGE.tag()}" + on.build((context / "tools" / "Dockerfile").read_text(), tools, on.platform, {}, + context=f"{builder.CONTEXT}/tools") + dockerfile = (context / "Dockerfile").read_text() + if base_disk is None: + progress.at(pair, "making the base's disk") + source, _ = _base(base, arch) + made = hashlib.sha256(f"{base_identity} {recipe}".encode()).hexdigest()[:16] + base_disk = f"agentenv-computer:{made}-base-disk-{arch}" + on.build(dockerfile, base_disk, platform, {}, build_args={ + "ROOTFS": source, "ROOTFS_CONFIG": _encoded(_config(on, source, arch)), "TOOLS": tools, + "SIZE_GB": str(base.forms.disk.min_disk_gb)}) + progress.at(pair, "making the disk") + local = f"agentenv-computer:{key[7:23]}-disk-{arch}" + on.build(dockerfile, local, platform, {KEY_LABEL: key}, build_args={ + "ROOTFS": container["image"], "ROOTFS_CONFIG": _encoded(_config(on, container["image"], arch)), + "ESP": base_disk, "TOOLS": tools}) + stage = "push" + progress.at(pair, "pushing the disk") + on.push(local, ref, store) + return {"status": "unverified", "key": key, "image": pin_digest(ref, store.auth(ref)), "reused": False, + "builder": on.name, "built_at": _now()} + except builder.Skipped as e: + return {"status": "skipped", "reason": str(e), "skipped_at": _now()} + except builder.Unbuildable as e: + return _failed(env, "disk", arch, stage, key, str(e)) + except DockerBuildError as e: + return _failed(env, "disk", arch, stage, key, "the build failed", log=str(e)) + except subprocess.CalledProcessError as e: + return _failed(env, "disk", arch, stage, key, f"{' '.join(e.cmd[:2])} failed: {(e.stderr or '').strip()}") + except (RuntimeError, ValueError, OSError) as e: + return _failed(env, "disk", arch, stage, key, f"{type(e).__name__}: {e}") + + +def _base_disk(base: ComputerArtifact, arch: str) -> tuple[str | None, str]: + """The base's disk for ``arch``, pinned to its digest, and its part of a disk's key: the base's disk form's image, + or the one published for the base's container sources; else None, and the base's container, which a disk is made + from with the boot recipe, the key's recipe.""" + if (image := base.disk_image()) is not None: + store = get_config().get_image_store() + pinned = image.pinned_name if "@" in image.pinned_name else pin_digest(image.pinned_name, + store.auth(image.pinned_name)) + return pinned, pinned + if (published := published_disk(base.image().source_digest or "")) is not None: + return published, published + return None, f"made from {_base(base, arch)[1]}" + + +def _config(on: builder.Builder, image: str, arch: str) -> dict: + """The config object of ``image``'s ``arch`` image: from its registry, or from the builder's Docker for one only it + holds.""" + if names_registry(image): + return image_config(image, arch, get_config().get_image_store().auth(image)) + return on.image_config(image) + + +def _encoded(config: dict) -> str: + return base64.b64encode(json.dumps(config, sort_keys=True).encode()).decode() + + def _compare(building_on: builder.Builder, tag: str, layers: list[_Layer], pair: str, progress: _Progress) -> str | None: """Why ``tag`` is refused: a part's run scripts or install steps certainly undoing what an earlier part's installed @@ -540,33 +667,42 @@ def _failed(env: ComputerEnv, form: str, arch: str, stage: str, key: str | None, return result -def _computer(env: ComputerEnv, base: ComputerArtifact, built: dict[str, str]) -> dict | None: +def _computer(env: ComputerEnv, base: ComputerArtifact, built: dict[str, str], disks: dict[str, str]) -> dict | None: """Write the env's computer over the images ``built``, by arch, joined into one index in the env's image store, signed in to it: its arches those, its OS and session the base's, its settings the base's with the env's values as their - defaults. None when nothing built.""" + defaults; with a disk form over ``disks`` when there's one for each of those arches. None when nothing built.""" if not built: return None arches = [arch for arch in base.arch if arch in built] - store = get_config().get_image_store_for(env.stored_id) - index = store.image_ref(REPOSITORY, "index-" + hashlib.sha256( - ",".join(built[arch] for arch in arches).encode()).hexdigest()) - write_index(index, {f"linux/{arch}": built[arch] for arch in arches}, store.auth(index)) - image = DockerImageArtifact.put_ref(derive_id(env.stored_id, "computer_image"), description=( - f"{env.stored_id}'s composed computer"), image_name=index) + image = _joined(env, "computer_image", "composed computer", built, arches) config = {name: setting.model_dump() for name, setting in base.config.items()} for name, value in _values(env).items(): config[name]["default"] = value + forms = {"container": {"image": {"id": image.id, "version": image.version}, "init": base.forms.container.init, + "input_channel": base.forms.container.input_channel}} + if base.forms.disk is not None and all(arch in disks for arch in arches): + disk = _joined(env, "computer_disk", "composed computer's disk", disks, arches) + forms["disk"] = {**base.forms.disk.model_dump(exclude={"image"}), "image": {"id": disk.id, "version": disk.version}} computer = ComputerArtifact.put( id=derive_id(env.stored_id, "computer"), description=f"{base.id} composed with {env.driver.id} for {env.stored_id}", - arch=arches, os=base.os.model_dump(), session=base.session.model_dump(), config=config, - forms={"container": {"image": {"id": image.id, "version": image.version}, "init": base.forms.container.init, - "input_channel": base.forms.container.input_channel}}) - composed = env.metadata.get("composed", {}).get("container", {}) - if probes := {arch: composed[arch]["probe"] for arch in arches if "probe" in composed.get(arch, {})}: - computer.merge_metadata({"validated_boot": {"container": probes}}) + arch=arches, os=base.os.model_dump(), session=base.session.model_dump(), config=config, forms=forms) + composed = env.metadata.get("composed", {}) + if probes := {form: by_arch for form in forms if (by_arch := { + arch: composed[form][arch]["probe"] for arch in arches if "probe" in composed.get(form, {}).get(arch, {})})}: + computer.merge_metadata({"validated_boot": probes}) return {"id": computer.id, "version": computer.version} +def _joined(env: ComputerEnv, name: str, what: str, images: dict[str, str], arches: list[str]) -> DockerImageArtifact: + """``images`` of ``arches`` joined into one index in the env's image store, signed in to it, as an image artifact.""" + store = get_config().get_image_store_for(env.stored_id) + index = store.image_ref(REPOSITORY, "index-" + hashlib.sha256( + ",".join(images[arch] for arch in arches).encode()).hexdigest()) + write_index(index, {f"linux/{arch}": images[arch] for arch in arches}, store.auth(index)) + return DockerImageArtifact.put_ref(derive_id(env.stored_id, name), description=f"{env.stored_id}'s {what}", + image_name=index) + + def _digest(folder: Path) -> str: """A digest of the files under ``folder``, by their paths, bytes and whether each is executable, which the composer alone decides (umask never sets it).""" @@ -596,16 +732,17 @@ def started(self, pairs: list[tuple[str, str]]) -> None: def finished(self, form: str, arch: str, result: dict) -> None: self._merge({(form, arch): result}) - def built(self) -> dict[str, str]: - """Each arch's image of the container form, of the pairs built or reused.""" - return {arch: result["image"] for arch, result in self.env.metadata.get("composed", {}).get("container", {}).items() + def built(self, form: str = "container") -> dict[str, str]: + """Each arch's image of ``form``, of the pairs built or reused.""" + return {arch: result["image"] for arch, result in self.env.metadata.get("composed", {}).get(form, {}).items() if result.get("status") in _BUILT} def computer(self, ref: dict | None) -> None: - """Record the env's computer, with the images it was written over.""" + """Record the env's computer, with the images it was written over, the disks' apart.""" if ref is not None: with self.lock: - self.env.merge_metadata({"computer": {**ref, "images": self.built()}}) + disks = self.built("disk") + self.env.merge_metadata({"computer": {**ref, "images": self.built(), **({"disks": disks} if disks else {})}}) def _merge(self, results: dict[tuple[str, str], dict]) -> None: with self.lock: diff --git a/src/agent_env/env/published_images.py b/src/agent_env/env/published_images.py index cf7d4ad1e..83d54821a 100644 --- a/src/agent_env/env/published_images.py +++ b/src/agent_env/env/published_images.py @@ -1,10 +1,12 @@ -"""The images agent-env publishes for the infra envs, the agents it ships, an example's env and its base computers, for -linux/amd64 and linux/arm64: one per Dockerfile, built from the Dockerfile's folder. Each is tagged by its build context's content, so -an install holding the same sources finds the image built from them; a release wheel also records the digest its -release published under each tag.""" +"""The images agent-env publishes for the infra envs, the agents it ships, an example's env, its base computers and their +disk forms, for linux/amd64 and linux/arm64: one per Dockerfile, built from the Dockerfile's folder. Each is tagged by +its build context's content, and an image built over other published images by theirs too, so an install holding the +same sources finds the image built from them; a release wheel also records the digest its release published under each +tag.""" from __future__ import annotations +import hashlib import json import logging import re @@ -28,24 +30,31 @@ _AGENTS = Path(__file__).parents[1] / "agents" / "agents" _EXAMPLES = Path(__file__).parents[1] / "examples" _COMPUTERS = Path(__file__).parents[1] / "computers" / "artifacts" +# The Linux disk form's boot recipe, disk tools and QEMU launcher. +DISK = Path(__file__).parents[1] / "computers" / "disk" / "linux" @dataclass(frozen=True) class PublishedImage: """An image agent-env publishes, under ``name`` in the registry, built from ``dockerfile`` with its folder as the - build context.""" + build context, and with each of ``args``, a build arg its Dockerfile builds from, set to that published image.""" name: str dockerfile: Path + args: tuple[tuple[str, PublishedImage], ...] = () @property def repository(self) -> str: return f"{REGISTRY}/{self.name}" def tag(self) -> str: - """``src-`` and the hex of the build context's source digest, the same for the same sources on any platform.""" - digest = BuildContext.of(self.dockerfile.parent, self.dockerfile).source_digest(None) - return f"src-{digest.removeprefix('sha256:')}" + """``src-`` and the hex of the build context's source digest, with each of ``args``' tags when it has any: the + same for the same sources on any platform.""" + digest = BuildContext.of(self.dockerfile.parent, self.dockerfile).source_digest(None).removeprefix("sha256:") + if self.args: + digest = hashlib.sha256("\0".join([digest, *(f"{arg}={image.tag()}" for arg, image in self.args)]) + .encode()).hexdigest() + return f"src-{digest}" def ref(self) -> str: """The repository and the tag of the image built from these sources.""" @@ -61,6 +70,13 @@ def ref(self) -> str: COMPUTER_IMAGES = (PublishedImage("agentenv-ubuntu-24.04", _COMPUTERS / "ubuntu-24.04" / "Dockerfile"),) # The shipped bundles' published images, whose requirements keep a protocol floor. BUNDLE_IMAGES = AGENT_IMAGES + EXAMPLE_IMAGES + COMPUTER_IMAGES +# What makes and boots a Linux disk form: the disk tools a build runs on its own platform, and the QEMU launcher. +DISK_TOOLS_IMAGE = PublishedImage("agentenv-disk-tools", DISK / "tools" / "Dockerfile") +LAUNCHER_IMAGE = PublishedImage("agentenv-qemu-launcher", DISK / "launcher" / "Dockerfile") +# Each base computer's disk form: its container form's root filesystem on the boot recipe, so a run pulls it rather than +# building it from the two. +DISK_IMAGES = tuple(PublishedImage(f"{image.name}-disk", DISK / "boot" / "Dockerfile", + (("ROOTFS", image), ("TOOLS", DISK_TOOLS_IMAGE))) for image in COMPUTER_IMAGES) def published_digest(image: PublishedImage) -> str | None: @@ -70,6 +86,19 @@ def published_digest(image: PublishedImage) -> str | None: return release_digests(RELEASE_FILE).get(ref) or _registry_digest(ref) +def published_pinned(image: PublishedImage) -> str | None: + """``image``'s ref pinned to the digest it's published under for the sources installed, or None.""" + digest = published_digest(image) + return f"{image.ref()}@{digest}" if digest else None + + +def published_disk(sources: str) -> str | None: + """The pinned disk form published for the shipped base computer whose container image's installed build context has + the source digest ``sources``, or None when no shipped base has those sources or its disk isn't published.""" + tag = f"src-{sources.removeprefix('sha256:')}" + return next((published_pinned(disk) for disk in DISK_IMAGES if dict(disk.args)["ROOTFS"].tag() == tag), None) + + def published_bundle_image(sources: str) -> tuple[str, str] | None: """The ref and digest of the image published for a shipped bundle's image (an agent's, an example env's or a base computer's) whose diff --git a/src/agent_env/providers/env_providers/env_computer_provider.py b/src/agent_env/providers/env_providers/env_computer_provider.py index 3db560b1b..fd6e5baa8 100644 --- a/src/agent_env/providers/env_providers/env_computer_provider.py +++ b/src/agent_env/providers/env_providers/env_computer_provider.py @@ -14,12 +14,14 @@ from agent_env.attribution import Attribution from agent_env.env.env import DeployedSandboxEnv, EnvCapabilityUnsupported from agent_env.env.envs.computer.utils import params as computer_params +from agent_env.env.envs.computer.utils.composer import probe_first_boot from agent_env.env.envs.computer.utils.boot import ( app_settings_environment, boot, control_environment, control_token, discard, + native_links, ready, ready_timeout, recorded_settings, @@ -70,7 +72,7 @@ async def deploy( params, values, problems = computer_params.checked(env, env_params) if problems: raise DeployRefused(f"{env.id}: " + "; ".join(problems)) - if not (builds := env.ready_builds(params.form)): + if not (builds := env.deployable_builds(params.form)): raise DeployRefused(f"{env.id} has no ready build to deploy; its builds are {env.build_statuses() or 'none yet'}") attempt = functools.partial(self._boot, env, builds, params, values, ttl_seconds=ttl_seconds, disk_size_gb=disk_size_gb, cpu=cpu, memory_mb=memory_mb, attribution=attribution) @@ -86,12 +88,15 @@ async def _boot(self, env: ComputerEnv, builds: list[tuple[str, str, dict]], par """One attempt on one sandbox provider: the first build it runs natively, booted and readied; its record.""" from agent_env.env.envs.computer.env import ComputerEnv - native = [(form, arch, build) for form, arch, build in builds - if f"linux/{arch}" in sandbox_provider.native_platforms()] + native = [(form, arch, build) for form, arch, build in builds if native_links(sandbox_provider, arch, form)] if not native: raise ValueError(f"{type(sandbox_provider).__name__} runs none of {env.id}'s ready builds natively " f"({', '.join(f'{form}/{arch}' for form, arch, _ in builds)})") form, arch, build = native[0] + if build["status"] == "unverified": + build = await probe_first_boot(env, form, arch, sandbox_provider) + if build["status"] != "ready": + raise RuntimeError(f"{env.id}'s {form}/{arch} failed its probe on its first boot: {build['reason']}") computer, name, token = env.computer_artifact(), env.environment_name, control_token() values = _values(computer, values) policy = NetworkPolicy.from_dict(env.network) if env.network else None @@ -99,7 +104,7 @@ async def _boot(self, env: ComputerEnv, builds: list[tuple[str, str, dict]], par sandbox_provider, image=build["image"], port=env.driver.port, network_policy=policy, token=token, environment=_environment(env, computer, values, params), timeout=ttl_seconds, attribution=attribution, disk_size_gb=disk_size_gb, cpu=cpu, memory_mb=memory_mb, - ready_s=ready_timeout(env.composed_apps(), env.computer.os.type)) + ready_s=ready_timeout(env.composed_apps(), env.computer.os.type), form=form) self._container_sandboxes.append(booted.sandbox) self._environment_sandboxes[name] = booted.sandbox try: @@ -124,9 +129,9 @@ async def _boot(self, env: ComputerEnv, builds: list[tuple[str, str, dict]], par return record async def reboot(self, env: ComputerEnv, deployed: DeployedSandboxEnv) -> None: - """Replace the computer ``deployed`` records with a fresh container of the image it booted, once its recordings - ended, its settings applied as that deploy applied them (a generated one minted afresh) and its control token - the record's, on the same sandbox and port, readied as a deploy readies it.""" + """Replace the computer ``deployed`` records with a fresh container of the image it booted, or its disk booted + afresh, once its recordings ended, its settings applied as that deploy applied them (a generated one minted + afresh) and its control token the record's, on the same sandbox and port, readied as a deploy readies it.""" record = deployed.metadata["computer"] params, values, _ = computer_params.checked(env, record["env_params"]) computer = ComputerArtifact.get(record["computer"]["id"], record["computer"]["version"]) @@ -134,10 +139,13 @@ async def reboot(self, env: ComputerEnv, deployed: DeployedSandboxEnv) -> None: await end_recordings(deployed, "reset") sandbox_provider = build_sandbox_provider(deployed.sandbox_type) sandbox = await sandbox_provider.get_sandbox(deployed.sandbox_id) + environment = {**_environment(env, computer, values, params), **control_environment(deployed.control_token)} try: - await sandbox_provider.reset_container(sandbox, image_name=record["image"], port=env.driver.port, - env={**_environment(env, computer, values, params), - **control_environment(deployed.control_token)}) + if record.get("form") == "disk": + await sandbox_provider.reset_vm(sandbox, image=record["image"], env=environment) + else: + await sandbox_provider.reset_container(sandbox, image_name=record["image"], port=env.driver.port, + env=environment) except NotImplementedError as e: # a reset the env supports, which this provider can't carry out raise RuntimeError(f"'{env.id}' can't be reset on {type(sandbox_provider).__name__}: {e}") from None card = await ready(deployed.environment_url, ready_timeout(env.composed_apps(), env.computer.os.type), diff --git a/src/agent_env/providers/sandbox_providers/AGENTS.md b/src/agent_env/providers/sandbox_providers/AGENTS.md index e25a2f38b..548bd5c56 100644 --- a/src/agent_env/providers/sandbox_providers/AGENTS.md +++ b/src/agent_env/providers/sandbox_providers/AGENTS.md @@ -50,13 +50,15 @@ with `exposed_ports=[port]` and returns the VM, and the deploy then loads and st | Method | Required | Called by | |---|---|---| -| `create_vm(*, image, boot_mode, cpu, memory, disk_size_gb, timeout, exposed_ports, setup_for_gateway, attribution, network_policy) -> VmSandbox` | VM backend | env deploys, `deploy_sandbox` in VM mode, `create_container`, and your `create_sandbox` | +| `create_vm(*, image, boot_mode, cpu, memory, disk_size_gb, timeout, exposed_ports, setup_for_gateway, attribution, network_policy, env) -> VmSandbox` | VM backend | env deploys, `deploy_sandbox` in VM mode, `create_container`, and your `create_sandbox`; with a computer's disk form as `image` (an image holding `/disk/disk.img`) and its `env`, a computer env's deploy and gate 3, on a provider whose `disk_image_platforms()` names the disk's platform: boot the disk, `env` in its NoCloud seed (`computers/disk/linux/launcher/` boots one under QEMU), reached on `exposed_ports`, with no exec channel into it | | `create_sandbox(*, image_name, port, env, cpu, memory, disk_size_gb, timeout, attribution, network_policy) -> Sandbox` | yes (a VM backend wraps `create_vm`) | an agent's sandbox (`provider.create_sandbox`); `create_container` of a container-only backend | | `get_sandbox(sandbox_id) -> Sandbox` | yes | every step after the one that created the sandbox, and teardown | | `terminate_sandbox(sandbox_id)` | no | teardown. The default calls `get_sandbox` then `terminate()`, and treats `SandboxNotFoundError` as done; override when you can terminate without a live handle | | `create_container(...)` | inherited | containers that run an env or agent image. Override only when the platform boots images natively | | `reset_container(sandbox, *, image_name, port, env)` | inherited | a computer env's reset, which replaces the container `create_container` ran with a fresh one on the same sandbox and port. The default works on a VM backend; one whose sandbox is the container itself raises `NotImplementedError` | | `native_platforms()` | no | a computer env's deploy and gate 3, which boot a build only on a provider that runs its arch without emulation. The default is `linux/amd64`; local Docker returns this machine's | +| `disk_image_platforms()` | no | the same for a computer's disk form: the platforms whose disks `create_vm` boots without emulation. The default is none; local Docker returns this machine's where it can pass `/dev/kvm` to a container | +| `reset_vm(sandbox, *, image, env)` | no | a computer env's reset of a disk `create_vm` booted: boot it afresh on the same sandbox and ports. The default raises `NotImplementedError` | | `prepare_image(image, *, attribution)` | no | before a sandbox is asked for an image, outside any creation deadline; build a context-only image here, and pull one with an `image_digest` by it, building it from its context when it can't be pulled | | `from_config(**config)` | no | receives `[sandbox.providers..config]` with `secret:` references resolved. Override to fail early with a `ConfigError` that names the missing key (`boxd` also rejects unknown keys) | | `close()` | no | release SDK clients | diff --git a/src/agent_env/providers/sandbox_providers/local_docker/disk.py b/src/agent_env/providers/sandbox_providers/local_docker/disk.py new file mode 100644 index 000000000..101a14f15 --- /dev/null +++ b/src/agent_env/providers/sandbox_providers/local_docker/disk.py @@ -0,0 +1,57 @@ +"""Booting a computer's disk form on this machine's Docker: the QEMU launcher (the published image for the sources +installed, else built here from its folder) with the disk copied in, as an image this machine keeps for the next boot, +run with /dev/kvm.""" + +import asyncio +import hashlib +import math +import os +import subprocess +import tempfile +from pathlib import Path + +from agent_env.env.published_images import LAUNCHER_IMAGE, published_pinned +from agent_env.providers.sandbox_providers.host_images import host_platform, present +from agent_env.utils.docker.build import build_image + +KVM = "/dev/kvm" +#: How the launcher takes the computer's environment, each variable under this prefix (see its launch script). +ENV_PREFIX = "VM_ENV_" + + +def kvm_platforms() -> frozenset[str]: + """This machine's platform when its Docker can pass KVM to a container: /dev/kvm here, readable and writable.""" + return frozenset({host_platform()}) if os.access(KVM, os.R_OK | os.W_OK) else frozenset() + + +async def machine_image(disk: str) -> str: + """An image of the launcher holding ``disk``'s /disk/disk.img for this machine's platform, built unless held.""" + launcher = await asyncio.to_thread(_launcher) + tag = "agentenv-vm:" + hashlib.sha256(f"{launcher} {disk}".encode()).hexdigest()[:16] + if not await asyncio.to_thread(present, tag): + await asyncio.to_thread(_build, f"FROM {launcher}\nCOPY --from={disk} /disk/ /disk/\n", tag) + return tag + + +def launcher_args(ports: list[int], env: dict[str, str], cpu: float, memory: int) -> list[str]: + """The ``docker run`` arguments that size the launcher's VM, forward ``ports`` and hand it ``env``.""" + variables = {"VM_PORTS": " ".join(map(str, ports)), "VM_CPUS": str(max(1, math.ceil(cpu))), + "VM_MEMORY_MB": str(memory), **{f"{ENV_PREFIX}{name}": value for name, value in env.items()}} + return ["--device", KVM, *(arg for name, value in variables.items() for arg in ("-e", f"{name}={value}"))] + + +def _launcher() -> str: + if (pinned := published_pinned(LAUNCHER_IMAGE)) is not None: + if not present(pinned): + subprocess.run(["docker", "pull", "-q", pinned], check=True, capture_output=True, text=True) + return pinned + tag = f"agentenv-qemu-launcher:{LAUNCHER_IMAGE.tag()}" + if not present(tag): + build_image(LAUNCHER_IMAGE.dockerfile, LAUNCHER_IMAGE.dockerfile.parent, tag, platform=None) + return tag + + +def _build(dockerfile: str, tag: str) -> None: + with tempfile.TemporaryDirectory(prefix="agentenv-vm-") as context: + (Path(context) / "Dockerfile").write_text(dockerfile) + build_image(Path(context) / "Dockerfile", Path(context), tag, platform=None) diff --git a/src/agent_env/providers/sandbox_providers/local_docker/provider.py b/src/agent_env/providers/sandbox_providers/local_docker/provider.py index a6c1dd74f..dee41ac9a 100644 --- a/src/agent_env/providers/sandbox_providers/local_docker/provider.py +++ b/src/agent_env/providers/sandbox_providers/local_docker/provider.py @@ -39,6 +39,7 @@ from agent_env.attribution import Attribution from agent_env.config.paths import state_root from agent_env.providers.sandbox_providers.host_images import build_here, built_here, host_platform, present, pull_here +from agent_env.providers.sandbox_providers.local_docker.disk import kvm_platforms, launcher_args, machine_image from agent_env.providers.sandbox_providers.local_docker.relay import relay_image from agent_env.providers.sandbox_providers.sandbox import ( SANDBOX_LABEL, @@ -115,6 +116,8 @@ def trust_env(directory: str) -> dict[str, str]: _PORT_MAP_FILE = ".port-map.json" # The internal network a container with no network runs on, so a sandbox rebuilt from its id removes it and its relay. _CLOSED_NETWORK_FILE = ".closed-network" +# The size of the VM a disk booted on, so a reset boots it at the same size. +_VM_SIZE_FILE = ".vm-size.json" # The host ports sandboxes publish on: below the default ranges outgoing connections take their local ports from (32768 # up on Linux, the VM Docker Desktop or Rancher Desktop runs Docker in included; 49152 up on macOS and Windows) and # below Kubernetes' node ports (30000-32767). @@ -657,6 +660,10 @@ def url_from_sandbox(self, url: str) -> str: def native_platforms(cls) -> frozenset[str]: return frozenset({host_platform()}) + @classmethod + def disk_image_platforms(cls) -> frozenset[str]: + return kvm_platforms() + @classmethod def supports_network_policy(cls, policy: NetworkPolicy) -> bool: """Unrestricted, or a container with no network at all (``create_container``).""" @@ -675,10 +682,15 @@ async def create_vm( setup_for_gateway: bool = True, attribution: Optional[Attribution] = None, network_policy: Optional[NetworkPolicy] = None, + env: Optional[dict[str, str]] = None, ) -> LocalSandbox: + """This machine as a VM, or with ``image``, a computer's disk form booted under KVM in the QEMU launcher's + container, reached on each of ``exposed_ports``.""" refuse_unenforceable_policy(self, network_policy) if network_policy is not None and network_policy.restricts_egress: raise NetworkPolicyUnsupportedError(f"{type(self).__name__} takes a container's network away, not a VM's") + if image is not None and host_platform() not in self.disk_image_platforms(): + raise ValueError(f"{type(self).__name__} boots a disk only where this machine's Docker can pass it /dev/kvm") # Allocated here rather than in the sandbox constructor: probing for a free port # is I/O, and callers that only need a handle must not pay for it. Off the event # loop, since a draw waits for any other process's draw to finish. @@ -687,8 +699,34 @@ async def create_vm( (sandbox.work_dir / _PORT_MAP_FILE).write_text(json.dumps(sandbox._port_map)) sandbox.network_policy = self.effective_network_policy(network_policy) sandbox.prepared = frozenset(self._prepared) + if image is not None: + (sandbox.work_dir / _VM_SIZE_FILE).write_text(json.dumps({"cpu": cpu, "memory": memory})) + try: + await self._boot_disk(sandbox, image=image, env=env or {}) + (sandbox.work_dir / _CONTAINER_MODE_MARKER).write_text(sandbox.container_name) + except BaseException: + with contextlib.suppress(Exception): + await sandbox.terminate() + await asyncio.to_thread(remove_local_work_dir, sandbox.sandbox_id) + raise return sandbox + async def reset_vm(self, sandbox: Sandbox, *, image: str, env: dict[str, str]) -> None: + await sandbox.exec_script(f"docker rm -f {shlex.quote(sandbox.container_name)} > /dev/null") + await self._boot_disk(sandbox, image=image, env=env) + + async def _boot_disk(self, sandbox: LocalSandbox, *, image: str, env: dict[str, str]) -> None: + """The launcher booting ``image`` as the sandbox's container, its ports published as a container's are.""" + size = json.loads((sandbox.work_dir / _VM_SIZE_FILE).read_text()) + await sandbox.pull_images([image]) + machine = await machine_image(image) + ports = list(sandbox._port_map) + publish = [arg for port in ports for spec in port_bindings(sandbox.host_ips, sandbox.host_port(port), port) + for arg in ("-p", spec)] + await asyncio.to_thread(_docker, "run", "-d", "--name", sandbox.container_name, "--label", + f"{SANDBOX_LABEL}={sandbox.sandbox_id}", *publish, + *launcher_args(ports, env, size["cpu"], size["memory"]), machine) + async def create_container( self, *, diff --git a/src/agent_env/providers/sandbox_providers/sandbox_provider.py b/src/agent_env/providers/sandbox_providers/sandbox_provider.py index 2ee4dfaef..b8928f796 100644 --- a/src/agent_env/providers/sandbox_providers/sandbox_provider.py +++ b/src/agent_env/providers/sandbox_providers/sandbox_provider.py @@ -90,7 +90,10 @@ async def create_vm( setup_for_gateway: bool = True, attribution: Optional[Attribution] = None, network_policy: Optional[NetworkPolicy] = None, + env: Optional[dict[str, str]] = None, ) -> VmSandbox: + """A VM. A provider whose ``disk_image_platforms`` names a platform boots a computer's disk form, ``image``, an + image holding /disk/disk.img, for it, its ``env`` in the disk's NoCloud seed; such a sandbox has no exec channel.""" raise NotImplementedError(f"{type(self).__name__} does not support create_vm") async def create_container( @@ -154,6 +157,11 @@ async def reset_container(self, sandbox: Sandbox, *, image_name: str, port: int, await sandbox.exec_script(f"docker rm -f {shlex.quote(sandbox.container_name)} > /dev/null") await self._start_container(sandbox, image_name=image_name, port=port, env=env) + async def reset_vm(self, sandbox: Sandbox, *, image: str, env: dict[str, str]) -> None: + """Boot the disk create_vm booted on ``sandbox`` afresh, from ``image`` with ``env``, reached on its ports as + before: the sandbox, its id and its URLs stay.""" + raise NotImplementedError(f"{type(self).__name__} can't boot a disk afresh in place") + async def prepare_image(self, image: DockerImageArtifact, *, attribution: Optional[Attribution] = None) -> None: """Get ready to run ``image`` by its name, before create_sandbox or create_container is asked for it and outside any deadline on creating one, for the deploy ``attribution`` describes, as it describes the sandbox. Does @@ -316,6 +324,11 @@ def native_platforms(cls) -> frozenset[str]: """The platforms its containers run without emulation.""" return frozenset({DEFAULT_BUILD_PLATFORM}) + @classmethod + def disk_image_platforms(cls) -> frozenset[str]: + """The platforms whose disk images (a computer's disk form) its create_vm boots without emulation: none here.""" + return frozenset() + @classmethod def effective_network_policy(cls, policy: Optional[NetworkPolicy]) -> NetworkPolicy: """``policy`` with the platform's own hosts unioned in; identity unless it allowlists.""" diff --git a/src/agent_env/store/image_store/image_contents.py b/src/agent_env/store/image_store/image_contents.py index 12f325a9f..d48aff834 100644 --- a/src/agent_env/store/image_store/image_contents.py +++ b/src/agent_env/store/image_store/image_contents.py @@ -1,21 +1,39 @@ -"""What an image a registry serves holds, read without pulling it: the image manifest each Linux architecture runs, and -an image's os-release, from the lowest layer that holds it.""" +"""What an image a registry serves holds, read without pulling it: the image manifest each Linux architecture runs, an +image's config, its os-release, from the lowest layer that holds it, and a disk form's disk, from the start of its +layer.""" from __future__ import annotations +import io import json +import struct import tarfile import tempfile +import zlib +from dataclasses import dataclass from pathlib import Path, PurePosixPath from agent_env.store.image_store.oci_registry_credentials import RegistryAuth -from agent_env.store.image_store.registry_api import pin_digest, read_blob, read_manifest +from agent_env.store.image_store.registry_api import pin_digest, read_blob, read_blob_start, read_manifest _INDEX_TYPES = ("application/vnd.oci.image.index.v1+json", "application/vnd.docker.distribution.manifest.list.v2+json") # os-release(5): /etc/os-release first, else /usr/lib/os-release; on Ubuntu the first links to the second. _OS_RELEASE = ("etc/os-release", "usr/lib/os-release") # tarfile reads zstd from Python 3.14. _READS_ZSTD = "zst" in tarfile.TarFile.OPEN_METH +# How much of a disk layer is read: its tar headers and the disk's own header lie well within it, compressed. +_DISK_START = 1 << 16 +_QCOW2 = b"QFI\xfb" + + +@dataclass(frozen=True) +class DiskLayer: + """What the start of a disk form's layer shows: the first file in it, its owner, and the virtual size of the disk + it holds, a qcow2's from its header, a raw one's its own.""" + + path: str + uid: int + virtual_size: int class UnreadableLayerError(ValueError): @@ -43,6 +61,13 @@ def platform_manifests(ref: str, auth: RegistryAuth | None) -> tuple[str, dict[s def labels(ref: str, arch: str, auth: RegistryAuth | None) -> dict[str, str]: """The labels of ``ref``'s image for ``arch``, read from its config, an index's or a single image's. Raises ValueError when it has none for ``arch``, or as ``read_manifest`` does.""" + return image_config(ref, arch, auth).get("Labels") or {} + + +def image_config(ref: str, arch: str, auth: RegistryAuth | None) -> dict: + """The ``config`` object of ``ref``'s image config for ``arch`` (its environment, entrypoint, command, user, working + folder and labels), an index's or a single image's. Raises ValueError when it has none for ``arch``, or as + ``read_manifest`` does.""" pinned, manifests = platform_manifests(ref, auth) if arch not in manifests: raise ValueError(f"{ref} has no linux/{arch} image") @@ -54,7 +79,28 @@ def labels(ref: str, arch: str, auth: RegistryAuth | None) -> dict[str, str]: config = json.loads(path.read_text()) except OSError as e: raise ValueError(f"its config {manifest['config']['digest']} couldn't be read: {e}") from e - return (config.get("config") or {}).get("Labels") or {} + return config.get("config") or {} + + +def disk_layer(ref: str, digest: str, auth: RegistryAuth | None) -> DiskLayer: + """The disk the image manifest ``digest`` in ``ref``'s repository holds, a disk form's, read from the start of its + one layer. Raises ValueError when it has another number of layers or its start isn't a disk's, + UnreadableLayerError when this Python can't decompress it.""" + layers = read_manifest(f"{ref.partition('@')[0]}@{digest}", auth).get("layers") or [] + if len(layers) != 1: + raise ValueError(f"it has {len(layers)} layers, not the one a disk's image has") + if layers[0].get("mediaType", "").endswith("+zstd"): + raise UnreadableLayerError(f"its layer {layers[0]['digest']} is zstd-compressed, which this reads no part of") + start = read_blob_start(ref, layers[0]["digest"], auth, _DISK_START) + data = zlib.decompressobj(zlib.MAX_WBITS | 32).decompress(start) if start[:2] == b"\x1f\x8b" else start + try: + with tarfile.open(fileobj=io.BytesIO(data), mode="r|") as tar: + member = next(member for member in tar if member.isfile()) + head = tar.extractfile(member).read(32) + except (tarfile.TarError, StopIteration, OSError, EOFError) as e: + raise ValueError(f"the start of its layer {layers[0]['digest']} holds no file: {type(e).__name__}: {e}") from e + size = struct.unpack(">Q", head[24:32])[0] if head[:4] == _QCOW2 and len(head) == 32 else member.size + return DiskLayer(member.name.removeprefix("./"), member.uid, size) def os_release(ref: str, digest: str, auth: RegistryAuth | None) -> dict[str, str]: diff --git a/src/agent_env/store/image_store/registry_api.py b/src/agent_env/store/image_store/registry_api.py index 583716407..61ff38bdb 100644 --- a/src/agent_env/store/image_store/registry_api.py +++ b/src/agent_env/store/image_store/registry_api.py @@ -163,6 +163,26 @@ def read_blob(ref: str, digest: str, auth: RegistryAuth | None, path: Path, *, raise ValueError(f"{ref}: the blob {image.host} served for {digest} hashes to sha256:{hashed.hexdigest()}") +def read_blob_start(ref: str, digest: str, auth: RegistryAuth | None, size: int, *, + timeout: float = _TIMEOUT_SECONDS) -> bytes: + """The first ``size`` bytes of the blob ``digest`` in ``ref``'s repository, or all of a smaller one, read without + fetching the rest, so unchecked against its digest.""" + image = _parse(ref) + if not digest.startswith("sha256:") or not _DIGEST.match(digest): + raise ValueError(f"{ref}: {digest!r} isn't a sha256 digest") + start = bytearray() + with _Repository(image, auth, timeout, ref) as repository: + try: + with repository.stream(f"blobs/{digest}") as response: + for chunk in response.iter_bytes(): + start += chunk + if len(start) >= size: + break + except httpx.HTTPError as e: + raise ValueError(f"{ref}: reading the blob {digest} from {image.host} failed: {type(e).__name__}: {e}") from e + return bytes(start[:size]) + + def write_index(ref: str, images: dict[str, str], auth: RegistryAuth | None, *, timeout: float = _TIMEOUT_SECONDS) -> None: """Write under ``ref``'s tag an index of ``images``, by their platforms, each pinned by its digest in ``ref``'s diff --git a/tst/integration/computer/disk/rootfs/Dockerfile b/tst/integration/computer/disk/rootfs/Dockerfile new file mode 100644 index 000000000..dba0242df --- /dev/null +++ b/tst/integration/computer/disk/rootfs/Dockerfile @@ -0,0 +1,6 @@ +# A small root filesystem, for either arch, that the disk tests make disks of. +FROM public.ecr.aws/docker/library/busybox:1.37@sha256:a7dbc319b1438115a207e11930dc4bb3e643dcb3ec0bdd694bb36cd8fad289ec +ENV GREETING="hello disk" +USER nobody +WORKDIR /tmp +CMD ["sh", "-c", "echo $GREETING"] diff --git a/tst/integration/computer/disk/tasks/probe.json b/tst/integration/computer/disk/tasks/probe.json new file mode 100644 index 000000000..e1e22b0ab --- /dev/null +++ b/tst/integration/computer/disk/tasks/probe.json @@ -0,0 +1,6 @@ +[ + {"id": "env", "type": "deploy_env", "env_id": "@local/agentenv-framework/desktop/desktop", "env_params": {"form": "disk"}}, + {"id": "probe", "type": "env_outcome_verifier", "env_id": "@local/agentenv-framework/desktop/desktop", "file_artifact_id": "@local/agentenv-framework/desktop/check_probe", "verifier_id": "probe", "depends_on": ["env"]}, + {"id": "reset", "type": "reset_env", "env_id": "@local/agentenv-framework/desktop/desktop", "depends_on": ["probe"], "fail_task_on_error": true}, + {"id": "again", "type": "env_outcome_verifier", "env_id": "@local/agentenv-framework/desktop/desktop", "file_artifact_id": "@local/agentenv-framework/desktop/check_probe", "verifier_id": "again", "depends_on": ["reset"]} +] diff --git a/tst/integration/computer/disk_boot_test.py b/tst/integration/computer/disk_boot_test.py new file mode 100644 index 000000000..16fb399b4 --- /dev/null +++ b/tst/integration/computer/disk_boot_test.py @@ -0,0 +1,36 @@ +"""The shipped desktop example's disk form, composed on this machine and booted under KVM in the QEMU launcher through +``agent-env run``: gate 3 probes this machine's arch's disk at full as it composes it, a deploy that names the disk +form boots it with its settings and control token in its seed, it passes the probe at full, and a reset boots it +afresh, where it passes again. Needs Docker, the local registry and /dev/kvm.""" + +from pathlib import Path + +import pytest +from click.testing import CliRunner + +from agent_env.cli import cli +from agent_env.env import Env +from agent_env.providers.sandbox_providers.host_images import host_platform +from agent_env.store.routing import namespace_routing +from tst.integration.computer.conftest import run_context +from tst.util.capabilities import skip_without_kvm + +pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow, skip_without_kvm()] + +BUNDLE = Path(__file__).parent / "disk" +DESKTOP = "@local/agentenv-framework/desktop/desktop" + + +def test_the_desktops_disk_is_probed_as_composed_then_deployed_probed_and_reset_under_kvm(state): + ran = CliRunner().invoke(cli, ["run", str(BUNDLE), "--task", "probe"]) + + assert ran.exit_code == 0, ran.output + arch = host_platform().rsplit("/", 1)[1] + context = run_context(ran.output) + verifications = context["metadata"]["verifications"] + assert all(result["result"] for step in ("probe", "again") for result in verifications[step]["results"]) + (deployed,) = context["deployed_envs"] + assert (deployed["metadata"]["computer"]["form"], deployed["metadata"]["computer"]["arch"]) == ("disk", arch) + with namespace_routing(): + disk = Env.get(DESKTOP).metadata["composed"]["disk"][arch] + assert (disk["status"], disk["probe"]["passed"], disk["probe"]["level"]) == ("ready", True, "full") diff --git a/tst/integration/computer/disk_test.py b/tst/integration/computer/disk_test.py new file mode 100644 index 000000000..ee660277f --- /dev/null +++ b/tst/integration/computer/disk_test.py @@ -0,0 +1,133 @@ +"""A Linux computer's disk form, made by the boot recipe's Dockerfile with the disk tools as a compose builds it: a GPT +disk of an EFI system partition and an ext4 root filesystem that is a container image's, with the image's start in +/etc/agentenv/vm/image.env, in an image holding only /disk/disk.img, owned by uid 107. A disk made over another keeps +its EFI system partition byte for byte and its size, and each arch's disk is made on this machine's own platform, +without emulating the other's. Needs Docker.""" + +import base64 +import io +import json +import subprocess +import tarfile +import uuid +from pathlib import Path + +import pytest + +from agent_env.env.published_images import DISK +from agent_env.providers.sandbox_providers.host_images import host_platform +from agent_env.utils.docker.build import build_image + +pytestmark = pytest.mark.integration + +ROOTFS = Path(__file__).parent / "disk" / "rootfs" +UKI = {"amd64": "BOOTX64.EFI", "arm64": "BOOTAA64.EFI"} +ROOT_TYPE = {"amd64": "4F68BCE3-E8CD-4DB1-96E7-FBCAF984B709", "arm64": "B921B045-1DF0-41C3-AF44-4C6F280D3FAE"} +ESP_TYPE = "C12A7328-F81F-11D2-BA4B-00A0C93EC93B" +# Each partition of a disk, written out with what the tools read of its filesystems. +_READ = """set -e +cd /work +qemu-img info --output=json disk.img > info.json +qemu-img convert -O raw disk.img disk.raw +sfdisk -J disk.raw > parts.json +python3 -c ' +import json +for i, p in enumerate(json.load(open("parts.json"))["partitiontable"]["partitions"]): + print(p["start"], p["size"], ("esp", "root")[i]) +' | while read -r start size name; do dd if=disk.raw of=$name.img bs=512 skip=$start count=$size status=none; done +mtype -i esp.img ::/EFI/BOOT/$1 > uki +debugfs -R "cat /etc/agentenv/vm/image.env" root.img > image.env 2>/dev/null +debugfs -R "stat /bin/busybox" root.img 2>/dev/null | head -1 > busybox +sha256sum esp.img root.img > sums +rm disk.raw esp.img root.img +""" + + +@pytest.fixture(scope="module") +def tools(): + tag = f"agentenv-test-disk-tools:{uuid.uuid4().hex[:8]}" + build_image(DISK / "tools" / "Dockerfile", DISK / "tools", tag, platform=None) + yield tag + subprocess.run(["docker", "rmi", "-f", tag], capture_output=True) + + +@pytest.fixture +def images(): + made = [] + yield made + subprocess.run(["docker", "rmi", "-f", *made], capture_output=True) + + +@pytest.mark.parametrize("arch", ["amd64", "arm64"]) +def test_a_disk_is_a_root_filesystem_with_its_start_beside_a_new_esp_and_one_made_over_it_keeps_that_esp( + tools, images, tmp_path, arch): + platform = f"linux/{arch}" + rootfs = _built(images, ROOTFS, platform) + uki = _built(images, _folder(tmp_path / "uki", {"Dockerfile": f"FROM scratch\nCOPY {UKI[arch]} /\n", + UKI[arch]: f"a UKI for {arch}"}), platform) + first = _disk(images, tools, rootfs, uki, platform, tmp_path, SIZE_GB="2") + changed = _built(images, _folder(tmp_path / "changed", {"Dockerfile": f"FROM {rootfs}\nENV EXTRA=1\n"}), platform) + second = _disk(images, tools, changed, first, platform, tmp_path, ESP=first) + + made, over = _read(tools, first, tmp_path / "first", arch), _read(tools, second, tmp_path / "second", arch) + + for disk in (made, over): + assert disk["file"] == ("disk.img", 107, 107) + assert [(p["type"], p["name"]) for p in disk["partitions"]] == [(ESP_TYPE, "ESP"), + (ROOT_TYPE[arch], "agentenv-root")] + assert disk["virtual_size"] == 2 << 30 and disk["uki"] == f"a UKI for {arch}" and disk["busybox"] + assert made["start"] == [("env", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"), + ("env", "GREETING=hello disk"), ("argv", "sh"), ("argv", "-c"), ("argv", "echo $GREETING"), + ("user", "nobody"), ("workdir", "/tmp")] + assert ("env", "EXTRA=1") in over["start"] and over["esp"] == made["esp"] and over["root"] != made["root"] + + +def _built(images: list, folder: Path, platform: str) -> str: + tag = f"agentenv-test-disk:{uuid.uuid4().hex[:12]}" + build_image(folder / "Dockerfile", folder, tag, platform=platform) + images.append(tag) + return tag + + +def _disk(images: list, tools: str, rootfs: str, esp: str, platform: str, tmp_path: Path, **args: str) -> str: + """The disk the boot recipe's Dockerfile makes of ``rootfs`` for ``platform``, its ESP from ``esp``, the image a UKI + or a disk is in.""" + config = subprocess.run(["docker", "image", "inspect", "--format", "{{json .Config}}", rootfs], capture_output=True, + text=True, check=True).stdout.strip() + tag = f"agentenv-test-disk:{uuid.uuid4().hex[:12]}" + build_image(DISK / "boot" / "Dockerfile", DISK / "boot", tag, platform=platform, build_args={ + "ROOTFS": rootfs, "TOOLS": tools, "ROOTFS_CONFIG": base64.b64encode(config.encode()).decode(), "ESP": esp, + **args}) + images.append(tag) + return tag + + +def _read(tools: str, image: str, folder: Path, arch: str) -> dict: + """What ``image`` holds: its file with its owner, its disk's virtual size and partitions, the UKI its EFI system + partition holds, the start its root filesystem's image.env gives, and each partition's digest.""" + folder.mkdir() + created = subprocess.run(["docker", "create", image, "none"], capture_output=True, text=True, check=True).stdout.strip() + try: + copied = subprocess.run(["docker", "cp", f"{created}:/disk/disk.img", "-"], capture_output=True, check=True) + finally: + subprocess.run(["docker", "rm", created], capture_output=True) + with tarfile.open(fileobj=io.BytesIO(copied.stdout)) as tar: + (member,) = tar.getmembers() + (folder / "disk.img").write_bytes(tar.extractfile(member).read()) + subprocess.run(["docker", "run", "--rm", "--platform", host_platform(), "-v", f"{folder}:/work", "--entrypoint", "sh", + tools, "-c", _READ, "read", UKI[arch]], check=True, capture_output=True) + start = [(key, base64.b64decode(value).decode()) for line in (folder / "image.env").read_text().splitlines() + for key, _, value in [line.partition("=")]] + return {"file": (member.name, member.uid, member.gid), "start": start, "uki": (folder / "uki").read_text(), + "virtual_size": json.loads((folder / "info.json").read_text())["virtual-size"], + "partitions": json.loads((folder / "parts.json").read_text())["partitiontable"]["partitions"], + "busybox": (folder / "busybox").read_text().startswith("Inode:"), + **{name.removesuffix(".img"): digest for digest, name in ( + line.split() for line in (folder / "sums").read_text().splitlines())}} + + +def _folder(folder: Path, files: dict[str, str]) -> Path: + folder.mkdir() + for name, text in files.items(): + (folder / name).write_text(text) + return folder diff --git a/tst/unit/artifact/test_computer_artifact.py b/tst/unit/artifact/test_computer_artifact.py index ff257825a..fde1294ac 100644 --- a/tst/unit/artifact/test_computer_artifact.py +++ b/tst/unit/artifact/test_computer_artifact.py @@ -137,7 +137,7 @@ def test_an_image_the_store_has_no_image_artifact_for_refuses_the_put_inspected_ @pytest.mark.parametrize("changes, message", [ ({"os": {"type": "windows", "version": "11"}}, "os.type 'windows' isn't supported yet"), - ({"forms": {"disk": {}}}, "the disk form isn't supported yet"), + ({"forms": {"lume": {}}}, "the lume form isn't supported yet"), ({"arch": ["amd64", "amd64"]}, "lists an architecture twice"), ({"arch": ["riscv64"]}, "arch.0"), ({"session": {"user": "Root User"}}, "session.user"), diff --git a/tst/unit/bundle/computer_env_folder_test.py b/tst/unit/bundle/computer_env_folder_test.py index 33efd2499..bf78f2791 100644 --- a/tst/unit/bundle/computer_env_folder_test.py +++ b/tst/unit/bundle/computer_env_folder_test.py @@ -107,15 +107,19 @@ def test_a_reused_computer_env_composes_what_it_has_left_on_the_runs_provider_an on_progress=lambda write, line: told.append((write.id, line))) again = materialize(_plan(root), build_here=local_builds(_plan(root)), composing=Composing(provider), on_progress=lambda write, line: told.append((write.id, line))) - env.update_metadata({**env.metadata, "computer": {"id": "c", "version": 1, "images": {"amd64": "a", "arm64": "b"}}, - "composed": {"container": {"amd64": {"status": "ready", "image": "a"}, - "arm64": {"status": "ready", "image": "b"}}}}) + built_ = {form: {arch: {"status": "ready", "image": f"{form}-{arch}"} for arch in ("amd64", "arm64")} + for form in ("container", "disk")} + env.update_metadata({**env.metadata, "composed": built_, "computer": { + "id": "c", "version": 1, "images": {"amd64": "container-amd64", "arm64": "container-arm64"}, + "disks": {"amd64": "disk-amd64", "arm64": "disk-arm64"}}}) built = materialize(_plan(root), build_here=local_builds(_plan(root)), composing=Composing(provider)) assert built.composing == () and checked.composing == again.composing == (env.id,) and all(item.reused for item in again.writes) - assert told == [(env.id, "would compose container/amd64, container/arm64"), (env.id, "composing")] + assert told == [(env.id, "would compose container/amd64, container/arm64, disk/amd64, disk/arm64"), + (env.id, "composing")] assert composed == [(env.id, UBUNTU, REFERENCE, None), - (env.id, UBUNTU, REFERENCE, provider, [("container", "amd64"), ("container", "arm64")])] + (env.id, UBUNTU, REFERENCE, provider, [("container", "amd64"), ("container", "arm64"), + ("disk", "amd64"), ("disk", "arm64")])] def test_a_dry_run_says_what_it_would_compose_of_a_reused_computer_env(tmp_path, composed): @@ -127,7 +131,7 @@ def test_a_dry_run_says_what_it_would_compose_of_a_reused_computer_env(tmp_path, env_id = f"{_plan(root).bundle.bundle.id_root}/desktop" assert dry.materialization.composing == (env_id,) and len(composed) == 1 - assert "envs/desktop: would compose container/amd64, container/arm64" in told + assert "envs/desktop: would compose container/amd64, container/arm64, disk/amd64, disk/arm64" in told def test_the_env_alone_is_planned_whatever_the_bundles_tasks_would_deploy(tmp_path, composed, monkeypatch): diff --git a/tst/unit/env/envs/_computer.py b/tst/unit/env/envs/_computer.py index 374d76484..9f404ae05 100644 --- a/tst/unit/env/envs/_computer.py +++ b/tst/unit/env/envs/_computer.py @@ -29,6 +29,9 @@ OS = {"type": "linux", "distribution": "ubuntu", "version": "24.04", "display": "x11"} +DISK_FORM = {"init": "supervisord", "input_channel": "nocloud", "min_disk_gb": 16} + + CONFIG = {"resolution": {"default": "1280x800", "phase": "deploy"}, "timezone": {"default": "UTC", "phase": "deploy"}, "theme": {"default": "light", "phase": "build"}} @@ -53,7 +56,7 @@ class FakeDocker: def __init__(self, registry: Registry): self.registry, self.builds, self.fail, self.emulates, self.labels = registry, [], {}, {"linux/amd64"}, {} self.probed, self.probe_fails, self.resolved, self.timeouts = [], {}, [], {} - self.writes, self.saved = {}, [] + self.writes, self.saved, self.disk_builds = {}, [], [] def resolve(self, request, context, source, arch, work): """The resolver in the base: each apt package as one .deb, each venv's requirements as one wheel.""" @@ -81,9 +84,9 @@ def resolving(on, request, source, arch, timeout): return (resolve or self.resolve)(request, on.work / builder.CONTEXT, source, arch, on.work / builder.RESOLVED) return resolving - async def probe(self, links, env, driver, image): + async def probe(self, links, env, driver, image, form="container"): arch = self._arch(self.registry.manifests[image.rpartition("@")[2]]["config"]["digest"]) - self.probed.append((arch, image)) + self.probed.append((arch, image) if form == "container" else (form, arch, image)) if arch in self.probe_fails: checks = [CheckResult(check="P4", name="clicks show their effect", status="failed", detail=self.probe_fails[arch])] else: @@ -96,6 +99,10 @@ def build_image(self, dockerfile, context, tag, *, platform, labels=None, build_ return if platform in self.fail: raise DockerBuildError(self.fail[platform]) + if build_args is not None or context.name == "tools": + self.disk_builds.append({"platform": platform, "tag": tag, "dockerfile": dockerfile.read_text(), + "args": build_args, "context": context.name, "labels": labels}) + return self.builds.append({"platform": platform, "tag": tag, "dockerfile": dockerfile.read_text(), "files": sorted(p.relative_to(context).as_posix() for p in context.rglob("*") if p.is_file()), "conf": next(context.glob("agentenv-*.conf")).read_text(), @@ -122,8 +129,11 @@ def save(self, tag, path): saved_image(path, layers) def push(self, local, ref, store): - self.registry.tags[ref] = self.registry.image(UBUNTU_LAYER, arch=local.rsplit("-", 1)[1], - labels=self.labels.get(local)) + layer = disk_layer() if "-disk-" in local else UBUNTU_LAYER + self.registry.tags[ref] = self.registry.image(layer, arch=local.rsplit("-", 1)[1], labels=self.labels.get(local)) + + def image_config(self, tag): + return {"Env": ["PATH=/usr/bin"], "Entrypoint": ["/usr/bin/tini", "--"], "User": "user", "Labels": {"tag": tag}} def joined(self, index, images, auth): self.registry.tags[index] = self.registry.index(**{ @@ -133,6 +143,20 @@ def _arch(self, config_digest): return json.loads(self.registry.blobs[config_digest])["architecture"] +def disk_layer(virtual_size=16 << 30, owner=107, path="disk/disk.img"): + """A disk form's one layer, gzipped: its folder, then a qcow2 of ``virtual_size`` owned by ``owner``.""" + head = b"QFI\xfb" + bytes(20) + virtual_size.to_bytes(8, "big") + bytes(480) + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w:gz") as tar: + folder = tarfile.TarInfo(path.rpartition("/")[0]) + folder.type = tarfile.DIRTYPE + tar.addfile(folder) + disk = tarfile.TarInfo(path) + disk.size, disk.uid = len(head), owner + tar.addfile(disk, io.BytesIO(head)) + return buffer.getvalue() + + def saved_image(path, layers): """A tar of an image as ``docker save`` writes one: each of ``layers``, bottom first, its history's ``created_by`` (None for a layer with no entry) and its files by path, a file's content as bytes, or with its mode as (bytes, mode), diff --git a/tst/unit/env/envs/conftest.py b/tst/unit/env/envs/conftest.py index 065f6a885..b9d3f1b3c 100644 --- a/tst/unit/env/envs/conftest.py +++ b/tst/unit/env/envs/conftest.py @@ -14,7 +14,7 @@ from agent_env.store.image_store.local_registry_image_store import LocalRegistryImageStore from agent_env.store.object_store.local.store import LocalFilesystemObjectStore from tst.util.oci_registry import UBUNTU_LAYER, Registry -from tst.unit.env.envs._computer import FakeDocker, _base, _driver +from tst.unit.env.envs._computer import DISK_FORM, FakeDocker, _base, _driver @pytest.fixture @@ -33,9 +33,11 @@ def docker(local_stores, cli_routing, monkeypatch): monkeypatch.setattr(composer, "_probe", fake.probe) monkeypatch.setattr(builder, "_emulated_platforms", lambda: {"linux/arm64", *fake.emulates}) monkeypatch.setattr(builder.LocalBuilder, "installed", lambda self, tag, prefix: {"mss": "10.1.0"}) + monkeypatch.setattr(builder.LocalBuilder, "image_config", lambda on, tag: fake.image_config(tag)) monkeypatch.setattr(composer, "_protocol_source", lambda: None) monkeypatch.setattr(builder.LocalBuilder, "resolve", fake.resolving()) monkeypatch.setattr(LocalRegistryImageStore, "ensure_repository", lambda self, repository: None) + monkeypatch.setattr(local_sandbox, "kvm_platforms", lambda: frozenset()) return fake @@ -48,6 +50,12 @@ def base(docker): return _base(image) +@pytest.fixture +def disk_base(base): + """``base`` with a disk form made from its container when composed.""" + return _base(base.image(), forms={**base.forms.model_dump(exclude_none=True), "disk": DISK_FORM}) + + @pytest.fixture def driver(docker, tmp_path): return _driver(tmp_path / "driver") diff --git a/tst/unit/env/envs/test_computer_disk.py b/tst/unit/env/envs/test_computer_disk.py new file mode 100644 index 000000000..5d250c02e --- /dev/null +++ b/tst/unit/env/envs/test_computer_disk.py @@ -0,0 +1,238 @@ +"""Computer envs of a base with a disk form: each arch's disk is made from its built container, on the base's disk or on +one made from the base's container and the boot recipe, by a build that runs on the builder's own platform; reused by +its key; probed where a provider boots disks natively, else left unverified for a deploy's first boot with KVM to +probe; and written as the computer's disk form. A deploy that names the disk form boots it in a VM, its settings in its +seed, and a reset boots it afresh. Docker and the registry are faked.""" + +import base64 +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest +from agentenv_protocol import CONTROL_TOKEN_ENV + +from agent_env.artifact.artifacts.computer import ComputerArtifact +from agent_env.artifact.artifacts.docker_image import DockerImageArtifact +from agent_env.artifact.store import get_artifact_store +from agent_env.env import Env +from agent_env.env.envs.computer.utils import boot, composer +from agent_env.env.published_images import DISK +from agent_env.providers.env_providers import env_computer_provider +from agent_env.providers.sandbox_providers.local_docker import provider as local_sandbox +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_steps.reset_env import ResetEnvTaskStep +from tst.unit.env.envs._computer import DISK_FORM, _base, _put, disk_layer +from tst.unit.env.envs.test_computer_env import DRIVER_CARD + +BOOT_DOCKERFILE = (DISK / "boot" / "Dockerfile").read_text() + + +def test_each_arch_has_a_disk_made_from_its_container_on_a_base_disk_made_from_the_bases_container(disk_base, driver, + docker): + env = Env.get(_put(disk_base, driver).id) + + containers, disks = env.metadata["composed"]["container"], env.metadata["composed"]["disk"] + assert {arch: r["status"] for arch, r in disks.items()} == {"amd64": "unverified", "arm64": "unverified"} + assert disks["arm64"]["reason"] == "no link of the sandbox provider boots linux/arm64 disks natively to probe it" + assert all(r["image"].startswith("localhost:5000/agentenv-computers:") for r in disks.values()) + built = {build["tag"].split(":")[1].split("-", 1)[1]: build for build in docker.disk_builds + if build["context"] != "tools"} + for arch in ("amd64", "arm64"): + made, disk = built[f"base-disk-{arch}"], built[f"disk-{arch}"] + assert made["dockerfile"] == disk["dockerfile"] == BOOT_DOCKERFILE and made["platform"] == f"linux/{arch}" + assert made["args"]["ROOTFS"] == disk_base.image().image_name and made["args"]["SIZE_GB"] == "16" + assert "ESP" not in made["args"] and made["args"]["TOOLS"] == disk["args"]["TOOLS"] + assert (disk["args"]["ROOTFS"], disk["args"]["ESP"]) == (containers[arch]["image"], made["tag"]) + assert json.loads(base64.b64decode(disk["args"]["ROOTFS_CONFIG"])) == {"Labels": docker.labels[ + f"agentenv-computer:{containers[arch]['key'][7:23]}-{arch}"]} + assert disk["labels"] == {composer.KEY_LABEL: disks[arch]["key"]} + tools = [build for build in docker.disk_builds if build["context"] == "tools"] + assert {(build["platform"], build["tag"]) for build in tools} == {("linux/arm64", built["disk-amd64"]["args"]["TOOLS"]), + ("linux/arm64", built["disk-arm64"]["args"]["TOOLS"])} + assert tools[0]["dockerfile"] == (DISK / "tools" / "Dockerfile").read_text() + computer = env.computer_artifact() + assert computer.forms.disk.model_dump(exclude={"image"}) == {**DISK_FORM, "firmware": ["efi"]} + assert computer.disk_image().image_name.startswith("localhost:5000/agentenv-computers:index-") + assert {arch: r["status"] for arch, r in computer.metadata["validated_images"]["disk"].items()} == { + "amd64": "passed", "arm64": "passed"} + assert env.metadata["computer"]["disks"] == {arch: r["image"] for arch, r in disks.items()} + assert composer.current(env) and composer.pending(env) == [] + + +def test_a_disk_is_reused_by_its_key_of_its_containers_key_the_base_disk_and_the_recipe(disk_base, driver, docker, + monkeypatch): + first = Env.get(_put(disk_base, driver).id).metadata["composed"]["disk"] + made = len(docker.disk_builds) + other = Env.get(_put(disk_base, driver, id="@local/~/kit/other", environment_name="other").id) + after_other = len(docker.disk_builds) + monkeypatch.setattr(composer, "FORMAT", composer.FORMAT + 1) + raised = Env.get(_put(disk_base, driver, id="@local/~/kit/third", environment_name="third").id) + + reused = other.metadata["composed"]["disk"] + assert {arch: (r["key"], r["image"], r["reused"]) for arch, r in reused.items()} == { + arch: (r["key"], r["image"], True) for arch, r in first.items()} and after_other == made + assert all(raised.metadata["composed"]["disk"][arch]["key"] != first[arch]["key"] for arch in first) + assert len(docker.disk_builds) > made + + +def test_a_base_with_a_disk_image_or_a_published_one_keeps_its_partition_and_makes_none(disk_base, driver, docker, + monkeypatch): + registry = docker.registry + disks = registry.index(amd64=registry.image(disk_layer()), arm64=registry.image(disk_layer(), arch="arm64")) + image = get_artifact_store().put_document(DockerImageArtifact(id="base-disk", description="", + image_name=f"ghcr.io/acme/base-disk:1@{disks}")) + with_disk = _base(disk_base.image(), id="@local/~/kit/with-disk", forms={ + **disk_base.forms.model_dump(exclude_none=True), "disk": {**DISK_FORM, "image": {"id": image.id, + "version": image.version}}}) + monkeypatch.setattr(composer, "published_disk", lambda sources: "ghcr.io/scaleapi/base-disk:src-1@sha256:" + "a" * 64) + + own = Env.get(_put(with_disk, driver).id).metadata["composed"]["disk"] + published = Env.get(_put(disk_base, driver, id="@local/~/kit/other", environment_name="other").id) + + esps = {build["args"]["ESP"] for build in docker.disk_builds if "ESP" in (build["args"] or {})} + assert esps == {image.image_name, "ghcr.io/scaleapi/base-disk:src-1@sha256:" + "a" * 64} + assert not [build for build in docker.disk_builds if "base-disk" in build["tag"]] + assert {arch: r["status"] for arch, r in own.items()} == {"amd64": "unverified", "arm64": "unverified"} + assert own["arm64"]["key"] != published.metadata["composed"]["disk"]["arm64"]["key"] + + +def test_a_disk_isnt_made_for_a_container_that_didnt_build_and_is_made_by_the_next_build(disk_base, driver, docker): + docker.fail["linux/amd64"] = "boom" + env = Env.get(_put(disk_base, driver).id) + disks, computer = env.metadata["composed"]["disk"], env.computer_artifact() + docker.fail.clear() + + composer.compose(env, composer.pending(env)) + + assert (disks["amd64"]["status"], disks["amd64"]["reason"]) == ("skipped", "its container form didn't build (failed)") + assert disks["arm64"]["status"] == "unverified" and computer.arch == ["arm64"] and computer.forms.disk is not None + rebuilt = Env.get(env.id) + assert rebuilt.metadata["composed"]["disk"]["amd64"]["status"] == "unverified" and composer.current(rebuilt) + assert rebuilt.computer_artifact().arch == ["amd64", "arm64"] + + +def test_a_disk_is_probed_on_a_provider_that_boots_its_arch_natively(disk_base, driver, docker, monkeypatch): + monkeypatch.setattr(local_sandbox, "kvm_platforms", lambda: frozenset({"linux/arm64"})) + + env = Env.get(_put(disk_base, driver).id) + + disks = env.metadata["composed"]["disk"] + assert {arch: r["status"] for arch, r in disks.items()} == {"amd64": "unverified", "arm64": "ready"} + assert ("disk", "arm64", disks["arm64"]["image"]) in docker.probed + assert env.computer_artifact().metadata["validated_boot"]["disk"] == {"arm64": disks["arm64"]["probe"]} + + +@pytest.fixture +def vms(monkeypatch): + """The VMs a deploy's boots of a disk ask the local provider for, each serving the desktop driver's card.""" + made = SimpleNamespace(created=[], reset=[]) + + async def create_vm(self, **kwargs): + made.created.append(kwargs) + sandbox = local_sandbox.LocalSandbox(port_map={18765: 20001}, sandbox_id=f"local-vm{len(made.created)}") + sandbox.terminate = AsyncMock() + return sandbox + + async def reset_vm(self, sandbox, **kwargs): + made.reset.append((sandbox.sandbox_id, kwargs)) + + monkeypatch.setattr(local_sandbox.LocalSandboxProvider, "create_vm", create_vm) + monkeypatch.setattr(local_sandbox.LocalSandboxProvider, "reset_vm", reset_vm) + monkeypatch.setattr(local_sandbox, "kvm_platforms", lambda: frozenset({"linux/arm64"})) + monkeypatch.setattr(boot, "ready", AsyncMock(side_effect=lambda url, timeout, token: { + **DRIVER_CARD, "name": "desktop"})) + monkeypatch.setattr(env_computer_provider, "ready", boot.ready) + monkeypatch.setattr(env_computer_provider, "_require_display", AsyncMock()) + return made + + +@pytest.mark.asyncio +async def test_a_deploy_of_the_disk_form_boots_it_in_a_vm_with_its_settings_and_token_in_its_seed(disk_base, driver, + docker, vms): + env = _put(disk_base, driver, display={"width": 1600, "height": 900}) + + deployed = await Env.get(env.id).deploy(env_params={"form": "disk", "computer_settings": {"timezone": "UTC"}}) + + (created,) = vms.created + disk = Env.get(env.id).metadata["composed"]["disk"]["arm64"] + assert (created["image"], created["exposed_ports"], created["cpu"], created["memory"]) == ( + disk["image"], [18765], boot.DISK_CPU, boot.DISK_MEMORY_MB) + assert created["env"]["AGENTENV_SETTING_RESOLUTION"] == "1600x900" and created["env"]["ENVIRONMENT_NAME"] == "desktop" + assert created["env"][CONTROL_TOKEN_ENV] == deployed.control_token + assert (deployed.metadata["computer"]["form"], deployed.metadata["computer"]["image"]) == ("disk", disk["image"]) + assert deployed.environment_url == "http://127.0.0.1:20001" + + +@pytest.mark.asyncio +async def test_an_unverified_disk_is_probed_on_its_first_boot_with_kvm_before_its_deploy_boots_it(disk_base, driver, + docker, vms, + monkeypatch): + monkeypatch.setattr(local_sandbox, "kvm_platforms", lambda: frozenset()) + env = _put(disk_base, driver) + failing = _put(disk_base, driver, id="@local/~/kit/failing", environment_name="failing") + monkeypatch.setattr(local_sandbox, "kvm_platforms", lambda: frozenset({"linux/arm64"})) + + await Env.get(env.id).deploy(env_params={"form": "disk"}) + docker.probe_fails["arm64"] = "the click never landed" + with pytest.raises(RuntimeError, match="failed its probe on its first boot: the probe failed: P4"): + await Env.get(failing.id).deploy(env_params={"form": "disk"}) + + assert Env.get(env.id).metadata["composed"]["disk"]["arm64"]["status"] == "ready" + assert [probe[0] for probe in docker.probed if probe[0] == "disk"] == ["disk", "disk"] + assert Env.get(failing.id).metadata["composed"]["disk"]["arm64"]["status"] == "failed" + assert len(vms.created) == 1 + + +def test_a_disk_form_is_refused_at_save_where_no_provider_boots_its_disk_natively(disk_base, driver, docker): + env = Env.get(_put(disk_base, driver).id) + + assert env.deploy_refusal(env_params={"form": "disk"}) == ( + f"computer env {env.id!r}: LocalSandboxProvider runs none of its arches (amd64, arm64) natively as disk") + assert env.deploy_refusal() is None + + +@pytest.mark.asyncio +async def test_a_reset_of_a_disk_boots_it_afresh_in_its_vm_with_the_records_settings_and_token(disk_base, driver, + docker, vms, monkeypatch): + env = _put(disk_base, driver, display={"width": 1600, "height": 900}) + deployed = await Env.get(env.id).deploy(env_params={"form": "disk"}) + monkeypatch.setattr(local_sandbox.LocalSandboxProvider, "get_sandbox", AsyncMock( + return_value=local_sandbox.LocalSandbox(port_map={18765: 20001}, sandbox_id="local-vm1"))) + monkeypatch.setattr(env_computer_provider, "end_recordings", AsyncMock()) + + await ResetEnvTaskStep(id="reset", version=None, env_id=env.id, fail_task_on_error=True).execute( + TaskStepContext(deployed_envs=[deployed])) + + ((sandbox_id, kwargs),) = vms.reset + assert (sandbox_id, kwargs["image"]) == ("local-vm1", deployed.metadata["computer"]["image"]) + assert kwargs["env"][CONTROL_TOKEN_ENV] == deployed.control_token + assert kwargs["env"]["AGENTENV_SETTING_RESOLUTION"] == "1600x900" + + +def test_a_computer_artifact_takes_a_disk_form_and_reads_its_image(base, docker): + registry = docker.registry + good = registry.index(amd64=registry.image(disk_layer()), arm64=registry.image(disk_layer(), arch="arm64")) + bad = registry.index(amd64=registry.image(disk_layer(virtual_size=8 << 30, owner=0)), + arm64=registry.image(disk_layer(path="disk/other.img"), arch="arm64")) + images = {name: get_artifact_store().put_document(DockerImageArtifact( + id=name, description="", image_name=f"ghcr.io/acme/{name}:1@{index}")) for name, index in (("good", good), + ("bad", bad))} + + def put(id, disk): + return ComputerArtifact.put(**{**base.model_dump(include={"arch", "os", "session", "config"}), "id": id, + "forms": {**base.forms.model_dump(exclude_none=True), "disk": {**DISK_FORM, **disk}}}) + + derived = put("@local/~/kit/derived", {}) + checked = put("@local/~/kit/good", {"image": {"id": "good", "version": images["good"].version}}) + with pytest.raises(ValueError) as refused: + put("@local/~/kit/bad", {"image": {"id": "bad", "version": images["bad"].version}}) + + assert derived.forms.disk.image is None and derived.disk_image() is None + assert {r["status"] for r in derived.metadata["validated_images"]["disk"].values()} == {"uninspected"} + assert {arch: r["status"] for arch, r in checked.metadata["validated_images"]["disk"].items()} == { + "amd64": "passed", "arm64": "passed"} + assert str(refused.value) == ( + "computer '@local/~/kit/bad': forms.disk (amd64): disk/disk.img is owned by uid 0, not 107; its virtual size is " + "8.0 GiB, under min_disk_gb (16); forms.disk (arm64): its first file is disk/other.img, not disk/disk.img") + assert ComputerArtifact.get(checked.id).forms.disk.image.id == "good" diff --git a/tst/unit/env/envs/test_disk_recipe.py b/tst/unit/env/envs/test_disk_recipe.py new file mode 100644 index 000000000..d1f7b7d42 --- /dev/null +++ b/tst/unit/env/envs/test_disk_recipe.py @@ -0,0 +1,106 @@ +"""The Linux disk form's recipe as the init reads what it writes: the image's start (image-env, run by the disk tools) and +the deploy's environment (the launcher's NoCloud seed), each a line per value base64'd so any bytes survive; and the +local provider booting a disk in the launcher, with KVM and the VM's ports, settings and size.""" + +import base64 +import json +import os +import subprocess +from unittest.mock import AsyncMock + +import pytest + +from agent_env.env.published_images import DISK +from agent_env.providers.sandbox_providers.local_docker import disk as local_disk +from agent_env.providers.sandbox_providers.local_docker import provider as local_sandbox +from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandbox, LocalSandboxProvider +from agent_env.providers.sandbox_providers.sandbox import SANDBOX_LABEL + +TRICKY = "a value with spaces, = signs, 'quotes', $dollars and\na second line" + + +def _decoded(text: str) -> list[tuple[str, str]]: + """Each line's key and value, as the disk's init decodes them.""" + return [(key, base64.b64decode(value).decode()) for line in text.splitlines() if line and not line.startswith("#") + for key, _, value in [line.partition("=")]] + + +def test_image_env_writes_an_images_start_a_line_a_value_root_and_slash_when_it_names_none(tmp_path): + config = tmp_path / "config.json" + config.write_text(json.dumps({"Env": ["PATH=/usr/bin", f"NOTE={TRICKY}"], "Entrypoint": ["/usr/bin/tini", "--"], + "Cmd": ["start-desktop", "init"], "User": "user", "WorkingDir": "/home/user", + "Labels": {"ignored": "yes"}})) + bare = tmp_path / "bare.json" + bare.write_text(json.dumps({"Cmd": ["sh"]})) + + start, plain = (subprocess.run(["python3", str(DISK / "tools" / "image-env"), str(path)], capture_output=True, + text=True, check=True).stdout for path in (config, bare)) + + assert _decoded(start) == [("env", "PATH=/usr/bin"), ("env", f"NOTE={TRICKY}"), ("argv", "/usr/bin/tini"), + ("argv", "--"), ("argv", "start-desktop"), ("argv", "init"), ("user", "user"), + ("workdir", "/home/user")] + assert _decoded(plain) == [("argv", "sh"), ("user", "root"), ("workdir", "/")] + + +def test_the_seed_holds_each_vm_env_variable_by_its_name_and_nothing_else(tmp_path): + environment = {"PATH": os.environ["PATH"], "VM_ENV_AGENTENV_SETTING_RESOLUTION": "1280x800", + "VM_ENV_AGENTENV_CONTROL_TOKEN": TRICKY, "VM_ENV_EMPTY": "", "VM_PORTS": "18765", "HOME": "/root"} + + subprocess.run(["sh", str(DISK / "launcher" / "seed"), str(tmp_path)], env=environment, check=True) + + user_data, meta_data = (tmp_path / "user-data").read_text(), (tmp_path / "meta-data").read_text() + assert user_data.splitlines()[0] == "#agentenv-environment" + assert _decoded(user_data) == [("AGENTENV_CONTROL_TOKEN", TRICKY), ("AGENTENV_SETTING_RESOLUTION", "1280x800"), + ("EMPTY", "")] + assert meta_data.endswith("\nlocal-hostname: computer\n") and meta_data.startswith("instance-id: ") + + +def test_the_launcher_gets_kvm_the_vms_ports_and_size_and_its_environment_under_its_prefix(): + args = local_disk.launcher_args([18765, 5900], {"ENVIRONMENT_NAME": "desk", "A": "b c"}, 1.5, 4096) + + assert args == ["--device", "/dev/kvm", "-e", "VM_PORTS=18765 5900", "-e", "VM_CPUS=2", "-e", "VM_MEMORY_MB=4096", + "-e", "VM_ENV_ENVIRONMENT_NAME=desk", "-e", "VM_ENV_A=b c"] + + +@pytest.fixture +def launched(monkeypatch): + """The ``docker`` commands the local provider runs for a disk, with the launcher image it builds faked.""" + ran = [] + monkeypatch.setattr(local_sandbox, "kvm_platforms", lambda: frozenset({"linux/arm64"})) + monkeypatch.setattr(local_sandbox, "host_platform", lambda: "linux/arm64") + monkeypatch.setattr(local_sandbox, "machine_image", AsyncMock(return_value="agentenv-vm:0123")) + monkeypatch.setattr(local_sandbox, "_free_host_port", lambda: 20001) + monkeypatch.setattr(local_sandbox, "_host_ips", lambda: ("127.0.0.1",)) + monkeypatch.setattr(local_sandbox, "_docker", lambda *args: ran.append(args) or "") + monkeypatch.setattr(LocalSandbox, "pull_images", AsyncMock()) + monkeypatch.setattr(LocalSandbox, "exec_script", AsyncMock(side_effect=lambda script: ran.append(script) or "")) + return ran + + +@pytest.mark.asyncio +async def test_a_disk_boots_in_the_launcher_with_kvm_its_ports_published_and_a_reset_boots_it_afresh(launched): + provider = LocalSandboxProvider() + + sandbox = await provider.create_vm(image="localhost:5000/disk@sha256:" + "a" * 64, exposed_ports=[18765], + env={"ENVIRONMENT_NAME": "desk"}, cpu=2, memory=4096) + await provider.reset_vm(sandbox, image="localhost:5000/disk@sha256:" + "a" * 64, env={"ENVIRONMENT_NAME": "again"}) + rebuilt = await provider.get_sandbox(sandbox.sandbox_id) + + first, removed, second = launched + assert first == ("run", "-d", "--name", sandbox.container_name, "--label", f"{SANDBOX_LABEL}={sandbox.sandbox_id}", + "-p", "127.0.0.1:20001:18765", *local_disk.launcher_args([18765], {"ENVIRONMENT_NAME": "desk"}, + 2, 4096), "agentenv-vm:0123") + assert removed == f"docker rm -f {sandbox.container_name} > /dev/null" + assert second[-2:] == ("VM_ENV_ENVIRONMENT_NAME=again", "agentenv-vm:0123") and "VM_CPUS=2" in second + LocalSandbox.pull_images.assert_awaited_with(["localhost:5000/disk@sha256:" + "a" * 64]) + assert sandbox.tunnel_urls == {18765: "http://127.0.0.1:20001"} and rebuilt.owns_container + + +@pytest.mark.asyncio +async def test_a_disk_isnt_booted_where_docker_cant_pass_kvm(launched, monkeypatch): + monkeypatch.setattr(local_sandbox, "kvm_platforms", lambda: frozenset()) + + with pytest.raises(ValueError, match="boots a disk only where this machine's Docker can pass it /dev/kvm"): + await LocalSandboxProvider().create_vm(image="disk", exposed_ports=[18765], env={}) + + assert launched == [] and LocalSandboxProvider.disk_image_platforms() == frozenset() diff --git a/tst/unit/env/published_images_test.py b/tst/unit/env/published_images_test.py index 3973041e3..d433d7534 100644 --- a/tst/unit/env/published_images_test.py +++ b/tst/unit/env/published_images_test.py @@ -1,5 +1,6 @@ -"""The images agent-env publishes for its infra envs, its agents and an example's env: one per shipped Dockerfile, -tagged by its sources, and the record of their digests a release wheel carries.""" +"""The images agent-env publishes for its infra envs, its agents, an example's env, its base computers and their disk +forms: one per shipped Dockerfile, tagged by its sources (a disk form's by its base's too), and the record of their +digests a release wheel carries.""" import hashlib import json @@ -10,12 +11,17 @@ from agent_env.env import bootstrap, published_images from agent_env.env.published_images import ( AGENT_IMAGES, + COMPUTER_IMAGES, + DISK, + DISK_IMAGES, + DISK_TOOLS_IMAGE, EXAMPLE_IMAGES, REGISTRY, PublishedImage, _registry_digest, published_bundle_image, published_digest, + published_disk, release_digests, write_release_digests, ) @@ -26,11 +32,14 @@ def test_every_shipped_dockerfile_is_published_once_under_a_name_of_its_own(): - """The infra envs', every shipped agent's, the env `a2a-agent validate` deploys and every shipped base computer's.""" + """The infra envs', every shipped agent's, the env `a2a-agent validate` deploys, every shipped base computer's, and + the Linux disk form's recipe: its disk tools, its QEMU launcher, and each base's disk, from the boot recipe.""" shipped = [dockerfile for dockerfiles in bootstrap._DOCKERFILES.values() for dockerfile in dockerfiles] shipped += published_images._AGENTS.glob("*/Dockerfile") shipped.append(published_images._EXAMPLES / "mail-mcp" / "envs" / "mail" / "Dockerfile") shipped += published_images._COMPUTERS.glob("*/Dockerfile") + shipped += DISK.glob("*/Dockerfile") + shipped += [DISK / "boot" / "Dockerfile"] * (len(COMPUTER_IMAGES) - 1) assert sorted(image.dockerfile for image in bootstrap.PUBLISHED_IMAGES) == sorted(shipped) assert len({image.name for image in bootstrap.PUBLISHED_IMAGES}) == len(bootstrap.PUBLISHED_IMAGES) @@ -163,3 +172,25 @@ def test_a_shipped_agents_sources_whose_image_isnt_published_find_none(monkeypat assert published_bundle_image(sources) is None + + +def test_a_disk_forms_tag_is_of_its_boot_recipe_its_bases_sources_and_the_disk_tools_and_it_is_found_by_its_bases( + tmp_path, monkeypatch): + (disk,), (base,) = DISK_IMAGES, COMPUTER_IMAGES + recipe = BuildContext.of(disk.dockerfile.parent, disk.dockerfile).source_digest(None).removeprefix("sha256:") + served = {} + monkeypatch.setattr(published_images, "_registry_digest", served.get) + other = tmp_path / "base" + shutil.copytree(base.dockerfile.parent, other) + (other / "start-desktop").write_text("#!/bin/sh\n") + moved = PublishedImage(disk.name, disk.dockerfile, (("ROOTFS", PublishedImage(base.name, other / "Dockerfile")), + ("TOOLS", DISK_TOOLS_IMAGE))) + sources = BuildContext.of(base.dockerfile.parent, base.dockerfile).source_digest(None) + + assert disk.name == "agentenv-ubuntu-24.04-disk" and dict(disk.args) == {"ROOTFS": base, "TOOLS": DISK_TOOLS_IMAGE} + assert disk.tag() == "src-" + hashlib.sha256("\0".join( + [recipe, f"ROOTFS={base.tag()}", f"TOOLS={DISK_TOOLS_IMAGE.tag()}"]).encode()).hexdigest() + assert moved.tag() != disk.tag() + assert published_disk(sources) is None + served[disk.ref()] = _DIGEST + assert published_disk(sources) == f"{disk.ref()}@{_DIGEST}" and published_disk("sha256:" + "0" * 64) is None diff --git a/tst/unit/test_infra_build_script.py b/tst/unit/test_infra_build_script.py index dfb2c44c6..ebb183949 100644 --- a/tst/unit/test_infra_build_script.py +++ b/tst/unit/test_infra_build_script.py @@ -12,7 +12,7 @@ import pytest from agent_env.env import bootstrap -from agent_env.env.published_images import REGISTRY, write_release_digests +from agent_env.env.published_images import DISK_IMAGES, REGISTRY, write_release_digests from tst.util.scripts import SCRIPTS, load_script REPO = SCRIPTS.parents[1] @@ -46,7 +46,8 @@ def test_it_imports_only_the_standard_library(): def test_its_images_are_the_ones_agent_env_ships(script): assert script.REGISTRY == REGISTRY assert script.IMAGES == {image.name: str(image.dockerfile.relative_to(REPO)) for image in bootstrap.PUBLISHED_IMAGES} - assert set(script._SMOKES) | {"agentenv-service-db"} == set(script.IMAGES) + assert set(script._SMOKES) | {"agentenv-service-db", *script.DISK_ARGS} == set(script.IMAGES) + assert script.DISK_ARGS == {image.name: {arg: built.name for arg, built in image.args} for image in DISK_IMAGES} def test_check_accepts_every_image_the_plan_lists(script, monkeypatch, capsys): @@ -62,8 +63,21 @@ def _gateway(**changes) -> str: return json.dumps({key: value for key, value in {**entry, **changes}.items() if value is not None}) +def _disk(**args) -> str: + image = "agentenv-ubuntu-24.04-disk" + return json.dumps({"name": image, "repository": f"{REGISTRY}/{image}", "tag": _TAG, + "dockerfile": "src/agent_env/computers/disk/linux/boot/Dockerfile", + "context": "src/agent_env/computers/disk/linux/boot", "args": { + "ROOTFS": f"{REGISTRY}/agentenv-ubuntu-24.04:{_TAG}", "TOOLS": f"{REGISTRY}/agentenv-disk-tools:{_TAG}", + **args}}) + + @pytest.mark.parametrize("entry, problem", [ (_gateway(name="agentenv-evil"), "'agentenv-evil' isn't an image agent-env ships"), + (_gateway(args={"ROOTFS": "evil"}), "agentenv-gateway: unexpected args"), + (_disk(ROOTFS="ghcr.io/someone/ubuntu:latest"), "agentenv-ubuntu-24.04-disk: ROOTFS is 'ghcr.io/someone/ubuntu:latest'"), + (_disk(TOOLS=f"{REGISTRY}/agentenv-disk-tools:latest"), "agentenv-ubuntu-24.04-disk: TOOLS is 'ghcr.io/scaleapi/"), + (_disk(EXTRA="x"), "agentenv-ubuntu-24.04-disk: args is "), (_gateway(repository="ghcr.io/someone/agentenv-gateway"), "agentenv-gateway: repository is 'ghcr.io/someone/"), (_gateway(dockerfile="tst/data/agentenv_mcp/Dockerfile"), "agentenv-gateway: dockerfile is 'tst/data/agentenv_mcp"), (_gateway(context="."), "agentenv-gateway: context is '.'"), @@ -197,6 +211,31 @@ def test_the_service_db_smoke_waits_for_its_health_check_and_removes_the_contain assert calls[0] == ["docker", "run", "-d", "pg"] and calls[-1] == ["docker", "rm", "-f", "c123"] +@pytest.mark.parametrize("kvm", [True, False], ids=["kvm", "tcg"]) +def test_a_disk_forms_smoke_boots_it_in_the_launcher_until_its_desktop_runs(script, monkeypatch, capsys, kvm): + calls, logs = [], iter(["agentenv-init: starting", "success: xvfb entered RUNNING\nsuccess: dbus entered RUNNING\n" + "success: desktop entered RUNNING"]) + + def run(command, **kw): + calls.append((command, kw.get("input"))) + answers = {"inspect": "arm64\n", "run": "c1\n", "logs": next(logs) if command[1] == "logs" else ""} + return subprocess.CompletedProcess(command, 0, answers.get(command[1], answers.get(command[2], "")), "") + + monkeypatch.setattr(script.subprocess, "run", run) + monkeypatch.setattr(script.time, "sleep", lambda seconds: None) + monkeypatch.setattr(script.Path, "exists", lambda self: kvm) + + assert _run(script, monkeypatch, "smoke", "agentenv-ubuntu-24.04-disk", "disk:1-arm64") == 0 + + commands = [command for command, _ in calls] + assert calls[1] == (["docker", "build", "-q", "-t", "disk:1-arm64-machine", "-"], + "FROM agentenv-qemu-launcher\nCOPY --from=disk:1-arm64 /disk/ /disk/\n") + assert commands[2] == ["docker", "run", "-d", *(["--device", "/dev/kvm"] if kvm else []), "-e", "VM_ARCH=arm64", + "disk:1-arm64-machine"] + assert commands[-1] == ["docker", "rm", "-f", "c1"] and commands.count(["docker", "logs", "c1"]) == 2 + assert f"booted to its desktop {'with KVM' if kvm else 'under TCG'}" in capsys.readouterr().out + + def test_a_service_db_that_never_turns_healthy_fails_with_its_logs(script, monkeypatch, capsys): calls = [] monkeypatch.setattr(script.subprocess, "run", _docker({}, calls, [])) diff --git a/tst/unit/test_infra_images_script.py b/tst/unit/test_infra_images_script.py index 5e9921857..d186195a5 100644 --- a/tst/unit/test_infra_images_script.py +++ b/tst/unit/test_infra_images_script.py @@ -50,19 +50,27 @@ def test_plan_lists_every_image_with_its_checkout_paths_and_tag(script, monkeypa "context": "src/agent_env/env/gateway"} -def test_plan_only_lists_the_infra_images_or_the_agents_images(script, monkeypatch, capsys): - """The release builds the infra images before it publishes the protocol, and the images that install it, the - bundles' and the Playwright MCP browser's, after.""" +def test_plan_only_lists_the_infra_images_the_agents_images_or_the_disk_forms(script, monkeypatch, capsys): + """The release builds the infra images before it publishes the protocol, the images that install it, the bundles' + and the Playwright MCP browser's, after, and the disk forms over their base computers' published images.""" planned = {} - for only in ("infra", "agents"): + for only in ("infra", "agents", "disks"): assert _run(script, monkeypatch, "plan", "--only", only) == 0 - planned[only] = [entry["name"] for entry in json.loads(capsys.readouterr().out)] + planned[only] = json.loads(capsys.readouterr().out) bundles = [image.name for image in BUNDLE_IMAGES] after = [*bundles, "agentenv-playwright-mcp"] # the browser's recorder installs the protocol too - assert planned == {"infra": [image.name for image in bootstrap.PUBLISHED_IMAGES if image.name not in after], - "agents": after} - assert "agentenv-gateway" in planned["infra"] and bundles == [ + disks = ["agentenv-ubuntu-24.04-disk"] + names = {only: [entry["name"] for entry in entries] for only, entries in planned.items()} + assert names == {"infra": [image.name for image in bootstrap.PUBLISHED_IMAGES if image.name not in after + disks], + "agents": after, "disks": disks} + assert "agentenv-disk-tools" in names["infra"] and "agentenv-qemu-launcher" in names["infra"] + (disk,) = planned["disks"] + assert disk["args"] == {"ROOTFS": next(image.ref() for image in BUNDLE_IMAGES if image.name == "agentenv-ubuntu-24.04"), + "TOOLS": next(image.ref() for image in bootstrap.PUBLISHED_IMAGES + if image.name == "agentenv-disk-tools")} + assert "args" not in planned["infra"][0] + assert "agentenv-gateway" in names["infra"] and bundles == [ "agentenv-claude-code", "agentenv-codex", "agentenv-gemini", "agentenv-hermes", "agentenv-letta-code", "agentenv-opencode", "agentenv-openhands", "agentenv-pi", "agentenv-proxy", "agentenv-pydantic-ai", "agentenv-mail-mcp", "agentenv-ubuntu-24.04"] diff --git a/tst/unit/test_release_infra_images.py b/tst/unit/test_release_infra_images.py index f143651ae..fa0e4faa7 100644 --- a/tst/unit/test_release_infra_images.py +++ b/tst/unit/test_release_infra_images.py @@ -85,7 +85,8 @@ def body(job: str) -> dict: assert yaml.safe_load(yaml.safe_dump(body(infra_job)).replace("missing_infra", "missing_agents")) == body(agent_job) -@pytest.mark.parametrize("job", ["infra-build", "infra-publish", "agent-build", "agent-publish"]) +@pytest.mark.parametrize("job", ["infra-build", "infra-publish", "agent-build", "agent-publish", "disk-build", + "disk-publish"]) def test_nothing_is_pushed_before_mains_ci_passes(job): """A condition with ``!cancelled()`` runs whatever the jobs it needs ended as, so it checks main-ci's itself.""" assert "main-ci" in JOBS[job]["needs"] @@ -113,7 +114,7 @@ def test_the_framework_waits_for_every_image_job(): assert _condition("framework").startswith("!cancelled()") assert ("(needs.infra-plan.result == 'skipped' || (needs.infra-plan.result == 'success' " "&& needs.infra-record.result == 'success'))") in _condition("framework") - for image_job in ("infra-build", "infra-publish", "agent-build", "agent-publish"): + for image_job in ("infra-build", "infra-publish", "agent-build", "agent-publish", "disk-build", "disk-publish"): assert image_job in JOBS["infra-record"]["needs"] and _finished(image_job) in _condition("infra-record") @@ -121,7 +122,7 @@ def test_every_job_that_reads_the_shipped_bundles_pins_the_protocol_first(): """The pin is a source of the shipped envs: a job without it would tag, build, record or ship other sources.""" readers = {job: "\n".join(_runs(job)) for job in JOBS if _READS_SHIPPED_SOURCES.search("\n".join(_runs(job)))} - assert set(readers) == {"infra-plan", "infra-build", "agent-build", "infra-record", "framework"} + assert set(readers) == {"infra-plan", "infra-build", "agent-build", "disk-build", "infra-record", "framework"} for job, runs in readers.items(): pins = [pin.start() for pin in _PIN.finditer(runs)] assert len(pins) == 1 and pins[0] < _READS_SHIPPED_SOURCES.search(runs).start(), job @@ -202,7 +203,8 @@ def test_what_a_registry_can_rate_limit_is_tried_again(): def test_every_infra_image_pins_its_bases_by_digest_and_dependabot_bumps_them(): """A release rebuilds an image only when its sources change, so a base behind a bare tag never picks up its - fixes; pinned, the pin is a source, and Dependabot bumping it is what rebuilds and republishes the image.""" + fixes; pinned, the pin is a source, and Dependabot bumping it is what rebuilds and republishes the image. A base a + build arg names is another published image, whose tag is a source of this one's.""" dependabot = yaml.safe_load((REPO / ".github" / "dependabot.yml").read_text()) (docker,) = [update for update in dependabot["updates"] if update["package-ecosystem"] == "docker"] @@ -210,7 +212,7 @@ def test_every_infra_image_pins_its_bases_by_digest_and_dependabot_bumps_them(): text = image.dockerfile.read_text() stages = set(re.findall(r"^FROM\s+\S+\s+AS\s+(\S+)", text, re.IGNORECASE | re.MULTILINE)) bases = [ref for ref in re.findall(r"^FROM\s+(?:--\S+\s+)*(\S+)", text, re.IGNORECASE | re.MULTILINE) - if ref not in stages] + if ref not in stages and ref != "scratch" and not ref.startswith("${")] assert bases and all(re.search(r"@sha256:[0-9a-f]{64}$", ref) for ref in bases), (image.name, bases) assert f"/{image.dockerfile.parent.relative_to(REPO)}" in docker["directories"], image.name @@ -223,3 +225,18 @@ def test_dependabot_proposes_a_bases_fixes_and_never_an_upgrade(): assert {"dependency-name": "*", "update-types": ["version-update:semver-major", "version-update:semver-minor"]} in ( docker["ignore"]) + + +def test_a_disk_form_is_built_over_its_bases_published_image_with_the_published_tools_and_booted_before_its_push(): + """Its base's image, an agent image, and the disk tools, an infra image, are published first; its build is checked + and pushed as the infra images' are.""" + build = "\n".join(_runs("disk-build")) + publish = "\n".join(_runs("disk-publish")) + + assert {"infra-publish", "agent-publish"} <= set(JOBS["disk-build"]["needs"]) + assert all(_finished(job) in _condition("disk-build") for job in ("infra-publish", "agent-publish")) + assert 'infra_build.py check "$ENTRY"' in build and build.index("infra_build.py check") < build.index("docker build") + assert '--build-arg "ROOTFS=$ROOTFS" --build-arg "TOOLS=$TOOLS"' in build + assert build.index('infra_build.py smoke "$NAME" "$BUILT"') < build.index('docker push "$BUILT"') + assert 'infra_build.py check "$entry"' in publish and publish.index("check") < publish.index("imagetools create") + assert JOBS["disk-build"]["env"]["ROOTFS"] == "${{ matrix.image.args.ROOTFS }}-${{ matrix.arch }}" diff --git a/tst/util/capabilities.py b/tst/util/capabilities.py index b75bff118..aca93d524 100644 --- a/tst/util/capabilities.py +++ b/tst/util/capabilities.py @@ -61,6 +61,10 @@ #: so collecting off a local sandbox needs GNU stat first on PATH. GNU_STAT = "gnu_stat" +#: Booting a computer's disk form on local Docker needs KVM: /dev/kvm on this host, which Docker passes to the launcher. +#: Docker Desktop on a Mac has none, nor has a host without nested virtualization. +KVM = "kvm" + def missing_capability_reason(capability: str) -> str: return f"{MISSING_CAPABILITY_PREFIX}{capability}" @@ -194,3 +198,15 @@ def skip_without_gnu_stat() -> pytest.MarkDecorator: not gnu_stat_is_available(), reason=missing_capability_reason(GNU_STAT), ) + + +def kvm_is_available() -> bool: + """Whether the local sandbox provider boots disks here: this host's /dev/kvm, readable and writable.""" + from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandboxProvider + + return bool(LocalSandboxProvider.disk_image_platforms()) + + +def skip_without_kvm() -> pytest.MarkDecorator: + """Collection-time ``skipif`` for tests that boot a computer's disk form on local Docker.""" + return pytest.mark.skipif(not kvm_is_available(), reason=missing_capability_reason(KVM)) diff --git a/tst/util/oci_registry.py b/tst/util/oci_registry.py index 8dc684d26..cea05dec3 100644 --- a/tst/util/oci_registry.py +++ b/tst/util/oci_registry.py @@ -22,7 +22,7 @@ def __init__(self): self.manifests, self.blobs, self.tags, self.fetched = {}, {}, {}, [] def serve(self, monkeypatch) -> "Registry": - for name in ("pin_digest", "read_manifest", "read_blob"): + for name in ("pin_digest", "read_manifest", "read_blob", "read_blob_start"): monkeypatch.setattr(image_contents, name, getattr(self, name)) return self @@ -63,6 +63,10 @@ def read_blob(self, ref, digest, auth, path): self.fetched.append(digest) path.write_bytes(self.blobs[digest]) + def read_blob_start(self, ref, digest, auth, size): + self.fetched.append(digest) + return self.blobs[digest][:size] + def layer(files: dict[str, bytes | str]) -> bytes: """A tar.gz layer of ``files`` by path; a str is a link to that path.""" From 23b772362ceab6a17fc7961f9225360f79781df5 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 16:23:01 -0800 Subject: [PATCH 02/21] fix(computer): clean-install compares a disk form's tag with the checkout's, its base's and tools' included Co-Authored-By: Claude Opus 5.5 --- .github/scripts/check_clean_install.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/scripts/check_clean_install.py b/.github/scripts/check_clean_install.py index 2be81aee9..82b99f24d 100644 --- a/.github/scripts/check_clean_install.py +++ b/.github/scripts/check_clean_install.py @@ -85,10 +85,8 @@ def problems(name: str, runs: int, before: dict[str, list[str]]) -> list[str]: found.append(f"agent-env found the config file {config}; the gate runs with none") for plugin in plugins(): found.append(f"{plugin} is installed; the gate tests agent-env without plugins") - installed_src = Path(agent_env.__file__).parent.parent for image in PUBLISHED_IMAGES: - checkout = PublishedImage(image.name, CHECKOUT_SRC / image.dockerfile.relative_to(installed_src)) - if (tag := image.tag()) != (expected := checkout.tag()): + if (tag := image.tag()) != (expected := _checkout(image).tag()): found.append(f"{image.name}: the installed sources tag it {tag}, the checkout's {expected}; an install " "wouldn't find the image published for its sources") bundle = find_bundle(name) @@ -121,6 +119,13 @@ def problems(name: str, runs: int, before: dict[str, list[str]]) -> list[str]: return found +def _checkout(image: PublishedImage) -> PublishedImage: + """``image`` as the checkout's sources make it, with the images it's built from.""" + installed_src = Path(agent_env.__file__).parent.parent + return PublishedImage(image.name, CHECKOUT_SRC / image.dockerfile.relative_to(installed_src), + tuple((arg, _checkout(built)) for arg, built in image.args)) + + def plugins() -> list[str]: """Entry points in an ``agent_env.*`` group from any distribution other than agent-env's own.""" return sorted(f"{dist.name}'s {ep.group} entry point {ep.name}" From a3e58e5632ccbf030f737816d7a643b61bc9b70a Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 16:36:27 -0800 Subject: [PATCH 03/21] fix(computer): tell whether Docker runs a platform by running it, and resolve with no bind mount Under colima with Rosetta, buildx inspect lists only linux/arm64 and linux/386, yet linux/amd64 containers run, so the composer refused amd64; it now runs a container of the base (or the image the shipped bases build from) for the platform, once per process. colima shares only $HOME and the composer's work folder is in $TMPDIR, so the resolver's bind mounts failed; its inputs and outputs now go through docker cp. Co-Authored-By: Claude Opus 5.5 --- .../env/envs/computer/utils/builder.py | 70 ++++++++++++++----- .../env/envs/computer/utils/composer.py | 4 +- tst/integration/computer/disk_test.py | 12 +++- tst/unit/env/envs/conftest.py | 2 +- tst/unit/env/envs/test_computer_builder.py | 67 ++++++++++++++++-- 5 files changed, 125 insertions(+), 30 deletions(-) diff --git a/src/agent_env/env/envs/computer/utils/builder.py b/src/agent_env/env/envs/computer/utils/builder.py index 7371f6c68..0bb01914e 100644 --- a/src/agent_env/env/envs/computer/utils/builder.py +++ b/src/agent_env/env/envs/computer/utils/builder.py @@ -7,6 +7,7 @@ import asyncio import contextlib +import functools import hashlib import io import json @@ -52,6 +53,10 @@ #: since the stores keep no file modes. RUN_ONLY = (b"#!", b"\x7fELF") EXECUTABLE = 0o755 +#: What shows whether this machine's Docker runs a platform, for a base that's only a build context: the image the shipped +#: Linux bases build from, which their build pulls for the platform anyway. +PLATFORM_CHECK = ("public.ecr.aws/docker/library/ubuntu:24.04" + "@sha256:24737cef17f4e22b1977df7d9a05e77c4e28b7da5b8911f8d8ccda136bf3573c") #: What registers QEMU in a builder VM's kernel, for a platform no link runs natively: Docker Hub's image through #: Google's mirror of it, which doesn't limit anonymous pulls by IP as Docker Hub does. BINFMT = ("mirror.gcr.io/tonistiigi/binfmt:qemu-v9.2.2" @@ -165,9 +170,9 @@ def __init__(self, provider: SandboxProvider, store_id: str, repository: str, em self.provider, self.emulate = provider, emulate self._refused = [(link, _refused(link, store_id, repository)) for link in provider.links] - def link(self, platform: str) -> SandboxProvider: - """The link that builds ``platform``. Raises Unbuildable, or Skipped when only emulation would build it and the - put turned emulation off.""" + def link(self, platform: str, base: DockerImageArtifact) -> SandboxProvider: + """The link that builds ``platform`` ``FROM`` ``base``. Raises Unbuildable, or Skipped when only emulation would + build it and the put turned emulation off.""" links = [link for link, problem in self._refused if problem is None] if native := [link for link in links if platform in link.native_platforms()]: return native[0] @@ -178,7 +183,7 @@ def link(self, platform: str) -> SandboxProvider: raise Skipped("emulation is off, and " + "; ".join( f"{type(link).__name__} doesn't run {platform} natively" for link, _ in self._refused)) for link in links: - if not _docker_here(link) or platform in _emulated_platforms(): + if not _docker_here(link) or _runs_here(platform, _platform_check(base)): return link unemulated = f"this machine can't build {platform}: its Docker has no emulation for it" raise Unbuildable("; ".join(problem or unemulated for _, problem in self._refused)) @@ -236,14 +241,24 @@ def _docker_here(link: SandboxProvider) -> bool: return link.ON_THIS_MACHINE and link.REACHES_HOST_AT is None -def _emulated_platforms() -> set[str]: - """The platforms this machine's Docker builds, its own and those it emulates.""" - inspected = subprocess.run(["docker", "buildx", "inspect", "--bootstrap"], capture_output=True, text=True) - platforms = set() - for line in inspected.stdout.splitlines(): - if line.strip().startswith("Platforms:"): - platforms |= {item.strip().rstrip("*") for item in line.split(":", 1)[1].split(",")} - return platforms +@functools.cache +def _runs_here(platform: str, image: str) -> bool: + """Whether this machine's Docker runs ``platform``'s programs, natively or emulated, as a container of ``image`` for + it shows: what its builder lists can leave out a platform it runs, as colima's does Rosetta's linux/amd64.""" + try: + return subprocess.run(["docker", "run", "--rm", "--platform", platform, "--entrypoint", "true", image], + capture_output=True).returncode == 0 + except FileNotFoundError: + return False + + +def _platform_check(base: DockerImageArtifact) -> str: + """The image ``_runs_here`` starts for a build ``FROM`` ``base``, one the build pulls anyway: ``base`` when it's a + registry's, signed in to it, else ``PLATFORM_CHECK``.""" + if not names_registry(base.image_name): + return PLATFORM_CHECK + docker_login(get_config().get_image_store().auth(base.pinned_name)) + return base.pinned_name def _resolver(source: str, arch: str, context: str, out: str) -> list[str]: @@ -262,17 +277,28 @@ def __init__(self, name: str, work: Path, base: DockerImageArtifact, platform: s build_for(base, platform) def resolve(self, request: dict, source: str, arch: str, timeout: int) -> dict: + """With the build context and what the resolver reads copied into its container, and what it wrote copied out, + rather than bind-mounted: a Docker in a VM, as on a Mac, mounts only the folders the VM shares, colima's only the + home, and keeps no owner on them.""" out = self.work / RESOLVED / "out" out.mkdir(parents=True) (out / "request.json").write_text(json.dumps({**request, "owner": [os.getuid(), os.getgid()]})) shutil.copy(RESOLVER, out / "resolve.py") + container = _docker("create", "--platform", f"linux/{arch}", "--user", "root", "--entrypoint", "python3", source, + "/out/resolve.py", "/out/request.json", "/out").strip() try: - ran = subprocess.run(_resolver(source, arch, str(self.work / CONTEXT), str(out)), capture_output=True, - text=True, timeout=timeout) - except subprocess.TimeoutExpired as e: - raise ResolveError(f"resolving took over {timeout} s", str(e.stdout or "")) from None - if ran.returncode != 0: - raise ResolveError("resolving the packages failed", ran.stdout + ran.stderr) + _docker("cp", f"{self.work / CONTEXT}/.", f"{container}:/ctx") + _docker("cp", f"{out}/.", f"{container}:/out") + try: + ran = subprocess.run(["docker", "start", "--attach", container], capture_output=True, text=True, + timeout=timeout) + except subprocess.TimeoutExpired as e: + raise ResolveError(f"resolving took over {timeout} s", str(e.stdout or "")) from None + if ran.returncode != 0: + raise ResolveError("resolving the packages failed", ran.stdout + ran.stderr) + _docker("cp", f"{container}:/out/.", str(out)) + finally: + subprocess.run(["docker", "rm", "--force", container], capture_output=True) return json.loads((out / "result.json").read_text()) def download(self, url: str, sha256: str, path: str) -> None: @@ -564,6 +590,14 @@ def _packages(listed: bytes) -> dict[str, str]: return {package["name"]: package["version"] for package in json.loads(listed)} +def _docker(*args: str) -> str: + """What ``docker`` printed for ``args``; a failure raises, with what it said.""" + ran = subprocess.run(["docker", *args], capture_output=True, text=True) + if ran.returncode != 0: + raise RuntimeError(f"docker {args[0]} failed: {ran.stderr.strip()}") + return ran.stdout + + def save_image(tag: str, path: Path) -> None: subprocess.run(["docker", "save", "-o", str(path), tag], check=True, capture_output=True, text=True) diff --git a/src/agent_env/env/envs/computer/utils/composer.py b/src/agent_env/env/envs/computer/utils/composer.py index 4fbf4cec6..7e129e3f8 100644 --- a/src/agent_env/env/envs/computer/utils/composer.py +++ b/src/agent_env/env/envs/computer/utils/composer.py @@ -261,7 +261,7 @@ def _built(env: ComputerEnv, base: ComputerArtifact, driver: ComputerDriverEnv, def on() -> builder.Builder: if not held: - link = builds.link(platform) + link = builds.link(platform, base.image()) native = platform in link.native_platforms() progress.at(pair, "starting its builder", f"building natively {builder.where(link)}" if native else f"building emulated, which is slow, {builder.where(link)}; " @@ -351,7 +351,7 @@ def _built_disk(env: ComputerEnv, base: ComputerArtifact, arch: str, container: if (pinned := _served(ref, store)) is not None: progress.at(pair, "reusing", "reusing the disk of the same key, made earlier") return {"status": "unverified", "key": key, "image": pinned, "reused": True, "built_at": _now()} - link = builds.link(platform) + link = builds.link(platform, base.image()) progress.at(pair, "starting its builder", f"making the disk {builder.where(link)}") on = opened.enter_context(builds.open(link, platform, Path(work), base.image(), {})) on.login(store.auth(container["image"])) diff --git a/tst/integration/computer/disk_test.py b/tst/integration/computer/disk_test.py index ee660277f..6ea3b65f9 100644 --- a/tst/integration/computer/disk_test.py +++ b/tst/integration/computer/disk_test.py @@ -39,7 +39,7 @@ debugfs -R "cat /etc/agentenv/vm/image.env" root.img > image.env 2>/dev/null debugfs -R "stat /bin/busybox" root.img 2>/dev/null | head -1 > busybox sha256sum esp.img root.img > sums -rm disk.raw esp.img root.img +rm disk.img disk.raw esp.img root.img """ @@ -114,8 +114,14 @@ def _read(tools: str, image: str, folder: Path, arch: str) -> dict: with tarfile.open(fileobj=io.BytesIO(copied.stdout)) as tar: (member,) = tar.getmembers() (folder / "disk.img").write_bytes(tar.extractfile(member).read()) - subprocess.run(["docker", "run", "--rm", "--platform", host_platform(), "-v", f"{folder}:/work", "--entrypoint", "sh", - tools, "-c", _READ, "read", UKI[arch]], check=True, capture_output=True) + reader = subprocess.run(["docker", "create", "--platform", host_platform(), "--entrypoint", "sh", tools, "-c", _READ, + "read", UKI[arch]], capture_output=True, text=True, check=True).stdout.strip() + try: + for command in (["cp", f"{folder}/.", f"{reader}:/work"], ["start", "--attach", reader], + ["cp", f"{reader}:/work/.", str(folder)]): + subprocess.run(["docker", *command], check=True, capture_output=True) + finally: + subprocess.run(["docker", "rm", "--force", reader], capture_output=True) start = [(key, base64.b64decode(value).decode()) for line in (folder / "image.env").read_text().splitlines() for key, _, value in [line.partition("=")]] return {"file": (member.name, member.uid, member.gid), "start": start, "uki": (folder / "uki").read_text(), diff --git a/tst/unit/env/envs/conftest.py b/tst/unit/env/envs/conftest.py index b9d3f1b3c..d8de90453 100644 --- a/tst/unit/env/envs/conftest.py +++ b/tst/unit/env/envs/conftest.py @@ -31,7 +31,7 @@ def docker(local_stores, cli_routing, monkeypatch): monkeypatch.setattr(builder, "host_platform", lambda: "linux/arm64") monkeypatch.setattr(local_sandbox, "host_platform", lambda: "linux/arm64") monkeypatch.setattr(composer, "_probe", fake.probe) - monkeypatch.setattr(builder, "_emulated_platforms", lambda: {"linux/arm64", *fake.emulates}) + monkeypatch.setattr(builder, "_runs_here", lambda platform, image: platform in {"linux/arm64", *fake.emulates}) monkeypatch.setattr(builder.LocalBuilder, "installed", lambda self, tag, prefix: {"mss": "10.1.0"}) monkeypatch.setattr(builder.LocalBuilder, "image_config", lambda on, tag: fake.image_config(tag)) monkeypatch.setattr(composer, "_protocol_source", lambda: None) diff --git a/tst/unit/env/envs/test_computer_builder.py b/tst/unit/env/envs/test_computer_builder.py index 146480fa3..1b104bea6 100644 --- a/tst/unit/env/envs/test_computer_builder.py +++ b/tst/unit/env/envs/test_computer_builder.py @@ -7,16 +7,19 @@ import io import json import logging +import os import re import subprocess import sys import tarfile import time +from pathlib import Path from types import SimpleNamespace import pytest from click.testing import CliRunner +import agent_env from agent_env.artifact.artifacts.docker_image import DockerImageArtifact from agent_env.artifact.store import get_artifact_store from agent_env.cli import cli @@ -83,12 +86,64 @@ def test_a_base_built_from_its_context_is_built_here_for_each_arch_and_keyed_by_ "amd64": "unverified", "arm64": "ready"} -def test_the_platforms_this_machine_builds_are_read_from_its_builder(monkeypatch): - inspected = ("Name: desktop-linux\nDriver: docker\nNodes:\nName: desktop-linux\n" - "Platforms: linux/arm64, linux/amd64, linux/amd64/v2, linux/riscv64*\nLabels:\n") - monkeypatch.setattr(subprocess, "run", lambda *a, **k: SimpleNamespace(stdout=inspected, returncode=0)) +def test_whether_this_machine_runs_a_platform_is_whether_a_container_of_it_runs_once_per_platform(monkeypatch): + """Not what its builder lists: colima's lists linux/arm64 and linux/386 alone, and runs Rosetta's linux/amd64.""" + ran = [] + monkeypatch.setattr(subprocess, "run", lambda command, **k: ran.append(command) or SimpleNamespace( + returncode=0 if "linux/amd64" in command else 125)) + builder._runs_here.cache_clear() - assert builder._emulated_platforms() == {"linux/arm64", "linux/amd64", "linux/amd64/v2", "linux/riscv64"} + answers = [builder._runs_here(platform, "img") for platform in ("linux/amd64", "linux/riscv64", "linux/amd64")] + builder._runs_here.cache_clear() + + assert answers == [True, False, True] + assert ran == [["docker", "run", "--rm", "--platform", platform, "--entrypoint", "true", "img"] + for platform in ("linux/amd64", "linux/riscv64")] + + +def test_the_platform_check_starts_the_base_or_the_image_the_shipped_bases_build_from(monkeypatch): + shipped = (Path(agent_env.__file__).parent / "computers" / "artifacts" / "ubuntu-24.04" / "Dockerfile").read_text() + logins = [] + monkeypatch.setattr(builder, "docker_login", logins.append) + registry = SimpleNamespace(image_name="ghcr.io/acme/base:1", pinned_name="ghcr.io/acme/base:1@sha256:" + "a" * 64) + context = SimpleNamespace(image_name="local/base-1234:abcd", pinned_name="local/base-1234:abcd") + + assert builder._platform_check(registry) == registry.pinned_name and len(logins) == 1 + assert builder._platform_check(context) == builder.PLATFORM_CHECK and f"FROM {builder.PLATFORM_CHECK}\n" in shipped + + +def test_a_resolve_copies_its_inputs_into_the_resolvers_container_and_its_outputs_out_with_no_bind_mount( + tmp_path, monkeypatch): + work = tmp_path / "work" + (work / builder.CONTEXT / "driver").mkdir(parents=True) + (work / builder.CONTEXT / "driver" / "requirements.txt").write_text("mss\n") + ran = [] + + def docker(command, **kwargs): + ran.append(command) + if command[1] == "create": + return subprocess.CompletedProcess(command, 0, "c1\n", "") + if command[1] == "cp" and command[2] == "c1:/out/.": + (Path(command[3]) / "result.json").write_text(json.dumps({"apt": [], "wheels": {"driver": []}})) + return subprocess.CompletedProcess(command, 0, "", "") + + monkeypatch.setattr(builder.subprocess, "run", docker) + monkeypatch.setattr(builder, "docker_login", lambda auth: None) + local = builder.LocalBuilder("LocalSandboxProvider", work, SimpleNamespace( + image_name="ghcr.io/acme/base:1", pinned_name="ghcr.io/acme/base:1"), "linux/arm64") + + result = local.resolve({"apt": [], "python": []}, "base@sha256:" + "b" * 64, "amd64", 60) + + out = str(work / builder.RESOLVED / "out") + assert result == {"apt": [], "wheels": {"driver": []}} + assert ran == [ + ["docker", "create", "--platform", "linux/amd64", "--user", "root", "--entrypoint", "python3", + "base@sha256:" + "b" * 64, "/out/resolve.py", "/out/request.json", "/out"], + ["docker", "cp", f"{work / builder.CONTEXT}/.", "c1:/ctx"], ["docker", "cp", f"{out}/.", "c1:/out"], + ["docker", "start", "--attach", "c1"], ["docker", "cp", "c1:/out/.", out], ["docker", "rm", "--force", "c1"]] + assert not [arg for command in ran for arg in command if arg in ("-v", "--volume", "--mount")] + assert json.loads((work / builder.RESOLVED / "out" / "request.json").read_text())["owner"] == [os.getuid(), + os.getgid()] def test_the_packages_a_build_installed_are_read_from_its_image_without_running_it(monkeypatch): @@ -185,7 +240,7 @@ def helper(*args): def test_a_put_on_a_vm_provider_builds_each_arch_in_a_vm_of_its_own_and_its_images_are_reused_by_key_here( base, driver, docker, signing, tmp_path, monkeypatch): vms = FakeVmProvider(docker, tmp_path / "vms") - for name in ("build_image", "push_local_image", "docker_login", "_emulated_platforms"): + for name in ("build_image", "push_local_image", "docker_login", "_runs_here"): monkeypatch.setattr(builder, name, lambda *args, **kwargs: pytest.fail("this machine's Docker was used")) told = [] From 9798675ef20feac09c42254a4e38f076f3a313ee Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 16:38:27 -0800 Subject: [PATCH 04/21] fix(computer): a disk's size in its key, sign-in for the base disk, a cancelled boot torn down, no enlarging, trailing newlines kept Co-Authored-By: Claude Opus 5.5 --- src/agent_env/computers/disk/linux/boot/init | 13 +++-- .../env/envs/computer/utils/composer.py | 8 ++-- .../env_providers/env_computer_provider.py | 6 ++- .../local_docker/provider.py | 14 ++++-- tst/unit/env/envs/test_computer_disk.py | 27 +++++++++++ tst/unit/env/envs/test_disk_recipe.py | 47 +++++++++++++++++++ 6 files changed, 102 insertions(+), 13 deletions(-) diff --git a/src/agent_env/computers/disk/linux/boot/init b/src/agent_env/computers/disk/linux/boot/init index 7b8fc8c50..ba43f8f97 100755 --- a/src/agent_env/computers/disk/linux/boot/init +++ b/src/agent_env/computers/disk/linux/boot/init @@ -9,6 +9,8 @@ mount -t sysfs sysfs /sys mount -t devtmpfs devtmpfs /dev log() { echo "agentenv-init: $*" > /dev/console; } fail() { log "$*"; sleep 2; poweroff -f; } +# decode VALUE: VALUE's base64 decoded into $decoded, its trailing newlines kept, which $(...) alone would drop. +decode() { decoded=$(echo "$1" | base64 -d; echo .); decoded=${decoded%.}; } while read -r module; do insmod "$module" 2>/dev/null; done < /modules.order root="" for _ in $(seq 200); do root=$(findfs LABEL=agentenv-root 2>/dev/null) && break; sleep 0.05; done @@ -28,17 +30,18 @@ set -- while read -r line; do key=${line%%=*} value=${line#*=} case "$key" in - env) export "$(echo "$value" | base64 -d)" ;; - argv) set -- "$@" "$(echo "$value" | base64 -d)" ;; - user) user=$(echo "$value" | base64 -d) ;; - workdir) workdir=$(echo "$value" | base64 -d) ;; + env) decode "$value"; export "$decoded" ;; + argv) decode "$value"; set -- "$@" "$decoded" ;; + user) decode "$value"; user=$decoded ;; + workdir) decode "$value"; workdir=$decoded ;; esac done < "$config" if [ -f /seed/user-data ]; then while read -r line; do key=${line%%=*} value=${line#*=} case "$key" in ''|\#*) continue ;; esac - export "$key=$(echo "$value" | base64 -d)" + decode "$value" + export "$key=$decoded" done < /seed/user-data fi hostname=$( [ -f /seed/meta-data ] && sed -n 's/^local-hostname: *//p' /seed/meta-data) diff --git a/src/agent_env/env/envs/computer/utils/composer.py b/src/agent_env/env/envs/computer/utils/composer.py index 7e129e3f8..9f4d27e80 100644 --- a/src/agent_env/env/envs/computer/utils/composer.py +++ b/src/agent_env/env/envs/computer/utils/composer.py @@ -355,6 +355,8 @@ def _built_disk(env: ComputerEnv, base: ComputerArtifact, arch: str, container: progress.at(pair, "starting its builder", f"making the disk {builder.where(link)}") on = opened.enter_context(builds.open(link, platform, Path(work), base.image(), {})) on.login(store.auth(container["image"])) + if base_disk is not None: + on.login(get_config().get_image_store().auth(base_disk)) if (tools := published_pinned(DISK_TOOLS_IMAGE)) is None: tools = f"agentenv-disk-tools:{DISK_TOOLS_IMAGE.tag()}" on.build((context / "tools" / "Dockerfile").read_text(), tools, on.platform, {}, @@ -392,8 +394,8 @@ def _built_disk(env: ComputerEnv, base: ComputerArtifact, arch: str, container: def _base_disk(base: ComputerArtifact, arch: str) -> tuple[str | None, str]: """The base's disk for ``arch``, pinned to its digest, and its part of a disk's key: the base's disk form's image, - or the one published for the base's container sources; else None, and the base's container, which a disk is made - from with the boot recipe, the key's recipe.""" + or the one published for the base's container sources; else None, and the base's container and disk size, which a + disk is made from with the boot recipe, the key's recipe.""" if (image := base.disk_image()) is not None: store = get_config().get_image_store() pinned = image.pinned_name if "@" in image.pinned_name else pin_digest(image.pinned_name, @@ -401,7 +403,7 @@ def _base_disk(base: ComputerArtifact, arch: str) -> tuple[str | None, str]: return pinned, pinned if (published := published_disk(base.image().source_digest or "")) is not None: return published, published - return None, f"made from {_base(base, arch)[1]}" + return None, f"made from {_base(base, arch)[1]} at {base.forms.disk.min_disk_gb} GB" def _config(on: builder.Builder, image: str, arch: str) -> dict: diff --git a/src/agent_env/providers/env_providers/env_computer_provider.py b/src/agent_env/providers/env_providers/env_computer_provider.py index fd6e5baa8..f304cbad7 100644 --- a/src/agent_env/providers/env_providers/env_computer_provider.py +++ b/src/agent_env/providers/env_providers/env_computer_provider.py @@ -93,11 +93,15 @@ async def _boot(self, env: ComputerEnv, builds: list[tuple[str, str, dict]], par raise ValueError(f"{type(sandbox_provider).__name__} runs none of {env.id}'s ready builds natively " f"({', '.join(f'{form}/{arch}' for form, arch, _ in builds)})") form, arch, build = native[0] + computer = env.computer_artifact() + if form == "disk" and disk_size_gb > computer.forms.disk.min_disk_gb: + raise ValueError(f"{env.id}'s disk is {computer.forms.disk.min_disk_gb} GB, which a deploy doesn't enlarge, " + f"and it asks for {disk_size_gb:g} GB") if build["status"] == "unverified": build = await probe_first_boot(env, form, arch, sandbox_provider) if build["status"] != "ready": raise RuntimeError(f"{env.id}'s {form}/{arch} failed its probe on its first boot: {build['reason']}") - computer, name, token = env.computer_artifact(), env.environment_name, control_token() + name, token = env.environment_name, control_token() values = _values(computer, values) policy = NetworkPolicy.from_dict(env.network) if env.network else None booted = await boot( diff --git a/src/agent_env/providers/sandbox_providers/local_docker/provider.py b/src/agent_env/providers/sandbox_providers/local_docker/provider.py index dee41ac9a..84c954c0d 100644 --- a/src/agent_env/providers/sandbox_providers/local_docker/provider.py +++ b/src/agent_env/providers/sandbox_providers/local_docker/provider.py @@ -685,7 +685,7 @@ async def create_vm( env: Optional[dict[str, str]] = None, ) -> LocalSandbox: """This machine as a VM, or with ``image``, a computer's disk form booted under KVM in the QEMU launcher's - container, reached on each of ``exposed_ports``.""" + container, reached on each of ``exposed_ports``, at the disk's own size: ``disk_size_gb`` is this machine's.""" refuse_unenforceable_policy(self, network_policy) if network_policy is not None and network_policy.restricts_egress: raise NetworkPolicyUnsupportedError(f"{type(self).__name__} takes a container's network away, not a VM's") @@ -723,9 +723,15 @@ async def _boot_disk(self, sandbox: LocalSandbox, *, image: str, env: dict[str, ports = list(sandbox._port_map) publish = [arg for port in ports for spec in port_bindings(sandbox.host_ips, sandbox.host_port(port), port) for arg in ("-p", spec)] - await asyncio.to_thread(_docker, "run", "-d", "--name", sandbox.container_name, "--label", - f"{SANDBOX_LABEL}={sandbox.sandbox_id}", *publish, - *launcher_args(ports, env, size["cpu"], size["memory"]), machine) + running = asyncio.ensure_future(asyncio.to_thread( + _docker, "run", "-d", "--name", sandbox.container_name, "--label", f"{SANDBOX_LABEL}={sandbox.sandbox_id}", + *publish, *launcher_args(ports, env, size["cpu"], size["memory"]), machine)) + try: + await asyncio.shield(running) + except asyncio.CancelledError: # the container docker is still starting must exist before teardown removes it + with contextlib.suppress(Exception): + await running + raise async def create_container( self, diff --git a/tst/unit/env/envs/test_computer_disk.py b/tst/unit/env/envs/test_computer_disk.py index 5d250c02e..e0bf72adf 100644 --- a/tst/unit/env/envs/test_computer_disk.py +++ b/tst/unit/env/envs/test_computer_disk.py @@ -20,6 +20,7 @@ from agent_env.env.published_images import DISK from agent_env.providers.env_providers import env_computer_provider from agent_env.providers.sandbox_providers.local_docker import provider as local_sandbox +from agent_env.store.image_store.local_registry_image_store import LocalRegistryImageStore from agent_env.task_step.context import TaskStepContext from agent_env.task_step.task_steps.reset_env import ResetEnvTaskStep from tst.unit.env.envs._computer import DISK_FORM, _base, _put, disk_layer @@ -76,6 +77,18 @@ def test_a_disk_is_reused_by_its_key_of_its_containers_key_the_base_disk_and_the assert len(docker.disk_builds) > made +def test_a_bases_disk_size_is_in_the_key_and_the_build_of_the_disk_made_for_it(disk_base, driver, docker): + bigger = _base(disk_base.image(), id="@local/~/kit/bigger", forms={ + **disk_base.forms.model_dump(exclude_none=True), "disk": {**DISK_FORM, "min_disk_gb": 32}}) + + small = Env.get(_put(disk_base, driver).id).metadata["composed"]["disk"]["arm64"] + large = Env.get(_put(bigger, driver, id="@local/~/kit/other", environment_name="other").id) + + assert large.metadata["composed"]["disk"]["arm64"]["key"] != small["key"] + assert sorted(build["args"]["SIZE_GB"] for build in docker.disk_builds if "base-disk-arm64" in build["tag"]) == [ + "16", "32"] + + def test_a_base_with_a_disk_image_or_a_published_one_keeps_its_partition_and_makes_none(disk_base, driver, docker, monkeypatch): registry = docker.registry @@ -86,6 +99,8 @@ def test_a_base_with_a_disk_image_or_a_published_one_keeps_its_partition_and_mak **disk_base.forms.model_dump(exclude_none=True), "disk": {**DISK_FORM, "image": {"id": image.id, "version": image.version}}}) monkeypatch.setattr(composer, "published_disk", lambda sources: "ghcr.io/scaleapi/base-disk:src-1@sha256:" + "a" * 64) + asked = [] + monkeypatch.setattr(LocalRegistryImageStore, "auth", lambda self, ref: asked.append(ref)) own = Env.get(_put(with_disk, driver).id).metadata["composed"]["disk"] published = Env.get(_put(disk_base, driver, id="@local/~/kit/other", environment_name="other").id) @@ -95,6 +110,7 @@ def test_a_base_with_a_disk_image_or_a_published_one_keeps_its_partition_and_mak assert not [build for build in docker.disk_builds if "base-disk" in build["tag"]] assert {arch: r["status"] for arch, r in own.items()} == {"amd64": "unverified", "arm64": "unverified"} assert own["arm64"]["key"] != published.metadata["composed"]["disk"]["arm64"]["key"] + assert {image.image_name, "ghcr.io/scaleapi/base-disk:src-1@sha256:" + "a" * 64} <= set(asked) def test_a_disk_isnt_made_for_a_container_that_didnt_build_and_is_made_by_the_next_build(disk_base, driver, docker): @@ -164,6 +180,17 @@ async def test_a_deploy_of_the_disk_form_boots_it_in_a_vm_with_its_settings_and_ assert deployed.environment_url == "http://127.0.0.1:20001" +@pytest.mark.asyncio +async def test_a_deploy_of_a_disk_bigger_than_the_disk_is_refused_rather_than_booted_smaller(disk_base, driver, docker, + vms): + env = _put(disk_base, driver) + + with pytest.raises(Exception, match="disk is 16 GB, which a deploy doesn't enlarge, and it asks for 64 GB"): + await Env.get(env.id).deploy(disk_size_gb=64, env_params={"form": "disk"}) + + assert vms.created == [] + + @pytest.mark.asyncio async def test_an_unverified_disk_is_probed_on_its_first_boot_with_kvm_before_its_deploy_boots_it(disk_base, driver, docker, vms, diff --git a/tst/unit/env/envs/test_disk_recipe.py b/tst/unit/env/envs/test_disk_recipe.py index d1f7b7d42..ca63685e7 100644 --- a/tst/unit/env/envs/test_disk_recipe.py +++ b/tst/unit/env/envs/test_disk_recipe.py @@ -2,10 +2,13 @@ the deploy's environment (the launcher's NoCloud seed), each a line per value base64'd so any bytes survive; and the local provider booting a disk in the launcher, with KVM and the VM's ports, settings and size.""" +import asyncio import base64 import json import os +import re import subprocess +import threading from unittest.mock import AsyncMock import pytest @@ -55,6 +58,26 @@ def test_the_seed_holds_each_vm_env_variable_by_its_name_and_nothing_else(tmp_pa assert meta_data.endswith("\nlocal-hostname: computer\n") and meta_data.startswith("instance-id: ") +def test_the_inits_decoder_keeps_every_byte_of_a_start_and_a_seed_trailing_newlines_included(tmp_path): + """As a container runtime would: $(...) alone drops a value's trailing newlines.""" + decoder = re.search(r"^decode\(\) .*$", (DISK / "boot" / "init").read_text(), re.MULTILINE).group(0) + values = ["plain", "ends with a newline\n", "\n", f"{TRICKY}\n\n"] + config = tmp_path / "config.json" + config.write_text(json.dumps({"Cmd": values, "User": "user\n"})) + start = subprocess.run(["python3", str(DISK / "tools" / "image-env"), str(config)], capture_output=True, text=True, + check=True).stdout + subprocess.run(["sh", str(DISK / "launcher" / "seed"), str(tmp_path)], check=True, env={ + "PATH": os.environ["PATH"], **{f"VM_ENV_V{i}": value for i, value in enumerate(values)}}) + seed = (tmp_path / "user-data").read_text() + + def decoded(text: str) -> list[str]: + script = f"{decoder}\n" + "".join(f"decode {line.partition('=')[2]}; printf '%s\\0' \"$decoded\"\n" + for line in text.splitlines() if line and not line.startswith("#")) + return subprocess.run(["sh", "-c", script], capture_output=True, text=True, check=True).stdout.split("\0")[:-1] + + assert decoded(start) == [*values, "user\n", "/"] and decoded(seed) == values + + def test_the_launcher_gets_kvm_the_vms_ports_and_size_and_its_environment_under_its_prefix(): args = local_disk.launcher_args([18765, 5900], {"ENVIRONMENT_NAME": "desk", "A": "b c"}, 1.5, 4096) @@ -96,6 +119,30 @@ async def test_a_disk_boots_in_the_launcher_with_kvm_its_ports_published_and_a_r assert sandbox.tunnel_urls == {18765: "http://127.0.0.1:20001"} and rebuilt.owns_container +@pytest.mark.asyncio +async def test_a_boot_cancelled_while_docker_starts_the_launcher_waits_for_it_then_tears_it_down(launched, monkeypatch): + started, release, order = threading.Event(), threading.Event(), [] + + def docker(*args): + started.set() + release.wait(5) + order.append("run") + return "" + + monkeypatch.setattr(local_sandbox, "_docker", docker) + monkeypatch.setattr(LocalSandbox, "terminate", AsyncMock(side_effect=lambda: order.append("terminate"))) + booting = asyncio.ensure_future(LocalSandboxProvider().create_vm(image="disk", exposed_ports=[18765], env={})) + await asyncio.to_thread(started.wait, 5) + + booting.cancel() + await asyncio.sleep(0.1) + release.set() + + with pytest.raises(asyncio.CancelledError): + await booting + assert order == ["run", "terminate"] + + @pytest.mark.asyncio async def test_a_disk_isnt_booted_where_docker_cant_pass_kvm(launched, monkeypatch): monkeypatch.setattr(local_sandbox, "kvm_platforms", lambda: frozenset()) From b122bde70a5d9a5316674cb892109bd167e557c8 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 16:38:51 -0800 Subject: [PATCH 05/21] ci: open /dev/kvm on the slow shards directly, so a runner without it says so Co-Authored-By: Claude Opus 5.5 --- .github/workflows/local-backends.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/local-backends.yml b/.github/workflows/local-backends.yml index 84c77616f..1ae3b2285 100644 --- a/.github/workflows/local-backends.yml +++ b/.github/workflows/local-backends.yml @@ -140,10 +140,7 @@ jobs: timeout 30 sh -c 'until curl -sf http://localhost:5000/v2/ >/dev/null; do sleep 1; done' # x64 hosted runners have /dev/kvm; the disk form's tests boot under it, and skip without it, which fails below. - name: Let the local provider boot disks with KVM - run: | - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm.rules - sudo udevadm control --reload-rules && sudo udevadm trigger --name-match=kvm - test -w /dev/kvm + run: sudo chmod 666 /dev/kvm - name: Slow integration tier on the local defaults env: AGENTENV_TEST_SHARD: ${{ matrix.shard }}/8 From 4f4b4eef079a98d7a0bb1a9b0e44b2467211bb65 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 16:44:19 -0800 Subject: [PATCH 06/21] test(computer): the fake registry's disks are the size their build made them; the platform check signs in Co-Authored-By: Claude Opus 5.5 --- tst/unit/env/envs/_computer.py | 6 ++++-- tst/unit/env/envs/test_computer_builder.py | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tst/unit/env/envs/_computer.py b/tst/unit/env/envs/_computer.py index 9f404ae05..4eb98155b 100644 --- a/tst/unit/env/envs/_computer.py +++ b/tst/unit/env/envs/_computer.py @@ -56,7 +56,7 @@ class FakeDocker: def __init__(self, registry: Registry): self.registry, self.builds, self.fail, self.emulates, self.labels = registry, [], {}, {"linux/amd64"}, {} self.probed, self.probe_fails, self.resolved, self.timeouts = [], {}, [], {} - self.writes, self.saved, self.disk_builds = {}, [], [] + self.writes, self.saved, self.disk_builds, self.disk_sizes = {}, [], [], {} def resolve(self, request, context, source, arch, work): """The resolver in the base: each apt package as one .deb, each venv's requirements as one wheel.""" @@ -102,6 +102,8 @@ def build_image(self, dockerfile, context, tag, *, platform, labels=None, build_ if build_args is not None or context.name == "tools": self.disk_builds.append({"platform": platform, "tag": tag, "dockerfile": dockerfile.read_text(), "args": build_args, "context": context.name, "labels": labels}) + args = build_args or {} + self.disk_sizes[tag] = int(args["SIZE_GB"]) if "SIZE_GB" in args else self.disk_sizes.get(args.get("ESP"), 16) return self.builds.append({"platform": platform, "tag": tag, "dockerfile": dockerfile.read_text(), "files": sorted(p.relative_to(context).as_posix() for p in context.rglob("*") if p.is_file()), @@ -129,7 +131,7 @@ def save(self, tag, path): saved_image(path, layers) def push(self, local, ref, store): - layer = disk_layer() if "-disk-" in local else UBUNTU_LAYER + layer = disk_layer(self.disk_sizes.get(local, 16) << 30) if "-disk-" in local else UBUNTU_LAYER self.registry.tags[ref] = self.registry.image(layer, arch=local.rsplit("-", 1)[1], labels=self.labels.get(local)) def image_config(self, tag): diff --git a/tst/unit/env/envs/test_computer_builder.py b/tst/unit/env/envs/test_computer_builder.py index 1b104bea6..18c52673c 100644 --- a/tst/unit/env/envs/test_computer_builder.py +++ b/tst/unit/env/envs/test_computer_builder.py @@ -201,12 +201,13 @@ def test_what_each_run_wrote_is_read_from_the_newest_layer_a_shell_ran_its_comma def test_a_registry_base_is_signed_in_to_before_its_image_is_built_on(base, driver, docker, monkeypatch): + """By each builder, and before the check that this machine runs the arch it emulates starts the base.""" signed = [] monkeypatch.setattr(builder, "docker_login", signed.append) _put(base, driver) - assert len(signed) == 2 and len(docker.builds) == 2 + assert len(signed) == 3 and len(docker.builds) == 2 def test_the_builder_vms_helper_lists_plain_files_marks_runnables_links_checks_and_downloads(tmp_path): From 3a21e9c3482f21df34561d5eaec3b975e8816bf0 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 17:06:54 -0800 Subject: [PATCH 07/21] fix(computer): a disk's size from the base's disk form, which a computer missing one arch's disk still has Co-Authored-By: Claude Opus 5.5 --- .../env_providers/env_computer_provider.py | 9 ++++----- tst/unit/env/envs/test_computer_disk.py | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/agent_env/providers/env_providers/env_computer_provider.py b/src/agent_env/providers/env_providers/env_computer_provider.py index f304cbad7..75592f464 100644 --- a/src/agent_env/providers/env_providers/env_computer_provider.py +++ b/src/agent_env/providers/env_providers/env_computer_provider.py @@ -93,15 +93,14 @@ async def _boot(self, env: ComputerEnv, builds: list[tuple[str, str, dict]], par raise ValueError(f"{type(sandbox_provider).__name__} runs none of {env.id}'s ready builds natively " f"({', '.join(f'{form}/{arch}' for form, arch, _ in builds)})") form, arch, build = native[0] - computer = env.computer_artifact() - if form == "disk" and disk_size_gb > computer.forms.disk.min_disk_gb: - raise ValueError(f"{env.id}'s disk is {computer.forms.disk.min_disk_gb} GB, which a deploy doesn't enlarge, " - f"and it asks for {disk_size_gb:g} GB") + if form == "disk" and disk_size_gb > (size := env.computer.forms.disk.min_disk_gb): + raise ValueError(f"{env.id}'s disk is {size} GB, which a deploy doesn't enlarge, and it asks for " + f"{disk_size_gb:g} GB") if build["status"] == "unverified": build = await probe_first_boot(env, form, arch, sandbox_provider) if build["status"] != "ready": raise RuntimeError(f"{env.id}'s {form}/{arch} failed its probe on its first boot: {build['reason']}") - name, token = env.environment_name, control_token() + computer, name, token = env.computer_artifact(), env.environment_name, control_token() values = _values(computer, values) policy = NetworkPolicy.from_dict(env.network) if env.network else None booted = await boot( diff --git a/tst/unit/env/envs/test_computer_disk.py b/tst/unit/env/envs/test_computer_disk.py index e0bf72adf..1bdf17e4e 100644 --- a/tst/unit/env/envs/test_computer_disk.py +++ b/tst/unit/env/envs/test_computer_disk.py @@ -180,6 +180,20 @@ async def test_a_deploy_of_the_disk_form_boots_it_in_a_vm_with_its_settings_and_ assert deployed.environment_url == "http://127.0.0.1:20001" +@pytest.mark.asyncio +async def test_an_arch_whose_disk_built_deploys_it_though_the_computer_has_no_disk_form_without_the_other( + disk_base, driver, docker, vms, monkeypatch): + built = composer._built_disk + monkeypatch.setattr(composer, "_built_disk", lambda env, base, arch, *rest: ( + {"status": "failed", "stage": "build", "reason": "boom"} if arch == "amd64" else built(env, base, arch, *rest))) + env = Env.get(_put(disk_base, driver).id) + + deployed = await env.deploy(env_params={"form": "disk"}) + + assert env.computer_artifact().forms.disk is None and env.metadata["composed"]["disk"]["arm64"]["status"] == "ready" + assert deployed.metadata["computer"]["form"] == "disk" and len(vms.created) == 1 + + @pytest.mark.asyncio async def test_a_deploy_of_a_disk_bigger_than_the_disk_is_refused_rather_than_booted_smaller(disk_base, driver, docker, vms): From 28fc53fb7055c2c5c5ec08beed1dd5b95c42f8d2 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 17:13:11 -0800 Subject: [PATCH 08/21] fix(release): the disk smoke copies the disk from its own platform, so another arch's disk boots under TCG Co-Authored-By: Claude Opus 5.5 --- .github/scripts/infra_build.py | 3 ++- tst/unit/test_infra_build_script.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/scripts/infra_build.py b/.github/scripts/infra_build.py index 155117641..64025fcc5 100644 --- a/.github/scripts/infra_build.py +++ b/.github/scripts/infra_build.py @@ -294,7 +294,8 @@ def _boots(image: str) -> int: text=True, check=True).stdout.strip() machine = f"{image}-machine" subprocess.run(["docker", "build", "-q", "-t", machine, "-"], text=True, check=True, capture_output=True, - input=f"FROM agentenv-qemu-launcher\nCOPY --from={image} /disk/ /disk/\n") + input=f"FROM --platform=linux/{arch} {image} AS disk\nFROM agentenv-qemu-launcher\n" + "COPY --from=disk /disk/ /disk/\n") kvm = ["--device", "/dev/kvm"] if Path("/dev/kvm").exists() else [] container = subprocess.run(["docker", "run", "-d", *kvm, "-e", f"VM_ARCH={arch}", machine], capture_output=True, text=True, check=True).stdout.strip() diff --git a/tst/unit/test_infra_build_script.py b/tst/unit/test_infra_build_script.py index ebb183949..1a8683e38 100644 --- a/tst/unit/test_infra_build_script.py +++ b/tst/unit/test_infra_build_script.py @@ -229,7 +229,8 @@ def run(command, **kw): commands = [command for command, _ in calls] assert calls[1] == (["docker", "build", "-q", "-t", "disk:1-arm64-machine", "-"], - "FROM agentenv-qemu-launcher\nCOPY --from=disk:1-arm64 /disk/ /disk/\n") + "FROM --platform=linux/arm64 disk:1-arm64 AS disk\nFROM agentenv-qemu-launcher\n" + "COPY --from=disk /disk/ /disk/\n") assert commands[2] == ["docker", "run", "-d", *(["--device", "/dev/kvm"] if kvm else []), "-e", "VM_ARCH=arm64", "disk:1-arm64-machine"] assert commands[-1] == ["docker", "rm", "-f", "c1"] and commands.count(["docker", "logs", "c1"]) == 2 From 5a462a5a1542cf9def3ef50721a84b1f9188e81a Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 17:14:51 -0800 Subject: [PATCH 09/21] fix(release): the disk smoke says KVM only for its own arch's disk Co-Authored-By: Claude Opus 5.5 --- .github/scripts/infra_build.py | 4 +++- tst/unit/test_infra_build_script.py | 9 ++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/scripts/infra_build.py b/.github/scripts/infra_build.py index 64025fcc5..2429b4d47 100644 --- a/.github/scripts/infra_build.py +++ b/.github/scripts/infra_build.py @@ -19,6 +19,7 @@ import argparse import base64 import json +import platform import posixpath import re import subprocess @@ -305,7 +306,8 @@ def _boots(image: str) -> int: logs = subprocess.run(["docker", "logs", container], capture_output=True, text=True, errors="replace") logs = logs.stdout + logs.stderr if len(set(_BOOTED.findall(logs))) == 3: - print(f"{image} booted to its desktop{' with KVM' if kvm else ' under TCG'}") + native = {"x86_64": "amd64", "aarch64": "arm64"}.get(platform.machine()) == arch + print(f"{image} booted to its desktop {'with KVM' if kvm and native else 'under TCG'}") return 0 time.sleep(_HEALTH_POLL_SECONDS) tail = "\n".join(logs.splitlines()[-_LOG_TAIL_LINES:]) diff --git a/tst/unit/test_infra_build_script.py b/tst/unit/test_infra_build_script.py index 1a8683e38..ec896ff41 100644 --- a/tst/unit/test_infra_build_script.py +++ b/tst/unit/test_infra_build_script.py @@ -211,8 +211,10 @@ def test_the_service_db_smoke_waits_for_its_health_check_and_removes_the_contain assert calls[0] == ["docker", "run", "-d", "pg"] and calls[-1] == ["docker", "rm", "-f", "c123"] -@pytest.mark.parametrize("kvm", [True, False], ids=["kvm", "tcg"]) -def test_a_disk_forms_smoke_boots_it_in_the_launcher_until_its_desktop_runs(script, monkeypatch, capsys, kvm): +@pytest.mark.parametrize("kvm, machine, how", [(True, "aarch64", "with KVM"), (False, "aarch64", "under TCG"), + (True, "x86_64", "under TCG")], ids=["kvm", "no-kvm", "another-arch"]) +def test_a_disk_forms_smoke_boots_it_in_the_launcher_until_its_desktop_runs(script, monkeypatch, capsys, kvm, machine, + how): calls, logs = [], iter(["agentenv-init: starting", "success: xvfb entered RUNNING\nsuccess: dbus entered RUNNING\n" "success: desktop entered RUNNING"]) @@ -224,6 +226,7 @@ def run(command, **kw): monkeypatch.setattr(script.subprocess, "run", run) monkeypatch.setattr(script.time, "sleep", lambda seconds: None) monkeypatch.setattr(script.Path, "exists", lambda self: kvm) + monkeypatch.setattr(script.platform, "machine", lambda: machine) assert _run(script, monkeypatch, "smoke", "agentenv-ubuntu-24.04-disk", "disk:1-arm64") == 0 @@ -234,7 +237,7 @@ def run(command, **kw): assert commands[2] == ["docker", "run", "-d", *(["--device", "/dev/kvm"] if kvm else []), "-e", "VM_ARCH=arm64", "disk:1-arm64-machine"] assert commands[-1] == ["docker", "rm", "-f", "c1"] and commands.count(["docker", "logs", "c1"]) == 2 - assert f"booted to its desktop {'with KVM' if kvm else 'under TCG'}" in capsys.readouterr().out + assert f"booted to its desktop {how}" in capsys.readouterr().out def test_a_service_db_that_never_turns_healthy_fails_with_its_logs(script, monkeypatch, capsys): From 67e9f60e91322d9fb6dc846f5db8372c9b43db7e Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 21:04:19 -0800 Subject: [PATCH 10/21] fix(computer): first deploys of one unverified disk at once share one probe A file lock per env, form, arch and image (holding_locks, so across processes too) lets the first deploy probe and record; the others wait, read the recorded result, and boot a ready disk or fail with the recorded reason without probing again. Co-Authored-By: Claude Opus 5.5 --- .../env/envs/computer/utils/composer.py | 33 ++++++++++++++--- tst/unit/env/envs/test_computer_disk.py | 37 +++++++++++++++++++ 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/src/agent_env/env/envs/computer/utils/composer.py b/src/agent_env/env/envs/computer/utils/composer.py index 9e944551f..cc8cb7352 100644 --- a/src/agent_env/env/envs/computer/utils/composer.py +++ b/src/agent_env/env/envs/computer/utils/composer.py @@ -27,7 +27,7 @@ from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path, PurePosixPath -from collections.abc import Callable +from collections.abc import AsyncIterator, Callable from typing import TYPE_CHECKING, Any from agentenv_protocol.computer import apps as installed_apps @@ -47,6 +47,7 @@ from agent_env.providers.sandbox_providers.host_images import platform_tag from agent_env.store.base import ConcurrentModificationError from agent_env.store.ids import derive_id +from agent_env.store.local_state import holding_locks from agent_env.store.image_store.image_contents import image_config, labels from agent_env.store.image_store.oci_registry_credentials import names_registry from agent_env.store.image_store.registry_api import NoSuchImageError, pin_digest, write_index @@ -179,10 +180,32 @@ def _composed_disk(env: ComputerEnv, base: ComputerArtifact, driver: ComputerDri async def probe_first_boot(env: ComputerEnv, form: str, arch: str, sandbox_provider: SandboxProvider) -> dict: """Gate 3 for a build its compose couldn't probe, on its first boot on ``sandbox_provider``, which boots it - natively: the result, recorded.""" - result = await _checked([sandbox_provider], env, env.driver, form, arch, env.metadata["composed"][form][arch]) - await asyncio.to_thread(_Record(env).finished, form, arch, result) - return result + natively: the result, recorded. Deploys of one build at once, in this process or another, probe it once: each + waits for the one probing, then takes the result it recorded.""" + image = env.metadata["composed"][form][arch]["image"] + async with _holding(f"computer-first-boot:{env.stored_id}:{form}:{arch}:{image}"): + recorded = (await asyncio.to_thread(Env.get, env.stored_id, env.version)).metadata["composed"][form][arch] + if recorded["image"] == image and recorded["status"] != "unverified": + return recorded + result = await _checked([sandbox_provider], env, env.driver, form, arch, recorded) + await asyncio.to_thread(_Record(env).finished, form, arch, result) + return result + + +@contextlib.asynccontextmanager +async def _holding(lock: str) -> AsyncIterator[None]: + """``holding_locks`` for ``lock``, taken off the event loop; a cancelled wait releases the lock once it's taken.""" + held = contextlib.ExitStack() + taking = asyncio.ensure_future(asyncio.to_thread(held.enter_context, holding_locks([lock]))) + try: + await asyncio.shield(taking) + except asyncio.CancelledError: + taking.add_done_callback(lambda _: held.close()) + raise + try: + yield + finally: + held.close() def _probed(env: ComputerEnv, driver: ComputerDriverEnv, form: str, arch: str, result: dict, diff --git a/tst/unit/env/envs/test_computer_disk.py b/tst/unit/env/envs/test_computer_disk.py index 1bdf17e4e..439551d3d 100644 --- a/tst/unit/env/envs/test_computer_disk.py +++ b/tst/unit/env/envs/test_computer_disk.py @@ -4,6 +4,7 @@ probe; and written as the computer's disk form. A deploy that names the disk form boots it in a VM, its settings in its seed, and a reset boots it afresh. Docker and the registry are faked.""" +import asyncio import base64 import json from types import SimpleNamespace @@ -194,6 +195,42 @@ async def test_an_arch_whose_disk_built_deploys_it_though_the_computer_has_no_di assert deployed.metadata["computer"]["form"] == "disk" and len(vms.created) == 1 +@pytest.mark.asyncio +@pytest.mark.parametrize("fails", [False, True], ids=["ready", "failed"]) +async def test_first_deploys_of_an_unverified_disk_at_once_share_one_probe_and_its_recorded_result( + disk_base, driver, docker, vms, monkeypatch, fails): + monkeypatch.setattr(local_sandbox, "kvm_platforms", lambda: frozenset()) + env = _put(disk_base, driver) + monkeypatch.setattr(local_sandbox, "kvm_platforms", lambda: frozenset({"linux/arm64"})) + probe, first_boot, waited = docker.probe, env_computer_provider.probe_first_boot, [] + + async def slow(*args, **kwargs): + await asyncio.sleep(0.3) + return await probe(*args, **kwargs) + + async def counted(*args): + waited.append(args[1:3]) + return await first_boot(*args) + + monkeypatch.setattr(composer, "_probe", slow) + monkeypatch.setattr(env_computer_provider, "probe_first_boot", counted) + if fails: + docker.probe_fails["arm64"] = "the click never landed" + + deployed = await asyncio.gather(*(Env.get(env.id).deploy(env_params={"form": "disk"}) for _ in range(3)), + return_exceptions=True) + + assert waited == [("disk", "arm64")] * 3 and len([probed for probed in docker.probed if probed[0] == "disk"]) == 1 + recorded = Env.get(env.id).metadata["composed"]["disk"]["arm64"] + if fails: + assert recorded["status"] == "failed" and vms.created == [] + assert {str(failure) for failure in deployed} == { + f"{env.id}'s disk/arm64 failed its probe on its first boot: {recorded['reason']}"} + else: + assert recorded["status"] == "ready" and len(vms.created) == 3 + assert all(record.metadata["computer"]["form"] == "disk" for record in deployed) + + @pytest.mark.asyncio async def test_a_deploy_of_a_disk_bigger_than_the_disk_is_refused_rather_than_booted_smaller(disk_base, driver, docker, vms): From b4c993912e5b73b56326c9f3f764fa5eaf382c4d Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 21:08:59 -0800 Subject: [PATCH 11/21] fix(computer): one deadline for a whole resolve, and the launcher's variables base64'd on the way in The resolver's container is created, filled, run and emptied by one deadline, so a stalled pull raises ResolveError at the requested timeout; the container, named up front, is removed whatever happened. The local provider hands the launcher each VM_ENV_ value base64'd, so seed copies them as they are: a value of several lines reaches the computer whole, and a line in one that looks like another variable makes none. Co-Authored-By: Claude Opus 5.5 --- .../computers/disk/linux/launcher/launch | 2 +- .../computers/disk/linux/launcher/seed | 8 +-- .../env/envs/computer/utils/builder.py | 28 ++++---- .../sandbox_providers/local_docker/disk.py | 9 ++- tst/unit/env/envs/test_computer_builder.py | 64 ++++++++++++++----- tst/unit/env/envs/test_disk_recipe.py | 31 ++++++--- 6 files changed, 93 insertions(+), 49 deletions(-) diff --git a/src/agent_env/computers/disk/linux/launcher/launch b/src/agent_env/computers/disk/linux/launcher/launch index 5a88b3a88..e28505419 100755 --- a/src/agent_env/computers/disk/linux/launcher/launch +++ b/src/agent_env/computers/disk/linux/launcher/launch @@ -2,7 +2,7 @@ # launch: boot /disk/disk.img, a Linux computer's disk form, under QEMU until it powers off, on a copy-on-write overlay # so the disk stays as it is: with KVM when /dev/kvm is here and the disk is this machine's arch, else with TCG, from # UEFI firmware. The guest's console is this container's output. -# VM_ENV_ the computer's environment, which goes in its NoCloud seed (CIDATA; see seed) as NAME +# VM_ENV_ the computer's variable NAME, its value base64'd, for its NoCloud seed (CIDATA; see seed) # VM_PORTS guest TCP ports, each forwarded from the same port here # VM_ARCH the disk's arch, amd64 or arm64; this machine's by default # VM_CPUS, VM_MEMORY_MB its size: 2 and 4096 by default diff --git a/src/agent_env/computers/disk/linux/launcher/seed b/src/agent_env/computers/disk/linux/launcher/seed index 98b8e4de8..2de80326b 100755 --- a/src/agent_env/computers/disk/linux/launcher/seed +++ b/src/agent_env/computers/disk/linux/launcher/seed @@ -1,12 +1,10 @@ #!/bin/sh # seed DIR: the NoCloud seed's files for the computer launch boots, in DIR: user-data holding each VM_ENV_ variable -# as NAME and its value base64'd, a line each, as the disk's init reads them, and meta-data naming the instance. +# as NAME and its value, base64'd as the variable already holds it, a line each, as the disk's init reads them, and +# meta-data naming the instance. A base64 value has no newline, so no line env prints can pass for another variable. set -eu { echo "#agentenv-environment" - for name in $(env | sed -n 's/^VM_ENV_\([A-Za-z_][A-Za-z0-9_]*\)=.*/\1/p' | sort -u); do - eval "value=\${VM_ENV_$name}" - printf '%s=%s\n' "$name" "$(printf %s "$value" | base64 | tr -d '\n')" - done + env | sed -n 's/^VM_ENV_\([A-Za-z_][A-Za-z0-9_]*\)=\([A-Za-z0-9+/=]*\)$/\1=\2/p' | sort } > "$1/user-data" printf 'instance-id: %s\nlocal-hostname: computer\n' "$(od -An -N8 -tx1 /dev/urandom | tr -d ' \n')" > "$1/meta-data" diff --git a/src/agent_env/env/envs/computer/utils/builder.py b/src/agent_env/env/envs/computer/utils/builder.py index 0bb01914e..10507233e 100644 --- a/src/agent_env/env/envs/computer/utils/builder.py +++ b/src/agent_env/env/envs/computer/utils/builder.py @@ -17,6 +17,7 @@ import shutil import subprocess import tarfile +import time import uuid from abc import ABC, abstractmethod from collections.abc import Iterator @@ -284,19 +285,19 @@ def resolve(self, request: dict, source: str, arch: str, timeout: int) -> dict: out.mkdir(parents=True) (out / "request.json").write_text(json.dumps({**request, "owner": [os.getuid(), os.getgid()]})) shutil.copy(RESOLVER, out / "resolve.py") - container = _docker("create", "--platform", f"linux/{arch}", "--user", "root", "--entrypoint", "python3", source, - "/out/resolve.py", "/out/request.json", "/out").strip() + container, deadline = f"agentenv-resolve-{uuid.uuid4().hex[:12]}", time.monotonic() + timeout try: - _docker("cp", f"{self.work / CONTEXT}/.", f"{container}:/ctx") - _docker("cp", f"{out}/.", f"{container}:/out") - try: - ran = subprocess.run(["docker", "start", "--attach", container], capture_output=True, text=True, - timeout=timeout) - except subprocess.TimeoutExpired as e: - raise ResolveError(f"resolving took over {timeout} s", str(e.stdout or "")) from None + _docker("create", "--name", container, "--platform", f"linux/{arch}", "--user", "root", "--entrypoint", + "python3", source, "/out/resolve.py", "/out/request.json", "/out", deadline=deadline) + _docker("cp", f"{self.work / CONTEXT}/.", f"{container}:/ctx", deadline=deadline) + _docker("cp", f"{out}/.", f"{container}:/out", deadline=deadline) + ran = subprocess.run(["docker", "start", "--attach", container], capture_output=True, text=True, + timeout=max(deadline - time.monotonic(), 0)) if ran.returncode != 0: raise ResolveError("resolving the packages failed", ran.stdout + ran.stderr) - _docker("cp", f"{container}:/out/.", str(out)) + _docker("cp", f"{container}:/out/.", str(out), deadline=deadline) + except subprocess.TimeoutExpired as e: + raise ResolveError(f"resolving took over {timeout} s", str(e.stdout or "")) from None finally: subprocess.run(["docker", "rm", "--force", container], capture_output=True) return json.loads((out / "result.json").read_text()) @@ -590,9 +591,10 @@ def _packages(listed: bytes) -> dict[str, str]: return {package["name"]: package["version"] for package in json.loads(listed)} -def _docker(*args: str) -> str: - """What ``docker`` printed for ``args``; a failure raises, with what it said.""" - ran = subprocess.run(["docker", *args], capture_output=True, text=True) +def _docker(*args: str, deadline: float) -> str: + """What ``docker`` printed for ``args``, by the ``time.monotonic()`` ``deadline``; a failure raises, with what it + said, and running past the deadline raises TimeoutExpired.""" + ran = subprocess.run(["docker", *args], capture_output=True, text=True, timeout=max(deadline - time.monotonic(), 0)) if ran.returncode != 0: raise RuntimeError(f"docker {args[0]} failed: {ran.stderr.strip()}") return ran.stdout diff --git a/src/agent_env/providers/sandbox_providers/local_docker/disk.py b/src/agent_env/providers/sandbox_providers/local_docker/disk.py index 101a14f15..d016dede2 100644 --- a/src/agent_env/providers/sandbox_providers/local_docker/disk.py +++ b/src/agent_env/providers/sandbox_providers/local_docker/disk.py @@ -3,6 +3,7 @@ run with /dev/kvm.""" import asyncio +import base64 import hashlib import math import os @@ -15,7 +16,7 @@ from agent_env.utils.docker.build import build_image KVM = "/dev/kvm" -#: How the launcher takes the computer's environment, each variable under this prefix (see its launch script). +#: How the launcher takes the computer's environment, each variable under this prefix, base64'd (see its launch script). ENV_PREFIX = "VM_ENV_" @@ -34,9 +35,11 @@ async def machine_image(disk: str) -> str: def launcher_args(ports: list[int], env: dict[str, str], cpu: float, memory: int) -> list[str]: - """The ``docker run`` arguments that size the launcher's VM, forward ``ports`` and hand it ``env``.""" + """The ``docker run`` arguments that size the launcher's VM, forward ``ports`` and hand it ``env``, each value + base64'd, so one of several lines reaches the computer whole.""" variables = {"VM_PORTS": " ".join(map(str, ports)), "VM_CPUS": str(max(1, math.ceil(cpu))), - "VM_MEMORY_MB": str(memory), **{f"{ENV_PREFIX}{name}": value for name, value in env.items()}} + "VM_MEMORY_MB": str(memory), + **{f"{ENV_PREFIX}{name}": base64.b64encode(value.encode()).decode() for name, value in env.items()}} return ["--device", KVM, *(arg for name, value in variables.items() for arg in ("-e", f"{name}={value}"))] diff --git a/tst/unit/env/envs/test_computer_builder.py b/tst/unit/env/envs/test_computer_builder.py index 18c52673c..a3874146a 100644 --- a/tst/unit/env/envs/test_computer_builder.py +++ b/tst/unit/env/envs/test_computer_builder.py @@ -114,38 +114,68 @@ def test_the_platform_check_starts_the_base_or_the_image_the_shipped_bases_build def test_a_resolve_copies_its_inputs_into_the_resolvers_container_and_its_outputs_out_with_no_bind_mount( tmp_path, monkeypatch): - work = tmp_path / "work" - (work / builder.CONTEXT / "driver").mkdir(parents=True) - (work / builder.CONTEXT / "driver" / "requirements.txt").write_text("mss\n") - ran = [] + ran, timeouts = [], [] def docker(command, **kwargs): ran.append(command) - if command[1] == "create": - return subprocess.CompletedProcess(command, 0, "c1\n", "") - if command[1] == "cp" and command[2] == "c1:/out/.": + timeouts.append(kwargs.get("timeout")) + if command[1] == "cp" and command[3] == str(tmp_path / "work" / builder.RESOLVED / "out"): (Path(command[3]) / "result.json").write_text(json.dumps({"apt": [], "wheels": {"driver": []}})) return subprocess.CompletedProcess(command, 0, "", "") - monkeypatch.setattr(builder.subprocess, "run", docker) - monkeypatch.setattr(builder, "docker_login", lambda auth: None) - local = builder.LocalBuilder("LocalSandboxProvider", work, SimpleNamespace( - image_name="ghcr.io/acme/base:1", pinned_name="ghcr.io/acme/base:1"), "linux/arm64") + work, local = _resolving(tmp_path, monkeypatch, docker) result = local.resolve({"apt": [], "python": []}, "base@sha256:" + "b" * 64, "amd64", 60) - out = str(work / builder.RESOLVED / "out") - assert result == {"apt": [], "wheels": {"driver": []}} + out, container = str(work / builder.RESOLVED / "out"), ran[0][3] + assert result == {"apt": [], "wheels": {"driver": []}} and container.startswith("agentenv-resolve-") assert ran == [ - ["docker", "create", "--platform", "linux/amd64", "--user", "root", "--entrypoint", "python3", - "base@sha256:" + "b" * 64, "/out/resolve.py", "/out/request.json", "/out"], - ["docker", "cp", f"{work / builder.CONTEXT}/.", "c1:/ctx"], ["docker", "cp", f"{out}/.", "c1:/out"], - ["docker", "start", "--attach", "c1"], ["docker", "cp", "c1:/out/.", out], ["docker", "rm", "--force", "c1"]] + ["docker", "create", "--name", container, "--platform", "linux/amd64", "--user", "root", "--entrypoint", + "python3", "base@sha256:" + "b" * 64, "/out/resolve.py", "/out/request.json", "/out"], + ["docker", "cp", f"{work / builder.CONTEXT}/.", f"{container}:/ctx"], + ["docker", "cp", f"{out}/.", f"{container}:/out"], ["docker", "start", "--attach", container], + ["docker", "cp", f"{container}:/out/.", out], ["docker", "rm", "--force", container]] + assert all(0 < timeout <= 60 for timeout in timeouts[:-1]) and timeouts == sorted(timeouts[:-1], reverse=True) + [None] assert not [arg for command in ran for arg in command if arg in ("-v", "--volume", "--mount")] assert json.loads((work / builder.RESOLVED / "out" / "request.json").read_text())["owner"] == [os.getuid(), os.getgid()] +@pytest.mark.parametrize("stalls", ["create", "start"]) +def test_one_deadline_bounds_a_resolve_from_its_containers_pull_to_its_output_and_its_container_goes( + tmp_path, monkeypatch, stalls): + """A pull that stalls, as a registry can, ends at the resolve's deadline as one that runs too long does.""" + now, ran = [0.0], [] + + def docker(command, **kwargs): + ran.append(command) + if command[1] == stalls: + now[0] += kwargs["timeout"] + raise subprocess.TimeoutExpired(command, kwargs["timeout"], output="Pulling from acme/base") + now[0] += 1 + return subprocess.CompletedProcess(command, 0, "", "") + + monkeypatch.setattr(builder.time, "monotonic", lambda: now[0]) + _, local = _resolving(tmp_path, monkeypatch, docker) + + with pytest.raises(builder.ResolveError, match="resolving took over 60 s") as failed: + local.resolve({"apt": [], "python": []}, "base", "amd64", 60) + + assert now[0] == 60 and ran[-1] == ["docker", "rm", "--force", ran[0][3]] + assert failed.value.log == "Pulling from acme/base" + + +def _resolving(tmp_path, monkeypatch, docker): + """A builder of this machine's Docker in ``tmp_path``, its build context a driver's, ``docker`` its subprocess.run.""" + work = tmp_path / "work" + (work / builder.CONTEXT / "driver").mkdir(parents=True) + (work / builder.CONTEXT / "driver" / "requirements.txt").write_text("mss\n") + monkeypatch.setattr(builder.subprocess, "run", docker) + monkeypatch.setattr(builder, "docker_login", lambda auth: None) + return work, builder.LocalBuilder("LocalSandboxProvider", work, SimpleNamespace( + image_name="ghcr.io/acme/base:1", pinned_name="ghcr.io/acme/base:1"), "linux/arm64") + + def test_the_packages_a_build_installed_are_read_from_its_image_without_running_it(monkeypatch): listed = json.dumps([{"name": "mss", "version": "10.2.0"}, {"name": "agentenv-framework-protocol", "version": "0.1.382"}]).encode() diff --git a/tst/unit/env/envs/test_disk_recipe.py b/tst/unit/env/envs/test_disk_recipe.py index ca63685e7..e3c83e8c4 100644 --- a/tst/unit/env/envs/test_disk_recipe.py +++ b/tst/unit/env/envs/test_disk_recipe.py @@ -45,15 +45,19 @@ def test_image_env_writes_an_images_start_a_line_a_value_root_and_slash_when_it_ assert _decoded(plain) == [("argv", "sh"), ("user", "root"), ("workdir", "/")] -def test_the_seed_holds_each_vm_env_variable_by_its_name_and_nothing_else(tmp_path): - environment = {"PATH": os.environ["PATH"], "VM_ENV_AGENTENV_SETTING_RESOLUTION": "1280x800", - "VM_ENV_AGENTENV_CONTROL_TOKEN": TRICKY, "VM_ENV_EMPTY": "", "VM_PORTS": "18765", "HOME": "/root"} +def test_the_seed_holds_each_variable_the_launcher_is_handed_by_its_name_whatever_its_value_holds(tmp_path): + """A value of several lines, a certificate's say, reaches it whole, and one holding a line that looks like another + variable makes none.""" + certificate = "-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----\n" + forged = "first line\nVM_ENV_FAKE=1" - subprocess.run(["sh", str(DISK / "launcher" / "seed"), str(tmp_path)], env=environment, check=True) + _seed(tmp_path, {"AGENTENV_SETTING_RESOLUTION": "1280x800", "AGENTENV_CONTROL_TOKEN": TRICKY, "EMPTY": "", + "AGENTENV_SETTING_CA": certificate, "AGENTENV_SETTING_NOTE": forged}) user_data, meta_data = (tmp_path / "user-data").read_text(), (tmp_path / "meta-data").read_text() assert user_data.splitlines()[0] == "#agentenv-environment" - assert _decoded(user_data) == [("AGENTENV_CONTROL_TOKEN", TRICKY), ("AGENTENV_SETTING_RESOLUTION", "1280x800"), + assert _decoded(user_data) == [("AGENTENV_CONTROL_TOKEN", TRICKY), ("AGENTENV_SETTING_CA", certificate), + ("AGENTENV_SETTING_NOTE", forged), ("AGENTENV_SETTING_RESOLUTION", "1280x800"), ("EMPTY", "")] assert meta_data.endswith("\nlocal-hostname: computer\n") and meta_data.startswith("instance-id: ") @@ -66,8 +70,7 @@ def test_the_inits_decoder_keeps_every_byte_of_a_start_and_a_seed_trailing_newli config.write_text(json.dumps({"Cmd": values, "User": "user\n"})) start = subprocess.run(["python3", str(DISK / "tools" / "image-env"), str(config)], capture_output=True, text=True, check=True).stdout - subprocess.run(["sh", str(DISK / "launcher" / "seed"), str(tmp_path)], check=True, env={ - "PATH": os.environ["PATH"], **{f"VM_ENV_V{i}": value for i, value in enumerate(values)}}) + _seed(tmp_path, {f"V{i}": value for i, value in enumerate([*values, "a\nVM_ENV_FAKE=1\n"])}) seed = (tmp_path / "user-data").read_text() def decoded(text: str) -> list[str]: @@ -75,14 +78,22 @@ def decoded(text: str) -> list[str]: for line in text.splitlines() if line and not line.startswith("#")) return subprocess.run(["sh", "-c", script], capture_output=True, text=True, check=True).stdout.split("\0")[:-1] - assert decoded(start) == [*values, "user\n", "/"] and decoded(seed) == values + assert decoded(start) == [*values, "user\n", "/"] and decoded(seed) == [*values, "a\nVM_ENV_FAKE=1\n"] def test_the_launcher_gets_kvm_the_vms_ports_and_size_and_its_environment_under_its_prefix(): args = local_disk.launcher_args([18765, 5900], {"ENVIRONMENT_NAME": "desk", "A": "b c"}, 1.5, 4096) assert args == ["--device", "/dev/kvm", "-e", "VM_PORTS=18765 5900", "-e", "VM_CPUS=2", "-e", "VM_MEMORY_MB=4096", - "-e", "VM_ENV_ENVIRONMENT_NAME=desk", "-e", "VM_ENV_A=b c"] + "-e", "VM_ENV_ENVIRONMENT_NAME=ZGVzaw==", "-e", "VM_ENV_A=YiBj"] + + +def _seed(folder, env: dict[str, str]) -> None: + """The seed of ``env`` in ``folder``, as the launcher makes it from what the local provider hands it.""" + args = local_disk.launcher_args([18765], env, 2, 4096) + handed = dict(arg.split("=", 1) for flag, arg in zip(args, args[1:]) if flag == "-e") + subprocess.run(["sh", str(DISK / "launcher" / "seed"), str(folder)], check=True, + env={"PATH": os.environ["PATH"], "HOME": "/root", **handed}) @pytest.fixture @@ -114,7 +125,7 @@ async def test_a_disk_boots_in_the_launcher_with_kvm_its_ports_published_and_a_r "-p", "127.0.0.1:20001:18765", *local_disk.launcher_args([18765], {"ENVIRONMENT_NAME": "desk"}, 2, 4096), "agentenv-vm:0123") assert removed == f"docker rm -f {sandbox.container_name} > /dev/null" - assert second[-2:] == ("VM_ENV_ENVIRONMENT_NAME=again", "agentenv-vm:0123") and "VM_CPUS=2" in second + assert second[-2:] == ("VM_ENV_ENVIRONMENT_NAME=YWdhaW4=", "agentenv-vm:0123") and "VM_CPUS=2" in second LocalSandbox.pull_images.assert_awaited_with(["localhost:5000/disk@sha256:" + "a" * 64]) assert sandbox.tunnel_urls == {18765: "http://127.0.0.1:20001"} and rebuilt.owns_container From 15b52090c8d917fc92db7a8ad8d341bb9de1e28d Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 21:21:31 -0800 Subject: [PATCH 12/21] fix(computer): the first-boot lock waited for on the event loop, records matched by key, the disk layout's numbers named Waiting deploys poll the file lock (awaiting_lock) instead of holding a worker thread each, which the probing deploy needs to finish; a waiter takes a failed record, which keeps the build's key but no image; the resolver deadline test's cleanup no longer moves its clock. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/computers/disk/linux/tools/disk | 20 +++++++------ .../env/envs/computer/utils/composer.py | 28 ++++--------------- src/agent_env/store/local_state.py | 28 +++++++++++++++++-- tst/unit/env/envs/test_computer_builder.py | 2 +- tst/unit/env/envs/test_computer_disk.py | 12 ++++++-- 5 files changed, 54 insertions(+), 36 deletions(-) diff --git a/src/agent_env/computers/disk/linux/tools/disk b/src/agent_env/computers/disk/linux/tools/disk index 08b770fc3..99e28dd47 100755 --- a/src/agent_env/computers/disk/linux/tools/disk +++ b/src/agent_env/computers/disk/linux/tools/disk @@ -12,6 +12,10 @@ case "$arch" in *) echo "disk: no architecture $arch" >&2; exit 2 ;; esac esptype=C12A7328-F81F-11D2-BA4B-00A0C93EC93B +sector=512 # bytes a disk sector holds, as sfdisk counts them +mib=$((1024 * 1024 / sector)) # sectors in a MiB, the alignment partitions start on +esp_mib=64 # a new EFI system partition's size +backup_gpt=$mib # sectors kept free at the disk's end, for the backup GPT work=$(mktemp -d) trap 'rm -rf "$work"' EXIT mkdir -p "$(dirname "$out")" @@ -22,26 +26,26 @@ if [ -f "$esp/disk/disk.img" ]; then esp_start=$(echo "$partition" | sed -n 's/.*start= *\([0-9]*\).*/\1/p') esp_sectors=$(echo "$partition" | sed -n 's/.*size= *\([0-9]*\).*/\1/p') [ -n "$esp_start" ] && [ -n "$esp_sectors" ] || { echo "disk: $esp/disk/disk.img has no EFI system partition" >&2; exit 1; } - dd if="$work/base.raw" of="$work/esp.img" bs=512 skip="$esp_start" count="$esp_sectors" status=none + dd if="$work/base.raw" of="$work/esp.img" bs=$sector skip="$esp_start" count="$esp_sectors" status=none rm "$work/base.raw" elif [ -f "$esp/$uki" ]; then - size=$((size_gb * 1024 * 1024 * 1024)) esp_start=2048 esp_sectors=$((64 * 2048)) - mkfs.vfat -C -n ESP "$work/esp.img" $((esp_sectors / 2)) >/dev/null + size=$((size_gb * 1024 * mib * sector)) esp_start=$mib esp_sectors=$((esp_mib * mib)) + mkfs.vfat -C -n ESP "$work/esp.img" $((esp_sectors * sector / 1024)) >/dev/null mmd -i "$work/esp.img" ::/EFI ::/EFI/BOOT mcopy -i "$work/esp.img" "$esp/$uki" "::/EFI/BOOT/$uki" else echo "disk: $esp holds neither disk/disk.img nor $uki" >&2 exit 1 fi -root_start=$(((esp_start + esp_sectors + 2047) / 2048 * 2048)) -root_sectors=$((size / 512 - root_start - 2048)) -mkfs.ext4 -q -F -L agentenv-root -E root_owner=0:0 -d "$rootfs" "$work/root.img" "$((root_sectors / 2))k" +root_start=$(((esp_start + esp_sectors + mib - 1) / mib * mib)) +root_sectors=$((size / sector - root_start - backup_gpt)) +mkfs.ext4 -q -F -L agentenv-root -E root_owner=0:0 -d "$rootfs" "$work/root.img" "$((root_sectors * sector / 1024))k" truncate -s "$size" "$work/disk.raw" sfdisk -q "$work/disk.raw" < dict: """Gate 3 for a build its compose couldn't probe, on its first boot on ``sandbox_provider``, which boots it natively: the result, recorded. Deploys of one build at once, in this process or another, probe it once: each - waits for the one probing, then takes the result it recorded.""" - image = env.metadata["composed"][form][arch]["image"] - async with _holding(f"computer-first-boot:{env.stored_id}:{form}:{arch}:{image}"): + waits for the one probing, then takes the result it recorded, a failure's too, which keeps the build's key.""" + build = env.metadata["composed"][form][arch] + async with awaiting_lock(f"computer-first-boot:{env.stored_id}:{form}:{arch}:{build['image']}"): recorded = (await asyncio.to_thread(Env.get, env.stored_id, env.version)).metadata["composed"][form][arch] - if recorded["image"] == image and recorded["status"] != "unverified": + if recorded.get("key") == build["key"] and recorded["status"] != "unverified": return recorded result = await _checked([sandbox_provider], env, env.driver, form, arch, recorded) await asyncio.to_thread(_Record(env).finished, form, arch, result) return result -@contextlib.asynccontextmanager -async def _holding(lock: str) -> AsyncIterator[None]: - """``holding_locks`` for ``lock``, taken off the event loop; a cancelled wait releases the lock once it's taken.""" - held = contextlib.ExitStack() - taking = asyncio.ensure_future(asyncio.to_thread(held.enter_context, holding_locks([lock]))) - try: - await asyncio.shield(taking) - except asyncio.CancelledError: - taking.add_done_callback(lambda _: held.close()) - raise - try: - yield - finally: - held.close() - - def _probed(env: ComputerEnv, driver: ComputerDriverEnv, form: str, arch: str, result: dict, sandbox_provider: SandboxProvider, progress: _Progress) -> dict: """Gate 3: the build of ``result`` booted on the first link of ``sandbox_provider`` that runs ``arch``'s ``form`` diff --git a/src/agent_env/store/local_state.py b/src/agent_env/store/local_state.py index d12899439..1fae7751d 100644 --- a/src/agent_env/store/local_state.py +++ b/src/agent_env/store/local_state.py @@ -2,10 +2,11 @@ from __future__ import annotations +import asyncio import hashlib import os -from collections.abc import Callable, Iterable, Iterator -from contextlib import contextmanager +from collections.abc import AsyncIterator, Callable, Iterable, Iterator +from contextlib import asynccontextmanager, contextmanager from pathlib import Path from agent_env.config.errors import ConfigError @@ -18,6 +19,8 @@ fcntl = resource = None _OTHER_FILES = 256 +#: How often a coroutine waiting for a lock another holder has tries it again. +_LOCK_POLL_S = 0.2 def ensure_state_dir(path: Path) -> None: @@ -68,6 +71,27 @@ def holding_locks(ids: Iterable[str], on_wait: Callable[[], None] | None = None) os.close(fd) +@asynccontextmanager +async def awaiting_lock(id: str) -> AsyncIterator[None]: + """``holding_locks`` for one id, waited for on the event loop: tried again every ``_LOCK_POLL_S`` while another + holder has it, so no thread is held waiting, which the holder may need to finish and release it.""" + if fcntl is None: + yield + return + ensure_state_dir(state_root() / "locks") + fd = os.open(_lock_path(id), os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600) + try: + while True: + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + await asyncio.sleep(_LOCK_POLL_S) + yield + finally: + os.close(fd) + + def _allow_open(locks: int) -> None: """Each lock holds a file open, so more locks than the soft limit on open files allows (1024 on Linux, 256 on macOS) raise it toward the hard limit, as Go's runtime does, leaving room for the process's other files.""" diff --git a/tst/unit/env/envs/test_computer_builder.py b/tst/unit/env/envs/test_computer_builder.py index a3874146a..e9f2efb3c 100644 --- a/tst/unit/env/envs/test_computer_builder.py +++ b/tst/unit/env/envs/test_computer_builder.py @@ -152,7 +152,7 @@ def docker(command, **kwargs): if command[1] == stalls: now[0] += kwargs["timeout"] raise subprocess.TimeoutExpired(command, kwargs["timeout"], output="Pulling from acme/base") - now[0] += 1 + now[0] += 0 if command[1] == "rm" else 1 return subprocess.CompletedProcess(command, 0, "", "") monkeypatch.setattr(builder.time, "monotonic", lambda: now[0]) diff --git a/tst/unit/env/envs/test_computer_disk.py b/tst/unit/env/envs/test_computer_disk.py index 439551d3d..abc18be2f 100644 --- a/tst/unit/env/envs/test_computer_disk.py +++ b/tst/unit/env/envs/test_computer_disk.py @@ -196,9 +196,10 @@ async def test_an_arch_whose_disk_built_deploys_it_though_the_computer_has_no_di @pytest.mark.asyncio -@pytest.mark.parametrize("fails", [False, True], ids=["ready", "failed"]) +@pytest.mark.parametrize("fails", [None, "probe", "boot"]) async def test_first_deploys_of_an_unverified_disk_at_once_share_one_probe_and_its_recorded_result( disk_base, driver, docker, vms, monkeypatch, fails): + """Whether its probe passes, fails, or never boots it, whose record keeps no image.""" monkeypatch.setattr(local_sandbox, "kvm_platforms", lambda: frozenset()) env = _put(disk_base, driver) monkeypatch.setattr(local_sandbox, "kvm_platforms", lambda: frozenset({"linux/arm64"})) @@ -206,6 +207,8 @@ async def test_first_deploys_of_an_unverified_disk_at_once_share_one_probe_and_i async def slow(*args, **kwargs): await asyncio.sleep(0.3) + if fails == "boot": + raise RuntimeError("LocalSandboxProvider: no boot") return await probe(*args, **kwargs) async def counted(*args): @@ -214,16 +217,19 @@ async def counted(*args): monkeypatch.setattr(composer, "_probe", slow) monkeypatch.setattr(env_computer_provider, "probe_first_boot", counted) - if fails: + if fails == "probe": docker.probe_fails["arm64"] = "the click never landed" deployed = await asyncio.gather(*(Env.get(env.id).deploy(env_params={"form": "disk"}) for _ in range(3)), return_exceptions=True) - assert waited == [("disk", "arm64")] * 3 and len([probed for probed in docker.probed if probed[0] == "disk"]) == 1 + assert waited == [("disk", "arm64")] * 3 + assert len([probed for probed in docker.probed if probed[0] == "disk"]) == (fails != "boot") recorded = Env.get(env.id).metadata["composed"]["disk"]["arm64"] if fails: assert recorded["status"] == "failed" and vms.created == [] + assert recorded["reason"] == ("it didn't boot: LocalSandboxProvider: no boot" if fails == "boot" else + "the probe failed: P4 clicks show their effect: the click never landed") assert {str(failure) for failure in deployed} == { f"{env.id}'s disk/arm64 failed its probe on its first boot: {recorded['reason']}"} else: From 4520267d3e97f64826942b34a81a41d7c73bc9c5 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 22:00:58 -0800 Subject: [PATCH 13/21] ci(release): give disk-build KVM with chmod, as the CI tiers do The udev rule and trigger raced on two CI shards, which moved to `chmod 666 /dev/kvm`; the release's disk-build used the same udev step, and a failed disk-build blocks the PyPI publish behind it. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/publish-pypi.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index ec971b824..b11f754a5 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -406,8 +406,7 @@ jobs: if: steps.wanted.outputs.build == 'true' run: | if [ -e /dev/kvm ]; then - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm.rules - sudo udevadm control --reload-rules && sudo udevadm trigger --name-match=kvm + sudo chmod 666 /dev/kvm fi - name: Build if: steps.wanted.outputs.build == 'true' From 8457d263634e424d304f6e5a4c29febc349a34ed Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 22:09:46 -0800 Subject: [PATCH 14/21] fix(computer): a disk's size comes only from its base's [forms.disk] The published base disk's PublishedImage carries SIZE_GB from the base's artifact.toml as a build arg, in its tag and its plan entry, which infra_build check holds to the base's min_disk_gb; the boot Dockerfile and the disk tool have no default, so a new disk without a size is refused. Tags are made by source_tag rather than by hand. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/infra_build.py | 20 +++++++-- .github/scripts/infra_images.py | 6 +-- .../computers/disk/linux/boot/Dockerfile | 2 +- src/agent_env/computers/disk/linux/tools/disk | 5 ++- src/agent_env/env/published_images.py | 42 ++++++++++++------- tst/integration/computer/disk_test.py | 13 +++++- tst/unit/env/published_images_test.py | 8 +++- tst/unit/test_infra_build_script.py | 3 +- tst/unit/test_infra_images_script.py | 9 ++-- 9 files changed, 75 insertions(+), 33 deletions(-) diff --git a/.github/scripts/infra_build.py b/.github/scripts/infra_build.py index 2429b4d47..d82fde2e6 100644 --- a/.github/scripts/infra_build.py +++ b/.github/scripts/infra_build.py @@ -25,10 +25,12 @@ import subprocess import sys import time +import tomllib from collections.abc import Callable from pathlib import Path REGISTRY = "ghcr.io/scaleapi" +REPO = Path(__file__).resolve().parents[2] # Each image agent-env ships, by name, and the Dockerfile it's built from, with its folder as the context. IMAGES = { "agentenv-gateway": "src/agent_env/env/gateway/Dockerfile", @@ -52,7 +54,8 @@ "agentenv-qemu-launcher": "src/agent_env/computers/disk/linux/launcher/Dockerfile", "agentenv-ubuntu-24.04-disk": "src/agent_env/computers/disk/linux/boot/Dockerfile", } -# Each disk form's build args, by the images agent-env ships that they name. +# Each disk form's build args, by the images agent-env ships that they name; its SIZE_GB is its base's [forms.disk] +# min_disk_gb, read from the artifact.toml beside the base's Dockerfile. DISK_ARGS = {"agentenv-ubuntu-24.04-disk": {"ROOTFS": "agentenv-ubuntu-24.04", "TOOLS": "agentenv-disk-tools"}} _TAG = re.compile(r"^src-[0-9a-f]{64}$") _DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$") @@ -164,18 +167,27 @@ def _check(entry: str) -> list[str]: def _check_args(name: str, args: object) -> list[str]: - """The problems with a disk form's build args: each must name the image agent-env ships it's for, by a source tag.""" + """The problems with a disk form's build args: each image arg must name the image agent-env ships it's for, by a + source tag, and SIZE_GB must be its base's size.""" wanted = DISK_ARGS[name] - if not isinstance(args, dict) or set(args) != set(wanted): - return [f"{name}: args is {args!r}, not {', '.join(sorted(wanted))}"] + if not isinstance(args, dict) or set(args) != {*wanted, "SIZE_GB"}: + return [f"{name}: args is {args!r}, not {', '.join(sorted({*wanted, 'SIZE_GB'}))}"] problems = [] for arg, image in wanted.items(): repository, _, tag = str(args[arg]).rpartition(":") if repository != f"{REGISTRY}/{image}" or not _TAG.match(tag): problems.append(f"{name}: {arg} is {args[arg]!r}, not {REGISTRY}/{image} under a source tag") + if args["SIZE_GB"] != (size := str(_disk_size(name))): + problems.append(f"{name}: SIZE_GB is {args['SIZE_GB']!r}, not its base's min_disk_gb, {size}") return problems +def _disk_size(name: str) -> int: + """The min_disk_gb of the base computer a disk form is made of, as its artifact.toml declares it.""" + toml = Path(IMAGES[DISK_ARGS[name]["ROOTFS"]]).with_name("artifact.toml") + return tomllib.loads((REPO / toml).read_text())["forms"]["disk"]["min_disk_gb"] + + def _tried_again(attempt: Callable[[], subprocess.CompletedProcess]) -> subprocess.CompletedProcess: """``attempt()``, and again after each of ``_RETRY_WAITS_SECONDS`` while what it printed says a later try can work (``_why_again``).""" diff --git a/.github/scripts/infra_images.py b/.github/scripts/infra_images.py index ea6bb22d4..8052fe044 100644 --- a/.github/scripts/infra_images.py +++ b/.github/scripts/infra_images.py @@ -3,7 +3,7 @@ installed, and no job that holds a token to push images or publish to PyPI runs it (those run ``infra_build.py``). ``plan`` prints each image's name, repository, tag, Dockerfile and context as JSON, and for an image built over other -published images, the build args naming each by its repository and tag, ``--missing`` only those the +published images, its build args, each naming one by its repository and tag or giving a value, ``--missing`` only those the registry hasn't published: it has no image under the tag, or refuses to show one, as GHCR does for a package that doesn't exist yet, so the first publish can start. ``--only infra`` lists only the infra images, ``--only agents`` only the images that install the protocol, the agents' and the website browser's, which a release builds after it @@ -66,8 +66,8 @@ def _described(image: PublishedImage) -> dict: described = {"name": image.name, "repository": image.repository, "tag": image.tag(), "dockerfile": str(image.dockerfile.relative_to(REPO)), "context": str(image.dockerfile.parent.relative_to(REPO))} - if image.args: - described["args"] = {arg: built.ref() for arg, built in image.args} + if image.args or image.build_args: + described["args"] = {**{arg: built.ref() for arg, built in image.args}, **dict(image.build_args)} return described diff --git a/src/agent_env/computers/disk/linux/boot/Dockerfile b/src/agent_env/computers/disk/linux/boot/Dockerfile index 06453d9c7..e91a97261 100644 --- a/src/agent_env/computers/disk/linux/boot/Dockerfile +++ b/src/agent_env/computers/disk/linux/boot/Dockerfile @@ -31,7 +31,7 @@ FROM ${ESP} AS esp FROM --platform=$BUILDPLATFORM ${TOOLS} AS disk ARG TARGETARCH -ARG SIZE_GB=16 +ARG SIZE_GB RUN --mount=type=bind,from=root,target=/rootfs --mount=type=bind,from=esp,target=/esp \ disk "$TARGETARCH" /rootfs /esp /out/disk.img "$SIZE_GB" diff --git a/src/agent_env/computers/disk/linux/tools/disk b/src/agent_env/computers/disk/linux/tools/disk index 99e28dd47..7616327cc 100755 --- a/src/agent_env/computers/disk/linux/tools/disk +++ b/src/agent_env/computers/disk/linux/tools/disk @@ -1,11 +1,11 @@ #!/bin/sh -# disk ARCH ROOTFS ESP OUT [SIZE_GB]: a Linux computer's disk form, written to OUT as qcow2: a GPT disk of an EFI system +# disk ARCH ROOTFS ESP OUT SIZE_GB: a Linux computer's disk form, written to OUT as qcow2: a GPT disk of an EFI system # partition and an ext4 root filesystem made from the folder ROOTFS, ownership kept. ESP is a folder holding either a # disk form's image, disk/disk.img, whose EFI system partition and size the disk keeps byte for byte, or a UKI, which a # new EFI system partition holds on a disk of SIZE_GB. Nothing is mounted or booted, so it runs unprivileged, on any # architecture, for either. set -eu -arch=$1 rootfs=$2 esp=$3 out=$4 size_gb=${5:-16} +arch=$1 rootfs=$2 esp=$3 out=$4 size_gb=$5 case "$arch" in amd64) uki=BOOTX64.EFI roottype=4F68BCE3-E8CD-4DB1-96E7-FBCAF984B709 ;; arm64) uki=BOOTAA64.EFI roottype=B921B045-1DF0-41C3-AF44-4C6F280D3FAE ;; @@ -29,6 +29,7 @@ if [ -f "$esp/disk/disk.img" ]; then dd if="$work/base.raw" of="$work/esp.img" bs=$sector skip="$esp_start" count="$esp_sectors" status=none rm "$work/base.raw" elif [ -f "$esp/$uki" ]; then + [ -n "$size_gb" ] || { echo "disk: a new disk needs its size, SIZE_GB" >&2; exit 2; } size=$((size_gb * 1024 * mib * sector)) esp_start=$mib esp_sectors=$((esp_mib * mib)) mkfs.vfat -C -n ESP "$work/esp.img" $((esp_sectors * sector / 1024)) >/dev/null mmd -i "$work/esp.img" ::/EFI ::/EFI/BOOT diff --git a/src/agent_env/env/published_images.py b/src/agent_env/env/published_images.py index 83d54821a..54a426c05 100644 --- a/src/agent_env/env/published_images.py +++ b/src/agent_env/env/published_images.py @@ -10,6 +10,7 @@ import json import logging import re +import tomllib from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path @@ -37,24 +38,27 @@ @dataclass(frozen=True) class PublishedImage: """An image agent-env publishes, under ``name`` in the registry, built from ``dockerfile`` with its folder as the - build context, and with each of ``args``, a build arg its Dockerfile builds from, set to that published image.""" + build context, with each of ``args``, a build arg its Dockerfile builds from, set to that published image, and each + of ``build_args`` set to its value.""" name: str dockerfile: Path args: tuple[tuple[str, PublishedImage], ...] = () + build_args: tuple[tuple[str, str], ...] = () @property def repository(self) -> str: return f"{REGISTRY}/{self.name}" def tag(self) -> str: - """``src-`` and the hex of the build context's source digest, with each of ``args``' tags when it has any: the - same for the same sources on any platform.""" - digest = BuildContext.of(self.dockerfile.parent, self.dockerfile).source_digest(None).removeprefix("sha256:") - if self.args: - digest = hashlib.sha256("\0".join([digest, *(f"{arg}={image.tag()}" for arg, image in self.args)]) - .encode()).hexdigest() - return f"src-{digest}" + """``src-`` and the hex of the build context's source digest, with each of ``args``' tags and ``build_args``' + values when it has any: the same for the same sources on any platform.""" + digest = BuildContext.of(self.dockerfile.parent, self.dockerfile).source_digest(None) + if self.args or self.build_args: + digest = hashlib.sha256("\0".join([digest.removeprefix("sha256:"), + *(f"{arg}={image.tag()}" for arg, image in self.args), + *(f"{arg}={value}" for arg, value in self.build_args)]).encode()).hexdigest() + return source_tag(digest) def ref(self) -> str: """The repository and the tag of the image built from these sources.""" @@ -73,10 +77,17 @@ def ref(self) -> str: # What makes and boots a Linux disk form: the disk tools a build runs on its own platform, and the QEMU launcher. DISK_TOOLS_IMAGE = PublishedImage("agentenv-disk-tools", DISK / "tools" / "Dockerfile") LAUNCHER_IMAGE = PublishedImage("agentenv-qemu-launcher", DISK / "launcher" / "Dockerfile") -# Each base computer's disk form: its container form's root filesystem on the boot recipe, so a run pulls it rather than -# building it from the two. -DISK_IMAGES = tuple(PublishedImage(f"{image.name}-disk", DISK / "boot" / "Dockerfile", - (("ROOTFS", image), ("TOOLS", DISK_TOOLS_IMAGE))) for image in COMPUTER_IMAGES) +# Each base computer's disk form: its container form's root filesystem on the boot recipe, at the size its [forms.disk] +# declares, so a run pulls it rather than building it from the two. +DISK_IMAGES = tuple(PublishedImage( + f"{image.name}-disk", DISK / "boot" / "Dockerfile", (("ROOTFS", image), ("TOOLS", DISK_TOOLS_IMAGE)), + (("SIZE_GB", str(tomllib.loads((image.dockerfile.parent / "artifact.toml").read_text())["forms"]["disk"][ + "min_disk_gb"])),)) for image in COMPUTER_IMAGES) + + +def source_tag(digest: str) -> str: + """The tag of a published image whose sources have the digest ``digest``.""" + return f"src-{digest.removeprefix('sha256:')}" def published_digest(image: PublishedImage) -> str | None: @@ -95,8 +106,8 @@ def published_pinned(image: PublishedImage) -> str | None: def published_disk(sources: str) -> str | None: """The pinned disk form published for the shipped base computer whose container image's installed build context has the source digest ``sources``, or None when no shipped base has those sources or its disk isn't published.""" - tag = f"src-{sources.removeprefix('sha256:')}" - return next((published_pinned(disk) for disk in DISK_IMAGES if dict(disk.args)["ROOTFS"].tag() == tag), None) + return next((published_pinned(disk) for disk in DISK_IMAGES if dict(disk.args)["ROOTFS"].tag() == source_tag(sources)), + None) def published_bundle_image(sources: str) -> tuple[str, str] | None: @@ -106,9 +117,8 @@ def published_bundle_image(sources: str) -> tuple[str, str] | None: shipped image has those sources or the registry doesn't serve it anonymously, as a sandbox pulls it: a new package stays private until a package admin makes it public, and a sandbox running an image by name can't build it instead.""" - tag = f"src-{sources.removeprefix('sha256:')}" for image in BUNDLE_IMAGES: - if image.tag() == tag: + if image.tag() == source_tag(sources): ref = image.ref() if (served := _registry_digest(ref)) is None: return None diff --git a/tst/integration/computer/disk_test.py b/tst/integration/computer/disk_test.py index 6ea3b65f9..4630d29a0 100644 --- a/tst/integration/computer/disk_test.py +++ b/tst/integration/computer/disk_test.py @@ -16,7 +16,7 @@ from agent_env.env.published_images import DISK from agent_env.providers.sandbox_providers.host_images import host_platform -from agent_env.utils.docker.build import build_image +from agent_env.utils.docker.build import DockerBuildError, build_image pytestmark = pytest.mark.integration @@ -82,6 +82,17 @@ def test_a_disk_is_a_root_filesystem_with_its_start_beside_a_new_esp_and_one_mad assert ("env", "EXTRA=1") in over["start"] and over["esp"] == made["esp"] and over["root"] != made["root"] +def test_a_new_disk_without_its_size_is_refused_rather_than_made_at_some_default(tools, images, tmp_path): + platform = host_platform() + arch = platform.rsplit("/", 1)[1] + rootfs = _built(images, ROOTFS, platform) + uki = _built(images, _folder(tmp_path / "uki", {"Dockerfile": f"FROM scratch\nCOPY {UKI[arch]} /\n", + UKI[arch]: "a UKI"}), platform) + + with pytest.raises(DockerBuildError, match="a new disk needs its size, SIZE_GB"): + _disk(images, tools, rootfs, uki, platform, tmp_path) + + def _built(images: list, folder: Path, platform: str) -> str: tag = f"agentenv-test-disk:{uuid.uuid4().hex[:12]}" build_image(folder / "Dockerfile", folder, tag, platform=platform) diff --git a/tst/unit/env/published_images_test.py b/tst/unit/env/published_images_test.py index d433d7534..7dfd67121 100644 --- a/tst/unit/env/published_images_test.py +++ b/tst/unit/env/published_images_test.py @@ -2,9 +2,11 @@ forms: one per shipped Dockerfile, tagged by its sources (a disk form's by its base's too), and the record of their digests a release wheel carries.""" +import dataclasses import hashlib import json import shutil +import tomllib import pytest @@ -187,10 +189,12 @@ def test_a_disk_forms_tag_is_of_its_boot_recipe_its_bases_sources_and_the_disk_t ("TOOLS", DISK_TOOLS_IMAGE))) sources = BuildContext.of(base.dockerfile.parent, base.dockerfile).source_digest(None) + size = tomllib.loads(base.dockerfile.with_name("artifact.toml").read_text())["forms"]["disk"]["min_disk_gb"] assert disk.name == "agentenv-ubuntu-24.04-disk" and dict(disk.args) == {"ROOTFS": base, "TOOLS": DISK_TOOLS_IMAGE} + assert disk.build_args == (("SIZE_GB", str(size)),) assert disk.tag() == "src-" + hashlib.sha256("\0".join( - [recipe, f"ROOTFS={base.tag()}", f"TOOLS={DISK_TOOLS_IMAGE.tag()}"]).encode()).hexdigest() - assert moved.tag() != disk.tag() + [recipe, f"ROOTFS={base.tag()}", f"TOOLS={DISK_TOOLS_IMAGE.tag()}", f"SIZE_GB={size}"]).encode()).hexdigest() + assert moved.tag() != disk.tag() != dataclasses.replace(disk, build_args=(("SIZE_GB", "32"),)).tag() assert published_disk(sources) is None served[disk.ref()] = _DIGEST assert published_disk(sources) == f"{disk.ref()}@{_DIGEST}" and published_disk("sha256:" + "0" * 64) is None diff --git a/tst/unit/test_infra_build_script.py b/tst/unit/test_infra_build_script.py index ec896ff41..56dbeb772 100644 --- a/tst/unit/test_infra_build_script.py +++ b/tst/unit/test_infra_build_script.py @@ -69,7 +69,7 @@ def _disk(**args) -> str: "dockerfile": "src/agent_env/computers/disk/linux/boot/Dockerfile", "context": "src/agent_env/computers/disk/linux/boot", "args": { "ROOTFS": f"{REGISTRY}/agentenv-ubuntu-24.04:{_TAG}", "TOOLS": f"{REGISTRY}/agentenv-disk-tools:{_TAG}", - **args}}) + "SIZE_GB": "16", **args}}) @pytest.mark.parametrize("entry, problem", [ @@ -78,6 +78,7 @@ def _disk(**args) -> str: (_disk(ROOTFS="ghcr.io/someone/ubuntu:latest"), "agentenv-ubuntu-24.04-disk: ROOTFS is 'ghcr.io/someone/ubuntu:latest'"), (_disk(TOOLS=f"{REGISTRY}/agentenv-disk-tools:latest"), "agentenv-ubuntu-24.04-disk: TOOLS is 'ghcr.io/scaleapi/"), (_disk(EXTRA="x"), "agentenv-ubuntu-24.04-disk: args is "), + (_disk(SIZE_GB="32"), "agentenv-ubuntu-24.04-disk: SIZE_GB is '32', not its base's min_disk_gb, 16"), (_gateway(repository="ghcr.io/someone/agentenv-gateway"), "agentenv-gateway: repository is 'ghcr.io/someone/"), (_gateway(dockerfile="tst/data/agentenv_mcp/Dockerfile"), "agentenv-gateway: dockerfile is 'tst/data/agentenv_mcp"), (_gateway(context="."), "agentenv-gateway: context is '.'"), diff --git a/tst/unit/test_infra_images_script.py b/tst/unit/test_infra_images_script.py index d186195a5..71be6dd6a 100644 --- a/tst/unit/test_infra_images_script.py +++ b/tst/unit/test_infra_images_script.py @@ -4,6 +4,7 @@ import json import sys +import tomllib import pytest @@ -66,9 +67,11 @@ def test_plan_only_lists_the_infra_images_the_agents_images_or_the_disk_forms(sc "agents": after, "disks": disks} assert "agentenv-disk-tools" in names["infra"] and "agentenv-qemu-launcher" in names["infra"] (disk,) = planned["disks"] - assert disk["args"] == {"ROOTFS": next(image.ref() for image in BUNDLE_IMAGES if image.name == "agentenv-ubuntu-24.04"), - "TOOLS": next(image.ref() for image in bootstrap.PUBLISHED_IMAGES - if image.name == "agentenv-disk-tools")} + base = next(image for image in BUNDLE_IMAGES if image.name == "agentenv-ubuntu-24.04") + assert disk["args"] == {"ROOTFS": base.ref(), "TOOLS": next(image.ref() for image in bootstrap.PUBLISHED_IMAGES + if image.name == "agentenv-disk-tools"), + "SIZE_GB": str(tomllib.loads(base.dockerfile.with_name("artifact.toml").read_text())[ + "forms"]["disk"]["min_disk_gb"])} assert "args" not in planned["infra"][0] assert "agentenv-gateway" in names["infra"] and bundles == [ "agentenv-claude-code", "agentenv-codex", "agentenv-gemini", "agentenv-hermes", "agentenv-letta-code", From 99c95a31d192ecddd4d10a6a20983e42ab5b7a0f Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 22:10:20 -0800 Subject: [PATCH 15/21] refactor(computer): the platform check's image is the one the shipped base's Dockerfile pins Read from that Dockerfile's FROM, so a Dependabot bump of the base moves it too. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/envs/computer/utils/builder.py | 16 ++++++++++------ tst/unit/env/envs/test_computer_builder.py | 7 +++---- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/agent_env/env/envs/computer/utils/builder.py b/src/agent_env/env/envs/computer/utils/builder.py index 10507233e..d71c7504c 100644 --- a/src/agent_env/env/envs/computer/utils/builder.py +++ b/src/agent_env/env/envs/computer/utils/builder.py @@ -11,6 +11,7 @@ import hashlib import io import json +import re import logging import os import shlex @@ -30,6 +31,7 @@ from agent_env.artifact.artifacts.docker_image import docker_login, push_local_image from agent_env.config import get_config from agent_env.env.envs.computer.utils import layers_in_image +from agent_env.env.published_images import COMPUTER_IMAGES from agent_env.providers.sandbox_providers.host_images import build_for, host_platform, platform_tag from agent_env.providers.sandbox_providers.sandbox import upload_vm_file from agent_env.providers.sandbox_providers.sandbox_provider import SANDBOX_MODE_VM @@ -54,10 +56,6 @@ #: since the stores keep no file modes. RUN_ONLY = (b"#!", b"\x7fELF") EXECUTABLE = 0o755 -#: What shows whether this machine's Docker runs a platform, for a base that's only a build context: the image the shipped -#: Linux bases build from, which their build pulls for the platform anyway. -PLATFORM_CHECK = ("public.ecr.aws/docker/library/ubuntu:24.04" - "@sha256:24737cef17f4e22b1977df7d9a05e77c4e28b7da5b8911f8d8ccda136bf3573c") #: What registers QEMU in a builder VM's kernel, for a platform no link runs natively: Docker Hub's image through #: Google's mirror of it, which doesn't limit anonymous pulls by IP as Docker Hub does. BINFMT = ("mirror.gcr.io/tonistiigi/binfmt:qemu-v9.2.2" @@ -255,13 +253,19 @@ def _runs_here(platform: str, image: str) -> bool: def _platform_check(base: DockerImageArtifact) -> str: """The image ``_runs_here`` starts for a build ``FROM`` ``base``, one the build pulls anyway: ``base`` when it's a - registry's, signed in to it, else ``PLATFORM_CHECK``.""" + registry's, signed in to it, else the image the shipped Linux base builds from, as its build pulls it.""" if not names_registry(base.image_name): - return PLATFORM_CHECK + return _shipped_base_from() docker_login(get_config().get_image_store().auth(base.pinned_name)) return base.pinned_name +@functools.cache +def _shipped_base_from() -> str: + """The image the shipped Linux base computer's Dockerfile builds ``FROM``, pinned by its digest there.""" + return re.search(r"^FROM\s+(\S+)", COMPUTER_IMAGES[0].dockerfile.read_text(), re.MULTILINE).group(1) + + def _resolver(source: str, arch: str, context: str, out: str) -> list[str]: return ["docker", "run", "--rm", "--platform", f"linux/{arch}", "--user", "root", "--entrypoint", "python3", "-v", f"{context}:/ctx:ro", "-v", f"{out}:/out", source, "/out/resolve.py", "/out/request.json", "/out"] diff --git a/tst/unit/env/envs/test_computer_builder.py b/tst/unit/env/envs/test_computer_builder.py index e9f2efb3c..fd0c4dd57 100644 --- a/tst/unit/env/envs/test_computer_builder.py +++ b/tst/unit/env/envs/test_computer_builder.py @@ -19,7 +19,6 @@ import pytest from click.testing import CliRunner -import agent_env from agent_env.artifact.artifacts.docker_image import DockerImageArtifact from agent_env.artifact.store import get_artifact_store from agent_env.cli import cli @@ -101,15 +100,15 @@ def test_whether_this_machine_runs_a_platform_is_whether_a_container_of_it_runs_ for platform in ("linux/amd64", "linux/riscv64")] -def test_the_platform_check_starts_the_base_or_the_image_the_shipped_bases_build_from(monkeypatch): - shipped = (Path(agent_env.__file__).parent / "computers" / "artifacts" / "ubuntu-24.04" / "Dockerfile").read_text() +def test_the_platform_check_starts_the_base_or_the_pinned_image_the_shipped_base_builds_from(monkeypatch): logins = [] monkeypatch.setattr(builder, "docker_login", logins.append) registry = SimpleNamespace(image_name="ghcr.io/acme/base:1", pinned_name="ghcr.io/acme/base:1@sha256:" + "a" * 64) context = SimpleNamespace(image_name="local/base-1234:abcd", pinned_name="local/base-1234:abcd") assert builder._platform_check(registry) == registry.pinned_name and len(logins) == 1 - assert builder._platform_check(context) == builder.PLATFORM_CHECK and f"FROM {builder.PLATFORM_CHECK}\n" in shipped + assert re.fullmatch(r"public\.ecr\.aws/docker/library/ubuntu:24\.04@sha256:[0-9a-f]{64}", + builder._platform_check(context)) def test_a_resolve_copies_its_inputs_into_the_resolvers_container_and_its_outputs_out_with_no_bind_mount( From 64e9744488b195c77a166d8ad51952d5fa34530e Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 22:10:45 -0800 Subject: [PATCH 16/21] refactor(computer): the disk's path named by DISK_FILE where Python names it Co-Authored-By: Claude Opus 5.5 --- .../providers/sandbox_providers/local_docker/disk.py | 8 +++++--- .../providers/sandbox_providers/sandbox_provider.py | 3 ++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/agent_env/providers/sandbox_providers/local_docker/disk.py b/src/agent_env/providers/sandbox_providers/local_docker/disk.py index d016dede2..a667bb367 100644 --- a/src/agent_env/providers/sandbox_providers/local_docker/disk.py +++ b/src/agent_env/providers/sandbox_providers/local_docker/disk.py @@ -9,8 +9,9 @@ import os import subprocess import tempfile -from pathlib import Path +from pathlib import Path, PurePosixPath +from agent_env.artifact.artifacts.computer import DISK_FILE from agent_env.env.published_images import LAUNCHER_IMAGE, published_pinned from agent_env.providers.sandbox_providers.host_images import host_platform, present from agent_env.utils.docker.build import build_image @@ -26,11 +27,12 @@ def kvm_platforms() -> frozenset[str]: async def machine_image(disk: str) -> str: - """An image of the launcher holding ``disk``'s /disk/disk.img for this machine's platform, built unless held.""" + """An image of the launcher holding ``disk``'s ``DISK_FILE`` for this machine's platform, built unless held.""" launcher = await asyncio.to_thread(_launcher) tag = "agentenv-vm:" + hashlib.sha256(f"{launcher} {disk}".encode()).hexdigest()[:16] if not await asyncio.to_thread(present, tag): - await asyncio.to_thread(_build, f"FROM {launcher}\nCOPY --from={disk} /disk/ /disk/\n", tag) + folder = f"/{PurePosixPath(DISK_FILE).parent}/" + await asyncio.to_thread(_build, f"FROM {launcher}\nCOPY --from={disk} {folder} {folder}\n", tag) return tag diff --git a/src/agent_env/providers/sandbox_providers/sandbox_provider.py b/src/agent_env/providers/sandbox_providers/sandbox_provider.py index b8928f796..4246ad97d 100644 --- a/src/agent_env/providers/sandbox_providers/sandbox_provider.py +++ b/src/agent_env/providers/sandbox_providers/sandbox_provider.py @@ -93,7 +93,8 @@ async def create_vm( env: Optional[dict[str, str]] = None, ) -> VmSandbox: """A VM. A provider whose ``disk_image_platforms`` names a platform boots a computer's disk form, ``image``, an - image holding /disk/disk.img, for it, its ``env`` in the disk's NoCloud seed; such a sandbox has no exec channel.""" + image holding ``computer.DISK_FILE``, for it, its ``env`` in the disk's NoCloud seed; such a sandbox has no exec + channel.""" raise NotImplementedError(f"{type(self).__name__} does not support create_vm") async def create_container( From ea1df28426cc77aa68807ed24899b79991041944 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 22:11:19 -0800 Subject: [PATCH 17/21] refactor(computer): one image inspection for both forms, each with its own reader The disk form's checks get the container's "only {served}" detail, and get_config is imported at the module's top. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/artifact/artifacts/computer.py | 92 +++++++++----------- 1 file changed, 41 insertions(+), 51 deletions(-) diff --git a/src/agent_env/artifact/artifacts/computer.py b/src/agent_env/artifact/artifacts/computer.py index 93185964e..61d5cc773 100644 --- a/src/agent_env/artifact/artifacts/computer.py +++ b/src/agent_env/artifact/artifacts/computer.py @@ -5,7 +5,9 @@ from __future__ import annotations +import functools import re +from collections.abc import Callable from datetime import datetime, timezone from typing import TYPE_CHECKING, Any, ClassVar, Literal, Self @@ -23,9 +25,10 @@ from agent_env.artifact.artifact import Artifact from agent_env.artifact.artifacts.docker_image import DockerImageArtifact from agent_env.artifact.ref import ArtifactRef +from agent_env.config import get_config from agent_env.entity.refs import EntityRef from agent_env.store.base import NotFoundError -from agent_env.store.image_store.oci_registry_credentials import names_registry +from agent_env.store.image_store.oci_registry_credentials import RegistryAuth, names_registry from agent_env.store.image_store.image_contents import UnreadableLayerError, disk_layer, os_release, platform_manifests if TYPE_CHECKING: @@ -191,13 +194,14 @@ def put(cls, *, inspect: bool = True, **fields: Any) -> Self: computer = cls(**fields) image, disk = computer.image(), computer.forms.disk - checks = {"container": _inspect_container(image, computer.arch, computer.os) if inspect - else _unread(computer.arch, "inspection was skipped")} + checks = {"container": _inspect(image, computer.arch, functools.partial(_read_container, computer.os)) + if inspect else _unread(computer.arch, "inspection was skipped")} if disk is not None and disk.image is None: checks["disk"] = _unread(computer.arch, "it's made from the container form when composed") elif disk is not None: - checks["disk"] = (_inspect_disk(computer.disk_image(), computer.arch, disk.min_disk_gb) if inspect - else _unread(computer.arch, "inspection was skipped")) + checks["disk"] = (_inspect(computer.disk_image(), computer.arch, + functools.partial(_read_disk, disk.min_disk_gb)) + if inspect else _unread(computer.arch, "inspection was skipped")) problems = [f"forms.{form} ({arch}): {check.problem}" for form, by_arch in checks.items() for arch, check in by_arch.items() if check.status == "failed"] if problems: @@ -240,9 +244,11 @@ def _image(self, form: str, ref: ArtifactRef) -> DockerImageArtifact: raise ValueError(f"computer {self.id!r}: forms.{form}.image: {e}") from e -def _inspect_container(image: DockerImageArtifact, arches: list[str], os: LinuxOS) -> dict[str, ImageCheck]: - from agent_env.config import get_config - +def _inspect(image: DockerImageArtifact, arches: list[str], + read: Callable[[str, str, RegistryAuth | None], ImageCheck]) -> dict[str, ImageCheck]: + """Each arch's image of a form, ``read`` from its registry without pulling it: failed when the registry serves none + for an arch or ``read`` refuses it, uninspected when this Python can't read its layers, so its first boot checks + it.""" if not names_registry(image.image_name): how = "only its build context is stored" if image.context_only else "it's stored under a tag no registry serves" return _unread(arches, f"{how}, so its first boot checks it") @@ -259,58 +265,42 @@ def _inspect_container(image: DockerImageArtifact, arches: list[str], os: LinuxO checks[arch] = ImageCheck(status="failed", image=ref, problem=f"it has no linux/{arch} image, only {served}") continue try: - release = os_release(ref, digest, auth) + checks[arch] = read(ref, digest, auth) except UnreadableLayerError as e: checks[arch] = ImageCheck(status="uninspected", image=ref, manifest=digest, reason=f"{e}, so its first boot checks it") - continue except ValueError as e: - checks[arch] = ImageCheck(status="failed", image=ref, manifest=digest, - problem=f"its os-release couldn't be read: {e}") - continue - expected = {"ID": os.distribution, "VERSION_ID": None if os.version == "rolling" else os.version} - problems = [f"os-release has {key}={release.get(key)!r}, not {want!r}" for key, want in expected.items() - if release.get(key) != want] - checks[arch] = ImageCheck(status="failed" if problems else "passed", image=ref, manifest=digest, - os_release={key: release[key] for key in expected if key in release}, - problem="; ".join(problems) or None) + checks[arch] = ImageCheck(status="failed", image=ref, manifest=digest, problem=str(e)) return checks -def _inspect_disk(image: DockerImageArtifact, arches: list[str], min_disk_gb: int) -> dict[str, ImageCheck]: - """Each arch's disk image read from the start of its one layer: its file, its owner and its virtual size.""" - from agent_env.config import get_config - - if not names_registry(image.image_name): - return _unread(arches, "it's stored under a tag no registry serves, so its first boot checks it") - ref = image.pinned_name - auth = get_config().get_image_store().auth(ref) +def _read_container(os: LinuxOS, ref: str, digest: str, auth: RegistryAuth | None) -> ImageCheck: + """The container form's image manifest ``digest``, by its os-release against ``os``.""" try: - ref, manifests = platform_manifests(ref, auth) + release = os_release(ref, digest, auth) + except UnreadableLayerError: + raise except ValueError as e: - return {arch: ImageCheck(status="failed", image=ref, problem=str(e)) for arch in arches} - checks = {} - for arch in arches: - if (digest := manifests.get(arch)) is None: - checks[arch] = ImageCheck(status="failed", image=ref, problem=f"it has no linux/{arch} image") - continue - try: - found = disk_layer(ref, digest, auth) - except UnreadableLayerError as e: - checks[arch] = ImageCheck(status="uninspected", image=ref, manifest=digest, - reason=f"{e}, so its first boot checks it") - continue - except ValueError as e: - checks[arch] = ImageCheck(status="failed", image=ref, manifest=digest, problem=str(e)) - continue - problems = [problem for problem, wrong in ( - (f"its first file is {found.path}, not {DISK_FILE}", found.path != DISK_FILE), - (f"{found.path} is owned by uid {found.uid}, not {DISK_OWNER}", found.uid != DISK_OWNER), - (f"its virtual size is {found.virtual_size / 2**30:.1f} GiB, under min_disk_gb ({min_disk_gb})", - found.virtual_size < min_disk_gb * 2**30)) if wrong] - checks[arch] = ImageCheck(status="failed" if problems else "passed", image=ref, manifest=digest, - problem="; ".join(problems) or None) - return checks + raise ValueError(f"its os-release couldn't be read: {e}") from e + expected = {"ID": os.distribution, "VERSION_ID": None if os.version == "rolling" else os.version} + problems = [f"os-release has {key}={release.get(key)!r}, not {want!r}" for key, want in expected.items() + if release.get(key) != want] + return ImageCheck(status="failed" if problems else "passed", image=ref, manifest=digest, + os_release={key: release[key] for key in expected if key in release}, + problem="; ".join(problems) or None) + + +def _read_disk(min_disk_gb: int, ref: str, digest: str, auth: RegistryAuth | None) -> ImageCheck: + """The disk form's image manifest ``digest``, by the start of its one layer: its file, its owner and its virtual + size.""" + found = disk_layer(ref, digest, auth) + problems = [problem for problem, wrong in ( + (f"its first file is {found.path}, not {DISK_FILE}", found.path != DISK_FILE), + (f"{found.path} is owned by uid {found.uid}, not {DISK_OWNER}", found.uid != DISK_OWNER), + (f"its virtual size is {found.virtual_size / 2**30:.1f} GiB, under min_disk_gb ({min_disk_gb})", + found.virtual_size < min_disk_gb * 2**30)) if wrong] + return ImageCheck(status="failed" if problems else "passed", image=ref, manifest=digest, + problem="; ".join(problems) or None) def _unread(arches: list[str], why: str) -> dict[str, ImageCheck]: From da6ba64c15253e2bdf47c3bdcc66f3a2829e337d Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 22:12:10 -0800 Subject: [PATCH 18/21] refactor(computer): the disk build shares the container build's key, reuse and failure handling _key, _stored and _attempting (one context manager mapping a build's failures to its result), so the disk build gets the "exec format error" reason too; making the base's disk is a helper of its own. Container keys are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../env/envs/computer/utils/composer.py | 283 ++++++++++-------- 1 file changed, 153 insertions(+), 130 deletions(-) diff --git a/src/agent_env/env/envs/computer/utils/composer.py b/src/agent_env/env/envs/computer/utils/composer.py index c3f4c1340..cbde770c8 100644 --- a/src/agent_env/env/envs/computer/utils/composer.py +++ b/src/agent_env/env/envs/computer/utils/composer.py @@ -27,7 +27,7 @@ from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path, PurePosixPath -from collections.abc import Callable +from collections.abc import Callable, Iterator from typing import TYPE_CHECKING, Any from agentenv_protocol.computer import apps as installed_apps @@ -255,85 +255,66 @@ def _built(env: ComputerEnv, base: ComputerArtifact, driver: ComputerDriverEnv, builds: builder.Builds, progress: _Progress) -> dict: """The status of ``form`` and ``arch`` once built and pushed, or reused, or the stage that failed and why, each stage reported to ``progress``. A builder from ``builds`` resolves and builds it, opened when first needed.""" - stage, key, pair = "build", None, f"{form}/{arch}" - try: - with tempfile.TemporaryDirectory(prefix="agentenv-compose-") as work, contextlib.ExitStack() as opened: - context, fetched, from_store = Path(work) / builder.CONTEXT, {}, {} - context.mkdir() - source, base_identity = _base(base, arch) - layer = _stage_driver(context, driver, base, from_store) - apps = [_stage_app(context, app, base, from_store) for app in env.composed_apps()] - layers = [layer, *apps] - wants = _wants(layers, _stage_protocol(context)) - platform, held = f"linux/{arch}", [] - - def on() -> builder.Builder: - if not held: - link = builds.link(platform, base.image()) - native = platform in link.native_platforms() - progress.at(pair, "starting its builder", f"building natively {builder.where(link)}" if native else - f"building emulated, which is slow, {builder.where(link)}; " - f"{builder.instead_of_emulating(builds.provider, platform)}") - held.append(opened.enter_context(builds.open(link, platform, Path(work), base.image(), from_store))) - return held[0] - - inputs = packages.inputs_digest(wants, context, base_identity, arch) - lock = packages.stored(inputs, env.stored_id) if inputs is not None else None - if inputs is not None and lock is None: - stage = "resolve" - resolving = on() - progress.at(pair, "resolving packages") - lock, fetched = packages.resolve(inputs, wants, resolving, source, arch, env.stored_id) - progress.at(pair, "resolved", f"resolved {len(fetched)} files and stored them") - stage = "build" - dockerfile = _dockerfile(source, layers, base, _build_settings(env, base), lock) - key = "sha256:" + hashlib.sha256(json.dumps({ - "base": base_identity, "dockerfile": dockerfile, "context": _digest(context), - "lock": lock.digest() if lock is not None else None, "form": form, "arch": arch, - "format": FORMAT, - }, sort_keys=True).encode()).hexdigest() - store = get_config().get_image_store_for(env.stored_id) - store.ensure_repository(REPOSITORY) - ref = store.image_ref(REPOSITORY, key.removeprefix("sha256:")) - if (pinned := _served(ref, store)) is not None: - progress.at(pair, "reusing", "reusing the image of the same key, built earlier") - kept = labels(pinned, arch, store.auth(pinned)) - return {"status": "unverified", "key": key, "image": pinned, "reused": True, - "emulated": kept.get(EMULATED_LABEL) == "true", "lock": inputs, - "installed": json.loads(kept.get(INSTALLED_LABEL, "{}")), "built_at": _now()} - building_on = on() - progress.at(pair, "building the image") - if lock is not None: - packages.stage(lock, building_on, env.stored_id, fetched) - local, emulated = f"agentenv-computer:{key[7:23]}-{arch}", platform != building_on.platform - building_on.build(f"{dockerfile}LABEL {KEY_LABEL}={key}\n", local, platform, {KEY_LABEL: key}) - if reason := _compare(building_on, local, layers, pair, progress): - return _failed(env, form, arch, stage, key, reason) - installed = building_on.installed(local, layer.prefix) - building_on.build(f"FROM {local}\n", local, platform, { - INSTALLED_LABEL: json.dumps(installed, sort_keys=True), EMULATED_LABEL: "true" if emulated else "false", - }, context="label") - stage = "push" - progress.at(pair, "pushing the image") - building_on.push(local, ref, store) - return {"status": "unverified", "key": key, "image": pin_digest(ref, store.auth(ref)), "reused": False, - "emulated": emulated, "builder": building_on.name, "lock": inputs, "installed": installed, - "built_at": _now()} - except builder.Skipped as e: - return {"status": "skipped", "reason": str(e), "skipped_at": _now()} - except builder.Unbuildable as e: - return _failed(env, form, arch, stage, key, str(e)) - except builder.ResolveError as e: - return _failed(env, form, arch, "resolve", None, str(e), log=e.log or None) - except DockerBuildError as e: - # Docker can list a platform it has lost the emulation for, until it restarts; the build says so. - unemulated = "exec format error" in str(e) - return _failed(env, form, arch, stage, key, f"the builder can't run linux/{arch} programs: its Docker has no " - "emulation for it" if unemulated else "the build failed", log=str(e)) - except subprocess.CalledProcessError as e: - return _failed(env, form, arch, stage, key, f"{' '.join(e.cmd[:2])} failed: {(e.stderr or '').strip()}") - except (RuntimeError, ValueError, OSError) as e: - return _failed(env, form, arch, stage, key, f"{type(e).__name__}: {e}") + pair = f"{form}/{arch}" + with (_attempting(env, form, arch) as attempt, tempfile.TemporaryDirectory(prefix="agentenv-compose-") as work, + contextlib.ExitStack() as opened): + context, fetched, from_store = Path(work) / builder.CONTEXT, {}, {} + context.mkdir() + source, base_identity = _base(base, arch) + layer = _stage_driver(context, driver, base, from_store) + apps = [_stage_app(context, app, base, from_store) for app in env.composed_apps()] + layers = [layer, *apps] + wants = _wants(layers, _stage_protocol(context)) + platform, held = f"linux/{arch}", [] + + def on() -> builder.Builder: + if not held: + link = builds.link(platform, base.image()) + native = platform in link.native_platforms() + progress.at(pair, "starting its builder", f"building natively {builder.where(link)}" if native else + f"building emulated, which is slow, {builder.where(link)}; " + f"{builder.instead_of_emulating(builds.provider, platform)}") + held.append(opened.enter_context(builds.open(link, platform, Path(work), base.image(), from_store))) + return held[0] + + inputs = packages.inputs_digest(wants, context, base_identity, arch) + lock = packages.stored(inputs, env.stored_id) if inputs is not None else None + if inputs is not None and lock is None: + attempt.stage = "resolve" + resolving = on() + progress.at(pair, "resolving packages") + lock, fetched = packages.resolve(inputs, wants, resolving, source, arch, env.stored_id) + progress.at(pair, "resolved", f"resolved {len(fetched)} files and stored them") + attempt.stage = "build" + dockerfile = _dockerfile(source, layers, base, _build_settings(env, base), lock) + key = attempt.key = _key(base=base_identity, dockerfile=dockerfile, context=_digest(context), + lock=lock.digest() if lock is not None else None, form=form, arch=arch, format=FORMAT) + store, ref, pinned = _stored(env, key) + if pinned is not None: + progress.at(pair, "reusing", "reusing the image of the same key, built earlier") + kept = labels(pinned, arch, store.auth(pinned)) + return {"status": "unverified", "key": key, "image": pinned, "reused": True, + "emulated": kept.get(EMULATED_LABEL) == "true", "lock": inputs, + "installed": json.loads(kept.get(INSTALLED_LABEL, "{}")), "built_at": _now()} + building_on = on() + progress.at(pair, "building the image") + if lock is not None: + packages.stage(lock, building_on, env.stored_id, fetched) + local, emulated = f"agentenv-computer:{key[7:23]}-{arch}", platform != building_on.platform + building_on.build(f"{dockerfile}LABEL {KEY_LABEL}={key}\n", local, platform, {KEY_LABEL: key}) + if reason := _compare(building_on, local, layers, pair, progress): + return _failed(env, form, arch, attempt.stage, key, reason) + installed = building_on.installed(local, layer.prefix) + building_on.build(f"FROM {local}\n", local, platform, { + INSTALLED_LABEL: json.dumps(installed, sort_keys=True), EMULATED_LABEL: "true" if emulated else "false", + }, context="label") + attempt.stage = "push" + progress.at(pair, "pushing the image") + building_on.push(local, ref, store) + return {"status": "unverified", "key": key, "image": pin_digest(ref, store.auth(ref)), "reused": False, + "emulated": emulated, "builder": building_on.name, "lock": inputs, "installed": installed, + "built_at": _now()} + return attempt.result def _built_disk(env: ComputerEnv, base: ComputerArtifact, arch: str, container: dict, builds: builder.Builds, @@ -342,62 +323,104 @@ def _built_disk(env: ComputerEnv, base: ComputerArtifact, arch: str, container: reused, or the stage that failed and why. Its key is of the container's key, the base disk and the disk recipe; the base disk is the base's own, the one published for it, or else one made on the builder from the base's container and the boot recipe. Its build runs no program of ``arch``'s but that recipe's.""" - stage, key, pair, platform = "build", None, f"disk/{arch}", f"linux/{arch}" + pair, platform = f"disk/{arch}", f"linux/{arch}" + with (_attempting(env, "disk", arch) as attempt, + tempfile.TemporaryDirectory(prefix="agentenv-compose-") as work, contextlib.ExitStack() as opened): + context = Path(work) / builder.CONTEXT + shutil.copytree(DISK / "boot", context) + shutil.copytree(DISK / "tools", context / "tools") + base_disk, base_identity, recipe = *_base_disk(base, arch), _digest(context) + key = attempt.key = _key(container=container["key"], base_disk=base_identity, recipe=recipe, form="disk", + arch=arch, format=FORMAT) + store, ref, pinned = _stored(env, key) + if pinned is not None: + progress.at(pair, "reusing", "reusing the disk of the same key, made earlier") + return {"status": "unverified", "key": key, "image": pinned, "reused": True, "built_at": _now()} + link = builds.link(platform, base.image()) + progress.at(pair, "starting its builder", f"making the disk {builder.where(link)}") + on = opened.enter_context(builds.open(link, platform, Path(work), base.image(), {})) + on.login(store.auth(container["image"])) + if base_disk is not None: + on.login(get_config().get_image_store().auth(base_disk)) + if (tools := published_pinned(DISK_TOOLS_IMAGE)) is None: + tools = f"agentenv-disk-tools:{DISK_TOOLS_IMAGE.tag()}" + on.build((context / "tools" / "Dockerfile").read_text(), tools, on.platform, {}, + context=f"{builder.CONTEXT}/tools") + dockerfile = (context / "Dockerfile").read_text() + if base_disk is None: + progress.at(pair, "making the base's disk") + base_disk = _made_base_disk(on, base, arch, f"{base_identity} {recipe}", tools, dockerfile) + progress.at(pair, "making the disk") + local = f"agentenv-computer:{key[7:23]}-disk-{arch}" + on.build(dockerfile, local, platform, {KEY_LABEL: key}, build_args={ + "ROOTFS": container["image"], "ROOTFS_CONFIG": _encoded(_config(on, container["image"], arch)), + "ESP": base_disk, "TOOLS": tools}) + attempt.stage = "push" + progress.at(pair, "pushing the disk") + on.push(local, ref, store) + return {"status": "unverified", "key": key, "image": pin_digest(ref, store.auth(ref)), "reused": False, + "builder": on.name, "built_at": _now()} + return attempt.result + + +def _made_base_disk(on: builder.Builder, base: ComputerArtifact, arch: str, made_from: str, tools: str, + dockerfile: str) -> str: + """The base's disk for ``arch``, made on ``on`` from the base's container with the boot recipe's ``dockerfile`` at + the base's size, under a tag of ``made_from``, what it's made from.""" + source, _ = _base(base, arch) + tag = f"agentenv-computer:{hashlib.sha256(made_from.encode()).hexdigest()[:16]}-base-disk-{arch}" + on.build(dockerfile, tag, f"linux/{arch}", {}, build_args={ + "ROOTFS": source, "ROOTFS_CONFIG": _encoded(_config(on, source, arch)), "TOOLS": tools, + "SIZE_GB": str(base.forms.disk.min_disk_gb)}) + return tag + + +@dataclass +class _Attempt: + """How far a pair's build got, its stage and key, and, when it failed, its result (``_attempting``).""" + + stage: str = "build" + key: str | None = None + result: dict | None = None + + +@contextlib.contextmanager +def _attempting(env: ComputerEnv, form: str, arch: str) -> Iterator[_Attempt]: + """A pair's build, whose failure becomes its result: skipped, or failed at the stage it reached, saying why.""" + attempt = _Attempt() try: - with tempfile.TemporaryDirectory(prefix="agentenv-compose-") as work, contextlib.ExitStack() as opened: - context = Path(work) / builder.CONTEXT - shutil.copytree(DISK / "boot", context) - shutil.copytree(DISK / "tools", context / "tools") - base_disk, base_identity, recipe = *_base_disk(base, arch), _digest(context) - key = "sha256:" + hashlib.sha256(json.dumps({ - "container": container["key"], "base_disk": base_identity, "recipe": recipe, "form": "disk", - "arch": arch, "format": FORMAT, - }, sort_keys=True).encode()).hexdigest() - store = get_config().get_image_store_for(env.stored_id) - store.ensure_repository(REPOSITORY) - ref = store.image_ref(REPOSITORY, key.removeprefix("sha256:")) - if (pinned := _served(ref, store)) is not None: - progress.at(pair, "reusing", "reusing the disk of the same key, made earlier") - return {"status": "unverified", "key": key, "image": pinned, "reused": True, "built_at": _now()} - link = builds.link(platform, base.image()) - progress.at(pair, "starting its builder", f"making the disk {builder.where(link)}") - on = opened.enter_context(builds.open(link, platform, Path(work), base.image(), {})) - on.login(store.auth(container["image"])) - if base_disk is not None: - on.login(get_config().get_image_store().auth(base_disk)) - if (tools := published_pinned(DISK_TOOLS_IMAGE)) is None: - tools = f"agentenv-disk-tools:{DISK_TOOLS_IMAGE.tag()}" - on.build((context / "tools" / "Dockerfile").read_text(), tools, on.platform, {}, - context=f"{builder.CONTEXT}/tools") - dockerfile = (context / "Dockerfile").read_text() - if base_disk is None: - progress.at(pair, "making the base's disk") - source, _ = _base(base, arch) - made = hashlib.sha256(f"{base_identity} {recipe}".encode()).hexdigest()[:16] - base_disk = f"agentenv-computer:{made}-base-disk-{arch}" - on.build(dockerfile, base_disk, platform, {}, build_args={ - "ROOTFS": source, "ROOTFS_CONFIG": _encoded(_config(on, source, arch)), "TOOLS": tools, - "SIZE_GB": str(base.forms.disk.min_disk_gb)}) - progress.at(pair, "making the disk") - local = f"agentenv-computer:{key[7:23]}-disk-{arch}" - on.build(dockerfile, local, platform, {KEY_LABEL: key}, build_args={ - "ROOTFS": container["image"], "ROOTFS_CONFIG": _encoded(_config(on, container["image"], arch)), - "ESP": base_disk, "TOOLS": tools}) - stage = "push" - progress.at(pair, "pushing the disk") - on.push(local, ref, store) - return {"status": "unverified", "key": key, "image": pin_digest(ref, store.auth(ref)), "reused": False, - "builder": on.name, "built_at": _now()} + yield attempt except builder.Skipped as e: - return {"status": "skipped", "reason": str(e), "skipped_at": _now()} + attempt.result = {"status": "skipped", "reason": str(e), "skipped_at": _now()} except builder.Unbuildable as e: - return _failed(env, "disk", arch, stage, key, str(e)) + attempt.result = _failed(env, form, arch, attempt.stage, attempt.key, str(e)) + except builder.ResolveError as e: + attempt.result = _failed(env, form, arch, "resolve", None, str(e), log=e.log or None) except DockerBuildError as e: - return _failed(env, "disk", arch, stage, key, "the build failed", log=str(e)) + # Docker can list a platform it has lost the emulation for, until it restarts; the build says so. + unemulated = "exec format error" in str(e) + attempt.result = _failed(env, form, arch, attempt.stage, attempt.key, + f"the builder can't run linux/{arch} programs: its Docker has no emulation for it" + if unemulated else "the build failed", log=str(e)) except subprocess.CalledProcessError as e: - return _failed(env, "disk", arch, stage, key, f"{' '.join(e.cmd[:2])} failed: {(e.stderr or '').strip()}") + attempt.result = _failed(env, form, arch, attempt.stage, attempt.key, + f"{' '.join(e.cmd[:2])} failed: {(e.stderr or '').strip()}") except (RuntimeError, ValueError, OSError) as e: - return _failed(env, "disk", arch, stage, key, f"{type(e).__name__}: {e}") + attempt.result = _failed(env, form, arch, attempt.stage, attempt.key, f"{type(e).__name__}: {e}") + + +def _key(**inputs: Any) -> str: + """A build's key: a digest of everything that goes into it.""" + return "sha256:" + hashlib.sha256(json.dumps(inputs, sort_keys=True).encode()).hexdigest() + + +def _stored(env: ComputerEnv, key: str) -> tuple[Any, str, str | None]: + """The env's image store, the ref there of the image of ``key``, and that ref pinned when the store serves one + already, for any env that built it.""" + store = get_config().get_image_store_for(env.stored_id) + store.ensure_repository(REPOSITORY) + ref = store.image_ref(REPOSITORY, key.removeprefix("sha256:")) + return store, ref, _served(ref, store) def _base_disk(base: ComputerArtifact, arch: str) -> tuple[str | None, str]: From 746d12e0947f0f97378a13d02da203c85debf532 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 22:17:35 -0800 Subject: [PATCH 19/21] refactor(computer): gate 3 in a module of its own, the disk's deploy preconditions beside deployable_builds probing.py holds the probe a compose runs and the first-boot probe a deploy of an unverified disk runs, so the computer env provider no longer imports the composer; composed.py holds the composed record they both write. The env's values of its base's settings are ComputerEnv.computer_values, and the refusal of a disk larger than its base makes it is ComputerEnv.disk_size_refusal, next to the statuses a deploy may boot. Behaviour is unchanged. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- src/agent_env/env/envs/computer/env.py | 12 ++ .../env/envs/computer/utils/composed.py | 90 ++++++++ .../env/envs/computer/utils/composer.py | 196 ++---------------- .../env/envs/computer/utils/probing.py | 90 ++++++++ .../env_providers/env_computer_provider.py | 7 +- tst/unit/env/envs/conftest.py | 4 +- tst/unit/env/envs/test_computer_disk.py | 4 +- tst/unit/env/envs/test_computer_env.py | 18 +- 9 files changed, 232 insertions(+), 191 deletions(-) create mode 100644 src/agent_env/env/envs/computer/utils/composed.py create mode 100644 src/agent_env/env/envs/computer/utils/probing.py diff --git a/AGENTS.md b/AGENTS.md index 0bb899864..0fd951ba4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -116,7 +116,7 @@ and deserialized through a registry. | Package | Contents | |---|---| -| `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]` (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`, `bundle register` and `run`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register that reuses a computer env composes what it has left (`pending`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference, over its env.toml's `[settings]` and a computer env's `settings` beside it in `apps`, each by top-level field (`params.app_settings`; checked at put, a required one left to the deploy; the deploy record keeps the merged values as `app_settings`, secrets withheld); its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | +| `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]` (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`, `bundle register` and `run`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register that reuses a computer env composes what it has left (`pending`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 (`utils/probing.py`) then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference, over its env.toml's `[settings]` and a computer env's `settings` beside it in `apps`, each by top-level field (`params.app_settings`; checked at put, a required one left to the deploy; the deploy record keeps the merged values as `app_settings`, secrets withheld); its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Every artifact but an environment universe, whose `metadata` key holds its legacy metadata file ids, carries a `metadata` dict outside its versioned fields, changed by compare-and-swap (`merge_metadata`). Built-ins: `CliArtifact` (`cli`), `ComputerArtifact` (`computer`, a base computer: its architectures, OS, session user, settings and container form, whose image `put` reads from its registry into `metadata.validated_images`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`: its environment's files by path, stored as the one wrapped `file_artifact_ref` every environment artifact had when it has one file at its filename, else as `file_artifact_refs`; `get_file_artifacts()` reads either, and a load hands each file to the env's `data/add` with its path, which a pre-v1 load, taking one file, refuses), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | | `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, `start_computer_recording` and `end_computer_recording`, and more). On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the `@local` namespace: while the CLI or a `build_app` server runs, an `@local/…` id and what an `@local` task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | diff --git a/src/agent_env/env/envs/computer/env.py b/src/agent_env/env/envs/computer/env.py index 6c4ebcdce..a82808e90 100644 --- a/src/agent_env/env/envs/computer/env.py +++ b/src/agent_env/env/envs/computer/env.py @@ -162,6 +162,11 @@ def app_defaults(self) -> dict[str, dict[str, Any]]: return {app.environment_name: {**app.settings, **self.app_settings.get(app.environment_name, {})} for app in self.composed_apps()} + def computer_values(self) -> dict[str, Any]: + """The env's values of its base's settings: its ``computer_settings``, and its display as the resolution.""" + display = self.display + return {**self.computer_settings, **({"resolution": f"{display['width']}x{display['height']}"} if display else {})} + def computer_artifact(self) -> ComputerArtifact: """The env's computer: its base composed with its driver, over the images built.""" return ComputerArtifact.get(self.metadata["computer"]["id"], self.metadata["computer"]["version"]) @@ -174,6 +179,13 @@ def deployable_builds(self, form: Optional[str] = None) -> list[tuple[str, str, return [(name, arch, composed[name][arch]) for name in gate.forms(self.computer) if form in (None, name) for arch in arches if composed.get(name, {}).get(arch, {}).get("status") in _DEPLOYABLE[name]] + def disk_size_refusal(self, form: str, disk_size_gb: float) -> Optional[str]: + """Why a deploy of ``form`` can't have ``disk_size_gb``: a disk is the size its base makes it, which a deploy + doesn't enlarge.""" + if form == "disk" and disk_size_gb > (size := self.computer.forms.disk.min_disk_gb): + return f"{self.id}'s disk is {size} GB, which a deploy doesn't enlarge, and it asks for {disk_size_gb:g} GB" + return None + def build_statuses(self) -> dict[str, str]: return {f"{form}/{arch}": result.get("status") for form, by_arch in self.metadata.get("composed", {}).items() for arch, result in by_arch.items()} diff --git a/src/agent_env/env/envs/computer/utils/composed.py b/src/agent_env/env/envs/computer/utils/composed.py new file mode 100644 index 000000000..4d4d924e7 --- /dev/null +++ b/src/agent_env/env/envs/computer/utils/composed.py @@ -0,0 +1,90 @@ +"""A computer env's ``composed`` metadata: each build's result by form and arch, recorded as it finishes.""" + +from __future__ import annotations + +import copy +import threading +from datetime import datetime, timezone +from typing import TYPE_CHECKING + +from agent_env.artifact.artifacts.file import FileArtifact +from agent_env.env.env import Env +from agent_env.store.base import ConcurrentModificationError +from agent_env.store.ids import derive_id + +if TYPE_CHECKING: + from agent_env.env.envs.computer.env import ComputerEnv + +# A pair's statuses once its image is in the store. +BUILT = ("unverified", "ready") +_WRITE_TRIES = 4 + + +class Record: + """The env's ``composed`` metadata, by form and arch. Each result is merged into the stored table pair by pair, the + table read again when another build wrote it first, so builds of one env at once keep each other's pairs, and a + result never undoes one that got further (``_rank``).""" + + def __init__(self, env: ComputerEnv): + self.env, self.lock = env, threading.Lock() + + def started(self, pairs: list[tuple[str, str]]) -> None: + at = now() + self._merge({pair: {"status": "building", "started_at": at} for pair in pairs}) + + def finished(self, form: str, arch: str, result: dict) -> None: + self._merge({(form, arch): result}) + + def built(self, form: str = "container") -> dict[str, str]: + """Each arch's image of ``form``, of the pairs built or reused.""" + return {arch: result["image"] for arch, result in self.env.metadata.get("composed", {}).get(form, {}).items() + if result.get("status") in BUILT} + + def computer(self, ref: dict | None) -> None: + """Record the env's computer, with the images it was written over, the disks' apart.""" + if ref is not None: + with self.lock: + disks = self.built("disk") + self.env.merge_metadata({"computer": {**ref, "images": self.built(), **({"disks": disks} if disks else {})}}) + + def _merge(self, results: dict[tuple[str, str], dict]) -> None: + with self.lock: + for attempt in range(_WRITE_TRIES): + composed = copy.deepcopy(self.env.metadata.get("composed", {})) + for (form, arch), result in results.items(): + stored = composed.setdefault(form, {}).get(arch, {}) + if _rank(result) >= _rank(stored): + composed[form][arch] = {"started_at": stored.get("started_at"), **result} + try: + self.env.update_metadata({**self.env.metadata, "composed": composed}) + return + except ConcurrentModificationError: + if attempt == _WRITE_TRIES - 1: + raise + self.env.metadata = Env.get(self.env.stored_id, self.env.version).metadata + + +def failed(env: ComputerEnv, form: str, arch: str, stage: str, key: str | None, reason: str, + log: str | None = None) -> dict: + result = {"status": "failed", "stage": stage, "reason": reason, "failed_at": now()} + if key is not None: + result["key"] = key + if log is not None: + stored = FileArtifact.put_bytes(derive_id(env.stored_id, f"build_log_{form}_{arch}"), description=( + f"the {form}/{arch} build of {env.stored_id} v{env.version}, which failed"), filename="build.log", + content=log.encode(), content_type="text/plain") + result["log"] = {"id": stored.id, "version": stored.version} + result["reason"] = f"{reason}: {log.strip().splitlines()[-1] if log.strip() else 'no output'}" + return result + + +def now() -> str: + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def _rank(result: dict) -> int: + """How far a pair got: ready, then built and either unprobed or failed at its probe, then anything short of an + image. A result takes a stored one's place only when it got at least as far.""" + if result.get("status") == "ready": + return 2 + return 1 if result.get("status") == "unverified" or result.get("stage") == "probe" else 0 diff --git a/src/agent_env/env/envs/computer/utils/composer.py b/src/agent_env/env/envs/computer/utils/composer.py index cbde770c8..e0c4112c0 100644 --- a/src/agent_env/env/envs/computer/utils/composer.py +++ b/src/agent_env/env/envs/computer/utils/composer.py @@ -11,7 +11,6 @@ import asyncio import base64 import contextlib -import copy import hashlib import importlib.metadata import json @@ -25,36 +24,30 @@ import urllib.parse from concurrent.futures import Future, ThreadPoolExecutor, as_completed from dataclasses import dataclass -from datetime import datetime, timezone from pathlib import Path, PurePosixPath from collections.abc import Callable, Iterator from typing import TYPE_CHECKING, Any from agentenv_protocol.computer import apps as installed_apps from agentenv_protocol.computer.app import BUILD_SETTINGS, SOCKET -from agentenv_protocol.computer.conformance import run_probe from agent_env.artifact.artifacts.computer import ComputerArtifact from agent_env.artifact.artifacts.docker_image import DockerImageArtifact -from agent_env.artifact.artifacts.file import FileArtifact from agent_env.artifact.artifacts.file_artifact_universe import FileArtifactUniverse from agent_env.config import get_config from agent_env.entity.export import Unwritable, checked_path -from agent_env.env.env import Env -from agent_env.env.envs.computer.utils import boot, builder, clashes, gate, packages +from agent_env.env.envs.computer.utils import boot, builder, clashes, gate, packages, probing +from agent_env.env.envs.computer.utils.composed import BUILT, Record, failed, now from agent_env.env.envs.computer.utils.install import Install from agent_env.env.published_images import DISK, DISK_TOOLS_IMAGE, published_disk, published_pinned from agent_env.providers.sandbox_providers.host_images import platform_tag -from agent_env.store.base import ConcurrentModificationError from agent_env.store.ids import derive_id -from agent_env.store.local_state import awaiting_lock from agent_env.store.image_store.image_contents import image_config, labels from agent_env.store.image_store.oci_registry_credentials import names_registry from agent_env.store.image_store.registry_api import NoSuchImageError, pin_digest, write_index from agent_env.utils.docker.build import DockerBuildError if TYPE_CHECKING: - from agentenv_protocol.computer.conformance import ProbeReport from agent_env.providers.sandbox_providers.sandbox_provider import SandboxProvider from agent_env.env.envs.computer.env import ComputerEnv @@ -76,13 +69,6 @@ KEY_LABEL = "dev.agentenv.computer-key" # What a build installed and whether it was emulated, kept on its image for an env that reuses it. INSTALLED_LABEL, EMULATED_LABEL = "dev.agentenv.installed", "dev.agentenv.emulated" -# A pair's statuses once its image is in the store. -_BUILT = ("unverified", "ready") -_WRITE_TRIES = 4 -#: How long a probe's computer may run before its sandbox provider stops it. -_PROBE_TTL_S = 1800 -#: How long the probe at full may take on a computer that booted: about ten seconds on a healthy one. -_PROBE_TIMEOUT_S = 120 #: How often a compose says a pair is still at a stage, which can take most of an hour under emulation. _HEARTBEAT_S = 600 _PROTOCOL = "agentenv-framework-protocol" @@ -115,7 +101,7 @@ def compose(env: ComputerEnv, pairs: list[tuple[str, str]] | None = None, compos builds = builder.Builds(provider, env.stored_id, REPOSITORY, composing.emulate) pairs = pairs if pairs is not None else [(form, arch) for form in gate.forms(base) for arch in gate.arches(base, driver, env.composed_apps())] - record = _Record(env) + record = Record(env) record.started(pairs) with _Progress(env, composing.on_progress) as progress, ThreadPoolExecutor(max_workers=max(len(pairs), 1)) as pool: progress.say(f"composing {', '.join(f'{form}/{arch}' for form, arch in pairs)} on " @@ -136,7 +122,7 @@ def compose(env: ComputerEnv, pairs: list[tuple[str, str]] | None = None, compos def current(env: ComputerEnv) -> bool: """Whether the env's computer was written over every pair built, as a put or env build that failed writing it leaves it not.""" - computer, record = env.metadata.get("computer"), _Record(env) + computer, record = env.metadata.get("computer"), Record(env) return computer is not None and computer.get("images") == record.built() and computer.get( "disks", {}) == record.built("disk") @@ -151,7 +137,7 @@ def pending(env: ComputerEnv, sandbox_provider: SandboxProvider | None = None) - for form in gate.forms(env.computer) for arch in gate.arches(env.computer, env.driver, env.composed_apps())} return [(form, arch) for (form, arch), status in statuses.items() - if status not in _BUILT or (status == "unverified" and boot.native_links(provider, arch, form))] + if status not in BUILT or (status == "unverified" and boot.native_links(provider, arch, form))] def _composed(env: ComputerEnv, base: ComputerArtifact, driver: ComputerDriverEnv, form: str, arch: str, @@ -169,86 +155,24 @@ def _composed_disk(env: ComputerEnv, base: ComputerArtifact, driver: ComputerDri when it built (gate 3).""" built = container.result() if container is not None else env.metadata.get("composed", {}).get( "container", {}).get(arch, {}) - if built.get("status") not in _BUILT: + if built.get("status") not in BUILT: return {"status": "skipped", "reason": f"its container form didn't build ({built.get('status', 'never')})", - "skipped_at": _now()} + "skipped_at": now()} result = _built_disk(env, base, arch, built, builds, progress) if result["status"] != "unverified": return result return _probed(env, driver, "disk", arch, result, builds.provider, progress) -async def probe_first_boot(env: ComputerEnv, form: str, arch: str, sandbox_provider: SandboxProvider) -> dict: - """Gate 3 for a build its compose couldn't probe, on its first boot on ``sandbox_provider``, which boots it - natively: the result, recorded. Deploys of one build at once, in this process or another, probe it once: each - waits for the one probing, then takes the result it recorded, a failure's too, which keeps the build's key.""" - build = env.metadata["composed"][form][arch] - async with awaiting_lock(f"computer-first-boot:{env.stored_id}:{form}:{arch}:{build['image']}"): - recorded = (await asyncio.to_thread(Env.get, env.stored_id, env.version)).metadata["composed"][form][arch] - if recorded.get("key") == build["key"] and recorded["status"] != "unverified": - return recorded - result = await _checked([sandbox_provider], env, env.driver, form, arch, recorded) - await asyncio.to_thread(_Record(env).finished, form, arch, result) - return result - - def _probed(env: ComputerEnv, driver: ComputerDriverEnv, form: str, arch: str, result: dict, sandbox_provider: SandboxProvider, progress: _Progress) -> dict: """Gate 3: the build of ``result`` booted on the first link of ``sandbox_provider`` that runs ``arch``'s ``form`` - natively, and probed at ``full`` (``_checked``). Still unverified, saying why, when no link does.""" + natively, and probed at ``full`` (``probing.checked``). Still unverified, saying why, when no link does.""" if not (links := boot.native_links(sandbox_provider, arch, form)): how = f"boots linux/{arch} disks" if form == "disk" else f"runs linux/{arch}" return {**result, "reason": f"no link of the sandbox provider {how} natively to probe it"} progress.at(f"{form}/{arch}", "probing", f"probing on {type(links[0]).__name__}") - return asyncio.run(_checked(links, env, driver, form, arch, result)) - - -async def _checked(links: list[SandboxProvider], env: ComputerEnv, driver: ComputerDriverEnv, form: str, arch: str, - result: dict) -> dict: - """The build of ``result`` booted on the first of ``links`` that boots it and probed at ``full``: ready, with the - driver's card, or failed at the probe; each with a summary of the probe and its report stored.""" - try: - report, card, provider = await _probe(links, env, driver, result["image"], form) - except TimeoutError: - return _failed(env, form, arch, "probe", result["key"], f"the probe didn't finish within {_PROBE_TIMEOUT_S} s") - except Exception as e: - return _failed(env, form, arch, "probe", result["key"], f"it didn't boot: {e}") - stored = FileArtifact.put_bytes(derive_id(env.stored_id, f"probe_report_{form}_{arch}"), description=( - f"the {form}/{arch} probe of {env.stored_id} v{env.version}"), filename="probe.json", - content=report.model_dump_json(indent=1).encode(), content_type="application/json") - probe = {"level": report.level, "provider": provider, "passed": report.passed, "checks": report.counts(), - "extensions": report.extensions, "report": {"id": stored.id, "version": stored.version}, - "probed_at": _now()} - if not report.passed: - failing = "; ".join(f"{c.check} {c.name}: {c.detail}" for c in report.checks if c.status == "failed") - return {**result, "status": "failed", "stage": "probe", "reason": f"the probe failed: {failing}", - "failed_at": _now(), "probe": probe} - return {**result, "status": "ready", "card": card, "probe": probe} - - -async def _probe(links: list[SandboxProvider], env: ComputerEnv, driver: ComputerDriverEnv, image: str, - form: str = "container") -> tuple[ProbeReport, dict, str]: - """``image``, of ``form``, booted at the env's settings and its apps' defaults with a control token of its own, as - a deploy with no ``env_params`` boots it, on the first of ``links`` that boots it, probed at ``full`` with that - token within ``_PROBE_TIMEOUT_S`` and torn down; the report, the driver's card and the provider's name.""" - token = boot.control_token() - environment = {**boot.settings_environment(env.computer, _values(env), environment_name=env.environment_name), - **boot.app_settings_environment(env.composed_apps(), env.app_defaults(), env.computer.os.type)} - problems = [] - for link in links: - try: - booted = await boot.boot(link, image=image, port=driver.port, environment=environment, token=token, - network_policy=None, timeout=_PROBE_TTL_S, attribution=None, - ready_s=boot.ready_timeout(env.composed_apps(), env.computer.os.type), form=form) - except Exception as e: - problems.append(f"{type(link).__name__}: {e}") - continue - try: - async with asyncio.timeout(_PROBE_TIMEOUT_S): - return await run_probe(booted.url, "full", token=token), booted.card, type(link).__name__ - finally: - await boot.discard(booted.sandbox) - raise RuntimeError("; ".join(problems)) + return asyncio.run(probing.checked(links, env, driver, form, arch, result)) def _built(env: ComputerEnv, base: ComputerArtifact, driver: ComputerDriverEnv, form: str, arch: str, @@ -295,7 +219,7 @@ def on() -> builder.Builder: kept = labels(pinned, arch, store.auth(pinned)) return {"status": "unverified", "key": key, "image": pinned, "reused": True, "emulated": kept.get(EMULATED_LABEL) == "true", "lock": inputs, - "installed": json.loads(kept.get(INSTALLED_LABEL, "{}")), "built_at": _now()} + "installed": json.loads(kept.get(INSTALLED_LABEL, "{}")), "built_at": now()} building_on = on() progress.at(pair, "building the image") if lock is not None: @@ -303,7 +227,7 @@ def on() -> builder.Builder: local, emulated = f"agentenv-computer:{key[7:23]}-{arch}", platform != building_on.platform building_on.build(f"{dockerfile}LABEL {KEY_LABEL}={key}\n", local, platform, {KEY_LABEL: key}) if reason := _compare(building_on, local, layers, pair, progress): - return _failed(env, form, arch, attempt.stage, key, reason) + return failed(env, form, arch, attempt.stage, key, reason) installed = building_on.installed(local, layer.prefix) building_on.build(f"FROM {local}\n", local, platform, { INSTALLED_LABEL: json.dumps(installed, sort_keys=True), EMULATED_LABEL: "true" if emulated else "false", @@ -313,7 +237,7 @@ def on() -> builder.Builder: building_on.push(local, ref, store) return {"status": "unverified", "key": key, "image": pin_digest(ref, store.auth(ref)), "reused": False, "emulated": emulated, "builder": building_on.name, "lock": inputs, "installed": installed, - "built_at": _now()} + "built_at": now()} return attempt.result @@ -335,7 +259,7 @@ def _built_disk(env: ComputerEnv, base: ComputerArtifact, arch: str, container: store, ref, pinned = _stored(env, key) if pinned is not None: progress.at(pair, "reusing", "reusing the disk of the same key, made earlier") - return {"status": "unverified", "key": key, "image": pinned, "reused": True, "built_at": _now()} + return {"status": "unverified", "key": key, "image": pinned, "reused": True, "built_at": now()} link = builds.link(platform, base.image()) progress.at(pair, "starting its builder", f"making the disk {builder.where(link)}") on = opened.enter_context(builds.open(link, platform, Path(work), base.image(), {})) @@ -359,7 +283,7 @@ def _built_disk(env: ComputerEnv, base: ComputerArtifact, arch: str, container: progress.at(pair, "pushing the disk") on.push(local, ref, store) return {"status": "unverified", "key": key, "image": pin_digest(ref, store.auth(ref)), "reused": False, - "builder": on.name, "built_at": _now()} + "builder": on.name, "built_at": now()} return attempt.result @@ -391,22 +315,22 @@ def _attempting(env: ComputerEnv, form: str, arch: str) -> Iterator[_Attempt]: try: yield attempt except builder.Skipped as e: - attempt.result = {"status": "skipped", "reason": str(e), "skipped_at": _now()} + attempt.result = {"status": "skipped", "reason": str(e), "skipped_at": now()} except builder.Unbuildable as e: - attempt.result = _failed(env, form, arch, attempt.stage, attempt.key, str(e)) + attempt.result = failed(env, form, arch, attempt.stage, attempt.key, str(e)) except builder.ResolveError as e: - attempt.result = _failed(env, form, arch, "resolve", None, str(e), log=e.log or None) + attempt.result = failed(env, form, arch, "resolve", None, str(e), log=e.log or None) except DockerBuildError as e: # Docker can list a platform it has lost the emulation for, until it restarts; the build says so. unemulated = "exec format error" in str(e) - attempt.result = _failed(env, form, arch, attempt.stage, attempt.key, + attempt.result = failed(env, form, arch, attempt.stage, attempt.key, f"the builder can't run linux/{arch} programs: its Docker has no emulation for it" if unemulated else "the build failed", log=str(e)) except subprocess.CalledProcessError as e: - attempt.result = _failed(env, form, arch, attempt.stage, attempt.key, + attempt.result = failed(env, form, arch, attempt.stage, attempt.key, f"{' '.join(e.cmd[:2])} failed: {(e.stderr or '').strip()}") except (RuntimeError, ValueError, OSError) as e: - attempt.result = _failed(env, form, arch, attempt.stage, attempt.key, f"{type(e).__name__}: {e}") + attempt.result = failed(env, form, arch, attempt.stage, attempt.key, f"{type(e).__name__}: {e}") def _key(**inputs: Any) -> str: @@ -598,16 +522,10 @@ def _protocol_source() -> Path | None: def _build_settings(env: ComputerEnv, base: ComputerArtifact) -> dict[str, Any]: """The env's values of the base's build-phase settings, which the composed image fixes.""" - values = _values(env) + values = env.computer_values() return {name: values[name] for name, setting in base.config.items() if setting.phase == "build" and name in values} -def _values(env: ComputerEnv) -> dict[str, Any]: - """The env's values of its base's settings: its ``computer_settings``, and its display as the resolution.""" - display = env.display - return {**env.computer_settings, **({"resolution": f"{display['width']}x{display['height']}"} if display else {})} - - def _dockerfile(source: str, layers: list[_Layer], base: ComputerArtifact, build_settings: dict, lock: packages.Lock | None) -> str: """The composed image's Dockerfile but its key's label, which the key is a digest of: the build-phase settings, the @@ -686,20 +604,6 @@ def _served(ref: str, store) -> str | None: return None -def _failed(env: ComputerEnv, form: str, arch: str, stage: str, key: str | None, reason: str, - log: str | None = None) -> dict: - result = {"status": "failed", "stage": stage, "reason": reason, "failed_at": _now()} - if key is not None: - result["key"] = key - if log is not None: - stored = FileArtifact.put_bytes(derive_id(env.stored_id, f"build_log_{form}_{arch}"), description=( - f"the {form}/{arch} build of {env.stored_id} v{env.version}, which failed"), filename="build.log", - content=log.encode(), content_type="text/plain") - result["log"] = {"id": stored.id, "version": stored.version} - result["reason"] = f"{reason}: {log.strip().splitlines()[-1] if log.strip() else 'no output'}" - return result - - def _computer(env: ComputerEnv, base: ComputerArtifact, built: dict[str, str], disks: dict[str, str]) -> dict | None: """Write the env's computer over the images ``built``, by arch, joined into one index in the env's image store, signed in to it: its arches those, its OS and session the base's, its settings the base's with the env's values as their @@ -709,7 +613,7 @@ def _computer(env: ComputerEnv, base: ComputerArtifact, built: dict[str, str], d arches = [arch for arch in base.arch if arch in built] image = _joined(env, "computer_image", "composed computer", built, arches) config = {name: setting.model_dump() for name, setting in base.config.items()} - for name, value in _values(env).items(): + for name, value in env.computer_values().items(): config[name]["default"] = value forms = {"container": {"image": {"id": image.id, "version": image.version}, "init": base.forms.container.init, "input_channel": base.forms.container.input_channel}} @@ -746,54 +650,6 @@ def _digest(folder: Path) -> str: return "sha256:" + hashed.hexdigest() -def _now() -> str: - return datetime.now(timezone.utc).isoformat(timespec="seconds") - - -class _Record: - """The env's ``composed`` metadata, by form and arch. Each result is merged into the stored table pair by pair, the - table read again when another build wrote it first, so builds of one env at once keep each other's pairs, and a - result never undoes one that got further (``_rank``).""" - - def __init__(self, env: ComputerEnv): - self.env, self.lock = env, threading.Lock() - - def started(self, pairs: list[tuple[str, str]]) -> None: - now = _now() - self._merge({pair: {"status": "building", "started_at": now} for pair in pairs}) - - def finished(self, form: str, arch: str, result: dict) -> None: - self._merge({(form, arch): result}) - - def built(self, form: str = "container") -> dict[str, str]: - """Each arch's image of ``form``, of the pairs built or reused.""" - return {arch: result["image"] for arch, result in self.env.metadata.get("composed", {}).get(form, {}).items() - if result.get("status") in _BUILT} - - def computer(self, ref: dict | None) -> None: - """Record the env's computer, with the images it was written over, the disks' apart.""" - if ref is not None: - with self.lock: - disks = self.built("disk") - self.env.merge_metadata({"computer": {**ref, "images": self.built(), **({"disks": disks} if disks else {})}}) - - def _merge(self, results: dict[tuple[str, str], dict]) -> None: - with self.lock: - for attempt in range(_WRITE_TRIES): - composed = copy.deepcopy(self.env.metadata.get("composed", {})) - for (form, arch), result in results.items(): - stored = composed.setdefault(form, {}).get(arch, {}) - if _rank(result) >= _rank(stored): - composed[form][arch] = {"started_at": stored.get("started_at"), **result} - try: - self.env.update_metadata({**self.env.metadata, "composed": composed}) - return - except ConcurrentModificationError: - if attempt == _WRITE_TRIES - 1: - raise - self.env.metadata = Env.get(self.env.stored_id, self.env.version).metadata - - class _Progress: """What a compose reports as it goes, to ``on_progress`` and the log: a line as each pair reaches a stage, one each ``_HEARTBEAT_S`` a stage goes on, and how each pair ended, with the time since the compose began.""" @@ -848,11 +704,3 @@ def _beat(self) -> None: def _took(seconds: float) -> str: minutes, seconds = divmod(round(seconds), 60) return f"{minutes} min {seconds} s" if minutes else f"{seconds} s" - - -def _rank(result: dict) -> int: - """How far a pair got: ready, then built and either unprobed or failed at its probe, then anything short of an - image. A result takes a stored one's place only when it got at least as far.""" - if result.get("status") == "ready": - return 2 - return 1 if result.get("status") == "unverified" or result.get("stage") == "probe" else 0 diff --git a/src/agent_env/env/envs/computer/utils/probing.py b/src/agent_env/env/envs/computer/utils/probing.py new file mode 100644 index 000000000..c5df75333 --- /dev/null +++ b/src/agent_env/env/envs/computer/utils/probing.py @@ -0,0 +1,90 @@ +"""Gate 3: a computer env's build booted on a provider that runs it natively and probed at ``full``, by its compose or, +for a build its compose couldn't boot, on its first deploy.""" + +from __future__ import annotations + +import asyncio +from typing import TYPE_CHECKING + +from agentenv_protocol.computer.conformance import run_probe + +from agent_env.artifact.artifacts.file import FileArtifact +from agent_env.env.env import Env +from agent_env.env.envs.computer.utils import boot +from agent_env.env.envs.computer.utils.composed import Record, failed, now +from agent_env.store.ids import derive_id +from agent_env.store.local_state import awaiting_lock + +if TYPE_CHECKING: + from agentenv_protocol.computer.conformance import ProbeReport + + from agent_env.env.envs.computer.driver_env import ComputerDriverEnv + from agent_env.env.envs.computer.env import ComputerEnv + from agent_env.providers.sandbox_providers.sandbox_provider import SandboxProvider + +#: How long a probe's computer may run before its sandbox provider stops it. +_PROBE_TTL_S = 1800 +#: How long the probe at full may take on a computer that booted: about ten seconds on a healthy one. +_PROBE_TIMEOUT_S = 120 + + +async def probe_first_boot(env: ComputerEnv, form: str, arch: str, sandbox_provider: SandboxProvider) -> dict: + """Gate 3 for a build its compose couldn't probe, on its first boot on ``sandbox_provider``, which boots it + natively: the result, recorded. Deploys of one build at once, in this process or another, probe it once: each + waits for the one probing, then takes the result it recorded, a failure's too, which keeps the build's key.""" + build = env.metadata["composed"][form][arch] + async with awaiting_lock(f"computer-first-boot:{env.stored_id}:{form}:{arch}:{build['image']}"): + recorded = (await asyncio.to_thread(Env.get, env.stored_id, env.version)).metadata["composed"][form][arch] + if recorded.get("key") == build["key"] and recorded["status"] != "unverified": + return recorded + result = await checked([sandbox_provider], env, env.driver, form, arch, recorded) + await asyncio.to_thread(Record(env).finished, form, arch, result) + return result + + +async def checked(links: list[SandboxProvider], env: ComputerEnv, driver: ComputerDriverEnv, form: str, arch: str, + result: dict) -> dict: + """The build of ``result`` booted on the first of ``links`` that boots it and probed at ``full``: ready, with the + driver's card, or failed at the probe; each with a summary of the probe and its report stored.""" + try: + report, card, provider = await _probe(links, env, driver, result["image"], form) + except TimeoutError: + return failed(env, form, arch, "probe", result["key"], f"the probe didn't finish within {_PROBE_TIMEOUT_S} s") + except Exception as e: + return failed(env, form, arch, "probe", result["key"], f"it didn't boot: {e}") + stored = FileArtifact.put_bytes(derive_id(env.stored_id, f"probe_report_{form}_{arch}"), description=( + f"the {form}/{arch} probe of {env.stored_id} v{env.version}"), filename="probe.json", + content=report.model_dump_json(indent=1).encode(), content_type="application/json") + probe = {"level": report.level, "provider": provider, "passed": report.passed, "checks": report.counts(), + "extensions": report.extensions, "report": {"id": stored.id, "version": stored.version}, + "probed_at": now()} + if not report.passed: + failing = "; ".join(f"{c.check} {c.name}: {c.detail}" for c in report.checks if c.status == "failed") + return {**result, "status": "failed", "stage": "probe", "reason": f"the probe failed: {failing}", + "failed_at": now(), "probe": probe} + return {**result, "status": "ready", "card": card, "probe": probe} + + +async def _probe(links: list[SandboxProvider], env: ComputerEnv, driver: ComputerDriverEnv, image: str, + form: str = "container") -> tuple[ProbeReport, dict, str]: + """``image``, of ``form``, booted at the env's settings and its apps' defaults with a control token of its own, as + a deploy with no ``env_params`` boots it, on the first of ``links`` that boots it, probed at ``full`` with that + token within ``_PROBE_TIMEOUT_S`` and torn down; the report, the driver's card and the provider's name.""" + token = boot.control_token() + environment = {**boot.settings_environment(env.computer, env.computer_values(), environment_name=env.environment_name), + **boot.app_settings_environment(env.composed_apps(), env.app_defaults(), env.computer.os.type)} + problems = [] + for link in links: + try: + booted = await boot.boot(link, image=image, port=driver.port, environment=environment, token=token, + network_policy=None, timeout=_PROBE_TTL_S, attribution=None, + ready_s=boot.ready_timeout(env.composed_apps(), env.computer.os.type), form=form) + except Exception as e: + problems.append(f"{type(link).__name__}: {e}") + continue + try: + async with asyncio.timeout(_PROBE_TIMEOUT_S): + return await run_probe(booted.url, "full", token=token), booted.card, type(link).__name__ + finally: + await boot.discard(booted.sandbox) + raise RuntimeError("; ".join(problems)) diff --git a/src/agent_env/providers/env_providers/env_computer_provider.py b/src/agent_env/providers/env_providers/env_computer_provider.py index 9fd9140ee..049592011 100644 --- a/src/agent_env/providers/env_providers/env_computer_provider.py +++ b/src/agent_env/providers/env_providers/env_computer_provider.py @@ -14,7 +14,7 @@ from agent_env.attribution import Attribution from agent_env.env.env import DeployedSandboxEnv, EnvCapabilityUnsupported from agent_env.env.envs.computer.utils import params as computer_params -from agent_env.env.envs.computer.utils.composer import probe_first_boot +from agent_env.env.envs.computer.utils.probing import probe_first_boot from agent_env.env.envs.computer.utils.boot import ( app_settings_environment, boot, @@ -94,9 +94,8 @@ async def _boot(self, env: ComputerEnv, builds: list[tuple[str, str, dict]], par raise ValueError(f"{type(sandbox_provider).__name__} runs none of {env.id}'s ready builds natively " f"({', '.join(f'{form}/{arch}' for form, arch, _ in builds)})") form, arch, build = native[0] - if form == "disk" and disk_size_gb > (size := env.computer.forms.disk.min_disk_gb): - raise ValueError(f"{env.id}'s disk is {size} GB, which a deploy doesn't enlarge, and it asks for " - f"{disk_size_gb:g} GB") + if refusal := env.disk_size_refusal(form, disk_size_gb): + raise ValueError(refusal) if build["status"] == "unverified": build = await probe_first_boot(env, form, arch, sandbox_provider) if build["status"] != "ready": diff --git a/tst/unit/env/envs/conftest.py b/tst/unit/env/envs/conftest.py index d8de90453..3f561951f 100644 --- a/tst/unit/env/envs/conftest.py +++ b/tst/unit/env/envs/conftest.py @@ -8,7 +8,7 @@ from agent_env.artifact.artifacts import docker_image from agent_env.artifact.artifacts.docker_image import DockerImageArtifact from agent_env.artifact.store import get_artifact_store -from agent_env.env.envs.computer.utils import builder, composer +from agent_env.env.envs.computer.utils import builder, composer, probing from agent_env.providers.sandbox_providers.local_docker import provider as local_sandbox from agent_env.store.image_store.image_store import OciRegistryImageStore from agent_env.store.image_store.local_registry_image_store import LocalRegistryImageStore @@ -30,7 +30,7 @@ def docker(local_stores, cli_routing, monkeypatch): monkeypatch.setattr(docker_image, "pin_digest", registry.pin_digest) monkeypatch.setattr(builder, "host_platform", lambda: "linux/arm64") monkeypatch.setattr(local_sandbox, "host_platform", lambda: "linux/arm64") - monkeypatch.setattr(composer, "_probe", fake.probe) + monkeypatch.setattr(probing, "_probe", fake.probe) monkeypatch.setattr(builder, "_runs_here", lambda platform, image: platform in {"linux/arm64", *fake.emulates}) monkeypatch.setattr(builder.LocalBuilder, "installed", lambda self, tag, prefix: {"mss": "10.1.0"}) monkeypatch.setattr(builder.LocalBuilder, "image_config", lambda on, tag: fake.image_config(tag)) diff --git a/tst/unit/env/envs/test_computer_disk.py b/tst/unit/env/envs/test_computer_disk.py index abc18be2f..ec4e59024 100644 --- a/tst/unit/env/envs/test_computer_disk.py +++ b/tst/unit/env/envs/test_computer_disk.py @@ -17,7 +17,7 @@ from agent_env.artifact.artifacts.docker_image import DockerImageArtifact from agent_env.artifact.store import get_artifact_store from agent_env.env import Env -from agent_env.env.envs.computer.utils import boot, composer +from agent_env.env.envs.computer.utils import boot, composer, probing from agent_env.env.published_images import DISK from agent_env.providers.env_providers import env_computer_provider from agent_env.providers.sandbox_providers.local_docker import provider as local_sandbox @@ -215,7 +215,7 @@ async def counted(*args): waited.append(args[1:3]) return await first_boot(*args) - monkeypatch.setattr(composer, "_probe", slow) + monkeypatch.setattr(probing, "_probe", slow) monkeypatch.setattr(env_computer_provider, "probe_first_boot", counted) if fails == "probe": docker.probe_fails["arm64"] = "the click never landed" diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index 8594ed34d..8a72a38f3 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -37,8 +37,10 @@ from agent_env.env import Env from agent_env.env import env as env_module from agent_env.env.env import DeployedEnv -from agent_env.env.envs.computer.utils import boot, builder, composer, gate, inside, packages, resolve_in_base -from agent_env.env.envs.computer.utils.composer import Composing, _probe +from agent_env.env.envs.computer.utils import (boot, builder, composed, composer, gate, inside, packages, probing, + resolve_in_base) +from agent_env.env.envs.computer.utils.composer import Composing +from agent_env.env.envs.computer.utils.probing import _probe from agent_env.env.envs.computer.env import ComputerEnv from agent_env.env.envs.computer import app_env from agent_env.env.envs.computer.app_env import ComputerAppEnv @@ -248,7 +250,7 @@ async def fake_boot(link, **kwargs): monkeypatch.setattr(boot, "boot", fake_boot) monkeypatch.setattr(boot, "discard", AsyncMock()) - monkeypatch.setattr(composer, "run_probe", AsyncMock(return_value="report")) + monkeypatch.setattr(probing, "run_probe", AsyncMock(return_value="report")) asyncio.run(_probe([SimpleNamespace()], env, driver, "img")) @@ -288,7 +290,7 @@ async def fake_boot(link, **kwargs): monkeypatch.setattr(boot, "boot", fake_boot) monkeypatch.setattr(boot, "discard", AsyncMock()) probed = AsyncMock(return_value="report") - monkeypatch.setattr(composer, "run_probe", probed) + monkeypatch.setattr(probing, "run_probe", probed) assert asyncio.run(_probe([SimpleNamespace()], env, driver, "img")) == ( "report", CARD, "SimpleNamespace") @@ -310,10 +312,10 @@ async def never_ready(*args): async def overran(*args): raise TimeoutError - monkeypatch.setattr(composer, "_probe", never_ready) + monkeypatch.setattr(probing, "_probe", never_ready) unbooted = Env.get(_put(base, driver, id="@local/~/kit/other", environment_name="other").id).metadata[ "composed"]["container"]["arm64"] - monkeypatch.setattr(composer, "_probe", overran) + monkeypatch.setattr(probing, "_probe", overran) stuck = Env.get(_put(base, driver, id="@local/~/kit/stuck", environment_name="stuck").id).metadata[ "composed"]["container"]["arm64"] @@ -324,7 +326,7 @@ async def overran(*args): assert unbooted["reason"].startswith("it didn't boot: LocalSandboxProvider: the computer at http://127.0.0.1:1 " "wasn't ready within 180 s") assert (stuck["status"], stuck["stage"], stuck["reason"]) == ( - "failed", "probe", f"the probe didn't finish within {composer._PROBE_TIMEOUT_S} s") + "failed", "probe", f"the probe didn't finish within {probing._PROBE_TIMEOUT_S} s") @pytest.mark.parametrize("result, stored, replaces", [ @@ -338,7 +340,7 @@ async def overran(*args): ], ids=["unprobed-over-ready", "build-failure-over-built", "building-over-built", "probe-failure-over-unprobed", "probe-failure-over-ready", "ready-over-probe-failure", "building-over-build-failure"]) def test_a_result_takes_a_pairs_place_only_when_it_got_at_least_as_far(result, stored, replaces): - assert (composer._rank(result) >= composer._rank(stored)) is replaces + assert (composed._rank(result) >= composed._rank(stored)) is replaces def test_env_build_builds_what_a_put_left_and_prints_each_pair(base, driver, docker, caplog): From 1e912c949dc3cf11ce2d71324a8542a99bbd13e5 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 22:23:31 -0800 Subject: [PATCH 20/21] ci(release): each image step one call of infra_build.py; the disk smoke boots in the launcher it's given The sign-in, registry lookup and step output that infra-build, agent-build and disk-build each ran as a copied shell block are `infra_build.py wanted`; the build, with its labels and, for a disk form, its base's image of the arch, that image's config, the disk tools and its size, is `infra_build.py build`, so disk-build now passes SIZE_GB from the plan; and the tagging the three publish jobs ran is `infra_build.py publish`. The script stays standard library only, and each does what its shell block did. `smoke` takes the launcher to boot a disk form in as --launcher, and waits for every program its base's supervisord.conf declares rather than three named ones. disk-publish is now infra-publish on the disks' list, and disk-build is infra-build but for its launcher's steps, which the tests check as they did the agents' jobs. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/infra_build.py | 130 ++++++++++++++++++++---- .github/workflows/publish-pypi.yml | 140 ++++---------------------- tst/unit/test_infra_build_script.py | 136 ++++++++++++++++++++++++- tst/unit/test_release_infra_images.py | 68 +++++++------ 4 files changed, 300 insertions(+), 174 deletions(-) diff --git a/.github/scripts/infra_build.py b/.github/scripts/infra_build.py index d82fde2e6..15d6e69a1 100644 --- a/.github/scripts/infra_build.py +++ b/.github/scripts/infra_build.py @@ -5,13 +5,18 @@ ``check ENTRY`` fails unless ENTRY, a JSON object as ``infra_images.py plan`` lists an image, names an image agent-env ships, with its Dockerfile, context, repository and a source tag, and for a disk form, the build args naming its base -computer's image and the disk tools by their repositories and source tags. ``exists REF`` exits 0 when the registry has -REF, 1 when it says it doesn't, and 2 when it can't tell. ``config IMAGE`` prints the config object of IMAGE's OCI -config, base64'd, as a disk form's build takes its base's. ``smoke NAME IMAGE`` checks that IMAGE, a local build of -NAME, runs: a disk form boots to its desktop in the QEMU launcher built here as ``agentenv-qemu-launcher``. -``verify-record PATH`` fails unless PATH records each image agent-env ships once, with the digest the registry serves -under its tag now. ``retry COMMAND...`` runs COMMAND, again while a registry rate-limits it or PyPI doesn't serve -yet the agentenv-framework-protocol an agent image's build installs. +computer's image and the disk tools by their repositories and source tags, and its size. ``exists REF`` exits 0 when +the registry has REF, 1 when it says it doesn't, and 2 when it can't tell. ``wanted ENTRY`` checks ENTRY, signs in to +GHCR and writes ``build=true`` to the step's output when the registry lacks its tag, ``build=false`` when it has it. +``build ENTRY ARCH`` checks ENTRY and builds it for linux/ARCH as its tag with ``-ARCH`` after it; a disk form from its +base computer's image of ARCH, which it pulls, with that image's config. ``config IMAGE`` prints the config object of +IMAGE's OCI config, base64'd, as a disk form's build takes its base's. ``smoke NAME IMAGE`` checks that IMAGE, a local +build of NAME, runs: a disk form boots in the QEMU launcher image ``--launcher`` names until every program its base's +supervisord.conf declares runs. ``publish MISSING`` signs in to GHCR and tags the two platform builds of each entry of +MISSING, a JSON list, as one multi-platform image under its tag, once the entry is checked, unless the registry has +that tag. ``verify-record PATH`` fails unless PATH records each image agent-env ships once, with the digest the +registry serves under its tag now. ``retry COMMAND...`` runs COMMAND, again while a registry rate-limits it or PyPI +doesn't serve yet the agentenv-framework-protocol an agent image's build installs. """ from __future__ import annotations @@ -19,6 +24,7 @@ import argparse import base64 import json +import os import platform import posixpath import re @@ -64,7 +70,9 @@ _LOG_TAIL_LINES = 30 # of a container that never turned healthy, in the error # How long a disk form may take to boot to its desktop: seconds with KVM, minutes under TCG. _BOOTED_WITHIN_SECONDS = 900 -_BOOTED = re.compile(r"success: (xvfb|dbus|desktop) entered RUNNING") +# What supervisord prints on a disk form's console as each program starts. +_RUNNING = re.compile(r"success: (\S+) entered RUNNING") +_PROGRAM = re.compile(r"^\[program:([^\]]+)\]", re.MULTILINE) _PRINTED_CHARS = 200 # of what docker printed in place of a manifest, in the error # What a registry answers when it rate-limits a request: ECR Public's 429, and the toomanyrequests error code GHCR and # Docker Hub send. @@ -123,10 +131,16 @@ def main() -> int: commands = parser.add_subparsers(dest="command", required=True) commands.add_parser("check").add_argument("entry") commands.add_parser("exists").add_argument("ref") + commands.add_parser("wanted").add_argument("entry") + build = commands.add_parser("build") + build.add_argument("entry") + build.add_argument("arch", choices=("amd64", "arm64")) commands.add_parser("config").add_argument("image") smoke = commands.add_parser("smoke") smoke.add_argument("name", choices=sorted(IMAGES)) smoke.add_argument("image") + smoke.add_argument("--launcher") + commands.add_parser("publish").add_argument("missing") commands.add_parser("verify-record").add_argument("path", type=Path) commands.add_parser("retry").add_argument("argv", nargs=argparse.REMAINDER, metavar="COMMAND") args = parser.parse_args() @@ -134,17 +148,98 @@ def main() -> int: return _tried_again(lambda: _streamed(args.argv)).returncode if args.command == "exists": return _exists(args.ref) + if args.command == "wanted": + return _wanted(args.entry) + if args.command == "build": + return _build(args.entry, args.arch) if args.command == "config": print(_config(args.image)) return 0 if args.command == "smoke": - return _smoke(args.name, args.image) + if args.name in DISK_ARGS and args.launcher is None: + parser.error(f"{args.name} is booted in a QEMU launcher, which --launcher names") + return _smoke(args.name, args.image, args.launcher) + if args.command == "publish": + return _publish(args.missing) problems = _check(args.entry) if args.command == "check" else _verify_record(args.path) for problem in problems: print(f"::error::{problem}") return 1 if problems else 0 +def _wanted(entry: str) -> int: + if not _checked(entry): + return 1 + _sign_in() + ref = _ref(json.loads(entry)) + if (code := _exists(ref)) == 2: + return 1 + with open(os.environ["GITHUB_OUTPUT"], "a") as output: + output.write(f"build={'true' if code == 1 else 'false'}\n") + if code == 0: + print(f"{ref} is published already, so it isn't built again") + return 0 + + +def _build(entry: str, arch: str) -> int: + """Build ENTRY for linux/``arch``, with no provenance attestation, which would show in the multi-platform image as an + unknown platform.""" + if not _checked(entry): + return 1 + image, target, build_args = json.loads(entry), f"linux/{arch}", [] + if image["name"] in DISK_ARGS: + args, rootfs = image["args"], f"{image['args']['ROOTFS']}-{arch}" + if code := _tried_again(lambda: _streamed(["docker", "pull", "--platform", target, rootfs])).returncode: + return code + for arg, value in {"ROOTFS": rootfs, "TOOLS": args["TOOLS"], "SIZE_GB": args["SIZE_GB"], + "ROOTFS_CONFIG": _config(rootfs)}.items(): + build_args += ["--build-arg", f"{arg}={value}"] + command = ["docker", "build", "--platform", target, "--provenance=false", "-f", image["dockerfile"], + "-t", f"{_ref(image)}-{arch}", *build_args, + "--label", f"org.opencontainers.image.source=https://github.com/{os.environ['GITHUB_REPOSITORY']}", + "--label", f"org.opencontainers.image.revision={os.environ['GITHUB_SHA']}", + "--label", f"io.agentenv.source-tag={image['tag']}", image["context"]] + return _tried_again(lambda: _streamed(command)).returncode + + +def _publish(missing: str) -> int: + _sign_in() + for image in json.loads(missing): + if not _checked(json.dumps(image)): + return 1 + ref = _ref(image) + if (code := _exists(ref)) == 0: + print(f"{ref} is published already, so it's left as it is") + continue + if code != 1: + return 1 + created = _tried_again(lambda: _streamed([ + "docker", "buildx", "imagetools", "create", "-t", ref, + "--annotation", f"index:org.opencontainers.image.source=https://github.com/{os.environ['GITHUB_REPOSITORY']}", + "--annotation", f"index:org.opencontainers.image.revision={os.environ['GITHUB_SHA']}", + f"{ref}-amd64", f"{ref}-arm64"])) + if created.returncode: + return created.returncode + return 0 + + +def _checked(entry: str) -> bool: + problems = _check(entry) + for problem in problems: + print(f"::error::{problem}") + return not problems + + +def _ref(image: dict) -> str: + return f"{image['repository']}:{image['tag']}" + + +def _sign_in() -> None: + """Sign in to GHCR with the job's token, as the workflow's actor.""" + subprocess.run(["docker", "login", "ghcr.io", "--username", os.environ["GITHUB_ACTOR"], "--password-stdin"], + input=os.environ["GHCR_TOKEN"], text=True, check=True) + + def _check(entry: str) -> list[str]: try: image = json.loads(entry) @@ -288,11 +383,11 @@ def _config(image: str) -> str: return base64.b64encode(inspected.stdout.strip().encode()).decode() -def _smoke(name: str, image: str) -> int: +def _smoke(name: str, image: str, launcher: str | None) -> int: if name == "agentenv-service-db": return _healthy(image) if name in DISK_ARGS: - return _boots(image) + return _boots(name, image, launcher) command = [part.replace("{image}", image) for part in _SMOKES[name]] result = subprocess.run(command, capture_output=True, text=True) if result.returncode != 0: @@ -300,15 +395,16 @@ def _smoke(name: str, image: str) -> int: return result.returncode -def _boots(image: str) -> int: - """Boot ``image``, a disk form, in the launcher built here, with KVM where it can, and wait for its desktop's - programs to run, which its console shows.""" +def _boots(name: str, image: str, launcher: str) -> int: + """Boot ``image``, a build of the disk form ``name``, in ``launcher``, with KVM where it can, and wait for every + program its base computer's supervisord.conf declares to run, which its console shows.""" + programs = set(_PROGRAM.findall((REPO / Path(IMAGES[DISK_ARGS[name]["ROOTFS"]]).with_name("supervisord.conf")) + .read_text())) arch = subprocess.run(["docker", "image", "inspect", "--format", "{{.Architecture}}", image], capture_output=True, text=True, check=True).stdout.strip() machine = f"{image}-machine" subprocess.run(["docker", "build", "-q", "-t", machine, "-"], text=True, check=True, capture_output=True, - input=f"FROM --platform=linux/{arch} {image} AS disk\nFROM agentenv-qemu-launcher\n" - "COPY --from=disk /disk/ /disk/\n") + input=f"FROM --platform=linux/{arch} {image} AS disk\nFROM {launcher}\nCOPY --from=disk /disk/ /disk/\n") kvm = ["--device", "/dev/kvm"] if Path("/dev/kvm").exists() else [] container = subprocess.run(["docker", "run", "-d", *kvm, "-e", f"VM_ARCH={arch}", machine], capture_output=True, text=True, check=True).stdout.strip() @@ -317,7 +413,7 @@ def _boots(image: str) -> int: while time.monotonic() < deadline: logs = subprocess.run(["docker", "logs", container], capture_output=True, text=True, errors="replace") logs = logs.stdout + logs.stderr - if len(set(_BOOTED.findall(logs))) == 3: + if programs <= set(_RUNNING.findall(logs)): native = {"x86_64": "amd64", "aarch64": "arm64"}.get(platform.machine()) == arch print(f"{image} booted to its desktop {'with KVM' if kvm and native else 'under TCG'}") return 0 diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index b11f754a5..a3b59d054 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -120,11 +120,8 @@ jobs: ENTRY: ${{ toJSON(matrix.image) }} NAME: ${{ matrix.image.name }} DOCKERFILE: ${{ matrix.image.dockerfile }} - CONTEXT: ${{ matrix.image.context }} - TAG: ${{ matrix.image.tag }} - REF: ${{ matrix.image.repository }}:${{ matrix.image.tag }} + ARCH: ${{ matrix.arch }} BUILT: ${{ matrix.image.repository }}:${{ matrix.image.tag }}-${{ matrix.arch }} - PLATFORM: linux/${{ matrix.arch }} steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: @@ -135,28 +132,13 @@ jobs: id: wanted env: GHCR_TOKEN: ${{ github.token }} - run: | - python3 .github/scripts/infra_build.py check "$ENTRY" - echo "$GHCR_TOKEN" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin - code=0 - python3 .github/scripts/infra_build.py exists "$REF" || code=$? - case "$code" in - 0) echo "build=false" >> "$GITHUB_OUTPUT"; echo "$REF is published already, so it isn't built again" ;; - 1) echo "build=true" >> "$GITHUB_OUTPUT" ;; - *) exit 1 ;; - esac + run: python3 .github/scripts/infra_build.py wanted "$ENTRY" - name: Pull its base images from mirror.gcr.io, as public.ecr.aws can rate-limit a runner for hours if: steps.wanted.outputs.build == 'true' run: python3 .github/scripts/mirror_base_images.py "$DOCKERFILE" - name: Build if: steps.wanted.outputs.build == 'true' - run: | - # No provenance attestation, which would show in the multi-platform image as an unknown platform. - python3 .github/scripts/infra_build.py retry \ - docker build --platform "$PLATFORM" --provenance=false -f "$DOCKERFILE" -t "$BUILT" \ - --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \ - --label "org.opencontainers.image.revision=${GITHUB_SHA}" \ - --label "io.agentenv.source-tag=${TAG}" "$CONTEXT" + run: python3 .github/scripts/infra_build.py build "$ENTRY" "$ARCH" - name: Check that it runs if: steps.wanted.outputs.build == 'true' run: python3 .github/scripts/infra_build.py smoke "$NAME" "$BUILT" @@ -184,23 +166,7 @@ jobs: - name: Tag each image's platform builds as one multi-platform image under its source tag env: GHCR_TOKEN: ${{ github.token }} - run: | - echo "$GHCR_TOKEN" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin - jq -c '.[]' <<< "$MISSING" | while read -r entry; do - python3 .github/scripts/infra_build.py check "$entry" - ref=$(jq -r '"\(.repository):\(.tag)"' <<< "$entry") - code=0 - python3 .github/scripts/infra_build.py exists "$ref" || code=$? - if [ "$code" = 0 ]; then - echo "$ref is published already, so it's left as it is" - continue - fi - [ "$code" = 1 ] || exit 1 - python3 .github/scripts/infra_build.py retry docker buildx imagetools create -t "$ref" \ - --annotation "index:org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \ - --annotation "index:org.opencontainers.image.revision=${GITHUB_SHA}" \ - "${ref}-amd64" "${ref}-arm64" - done + run: python3 .github/scripts/infra_build.py publish "$MISSING" # When images are published, the protocol waits for the infra images, so a failed infra image job sends neither # package to PyPI. A failed agent image job, which runs after it, keeps agentenv-framework off PyPI. @@ -265,11 +231,8 @@ jobs: ENTRY: ${{ toJSON(matrix.image) }} NAME: ${{ matrix.image.name }} DOCKERFILE: ${{ matrix.image.dockerfile }} - CONTEXT: ${{ matrix.image.context }} - TAG: ${{ matrix.image.tag }} - REF: ${{ matrix.image.repository }}:${{ matrix.image.tag }} + ARCH: ${{ matrix.arch }} BUILT: ${{ matrix.image.repository }}:${{ matrix.image.tag }}-${{ matrix.arch }} - PLATFORM: linux/${{ matrix.arch }} steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: @@ -280,28 +243,13 @@ jobs: id: wanted env: GHCR_TOKEN: ${{ github.token }} - run: | - python3 .github/scripts/infra_build.py check "$ENTRY" - echo "$GHCR_TOKEN" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin - code=0 - python3 .github/scripts/infra_build.py exists "$REF" || code=$? - case "$code" in - 0) echo "build=false" >> "$GITHUB_OUTPUT"; echo "$REF is published already, so it isn't built again" ;; - 1) echo "build=true" >> "$GITHUB_OUTPUT" ;; - *) exit 1 ;; - esac + run: python3 .github/scripts/infra_build.py wanted "$ENTRY" - name: Pull its base images from mirror.gcr.io, as public.ecr.aws can rate-limit a runner for hours if: steps.wanted.outputs.build == 'true' run: python3 .github/scripts/mirror_base_images.py "$DOCKERFILE" - name: Build if: steps.wanted.outputs.build == 'true' - run: | - # No provenance attestation, which would show in the multi-platform image as an unknown platform. - python3 .github/scripts/infra_build.py retry \ - docker build --platform "$PLATFORM" --provenance=false -f "$DOCKERFILE" -t "$BUILT" \ - --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \ - --label "org.opencontainers.image.revision=${GITHUB_SHA}" \ - --label "io.agentenv.source-tag=${TAG}" "$CONTEXT" + run: python3 .github/scripts/infra_build.py build "$ENTRY" "$ARCH" - name: Check that it runs if: steps.wanted.outputs.build == 'true' run: python3 .github/scripts/infra_build.py smoke "$NAME" "$BUILT" @@ -329,23 +277,7 @@ jobs: - name: Tag each image's platform builds as one multi-platform image under its source tag env: GHCR_TOKEN: ${{ github.token }} - run: | - echo "$GHCR_TOKEN" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin - jq -c '.[]' <<< "$MISSING" | while read -r entry; do - python3 .github/scripts/infra_build.py check "$entry" - ref=$(jq -r '"\(.repository):\(.tag)"' <<< "$entry") - code=0 - python3 .github/scripts/infra_build.py exists "$ref" || code=$? - if [ "$code" = 0 ]; then - echo "$ref is published already, so it's left as it is" - continue - fi - [ "$code" = 1 ] || exit 1 - python3 .github/scripts/infra_build.py retry docker buildx imagetools create -t "$ref" \ - --annotation "index:org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \ - --annotation "index:org.opencontainers.image.revision=${GITHUB_SHA}" \ - "${ref}-amd64" "${ref}-arm64" - done + run: python3 .github/scripts/infra_build.py publish "$MISSING" # Each base computer's disk form whose tag the registry lacks, built on a runner of each platform from the base's # published image of that platform and the published disk tools, booted with the QEMU launcher built from this @@ -372,13 +304,8 @@ jobs: ENTRY: ${{ toJSON(matrix.image) }} NAME: ${{ matrix.image.name }} DOCKERFILE: ${{ matrix.image.dockerfile }} - CONTEXT: ${{ matrix.image.context }} - TAG: ${{ matrix.image.tag }} - REF: ${{ matrix.image.repository }}:${{ matrix.image.tag }} + ARCH: ${{ matrix.arch }} BUILT: ${{ matrix.image.repository }}:${{ matrix.image.tag }}-${{ matrix.arch }} - ROOTFS: ${{ matrix.image.args.ROOTFS }}-${{ matrix.arch }} - TOOLS: ${{ matrix.image.args.TOOLS }} - PLATFORM: linux/${{ matrix.arch }} steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: @@ -389,16 +316,7 @@ jobs: id: wanted env: GHCR_TOKEN: ${{ github.token }} - run: | - python3 .github/scripts/infra_build.py check "$ENTRY" - echo "$GHCR_TOKEN" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin - code=0 - python3 .github/scripts/infra_build.py exists "$REF" || code=$? - case "$code" in - 0) echo "build=false" >> "$GITHUB_OUTPUT"; echo "$REF is published already, so it isn't built again" ;; - 1) echo "build=true" >> "$GITHUB_OUTPUT" ;; - *) exit 1 ;; - esac + run: python3 .github/scripts/infra_build.py wanted "$ENTRY" - name: Pull its base images, and the launcher's, from mirror.gcr.io, as public.ecr.aws can rate-limit a runner if: steps.wanted.outputs.build == 'true' run: python3 .github/scripts/mirror_base_images.py "$DOCKERFILE" src/agent_env/computers/disk/linux/launcher/Dockerfile @@ -410,21 +328,13 @@ jobs: fi - name: Build if: steps.wanted.outputs.build == 'true' - run: | - python3 .github/scripts/infra_build.py retry docker pull --platform "$PLATFORM" "$ROOTFS" - config=$(python3 .github/scripts/infra_build.py config "$ROOTFS") - # No provenance attestation, which would show in the multi-platform image as an unknown platform. - python3 .github/scripts/infra_build.py retry \ - docker build --platform "$PLATFORM" --provenance=false -f "$DOCKERFILE" -t "$BUILT" \ - --build-arg "ROOTFS=$ROOTFS" --build-arg "TOOLS=$TOOLS" --build-arg "ROOTFS_CONFIG=$config" \ - --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \ - --label "org.opencontainers.image.revision=${GITHUB_SHA}" \ - --label "io.agentenv.source-tag=${TAG}" "$CONTEXT" - - name: Check that it runs + run: python3 .github/scripts/infra_build.py build "$ENTRY" "$ARCH" + - name: Build the QEMU launcher from this checkout, to boot it in if: steps.wanted.outputs.build == 'true' - run: | - docker build -q -t agentenv-qemu-launcher src/agent_env/computers/disk/linux/launcher - python3 .github/scripts/infra_build.py smoke "$NAME" "$BUILT" + run: docker build -q -t agentenv-qemu-launcher src/agent_env/computers/disk/linux/launcher + - name: Check that it boots + if: steps.wanted.outputs.build == 'true' + run: python3 .github/scripts/infra_build.py smoke "$NAME" "$BUILT" --launcher agentenv-qemu-launcher - name: Push it under its tag and platform if: steps.wanted.outputs.build == 'true' run: python3 .github/scripts/infra_build.py retry docker push "$BUILT" @@ -448,23 +358,7 @@ jobs: - name: Tag each image's platform builds as one multi-platform image under its source tag env: GHCR_TOKEN: ${{ github.token }} - run: | - echo "$GHCR_TOKEN" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin - jq -c '.[]' <<< "$MISSING" | while read -r entry; do - python3 .github/scripts/infra_build.py check "$entry" - ref=$(jq -r '"\(.repository):\(.tag)"' <<< "$entry") - code=0 - python3 .github/scripts/infra_build.py exists "$ref" || code=$? - if [ "$code" = 0 ]; then - echo "$ref is published already, so it's left as it is" - continue - fi - [ "$code" = 1 ] || exit 1 - python3 .github/scripts/infra_build.py retry docker buildx imagetools create -t "$ref" \ - --annotation "index:org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \ - --annotation "index:org.opencontainers.image.revision=${GITHUB_SHA}" \ - "${ref}-amd64" "${ref}-arm64" - done + run: python3 .github/scripts/infra_build.py publish "$MISSING" # The digest the registry serves under every image's tag, for the wheel and the release notes, which the framework # job checks against the registry before the wheel carries it; and a warning for each image an anonymous pull, as an diff --git a/tst/unit/test_infra_build_script.py b/tst/unit/test_infra_build_script.py index 56dbeb772..da1246a5c 100644 --- a/tst/unit/test_infra_build_script.py +++ b/tst/unit/test_infra_build_script.py @@ -4,6 +4,7 @@ from __future__ import annotations import ast +import base64 import json import subprocess import sys @@ -26,6 +27,18 @@ def script(): return load_script(SCRIPT.name) +@pytest.fixture +def workflow(monkeypatch, tmp_path) -> Path: + """What a workflow step's environment holds: its output file, which is returned, the actor and the job's token, the + repository and the commit.""" + output = tmp_path / "output" + output.touch() + for name, value in {"GITHUB_OUTPUT": str(output), "GITHUB_ACTOR": "bot", "GHCR_TOKEN": "t0ken", + "GITHUB_REPOSITORY": "scaleapi/agentenv-framework", "GITHUB_SHA": "c0ffee"}.items(): + monkeypatch.setenv(name, value) + return output + + def _run(script, monkeypatch, *args: str) -> int: monkeypatch.setattr(sys, "argv", [SCRIPT.name, *args]) return script.main() @@ -124,6 +137,100 @@ def test_exists_says_yes_no_or_that_it_cant_tell(script, monkeypatch, capsys, an assert ("::error::couldn't tell whether the registry has" in capsys.readouterr().out) is (code == 2) +@pytest.mark.parametrize("answer, code, output", [(_DIGEST, 0, "build=false\n"), ("ERROR: not found", 0, "build=true\n"), + ("ERROR: connection refused", 1, "")], ids=["served", "missing", "unknown"]) +def test_wanted_signs_in_and_builds_only_a_tag_the_registry_says_it_lacks(script, monkeypatch, capsys, workflow, answer, + code, output): + calls, tokens = [], [] + docker = _docker({f"{REGISTRY}/agentenv-gateway:{_TAG}": answer}, calls) + + def run(command, **kw): + tokens.append(kw.get("input")) + return docker(command, **kw) + + monkeypatch.setattr(script.subprocess, "run", run) + + assert _run(script, monkeypatch, "wanted", _gateway()) == code + + assert (calls[0], tokens[0]) == (["docker", "login", "ghcr.io", "--username", "bot", "--password-stdin"], "t0ken") + assert workflow.read_text() == output + assert ("is published already, so it isn't built again" in capsys.readouterr().out) is (answer == _DIGEST) + + +@pytest.mark.parametrize("command", [["wanted"], ["build", "amd64"]]) +def test_wanted_and_build_refuse_an_entry_before_signing_in_or_building(script, monkeypatch, capsys, workflow, command): + calls, runs = [], [] + monkeypatch.setattr(script.subprocess, "run", _docker({}, calls)) + monkeypatch.setattr(script.subprocess, "Popen", lambda argv, **kw: runs.append(argv)) + + assert _run(script, monkeypatch, command[0], _gateway(context="."), *command[1:]) == 1 + + assert calls == runs == [] and workflow.read_text() == "" + assert "::error::agentenv-gateway: context is '.'" in capsys.readouterr().out + + +def test_build_builds_an_image_for_its_platform_under_its_tag_with_its_source_on_it(script, monkeypatch, workflow): + runs, waits = _attempts(script, monkeypatch, [(_LIMITED, 1), ("#1 DONE\n", 0)]) + + assert _run(script, monkeypatch, "build", _gateway(), "arm64") == 0 + + assert runs[1] == ["docker", "build", "--platform", "linux/arm64", "--provenance=false", "-f", + "src/agent_env/env/gateway/Dockerfile", "-t", f"{REGISTRY}/agentenv-gateway:{_TAG}-arm64", + "--label", "org.opencontainers.image.source=https://github.com/scaleapi/agentenv-framework", + "--label", "org.opencontainers.image.revision=c0ffee", "--label", f"io.agentenv.source-tag={_TAG}", + "src/agent_env/env/gateway"] + assert runs[0] == runs[1] and waits == [script._RETRY_WAITS_SECONDS[0]] + + +def test_build_makes_a_disk_form_of_its_bases_image_of_the_arch_with_that_images_config_the_tools_and_its_size( + script, monkeypatch, workflow): + runs, _ = _attempts(script, monkeypatch, [("pulled\n", 0), ("#1 DONE\n", 0)]) + inspected = [] + config = '{"Env": ["A=1"]}' + monkeypatch.setattr(script.subprocess, "run", lambda command, **kw: inspected.append(command) or ( + subprocess.CompletedProcess(command, 0, config + "\n", ""))) + rootfs = f"{REGISTRY}/agentenv-ubuntu-24.04:{_TAG}-amd64" + + assert _run(script, monkeypatch, "build", _disk(), "amd64") == 0 + + assert runs[0] == ["docker", "pull", "--platform", "linux/amd64", rootfs] + assert inspected == [["docker", "image", "inspect", "--format", "{{json .Config}}", rootfs]] + assert runs[1][runs[1].index("-t"):runs[1].index("--label")] == [ + "-t", f"{REGISTRY}/agentenv-ubuntu-24.04-disk:{_TAG}-amd64", "--build-arg", f"ROOTFS={rootfs}", + "--build-arg", f"TOOLS={REGISTRY}/agentenv-disk-tools:{_TAG}", "--build-arg", "SIZE_GB=16", + "--build-arg", f"ROOTFS_CONFIG={base64.b64encode(config.encode()).decode()}"] + + +def test_publish_tags_each_new_images_platform_builds_as_one_and_leaves_a_published_tag_as_it_is(script, monkeypatch, + capsys, workflow): + published, new = f"{REGISTRY}/agentenv-gateway:{_TAG}", f"{REGISTRY}/agentenv-ubuntu-24.04-disk:{_TAG}" + calls = [] + monkeypatch.setattr(script.subprocess, "run", _docker({published: _DIGEST}, calls)) + runs, _ = _attempts(script, monkeypatch, [("pushed\n", 0)]) + + assert _run(script, monkeypatch, "publish", json.dumps([json.loads(_gateway()), json.loads(_disk())])) == 0 + + assert calls[0] == ["docker", "login", "ghcr.io", "--username", "bot", "--password-stdin"] + assert runs == [["docker", "buildx", "imagetools", "create", "-t", new, + "--annotation", "index:org.opencontainers.image.source=https://github.com/scaleapi/agentenv-framework", + "--annotation", "index:org.opencontainers.image.revision=c0ffee", f"{new}-amd64", f"{new}-arm64"]] + assert f"{published} is published already, so it's left as it is" in capsys.readouterr().out + + +@pytest.mark.parametrize("first, served", [ + (_gateway(context="."), {}), + (_gateway(), {f"{REGISTRY}/agentenv-gateway:{_TAG}": "ERROR: connection refused"}), +], ids=["refused", "unknown"]) +def test_publish_tags_nothing_more_after_an_entry_it_refuses_or_whose_tag_it_cant_look_up(script, monkeypatch, workflow, + first, served): + monkeypatch.setattr(script.subprocess, "run", _docker(served)) + runs, _ = _attempts(script, monkeypatch, []) + + assert _run(script, monkeypatch, "publish", json.dumps([json.loads(first), json.loads(_disk())])) == 1 + + assert runs == [] + + def _record(tmp_path, digests: dict[str, str] | None = None) -> Path: path = tmp_path / "published_images.json" write_release_digests(digests or {image.ref(): _DIGEST for image in bootstrap.PUBLISHED_IMAGES}, path) @@ -229,18 +336,41 @@ def run(command, **kw): monkeypatch.setattr(script.Path, "exists", lambda self: kvm) monkeypatch.setattr(script.platform, "machine", lambda: machine) - assert _run(script, monkeypatch, "smoke", "agentenv-ubuntu-24.04-disk", "disk:1-arm64") == 0 + assert _run(script, monkeypatch, "smoke", "agentenv-ubuntu-24.04-disk", "disk:1-arm64", "--launcher", "launcher:1") == 0 commands = [command for command, _ in calls] assert calls[1] == (["docker", "build", "-q", "-t", "disk:1-arm64-machine", "-"], - "FROM --platform=linux/arm64 disk:1-arm64 AS disk\nFROM agentenv-qemu-launcher\n" - "COPY --from=disk /disk/ /disk/\n") + "FROM --platform=linux/arm64 disk:1-arm64 AS disk\nFROM launcher:1\nCOPY --from=disk /disk/ /disk/\n") assert commands[2] == ["docker", "run", "-d", *(["--device", "/dev/kvm"] if kvm else []), "-e", "VM_ARCH=arm64", "disk:1-arm64-machine"] assert commands[-1] == ["docker", "rm", "-f", "c1"] and commands.count(["docker", "logs", "c1"]) == 2 assert f"booted to its desktop {how}" in capsys.readouterr().out +def test_a_disk_forms_smoke_waits_for_every_program_its_bases_supervisord_conf_declares(script, monkeypatch, capsys, + tmp_path): + conf = tmp_path / Path(script.IMAGES["agentenv-ubuntu-24.04"]).with_name("supervisord.conf") + conf.parent.mkdir(parents=True) + conf.write_text("[supervisord]\nnodaemon=true\n\n[program:xvfb]\ncommand=Xvfb\n\n[program:recorder]\ncommand=rec\n") + monkeypatch.setattr(script, "REPO", tmp_path) + monkeypatch.setattr(script, "_BOOTED_WITHIN_SECONDS", 0.01) + monkeypatch.setattr(script.time, "sleep", lambda seconds: None) + console = "success: xvfb entered RUNNING\nsuccess: dbus entered RUNNING\nsuccess: desktop entered RUNNING\n" + monkeypatch.setattr(script.subprocess, "run", lambda command, **kw: subprocess.CompletedProcess( + command, 0, {"image": "amd64\n", "run": "c1\n", "logs": console}.get(command[1], ""), "")) + + assert _run(script, monkeypatch, "smoke", "agentenv-ubuntu-24.04-disk", "disk:1", "--launcher", "launcher:1") == 1 + + assert "::error::disk:1 didn't boot to its desktop within 0.01s" in capsys.readouterr().out + + +def test_a_disk_forms_smoke_needs_the_launcher_to_boot_it_in(script, monkeypatch, capsys): + with pytest.raises(SystemExit) as exited: + _run(script, monkeypatch, "smoke", "agentenv-ubuntu-24.04-disk", "disk:1") + + assert exited.value.code == 2 and "--launcher" in capsys.readouterr().err + + def test_a_service_db_that_never_turns_healthy_fails_with_its_logs(script, monkeypatch, capsys): calls = [] monkeypatch.setattr(script.subprocess, "run", _docker({}, calls, [])) diff --git a/tst/unit/test_release_infra_images.py b/tst/unit/test_release_infra_images.py index fa0e4faa7..db7f189a2 100644 --- a/tst/unit/test_release_infra_images.py +++ b/tst/unit/test_release_infra_images.py @@ -21,9 +21,11 @@ # What installs or runs agent-env and its dependencies in a job. _RUNS_DEPENDENCIES = re.compile(r"\buv (sync|run)\b|infra_images\.py") # What reads the shipped bundles' folders: an image's tag, its build, and the wheel. -_READS_SHIPPED_SOURCES = re.compile(r"infra_images\.py (plan|write)|docker build\b|uv build --no-sources --out-dir dist") +_READS_SHIPPED_SOURCES = re.compile( + r"infra_images\.py (plan|write)|infra_build\.py build\b|docker build\b|uv build --no-sources --out-dir dist") # The pin command, and not pin-pyproject, which pins the framework's own requirement. _PIN = re.compile(r"python3 \.github/scripts/shipped_bundles\.py pin(?![\w-])") +_SCRIPT = "python3 .github/scripts/infra_build.py" def _runs(job: str) -> list[str]: @@ -56,33 +58,42 @@ def test_a_job_that_installs_agent_env_holds_no_token_that_writes(job): def test_the_jobs_that_push_or_publish_check_what_they_are_handed(): - build = "\n".join(_runs("infra-build")) - publish = "\n".join(_runs("infra-publish")) - - assert 'infra_build.py check "$ENTRY"' in build and 'infra_build.py check "$entry"' in publish - assert build.index("infra_build.py check") < build.index("docker build") - assert publish.index("infra_build.py check") < publish.index("imagetools create") + """The script's wanted, build and publish check each entry before anything else (``test_infra_build_script``).""" + assert _step("infra-build", "wanted")["run"] == f'{_SCRIPT} wanted "$ENTRY"' + assert f'{_SCRIPT} build "$ENTRY" "$ARCH"' in _runs("infra-build") + assert _runs("infra-publish")[-1] == f'{_SCRIPT} publish "$MISSING"' assert "infra_build.py verify-record " + RECORD in _step("framework", "record")["run"] -def test_a_published_tag_is_never_built_again_or_moved(): - wanted = _step("infra-build", "wanted")["run"] - publish = "\n".join(_runs("infra-publish")) +@pytest.mark.parametrize("job", ["infra-build", "agent-build", "disk-build"]) +def test_a_published_tag_is_never_built_again(job): + """Nor moved: publish leaves a tag the registry serves as it is (``test_infra_build_script``).""" + steps = JOBS[job]["steps"] - assert 'infra_build.py exists "$REF"' in wanted assert all(step.get("if") == "steps.wanted.outputs.build == 'true'" - for step in JOBS["infra-build"]["steps"] if step.get("name") in ("Build", "Check that it runs", - "Push it under its tag and platform")) - assert publish.index('infra_build.py exists "$ref"') < publish.index("imagetools create") + for step in steps[steps.index(_step(job, "wanted")) + 1:]) + + +@pytest.mark.parametrize(("job", "infra_job", "listed"), [("agent-build", "infra-build", "missing_agents"), + ("agent-publish", "infra-publish", "missing_agents"), + ("disk-publish", "infra-publish", "missing_disks")]) +def test_the_agent_and_disk_image_jobs_are_the_infra_image_jobs_on_their_lists(job, infra_job, listed): + """So each check here on how the infra images are built, pushed and tagged holds for the agents' images and the + disk forms too.""" + def body(name: str) -> dict: + return {key: value for key, value in JOBS[name].items() if key not in ("needs", "if")} + + assert yaml.safe_load(yaml.safe_dump(body(infra_job)).replace("missing_infra", listed)) == body(job) -@pytest.mark.parametrize(("agent_job", "infra_job"), [("agent-build", "infra-build"), ("agent-publish", "infra-publish")]) -def test_the_agent_image_jobs_are_the_infra_image_jobs_on_the_agents_list(agent_job, infra_job): - """So each check here on how the infra images are built, pushed and tagged holds for the agents' images too.""" - def body(job: str) -> dict: - return {key: value for key, value in JOBS[job].items() if key not in ("needs", "if")} +def test_the_disk_build_is_the_infra_build_but_for_the_launcher_it_pulls_builds_and_boots_the_disk_in(): + infra, disk = JOBS["infra-build"], JOBS["disk-build"] + own = re.compile(r"mirror_base_images\.py|infra_build\.py smoke") - assert yaml.safe_load(yaml.safe_dump(body(infra_job)).replace("missing_infra", "missing_agents")) == body(agent_job) + assert {key: disk[key] for key in ("runs-on", "timeout-minutes", "permissions", "env")} == { + key: infra[key] for key in ("runs-on", "timeout-minutes", "permissions", "env")} + assert [step for step in disk["steps"] if step in infra["steps"]] == [ + step for step in infra["steps"] if not own.search(step.get("run", ""))] @pytest.mark.parametrize("job", ["infra-build", "infra-publish", "agent-build", "agent-publish", "disk-build", @@ -187,17 +198,15 @@ def test_a_build_pulls_its_base_images_from_the_mirror_first(): assert step["run"] == 'python3 .github/scripts/mirror_base_images.py "$DOCKERFILE"' assert step["if"] == "steps.wanted.outputs.build == 'true'" - assert build.index("mirror_base_images.py") < build.index("docker build") + assert build.index("mirror_base_images.py") < build.index("infra_build.py build") def test_what_a_registry_can_rate_limit_is_tried_again(): - """A 429 on a base image's pull would fail the build, and with it the release's PyPI jobs.""" + """A 429 on a base image's pull would fail the build, and with it the release's PyPI jobs; the script's build and + publish try again themselves (``test_infra_build_script``).""" build = "\n".join(_runs("infra-build")) - publish = "\n".join(_runs("infra-publish")) - assert re.search(r"infra_build\.py retry \\\s+docker build", build) assert 'infra_build.py retry docker push "$BUILT"' in build - assert "infra_build.py retry docker buildx imagetools create" in publish assert JOBS["infra-build"]["strategy"]["max-parallel"] <= 4 @@ -231,12 +240,9 @@ def test_a_disk_form_is_built_over_its_bases_published_image_with_the_published_ """Its base's image, an agent image, and the disk tools, an infra image, are published first; its build is checked and pushed as the infra images' are.""" build = "\n".join(_runs("disk-build")) - publish = "\n".join(_runs("disk-publish")) assert {"infra-publish", "agent-publish"} <= set(JOBS["disk-build"]["needs"]) assert all(_finished(job) in _condition("disk-build") for job in ("infra-publish", "agent-publish")) - assert 'infra_build.py check "$ENTRY"' in build and build.index("infra_build.py check") < build.index("docker build") - assert '--build-arg "ROOTFS=$ROOTFS" --build-arg "TOOLS=$TOOLS"' in build - assert build.index('infra_build.py smoke "$NAME" "$BUILT"') < build.index('docker push "$BUILT"') - assert 'infra_build.py check "$entry"' in publish and publish.index("check") < publish.index("imagetools create") - assert JOBS["disk-build"]["env"]["ROOTFS"] == "${{ matrix.image.args.ROOTFS }}-${{ matrix.arch }}" + assert (build.index(f'{_SCRIPT} build "$ENTRY" "$ARCH"') < build.index("docker build -q -t agentenv-qemu-launcher ") + < build.index(f'{_SCRIPT} smoke "$NAME" "$BUILT" --launcher agentenv-qemu-launcher') + < build.index('docker push "$BUILT"')) From b8bb67897a61022b41a34f68624e57c907541ee2 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 22:33:27 -0800 Subject: [PATCH 21/21] fix(ci): the clean-install check's checkout copy of an image keeps its build args _checkout rebuilt each PublishedImage from its name, Dockerfile and args, so the disk image's copy lost SIZE_GB and its tag never matched the installed one's. It now replaces only the Dockerfile and the images it's built from, keeping every other field. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/check_clean_install.py | 5 +++-- tst/unit/test_clean_install_wheel.py | 17 ++++++++++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/.github/scripts/check_clean_install.py b/.github/scripts/check_clean_install.py index 82b99f24d..0b0c6c54f 100644 --- a/.github/scripts/check_clean_install.py +++ b/.github/scripts/check_clean_install.py @@ -15,6 +15,7 @@ import argparse import compileall +import dataclasses import importlib.metadata import json import os @@ -122,8 +123,8 @@ def problems(name: str, runs: int, before: dict[str, list[str]]) -> list[str]: def _checkout(image: PublishedImage) -> PublishedImage: """``image`` as the checkout's sources make it, with the images it's built from.""" installed_src = Path(agent_env.__file__).parent.parent - return PublishedImage(image.name, CHECKOUT_SRC / image.dockerfile.relative_to(installed_src), - tuple((arg, _checkout(built)) for arg, built in image.args)) + return dataclasses.replace(image, dockerfile=CHECKOUT_SRC / image.dockerfile.relative_to(installed_src), + args=tuple((arg, _checkout(built)) for arg, built in image.args)) def plugins() -> list[str]: diff --git a/tst/unit/test_clean_install_wheel.py b/tst/unit/test_clean_install_wheel.py index c1d40778b..44522dbdd 100644 --- a/tst/unit/test_clean_install_wheel.py +++ b/tst/unit/test_clean_install_wheel.py @@ -1,5 +1,6 @@ """The wheel check in ``.github/scripts/clean_install.py``: every tracked file of each shipped bundle, and every bundle -entry point, must be in the built wheel.""" +entry point, must be in the built wheel; and the published images as the checkout makes them, which +``check_clean_install.py`` compares the installed ones' tags with.""" from __future__ import annotations @@ -10,6 +11,10 @@ import pytest +import agent_env +from agent_env.env import bootstrap +from tst.util.scripts import load_script + SCRIPT = Path(__file__).resolve().parents[2] / ".github" / "scripts" / "clean_install.py" sys.path.insert(0, str(SCRIPT.parent)) # as running the script does, so it imports its sibling shipped_bundles _spec = importlib.util.spec_from_file_location("clean_install", SCRIPT) @@ -69,3 +74,13 @@ def test_the_gate_checks_the_folder_of_every_shipped_bundle(): "echo-mcp": "src/agent_env/examples/echo-mcp", "hello": "src/agent_env/examples/hello", "mail-mcp": "src/agent_env/examples/mail-mcp", "next-env-event": "src/agent_env/examples/next-env-event"} + + +def test_the_checkout_makes_each_published_image_as_the_installed_sources_do_when_they_are_its_own(monkeypatch): + """Every field of an image is kept in the checkout's copy, its build args too, so the tags differ only when the + sources do.""" + check = load_script("check_clean_install.py") + monkeypatch.setattr(check, "CHECKOUT_SRC", Path(agent_env.__file__).parent.parent) + + assert [check._checkout(image) for image in bootstrap.PUBLISHED_IMAGES] == list(bootstrap.PUBLISHED_IMAGES) + assert any(image.build_args for image in bootstrap.PUBLISHED_IMAGES)