Repository navigation
[feature] Import AgentEnv bundles with openenv import #1493
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
earakely-scale
wants to merge
11
commits into
openenv:main
Choose a base branch
from
earakely-scale:agentenv-importer
base: main
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
+814
−7
Open
Changes from all commits
Commits
Show all changes
11 commits
Select commit
Hold shift + click to select a range
ab6b831
Import AgentEnv bundles with openenv import
earakely-scale 77369a4
Vendor only the picked AgentEnv bundle and keep the framework floor
earakely-scale 4319a10
Point the AgentEnv import docs at AgentEnv's own page
earakely-scale 3294418
Depend on agentenv-framework's openenv extra in imported bundles
earakely-scale 35e7721
Document delegated AgentEnv API and Docker prerequisites (#1493)
burtenshaw b9a7d75
Merge main into AgentEnv importer, preserving local serve guidance
burtenshaw c834349
Link the delegated adapter's source and require step ids in detection
earakely-scale bc98140
fix(cli): reject unverifiable AgentEnv framework direct references (#…
burtenshaw a72456a
Generate an MCPToolClient for AgentEnv bundles, read MAX_CONCURRENT_E…
earakely-scale c1d55ca
Merge remote-tracking branch 'origin/main' into fix/1493-tool-discovery
burtenshaw 598514f
Fix generated AgentEnv client discovery across tasks and turns (#1493)
burtenshaw File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,163 @@ | ||
| """AgentEnv bundle source importer.""" | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| import json | ||
| from pathlib import Path | ||
|
|
||
| from packaging.requirements import InvalidRequirement, Requirement | ||
| from packaging.specifiers import SpecifierSet | ||
| from packaging.utils import canonicalize_name | ||
|
|
||
| from .._cli_utils import console | ||
| from .base import ( | ||
| append_dependency_files, | ||
| collect_source_dependencies, | ||
| DetectedEnvironment, | ||
| iter_directories, | ||
| prepare_import_package, | ||
| render_importer_template, | ||
| write_text, | ||
| ) | ||
|
|
||
|
|
||
| _FRAMEWORK = "agentenv-framework" | ||
| _FRAMEWORK_EXTRA = "openenv" | ||
| _FRAMEWORK_VERSION = "0.9.1366" | ||
| _FRAMEWORK_FLOOR = f">={_FRAMEWORK_VERSION}" | ||
| _ADAPTER_SOURCE = ( | ||
| "https://github.com/scaleapi/agentenv-framework/blob/" | ||
| f"v{_FRAMEWORK_VERSION}/src/agent_env/openenv" | ||
| ) | ||
| _REQUIRES_PYTHON = ">=3.11.4" | ||
|
|
||
|
|
||
| def _is_task_file(path: Path) -> bool: | ||
| try: | ||
| steps = json.loads(path.read_text(encoding="utf-8")) | ||
| except (OSError, UnicodeDecodeError, ValueError): | ||
| return False | ||
| return ( | ||
| isinstance(steps, list) | ||
| and bool(steps) | ||
| and all( | ||
| isinstance(step, dict) | ||
| and isinstance(step.get("id"), str) | ||
| and isinstance(step.get("type"), str) | ||
| for step in steps | ||
| ) | ||
| ) | ||
|
|
||
|
|
||
| def detect_agentenv_bundles(source: Path) -> list[DetectedEnvironment]: | ||
| """Detect AgentEnv bundles, folders whose tasks/ holds task files, without importing source.""" | ||
| source = source.resolve() | ||
| matches: list[DetectedEnvironment] = [] | ||
|
|
||
| for tasks_dir in iter_directories(source, "tasks"): | ||
| if not any(_is_task_file(path) for path in sorted(tasks_dir.glob("*.json"))): | ||
| continue | ||
| bundle = tasks_dir.parent | ||
| matches.append( | ||
| DetectedEnvironment( | ||
| source_type="agentenv", | ||
| class_name=bundle.name, | ||
| module_path="" | ||
| if bundle == source | ||
| else bundle.relative_to(source).as_posix(), | ||
| file_path=bundle, | ||
| ) | ||
| ) | ||
|
|
||
| return matches | ||
|
|
||
|
|
||
| def _with_framework_floor(dependencies: list[str]) -> list[str]: | ||
| """The bundle's dependencies, with agentenv-framework's openenv extra at least the version that serves bundles.""" | ||
| for index, dependency in enumerate(dependencies): | ||
| try: | ||
| requirement = Requirement(dependency) | ||
| except InvalidRequirement: | ||
| continue | ||
| if canonicalize_name(requirement.name) == _FRAMEWORK: | ||
| if requirement.url is not None: | ||
| raise ValueError( | ||
| f"Cannot verify {_FRAMEWORK}{_FRAMEWORK_FLOOR} from a direct " | ||
| "reference. In the bundle's pyproject.toml or requirements.txt, " | ||
| "replace the framework direct reference with a version constraint " | ||
| f"such as {_FRAMEWORK}[{_FRAMEWORK_EXTRA}]{_FRAMEWORK_FLOOR}, " | ||
| "preserving any additional extras and environment marker." | ||
| ) | ||
| requirement.extras.add(_FRAMEWORK_EXTRA) | ||
| requirement.specifier &= SpecifierSet(_FRAMEWORK_FLOOR) | ||
| dependencies[index] = str(requirement) | ||
| return dependencies | ||
| return [*dependencies, f"{_FRAMEWORK}[{_FRAMEWORK_EXTRA}]{_FRAMEWORK_FLOOR}"] | ||
|
|
||
|
|
||
| class AgentEnvImporter: | ||
| """Importer for AgentEnv bundles, served by agentenv-framework's OpenEnv server.""" | ||
|
|
||
| source_type = "agentenv" | ||
|
|
||
| def detect(self, source: Path) -> list[DetectedEnvironment]: | ||
| return detect_agentenv_bundles(source) | ||
|
|
||
| def generate( | ||
| self, | ||
| *, | ||
| source: Path, | ||
| destination: Path, | ||
| env_name: str, | ||
| detected: DetectedEnvironment, | ||
| ) -> None: | ||
| bundle = detected.file_path | ||
| dependencies = _with_framework_floor(collect_source_dependencies(bundle)) | ||
| package = prepare_import_package( | ||
| bundle, destination, env_name, vendor_dir=bundle.name | ||
| ) | ||
| client = f"{package.class_name_prefix}Env" | ||
| (destination / "server" / f"{env_name}_environment.py").unlink() | ||
| (destination / "models.py").unlink() | ||
| for name, template in ( | ||
| ("client.py", "agentenv_client.py.tpl"), | ||
| ("__init__.py", "agentenv_init.py.tpl"), | ||
| ): | ||
| write_text( | ||
| destination / name, | ||
| render_importer_template(template, env_name=env_name, client=client), | ||
| ) | ||
| write_text( | ||
| destination / "server" / "__init__.py", | ||
| f'"""{env_name} environment server: an AgentEnv bundle."""\n', | ||
| ) | ||
| write_text( | ||
| destination / "server" / "app.py", | ||
| render_importer_template( | ||
| "agentenv_app.py.tpl", bundle=json.dumps(bundle.name) | ||
| ), | ||
| ) | ||
| write_text( | ||
| destination / "README.md", | ||
| render_importer_template( | ||
| "agentenv_README.md.tpl", | ||
| env_name=env_name, | ||
| bundle=bundle.name, | ||
| framework=f"{_FRAMEWORK}[{_FRAMEWORK_EXTRA}]{_FRAMEWORK_FLOOR}", | ||
| adapter=_ADAPTER_SOURCE, | ||
| client=client, | ||
| ), | ||
| ) | ||
| console.print( | ||
| "[yellow]Warning:[/yellow] AgentEnv requires local Docker: a Docker CLI " | ||
| "and a running daemon accessible to the server for image builds and episode " | ||
| "containers. The generated server/Dockerfile does not provide this " | ||
| "runtime; openenv build / openenv push alone do not produce a working " | ||
| "deployment. See README.md for prerequisites." | ||
| ) | ||
| append_dependency_files( | ||
| destination, | ||
| env_name, | ||
| dependencies, | ||
| requires_python=_REQUIRES_PYTHON, | ||
| ) | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
82 changes: 82 additions & 0 deletions
82
src/openenv/cli/importers/templates/agentenv_README.md.tpl
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,82 @@ | ||
| # ${env_name}: AgentEnv bundle | ||
|
|
||
| This package serves the vendored `${bundle}` bundle through | ||
| `agent_env.openenv.build_app`. The bundle keeps its original folder name at | ||
| `vendor/${bundle}`; the generated package name is `${env_name}`. | ||
|
|
||
| ## Prerequisites | ||
|
|
||
| - Python >=3.11.4 and `${framework}` (declared in | ||
| `pyproject.toml` and `server/requirements.txt`). | ||
| - Local Docker: a Docker CLI and a running daemon accessible to the server. | ||
| The delegated server builds the bundle's environment images at startup and | ||
| deploys episode containers using local Docker by default. | ||
| - A self-contained bundle: tasks, environment definitions, verifiers, data and | ||
| dependencies must be included in the bundle or provided by agentenv-framework. | ||
| Only the selected bundle is vendored; external plugin registrations and files | ||
| elsewhere in the source repository are not included. | ||
|
|
||
| ## Run locally | ||
|
|
||
| From this generated package directory, after setting up Docker: | ||
|
|
||
| ```bash | ||
| uv sync | ||
| MAX_CONCURRENT_ENVS=8 uv run server --host 127.0.0.1 --port 8000 | ||
| ``` | ||
|
|
||
| `MAX_CONCURRENT_ENVS` (default 8) caps the concurrent sessions. Each session | ||
| plays its own episode on its own deployment. | ||
|
|
||
| The `server` entry point runs `server/app.py`, which passes | ||
| `vendor/${bundle}` to `build_app`. Task selection, splits, tools and rewards | ||
| are defined by the bundle and agentenv-framework. The served environment is | ||
| `agent_env.openenv`'s `AgentEnvEnvironment`; read it, and `build_app`, at the | ||
| minimum supported version: [environment.py](${adapter}/environment.py) and | ||
| [app.py](${adapter}/app.py). Follow | ||
| [AgentEnv's OpenEnv documentation](https://www.agentenvframework.com/docs/openenv) | ||
| for its API and client usage. | ||
|
|
||
| ## Client | ||
|
|
||
| `client.py` defines `${client}`, an `MCPToolClient`: the served environment | ||
| takes OpenEnv's MCP actions (`ListToolsAction`, `CallToolAction`), which | ||
| `__init__.py` also exports. | ||
|
|
||
| ```python | ||
| from ${env_name} import ${client} | ||
|
|
||
| async with ${client}(base_url="http://127.0.0.1:8000") as env: | ||
| await env.reset(split="all", index=0) | ||
| tools = await env.list_tools() | ||
| ended = await env.call_tool("end_turn", response="Done.") | ||
| print(ended["done"], ended["reward"]) | ||
| ``` | ||
|
|
||
| The tools are the bundle environment's MCP tools plus `end_turn`, which ends | ||
| the turn with its `response`. Its result carries the next turn's prompt, or, | ||
| once `done`, the task's reward. A step before `reset` raises. | ||
|
|
||
| `list_tools()` discovers fresh tools by default because tools can change after | ||
| reset, `end_turn`, or a bundle tool call. `get_tool()` and `has_tool()` also use | ||
| fresh discovery. Pass `list_tools(use_cache=True)` explicitly to reuse the last | ||
| discovery result. | ||
|
|
||
| ## Deployment limitations | ||
|
|
||
| **Warning:** `server/Dockerfile` does not provide the Docker runtime required | ||
| by the delegated server. `openenv build` / `openenv push` alone do not produce | ||
| a working deployment, including on Hugging Face Spaces. Deployment requires | ||
| separately provisioning the server's image-build and episode-container runtime; | ||
| this importer does not configure it. Review AgentEnv's runtime requirements | ||
| before choosing a deployment host. | ||
|
|
||
| ## Generated files | ||
|
|
||
| - `server/app.py`: delegated application and local server entry point. | ||
| - `vendor/${bundle}`: the selected bundle and its data. | ||
| - `pyproject.toml`: package dependencies and the `server` entry point. | ||
| - `server/requirements.txt`: server dependencies. | ||
| - `openenv.yaml`: generated OpenEnv manifest. | ||
| - `server/Dockerfile`: shared image template with the limitation described above. | ||
| - `client.py`, `__init__.py`: the client and the MCP action types it sends. |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| from __future__ import annotations | ||
|
|
||
| import os | ||
| from pathlib import Path | ||
|
|
||
| from agent_env.openenv import build_app | ||
|
|
||
|
|
||
| BUNDLE = Path(__file__).resolve().parents[1] / "vendor" / ${bundle} | ||
|
|
||
| app = build_app( | ||
| BUNDLE, max_concurrent_envs=int(os.getenv("MAX_CONCURRENT_ENVS", "8")) | ||
| ) | ||
|
|
||
|
|
||
| def main(): | ||
| import argparse | ||
|
|
||
| import uvicorn | ||
|
|
||
| parser = argparse.ArgumentParser() | ||
| parser.add_argument("--host", default="0.0.0.0") | ||
| parser.add_argument("--port", type=int, default=8000) | ||
| args = parser.parse_args() | ||
| uvicorn.run(app, host=args.host, port=args.port) | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| main() |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.