diff --git a/dokimi/changelog.d/20261024_100000_napetrov_run-tbench-v1.md b/dokimi/changelog.d/20261024_100000_napetrov_run-tbench-v1.md new file mode 100644 index 000000000..51c312291 --- /dev/null +++ b/dokimi/changelog.d/20261024_100000_napetrov_run-tbench-v1.md @@ -0,0 +1,16 @@ +### Added + +- **terminal-bench v1 tasks run natively** — a `task.yaml` task + (laude-institute/terminal-bench `original-tasks/`) loads as a `TaskSpec` and + runs under `docker-oracle` / `docker-claude` as terminal-bench runs it: + `solution.sh` at `/oracle`, `run-tests.sh` and `tests/` at `/tests`, the + image's user, `TEST_DIR` and the compose environment, and the pytest summary + as the verdict. A task one `docker run` cannot reproduce (extra services, + volumes, tmpfs, ports, a service command, build args or target, a host + environment value; `solution.yaml`; `run_tests_in_same_shell`) is refused + with the reason. Harbor-backed harnesses and `benchflow*` run a Harbor copy + written lazily into a content-keyed cache; its `test.sh` scores the same + pytest summary. `tasks inspect` lists `docker-*`, `harbor` and `benchflow` as + consumers (only `docker-*` for a `solution.yaml` task, which has no Harbor export). A v1 task gets 8 GB of memory + (its compose `deploy.resources.limits.memory` when set; the default is + `DOKIMI_TBENCH_V1_MEMORY_MB`, else 8192), where other formats default to 2 GB. diff --git a/dokimi/docs/coding-harnesses.md b/dokimi/docs/coding-harnesses.md index d2428656a..78b1ab613 100644 --- a/dokimi/docs/coding-harnesses.md +++ b/dokimi/docs/coding-harnesses.md @@ -69,8 +69,8 @@ Jekyll or Hugo, an unclosed block) is an author's note, and the task loads from `task.toml` with a warning (none for a note without frontmatter, or with Hugo's TOML `+++` frontmatter, which is not frontmatter here). -Discovery skips a lone `task.md` without frontmatter with a warning; -`load_task` on it raises. A lone `task.md` with frontmatter is a task, whatever +Discovery skips a lone `task.md` without frontmatter, and a `task.yaml` +mapping with no `instruction`, with a warning; `load_task` on either raises. A lone `task.md` with frontmatter is a task, whatever its keys. One that opens with `---` but does not parse (a horizontal rule, an unclosed block, invalid YAML, a non-mapping) fails discovery of the whole root, as a malformed `task.toml` does; `coverage` skips it. YAML aliases are @@ -107,7 +107,7 @@ real files in each format, with their sources in its `MANIFEST.yaml`. |---|---|---|---| | `harbor` | directory with `task.toml` | executable | Harbor-backed harnesses (unless Harbor would refuse it), `docker-*` (with `environment/Dockerfile`) | | `task-md` | directory with a BenchFlow `task.md` | executable | `benchflow*`, `docker-*` (with `environment/Dockerfile`) | -| `terminal-bench-v1` | directory with `task.yaml` + `run-tests.sh` / `Dockerfile` / `solution.sh` | executable | none yet | +| `terminal-bench-v1` | directory with `task.yaml` + `run-tests.sh` / `Dockerfile` / `solution.sh` | executable | `docker-*` run it as it is; `harbor`, `benchflow` through a cached Harbor export (not a `solution.yaml` task) | | `swe-bench` | JSON/JSONL rows with `instance_id`, `repo`, `base_commit`, `problem_statement` (`FAIL_TO_PASS`) | executable | none yet | | `inspect-samples` | JSON/JSONL/CSV rows with `input` and `target` (or the keys a `FieldSpec` names) | answer-only | none yet | | `inspect-task` | `.py` importing `inspect_ai` with a `@task` function | answer-only (executable with a `sandbox=`) | none yet | @@ -122,7 +122,30 @@ real files in each format, with their sources in its `MANIFEST.yaml`. | `dokimi-explore-task` | YAML with `id`, `query` and `references[].path` | localization | explore track | | `git-lfs-pointer` | a git-LFS spec v1 pointer stub | unknown | none (fetch it first) | -`tasks list` and `tasks run` still read only the first two. +A terminal-bench v1 task (`task.yaml`, laude-institute/terminal-bench +`original-tasks/`) runs as it is under `docker-*`: `solution.sh` is copied to +`/oracle/solution.sh`, `run-tests.sh` and `tests/` to `/tests`, both run as the +image's user with `TEST_DIR=/tests` and the compose `environment`, and the +verdict is the pytest `short test summary info` (every listed test passed, skipped or +xfailed), not the exit code, as terminal-bench reads it. `docker-*` refuses, +with the reason, a task whose `docker-compose.yaml` needs more than one +`docker run` of its default build stage (a second service, volumes, tmpfs, +ports, a service `command` other than `sleep infinity`, `build.args` or +`build.target`, an `environment` value read from the host), a `parser_name` other +than `pytest`, `run_tests_in_same_shell`, or (oracle only) a `solution.yaml`. +The Harbor-backed harnesses and `benchflow*` run a Harbor copy written on first +use to `~/.cache/dokimi/harbor-export//` (mode 0700) by +[`harbor_export.py`](../dokimi/harnesses/harbor_export.py); its `test.sh` scores +the same pytest summary, so a task gets one verdict on every backend. + +Memory: terminal-bench sets no limit, so a v1 task gets +`environment.memory_mb = 8192` (not the 2048 of other formats; tasks such as +`predicate-pushdown-bench` are OOM-killed at 2 GB), unless its compose file +sets `deploy.resources.limits.memory`, which `docker compose` honours and +dokimi reads as the task's own value. `DOKIMI_TBENCH_V1_MEMORY_MB` changes the +8192 default. The Harbor export writes the same value to `task.toml`. + +`tasks list` and `tasks run` still read only the first three. The `docker-*` harnesses mount the oracle at `/solution` and `/oracle` and the verifier at `/tests` and `/verifier`, because a script written for one layout diff --git a/dokimi/docs/contributing-terminal-bench-task.md b/dokimi/docs/contributing-terminal-bench-task.md index 19c814a82..8008a4ab9 100644 --- a/dokimi/docs/contributing-terminal-bench-task.md +++ b/dokimi/docs/contributing-terminal-bench-task.md @@ -30,6 +30,8 @@ read it. Contribute the task itself to the content repository; contribute change to this format here, next to the code that parses it ([`dokimi/harnesses/tasks.py`](../dokimi/harnesses/tasks.py), [`dokimi/harnesses/docker_solver.py`](../dokimi/harnesses/docker_solver.py)). +An existing terminal-bench v1 task (`task.yaml` + `run-tests.sh`) runs without +conversion; see the format table in [coding-harnesses.md](coding-harnesses.md). ## 1 — Pick a task diff --git a/dokimi/dokimi/harnesses/benchflow.py b/dokimi/dokimi/harnesses/benchflow.py index b00236643..fd460fe5b 100644 --- a/dokimi/dokimi/harnesses/benchflow.py +++ b/dokimi/dokimi/harnesses/benchflow.py @@ -44,6 +44,7 @@ from dokimi import process from .command import clear_run_dir +from .harbor_export import harbor_view from .models import HarnessResult, OperationRecord, PreflightError from .tasks import TaskSpec @@ -95,8 +96,7 @@ # the LiteLLM it starts all the same. _BRIDGE_NO_PROXY = ("172.17.0.1", "host.docker.internal") _PROXY_VARS = ("http_proxy", "https_proxy", "HTTP_PROXY", "HTTPS_PROXY") -# Paths bench and docker read from the env; see _subprocess_env. -_PATH_VARS = ("BENCHFLOW_SKILL_CREATOR_DIR", "DOCKER_CONFIG") +_PATH_VARS = ("BENCHFLOW_SKILL_CREATOR_DIR", "DOCKER_CONFIG") # read from the env; see _subprocess_env @dataclass(frozen=True) @@ -122,8 +122,8 @@ def run( timeout_sec: Optional[float] = None, dry_run: bool = False, ) -> HarnessResult: - # bench runs with cwd=: every path it is given must be absolute. - # The row records the dir as given, like a docker-* row. + # bench runs with cwd=: every path absolute; the row records the dir as given. + task = harbor_view(task) # a terminal-bench v1 task runs from its Harbor copy given_dir = Path(output_dir) if output_dir else None output_dir = given_dir.resolve() if given_dir else None if output_dir: diff --git a/dokimi/dokimi/harnesses/command.py b/dokimi/dokimi/harnesses/command.py index a79e37975..3600b9744 100644 --- a/dokimi/dokimi/harnesses/command.py +++ b/dokimi/dokimi/harnesses/command.py @@ -20,6 +20,7 @@ from .models import HarnessResult, OperationRecord from .operations import load_operations +from .harbor_export import harbor_view from .tasks import TaskSpec, harbor_task_problem @@ -64,6 +65,7 @@ def run( dry_run: bool = False, ) -> HarnessResult: if self.kind == "harbor": + task = harbor_view(task) # a terminal-bench v1 task runs from its Harbor copy # Harbor's own rejection of the directory exits 1, which would read # as a fail; refuse first, so the cell is an error (missing data). problem = harbor_task_problem(task, needs_solution=self.agent == "oracle") diff --git a/dokimi/dokimi/harnesses/docker_solver.py b/dokimi/dokimi/harnesses/docker_solver.py index 0d8416392..52d5ae20b 100644 --- a/dokimi/dokimi/harnesses/docker_solver.py +++ b/dokimi/dokimi/harnesses/docker_solver.py @@ -75,13 +75,17 @@ from .container_agent import AGENT_BIN, DOCKER_STEP_TIMEOUT, FILES_DIR, ContainerSandbox, Docker from .models import HarnessResult, OperationRecord from .network_policy import (DEFAULTS, NetworkPlan, PhasePolicy, build_args, enter_phase, needs_sandbox, - proxy_env_flags, resolve, resolve_recorded, skill_needs, start_args, redact as _redact) + resolve_recorded, skill_needs, start_args, redact as _redact) from .skill_treatment import Treatment, build_plugin, check_skill, skill_listing, treatment_notice from .solver_stream import record_solver, run_solver +from .task_env import _exec_env, unresolved_env # noqa: F401 (re-exported) +from .tbench_v1 import reward as _pytest_reward from .tasks import ( # noqa: F401 (timeouts re-exported) _DEFAULT_BUILD_TIMEOUT, _DEFAULT_VERIFY_TIMEOUT, FORMAT_TASK_MD, + FORMAT_TBENCH_V1, + ContainerStep, TaskSpec, _task_build_timeout, _task_verify_timeout, @@ -298,7 +302,7 @@ def _build_image(self, task: TaskSpec, image: str, log, net: Optional[PhasePolic built from a copy of ``environment/`` without that directory — the same rule BenchFlow applies to its no-skill runs. """ - env_dir = task.path / "environment" + env_dir = task.build_context flags = build_args(net or DEFAULTS["build"]) # Honor the task's own build_timeout_sec (some images need >10 min); # _DEFAULT_BUILD_TIMEOUT is only the fallback when the task omits it. @@ -318,7 +322,7 @@ def _build_image(self, task: TaskSpec, image: str, log, net: Optional[PhasePolic (context / "skills").mkdir() self._docker( ["build", "-q", *flags, "-t", image, - "-f", str(context / "Dockerfile"), str(context)], + "-f", str(context / task.dockerfile), str(context)], log, timeout=timeout, ) @@ -347,8 +351,9 @@ def _start_container(self, task: TaskSpec, image: str, container: str, log, lifetime = int(max(1800.0, (task.timeout_sec or _DEFAULT_SOLVE_TIMEOUT) + _task_verify_timeout(task) + 600)) self._docker(["rm", "-f", container], log, check=False) + env_args = [a for k, v in task.container_env.items() for a in ("-e", f"{k}={v}")] self._docker( - ["run", "-d", "--name", container, *net_args, + ["run", "-d", "--name", container, *net_args, *env_args, "--cpus", cpus, "--memory", memory, image, "sleep", str(lifetime)], log, ) @@ -367,10 +372,10 @@ def _solve_oracle(self, task: TaskSpec, container: str, log, net: Optional[Phase ``/solution`` (Harbor) and ``/oracle`` (BenchFlow), because a script written for one layout refers to its siblings by that layout's path. """ - _copy_dir_into(task.solution_dir, container, ("/solution", "/oracle"), log, self._docker) + step = task.oracle_step + _copy_into(step, container, log, self._docker) self._docker( - ["exec", "-u", "root", *_exec_env(task, "oracle", net), container, "bash", "-lc", - "chmod +x /solution/solve.sh && /solution/solve.sh"], + ["exec", *_user(step), *_exec_env(task, "oracle", net), container, "bash", "-lc", step.command], log, check=False, timeout=timeout, ) @@ -627,16 +632,21 @@ def _verify(self, task: TaskSpec, container: str, output_dir: Path, log, SkillsBench ``test.sh`` runs ``pytest /verifier/test_outputs.py`` even after ``bench tasks export`` has renamed the directory to ``tests/``. """ - _copy_dir_into(task.tests_dir, container, ("/tests", "/verifier"), log, self._docker) + step = task.verifier_step + _copy_into(step, container, log, self._docker) # After an in-container agent the verifier runs its /app code as root: no # proxy credentials then (the sidecar kept them from the agent; see :func:`_exec_env`). agent_code = self.mode == "claude" and self.agent_location == "container" proc = self._docker( - ["exec", "-u", "root", *_exec_env(task, "verifier", net, credentials=not agent_code), - container, "bash", "-lc", - "mkdir -p /logs/verifier && chmod +x /tests/test.sh && /tests/test.sh"], + ["exec", *_user(step), *_exec_env(task, "verifier", net, credentials=not agent_code), + container, "bash", "-lc", step.command], log, check=False, timeout=_task_verify_timeout(task), ) + if step.parser != "reward-file": # terminal-bench: the pytest summary is the verdict + verdict, tests = _pytest_reward((proc.stdout or "") + (proc.stderr or "")) + (output_dir / "verifier_tests.json").write_text(json.dumps( + {"parser": step.parser, "exit_code": proc.returncode, "tests": tests}, indent=2), encoding="utf-8") + return verdict reward_file = output_dir / "reward.txt" pulled = self._docker( ["cp", f"{container}:/logs/verifier/reward.txt", str(reward_file)], @@ -736,16 +746,20 @@ def _refuse_unrunnable(task: TaskSpec, mode: str, agent_location: str) -> None: that never runs. It copies neither directory. Oracle mode is refused only for a missing directory. """ + problem = task.local_docker_problem(oracle=mode == "oracle") + if problem: + raise ValueError(f"{task.name}: a docker-* harness cannot run it: {problem}") if mode != "probe": - needed = [("tests", task.tests_dir)] + ([("solution", task.solution_dir)] if mode == "oracle" else []) - for role, directory in needed: - if not directory.is_dir(): - raise ValueError(f"{task.name}: no {directory.name}/ directory to copy to /{role}") + steps = [task.verifier_step] + ([task.oracle_step] if mode == "oracle" else []) + for src, dest in (c for step in steps for c in step.copies): + if not src.exists(): + what = f"{src.name}/ directory" if src in (task.solution_dir, task.tests_dir) else src.name + raise ValueError(f"{task.name}: no {what} to copy to {dest}") if mode not in ("claude", "probe"): return - if agent_location == "host" and task.format == FORMAT_TASK_MD: + if agent_location == "host" and task.format in (FORMAT_TASK_MD, FORMAT_TBENCH_V1): raise ValueError( - f"{task.name}: a task.md task cannot be solved by a host-side agent " + f"{task.name}: a {task.format} task cannot be solved by a host-side agent " f"(its instruction names absolute container paths); {_IN_CONTAINER}" ) declared = _declared_workdir(task) @@ -819,59 +833,18 @@ def _task_workdir(task: TaskSpec, container: str, log, docker: "Docker") -> str: return found if found != "/" else "/app" -def _copy_dir_into(src: Path, container: str, dests: tuple[str, ...], log, docker: "Docker") -> None: - """Copy *src*'s contents to each dest: ``docker cp src c:dest`` nests ``src`` - inside a dest the image already has (``RUN mkdir -p /tests/...``), so the +def _copy_into(step: ContainerStep, container: str, log, docker: "Docker") -> None: + """Copy a step's files in. A directory's contents go to its dest: ``docker cp src c:dest`` + nests ``src`` inside a dest the image already has (``RUN mkdir -p /tests/...``), so the entry script would be missing. Same as BenchFlow: ``mkdir -p``, then ``src/.``.""" - docker(["exec", "-u", "root", container, "mkdir", "-p", *dests], log) - for dest in dests: - docker(["cp", f"{src}/.", f"{container}:{dest}"], log) - - -def _exec_env(task: TaskSpec, table: str, net: Optional[PhasePolicy] = None, *, - credentials: bool = True) -> list[str]: - """``docker exec -e`` flags for the oracle or verifier step. - - First the proxy variables of the step's network policy (:func:`.network_policy.proxy_env_flags`), - on the exec, not on ``docker run``, so they reach only that step: a public verifier behind - a corporate proxy that installs pytest at verify time (52 of 101 SkillsBench ``test.sh``) - otherwise fails with reward 0. With ``credentials=False`` (the verifier after an - in-container agent runs the agent's code) their ``user:pw@`` is dropped. Without *net*, - the task's own plan. - - Then the task's ``[verifier.env]`` / ``[oracle.env]``: only literal values - are forwarded; BenchFlow's ``${VAR}`` host references are left to - BenchFlow, so a secret is never read from our environment on a task - file's say-so. - """ - if net is None: - plan = resolve(task) - net = plan.verify if table == "verifier" else plan.solve - flags = proxy_env_flags(net, credentials=credentials) - source = task.verifier if table == "verifier" else task.oracle - env = source.get("env") - if not isinstance(env, dict): - return flags - for key, value in env.items(): - if isinstance(value, bool): - flags += ["-e", f"{key}={'true' if value else 'false'}"] # YAML true, not Python True - elif isinstance(value, (str, int, float)) and "${" not in str(value): - flags += ["-e", f"{key}={value}"] - return flags - - -def unresolved_env(task: TaskSpec, table: str) -> list[str]: - """Names in ``[verifier.env]`` / ``[oracle.env]`` that :func:`_exec_env` - does not forward: a ``${VAR}`` host reference (with or without a default), - or a value that is not a scalar (list, table, null).""" - source = task.verifier if table == "verifier" else task.oracle - env = source.get("env") - if not isinstance(env, dict): - return [] - return sorted( - str(k) for k, v in env.items() - if not isinstance(v, (str, int, float)) or (isinstance(v, str) and "${" in v) - ) + dirs = [d if src.is_dir() else posixpath.dirname(d) for src, d in step.copies] + docker(["exec", "-u", "root", container, "mkdir", "-p", *dict.fromkeys(dirs)], log) + for src, dest in step.copies: + docker(["cp", f"{src}/." if src.is_dir() else str(src), f"{container}:{dest}"], log) + + +def _user(step: ContainerStep) -> list[str]: + return ["-u", step.user] if step.user else [] def _ignore_top_level_skills(env_dir: Path): diff --git a/dokimi/dokimi/harnesses/harbor_export.py b/dokimi/dokimi/harnesses/harbor_export.py new file mode 100644 index 000000000..bf83c0d71 --- /dev/null +++ b/dokimi/dokimi/harnesses/harbor_export.py @@ -0,0 +1,198 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Write a non-Harbor task in the Harbor layout, only when a Harbor-backed harness runs it. + +The :class:`~.tasks.TaskSpec` is the task; Harbor's directory layout is one +place it can be written to. The docker-* harnesses run a terminal-bench v1 +task as it is; ``harbor run`` and ``bench eval run`` read ``task.toml`` + +``instruction.md`` + ``environment/`` + ``tests/test.sh`` (+ ``solution/``), so +:func:`harbor_view` writes that copy for them, once per task content, into a +cache keyed by a hash of the task's files. + +The mapping follows Harbor's own ``harbor.mappers.terminal_bench`` (harbor +6099b8c, 2026-10-23): the build context goes to ``environment/``, with +``ENV TEST_DIR=/tests``, the compose environment and ``working_dir`` appended to +the Dockerfile; a compose file with more than the client service is kept, +its client renamed ``main`` with Harbor's build/image/resources; ``solution.sh`` +becomes ``solution/solve.sh``; ``tests/`` is copied and ``run-tests.sh`` +wrapped as ``tests/test.sh``. One difference: Harbor's mapper writes the reward +from ``run-tests.sh``'s exit code, while terminal-bench reads the pytest +summary (:func:`.tbench_v1.reward`). Here ``test.sh`` applies the summary rule +too (the awk below), so a task gets the same verdict on every backend. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import tempfile +from dataclasses import replace +from pathlib import Path +from typing import Any + +import yaml + +from . import tbench_v1 +from .tasks import FORMAT_HARBOR, FORMAT_TBENCH_V1, TaskSpec + +#: Bump when the written layout changes, so a cached export is rewritten. +EXPORT_VERSION = "1" +_SOURCE_ONLY = {tbench_v1.CONFIG, tbench_v1.COMPOSE, "solution.sh", "solution.yaml", "run-tests.sh", "tests"} + +# terminal-bench's PytestParser + _is_resolved in awk: a summary section must +# be present and list no FAILED/XPASS/ERROR test. +_VERDICT_AWK = ( + r"""awk 'BEGIN{s=0;f=0} !s && tolower($0) ~ /=+[ \t]*short test summary info[ \t]*=+/ {s=1; next} """ + r"""s {st=$1; sub(/:$/,"",st); if ((st=="FAILED"||st=="XPASS"||st=="ERROR") && NF>1) f=1} """ + r"""END{print (s && !f) ? 1 : 0}'""" +) +_TEST_SH = f"""#!/bin/bash +# Written by dokimi from a terminal-bench v1 task: run-tests.sh, scored as terminal-bench scores it. +export TEST_DIR=/tests +mkdir -p /logs/verifier +bash /tests/run-tests.sh 2>&1 | tee /logs/verifier/terminal-bench-output.txt +code=${{PIPESTATUS[0]}} +{_VERDICT_AWK} /logs/verifier/terminal-bench-output.txt > /logs/verifier/reward.txt +exit $code +""" +_HARBOR_MAIN = { + "build": {"context": "${CONTEXT_DIR}"}, + "image": "${MAIN_IMAGE_NAME}", + "command": ["sh", "-c", "sleep infinity"], + "environment": ["TEST_DIR=${TEST_DIR}"], + "deploy": {"resources": {"limits": {"cpus": "${CPUS}", "memory": "${MEMORY}"}}}, +} +_HARBOR_REPLACES = {"build", "image", "container_name", "command", "environment", "volumes", "deploy"} + + +def harbor_view(task: TaskSpec, cache_root: Path | None = None) -> TaskSpec: + """*task* as a Harbor-layout task: itself, or its cached Harbor copy for a terminal-bench v1 task.""" + if task.format != FORMAT_TBENCH_V1: + return task + root = Path(cache_root) if cache_root else default_cache_root() + dest = root / _content_key(task.path, _task_toml(task)) / task.name + if not (dest / "task.toml").is_file(): + dest.parent.mkdir(parents=True, exist_ok=True) + staging = Path(tempfile.mkdtemp(prefix=".staging-", dir=dest.parent)) / task.name + try: + export(task, staging) + os.replace(staging, dest) # atomic: a concurrent run sees all of it or none + except OSError: + if not (dest / "task.toml").is_file(): + raise + finally: + shutil.rmtree(staging.parent, ignore_errors=True) + return replace(task, path=dest, format=FORMAT_HARBOR) + + +def default_cache_root() -> Path: + """``~/.cache/dokimi/harbor-export``, private: a shared temp dir lets another user plant an export.""" + root = Path.home() / ".cache" / "dokimi" / "harbor-export" + root.mkdir(mode=0o700, parents=True, exist_ok=True) + return root + + +def _content_key(path: Path, resolved: str = "") -> str: + """Hash of the task's files and of *resolved*, what the export writes from runtime state.""" + digest = hashlib.sha256(EXPORT_VERSION.encode() + b"\0" + resolved.encode() + b"\0") + for file in sorted(p for p in path.rglob("*") if p.is_file() and not p.is_symlink()): + digest.update(file.relative_to(path).as_posix().encode() + b"\0") + digest.update(hashlib.sha256(file.read_bytes()).digest()) + return digest.hexdigest()[:16] + + +def export(task: TaskSpec, dest: Path) -> Path: + """Write the terminal-bench v1 *task* as a Harbor task directory at *dest* (which must not exist).""" + if task.format != FORMAT_TBENCH_V1: + raise ValueError(f"{task.name}: only a {FORMAT_TBENCH_V1} task is exported, not {task.format}") + tb = task.environment.get("terminal_bench") or {} + if tb.get("solution") == "solution.yaml": + raise ValueError(f"{task.name}: solution.yaml (tmux keystrokes) has no Harbor equivalent; " + "Harbor's own mapper refuses it too") + dest.mkdir(parents=True) + env_dir = dest / "environment" + _copy_context(task, env_dir) + dockerfile = env_dir / "Dockerfile" # the one path every Harbor environment builds + if (env_dir / task.dockerfile).resolve() != dockerfile.resolve(): + if dockerfile.exists(): + raise ValueError(f"{task.name}: the build context has a Dockerfile besides {task.dockerfile}") + (env_dir / task.dockerfile).rename(dockerfile) + _append_dockerfile(dockerfile, tb) + compose = _harbor_compose(task) + if compose is not None: + (env_dir / "docker-compose.yaml").write_text(yaml.safe_dump(compose, sort_keys=False), encoding="utf-8") + if (task.path / "solution.sh").is_file(): + (dest / "solution").mkdir() + shutil.copy(task.path / "solution.sh", dest / "solution" / "solve.sh") + tests = dest / "tests" + if task.tests_dir.is_dir(): + shutil.copytree(task.tests_dir, tests, symlinks=True) + tests.mkdir(exist_ok=True) + shutil.copy(task.path / "run-tests.sh", tests / "run-tests.sh") + (tests / "test.sh").write_text(_TEST_SH, encoding="utf-8") + for script in (tests / "test.sh", tests / "run-tests.sh", dest / "solution" / "solve.sh"): + if script.is_file(): + script.chmod(0o755) + instruction = task.instruction if task.instruction.endswith("\n") else task.instruction + "\n" + (dest / "instruction.md").write_text(instruction, encoding="utf-8") + (dest / "task.toml").write_text(_task_toml(task), encoding="utf-8") + return dest + + +def _copy_context(task: TaskSpec, env_dir: Path) -> None: + """The task's files into ``environment/``, the build context's at its top, as Harbor's mapper does. + + Left out: the files only the runner reads (``task.yaml``, ``run-tests.sh``, ...), except + ``tests/`` and ``solution.sh`` when the Dockerfile copies them. + """ + context = task.build_context + dockerfile = (context / task.dockerfile).read_text(encoding="utf-8", errors="replace") + keep = {n for n in ("tests", "solution.sh") for verb in ("COPY", "ADD") + if f"{verb} {n}" in dockerfile or f"{verb} ./{n}" in dockerfile} + inside = context.resolve() != task.path.resolve() + skip = (_SOURCE_ONLY - keep) | ({context.relative_to(task.path).parts[0]} if inside else set()) + shutil.copytree(task.path, env_dir, symlinks=True, + ignore=lambda d, names: [n for n in names if Path(d) == task.path and n in skip]) + if inside: + shutil.copytree(context, env_dir, symlinks=True, dirs_exist_ok=True) + + +def _append_dockerfile(dockerfile: Path, tb: dict[str, Any]) -> None: + lines = ["ENV TEST_DIR=/tests"] + [f"ENV {k}={json.dumps(v)}" for k, v in (tb.get("env") or {}).items()] + if tb.get("workdir"): + lines.append(f"WORKDIR {tb['workdir']}") + text = dockerfile.read_text(encoding="utf-8") + dockerfile.write_text(text.rstrip("\n") + "\n\n# Mapped from docker-compose.yaml by dokimi\n" + + "\n".join(lines) + "\n", encoding="utf-8") + + +def _harbor_compose(task: TaskSpec) -> dict[str, Any] | None: + """Harbor's compose for a multi-container task, or ``None`` when the Dockerfile is enough.""" + tb = task.environment.get("terminal_bench") or {} + if not tb.get("unsupported"): + return None + data = yaml.safe_load((task.path / tbench_v1.COMPOSE).read_text(encoding="utf-8")) or {} + services = data.get("services") or {} + main = dict(_HARBOR_MAIN) + main.update({k: v for k, v in (services.get(tb["service"]) or {}).items() if k not in _HARBOR_REPLACES}) + out: dict[str, Any] = {k: v for k, v in data.items() if k not in ("services", "version")} + out["services"] = {"main": main, **{n: s for n, s in services.items() if n != tb["service"]}} + return json.loads(json.dumps(out).replace("${T_BENCH_TASK_DOCKER_NAME_PREFIX}", "hb")) + + +def _task_toml(task: TaskSpec) -> str: + def value(v: Any) -> str: + # A TOML basic string, number, bool or array of them; ASCII escaping would + # write a non-BMP character as a surrogate pair, which TOML rejects. + return json.dumps(v, ensure_ascii=False) + + lines = ["[metadata]"] + lines += [f"{k} = {value(v)}" for k, v in task.metadata.items() if v is not None] + lines += ["", "[verifier]", f"timeout_sec = {float(task.verifier['timeout_sec'])}"] + lines += ["", "[agent]", f"timeout_sec = {float(task.agent['timeout_sec'])}"] + if task.environment.get("memory_mb"): + lines += ["", "[environment]", f"memory_mb = {int(task.environment['memory_mb'])}"] + return "\n".join(lines) + "\n" diff --git a/dokimi/dokimi/harnesses/task_env.py b/dokimi/dokimi/harnesses/task_env.py new file mode 100644 index 000000000..36b016c66 --- /dev/null +++ b/dokimi/dokimi/harnesses/task_env.py @@ -0,0 +1,60 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The environment the docker-* harnesses give a task's oracle and verifier steps.""" + +from __future__ import annotations + +from typing import Optional + +from .network_policy import PROXY_VARS as _PROXY_VARS, _USERINFO_RE, PhasePolicy, proxy_env_flags, resolve # noqa: F401 +from .tasks import TaskSpec + + +def _exec_env(task: TaskSpec, table: str, net: Optional[PhasePolicy] = None, *, + credentials: bool = True) -> list[str]: + """``docker exec -e`` flags for the oracle or verifier step. + + A public-network task gets the host's proxy variables: behind a corporate + proxy a verifier that installs ``uv`` or pytest at verify time (52 of 101 + SkillsBench ``test.sh`` install both, 90 install something) otherwise fails + with reward 0 (measured on ``parallel-tfidf-search``). They go on the exec, + not on ``docker run``, so they reach only the steps the task's + ``network_mode`` is about. With ``credentials=False`` (the verifier after + an in-container agent, which runs the agent's code) their ``user:pw@`` is + dropped: the egress sidecar holds the credentials so the agent never sees + them, and a planted ``conftest.py`` must not read them at verify time. + + Then the task's ``[verifier.env]`` / ``[oracle.env]``: only literal values + are forwarded; BenchFlow's ``${VAR}`` host references are left to + BenchFlow, so a secret is never read from our environment on a task + file's say-so. + """ + if net is None: # the step's policy from the task alone (see .network_policy) + plan = resolve(task) + net = plan.verify if table == "verifier" else plan.solve + flags = proxy_env_flags(net, credentials=credentials) + source = task.verifier if table == "verifier" else task.oracle + env = source.get("env") + if not isinstance(env, dict): + return flags + for key, value in env.items(): + if isinstance(value, bool): + flags += ["-e", f"{key}={'true' if value else 'false'}"] # YAML true, not Python True + elif isinstance(value, (str, int, float)) and "${" not in str(value): + flags += ["-e", f"{key}={value}"] + return flags + + +def unresolved_env(task: TaskSpec, table: str) -> list[str]: + """Names in ``[verifier.env]`` / ``[oracle.env]`` that :func:`_exec_env` + does not forward: a ``${VAR}`` host reference (with or without a default), + or a value that is not a scalar (list, table, null).""" + source = task.verifier if table == "verifier" else task.oracle + env = source.get("env") + if not isinstance(env, dict): + return [] + return sorted( + str(k) for k, v in env.items() + if not isinstance(v, (str, int, float)) or (isinstance(v, str) and "${" in v) + ) diff --git a/dokimi/dokimi/harnesses/tasks.py b/dokimi/dokimi/harnesses/tasks.py index 4b696efc9..a3d26da64 100644 --- a/dokimi/dokimi/harnesses/tasks.py +++ b/dokimi/dokimi/harnesses/tasks.py @@ -20,6 +20,13 @@ is a BenchFlow document (every top-level key one BenchFlow knows); any other ``task.md`` there is an author's note and the task loads from ``task.toml``. When both definitions load, :func:`legacy_drift` names where they differ. + +``terminal-bench-v1`` + terminal-bench 0.x: ``task.yaml`` + ``Dockerfile``/``docker-compose.yaml`` + + ``solution.sh`` + ``run-tests.sh`` + ``tests/``, read by :mod:`.tbench_v1`. + The docker-* harnesses run it as it is (:attr:`TaskSpec.oracle_step`, + :attr:`TaskSpec.verifier_step`); the Harbor- and BenchFlow-backed ones get + a Harbor copy (:mod:`.harbor_export`). """ from __future__ import annotations @@ -34,6 +41,7 @@ import yaml +from . import tbench_v1 # Executable tasks live in a separate evaluation-content checkout, so there is # no in-repo default root: the caller always names one. @@ -44,14 +52,34 @@ FORMAT_HARBOR = "harbor" FORMAT_TASK_MD = "task-md" +FORMAT_TBENCH_V1 = tbench_v1.FORMAT +PARSER_REWARD_FILE = "reward-file" # Where each layout keeps its oracle and its verifier, in lookup order. Both # names are tried for either format, as BenchFlow accepts both. The order # differs from BenchFlow's in one case: BenchFlow always prefers oracle/ and # verifier/, while a Harbor-layout directory that holds both pairs uses # solution/ and tests/ here, the names Harbor itself reads. -_SOLUTION_DIRS = {FORMAT_HARBOR: ("solution", "oracle"), FORMAT_TASK_MD: ("oracle", "solution")} -_TESTS_DIRS = {FORMAT_HARBOR: ("tests", "verifier"), FORMAT_TASK_MD: ("verifier", "tests")} +_SOLUTION_DIRS = {FORMAT_HARBOR: ("solution", "oracle"), FORMAT_TASK_MD: ("oracle", "solution"), + FORMAT_TBENCH_V1: ("solution",)} +_TESTS_DIRS = {FORMAT_HARBOR: ("tests", "verifier"), FORMAT_TASK_MD: ("verifier", "tests"), + FORMAT_TBENCH_V1: ("tests",)} + + +@dataclass(frozen=True) +class ContainerStep: + """What a docker-* run copies into the task container and runs, for the oracle or the verifier. + + ``copies`` maps a host file or directory to a container path (a + directory's contents go into it); ``user`` ``None`` is the image's own + user; ``parser`` says how the verdict is read: the reward file, or + ``pytest`` (terminal-bench's summary parser, :func:`.tbench_v1.reward`). + """ + + copies: tuple[tuple[Path, str], ...] + command: str + user: str | None = "root" + parser: str = PARSER_REWARD_FILE @dataclass(frozen=True) @@ -82,6 +110,55 @@ def solution_dir(self) -> Path: def tests_dir(self) -> Path: return _first_dir(self.path, _TESTS_DIRS[self.format]) + @property + def _tb(self) -> dict[str, Any]: + return _table(self.environment.get("terminal_bench")) if self.format == FORMAT_TBENCH_V1 else {} + + @property + def build_context(self) -> Path: + """The ``docker build`` context: ``environment/``, or the compose file's.""" + if self.format == FORMAT_TBENCH_V1: + return self.path / str(self._tb.get("build_context") or ".") + return self.path / "environment" + + @property + def dockerfile(self) -> str: + """The Dockerfile, relative to :attr:`build_context`.""" + return str(self._tb.get("dockerfile") or "Dockerfile") + + @property + def container_env(self) -> dict[str, str]: + """Environment the task container starts with (terminal-bench's ``TEST_DIR`` and compose env).""" + if self.format != FORMAT_TBENCH_V1: + return {} + return {**_table(self._tb.get("env")), "TEST_DIR": tbench_v1.TEST_DIR} + + @property + def oracle_step(self) -> ContainerStep: + if self.format == FORMAT_TBENCH_V1: + return ContainerStep(((self.path / "solution.sh", "/oracle/solution.sh"),), + "bash /oracle/solution.sh", user=None) + return ContainerStep(((self.solution_dir, "/solution"), (self.solution_dir, "/oracle")), + "chmod +x /solution/solve.sh && /solution/solve.sh") + + @property + def verifier_step(self) -> ContainerStep: + if self.format == FORMAT_TBENCH_V1: + copies: tuple[tuple[Path, str], ...] = ((self.path / "run-tests.sh", f"{tbench_v1.TEST_DIR}/run-tests.sh"),) + if self.tests_dir.is_dir(): + copies += ((self.tests_dir, tbench_v1.TEST_DIR),) + # 2>&1: one ordered stream, as terminal-bench reads it from the tmux pane + return ContainerStep(copies, f"bash {tbench_v1.TEST_DIR}/run-tests.sh 2>&1", user=None, + parser=tbench_v1.PARSER_PYTEST) + return ContainerStep(((self.tests_dir, "/tests"), (self.tests_dir, "/verifier")), + "mkdir -p /logs/verifier && chmod +x /tests/test.sh && /tests/test.sh") + + def local_docker_problem(self, *, oracle: bool) -> str | None: + """Why a docker-* harness cannot run this task as its own runner would, or ``None``.""" + if self.format != FORMAT_TBENCH_V1: + return None + return tbench_v1.local_docker_problem(self._tb, oracle=oracle) + @property def skills_dir(self) -> Path | None: """Task-bundled skills (``environment/skills``), if the task ships any.""" @@ -117,6 +194,8 @@ def task_format(path: Path) -> str | None: return FORMAT_TASK_MD if has_toml: return FORMAT_HARBOR + if tbench_v1.is_task_dir(path): + return FORMAT_TBENCH_V1 return None @@ -207,10 +286,25 @@ def discover_tasks(root: Path | None = None) -> list[TaskSpec]: # that directory still raises. _warn_once(f"skipping {path}: task.md has no YAML frontmatter") continue + if fmt == FORMAT_TBENCH_V1 and not tbench_v1.has_instruction(path, _load_yaml): + _warn_once(f"skipping {path}: task.yaml has no instruction") # some other tool's task.yaml + continue tasks.append(load_task(path)) return tasks +def _read_tables(path: Path, fmt: str) -> tuple[dict[str, Any], str]: + """The raw tables and instruction of the task at *path*, read by its *fmt*.""" + if fmt == FORMAT_TASK_MD: + return _read_task_md(path / "task.md") + if fmt == FORMAT_TBENCH_V1: + return tbench_v1.read(path, _load_yaml) + instruction_path = path / "instruction.md" + if not instruction_path.is_file(): + raise FileNotFoundError(f"Task has no instruction.md: {path}") + return read_task_config(path), instruction_path.read_text(encoding="utf-8") + + def load_task(path_or_name: str | Path, root: Path | None = None) -> TaskSpec: """Load one task by directory path, or by task name relative to *root*. @@ -231,16 +325,9 @@ def load_task(path_or_name: str | Path, root: Path | None = None) -> TaskSpec: raise FileNotFoundError(f"Task directory not found: {path_or_name}") fmt = task_format(path) - if fmt == FORMAT_TASK_MD: - data, instruction = _read_task_md(path / "task.md") - elif fmt == FORMAT_HARBOR: - instruction_path = path / "instruction.md" - if not instruction_path.is_file(): - raise FileNotFoundError(f"Task has no instruction.md: {path}") - data = read_task_config(path) - instruction = instruction_path.read_text(encoding="utf-8") - else: - raise FileNotFoundError(f"Task has no task.toml or task.md: {path}") + if fmt is None: + raise FileNotFoundError(f"Task has no task.toml, task.md or task.yaml: {path}") + data, instruction = _read_tables(path, fmt) # BenchFlow schema 1.3 renamed [environment] to [sandbox]; either feeds # TaskSpec.environment. BenchFlow rejects 'environment:' inside a task.md; @@ -290,7 +377,9 @@ def read_task_config(path: Path) -> dict[str, Any]: return _read_task_md(path / "task.md")[0] if fmt == FORMAT_HARBOR: return _read_task_toml(path) - raise FileNotFoundError(f"Task has no task.toml or task.md: {path}") + if fmt == FORMAT_TBENCH_V1: + return tbench_v1.read(path, _load_yaml)[0] + raise FileNotFoundError(f"Task has no task.toml, task.md or task.yaml: {path}") def _read_task_toml(path: Path) -> dict[str, Any]: diff --git a/dokimi/dokimi/harnesses/tbench_v1.py b/dokimi/dokimi/harnesses/tbench_v1.py new file mode 100644 index 000000000..f2d5c434a --- /dev/null +++ b/dokimi/dokimi/harnesses/tbench_v1.py @@ -0,0 +1,305 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""terminal-bench v1 tasks (``task.yaml``), read as they are, no conversion. + +A terminal-bench 0.x task (laude-institute/terminal-bench, ``original-tasks/``) +is a directory with ``task.yaml``, ``Dockerfile`` and ``docker-compose.yaml``, +``solution.sh`` (or ``solution.yaml``), ``run-tests.sh`` and ``tests/``. This +module reads one into the tables of a :class:`~.tasks.TaskSpec` and parses its +verifier output; :mod:`.tasks` builds the spec, :mod:`.harbor_export` writes the +Harbor layout for the Harbor- and BenchFlow-backed harnesses. + +The semantics follow terminal-bench's own harness (``terminal_bench/``, commit +d28711d of 2026-07-10): + +- ``handlers/trial_handler.py`` ``Task``: the ``task.yaml`` fields and their + defaults (``max_agent_timeout_sec`` 360, ``max_test_timeout_sec`` 60, + ``parser_name`` pytest). +- ``agents/oracle_agent.py``: ``solution.sh`` is copied to + ``/oracle/solution.sh`` and run as ``bash /oracle/solution.sh`` in the + container's shell (its ``WORKDIR``, its user). +- ``harness/harness.py`` ``_setup_test_env``/``_run_tests``: ``run-tests.sh`` + and the *contents* of ``tests/`` are copied into ``/tests``, then + ``bash /tests/run-tests.sh`` runs with ``TEST_DIR=/tests`` (set by the + compose file), within ``max_test_timeout_sec``. +- ``parsers/pytest_parser.py`` and ``harness._is_resolved``: the verdict is + read from the ``short test summary info`` of ``pytest -rA``, not from the + exit code; a task passes when every listed test passed (skipped and xfail + count as passed), and output without that section is a fail. + +Memory: terminal-bench sets no limit of its own; a compose file may set one +(``deploy.resources.limits.memory``, honoured by ``docker compose``). That +limit becomes ``environment.memory_mb``; a task without one gets +:data:`DEFAULT_MEMORY_MB` (8 GB, not the 2 GB default of other formats, which +OOM-kills tasks such as predicate-pushdown-bench), or +``$DOKIMI_TBENCH_V1_MEMORY_MB`` when set. +""" + +from __future__ import annotations + +import os +import re +import shlex +from collections.abc import Callable +from pathlib import Path +from typing import Any + +FORMAT = "terminal-bench-v1" +CONFIG = "task.yaml" +COMPOSE = "docker-compose.yaml" +TEST_DIR = "/tests" +PARSER_PYTEST = "pytest" + +# trial_handler.Task defaults. +_DEFAULT_AGENT_TIMEOUT = 360.0 +_DEFAULT_TEST_TIMEOUT = 60.0 +#: ``environment.memory_mb`` of a task whose compose file sets no memory limit. +DEFAULT_MEMORY_MB = 8192 +MEMORY_ENV = "DOKIMI_TBENCH_V1_MEMORY_MB" +_SIZE_RE = re.compile(r"^\s*(\d+(?:\.\d+)?)\s*([kmgt]?)i?b?\s*$", re.IGNORECASE) +_SIZE_MB = {"": 1 / (1024 * 1024), "k": 1 / 1024, "m": 1, "g": 1024, "t": 1024 * 1024} +_METADATA_KEYS = ( + "author_name", "author_email", "difficulty", "category", "tags", + "estimated_duration_sec", "expert_time_estimate_min", "junior_time_estimate_min", +) +# What terminal-bench's own compose files give the client service; the docker-* +# harnesses provide the same (TEST_DIR, the logs mounts are not used). +_DEFAULT_ENV = {"TEST_DIR=${T_BENCH_TEST_DIR}"} +_DEFAULT_VOLUMES = { + "${T_BENCH_TASK_LOGS_PATH}:${T_BENCH_CONTAINER_LOGS_PATH}", + "${T_BENCH_TASK_AGENT_LOGS_PATH}:${T_BENCH_CONTAINER_AGENT_LOGS_PATH}", +} +# Service keys a single `docker run` reproduces or that do not change the +# verdict (resource limits, a TTY, the restart policy); command, build and +# environment within what _not_reproduced allows. +_RUN_KEYS = { + "build", "image", "container_name", "command", "environment", "volumes", "working_dir", + "deploy", "tty", "stdin_open", "restart", "platform", "expose", +} +_IDLE_COMMANDS = (["sleep", "infinity"], ["sh", "-c", "sleep infinity"], ["bash", "-c", "sleep infinity"]) +_SIGNALS = ("run-tests.sh", COMPOSE, "Dockerfile", "solution.sh", "solution.yaml") + + +def is_task_dir(path: Path) -> bool: + """A ``task.yaml`` next to at least one other terminal-bench file.""" + return (path / CONFIG).is_file() and any((path / n).is_file() for n in _SIGNALS) + + +def has_instruction(path: Path, load_yaml: Callable[[str], Any]) -> bool: + """False for a ``task.yaml`` that parses as a mapping without an instruction (not a task).""" + try: + config = _load(path / CONFIG, load_yaml) + except ValueError: + return True # broken, not absent: loading it reports why + return not isinstance(config, dict) or bool(config.get("instruction")) + + +def read(path: Path, load_yaml: Callable[[str], Any]) -> tuple[dict[str, Any], str]: + """The TaskSpec tables of the task at *path*, and its instruction. + + ``load_yaml`` parses YAML text (the alias-refusing loader of :mod:`.tasks`). + Raises ``ValueError`` for a ``task.yaml`` or compose file that does not parse + or has no instruction. + """ + config = _mapping(_load(path / CONFIG, load_yaml), CONFIG, path) + instruction = config.get("instruction") + if not isinstance(instruction, str) or not instruction.strip(): + raise ValueError(f"task.yaml has no instruction: {path}") + compose = _mapping(_load(path / COMPOSE, load_yaml), COMPOSE, path) if (path / COMPOSE).is_file() else {} + tb = _compose_layout(compose) + tb["run_tests_in_same_shell"] = bool(config.get("run_tests_in_same_shell")) + tb["parser"] = str(config.get("parser_name") or PARSER_PYTEST) + tb["solution"] = next((n for n in ("solution.sh", "solution.yaml") if (path / n).is_file()), None) + tables = { + "metadata": {k: config[k] for k in _METADATA_KEYS if k in config}, + "agent": {"timeout_sec": _float(config.get("max_agent_timeout_sec"), _DEFAULT_AGENT_TIMEOUT)}, + "verifier": { + "timeout_sec": _float(config.get("max_test_timeout_sec"), _DEFAULT_TEST_TIMEOUT), + "parser": tb["parser"], + }, + # terminal-bench runs on compose's default network, open to the internet: + # run-tests.sh installs curl, uv and pytest at verify time. + "environment": {"network_mode": "public", **({"workdir": tb["workdir"]} if tb.get("workdir") else {}), + "memory_mb": tb.pop("memory_mb") or default_memory_mb(), "terminal_bench": tb}, + } + return tables, instruction + + +def _load(file: Path, load_yaml: Callable[[str], Any]) -> Any: + try: + return load_yaml(file.read_text(encoding="utf-8")) + except RecursionError as exc: + raise ValueError(f"{file.name} is nested too deeply: {file.parent}") from exc + except Exception as exc: # yaml.YAMLError and the alias refusal, without importing yaml here + raise ValueError(f"{file.name} is not valid YAML: {file.parent}: {exc}") from exc + + +def _mapping(data: Any, name: str, path: Path) -> dict[str, Any]: + if data is None: + return {} + if not isinstance(data, dict): + raise ValueError(f"{name} must be a mapping: {path}") + return data + + +def _compose_layout(compose: dict[str, Any]) -> dict[str, Any]: + """The client service of a terminal-bench compose file, as one ``docker run`` sees it. + + ``unsupported`` lists what the docker-* harnesses' single ``docker run`` does + not reproduce (a second service, a volume, a network, ports, tmpfs, a + service command, a build arg, a host environment value): they + refuse such a task, the Harbor export keeps the compose file. + """ + raw = compose.get("services") + services: dict[str, Any] = raw if isinstance(raw, dict) else {} + name = "client" if "client" in services else next(iter(services), None) + found = services.get(name) if name is not None else None + service: dict[str, Any] = found if isinstance(found, dict) else {} + build = service.get("build") + context = build if isinstance(build, str) else (build or {}).get("context", ".") if isinstance(build, dict) else "." + dockerfile = (build.get("dockerfile") if isinstance(build, dict) else None) or "Dockerfile" + unsupported = [f"service {s}" for s in services if s != name] + unsupported += [f"top-level {k}" for k in compose if k not in ("services", "version")] + unsupported += [f"{name}.{k}" for k in service if k not in _RUN_KEYS] + volumes = [v for v in service.get("volumes") or [] if v not in _DEFAULT_VOLUMES] + unsupported += [f"{name}.volumes {v}" for v in volumes] + unsupported += _not_reproduced(str(name), service) + workdir = service.get("working_dir") + return { + "service": name, + "build_context": str(context), + "dockerfile": str(dockerfile), + "env": _env(service.get("environment")), + "workdir": workdir if isinstance(workdir, str) and "${" not in workdir else None, + "unsupported": unsupported, + "memory_mb": _size_mb(_dig(service, "deploy", "resources", "limits", "memory")), + } + + +def _not_reproduced(name: str, service: dict[str, Any]) -> list[str]: + """What of an accepted key one ``docker run`` of the default build stage, idling, does not do. + + A command other than terminal-bench's idle one, a build arg or target, and an + environment value compose reads from the host (a bare name, a ``$`` reference). + """ + out: list[str] = [] + command = service.get("command") + if command is not None and _words(command) not in _IDLE_COMMANDS: + out.append(f"{name}.command") + build = service.get("build") + if isinstance(build, dict): + out += [f"{name}.build.{k}" for k in build if k not in ("context", "dockerfile")] + raw = service.get("environment") + items = ([(str(k), None if v is None else str(v)) for k, v in raw.items()] if isinstance(raw, dict) + else [(e.partition("=")[0], e.partition("=")[2] if "=" in e else None) for e in map(str, raw or [])]) + out += [f"{name}.environment {k}" for k, v in items if k != "TEST_DIR" and (v is None or "$" in v)] + return out + + +def _words(command: Any) -> list[str]: + if not isinstance(command, str): + return [str(w) for w in command] if isinstance(command, list) else [str(command)] + try: + return shlex.split(command) + except ValueError: # an unclosed quote: not the idle command either + return [command] + + +def default_memory_mb() -> int: + """:data:`DEFAULT_MEMORY_MB`, or ``$DOKIMI_TBENCH_V1_MEMORY_MB`` when it is a positive integer.""" + raw = os.environ.get(MEMORY_ENV, "").strip() + if not raw: + return DEFAULT_MEMORY_MB + if not raw.isdigit() or int(raw) <= 0: + raise ValueError(f"{MEMORY_ENV} must be a positive number of MB, not {raw!r}") + return int(raw) + + +def _dig(data: Any, *keys: str) -> Any: + for key in keys: + data = data.get(key) if isinstance(data, dict) else None + return data + + +def _size_mb(value: Any) -> int | None: + """A compose size (``4.0G``, ``512m``, bytes as a number) in MB, or ``None``.""" + if isinstance(value, bool) or value is None: + return None + match = _SIZE_RE.match(str(value)) + if not match: + return None + mb = int(float(match.group(1)) * _SIZE_MB[match.group(2).lower()]) + return mb if mb > 0 else None + + +def _env(value: Any) -> dict[str, str]: + """The service's environment without terminal-bench's own TEST_DIR, which is fixed.""" + items = value.items() if isinstance(value, dict) else ( + (str(e).partition("=")[0], str(e).partition("=")[2]) for e in value or [] if str(e) not in _DEFAULT_ENV) + return {str(k): "" if v is None else str(v) for k, v in items if k != "TEST_DIR"} + + +def _float(value: Any, default: float) -> float: + try: + return float(value) if value is not None else default + except (TypeError, ValueError): + return default + + +def local_docker_problem(tb: dict[str, Any], *, oracle: bool) -> str | None: + """Why a docker-* harness cannot run this task as terminal-bench would, or ``None``.""" + if tb.get("unsupported"): + return (f"its docker-compose.yaml uses what one docker run does not reproduce " + f"({', '.join(tb['unsupported'])}); run it with a Harbor-backed harness") + if tb.get("parser", PARSER_PYTEST) != PARSER_PYTEST: + return f"its parser_name is {tb['parser']}; the docker-* harnesses read pytest output only" + if tb.get("run_tests_in_same_shell"): + return "run_tests_in_same_shell: its tests read the agent's shell, which a docker exec does not keep" + if oracle and tb.get("solution") != "solution.sh": + return ("its oracle is solution.yaml (tmux keystrokes), not solution.sh" if tb.get("solution") + else "no solution.sh to copy to /oracle/solution.sh") + return None + + +# ── verdict ───────────────────────────────────────────────────────────────── + +_SUMMARY_RE = re.compile(r"=+\s*short test summary info\s*=+", re.IGNORECASE) +# PytestTestStatus: passed, skipped and xfail resolve a test; the rest fail it. +_PASSING = {"PASSED", "SKIPPED", "XFAIL"} +_STATUSES = _PASSING | {"FAILED", "XPASS", "ERROR"} + + +def parse_pytest(output: str) -> dict[str, bool] | None: + """terminal-bench's ``PytestParser``: test id -> passed, or ``None`` with no summary. + + Keyed by the full node id, where terminal-bench keys by the part after the + file: there a later ``PASSED b.py::test_x`` hides an earlier ``FAILED + a.py::test_x``, and the export's ``test.sh`` (any failed line fails) would + disagree with the docker-* verdict. + """ + parts = _SUMMARY_RE.split(output, maxsplit=1) + if len(parts) < 2: + return None + results: dict[str, bool] = {} + for line in parts[1].splitlines(): + if line.startswith("FAILED") and " - " in line: + line = " - ".join(line.split(" - ")[:-1]) + fields = line.split(maxsplit=1) + if len(fields) < 2: + continue + status = fields[0].strip().strip(":") + name = fields[1].strip() + if status in _STATUSES and name: + results[name] = status in _PASSING + return results + + +def reward(output: str) -> tuple[float, dict[str, bool] | None]: + """The terminal-bench verdict of a verifier's output: 1.0 when every parsed test passed. + + As in ``harness._is_resolved``, a summary that lists no test resolves the + task (``all()`` of nothing) and output with no summary does not. + """ + results = parse_pytest(output) + return (1.0 if results is not None and all(results.values()) else 0.0), results diff --git a/dokimi/dokimi/task_formats/dirs.py b/dokimi/dokimi/task_formats/dirs.py index fb5e028eb..4d03d1299 100644 --- a/dokimi/dokimi/task_formats/dirs.py +++ b/dokimi/dokimi/task_formats/dirs.py @@ -158,6 +158,19 @@ def describe(self, path: Path) -> Report: if config and not instruction: report.problems.append("task.yaml: no instruction") report.details["metadata"] = _summary(config, ("difficulty", "category", "tags", "parser_name")) + spec = _load(path, report) if config and instruction else None + if spec is None: + return report + # The docker-* harnesses run it as it is; Harbor and BenchFlow from its Harbor export. + problem = spec.local_docker_problem(oracle=False) + if problem: + report.problems.append(f"docker-* would refuse it: {problem}") + else: + report.consumers.append(BY_DOCKER) + if (spec.environment.get("terminal_bench") or {}).get("solution") == "solution.yaml": + report.problems.append("no Harbor export: solution.yaml (tmux keystrokes) has no Harbor equivalent") + else: + report.consumers += [BY_HARBOR, BY_BENCHFLOW] return report diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index 21fd4b51b..d0d35275e 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -2,7 +2,7 @@ "files": { "dokimi/commands/novelty.py": {"lines": 922, "rationale": "W3 adds routing no_queries handling, W8 a routing flag, W10 a --catalog help line, #173 a stdout coverage line and warning; file split tracked separately to avoid conflicts across the W-series"}, "dokimi/harnesses/benchflow.py": {"lines": 869}, - "dokimi/harnesses/docker_solver.py": {"lines": 927, "rationale": "pre-existing when the gate was introduced"}, + "dokimi/harnesses/docker_solver.py": {"lines": 900, "rationale": "pre-existing when the gate was introduced"}, "dokimi/llm.py": {"lines": 1118, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"}, "dokimi/report/model_compare.py": {"lines": 1210, "rationale": "grew on main in #73 (judge-validity --context stored) before the ratchet landed; split it in its own PR"}, "tests/test_arm_runner.py": {"lines": 1176, "rationale": "pre-existing when the gate was introduced"}, diff --git a/dokimi/tests/fixtures/task_formats/MANIFEST.yaml b/dokimi/tests/fixtures/task_formats/MANIFEST.yaml index cc2f1e86b..4e743b36f 100644 --- a/dokimi/tests/fixtures/task_formats/MANIFEST.yaml +++ b/dokimi/tests/fixtures/task_formats/MANIFEST.yaml @@ -54,7 +54,7 @@ items: - path: terminal-bench/original-tasks/hello-world format: terminal-bench-v1 kind: executable - consumers: [none-yet] + consumers: [docker-*, harbor, benchflow] items: null source: terminal-bench upstream: original-tasks/hello-world diff --git a/dokimi/tests/test_task_format_detect.py b/dokimi/tests/test_task_format_detect.py index fae2168f6..0c8beaa9f 100644 --- a/dokimi/tests/test_task_format_detect.py +++ b/dokimi/tests/test_task_format_detect.py @@ -133,10 +133,10 @@ def test_task_without_tests_or_verifier_is_not_claimed_for_docker(tmp_path): assert "verifier" in r.missing and "docker-*" not in r.consumers -def test_terminal_bench_v1_task_has_every_part_and_no_consumer_yet(tmp_path): +def test_terminal_bench_v1_task_has_every_part_and_runs_natively_or_exported(tmp_path): r = _one(_tb1(tmp_path)) assert (r.format, r.kind, r.missing) == ("terminal-bench-v1", "executable", []) - assert r.as_dict()["consumers"] == ["none-yet"] + assert r.as_dict()["consumers"] == ["docker-*", "harbor", "benchflow"] assert r.details["metadata"]["difficulty"] == "easy" @@ -372,11 +372,12 @@ def test_inspect_missing_path_raises(tmp_path): inspect_path(tmp_path / "nope") -def test_existing_discovery_is_unchanged_by_new_formats_in_the_root(tmp_path): +def test_discovery_runs_terminal_bench_v1_next_to_the_other_layouts(tmp_path): _harbor(tmp_path, "a") _task_md(tmp_path, "b") _tb1(tmp_path, "c") - assert [(t.name, t.format) for t in discover_tasks(tmp_path)] == [("a", "harbor"), ("b", "task-md")] + assert [(t.name, t.format) for t in discover_tasks(tmp_path)] == [ + ("a", "harbor"), ("b", "task-md"), ("c", "terminal-bench-v1")] # ── CLI ─────────────────────────────────────────────────────────────────────── @@ -402,7 +403,7 @@ def test_cli_inspect_json(tmp_path, capsys): _run(["tasks", "inspect", str(tmp_path), "--json"]) payload = json.loads(capsys.readouterr().out) assert [i["format"] for i in payload["items"]] == ["terminal-bench-v1"] - assert payload["items"][0]["consumers"] == ["none-yet"] + assert payload["items"][0]["consumers"] == ["docker-*", "harbor", "benchflow"] def test_cli_inspect_shows_ambiguity_and_problems(tmp_path, capsys): diff --git a/dokimi/tests/test_tbench_v1.py b/dokimi/tests/test_tbench_v1.py new file mode 100644 index 000000000..beed569d1 --- /dev/null +++ b/dokimi/tests/test_tbench_v1.py @@ -0,0 +1,449 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""terminal-bench v1 tasks (task.yaml) run as they are by the docker-* harnesses, +and through a cached Harbor export by the Harbor-backed ones.""" + +from __future__ import annotations + +import os +import shlex +import shutil +import subprocess +from pathlib import Path + +import pytest + +from dokimi.harnesses import TaskSuiteRunner, discover_tasks, load_task +from dokimi.harnesses import harbor_export, tbench_v1 +from dokimi.harnesses.network_policy import resolve +from dokimi.process import reply +from dokimi.task_formats import inspect_path + + +def _write(path: Path, text: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + +_COMPOSE = """services: + client: + build: + dockerfile: Dockerfile + image: ${T_BENCH_TASK_DOCKER_CLIENT_IMAGE_NAME} + container_name: ${T_BENCH_TASK_DOCKER_CLIENT_CONTAINER_NAME} + command: [ "sh", "-c", "sleep infinity" ] + environment: + - TEST_DIR=${T_BENCH_TEST_DIR} + - GREETING=hi there + volumes: + - ${T_BENCH_TASK_LOGS_PATH}:${T_BENCH_CONTAINER_LOGS_PATH} + - ${T_BENCH_TASK_AGENT_LOGS_PATH}:${T_BENCH_CONTAINER_AGENT_LOGS_PATH} +""" + + +def _tb1(root: Path, name: str = "hello-world", *, compose: str = _COMPOSE, yaml_extra: str = "") -> Path: + task = root / name + _write(task / "task.yaml", "instruction: |\n Create hello.txt with 'Hello, world!'.\n" + "author_name: a\ndifficulty: easy\ncategory: file-operations\n" + yaml_extra) + _write(task / "Dockerfile", "FROM ubuntu:24.04\nWORKDIR /app\n") + _write(task / "docker-compose.yaml", compose) + _write(task / "run-tests.sh", "#!/bin/bash\ncd /tests && python3 -m pytest -rA test_outputs.py\n") + _write(task / "solution.sh", "echo 'Hello, world!' > hello.txt\n") + _write(task / "tests" / "test_outputs.py", "def test_hello():\n pass\n") + return task + + +_PASS = "===== short test summary info =====\nPASSED test_outputs.py::test_hello\n" +_FAIL = ("===== short test summary info =====\nPASSED test_outputs.py::test_a\n" + "FAILED test_outputs.py::test_b - AssertionError: x\n") +_SAME_NAME = "===== short test summary info =====\nFAILED tests/a.py::test_x\nPASSED tests/b.py::test_x\n" + + +# ── loading ─────────────────────────────────────────────────────────────────── + +def test_load_reads_task_yaml_with_terminal_bench_defaults(tmp_path): + task = load_task(_tb1(tmp_path)) + assert task.format == "terminal-bench-v1" + assert task.instruction.startswith("Create hello.txt") + assert task.agent["timeout_sec"] == 360.0 and task.verifier["timeout_sec"] == 60.0 + assert task.verifier["parser"] == "pytest" + assert task.metadata["difficulty"] == "easy" + assert task.container_env == {"GREETING": "hi there", "TEST_DIR": "/tests"} + assert task.local_docker_problem(oracle=True) is None + assert task.environment["memory_mb"] == tbench_v1.DEFAULT_MEMORY_MB == 8192 + + + +def test_network_is_public_for_oracle_and_verifier_not_for_an_agent(tmp_path): + task = load_task(_tb1(tmp_path)) + oracle, agent = resolve(task), resolve(task, agent=True) + assert oracle.solve.mode == oracle.verify.mode == agent.verify.mode == "public" + assert agent.solve.mode != "public" + + +_LIMITS = " deploy:\n resources:\n limits:\n memory: {}\n" + + +@pytest.mark.parametrize(("limit", "mb"), [("4.0G", 4096), ("512m", 512), ("1GiB", 1024), ("junk", 8192)]) +def test_memory_limit_comes_from_compose_deploy_limits_else_the_tb1_default(tmp_path, limit, mb): + task = load_task(_tb1(tmp_path, compose=_COMPOSE + _LIMITS.format(limit))) + assert task.environment["memory_mb"] == mb + assert "memory_mb" not in task.environment["terminal_bench"] + + +def test_memory_default_is_configurable_by_env(tmp_path, monkeypatch): + monkeypatch.setenv(tbench_v1.MEMORY_ENV, "16384") + assert load_task(_tb1(tmp_path)).environment["memory_mb"] == 16384 + assert load_task(_tb1(tmp_path, "b", compose=_COMPOSE + _LIMITS.format("4G"))).environment["memory_mb"] == 4096 + monkeypatch.setenv(tbench_v1.MEMORY_ENV, "8g") + with pytest.raises(ValueError, match=tbench_v1.MEMORY_ENV): + tbench_v1.default_memory_mb() + + +def test_load_reads_timeouts_and_a_subdir_build_context(tmp_path): + compose = "services:\n client:\n build:\n context: ./env\n dockerfile: Dockerfile.x\n" + path = _tb1(tmp_path, compose=compose, yaml_extra="max_agent_timeout_sec: 900\nmax_test_timeout_sec: 120\n") + _write(path / "env" / "Dockerfile.x", "FROM ubuntu:24.04\n") + task = load_task(path) + assert task.agent["timeout_sec"] == 900.0 and task.verifier["timeout_sec"] == 120.0 + assert task.build_context == path / "env" and task.dockerfile == "Dockerfile.x" + + +@pytest.mark.parametrize("text, match", [ + ("instruction: [unclosed\n", "not valid YAML"), + ("- a list\n", "must be a mapping"), + ("author_name: a\n", "no instruction"), +]) +def test_invalid_task_yaml_is_refused(tmp_path, text, match): + path = _tb1(tmp_path) + _write(path / "task.yaml", text) + with pytest.raises(ValueError, match=match): + load_task(path) + + +def test_discovery_finds_terminal_bench_v1_tasks(tmp_path): + _tb1(tmp_path, "a") + _tb1(tmp_path, "b") + assert [(t.name, t.format) for t in discover_tasks(tmp_path)] == [ + ("a", "terminal-bench-v1"), ("b", "terminal-bench-v1")] + + +def test_discovery_skips_a_task_yaml_without_an_instruction(tmp_path): + _tb1(tmp_path, "hello-world") + _write(tmp_path / "not-a-task" / "task.yaml", "name: foo\nsteps: []\n") + _write(tmp_path / "not-a-task" / "Dockerfile", "FROM ubuntu:24.04\n") + with pytest.warns(UserWarning, match="not-a-task: task.yaml has no instruction"): + assert [t.name for t in discover_tasks(tmp_path)] == ["hello-world"] + with pytest.raises(ValueError, match="no instruction"): + load_task(tmp_path / "not-a-task") + + +def _compose_with(old: str, new: str): + return lambda p: _write(p / "docker-compose.yaml", _COMPOSE.replace(old, new)) + + +@pytest.mark.parametrize("edit, oracle, match", [ + (lambda p: _write(p / "docker-compose.yaml", _COMPOSE + " db:\n image: postgres\n"), False, "service db"), + (lambda p: _write(p / "docker-compose.yaml", _COMPOSE + " tmpfs: [/run]\n"), False, "client.tmpfs"), + (_compose_with("services:", "networks:\n x: {}\nservices:"), False, "top-level networks"), + (_compose_with(" volumes:\n", " volumes:\n - ./d:/d\n"), False, "client.volumes ./d:/d"), + (_compose_with('"sleep infinity"', '"service nginx start"'), False, "client.command"), + (_compose_with(" dockerfile:", " args: {A: b}\n dockerfile:"), False, "client.build.args"), + (_compose_with(" dockerfile:", " target: dev\n dockerfile:"), False, "client.build.target"), + (_compose_with(" - GREETING", " - PASSTHRU\n - GREETING"), False, "client.environment PASSTHRU"), + (_compose_with(" - GREETING", " - D=${HOME}/x\n - GREETING"), False, "client.environment D"), + (lambda p: _write(p / "task.yaml", (p / "task.yaml").read_text() + "run_tests_in_same_shell: true\n"), + False, "run_tests_in_same_shell"), + (lambda p: _write(p / "task.yaml", (p / "task.yaml").read_text() + "parser_name: swebench\n"), + False, "parser_name is swebench"), + (lambda p: ((p / "solution.sh").unlink(), _write(p / "solution.yaml", "- keys: ls\n")), True, "solution.yaml"), + (lambda p: (p / "solution.sh").unlink(), True, "no solution.sh"), +]) +def test_local_docker_problem_names_what_one_docker_run_cannot_do(tmp_path, edit, oracle, match): + path = _tb1(tmp_path) + edit(path) + assert match in load_task(path).local_docker_problem(oracle=oracle) + + +@pytest.mark.parametrize("command", ['"sleep infinity"', '["sleep", "infinity"]', '["bash", "-c", "sleep infinity"]']) +def test_terminal_bench_idle_command_and_a_literal_environment_run(tmp_path, command): + compose = _COMPOSE.replace('[ "sh", "-c", "sleep infinity" ]', command) + " working_dir: ${WD}\n" + task = load_task(_tb1(tmp_path, compose=compose)) + assert task.local_docker_problem(oracle=True) is None + assert "workdir" not in task.environment, "a working_dir compose fills in is not the image's" + + +def test_solution_is_needed_by_the_oracle_only(tmp_path): + path = _tb1(tmp_path) + (path / "solution.sh").unlink() + assert load_task(path).local_docker_problem(oracle=False) is None + + +# ── verdict ─────────────────────────────────────────────────────────────────── + +@pytest.mark.parametrize("output, reward", [ + (_PASS, 1.0), + (_FAIL, 0.0), + (_SAME_NAME, 0.0), # one file's pass does not hide another's failure + ("===== short test summary info =====\nSKIPPED [1] t.py:3: why\nXFAIL t.py::test_x\n", 1.0), + ("===== short test summary info =====\nXPASS t.py::test_x\n", 0.0), + ("===== short test summary info =====\nERROR t.py::test_x - ImportError\n", 0.0), + ("===== short test summary info =====\n", 1.0), # all() of nothing, as terminal-bench + ("1 passed in 0.1s\n", 0.0), # no summary section + ("", 0.0), +]) +def test_reward_follows_the_pytest_summary(output, reward): + assert tbench_v1.reward(output)[0] == reward + + +def test_parse_pytest_names_tests_and_strips_failure_messages(): + assert tbench_v1.parse_pytest(_FAIL) == {"test_outputs.py::test_a": True, "test_outputs.py::test_b": False} + assert tbench_v1.parse_pytest("no summary") is None + + +@pytest.mark.skipif(shutil.which("awk") is None, reason="needs awk") +@pytest.mark.parametrize("output", [ + _PASS, _FAIL, _SAME_NAME, "===== short test summary info =====\n", "1 passed\n", "", + "== SHORT TEST SUMMARY INFO ==\nERROR t.py::x\n", "===== short test summary info =====\nXPASS t.py::y\n", +]) +def test_exported_awk_verdict_matches_the_python_parser(tmp_path, output): + awk = harbor_export._VERDICT_AWK + log = _write(tmp_path / "out.txt", output) + got = subprocess.run(["sh", "-c", f"{awk} {log}"], capture_output=True, text=True, check=True).stdout + assert float(got.strip()) == tbench_v1.reward(output)[0] + + +@pytest.mark.skipif(os.name == "nt" or shutil.which("bash") is None, reason="needs bash and POSIX paths") +@pytest.mark.parametrize("summary, code, reward", [(_PASS, 0, "1"), (_FAIL, 3, "0")]) +def test_exported_test_sh_scores_run_tests_and_keeps_its_exit_code(tmp_path, summary, code, reward): + tests, logs = tmp_path / "tests", tmp_path / "logs" + _write(tests / "run-tests.sh", f'echo "TEST_DIR=$TEST_DIR"\necho "on stderr" >&2\n' + f"printf %s {shlex.quote(summary)}\nexit {code}\n") + script = harbor_export._TEST_SH.replace("/tests", str(tests)).replace("/logs", str(logs)) + env = {k: v for k, v in os.environ.items() if k != "TEST_DIR"} + proc = subprocess.run(["bash", "-c", script], capture_output=True, text=True, env=env, check=False) + written = (logs / "verifier" / "terminal-bench-output.txt").read_text(encoding="utf-8") + assert proc.returncode == code + assert (logs / "verifier" / "reward.txt").read_text(encoding="utf-8").strip() == reward + assert f"TEST_DIR={tests}" in written and "on stderr" in written and summary in written + assert proc.stdout == written # tee: Harbor's own log of test.sh keeps the output + + +# ── docker-oracle ──────────────────────────────────────────────────────────── + +def _run_oracle(tmp_path, verifier_output: str, returncode: int = 0, verifier_stderr: str = ""): + from docker_cli_fake import DockerCLI + + def answer(call): # bash: the step's own 2>&1 puts stderr, printed first, in the one stream + if call.argv[:2] == ("docker", "exec") and call.argv[-1].startswith("bash /tests/run-tests.sh"): + if call.argv[-1].endswith(" 2>&1"): + return reply(verifier_stderr + verifier_output, returncode=returncode) + return reply(verifier_output, verifier_stderr, returncode=returncode) + return None + + cli = DockerCLI(reward=None, answer=answer) + task = load_task(_tb1(tmp_path / "tasks")) + out = TaskSuiteRunner([task], ["docker-oracle"], output_root=tmp_path / "runs", runner=cli).run() + (row,) = out["results"] + return task, cli, row + + +def test_docker_oracle_runs_a_tb1_task_as_terminal_bench_does(tmp_path): + task, cli, row = _run_oracle(tmp_path, _PASS) + assert "error" not in row["metrics"] and row["reward"] == 1.0 and row["passed"] is True + copies = {(c[1], c[2].split(":", 1)[1]) for c in cli.args if c[0] == "cp" and ":" in c[2]} + assert (str(task.path / "solution.sh"), "/oracle/solution.sh") in copies + assert (str(task.path / "run-tests.sh"), "/tests/run-tests.sh") in copies + assert (f"{task.path / 'tests'}/.", "/tests") in copies + (build,) = [c for c in cli.args if c[0] == "build"] + assert build[-1] == str(task.path) + (run,) = [c for c in cli.args if c[0] == "run" and "sleep" in " ".join(c)] + assert "TEST_DIR=/tests" in run and "GREETING=hi there" in run + for step in ("bash /oracle/solution.sh", "bash /tests/run-tests.sh 2>&1"): + (call,) = [c for c in cli.args if c[0] == "exec" and c[-1] == step] + assert "root" not in call[: call.index("bash")], "runs as the image user, as terminal-bench does" + + +def test_docker_oracle_verdict_is_the_summary_not_the_exit_code(tmp_path): + _, _, row = _run_oracle(tmp_path / "a", _FAIL, returncode=0) + assert row["reward"] == 0.0 + _, _, row = _run_oracle(tmp_path / "b", _PASS, returncode=1) + assert row["reward"] == 1.0 + + +def test_docker_oracle_verdict_ignores_stderr_printed_before_the_summary(tmp_path): + pip = "ERROR: pip's dependency resolver does not currently take into account all the packages.\n" + _, _, row = _run_oracle(tmp_path, _PASS, verifier_stderr=pip) + assert row["reward"] == 1.0, row + + +def test_docker_oracle_refuses_a_multi_service_task_without_docker(tmp_path): + from docker_cli_fake import DockerCLI + + cli = DockerCLI() + path = _tb1(tmp_path / "tasks", compose=_COMPOSE + " db:\n image: postgres\n") + out = TaskSuiteRunner([load_task(path)], ["docker-oracle"], output_root=tmp_path / "runs", runner=cli).run() + (row,) = out["results"] + assert row["reward"] is None and "service db" in row["metrics"]["error"] + assert cli.args == [] + + +# ── Harbor export ──────────────────────────────────────────────────────────── + +def test_export_writes_the_harbor_layout(tmp_path): + task = load_task(_tb1(tmp_path / "tasks")) + dest = harbor_export.export(task, tmp_path / "out") + assert sorted(p.relative_to(dest).as_posix() for p in dest.rglob("*") if p.is_file()) == [ + "environment/Dockerfile", "instruction.md", "solution/solve.sh", "task.toml", + "tests/run-tests.sh", "tests/test.sh", "tests/test_outputs.py"] + dockerfile = (dest / "environment" / "Dockerfile").read_text() + assert "ENV TEST_DIR=/tests" in dockerfile and 'ENV GREETING="hi there"' in dockerfile + toml = (dest / "task.toml").read_text() + assert "[verifier]\ntimeout_sec = 60.0" in toml and "[agent]\ntimeout_sec = 360.0" in toml + assert "[environment]\nmemory_mb = 8192" in toml + assert os.access(dest / "tests" / "test.sh", os.X_OK) + + +def test_exported_task_toml_parses_with_a_non_bmp_character(tmp_path): + import tomllib + + author = "Zo\u00eb \U0001f600" + task = load_task(_tb1(tmp_path / "tasks", yaml_extra=f'author_email: "{author}"\n')) + dest = harbor_export.export(task, tmp_path / "out") + assert tomllib.loads((dest / "task.toml").read_text(encoding="utf-8"))["metadata"]["author_email"] == author + + +def test_export_puts_a_subdir_context_and_its_named_dockerfile_where_harbor_builds(tmp_path): + compose = _COMPOSE.replace(" dockerfile: Dockerfile\n", + " context: ./env\n dockerfile: Dockerfile.x\n working_dir: /srv\n") + path = _tb1(tmp_path / "tasks", compose=compose) + (path / "Dockerfile").unlink() + _write(path / "env" / "Dockerfile.x", "FROM ubuntu:24.04\nCOPY data.txt /data.txt\n") + _write(path / "env" / "data.txt", "x\n") + task = load_task(path) + assert task.environment["terminal_bench"]["workdir"] == "/srv" + dest = harbor_export.export(task, tmp_path / "out") + env = sorted(p.relative_to(dest / "environment").as_posix() for p in (dest / "environment").rglob("*")) + assert env == ["Dockerfile", "data.txt"] + dockerfile = (dest / "environment" / "Dockerfile").read_text() + assert dockerfile.startswith("FROM ubuntu:24.04\nCOPY data.txt") and "\nWORKDIR /srv\n" in dockerfile + + +def test_export_keeps_tests_and_solution_the_dockerfile_copies(tmp_path): + path = _tb1(tmp_path / "tasks") + _write(path / "Dockerfile", "FROM ubuntu:24.04\nCOPY tests /tests\nCOPY ./solution.sh /s.sh\n") + dest = harbor_export.export(load_task(path), tmp_path / "out") + assert (dest / "environment" / "tests" / "test_outputs.py").is_file() + assert (dest / "environment" / "solution.sh").is_file() + assert not (dest / "environment" / "run-tests.sh").exists() + + +def test_export_keeps_a_multi_service_compose_with_main_renamed(tmp_path): + import yaml + + task = load_task(_tb1(tmp_path / "tasks", compose=_COMPOSE + " db:\n image: postgres\n")) + dest = harbor_export.export(task, tmp_path / "out") + compose = yaml.safe_load((dest / "environment" / "docker-compose.yaml").read_text()) + assert list(compose["services"]) == ["main", "db"] + assert compose["services"]["main"]["image"] == "${MAIN_IMAGE_NAME}" + + +def test_export_refuses_solution_yaml(tmp_path): + path = _tb1(tmp_path / "tasks") + (path / "solution.sh").unlink() + _write(path / "solution.yaml", "- keys: ls\n") + with pytest.raises(ValueError, match="solution.yaml"): + harbor_export.export(load_task(path), tmp_path / "out") + with pytest.raises(ValueError, match="solution.yaml"): + harbor_export.harbor_view(load_task(path), cache_root=tmp_path / "cache") + assert not list((tmp_path / "cache").rglob(".staging-*")), "a refused export leaves no staging dir" + (report,) = inspect_path(path) + assert report.consumers == ["docker-*"] and any("solution.yaml" in p for p in report.problems) + + +def test_harbor_view_caches_by_content_and_passes_other_formats_through(tmp_path): + task = load_task(_tb1(tmp_path / "tasks")) + first = harbor_export.harbor_view(task, cache_root=tmp_path / "cache") + assert first.format == "harbor" and (first.path / "task.toml").is_file() + assert harbor_export.harbor_view(task, cache_root=tmp_path / "cache").path == first.path + _write(task.path / "tests" / "test_outputs.py", "def test_other():\n pass\n") + assert harbor_export.harbor_view(task, cache_root=tmp_path / "cache").path != first.path + assert harbor_export.harbor_view(first) is first + + +def test_harbor_view_rewrites_the_export_when_the_memory_default_changes(tmp_path, monkeypatch): + path = _tb1(tmp_path / "tasks") + harbor_export.harbor_view(load_task(path), cache_root=tmp_path / "cache") + monkeypatch.setenv("DOKIMI_TBENCH_V1_MEMORY_MB", "16384") + view = harbor_export.harbor_view(load_task(path), cache_root=tmp_path / "cache") + assert "memory_mb = 16384" in (view.path / "task.toml").read_text() + + +def test_harbor_view_ignores_an_export_planted_in_the_shared_temp_dir(tmp_path, monkeypatch): + monkeypatch.setenv("HOME", str(tmp_path / "home")) + monkeypatch.setattr(harbor_export.tempfile, "gettempdir", lambda: str(tmp_path / "tmp")) + task = load_task(_tb1(tmp_path / "tasks")) + key = harbor_export._content_key(task.path, harbor_export._task_toml(task)) + planted = tmp_path / "tmp" / "dokimi-harbor-export" / key / task.name + _write(planted / "task.toml", "[verifier]\ntimeout_sec = 1.0\n") + view = harbor_export.harbor_view(task) + assert "timeout_sec = 60.0" in (view.path / "task.toml").read_text() + assert (view.path.parents[1].stat().st_mode & 0o777) == 0o700 + + +def test_harbor_harness_runs_the_export_not_the_source(tmp_path, monkeypatch): + from docker_cli_fake import DockerCLI + + monkeypatch.setenv("HOME", str(tmp_path / "home")) + cli = DockerCLI(reward="1") + task = load_task(_tb1(tmp_path / "tasks")) + out = TaskSuiteRunner([task], ["terminal-bench:oracle"], output_root=tmp_path / "runs", runner=cli).run() + (row,) = out["results"] + assert row["reward"] == 1.0, row + (harbor,) = [c for c in cli.calls if Path(c.argv[0]).name == "harbor"] + argv = " ".join(harbor.argv) + assert str(tmp_path / "home" / ".cache" / "dokimi" / "harbor-export") in argv and str(task.path) + " " not in argv + " " + + +def test_benchflow_harness_runs_the_export_not_the_source(tmp_path, monkeypatch): + from dokimi.harnesses.registry import create_harness + + monkeypatch.setenv("HOME", str(tmp_path / "home")) + task = load_task(_tb1(tmp_path / "tasks")) + res = create_harness("benchflow:oracle").run(task, output_dir=tmp_path / "o", dry_run=True) + tasks_dir = Path(res.command[res.command.index("--tasks-dir") + 1]) + assert tasks_dir != task.path and (tasks_dir / "task.toml").is_file() + + +def test_host_side_agent_is_refused_for_a_tb1_task(tmp_path): + from dokimi.harnesses.docker_solver import _refuse_unrunnable + + task = load_task(_tb1(tmp_path)) + with pytest.raises(ValueError, match="host-side agent"): + _refuse_unrunnable(task, "claude", "host") + _refuse_unrunnable(task, "claude", "container") + + +# ── real docker ────────────────────────────────────────────────────────────── + +@pytest.mark.timeout(900) +@pytest.mark.skipif(not os.environ.get("DOKIMI_DOCKER_TESTS") or shutil.which("docker") is None, + reason="set DOKIMI_DOCKER_TESTS=1 with docker available") +def test_docker_oracle_passes_and_a_no_op_fails_on_real_docker(tmp_path): + path = _tb1(tmp_path / "tasks") + _write(path / "Dockerfile", "FROM python:3.12-slim\nRUN pip install -q pytest==8.3.3\nWORKDIR /app\n") + _write(path / "run-tests.sh", "#!/bin/bash\necho \"ERROR: pip's resolver warning\" >&2\n" + "cd /tests && python3 -m pytest -rA test_outputs.py\n") + _write(path / "tests" / "test_outputs.py", + "from pathlib import Path\n\ndef test_hello():\n" + " assert Path('/app/hello.txt').read_text().strip() == 'Hello, world!'\n") + task = load_task(path) + out = TaskSuiteRunner([task], ["docker-oracle"], output_root=tmp_path / "runs").run() + (row,) = out["results"] + assert row["reward"] == 1.0, row + _write(path / "solution.sh", "true\n") + out = TaskSuiteRunner([load_task(path)], ["docker-oracle"], output_root=tmp_path / "runs2").run() + (row,) = out["results"] + assert row["reward"] == 0.0, row