diff --git a/dokimi/changelog.d/20261024_100000_napetrov_run-swebench.md b/dokimi/changelog.d/20261024_100000_napetrov_run-swebench.md new file mode 100644 index 000000000..c46886897 --- /dev/null +++ b/dokimi/changelog.d/20261024_100000_napetrov_run-swebench.md @@ -0,0 +1,13 @@ +### Added + +- **`tasks run --swe-bench `** runs SWE-bench rows (Lite, + Verified) with the `docker-*` harnesses: each row runs in its prebuilt + `swebench/sweb.eval.*` image, `docker-oracle` applies the gold patch, and the + verifier runs SWE-bench's eval script and grades the log with the repo's + SWE-bench parser (reward 1 only when every `FAIL_TO_PASS` and `PASS_TO_PASS` + test passes). `tasks inspect` now names `docker-*` as the consumer of a + SWE-bench file whose repos and versions all have a test spec. + The solve runs offline and the verifier on a PyPI allowlist by default; + `--all` skips the 17 rows in `swebench.KNOWN_BAD` known to fail with the + gold patch in the prebuilt image (live-network tests, Harbor's known + issues). Every row of a file is checked before anything is staged. diff --git a/dokimi/docs/coding-harnesses.md b/dokimi/docs/coding-harnesses.md index 9f5260490..62cd502d5 100644 --- a/dokimi/docs/coding-harnesses.md +++ b/dokimi/docs/coding-harnesses.md @@ -78,6 +78,56 @@ refused (a few hundred bytes of nested aliases expand without bound), and frontmatter nested too deeply to parse is a parse error. YAML and TOML dates and times become ISO strings, and YAML sets sorted lists. +### SWE-bench rows: `tasks run --swe-bench` + +`python cli.py tasks run --swe-bench --tasks [,...] +--harnesses docker-oracle,docker-claude` runs SWE-bench rows (Lite, Verified, +or any file with the same fields) with the `docker-*` harnesses; `--all` +runs every row. Each row is staged under `/swe-bench-tasks//` +as a task the harness already runs, by `dokimi/harnesses/swebench.py`: + +- environment: the prebuilt SWE-bench instance image + (`swebench/sweb.eval.x86_64.:latest`, or the row's own + `image_name`/`docker_image`), with the checkout moved to `/app` and `/testbed` + left as a link to it. Nothing is built but that one layer; the first run pulls + the image (about 4 GB, 1-2.5 GB of it unique per instance). +- oracle (`docker-oracle`): the row's gold `patch`, applied as SWE-bench applies + a prediction (`git apply`, then `patch --fuzz=5`). +- verifier: SWE-bench's eval script (reinstall the repo, reset and apply + `test_patch`, run the repo's test command on the files it touches), then + `swebench_grade.py`, which parses the log with the repo's SWE-bench parser: + reward 1 only when every `FAIL_TO_PASS` and `PASS_TO_PASS` test passes. + Unlike upstream, a skipped `FAIL_TO_PASS` test (or a row with none) does + not resolve. The end of the eval log goes to `run.log`; the grader's exit + code is the verdict too. + +The image's `testbed` conda env is first on `PATH`, so an in-container agent's +`python` and `pytest` are the repo's. A host-side agent +(`docker-claude-host`) is refused: the instruction names container paths and +the host has no `testbed` env. + +The per-repo test commands and parsers are a table, +`dokimi/harnesses/swebench_specs.json`, generated from the `swebench` +package by `scripts/gen_swebench_specs.py`; dokimi does not depend on +`swebench`. Rows from a repo outside the table (SWE-bench Multilingual, +SWE-smith, SWE-Gym repos) are refused with the repo's name, and so is a +version the table lacks. The solve runs offline (see +[Network policy](#network-policy)); the verifier gets an allowlist of +`pypi.org` and `files.pythonhosted.org`, because the eval script's install +step fetches build requirements and the packages a gold patch adds +(`pylint-dev__pylint-4661`, `astropy__astropy-12907` fail offline). +`--network verify=none` runs fully offline, `--network verify=public` gives +the verifier upstream's network. A file is checked row by row before any task +is written. + +`--all` skips the rows in `swebench.KNOWN_BAD`, whose gold patch fails in the +prebuilt image under that default: `psf__requests-863`, `-1724`, `-1766`, +`-1921`, `-1963`, `-2148`, `-2317` and `-2674` (their tests fetch httpbin.org), +`sphinx-doc__sphinx-7985`, `-8269` and `-8475` (`test_build_linkcheck.py` +checks live links), and the six Harbor's SWE-bench adapter lists as failing under its +oracle (harbor 6099b8c, `adapters/swebench/README.md`, Known Issues). Naming +one in `--tasks` still runs it. + ### Other eval formats: `tasks inspect` `python cli.py tasks inspect [--json] [--out FILE]` names the eval-task format of a @@ -108,7 +158,7 @@ real files in each format, with their sources in its `MANIFEST.yaml`. | `harbor` | directory with `task.toml` | executable | Harbor-backed harnesses (unless Harbor would refuse it), `docker-*` (with `environment/Dockerfile`) | | `task-md` | directory with a BenchFlow `task.md` | executable | `benchflow*`, `docker-*` (with `environment/Dockerfile`) | | `terminal-bench-v1` | directory with `task.yaml` + `run-tests.sh` / `Dockerfile` / `solution.sh` | executable | `docker-*` run it as it is; `harbor`, `benchflow` through a cached Harbor export (not a `solution.yaml` task) | -| `swe-bench` | JSON/JSONL rows with `instance_id`, `repo`, `base_commit`, `problem_statement` (`FAIL_TO_PASS`) | executable | none yet | +| `swe-bench` | JSON/JSONL rows with `instance_id`, `repo`, `base_commit`, `problem_statement` (`FAIL_TO_PASS`) | executable | `docker-*` via `tasks run --swe-bench` (Python SWE-bench repos) | | `inspect-samples` | JSON/JSONL/CSV rows with `input` and `target` (or the keys a `FieldSpec` names) | answer-only | `questions import` ([answer-only formats](answer-formats.md)) | | `inspect-task` | `.py` importing `inspect_ai` with a `@task` function | answer-only (executable with a `sandbox=`) | `questions import` ([answer-only formats](answer-formats.md)) | | `openai-evals` | JSONL rows with `input` (chat messages) and `ideal` | answer-only | `questions import` ([answer-only formats](answer-formats.md)) | @@ -145,7 +195,8 @@ sets `deploy.resources.limits.memory`, which `docker compose` honours and dokimi reads as the task's own value. `DOKIMI_TBENCH_V1_MEMORY_MB` changes the 8192 default. The Harbor export writes the same value to `task.toml`. -`tasks list` and `tasks run` still read only the first three. +`tasks list` and `tasks run` still read only the first three; `tasks run --swe-bench` +also takes a SWE-bench file. The `docker-*` harnesses mount the oracle at `/solution` and `/oracle` and the verifier at `/tests` and `/verifier`, because a script written for one layout diff --git a/dokimi/dokimi/commands/tasks.py b/dokimi/dokimi/commands/tasks.py index bb082c52e..d4e0d9838 100644 --- a/dokimi/dokimi/commands/tasks.py +++ b/dokimi/dokimi/commands/tasks.py @@ -122,9 +122,33 @@ def location(name: str) -> str | None: for h in harnesses if h != baseline and (loc := location(h)) not in (None, base)] +def _select_swebench(args: argparse.Namespace) -> list: + """Stage the selected rows of a SWE-bench dataset file under the output dir.""" + from dokimi.harnesses.swebench import KNOWN_BAD, load_swebench, read_instances + + ids = None if args.all else [t.strip() for t in (args.tasks or "").split(",") if t.strip()] + if ids == []: + raise SystemExit("No tasks selected. Pass --tasks or --all.") + try: + if ids is None: + skipped = sorted(KNOWN_BAD.keys() & {str(r.get("instance_id")) for r in read_instances(Path(args.swe_bench))}) + if skipped: + print(f"--swe-bench --all skips {len(skipped)} known-bad rows (name them in --tasks to run them): " + + ", ".join(skipped), file=sys.stderr) + tasks = load_swebench(Path(args.swe_bench), Path(args.output_dir) / "swe-bench-tasks", ids) + except (OSError, ValueError) as exc: # SweBenchError and bad JSON are ValueErrors + raise SystemExit(f"--swe-bench: {exc}") from None + if not tasks: # an empty run must not exit 0 with an empty artifact + raise SystemExit(f"--swe-bench: no runnable rows in {args.swe_bench} " + "(--all skips the known-bad rows)") + return tasks + + def _select_tasks(args: argparse.Namespace) -> list: from dokimi.harnesses import discover_tasks, load_task + if getattr(args, "swe_bench", None): + return _select_swebench(args) root = _tasks_root(args) try: if args.all: @@ -254,6 +278,11 @@ def register(sub, positive_int) -> None: run_p.add_argument("--baseline-harness", default=None, dest="baseline_harness") run_p.add_argument("--model", default=None, help="Model ref passed to the harness") run_p.add_argument("--tasks-root", default=None, help=_TASKS_ROOT_HELP) + run_p.add_argument( + "--swe-bench", default=None, dest="swe_bench", metavar="FILE", + help=("A SWE-bench dataset (.jsonl or .json rows); --tasks names instance ids. " + "Each row runs in its prebuilt swebench/sweb.eval.* image (docker-* harnesses)."), + ) run_p.add_argument("--output-dir", default="results/task-runs", dest="output_dir") run_p.add_argument("--out-json", default=None, dest="out_json") run_p.add_argument("--out-md", default=None, dest="out_md", diff --git a/dokimi/dokimi/harnesses/docker_solver.py b/dokimi/dokimi/harnesses/docker_solver.py index bcb5c472a..bf68c4211 100644 --- a/dokimi/dokimi/harnesses/docker_solver.py +++ b/dokimi/dokimi/harnesses/docker_solver.py @@ -83,6 +83,7 @@ from .tasks import ( # noqa: F401 (timeouts re-exported) _DEFAULT_BUILD_TIMEOUT, _DEFAULT_VERIFY_TIMEOUT, + FORMAT_SWE_BENCH, FORMAT_TASK_MD, FORMAT_TBENCH_V1, ContainerStep, @@ -728,15 +729,14 @@ def _describe_command(self, task: TaskSpec, model: Optional[str]) -> list[str]: def _refuse_unrunnable(task: TaskSpec, mode: str, agent_location: str) -> None: """Refuse, from the task file alone, what a docker-* agent cannot run. - - A ``task.md`` task, for a host-side agent: its instruction names absolute - container paths (``/root/...``, ``/app/...``) and the agent runs on the - host in a copy of ``/app``, so it writes to the operator's machine and the - verifier, in the container, finds nothing: a false fail plus a host write. - The in-container agent runs such a task when it works in ``/app``. + - A ``task.md``, terminal-bench-v1 or SWE-bench task, for a host-side + agent: its instruction names container paths (``/root/...``, ``/app/...``) + and the agent runs on the host in a copy of ``/app``: a host write plus a + false fail, as the verifier in the container finds nothing. The + in-container agent runs such a task when it works in ``/app``. - A declared ``environment.workdir`` outside ``/app``, for either agent: both edit ``/app`` only (see :func:`_require_agent_workdir` for the image's own ``WORKDIR``, which needs the built image). - - A task without the directory a mode copies in: ``tests/`` (``verifier/``) for the verifier, and ``solution/`` (``oracle/``) for ``docker-oracle``. ``docker cp`` would fail only at that step, after the image build and, @@ -757,7 +757,7 @@ def _refuse_unrunnable(task: TaskSpec, mode: str, agent_location: str) -> None: raise ValueError(f"{task.name}: no {what} to copy to {dest}") if mode not in ("claude", "probe"): return - if agent_location == "host" and task.format in (FORMAT_TASK_MD, FORMAT_TBENCH_V1): + if agent_location == "host" and task.format in (FORMAT_TASK_MD, FORMAT_TBENCH_V1, FORMAT_SWE_BENCH): raise ValueError( f"{task.name}: a {task.format} task cannot be solved by a host-side agent " f"(its instruction names absolute container paths); {_IN_CONTAINER}" diff --git a/dokimi/dokimi/harnesses/swebench.py b/dokimi/dokimi/harnesses/swebench.py new file mode 100644 index 000000000..acd28cbcd --- /dev/null +++ b/dokimi/dokimi/harnesses/swebench.py @@ -0,0 +1,311 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""SWE-bench instances as :class:`TaskSpec`: run a dataset row with a ``docker-*`` harness. + +A SWE-bench row (``instance_id``, ``repo``, ``base_commit``, ``problem_statement``, +``patch``, ``test_patch``, ``FAIL_TO_PASS``, ``PASS_TO_PASS``, ``version``) is staged +as a task directory the Docker harness already runs: + +``environment/Dockerfile`` + ``FROM`` the prebuilt SWE-bench instance image (``swebench/sweb.eval.x86_64.*`` + on Docker Hub, or the row's own ``image_name``/``docker_image``), with the + checkout moved from ``/testbed`` to ``/app`` (the tree a docker-* agent edits) + and ``/testbed`` left as a link to it, so upstream paths still resolve. +``solution/`` + The gold ``patch``, applied as SWE-bench applies a prediction. +``tests/`` + The upstream eval script (reset the test files, apply ``test_patch``, run the + repo's test command on the files it touches) and ``grade.py``, which parses + the log with the repo's SWE-bench parser and writes ``reward.txt``. + +The repo/version table (test command, install step, parser) is +``swebench_specs.json``, generated from the ``swebench`` package by +``scripts/gen_swebench_specs.py``; dokimi does not import ``swebench``. +The staged directory is the harness's working copy, not a Harbor export: the +spec is built here, in memory, with ``format = "swe-bench"``. +""" + +from __future__ import annotations + +import json +import re +import shlex +from collections import Counter +from functools import lru_cache +from pathlib import Path +from typing import Any, Iterable + +from .tasks import FORMAT_SWE_BENCH, TaskSpec + +_SPECS = Path(__file__).with_name("swebench_specs.json") +_GRADER = Path(__file__).with_name("swebench_grade.py") +REQUIRED = ("instance_id", "repo", "base_commit", "patch", "test_patch", "problem_statement", + "version", "FAIL_TO_PASS", "PASS_TO_PASS") +WORKDIR = "/app" +UPSTREAM_DIR = "/testbed" +# SWE-bench's NON_TEST_EXTS: files of a test patch that are not test directives. +_NON_TEST_EXTS = (".json", ".png", "csv", ".txt", ".md", ".jpg", ".jpeg", ".pkl", ".yml", + ".yaml", ".toml") +# SWE-bench's GIT_APPLY_CMDS, tried in order on a prediction. +_APPLY_CMDS = ("git apply --verbose", "git apply --verbose --reject", "patch --batch --fuzz=5 -p1 -i") +_HEREDOC = "EOF_114329324912" +_DEFAULT_TIMEOUT = 1800.0 +# The verifier's default network: eval.sh's install step fetches build requirements +# and the packages a gold patch adds (pylint-4661 `appdirs`, astropy-12907 setuptools). +PYPI_HOSTS = ("pypi.org", "files.pythonhosted.org") +_HARBOR = "harbor 6099b8c adapters/swebench/README.md, Known Issues: fails under the oracle" +# Rows whose gold patch does not pass in the prebuilt image under the default network +# (verify on the PyPI allowlist): --all skips them, --tasks still runs them. +KNOWN_BAD = { + "psf__requests-863": "tests fetch httpbin.org live; fail offline and when httpbin is down", + "psf__requests-2317": "tests fetch httpbin.org live; fail offline and when httpbin is down", + "psf__requests-1724": "tests fetch httpbin.org live; fail offline and when httpbin is down", + "psf__requests-1766": "tests fetch httpbin.org live; fail offline and when httpbin is down", + "psf__requests-1921": "tests fetch httpbin.org live; fail offline and when httpbin is down", + "psf__requests-1963": "tests fetch httpbin.org live; fail offline and when httpbin is down", + "psf__requests-2148": "tests fetch httpbin.org live; fail offline and when httpbin is down", + "psf__requests-2674": "tests fetch httpbin.org live; fail offline and when httpbin is down", + "sphinx-doc__sphinx-7985": "test_build_linkcheck.py::test_defaults checks live google.com and sphinx-doc.org links", + "sphinx-doc__sphinx-8269": "test_build_linkcheck.py::test_defaults checks live google.com and sphinx-doc.org links", + "sphinx-doc__sphinx-8475": "test_build_linkcheck.py::test_defaults checks live google.com and sphinx-doc.org links", + "astropy__astropy-7606": f"missing test in the dataset ({_HARBOR})", + "astropy__astropy-8707": f"test infrastructure failure ({_HARBOR})", + "astropy__astropy-8872": f"distutils/pytest deprecation ({_HARBOR})", + "django__django-10097": f"INSTALLED_APPS settings mismatch ({_HARBOR})", + "sphinx-doc__sphinx-8595": f"harness marks a passing test failed, SWE-bench PR #475 ({_HARBOR})", + "sphinx-doc__sphinx-9711": f"harness marks a passing test failed, SWE-bench PR #475 ({_HARBOR})", +} + + +# An instance_id names a directory and an image: no path separators, no leading dot. +_SAFE_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*") +# A row's own image reference: registry/repo, optional :tag or @digest, no whitespace. +_IMAGE_REF = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/:@-]*") + + +class SweBenchError(ValueError): + """A row the loader cannot turn into a runnable task.""" + + +@lru_cache(maxsize=1) +def specs() -> dict[str, Any]: + """The repo -> {parser, versions} table (see ``swebench_specs.json``).""" + return json.loads(_SPECS.read_text(encoding="utf-8")) + + +def read_instances(path: Path) -> list[dict[str, Any]]: + """Rows of a SWE-bench ``.jsonl`` (one per line) or ``.json`` (a list) file.""" + text = Path(path).read_text(encoding="utf-8") + if Path(path).suffix == ".jsonl": + rows = [json.loads(line) for line in text.splitlines() if line.strip()] + else: + data = json.loads(text) + rows = data if isinstance(data, list) else [data] + if not all(isinstance(r, dict) for r in rows): + raise SweBenchError(f"{path}: every row must be a JSON object") + return rows + + +def _test_list(value: Any, key: str, instance_id: str) -> list[str]: + """FAIL_TO_PASS/PASS_TO_PASS: a list, or (the Hugging Face export) a JSON string of one.""" + if isinstance(value, str): + value = json.loads(value) + if not (isinstance(value, list) and all(isinstance(t, str) for t in value)): + raise SweBenchError(f"{instance_id}: {key} must be a list of test names") + return value + + +def instance_image(row: dict[str, Any], namespace: str = "swebench", arch: str = "x86_64") -> str: + """The prebuilt instance image: the row's own, else SWE-bench's Docker Hub name. + + Upstream ``TestSpec.instance_image_key`` with a namespace: ``__`` becomes + ``_1776_`` because Docker Hub repository names cannot hold ``__``. + """ + for key in ("image_name", "docker_image"): # SWE-smith, SWE-rebench rows + if isinstance(row.get(key), str) and row[key]: + # The value goes into the Dockerfile's FROM line: a newline would add instructions. + if not _IMAGE_REF.fullmatch(row[key]): + raise SweBenchError(f"{row.get('instance_id', '?')}: {key} {row[key]!r} is not an image reference") + return str(row[key]) + key = f"sweb.eval.{arch}.{row['instance_id'].lower()}:latest" + return f"{namespace}/{key}".replace("__", "_1776_") + + +def repo_spec(repo: str, version: str) -> tuple[str, dict[str, Any]]: + """(parser name, {test_cmd, install, eval_commands}) for one repo version.""" + entry = specs()["repos"].get(repo) + if entry is None: + raise SweBenchError(f"no SWE-bench test spec for repo {repo!r} (known: Python SWE-bench repos)") + spec = entry["versions"].get(str(version)) + if spec is None or "test_cmd" not in spec: + raise SweBenchError(f"no SWE-bench test spec for {repo} version {version!r}") + return entry["parser"], spec + + +def patch_files(patch: str) -> tuple[list[str], list[str]]: + """(files the patch modifies or deletes, every file it names), from its headers.""" + # A name with a space is followed by a tab in git's ---/+++ lines; the tab is not part of it. + old = re.findall(r"^--- a/(.+?)\t?$", patch, re.MULTILINE) + new = re.findall(r"^\+\+\+ b/(.+?)\t?$", patch, re.MULTILINE) + return list(dict.fromkeys(old)), list(dict.fromkeys(old + new)) + + +def test_directives(repo: str, test_patch: str) -> list[str]: + """Upstream ``get_test_directives``: the test files to run, Django as dotted modules.""" + found = re.findall(r"diff --git a/.* b/(.*)", test_patch) + found = [d for d in found if not d.endswith(_NON_TEST_EXTS)] + if repo != "django/django": + return found + out = [] + for d in found: + d = d[: -len(".py")] if d.endswith(".py") else d + d = d[len("tests/"):] if d.startswith("tests/") else d + out.append(d.replace("/", ".")) + return out + + +def eval_script(row: dict[str, Any], spec: dict[str, Any]) -> str: + """Upstream ``make_eval_script_list_py``, rooted at ``/testbed`` like the image.""" + base = shlex.quote(row["base_commit"]) + tracked, touched = patch_files(row["test_patch"]) + reset = f"git checkout {base} {' '.join(shlex.quote(f) for f in tracked)}" if tracked else ":" + # A file the test patch adds is left over by a first run; remove it so the apply holds. + added = [f for f in touched if f not in tracked] + clean = f"rm -f -- {' '.join(shlex.quote(f) for f in added)}" if added else ":" + if "\0" in row["test_patch"]: # bash drops NUL bytes as it reads, so "EOF\0..." would end the heredoc + raise SweBenchError(f"{row.get('instance_id', '?')}: test_patch has a NUL byte") + if _HEREDOC in row["test_patch"].splitlines(): # the patch is a heredoc body: this line would end it + raise SweBenchError(f"{row.get('instance_id', '?')}: test_patch has a line equal to the heredoc delimiter") + directives = [shlex.quote(d) for d in test_directives(row["repo"], row["test_patch"])] + test_cmd = " ".join([spec["test_cmd"], *directives]) + activate = ["source /opt/miniconda3/bin/activate", "conda activate testbed"] + lines = ["#!/bin/bash", "set -uxo pipefail", *activate, f"cd {UPSTREAM_DIR}", + *spec.get("eval_commands", []), + "git config --global --add safe.directory '*'", + "git status", f"git -c core.fileMode=false diff {base}", *activate] + if spec.get("install"): + lines.append(spec["install"]) + lines += [reset, clean, + f"git apply -v - <<'{_HEREDOC}'\n{row['test_patch']}\n{_HEREDOC}", + ": '>>>>> Start Test Output'", test_cmd, ": '>>>>> End Test Output'", reset, clean] + return "\n".join(lines) + "\n" + + +def _solve_script() -> str: + tries = " || ".join(f"{cmd} /solution/patch.diff" for cmd in _APPLY_CMDS) + return ("#!/bin/bash\nset -uo pipefail\ncd /testbed\n" + "git config --global --add safe.directory '*'\n" + "[ -s /solution/patch.diff ] || { echo 'empty patch: nothing to apply'; exit 0; }\n" + f"{tries} || {{ echo '>>>>> Patch Apply Failed'; exit 1; }}\n") + + +_TEST_SH = """#!/bin/bash +mkdir -p /logs/verifier +bash /tests/eval.sh > /logs/verifier/test_output.log 2>&1 +# The log stays in the container: its end goes to run.log for a row that scores 0. +tail -n 200 /logs/verifier/test_output.log +PY=/opt/miniconda3/bin/python; [ -x "$PY" ] || PY=python3 +"$PY" /tests/grade.py /tests/config.json /logs/verifier/test_output.log /logs/verifier +""" + +_DOCKERFILE = """FROM {image} +# A docker-* agent edits /app; the eval scripts and the conda env name /testbed. +RUN mv {upstream} {workdir} && ln -s {workdir} {upstream} +WORKDIR {workdir} +# `conda activate testbed` is only in /root/.bashrc; the agent (uid 10001, tmpfs HOME) needs it on PATH. +ENV PATH=/opt/miniconda3/envs/testbed/bin:$PATH +""" + + +def instruction(row: dict[str, Any]) -> str: + return (f"{row['problem_statement'].rstrip()}\n\n---\n\n" + f"The repository {row['repo']} is checked out at {WORKDIR} (also reachable as " + f"{UPSTREAM_DIR}) at commit {row['base_commit'][:12]}. Edit the source so that the " + "issue above is resolved. Do not modify the existing tests.\n") + + +def stage(row: dict[str, Any], root: Path, *, patch: str | None = None, + namespace: str = "swebench") -> TaskSpec: + """Write *row*'s task directory under *root* and return its spec. + + *patch* replaces the gold patch as the oracle (``""``: the empty-patch control). + """ + task, files = _build(row, root, patch=patch, namespace=namespace) + _write(task.path, files) + return task + + +def _write(task_dir: Path, files: dict[str, str]) -> None: + for rel, text in files.items(): + dest = task_dir / rel + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text(text, encoding="utf-8") + + +def _build(row: dict[str, Any], root: Path, *, patch: str | None, + namespace: str) -> tuple[TaskSpec, dict[str, str]]: + """*row*'s spec and files, checked but not written.""" + missing = [k for k in REQUIRED if k not in row] + if missing: + raise SweBenchError(f"{row.get('instance_id', '?')}: missing {', '.join(missing)}") + iid = str(row["instance_id"]) + if not _SAFE_ID.fullmatch(iid): + raise SweBenchError(f"{iid!r}: instance_id must match {_SAFE_ID.pattern}") + not_text = [k for k in ("repo", "base_commit", "problem_statement", "patch", "test_patch") if not isinstance(row[k], str)] + if not_text: # a null or a number here would escape the CLI's error handling as a traceback + raise SweBenchError(f"{iid}: {', '.join(not_text)} must be a string") + tests = {k: _test_list(row[k], k, iid) for k in ("FAIL_TO_PASS", "PASS_TO_PASS")} + parser, spec = repo_spec(row["repo"], row["version"]) + image = instance_image(row, namespace) + task_dir = Path(root) / iid + files = { + "environment/Dockerfile": _DOCKERFILE.format(image=image, upstream=UPSTREAM_DIR, workdir=WORKDIR), + "solution/solve.sh": _solve_script(), + "solution/patch.diff": row["patch"] if patch is None else patch, + "tests/test.sh": _TEST_SH, + "tests/eval.sh": eval_script(row, spec), + "tests/grade.py": _GRADER.read_text(encoding="utf-8"), + "tests/config.json": json.dumps({"instance_id": iid, "parser": parser, **tests}, indent=1), + "instruction.md": instruction(row), + } + return TaskSpec( + name=iid, + path=task_dir.resolve(), + instruction=files["instruction.md"], + metadata={"source": "swe-bench", "instance_id": iid, "repo": row["repo"], + "base_commit": row["base_commit"], "version": str(row["version"]), + "parser": parser, "oracle_patch": "gold" if patch is None else "override"}, + # Solve stays offline (the agent gets no network); the verifier reaches PyPI only. + # `--network verify=none` runs fully offline, `verify=public` is upstream's network. + verifier={"timeout_sec": _DEFAULT_TIMEOUT, "network_mode": "allowlist", + "allowed_hosts": list(PYPI_HOSTS)}, + agent={"timeout_sec": _DEFAULT_TIMEOUT}, + environment={"docker_image": image, "workdir": WORKDIR, + "cpus": 4, "memory_mb": 8192, "build_timeout_sec": _DEFAULT_TIMEOUT}, + format=FORMAT_SWE_BENCH, + ), files + + +def load_swebench(path: Path, root: Path, ids: Iterable[str] | None = None, *, + namespace: str = "swebench") -> list[TaskSpec]: + """Stage the rows of *path* named in *ids* (every row when ``None``) under *root*.""" + rows = read_instances(path) + counts = Counter(str(r["instance_id"]) for r in rows if "instance_id" in r) + dups = sorted(i for i, n in counts.items() if n > 1) + if dups: # two rows would stage into one directory and both run the last one + raise SweBenchError(f"{path}: duplicate instance_id {', '.join(dups)}") + if ids is None: + selected = [r for r in rows if str(r.get("instance_id")) not in KNOWN_BAD] + else: + by_id = {str(r.get("instance_id")): r for r in rows} + wanted = list(ids) + unknown = [i for i in wanted if i not in by_id] + if unknown: + raise SweBenchError(f"{path}: no instance {', '.join(unknown)}") + selected = [by_id[i] for i in wanted] + # Every row is checked before any is written: a bad row late in the file stages nothing. + built = [_build(r, root, patch=None, namespace=namespace) for r in selected] + for task, files in built: + _write(task.path, files) + return [task for task, _ in built] diff --git a/dokimi/dokimi/harnesses/swebench_grade.py b/dokimi/dokimi/harnesses/swebench_grade.py new file mode 100644 index 000000000..e61cb16a4 --- /dev/null +++ b/dokimi/dokimi/harnesses/swebench_grade.py @@ -0,0 +1,232 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +# The log parsers and the resolution rule below follow SWE-bench +# (swebench==4.1.0, swebench/harness/log_parsers/python.py and grading.py), +# Copyright (c) 2023 Carlos E Jimenez, John Yang, Alexander Wettig, Shunyu Yao, +# Kexin Pei, Ofir Press, Karthik R Narasimhan; MIT License. + +"""Grade one SWE-bench test log: the verifier half of a SWE-bench task. + +This file is copied into the task container and run there by the task's +``test.sh`` (``python grade.py config.json test.log /logs/verifier``), so it +imports nothing outside the standard library and nothing from dokimi. It +writes ``report.json`` (per-test outcome) and ``reward.txt`` (1 when every +FAIL_TO_PASS and PASS_TO_PASS test passes, else 0), as SWE-bench's +``get_resolution_status`` decides ``RESOLVED_FULL``. +""" + +import json +import re +import sys +from pathlib import Path +from typing import Callable, Dict, List + +START = ">>>>> Start Test Output" +END = ">>>>> End Test Output" +# A log holding any of these never reached a test run (SWE-bench's bad codes). +BAD_CODES = (">>>>> Patch Apply Failed", ">>>>> Reset Failed", ">>>>> Tests Errored", + ">>>>> Tests Timed Out") +STATUSES = ("FAILED", "PASSED", "SKIPPED", "ERROR", "XFAIL") + +StatusMap = Dict[str, str] + + +def _starts_with_status(line: str) -> bool: + return any(line.startswith(s) for s in STATUSES) + + +def parse_pytest(log: str) -> StatusMap: + found: StatusMap = {} + for line in log.split("\n"): + if _starts_with_status(line): + if line.startswith("FAILED"): + line = line.replace(" - ", " ") + parts = line.split() + if len(parts) > 1: + found[parts[1]] = parts[0] + return found + + +def parse_pytest_options(log: str) -> StatusMap: + option = re.compile(r"(.*?)\[(.*)\]") + found: StatusMap = {} + for line in log.split("\n"): + if not _starts_with_status(line): + continue + if line.startswith("FAILED"): + line = line.replace(" - ", " ") + parts = line.split() + if len(parts) <= 1: + continue + match = option.search(parts[1]) + name = parts[1] + if match: + main, opt = match.groups() + if opt.startswith("/") and not opt.startswith("//") and "*" not in opt: + opt = "/" + opt.split("/")[-1] + name = f"{main}[{opt}]" + found[name] = parts[0] + return found + + +def parse_pytest_v2(log: str) -> StatusMap: + found: StatusMap = {} + drop = str.maketrans("", "", "".join(chr(c) for c in range(1, 32))) + for raw in log.split("\n"): + line = re.sub(r"\[(\d+)m", "", raw).translate(drop) + parts = line.split() + if _starts_with_status(line): + if line.startswith("FAILED"): + parts = line.replace(" - ", " ").split() + if len(parts) >= 2: + found[parts[1]] = parts[0] + elif any(line.endswith(s) for s in STATUSES) and len(parts) >= 2: + found[parts[0]] = parts[1] # older pytest: " PASSED" + return found + + +def parse_matplotlib(log: str) -> StatusMap: + return parse_pytest(log.replace("MouseButton.LEFT", "1").replace("MouseButton.RIGHT", "3")) + + +def parse_seaborn(log: str) -> StatusMap: + found: StatusMap = {} + for line in log.split("\n"): + parts = line.split() + if line.startswith("FAILED") and len(parts) > 1: + found[parts[1]] = "FAILED" + elif " PASSED " in line: + if len(parts) > 1 and parts[1] == "PASSED": + found[parts[0]] = "PASSED" + elif line.startswith("PASSED") and len(parts) > 1: + found[parts[1]] = "PASSED" + return found + + +def parse_sympy(log: str) -> StatusMap: + found: StatusMap = {} + for match in re.findall(r"(_*) (.*)\.py:(.*) (_*)", log): + found[f"{match[1]}.py:{match[2]}"] = "FAILED" + endings = {" E": "ERROR", " F": "FAILED", " ok": "PASSED"} + for line in log.split("\n"): + line = line.strip() + if line.startswith("test_"): + for suffix, status in endings.items(): + if line.endswith(suffix): + found[line.split()[0]] = status + return found + + +_DJANGO_PASS_SUFFIXES = (" ... ok", " ... OK", " ... OK") +_DJANGO_LINE_STATUS = ((" ... FAIL", "FAILED"), (" ... ERROR", "ERROR")) +_DJANGO_PREFIX_STATUS = (("FAIL:", "FAILED"), ("ERROR:", "ERROR")) +# A long print can split " ... ok" over lines; SWE-bench recovers three forms. +_DJANGO_SPLIT_PASS = ( + r"^(.*?)\s\.\.\.\sTesting\ against\ Django\ installed\ in\ ((?s:.*?))\ silenced\)\.\nok$", + r"^(.*?)\s\.\.\.\sInternal\ Server\ Error:\ \/(.*)\/\nok$", + r"^(.*?)\s\.\.\.\sSystem check identified no issues \(0 silenced\)\nok$", +) + + +def _django_line(line: str, found: StatusMap) -> None: + for suffix in _DJANGO_PASS_SUFFIXES: + if line.endswith(suffix): + if line.startswith("Applying sites.0002_alter_domain_unique...test_no_migrations"): + line = line.split("...", 1)[-1].strip() # django__django-7188 + found[line.rsplit(suffix, 1)[0]] = "PASSED" + break + if " ... skipped" in line: + found[line.split(" ... skipped")[0]] = "SKIPPED" + for suffix, status in _DJANGO_LINE_STATUS: + if line.endswith(suffix): + found[line.split(suffix)[0]] = status + for prefix, status in _DJANGO_PREFIX_STATUS: + if line.startswith(prefix): + found[line.split()[1].strip()] = status + + +def parse_django(log: str) -> StatusMap: + found: StatusMap = {} + prev = None + for raw in log.split("\n"): + line = raw.strip() + if "--version is equivalent to version" in line: + found["--version is equivalent to version"] = "PASSED" + if " ... " in line: + prev = line.split(" ... ")[0] + _django_line(line, found) + if line.startswith("ok") and prev is not None: + found[prev] = "PASSED" # output between "..." and "ok" + for pattern in _DJANGO_SPLIT_PASS: + for match in re.finditer(pattern, log, re.MULTILINE): + found[match.group(1)] = "PASSED" + return found + + +PARSERS: Dict[str, Callable[[str], StatusMap]] = { + "pytest": parse_pytest, + "pytest_options": parse_pytest_options, + "pytest_v2": parse_pytest_v2, + "matplotlib": parse_matplotlib, + "seaborn": parse_seaborn, + "sympy": parse_sympy, + "django": parse_django, +} + + +def status_map(log: str, parser: str) -> "StatusMap | None": + """Test -> status from the log, or ``None`` if the log holds no test run.""" + if any(code in log for code in BAD_CODES) or START not in log or END not in log: + return None + parse = PARSERS[parser] + # Like SWE-bench: the marked section first, the whole log if that finds nothing. + return parse(log.split(START)[1].split(END)[0]) or parse(log) + + +def grade(log: str, config: dict) -> dict: + """The SWE-bench report for one log: ``resolved`` and each test's outcome.""" + found = status_map(log, config["parser"]) + report: dict = {"instance_id": config.get("instance_id"), "tests_ran": found is not None} + if found is None: + report["resolved"] = False + return report + resolved = True + for key in ("FAIL_TO_PASS", "PASS_TO_PASS"): + tests: List[str] = config[key] + passed = [t for t in tests if found.get(t) in ("PASSED", "XFAIL")] + # Missing counts as failed; a SKIPPED PASS_TO_PASS test is neither, as upstream. + failed = [t for t in tests if found.get(t, "FAILED") in ("FAILED", "ERROR")] + if key == "FAIL_TO_PASS": + # Unlike upstream: a fix is shown only by a test that ran and passed, so a + # skipped FAIL_TO_PASS test (an offline skip) is a failure, and so is none. + failed = [t for t in tests if t not in passed] + resolved = resolved and bool(tests) + report[key] = {"success": passed, "failure": failed} + resolved = resolved and not failed + report["resolved"] = resolved + return report + + +def main(argv: List[str]) -> int: + config_path, log_path, out_dir = (Path(a) for a in argv[1:4]) + config = json.loads(config_path.read_text(encoding="utf-8")) + report = grade(log_path.read_text(encoding="utf-8", errors="replace"), config) + out_dir.mkdir(parents=True, exist_ok=True) + (out_dir / "report.json").write_text(json.dumps(report, indent=1), encoding="utf-8") + (out_dir / "reward.txt").write_text("1\n" if report["resolved"] else "0\n", encoding="utf-8") + print("SWE-bench: " + ("RESOLVED" if report["resolved"] else "NOT RESOLVED")) + if not report["tests_ran"]: + print(" no test run in the log (test patch did not apply, or no output markers)") + for key in ("FAIL_TO_PASS", "PASS_TO_PASS"): + failed = report.get(key, {}).get("failure", []) + for test in failed[:10]: + print(f" {key} failed: {test}") + if len(failed) > 10: + print(f" {key}: {len(failed) - 10} more failed") + # The verdict is the exit code too: a reward.txt that cannot be copied out reads as a fail. + return 0 if report["resolved"] else 1 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/dokimi/dokimi/harnesses/swebench_specs.json b/dokimi/dokimi/harnesses/swebench_specs.json new file mode 100644 index 000000000..4b81212a6 --- /dev/null +++ b/dokimi/dokimi/harnesses/swebench_specs.json @@ -0,0 +1,1289 @@ +{ + "repos": { + "astropy/astropy": { + "parser": "pytest_v2", + "versions": { + "0.1": { + "install": "python -m pip install -e .[test] --verbose", + "test_cmd": "pytest -rA -vv -o console_output_style=classic --tb=no" + }, + "0.2": { + "install": "python -m pip install -e .[test] --verbose", + "test_cmd": "pytest -rA -vv -o console_output_style=classic --tb=no" + }, + "0.3": { + "install": "python -m pip install -e .[test] --verbose", + "test_cmd": "pytest -rA -vv -o console_output_style=classic --tb=no" + }, + "0.4": { + "install": "python -m pip install -e .[test] --verbose", + "test_cmd": "pytest -rA -vv -o console_output_style=classic --tb=no" + }, + "1.1": { + "install": "python -m pip install -e .[test] --verbose", + "test_cmd": "pytest -rA -vv -o console_output_style=classic --tb=no" + }, + "1.2": { + "install": "python -m pip install -e .[test] --verbose", + "test_cmd": "pytest -rA -vv -o console_output_style=classic --tb=no" + }, + "1.3": { + "install": "python -m pip install -e .[test] --verbose", + "test_cmd": "pytest -rA -vv -o console_output_style=classic --tb=no" + }, + "3.0": { + "install": "python -m pip install -e .[test] --verbose", + "test_cmd": "pytest -rA" + }, + "3.1": { + "install": "python -m pip install -e .[test] --verbose", + "test_cmd": "pytest -rA" + }, + "3.2": { + "install": "python -m pip install -e .[test] --verbose", + "test_cmd": "pytest -rA" + }, + "4.1": { + "install": "python -m pip install -e .[test] --verbose", + "test_cmd": "pytest -rA" + }, + "4.2": { + "install": "python -m pip install -e .[test] --verbose", + "test_cmd": "pytest -rA" + }, + "4.3": { + "install": "python -m pip install -e .[test] --verbose", + "test_cmd": "pytest -rA" + }, + "5.0": { + "install": "python -m pip install -e .[test] --verbose", + "test_cmd": "pytest -rA" + }, + "5.1": { + "install": "python -m pip install -e .[test] --verbose", + "test_cmd": "pytest -rA" + }, + "5.2": { + "install": "python -m pip install -e .[test] --verbose", + "test_cmd": "pytest -rA" + }, + "v5.3": { + "install": "python -m pip install -e .[test] --verbose", + "test_cmd": "pytest -rA" + } + } + }, + "django/django": { + "parser": "django", + "versions": { + "1.10": { + "eval_commands": [ + "export LANG=en_US.UTF-8", + "export LC_ALL=en_US.UTF-8", + "export PYTHONIOENCODING=utf8", + "export LANGUAGE=en_US:en" + ], + "install": "python setup.py install", + "test_cmd": "./tests/runtests.py --verbosity 2 --settings=test_sqlite --parallel 1" + }, + "1.11": { + "eval_commands": [ + "export LANG=en_US.UTF-8", + "export LC_ALL=en_US.UTF-8", + "export PYTHONIOENCODING=utf8", + "export LANGUAGE=en_US:en" + ], + "install": "python setup.py install", + "test_cmd": "./tests/runtests.py --verbosity 2 --settings=test_sqlite --parallel 1" + }, + "1.4": { + "install": "python setup.py install", + "test_cmd": "./tests/runtests.py --verbosity 2 --settings=test_sqlite --parallel 1" + }, + "1.5": { + "install": "python setup.py install", + "test_cmd": "./tests/runtests.py --verbosity 2 --settings=test_sqlite --parallel 1" + }, + "1.6": { + "install": "python setup.py install", + "test_cmd": "./tests/runtests.py --verbosity 2 --settings=test_sqlite --parallel 1" + }, + "1.7": { + "eval_commands": [ + "export LANG=en_US.UTF-8", + "export LC_ALL=en_US.UTF-8", + "export PYTHONIOENCODING=utf8", + "export LANGUAGE=en_US:en" + ], + "install": "python setup.py install", + "test_cmd": "./tests/runtests.py --verbosity 2 --settings=test_sqlite --parallel 1" + }, + "1.8": { + "eval_commands": [ + "export LANG=en_US.UTF-8", + "export LC_ALL=en_US.UTF-8", + "export PYTHONIOENCODING=utf8", + "export LANGUAGE=en_US:en" + ], + "install": "python setup.py install", + "test_cmd": "./tests/runtests.py --verbosity 2 --settings=test_sqlite --parallel 1" + }, + "1.9": { + "eval_commands": [ + "export LANG=en_US.UTF-8", + "export LC_ALL=en_US.UTF-8", + "export PYTHONIOENCODING=utf8", + "export LANGUAGE=en_US:en" + ], + "install": "python setup.py install", + "test_cmd": "./tests/runtests.py --verbosity 2" + }, + "2.0": { + "eval_commands": [ + "export LANG=en_US.UTF-8", + "export LC_ALL=en_US.UTF-8", + "export PYTHONIOENCODING=utf8", + "export LANGUAGE=en_US:en" + ], + "install": "python setup.py install", + "test_cmd": "./tests/runtests.py --verbosity 2 --settings=test_sqlite --parallel 1" + }, + "2.1": { + "eval_commands": [ + "export LANG=en_US.UTF-8", + "export LC_ALL=en_US.UTF-8", + "export PYTHONIOENCODING=utf8", + "export LANGUAGE=en_US:en" + ], + "install": "python setup.py install", + "test_cmd": "./tests/runtests.py --verbosity 2 --settings=test_sqlite --parallel 1" + }, + "2.2": { + "eval_commands": [ + "export LANG=en_US.UTF-8", + "export LC_ALL=en_US.UTF-8", + "export PYTHONIOENCODING=utf8", + "export LANGUAGE=en_US:en" + ], + "install": "python setup.py install", + "test_cmd": "./tests/runtests.py --verbosity 2 --settings=test_sqlite --parallel 1" + }, + "3.0": { + "eval_commands": [ + "sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen", + "export LANG=en_US.UTF-8", + "export LANGUAGE=en_US:en", + "export LC_ALL=en_US.UTF-8" + ], + "install": "python -m pip install -e .", + "test_cmd": "./tests/runtests.py --verbosity 2 --settings=test_sqlite --parallel 1" + }, + "3.1": { + "eval_commands": [ + "sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen", + "export LANG=en_US.UTF-8", + "export LANGUAGE=en_US:en", + "export LC_ALL=en_US.UTF-8" + ], + "install": "python -m pip install -e .", + "test_cmd": "./tests/runtests.py --verbosity 2 --settings=test_sqlite --parallel 1" + }, + "3.2": { + "eval_commands": [ + "sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen", + "export LANG=en_US.UTF-8", + "export LANGUAGE=en_US:en", + "export LC_ALL=en_US.UTF-8" + ], + "install": "python -m pip install -e .", + "test_cmd": "./tests/runtests.py --verbosity 2 --settings=test_sqlite --parallel 1" + }, + "4.0": { + "install": "python -m pip install -e .", + "test_cmd": "./tests/runtests.py --verbosity 2 --settings=test_sqlite --parallel 1" + }, + "4.1": { + "install": "python -m pip install -e .", + "test_cmd": "./tests/runtests.py --verbosity 2 --settings=test_sqlite --parallel 1" + }, + "4.2": { + "install": "python -m pip install -e .", + "test_cmd": "./tests/runtests.py --verbosity 2 --settings=test_sqlite --parallel 1" + }, + "5.0": { + "install": "python -m pip install -e .", + "test_cmd": "./tests/runtests.py --verbosity 2 --settings=test_sqlite --parallel 1" + }, + "5.1": { + "install": "python -m pip install -e .", + "test_cmd": "./tests/runtests.py --verbosity 2 --settings=test_sqlite --parallel 1" + }, + "5.2": { + "install": "python -m pip install -e .", + "test_cmd": "./tests/runtests.py --verbosity 2 --settings=test_sqlite --parallel 1" + } + } + }, + "marshmallow-code/marshmallow": { + "parser": "pytest", + "versions": { + "2.18": { + "install": "python -m pip install -e '.[dev]'", + "test_cmd": "pytest -rA" + }, + "2.19": { + "install": "python -m pip install -e '.[dev]'", + "test_cmd": "pytest -rA" + }, + "2.20": { + "install": "python -m pip install -e '.[dev]'", + "test_cmd": "pytest -rA" + }, + "3.0": { + "install": "python -m pip install -e '.[dev]'", + "test_cmd": "pytest -rA" + }, + "3.1": { + "install": "python -m pip install -e '.[dev]'", + "test_cmd": "pytest -rA" + }, + "3.10": { + "install": "python -m pip install -e '.[dev]'", + "test_cmd": "pytest -rA" + }, + "3.11": { + "install": "python -m pip install -e '.[dev]'", + "test_cmd": "pytest -rA" + }, + "3.12": { + "install": "python -m pip install -e '.[dev]'", + "test_cmd": "pytest -rA" + }, + "3.13": { + "install": "python -m pip install -e '.[dev]'", + "test_cmd": "pytest -rA" + }, + "3.15": { + "install": "python -m pip install -e '.[dev]'", + "test_cmd": "pytest -rA" + }, + "3.16": { + "install": "python -m pip install -e '.[dev]'", + "test_cmd": "pytest -rA" + }, + "3.19": { + "install": "python -m pip install -e '.[dev]'", + "test_cmd": "pytest -rA" + }, + "3.2": { + "install": "python -m pip install -e '.[dev]'", + "test_cmd": "pytest -rA" + }, + "3.4": { + "install": "python -m pip install -e '.[dev]'", + "test_cmd": "pytest -rA" + }, + "3.8": { + "install": "python -m pip install -e '.[dev]'", + "test_cmd": "pytest -rA" + }, + "3.9": { + "install": "python -m pip install -e '.[dev]'", + "test_cmd": "pytest -rA" + } + } + }, + "matplotlib/matplotlib": { + "parser": "matplotlib", + "versions": { + "1.0": { + "install": "python setup.py build; python setup.py install", + "test_cmd": "pytest -rA" + }, + "1.1": { + "install": "python setup.py build; python setup.py install", + "test_cmd": "pytest -rA" + }, + "1.2": { + "install": "python setup.py build; python setup.py install", + "test_cmd": "pytest -rA" + }, + "1.3": { + "install": "python setup.py build; python setup.py install", + "test_cmd": "pytest -rA" + }, + "1.4": { + "install": "python setup.py build; python setup.py install", + "test_cmd": "pytest -rA" + }, + "1.5": { + "install": "python setup.py build; python setup.py install", + "test_cmd": "pytest -rA" + }, + "2.0": { + "install": "python setup.py build; python setup.py install", + "test_cmd": "pytest -rA" + }, + "2.1": { + "install": "python setup.py build; python setup.py install", + "test_cmd": "pytest -rA" + }, + "2.2": { + "install": "python setup.py build; python setup.py install", + "test_cmd": "pytest -rA" + }, + "3.0": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "3.1": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "3.2": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "3.3": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "3.4": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "3.5": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "3.6": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "3.7": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "3.8": { + "install": "python -m pip install --no-build-isolation -e \".[dev]\"", + "test_cmd": "pytest -rA" + }, + "3.9": { + "install": "python -m pip install --no-build-isolation -e \".[dev]\"", + "test_cmd": "pytest -rA" + } + } + }, + "mwaskom/seaborn": { + "parser": "seaborn", + "versions": { + "0.11": { + "install": "python -m pip install -e .", + "test_cmd": "pytest --no-header -rA" + }, + "0.12": { + "install": "python -m pip install -e .[dev]", + "test_cmd": "pytest --no-header -rA" + }, + "0.13": { + "install": "python -m pip install -e .[dev]", + "test_cmd": "pytest --no-header -rA" + }, + "0.14": { + "install": "python -m pip install -e .[dev]", + "test_cmd": "pytest --no-header -rA" + } + } + }, + "pallets/flask": { + "parser": "pytest", + "versions": { + "2.0": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2.1": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2.2": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2.3": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "3.0": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "3.1": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + } + } + }, + "psf/requests": { + "parser": "pytest_options", + "versions": { + "0.11": { + "install": "python -m pip install .", + "test_cmd": "pytest -rA" + }, + "0.13": { + "install": "python -m pip install .", + "test_cmd": "pytest -rA" + }, + "0.14": { + "install": "python -m pip install .", + "test_cmd": "pytest -rA" + }, + "0.7": { + "install": "python -m pip install .", + "test_cmd": "pytest -rA" + }, + "0.8": { + "install": "python -m pip install .", + "test_cmd": "pytest -rA" + }, + "0.9": { + "install": "python -m pip install .", + "test_cmd": "pytest -rA" + }, + "1.1": { + "install": "python -m pip install .", + "test_cmd": "pytest -rA" + }, + "1.2": { + "install": "python -m pip install .", + "test_cmd": "pytest -rA" + }, + "2.0": { + "install": "python -m pip install .", + "test_cmd": "pytest -rA" + }, + "2.10": { + "install": "python -m pip install .", + "test_cmd": "pytest -rA" + }, + "2.11": { + "install": "python -m pip install .", + "test_cmd": "pytest -rA" + }, + "2.12": { + "install": "python -m pip install .", + "test_cmd": "pytest -rA" + }, + "2.17": { + "install": "python -m pip install .", + "test_cmd": "pytest -rA" + }, + "2.18": { + "install": "python -m pip install .", + "test_cmd": "pytest -rA" + }, + "2.19": { + "install": "python -m pip install .", + "test_cmd": "pytest -rA" + }, + "2.2": { + "install": "python -m pip install .", + "test_cmd": "pytest -rA" + }, + "2.22": { + "install": "python -m pip install .", + "test_cmd": "pytest -rA" + }, + "2.25": { + "install": "python -m pip install .", + "test_cmd": "pytest -rA" + }, + "2.26": { + "install": "python -m pip install .", + "test_cmd": "pytest -rA" + }, + "2.27": { + "install": "python -m pip install .", + "test_cmd": "pytest -rA" + }, + "2.3": { + "install": "python -m pip install .", + "test_cmd": "pytest -rA" + }, + "2.31": { + "install": "python -m pip install .", + "test_cmd": "pytest -rA" + }, + "2.4": { + "install": "python -m pip install .", + "test_cmd": "pytest -rA" + }, + "2.5": { + "install": "python -m pip install .", + "test_cmd": "pytest -rA" + }, + "2.7": { + "install": "python -m pip install .", + "test_cmd": "pytest -rA" + }, + "2.8": { + "install": "python -m pip install .", + "test_cmd": "pytest -rA" + }, + "2.9": { + "install": "python -m pip install .", + "test_cmd": "pytest -rA" + }, + "3.0": { + "install": "python -m pip install .", + "test_cmd": "pytest -rA" + } + } + }, + "pvlib/pvlib-python": { + "parser": "pytest", + "versions": { + "0.1": { + "install": "python -m pip install -e .[all]", + "test_cmd": "pytest -rA" + }, + "0.2": { + "install": "python -m pip install -e .[all]", + "test_cmd": "pytest -rA" + }, + "0.3": { + "install": "python -m pip install -e .[all]", + "test_cmd": "pytest -rA" + }, + "0.4": { + "install": "python -m pip install -e .[all]", + "test_cmd": "pytest -rA" + }, + "0.5": { + "install": "python -m pip install -e .[all]", + "test_cmd": "pytest -rA" + }, + "0.6": { + "install": "python -m pip install -e .[all]", + "test_cmd": "pytest -rA" + }, + "0.7": { + "install": "python -m pip install -e .[all]", + "test_cmd": "pytest -rA" + }, + "0.8": { + "install": "python -m pip install -e .[all]", + "test_cmd": "pytest -rA" + }, + "0.9": { + "install": "python -m pip install -e .[all]", + "test_cmd": "pytest -rA" + } + } + }, + "pydata/xarray": { + "parser": "pytest", + "versions": { + "0.12": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.18": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.19": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.20": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2022.03": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2022.06": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2022.09": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2023.07": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2024.05": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + } + } + }, + "pydicom/pydicom": { + "parser": "pytest_options", + "versions": { + "1.0": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "1.1": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "1.2": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "1.3": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "1.4": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2.0": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2.1": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2.2": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2.3": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2.4": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "3.0": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + } + } + }, + "pylint-dev/astroid": { + "parser": "pytest", + "versions": { + "2.10": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2.12": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2.13": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2.14": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2.15": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2.16": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2.5": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2.6": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2.7": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2.8": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2.9": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "3.0": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + } + } + }, + "pylint-dev/pylint": { + "parser": "pytest_options", + "versions": { + "2.10": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2.11": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2.13": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2.14": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2.15": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2.16": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2.17": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2.8": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2.9": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "3.0": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "3.1": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "3.2": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "3.3": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "4.0": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + } + } + }, + "pytest-dev/pytest": { + "parser": "pytest", + "versions": { + "4.4": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "4.5": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "4.6": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "5.0": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "5.1": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "5.2": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "5.3": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "5.4": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "6.0": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "6.2": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "6.3": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "7.0": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "7.1": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "7.2": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "7.4": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "8.0": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "8.1": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "8.2": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "8.3": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "8.4": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + } + } + }, + "pyvista/pyvista": { + "parser": "pytest", + "versions": { + "0.20": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.21": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.22": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.23": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.24": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.25": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.26": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.27": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.28": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.29": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.30": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.31": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.32": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.33": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.34": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.35": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.36": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.37": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.38": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.39": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.40": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.41": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.42": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.43": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + } + } + }, + "scikit-learn/scikit-learn": { + "parser": "pytest_v2", + "versions": { + "0.20": { + "install": "python -m pip install -v --no-use-pep517 --no-build-isolation -e .", + "test_cmd": "pytest -rA" + }, + "0.21": { + "install": "python -m pip install -v --no-use-pep517 --no-build-isolation -e .", + "test_cmd": "pytest -rA" + }, + "0.22": { + "install": "python -m pip install -v --no-use-pep517 --no-build-isolation -e .", + "test_cmd": "pytest -rA" + }, + "1.3": { + "install": "python -m pip install -v --no-use-pep517 --no-build-isolation -e .", + "test_cmd": "pytest -rA" + }, + "1.4": { + "install": "python -m pip install -v --no-use-pep517 --no-build-isolation -e .", + "test_cmd": "pytest -rA" + }, + "1.5": { + "install": "python -m pip install -v --no-use-pep517 --no-build-isolation -e .", + "test_cmd": "pytest -rA" + }, + "1.6": { + "install": "python -m pip install -v --no-use-pep517 --no-build-isolation -e .", + "test_cmd": "pytest -rA" + } + } + }, + "sphinx-doc/sphinx": { + "parser": "pytest_v2", + "versions": { + "1.5": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + }, + "1.6": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + }, + "1.7": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + }, + "1.8": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + }, + "2.0": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + }, + "2.1": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + }, + "2.2": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + }, + "2.3": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + }, + "2.4": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + }, + "3.0": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + }, + "3.1": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + }, + "3.2": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + }, + "3.3": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + }, + "3.4": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + }, + "3.5": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + }, + "4.0": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + }, + "4.1": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + }, + "4.2": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + }, + "4.3": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + }, + "4.4": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + }, + "4.5": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + }, + "5.0": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + }, + "5.1": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + }, + "5.2": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + }, + "5.3": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + }, + "6.0": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + }, + "6.2": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + }, + "7.0": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + }, + "7.1": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + }, + "7.2": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + }, + "7.3": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + }, + "7.4": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + }, + "8.0": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + }, + "8.1": { + "install": "python -m pip install -e .[test]", + "test_cmd": "tox --current-env -epy39 -v --" + } + } + }, + "sqlfluff/sqlfluff": { + "parser": "pytest", + "versions": { + "0.10": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.11": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.12": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.13": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.4": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.5": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.6": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.8": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "0.9": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "1.0": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "1.1": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "1.2": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "1.3": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "1.4": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2.0": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2.1": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + }, + "2.2": { + "install": "python -m pip install -e .", + "test_cmd": "pytest -rA" + } + } + }, + "sympy/sympy": { + "parser": "sympy", + "versions": { + "0.7": { + "install": "python -m pip install -e .", + "test_cmd": "PYTHONWARNINGS='ignore::UserWarning,ignore::SyntaxWarning' bin/test -C --verbose" + }, + "1.0": { + "install": "python -m pip install -e .", + "test_cmd": "PYTHONWARNINGS='ignore::UserWarning,ignore::SyntaxWarning' bin/test -C --verbose" + }, + "1.1": { + "install": "python -m pip install -e .", + "test_cmd": "PYTHONWARNINGS='ignore::UserWarning,ignore::SyntaxWarning' bin/test -C --verbose" + }, + "1.10": { + "install": "python -m pip install -e .", + "test_cmd": "PYTHONWARNINGS='ignore::UserWarning,ignore::SyntaxWarning' bin/test -C --verbose" + }, + "1.11": { + "install": "python -m pip install -e .", + "test_cmd": "PYTHONWARNINGS='ignore::UserWarning,ignore::SyntaxWarning' bin/test -C --verbose" + }, + "1.12": { + "install": "python -m pip install -e .", + "test_cmd": "PYTHONWARNINGS='ignore::UserWarning,ignore::SyntaxWarning' bin/test -C --verbose" + }, + "1.13": { + "install": "python -m pip install -e .", + "test_cmd": "PYTHONWARNINGS='ignore::UserWarning,ignore::SyntaxWarning' bin/test -C --verbose" + }, + "1.14": { + "install": "python -m pip install -e .", + "test_cmd": "PYTHONWARNINGS='ignore::UserWarning,ignore::SyntaxWarning' bin/test -C --verbose" + }, + "1.2": { + "install": "python -m pip install -e .", + "test_cmd": "PYTHONWARNINGS='ignore::UserWarning,ignore::SyntaxWarning' bin/test -C --verbose" + }, + "1.4": { + "install": "python -m pip install -e .", + "test_cmd": "PYTHONWARNINGS='ignore::UserWarning,ignore::SyntaxWarning' bin/test -C --verbose" + }, + "1.5": { + "install": "python -m pip install -e .", + "test_cmd": "PYTHONWARNINGS='ignore::UserWarning,ignore::SyntaxWarning' bin/test -C --verbose" + }, + "1.6": { + "install": "python -m pip install -e .", + "test_cmd": "PYTHONWARNINGS='ignore::UserWarning,ignore::SyntaxWarning' bin/test -C --verbose" + }, + "1.7": { + "install": "python -m pip install -e .", + "test_cmd": "PYTHONWARNINGS='ignore::UserWarning,ignore::SyntaxWarning' bin/test -C --verbose" + }, + "1.8": { + "install": "python -m pip install -e .", + "test_cmd": "PYTHONWARNINGS='ignore::UserWarning,ignore::SyntaxWarning' bin/test -C --verbose" + }, + "1.9": { + "install": "python -m pip install -e .", + "test_cmd": "PYTHONWARNINGS='ignore::UserWarning,ignore::SyntaxWarning' bin/test -C --verbose" + } + } + } + }, + "source": "swebench==4.1.0" +} diff --git a/dokimi/dokimi/harnesses/tasks.py b/dokimi/dokimi/harnesses/tasks.py index a3d26da64..dba0afff4 100644 --- a/dokimi/dokimi/harnesses/tasks.py +++ b/dokimi/dokimi/harnesses/tasks.py @@ -52,6 +52,8 @@ FORMAT_HARBOR = "harbor" FORMAT_TASK_MD = "task-md" +# A SWE-bench dataset row staged by dokimi.harnesses.swebench (solution/ and tests/). +FORMAT_SWE_BENCH = "swe-bench" FORMAT_TBENCH_V1 = tbench_v1.FORMAT PARSER_REWARD_FILE = "reward-file" @@ -61,9 +63,9 @@ # verifier/, while a Harbor-layout directory that holds both pairs uses # solution/ and tests/ here, the names Harbor itself reads. _SOLUTION_DIRS = {FORMAT_HARBOR: ("solution", "oracle"), FORMAT_TASK_MD: ("oracle", "solution"), - FORMAT_TBENCH_V1: ("solution",)} + FORMAT_TBENCH_V1: ("solution",), FORMAT_SWE_BENCH: ("solution",)} _TESTS_DIRS = {FORMAT_HARBOR: ("tests", "verifier"), FORMAT_TASK_MD: ("verifier", "tests"), - FORMAT_TBENCH_V1: ("tests",)} + FORMAT_TBENCH_V1: ("tests",), FORMAT_SWE_BENCH: ("tests",)} @dataclass(frozen=True) diff --git a/dokimi/dokimi/task_formats/datasets.py b/dokimi/dokimi/task_formats/datasets.py index d2011fba6..2c5d7bd73 100644 --- a/dokimi/dokimi/task_formats/datasets.py +++ b/dokimi/dokimi/task_formats/datasets.py @@ -21,8 +21,8 @@ from typing import Any from dokimi.task_formats.model import ( - BY_EXPLORE, BY_QUESTIONS, BY_TRIGGER_SET, KIND_ANSWER_ONLY, KIND_EXECUTABLE, KIND_LOCALIZATION, - KIND_TRIGGER, Fingerprint, Report, + BY_DOCKER, BY_EXPLORE, BY_QUESTIONS, BY_TRIGGER_SET, KIND_ANSWER_ONLY, KIND_EXECUTABLE, + KIND_LOCALIZATION, KIND_TRIGGER, Fingerprint, Report, ) from dokimi.task_formats.source import FileSource, first_rows, key_share @@ -109,9 +109,28 @@ def describe(self, source: FileSource) -> Report: report.problems.append(f"{bad} items have a FAIL_TO_PASS that is not a list of test ids") _lint_duplicate_ids(report, rows, "instance_id") report.details["repos"] = sorted({str(r.get("repo")) for r in rows if r.get("repo")})[:20] + _swe_bench_consumers(report, rows) return report +def _swe_bench_consumers(report: Report, rows: list[dict[str, Any]]) -> None: + """``docker-*`` (``tasks run --swe-bench``) when every row's repo and version has a test spec.""" + from dokimi.harnesses.swebench import specs + + known = specs()["repos"] + unsupported = sorted({str(r.get("repo")) for r in rows if r.get("repo") not in known}) + if unsupported: + report.details["repos_without_test_spec"] = unsupported[:20] + return + # `tasks run` refuses a version the table lacks, as `repo_spec` does. + versions = sorted({f"{r.get('repo')} {r.get('version')}" for r in rows + if "test_cmd" not in known[r.get("repo")]["versions"].get(str(r.get("version")), {})}) + if versions: + report.details["versions_without_test_spec"] = versions[:20] + elif rows and not report.problems: + report.consumers.append(BY_DOCKER) + + # --- Inspect AI samples and OpenAI evals -------------------------------------- diff --git a/dokimi/pyproject.toml b/dokimi/pyproject.toml index 39d24d61f..809b3e0fc 100644 --- a/dokimi/pyproject.toml +++ b/dokimi/pyproject.toml @@ -103,6 +103,7 @@ dokimi = [ "schemas/*.json", "repos/profiles/*.yaml", "repos/support/*.py", + "harnesses/swebench_specs.json", ] [tool.scriv] diff --git a/dokimi/scripts/gen_swebench_specs.py b/dokimi/scripts/gen_swebench_specs.py new file mode 100644 index 000000000..782a18c9a --- /dev/null +++ b/dokimi/scripts/gen_swebench_specs.py @@ -0,0 +1,43 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Regenerate ``dokimi/harnesses/swebench_specs.json`` from the ``swebench`` package. + +dokimi does not depend on ``swebench`` (it pulls in modal, datasets, docker and +pre-commit); the loader reads this table instead. Run it from a throwaway env: + + uv run --no-project --with swebench==4.1.0 python scripts/gen_swebench_specs.py +""" + +from __future__ import annotations + +import importlib +import importlib.metadata +import json +import sys +from pathlib import Path + +OUT = Path(__file__).resolve().parent.parent / "dokimi" / "harnesses" / "swebench_specs.json" +_KEYS = ("test_cmd", "eval_commands", "install") + + +def main() -> int: + specs_py = importlib.import_module("swebench.harness.constants.python").MAP_REPO_VERSION_TO_SPECS_PY + parsers = importlib.import_module("swebench.harness.log_parsers").MAP_REPO_TO_PARSER + repos = {} + for repo, versions in sorted(specs_py.items()): + parser = parsers.get(repo) + if parser is None: # no log parser upstream either: the repo cannot be graded + continue + repos[repo] = { + "parser": parser.__name__.removeprefix("parse_log_"), + "versions": {v: {k: s[k] for k in _KEYS if s.get(k)} for v, s in sorted(versions.items())}, + } + table = {"source": f"swebench=={importlib.metadata.version('swebench')}", "repos": repos} + OUT.write_text(json.dumps(table, indent=1, sort_keys=True) + "\n", encoding="utf-8") + print(f"wrote {OUT} ({len(repos)} repos)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/dokimi/tests/fixtures/swebench/lite-2.json b/dokimi/tests/fixtures/swebench/lite-2.json new file mode 100644 index 000000000..8b8a9b9b5 --- /dev/null +++ b/dokimi/tests/fixtures/swebench/lite-2.json @@ -0,0 +1,32 @@ +[ + { + "repo": "pytest-dev/pytest", + "instance_id": "pytest-dev__pytest-5227", + "base_commit": "2051e30b9b596e944524ccb787ed20f9f5be93e3", + "patch": "diff --git a/src/x.py b/src/x.py\n--- a/src/x.py\n+++ b/src/x.py\n@@ -1 +1 @@\n-X = 1\n+X = 2\n", + "test_patch": "diff --git a/testing/test_x.py b/testing/test_x.py\n--- a/testing/test_x.py\n+++ b/testing/test_x.py\n@@ -1 +1,2 @@\n def test_a(): pass\n+def test_b(): assert 1\ndiff --git a/testing/data.json b/testing/data.json\nnew file mode 100644\n--- /dev/null\n+++ b/testing/data.json\n@@ -0,0 +1 @@\n+{}\n", + "problem_statement": "Improve default logging format\n", + "hints_text": "", + "created_at": "2019-05-07T20:27:24Z", + "version": "4.4", + "FAIL_TO_PASS": "[\"testing/test_x.py::test_b\"]", + "PASS_TO_PASS": "[\"testing/test_x.py::test_a\"]", + "environment_setup_commit": "4ccaa987d47566e3907f2f74167c4ab7997f622f" + }, + { + "repo": "django/django", + "instance_id": "django__django-11099", + "base_commit": "d26b2424437dabeeca94d7900b37d2df4410da0c", + "patch": "diff --git a/src/x.py b/src/x.py\n--- a/src/x.py\n+++ b/src/x.py\n@@ -1 +1 @@\n-X = 1\n+X = 2\n", + "test_patch": "diff --git a/tests/auth_tests/test_validators.py b/tests/auth_tests/test_validators.py\n--- a/tests/auth_tests/test_validators.py\n+++ b/tests/auth_tests/test_validators.py\n@@ -1 +1,2 @@\n x = 1\n+y = 2\n", + "problem_statement": "UsernameValidator allows trailing newline\n", + "hints_text": "", + "created_at": "2019-03-20T03:46:18Z", + "version": "3.0", + "FAIL_TO_PASS": [ + "test_unicode_validator (auth_tests.test_validators.UsernameValidatorsTests)" + ], + "PASS_TO_PASS": [], + "environment_setup_commit": "419a78300f7cd27611196e1e464d50fd0385ff27" + } +] diff --git a/dokimi/tests/test_swebench_tasks.py b/dokimi/tests/test_swebench_tasks.py new file mode 100644 index 000000000..35f40fe71 --- /dev/null +++ b/dokimi/tests/test_swebench_tasks.py @@ -0,0 +1,455 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""SWE-bench rows as executable tasks (dokimi.harnesses.swebench) and their grader. + +The fixture rows keep the SWE-bench Lite field set (FAIL_TO_PASS as the Hugging Face +JSON string in one row, a list in the other) with tiny synthetic patches. The docker +test at the end runs one real instance image and is skipped unless +``DOKIMI_SWEBENCH_DOCKER=1``: the image is ~4 GB. +""" + +from __future__ import annotations + +import json +import os +import shlex +import shutil +from pathlib import Path + +import pytest + +from dokimi.cli import build_parser +from dokimi.harnesses import create_harness +from dokimi.harnesses import swebench_grade as grader +from dokimi.harnesses.swebench import ( + SweBenchError, + eval_script, + instance_image, + load_swebench, + patch_files, + read_instances, + repo_spec, + specs, + stage, + test_directives as directives, +) +from dokimi.harnesses.tasks import FORMAT_SWE_BENCH + +FIXTURE = Path(__file__).parent / "fixtures" / "swebench" / "lite-2.json" +PYTEST_ID = "pytest-dev__pytest-5227" +DJANGO_ID = "django__django-11099" + + +def _row(iid: str) -> dict: + return next(r for r in read_instances(FIXTURE) if r["instance_id"] == iid) + + +def _log(body: str) -> str: + return f"+ git apply\n>>>>> Start Test Output\n{body}\n>>>>> End Test Output\n" + + +# ── loading and staging ────────────────────────────────────────────────────── + + +def test_a_row_stages_as_a_swe_bench_task_on_its_prebuilt_image(tmp_path): + task = stage(_row(PYTEST_ID), tmp_path) + assert task.format == FORMAT_SWE_BENCH + assert task.environment["docker_image"] == "swebench/sweb.eval.x86_64.pytest-dev_1776_pytest-5227:latest" + dockerfile = (task.path / "environment" / "Dockerfile").read_text() + assert dockerfile.startswith("FROM swebench/sweb.eval.x86_64.pytest-dev_1776_pytest-5227:latest\n") + assert "mv /testbed /app && ln -s /app /testbed" in dockerfile + assert task.solution_dir == task.path / "solution" and task.tests_dir == task.path / "tests" + assert (task.solution_dir / "patch.diff").read_text() == _row(PYTEST_ID)["patch"] + assert task.instruction.startswith("Improve default logging format") + + +def test_the_hugging_face_json_string_test_lists_are_decoded(tmp_path): + task = stage(_row(PYTEST_ID), tmp_path) + config = json.loads((task.tests_dir / "config.json").read_text()) + assert config == {"instance_id": PYTEST_ID, "parser": "pytest", + "FAIL_TO_PASS": ["testing/test_x.py::test_b"], + "PASS_TO_PASS": ["testing/test_x.py::test_a"]} + + +def test_an_empty_patch_override_is_the_negative_control(tmp_path): + task = stage(_row(PYTEST_ID), tmp_path, patch="") + assert (task.solution_dir / "patch.diff").read_text() == "" + assert task.metadata["oracle_patch"] == "override" + + +def test_a_row_naming_its_own_image_keeps_it(): + row = {"instance_id": "a__b-1", "image_name": "jyangballin/swesmith.x86_64.a_1776_b.abc"} + assert instance_image(row) == "jyangballin/swesmith.x86_64.a_1776_b.abc" + assert instance_image({"instance_id": "Org__Repo-2"}, namespace="me") == "me/sweb.eval.x86_64.org_1776_repo-2:latest" + + +def test_a_row_image_name_with_a_newline_is_refused_not_written_into_the_dockerfile(tmp_path): + # The image goes into the Dockerfile's FROM line; a newline would add instructions. + row = dict(_row(PYTEST_ID), image_name="ubuntu:22.04\nRUN echo injected > /tmp/x") + with pytest.raises(SweBenchError, match="image reference"): + stage(row, tmp_path / "root") + assert list(tmp_path.iterdir()) == [] + + +def test_load_selects_ids_in_order_and_names_an_unknown_one(tmp_path): + tasks = load_swebench(FIXTURE, tmp_path, [DJANGO_ID, PYTEST_ID]) + assert [t.name for t in tasks] == [DJANGO_ID, PYTEST_ID] + with pytest.raises(SweBenchError, match="no instance nope__nope-1"): + load_swebench(FIXTURE, tmp_path, ["nope__nope-1"]) + + +def test_all_skips_the_known_bad_rows_and_an_explicit_id_still_runs(tmp_path): + from dokimi.harnesses.network_policy import resolve + from dokimi.harnesses.swebench import KNOWN_BAD + + rows = json.loads(FIXTURE.read_text(encoding="utf-8")) + bad = dict(rows[0], instance_id="psf__requests-863") + data = tmp_path / "rows.json" + data.write_text(json.dumps([*rows, bad]), encoding="utf-8") + assert "psf__requests-863" in KNOWN_BAD + assert "psf__requests-863" not in {t.name for t in load_swebench(data, tmp_path / "a")} + (task,) = load_swebench(data, tmp_path / "b", ["psf__requests-863"]) + plan = resolve(task, agent=True) + assert plan.solve.mode == "none" # the agent gets no network + assert (plan.verify.mode, plan.verify.hosts) == ("allowlist", ("files.pythonhosted.org:443", "pypi.org:443")) + assert resolve(task, agent=False).solve.mode == "none" # nor does the oracle's apply + assert resolve(task, ["verify=none"]).verify.mode == "none" + + +def test_the_testbed_env_is_first_on_the_agents_path(tmp_path): + task = stage(_row(PYTEST_ID), tmp_path) + dockerfile = (task.path / "environment" / "Dockerfile").read_text() + assert "ENV PATH=/opt/miniconda3/envs/testbed/bin:$PATH" in dockerfile.splitlines() + + +def test_a_host_side_agent_is_refused_a_swe_bench_task(tmp_path): + from dokimi.harnesses.docker_solver import _refuse_unrunnable + + task = stage(_row(PYTEST_ID), tmp_path) + with pytest.raises(ValueError, match="cannot be solved by a host-side agent"): + _refuse_unrunnable(task, "claude", "host") + _refuse_unrunnable(task, "claude", "container") + + +def test_a_bad_row_late_in_the_file_stages_nothing(tmp_path): + rows = json.loads(FIXTURE.read_text(encoding="utf-8")) + data = tmp_path / "rows.json" + data.write_text(json.dumps([*rows, dict(rows[0], instance_id="x__y-1", version="0.0")]), encoding="utf-8") + with pytest.raises(SweBenchError, match="version '0.0'"): + load_swebench(data, tmp_path / "root") + assert not (tmp_path / "root").exists() + + +def test_the_all_banner_names_only_the_known_bad_rows_the_file_holds(tmp_path, capsys): + import argparse + + from dokimi.commands.tasks import _select_swebench + + rows = json.loads(FIXTURE.read_text(encoding="utf-8")) + data = tmp_path / "rows.json" + args = argparse.Namespace(all=True, tasks=None, swe_bench=str(data), output_dir=str(tmp_path / "out")) + data.write_text(json.dumps([*rows, dict(rows[0], instance_id="psf__requests-863")]), encoding="utf-8") + _select_swebench(args) + err = capsys.readouterr().err + assert "skips 1 known-bad rows" in err and "psf__requests-863" in err + assert "astropy__astropy-7606" not in err # a KNOWN_BAD id the file does not hold is not named + data.write_text(json.dumps(rows), encoding="utf-8") + _select_swebench(args) + assert "skips" not in capsys.readouterr().err + + +def test_a_row_without_a_test_spec_or_a_field_is_refused(tmp_path): + row = dict(_row(PYTEST_ID), repo="someone/else") + with pytest.raises(SweBenchError, match="no SWE-bench test spec for repo 'someone/else'"): + stage(row, tmp_path) + row = {k: v for k, v in _row(PYTEST_ID).items() if k != "test_patch"} + with pytest.raises(SweBenchError, match="missing test_patch"): + stage(row, tmp_path) + with pytest.raises(SweBenchError, match="version '0.0'"): + repo_spec("pytest-dev/pytest", "0.0") + + + +@pytest.mark.parametrize("field", ["repo", "base_commit", "problem_statement", "patch", "test_patch"]) +def test_a_null_or_non_string_text_field_is_a_row_error_not_a_traceback(tmp_path, field): + with pytest.raises(SweBenchError, match=f"{field} must be a string"): + stage(dict(_row(PYTEST_ID), **{field: None}), tmp_path) + + +def test_an_unsafe_or_duplicate_instance_id_is_refused(tmp_path): + row = _row(PYTEST_ID) + for bad in ("../../escaped__x-1", str(tmp_path / "abs__x-1"), "a/b", ".hidden"): + with pytest.raises(SweBenchError, match="instance_id must match"): + stage(dict(row, instance_id=bad), tmp_path / "root") + assert list(tmp_path.iterdir()) == [] + data = tmp_path / "dup.json" + data.write_text(json.dumps([row, dict(row, patch="SECOND")]), encoding="utf-8") + for ids in (None, [PYTEST_ID]): + with pytest.raises(SweBenchError, match=f"duplicate instance_id {PYTEST_ID}"): + load_swebench(data, tmp_path / "d", ids) + assert not (tmp_path / "d").exists() + + +# ── the eval script ────────────────────────────────────────────────────────── + + +def test_the_eval_script_resets_applies_and_runs_the_touched_test_files(): + script = eval_script(_row(PYTEST_ID), repo_spec("pytest-dev/pytest", "4.4")[1]) + base = _row(PYTEST_ID)["base_commit"] + assert script.count(f"git checkout {base} testing/test_x.py") == 2 # before and after + assert "rm -f -- testing/data.json" in script # added by the test patch + assert "python -m pip install -e ." in script # upstream reinstalls before the run + start, run, end = script.index("Start Test Output"), script.index("pytest -rA testing/test_x.py"), \ + script.index("End Test Output") + assert start < run < end + assert "testing/data.json" not in script[start:end] # a .json file is not a directive + + +def test_the_eval_script_passes_row_values_as_words_not_shell(): + test_patch = ("diff --git a/t$(touch pwned2).py b/t$(touch pwned2).py\n" + "diff --git a/a b.py b/a b.py\n" + "diff --git a/t.py b/t.py\n--- a/t.py\n+++ b/t.py\n") + row = dict(_row(PYTEST_ID), base_commit="abc; touch pwned #", test_patch=test_patch) + lines = eval_script(row, repo_spec("pytest-dev/pytest", "4.4")[1]).splitlines() + words = [shlex.split(line) for line in lines if line.startswith(("git checkout", "git -c", "pytest"))] + assert ["git", "-c", "core.fileMode=false", "diff", "abc; touch pwned #"] in words + assert words.count(["git", "checkout", "abc; touch pwned #", "t.py"]) == 2 + assert ["pytest", "-rA", "t$(touch pwned2).py", "a b.py", "t.py"] in words + + +def test_a_tracked_test_file_with_a_space_is_reset_by_its_whole_name(): + # git puts a tab after a name with a space in its ---/+++ header lines. + test_patch = "diff --git a/a b.py b/a b.py\n--- a/a b.py\t\n+++ b/a b.py\t\n" + row = dict(_row(PYTEST_ID), test_patch=test_patch) + lines = eval_script(row, repo_spec("pytest-dev/pytest", "4.4")[1]).splitlines() + words = [shlex.split(line) for line in lines if line.startswith("git checkout")] + assert words == [["git", "checkout", row["base_commit"], "a b.py"]] * 2 + + +def test_an_added_test_file_with_a_space_is_cleaned_by_its_whole_name(): + # An added file has no "--- a/" name, so only the tab-stripped "+++ b/" side supplies it. + test_patch = "diff --git a/new file.py b/new file.py\n--- /dev/null\n+++ b/new file.py\t\n" + assert patch_files(test_patch) == ([], ["new file.py"]) + row = dict(_row(PYTEST_ID), test_patch=test_patch) + lines = eval_script(row, repo_spec("pytest-dev/pytest", "4.4")[1]).splitlines() + words = [shlex.split(line) for line in lines if line.startswith("rm -f")] + assert words and all(w == ["rm", "-f", "--", "new file.py"] for w in words) + + +def test_a_test_patch_that_ends_the_heredoc_is_refused_not_run_as_shell(): + # The patch is fed to git apply by a heredoc; a line equal to its delimiter ends the heredoc. + test_patch = _row(PYTEST_ID)["test_patch"] + "EOF_114329324912\ntouch pwned\n" + row = dict(_row(PYTEST_ID), test_patch=test_patch) + with pytest.raises(SweBenchError, match="heredoc delimiter"): + eval_script(row, repo_spec("pytest-dev/pytest", "4.4")[1]) + + +@pytest.mark.parametrize("line", ["EOF_1143293\x0024912", "EOF_114329324912\x00"]) +def test_a_nul_byte_that_bash_would_drop_cannot_end_the_heredoc(line): + # bash drops NUL bytes while reading a script, so either line ends the heredoc there. + row = dict(_row(PYTEST_ID), test_patch=_row(PYTEST_ID)["test_patch"] + line + "\ntouch pwned\n") + with pytest.raises(SweBenchError, match="NUL byte"): + eval_script(row, repo_spec("pytest-dev/pytest", "4.4")[1]) + + +def test_django_directives_are_dotted_test_modules(): + assert directives("django/django", _row(DJANGO_ID)["test_patch"]) == ["auth_tests.test_validators"] + script = eval_script(_row(DJANGO_ID), repo_spec("django/django", "3.0")[1]) + assert "./tests/runtests.py --verbosity 2 --settings=test_sqlite --parallel 1 auth_tests.test_validators" in script + + +def test_every_repo_in_the_spec_table_has_a_grader_parser(): + parsers = {entry["parser"] for entry in specs()["repos"].values()} + assert parsers <= set(grader.PARSERS) + assert specs()["source"].startswith("swebench==") + + +# ── the grader (SWE-bench resolution semantics) ────────────────────────────── + +_CONFIG = {"instance_id": "x", "parser": "pytest", "FAIL_TO_PASS": ["t.py::f"], "PASS_TO_PASS": ["t.py::p"]} + + +def test_all_tests_passing_resolves(): + report = grader.grade(_log("PASSED t.py::f\nPASSED t.py::p"), _CONFIG) + assert report["resolved"] is True + + +def test_a_missing_or_failed_fail_to_pass_test_does_not_resolve(): + assert grader.grade(_log("PASSED t.py::p"), _CONFIG)["FAIL_TO_PASS"]["failure"] == ["t.py::f"] + report = grader.grade(_log("FAILED t.py::f - AssertionError\nPASSED t.py::p"), _CONFIG) + assert report["resolved"] is False + + +def test_a_skipped_pass_to_pass_test_counts_neither_way_as_upstream(): + report = grader.grade(_log("PASSED t.py::f\nSKIPPED t.py::p"), _CONFIG) + assert report["PASS_TO_PASS"] == {"success": [], "failure": []} + assert report["resolved"] is True + + +def test_a_skipped_fail_to_pass_test_or_none_at_all_does_not_resolve(): + report = grader.grade(_log("SKIPPED t.py::f\nPASSED t.py::p"), _CONFIG) + assert report["FAIL_TO_PASS"] == {"success": [], "failure": ["t.py::f"]} + assert report["resolved"] is False + empty = dict(_CONFIG, FAIL_TO_PASS=[], PASS_TO_PASS=[]) + assert grader.grade(_log("PASSED t.py::f"), empty)["resolved"] is False + + +def test_an_xfail_fail_to_pass_test_counts_as_a_success(): + report = grader.grade(_log("XFAIL t.py::f\nPASSED t.py::p"), _CONFIG) + assert report["FAIL_TO_PASS"] == {"success": ["t.py::f"], "failure": []} + assert report["resolved"] is True + + +def test_an_empty_marked_section_falls_back_to_the_whole_log(): + log = "PASSED t.py::f\nPASSED t.py::p\n" + _log("") + assert grader.grade(log, _CONFIG)["resolved"] is True + + +def test_a_bare_seaborn_status_word_is_skipped_not_an_index_error(): + # A line that is only FAILED or PASSED has no test name; it must not raise. + log = "FAILED\nPASSED\n PASSED \nPASSED t.py::f\n" + assert grader.parse_seaborn(log) == {"t.py::f": "PASSED"} + + +def test_pytest_v2_reads_coloured_and_old_style_lines(): + config = dict(_CONFIG, parser="pytest_v2") + coloured = _log("\x1b[32mPASSED\x1b[0m t.py::f\nPASSED t.py::p") + assert grader.grade(coloured, config)["resolved"] is True + old_style = _log("t.py::f PASSED\nPASSED t.py::p") + assert grader.grade(old_style, config)["resolved"] is True + + +def test_a_log_without_a_test_run_does_not_resolve(): + assert grader.grade("error: patch failed\n", _CONFIG) == {"instance_id": "x", "tests_ran": False, + "resolved": False} + + +@pytest.mark.parametrize("code", ["Patch Apply Failed", "Reset Failed", "Tests Errored", "Tests Timed Out"]) +def test_a_bad_code_in_the_log_does_not_resolve_even_with_passing_lines(code): + log = _log("PASSED t.py::f\nPASSED t.py::p") + f">>>>> {code}\n" + assert grader.grade(log, _CONFIG) == {"instance_id": "x", "tests_ran": False, "resolved": False} + + +@pytest.mark.parametrize("body", ["ERROR t.py::f\nPASSED t.py::p", "PASSED t.py::f\nERROR t.py::p"]) +def test_an_error_status_counts_as_a_failure(body): + assert grader.grade(_log(body), _CONFIG)["resolved"] is False + + +def test_django_fail_and_error_headers_are_read(): + # Upstream keys a "FAIL:"/"ERROR:" header by the word after it, as here. + found = grader.parse_django("FAIL: test_y (m.T)\nERROR: test_z (m.T)\n") + assert found == {"test_y": "FAILED", "test_z": "ERROR"} + + +def test_the_scripts_trust_the_image_checkout_and_fall_back_through_the_apply_commands(tmp_path): + task = stage(_row(PYTEST_ID), tmp_path) + trust = "git config --global --add safe.directory '*'" + eval_sh = (task.tests_dir / "eval.sh").read_text() + assert eval_sh.index(trust) < eval_sh.index("git status") + solve = (task.solution_dir / "solve.sh").read_text() + assert solve.index(trust) < solve.index("git apply") + assert ("git apply --verbose /solution/patch.diff || git apply --verbose --reject /solution/patch.diff" + " || patch --batch --fuzz=5 -p1 -i /solution/patch.diff || ") in solve + + +def test_django_and_sympy_logs_parse_to_statuses(): + django = grader.parse_django( + "test_a (m.T) ... ok\ntest_b (m.T) ... FAIL\ntest_c (m.T) ... skipped 'x'\n" + "test_d (m.T) ... Internal noise\nok\n") + assert django == {"test_a (m.T)": "PASSED", "test_b (m.T)": "FAILED", + "test_c (m.T)": "SKIPPED", "test_d (m.T)": "PASSED"} + assert grader.parse_sympy("test_one ok\ntest_two F\ntest_three E\n") == { + "test_one": "PASSED", "test_two": "FAILED", "test_three": "ERROR"} + + +def test_pytest_options_shortens_a_path_parameter(): + found = grader.parse_pytest_options("PASSED t.py::f[/tmp/a/b.txt]\nPASSED t.py::g[x-1]\n") + assert found == {"t.py::f[/b.txt]": "PASSED", "t.py::g[x-1]": "PASSED"} + + +def test_the_grader_script_writes_reward_and_report(tmp_path, capsys): + (tmp_path / "config.json").write_text(json.dumps(_CONFIG)) + (tmp_path / "test.log").write_text(_log("PASSED t.py::f\nFAILED t.py::p")) + out = tmp_path / "verifier" + argv = ["grade.py", str(tmp_path / "config.json"), str(tmp_path / "test.log"), str(out)] + assert grader.main(argv) == 1 # the exit code carries the verdict when reward.txt is lost + assert (out / "reward.txt").read_text() == "0\n" + assert json.loads((out / "report.json").read_text())["PASS_TO_PASS"]["failure"] == ["t.py::p"] + assert "PASS_TO_PASS failed: t.py::p" in capsys.readouterr().out + (tmp_path / "test.log").write_text(_log("PASSED t.py::f\nPASSED t.py::p")) + assert grader.main(argv) == 0 + + +def test_the_verifier_prints_the_end_of_the_eval_log(tmp_path): + test_sh = (stage(_row(PYTEST_ID), tmp_path).tests_dir / "test.sh").read_text().splitlines() + tail = test_sh.index("tail -n 200 /logs/verifier/test_output.log") + assert test_sh.index("bash /tests/eval.sh > /logs/verifier/test_output.log 2>&1") < tail + assert tail < next(i for i, line in enumerate(test_sh) if "grade.py" in line) + + +# ── the command line ───────────────────────────────────────────────────────── + + +def _run(argv: list[str]) -> None: + args = build_parser().parse_args(argv) + args.func(args) + + +def test_tasks_run_stages_swe_bench_rows_for_a_dry_run(tmp_path): + out = tmp_path / "out" + _run(["tasks", "run", "--swe-bench", str(FIXTURE), "--tasks", PYTEST_ID, + "--harnesses", "docker-oracle", "--output-dir", str(out), "--dry-run"]) + assert (out / "swe-bench-tasks" / PYTEST_ID / "tests" / "eval.sh").is_file() + runs = json.loads((out / "task_runs.json").read_text()) + assert [r["task"] for r in runs["results"]] == [PYTEST_ID] + + +def test_tasks_run_names_a_missing_swe_bench_instance(tmp_path): + with pytest.raises(SystemExit, match="--swe-bench: .*no instance x__y-1"): + _run(["tasks", "run", "--swe-bench", str(FIXTURE), "--tasks", "x__y-1", + "--harnesses", "docker-oracle", "--output-dir", str(tmp_path)]) + + + +def test_tasks_run_refuses_an_all_selection_with_no_runnable_row(tmp_path): + rows = json.loads(FIXTURE.read_text(encoding="utf-8")) + data = tmp_path / "bad.json" + data.write_text(json.dumps([dict(rows[0], instance_id="psf__requests-863")]), encoding="utf-8") + out = tmp_path / "out" + with pytest.raises(SystemExit, match="no runnable rows"): + _run(["tasks", "run", "--swe-bench", str(data), "--all", + "--harnesses", "docker-oracle", "--output-dir", str(out), "--dry-run"]) + assert not (out / "task_runs.json").exists() + + +# ── one real instance image (opt-in) ───────────────────────────────────────── + +_REAL = Path(os.environ.get("DOKIMI_SWEBENCH_ROWS", "")) + + +@pytest.mark.skipif(os.environ.get("DOKIMI_SWEBENCH_DOCKER") != "1" or shutil.which("docker") is None + or not _REAL.is_file(), + reason="pulls a ~4 GB SWE-bench image: set DOKIMI_SWEBENCH_DOCKER=1 and " + "DOKIMI_SWEBENCH_ROWS=") +@pytest.mark.allow_hosts(["127.0.0.1"]) +def test_gold_patch_resolves_and_empty_patch_does_not_in_docker(tmp_path): + row = next(r for r in read_instances(_REAL) if r["instance_id"] == PYTEST_ID) + harness = create_harness("docker-oracle") + gold = harness.run(stage(row, tmp_path / "gold"), output_dir=tmp_path / "run-gold") + empty = harness.run(stage(row, tmp_path / "empty", patch=""), output_dir=tmp_path / "run-empty") + assert (gold.reward, empty.reward) == (1.0, 0.0) + + +def test_tasks_inspect_names_docker_as_the_consumer_of_a_supported_dataset(tmp_path): + from dokimi.task_formats import inspect_path + + [report] = inspect_path(FIXTURE) + assert (report.format, report.consumers) == ("swe-bench", ["docker-*"]) + other = tmp_path / "other.jsonl" + other.write_text(json.dumps(dict(_row(PYTEST_ID), repo="someone/else")) + "\n") + [report] = inspect_path(other) + assert report.consumers == [] and report.details["repos_without_test_spec"] == ["someone/else"] + other.write_text(json.dumps(dict(_row(PYTEST_ID), version="9.9")) + "\n") + [report] = inspect_path(other) + assert report.consumers == [] and report.details["versions_without_test_spec"] == ["pytest-dev/pytest 9.9"]