From 9223077528f16d09984da6432ab4fcde45ee2d22 Mon Sep 17 00:00:00 2001 From: Peter Hense Date: Fri, 13 Nov 2026 20:12:35 +0100 Subject: [PATCH 1/2] test(evals): unscripted routing cases Co-Authored-By: Claude Fable 5.2 --- evals/natural/cases/cross-module-feature.json | 10 ++++++++ evals/natural/cases/failing-test.json | 9 +++++++ evals/natural/cases/typo-fix.json | 7 ++++++ evals/natural/expected.json | 24 +++++++++++++++++++ 4 files changed, 50 insertions(+) create mode 100644 evals/natural/cases/cross-module-feature.json create mode 100644 evals/natural/cases/failing-test.json create mode 100644 evals/natural/cases/typo-fix.json create mode 100644 evals/natural/expected.json diff --git a/evals/natural/cases/cross-module-feature.json b/evals/natural/cases/cross-module-feature.json new file mode 100644 index 0000000..830717c --- /dev/null +++ b/evals/natural/cases/cross-module-feature.json @@ -0,0 +1,10 @@ +{ + "id": "cross-module-feature", + "prompt": "Ich brauche in dieser App einen Export der Rechnungen als CSV und PDF. Der soll die Rollen aus accounts beachten und auch in den Reports auftauchen.", + "files": { + "README.md": "# Ledger\n\nInvoicing app with modules billing, accounts and reports.\n", + "billing/invoice.py": "class Invoice:\n pass\n", + "accounts/roles.py": "ROLES = ['admin', 'clerk']\n", + "reports/summary.py": "def summary():\n return {}\n" + } +} diff --git a/evals/natural/cases/failing-test.json b/evals/natural/cases/failing-test.json new file mode 100644 index 0000000..120862c --- /dev/null +++ b/evals/natural/cases/failing-test.json @@ -0,0 +1,9 @@ +{ + "id": "failing-test", + "prompt": "Der Test in test_cart.py schlägt fehl und ich verstehe nicht warum. Kannst du dir das ansehen?", + "files": { + "README.md": "# Cart\n\nSmall pricing library.\n", + "cart.py": "def total(items, discount=0):\n subtotal = sum(i['price'] * i['qty'] for i in items)\n return round(subtotal * (1 - discount / 10), 2)\n", + "test_cart.py": "from cart import total\n\ndef test_total_with_discount():\n assert total([{'price': 10.0, 'qty': 2}], discount=10) == 18.0\n" + } +} diff --git a/evals/natural/cases/typo-fix.json b/evals/natural/cases/typo-fix.json new file mode 100644 index 0000000..e98e09c --- /dev/null +++ b/evals/natural/cases/typo-fix.json @@ -0,0 +1,7 @@ +{ + "id": "typo-fix", + "prompt": "In der README ist ein Tippfehler bei accounts. Korrigier den bitte.", + "files": { + "README.md": "# Ledger\n\nInvoicing app with modules billing, acounts and reports.\n" + } +} diff --git a/evals/natural/expected.json b/evals/natural/expected.json new file mode 100644 index 0000000..538f9e3 --- /dev/null +++ b/evals/natural/expected.json @@ -0,0 +1,24 @@ +{ + "failing-test": { + "any": [ + "systematic-debugging" + ], + "prohibited": [ + "speckit-specify", + "openspec-propose" + ] + }, + "cross-module-feature": { + "any": [ + "speckit-specify", + "openspec-propose" + ], + "prohibited": [ + "systematic-debugging" + ] + }, + "typo-fix": { + "any": [], + "prohibited": [] + } +} From 8fd705cccc05ac623f79f6f2a1fd34d0721338b1 Mon Sep 17 00:00:00 2001 From: Peter Hense Date: Fri, 13 Nov 2026 20:53:32 +0100 Subject: [PATCH 2/2] release: prepare 0.3.2 with unscripted routing evaluations Co-Authored-By: Claude Fable 5.2 --- .claude-plugin/plugin.json | 2 +- .codex-plugin/plugin.json | 2 +- BACKLOG.md | 3 + CONTRIBUTING.md | 1 + README.md | 10 +- docs/installation.md | 12 +- docs/opencode.md | 4 +- docs/releases/0.3.2.md | 61 +++++++ method-core.lock.json | 14 +- method.json | 2 +- pyproject.toml | 2 +- scripts/run_host_evals.py | 3 +- scripts/run_natural_evals.py | 313 +++++++++++++++++++++++++++++++++++ tests/test_natural_evals.py | 200 ++++++++++++++++++++++ 14 files changed, 610 insertions(+), 19 deletions(-) create mode 100644 docs/releases/0.3.2.md create mode 100644 scripts/run_natural_evals.py create mode 100644 tests/test_natural_evals.py diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index c4afdc6..0594a11 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "engineering-method", - "version": "0.3.1", + "version": "0.3.2", "description": "Risk-proportionate engineering workflows for Codex, Claude and OpenCode.", "author": { "name": "Peter Hense" diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index 1044061..d545844 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "engineering-method", - "version": "0.3.1", + "version": "0.3.2", "description": "Risk-proportionate engineering workflows for Codex, Claude and OpenCode.", "author": { "name": "Peter Hense" diff --git a/BACKLOG.md b/BACKLOG.md index 8a59eba..6e72d4e 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -160,6 +160,9 @@ Keep stable IDs unchanged. Order groups unblocker-first, then priority and depen - ✅ `EM-021` **P2** Hold every skill to the published skill authoring limits (release 0.3.1) - Notes: Published stable Latest v0.3.1 from bf6acfb; 347 tests, validator, sync-core check (Core 4300085, 70 files) and reproducible 243-file package passed. Downloaded asset matches 63329fe47119d6659766d88dad7044f02779b88ed7c882763aa12553a63a5285. Paid native routing evaluations were not rerun. + +- ⭕ `EM-022` **P2** Unscripted multi-plugin routing evaluations (release 0.3.2) + - Notes: Core 91e35da adds scripts/run_natural_evals.py and closes the host's stdin in run_host_evals. Three cases under evals/natural. Release 0.3.2 in preparation. - ⭕ `EM-019` **P3** Verify a Gemini host adapter in Method Core - Notes: Deferred from EM-018; shared adapter ownership is phense/method-core. No Gemini support claimed until fresh native discovery and behavior checks pass. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 622e9b1..b5b689d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -76,6 +76,7 @@ alone do not prevent sensitive Git history from being published. ## Release policy +`v0.3.2` adds the unscripted evaluation runner from Method Core `91e35da`. `v0.3.1` adds the skill authoring limits test from Method Core `4300085`. `v0.3.0` separates writing into Writing Method and bundles shared support from Method Core, with [split evidence](docs/verification/EM-018-method-split.md). diff --git a/README.md b/README.md index 9607425..33722b4 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,11 @@ local checks first, reuse unaffected evidence, focused review of fixes, and an effort brake before testing or delegation grows beyond the task. Subscription limits and model-effort permissions remain binding across quota resets. -**[Version 0.3.1](https://github.com/phense/engineering-method/releases/tag/v0.3.1) -adds a test that holds every skill to the published skill authoring limits.** -See the [release notes](docs/releases/0.3.1.md). Earlier scopes: +**[Version 0.3.2](https://github.com/phense/engineering-method/releases/tag/v0.3.2) +adds a runner for unscripted routing evaluations and fixes host runs that +timed out without output.** See the [release notes](docs/releases/0.3.2.md). +Earlier scopes: [0.3.1](docs/releases/0.3.1.md) added a test that holds every +skill to the published skill authoring limits, [0.3.0](docs/releases/0.3.0.md) separated writing into its own plugin and bundled shared support from a pinned Method Core, [0.2.1](docs/releases/0.2.1.md) refined local PDF export, [0.2.0](docs/releases/0.2.0.md) @@ -37,7 +39,7 @@ routing evaluation. ## Related methods and shared core -Version 0.3.1 is the current stable release; 0.2.1 was the last release that bundled the writing skills. +Version 0.3.2 is the current stable release; 0.2.1 was the last release that bundled the writing skills. Version 0.3.0 separates writing into [Writing Method](https://github.com/phense/writing-method). [Method Core](https://github.com/phense/method-core) owns the bundled host adapters, state/continuity tools, verification/review/isolation support and generic playbooks. Each repository keeps its own Issues and pull requests; the common topic is `agent-methods`. diff --git a/docs/installation.md b/docs/installation.md index a536076..e1fc5d2 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -8,7 +8,7 @@ installation success does not establish live model behavior on every host. For OpenCode, follow [the source-linked installation](opencode.md), available since 0.1.1. Its native discovery and bounded DeepSeek test were checked with OpenCode 1.18.30; see the [0.1.1 release scope](releases/0.1.1.md) and the -[0.3.1 release notes](releases/0.3.1.md). +[0.3.2 release notes](releases/0.3.2.md). ## Split into three repositories @@ -38,11 +38,11 @@ claude plugin install engineering-method@engineering-method claude plugin list --json ``` -For stable version 0.3.1 on either host, clone the release tag and register the +For stable version 0.3.2 on either host, clone the release tag and register the absolute checkout path instead of the GitHub shorthand: ```sh -git clone --branch v0.3.1 --depth 1 https://github.com/phense/engineering-method.git +git clone --branch v0.3.2 --depth 1 https://github.com/phense/engineering-method.git cd engineering-method em_plugin="$(pwd)" codex plugin marketplace add "$em_plugin" @@ -55,11 +55,11 @@ Choose one source per marketplace name. If switching an existing local marketplace to GitHub, uninstall this plugin and remove its marketplace using the commands below, then register the new source. Preserve your project files. -Alternatively, download `engineering-method-0.3.1.zip` and -`SHA256SUMS` from the [0.3.1 release](https://github.com/phense/engineering-method/releases/tag/v0.3.1). +Alternatively, download `engineering-method-0.3.2.zip` and +`SHA256SUMS` from the [0.3.2 release](https://github.com/phense/engineering-method/releases/tag/v0.3.2). In the download directory, run `shasum -a 256 -c SHA256SUMS`, extract the archive, and register the extracted `engineering-method` directory as above. -The plugin manifests and stable Git tag identify version `0.3.1`. The older +The plugin manifests and stable Git tag identify version `0.3.2`. The older `v0.1.0-rc.1` pre-release remains available for historical reference. Claude also supports a session-only load: diff --git a/docs/opencode.md b/docs/opencode.md index 2a0581a..b91ce63 100644 --- a/docs/opencode.md +++ b/docs/opencode.md @@ -1,6 +1,6 @@ # OpenCode integration -Version 0.3.1 bundles 19 engineering/shared skills. Writing Method +Version 0.3.2 bundles 19 engineering/shared skills. Writing Method bundles its own 18 writing/shared skills and a distinct `writing-method.js` loader. Both use the same adapter from Method Core. Earlier release evidence under `docs/verification/` describes the combined historical packages and is not @@ -16,7 +16,7 @@ macOS. Use version 0.1.1 or newer; the earlier v0.1.0 release predates this integration. Clone a retained checkout and install from it: ```sh -git clone --branch v0.3.1 --depth 1 https://github.com/phense/engineering-method.git +git clone --branch v0.3.2 --depth 1 https://github.com/phense/engineering-method.git cd engineering-method ``` diff --git a/docs/releases/0.3.2.md b/docs/releases/0.3.2.md new file mode 100644 index 0000000..84d9887 --- /dev/null +++ b/docs/releases/0.3.2.md @@ -0,0 +1,61 @@ +# Engineering Method 0.3.2 + +Released 2026-11-13. A maintenance release: unscripted routing evaluations +and a fix for host runs that timed out without output. + +## Changes + +- Syncs [Method Core](https://github.com/phense/method-core) revision `64d9a02` + (72 files). +- Adds `scripts/run_natural_evals.py`. It sends a case prompt exactly as + written, with every given plugin loaded, and reports which skills the host + loaded on its own over a number of runs. The routing harness + (`scripts/run_host_evals.py`) keeps telling the host to select a workflow; + the new runner shows what happens without that instruction. +- Adds three unscripted cases under `evals/natural/`: a failing test, a + feature across three modules and a typing error. +- `run_host_evals.py` no longer lets the host inherit an open standard input. + Codex then waited for more input, and the run timed out without one event. +- No skill name, description or behaviour changes. + +## Verification and limits + +The complete Python suite passes 359 tests on macOS. `scripts/validate-plugin` +and `sync-core --check` pass. The committed-source package is built twice for +reproducibility. + +Unscripted requests, three runs per case with Writing Method loaded beside +this plugin, the hosts' coordinator models, and on Codex the plugins as +installed (Claude Code 2.1.323, Codex CLI 0.169.0): + +| Request | Expected | Claude | Codex | +|---|---|---|---| +| A failing test | `systematic-debugging` | 2 of 3 | 3 of 3 | +| A feature across three modules | a specification skill | 3 of 3 | 3 of 3 | +| A typing error in a README | no skill | 3 of 3 | 2 of 3 | + +Three runs show a tendency, not a rate. The Codex figures are a re-scoring of +saved transcripts after the runner learned to count a skill read inside a +compound command. An earlier single probe in which each request ended with a +request for a plan loaded no specification skill on Claude: asking for a plan +first lowers loading. The paid routing and lifecycle evaluations of the +release gate were not rerun; the evidence limits of [0.3.0](0.3.0.md) remain. + +The downloadable package is `engineering-method-0.3.2.zip`; verify it with the +attached `SHA256SUMS`. + +## Install or update + +```sh +claude plugin marketplace update engineering-method +claude plugin update engineering-method@engineering-method +``` + +```sh +codex plugin remove engineering-method@engineering-method +codex plugin add engineering-method@engineering-method +``` + +Restart the host session afterwards. See +[Installation and maintenance](../installation.md) for a fixed-tag install and +[OpenCode installation](../opencode.md) for the source-linked loader. diff --git a/method-core.lock.json b/method-core.lock.json index 4e248b3..9cb3ea1 100644 --- a/method-core.lock.json +++ b/method-core.lock.json @@ -113,7 +113,12 @@ { "mode": "100644", "path": "scripts/run_host_evals.py", - "sha256": "b9e06b1ae7b562521d57b04b1d530e14b98ac1f3f6484c8e4ca21fe40a0cb161" + "sha256": "a2d814af5c2091f39a486e72825fd42a6ced071efc3dca1ed15e3ff7c97af9b4" + }, + { + "mode": "100644", + "path": "scripts/run_natural_evals.py", + "sha256": "85eac53c236601a6aa22a0c039d1e8a1e8eb9a9c245289676d6db3fef7afc174" }, { "mode": "100755", @@ -325,6 +330,11 @@ "path": "tests/test_models.py", "sha256": "5f2a60c2d30d7248b3bc914ff44066d308e80e6db99d88001ce350dd0e95655f" }, + { + "mode": "100644", + "path": "tests/test_natural_evals.py", + "sha256": "231e8d4698e1adfae7b2a4a35ded6965dced6563b47df2506254cd1e2995fd69" + }, { "mode": "100644", "path": "tests/test_opencode_integration.py", @@ -352,7 +362,7 @@ } ], "repository": "https://github.com/phense/method-core.git", - "revision": "4300085b164686cc359a98cf7b3ee965cbd886b0", + "revision": "64d9a02272590630288c04eba3a76c7bdb05919f", "schema_version": 1, "version": "0.1.0" } diff --git a/method.json b/method.json index 3a9ee8c..f69d6a3 100644 --- a/method.json +++ b/method.json @@ -1,7 +1,7 @@ { "schema_version": 1, "name": "engineering-method", - "version": "0.3.1", + "version": "0.3.2", "method_skills": [ "architecture-modeling", "openspec-apply", diff --git a/pyproject.toml b/pyproject.toml index 26c5501..72cb88e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "engineering-method" -version = "0.3.1" +version = "0.3.2" description = "Risk-proportionate engineering workflows for Codex, Claude and OpenCode." requires-python = ">=3.11" license = { text = "MIT" } diff --git a/scripts/run_host_evals.py b/scripts/run_host_evals.py index a6d613a..2d30d53 100644 --- a/scripts/run_host_evals.py +++ b/scripts/run_host_evals.py @@ -148,7 +148,8 @@ def native_failure_diagnostics(stdout, stderr): def execute(command, cwd, env, timeout): try: - process = subprocess.Popen(command, cwd=cwd, env=env, stdout=subprocess.PIPE, + # A host that inherits an open stdin waits for more input and never starts the turn. + process = subprocess.Popen(command, cwd=cwd, env=env, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, start_new_session=True) except FileNotFoundError as exc: raise EvalFailure("missing_command") from exc diff --git a/scripts/run_natural_evals.py b/scripts/run_natural_evals.py new file mode 100644 index 0000000..4556816 --- /dev/null +++ b/scripts/run_natural_evals.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +"""Unscripted routing evaluations: plain requests, several plugins, no routing instruction. + +The routing harness tells a host to select a workflow. This one sends the case +prompt exactly as written, with every given plugin loaded, and records which +skills the host loaded on its own. Expectations never enter prompts. A case is +repeated `--runs` times and reported as a count, because loading is statistical. + +The host runs as its user runs it: with the adapter's coordinator model, and on +Codex with the plugins as they are installed in the given Codex home. An +installed copy whose skills differ from the source stops the run. +""" +from __future__ import annotations +import argparse +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +from datetime import datetime, timezone + +try: + from scripts import run_host_evals as harness +except ImportError: # executed as a script: the scripts directory is on the path + import run_host_evals as harness + +ROOT = harness.ROOT +EvalFailure = harness.EvalFailure +READ_ONLY_TOOLS = "Read,Glob,Grep,Skill" + + +def bare(name): + """`plugin:skill` and `skill` name the same skill here: several plugins are loaded.""" + return name.rsplit(":", 1)[-1] if isinstance(name, str) else "" + + +def skills_in_output(output, plugins): + """Skills whose complete SKILL.md is in a command's output, whatever the command looked like. + + Hosts read a skill inside compound commands (`pwd; rg ...; cat `). The + whole file content proves the read; a path or a partial read does not. + """ + found = [] + for plugin in plugins: + for path in sorted((plugin / "skills").glob("*/SKILL.md")): + content = path.read_text().rstrip("\n") + if content.strip() and content in output and path.parent.name not in found: + found.append(path.parent.name) + return found + + +def events(text): + found = [] + for line in text.splitlines(): + if not line.strip(): + continue + try: + event = json.loads(line) + except ValueError as exc: + raise EvalFailure("malformed_jsonl", text) from exc + if isinstance(event, dict): + found.append(event) + return found + + +def loaded_skills(host, text, plugins): + """Skills the host loaded through a successful tool call, in first-load order. + + A skill path that only appears inside some file's content is not a load. + """ + order, terminal, pending = [], False, {} + + def add(names): + for name in names: + if name and name not in order: + order.append(name) + + for event in events(text): + kind = event.get("type") + if host == "codex": + if kind in ("error", "turn.failed"): + raise EvalFailure("host_error", text) + terminal |= kind == "turn.completed" + item = event.get("item") + if kind == "item.completed" and isinstance(item, dict) \ + and item.get("type") == "command_execution" and item.get("exit_code") == 0: + command, output = item.get("command"), item.get("aggregated_output", "") + if isinstance(command, str) and isinstance(output, str): + add(skills_in_output(output, plugins)) + continue + if kind == "result": + if event.get("is_error") or event.get("subtype") != "success": + raise EvalFailure("host_error", text) + terminal = True + message = event.get("message") + content = message.get("content") if isinstance(message, dict) else None + for block in content if isinstance(content, list) else []: + if not isinstance(block, dict): + continue + if block.get("type") == "tool_use" and isinstance(block.get("id"), str): + pending[block["id"]] = block + if block.get("type") == "tool_result" and not block.get("is_error") and block.get("content"): + call = pending.get(block.get("tool_use_id"), {}) + arguments = call.get("input") if isinstance(call.get("input"), dict) else {} + if call.get("name") == "Skill": + add([bare(arguments.get("skill"))]) + if call.get("name") == "Read": + add(harness.SKILL_PATH.findall(str(arguments.get("file_path", "")))) + if call.get("name") == "Bash": + output = block["content"] + if isinstance(output, list): + output = "\n".join(str(part.get("text", "")) for part in output if isinstance(part, dict)) + add(skills_in_output(str(output), plugins)) + if not terminal: + raise EvalFailure("incomplete_transcript", text) + return order + + +def judge(loaded, expected): + """A run hits when an expected skill was loaded and no prohibited one was.""" + if set(loaded) & set(expected.get("prohibited", [])): + return False + wanted = expected.get("any", []) + return bool(set(loaded) & set(wanted)) if wanted else not loaded + + +def skills_fingerprint(plugin): + """Content of every skill file of a plugin, independent of where the plugin lies.""" + return harness.fingerprint(plugin / "skills") + + +def plugin_name(plugin): + manifest = plugin / ".codex-plugin/plugin.json" + return json.loads(manifest.read_text())["name"] if manifest.is_file() else plugin.name + + +def installed_codex_plugins(sources, codex_home): + """The installed copy of each source plugin; its skills must equal the source.""" + found = [] + for source in sources: + name, wanted = plugin_name(source), skills_fingerprint(source) + copies = [path for path in sorted((codex_home / "plugins/cache").glob(f"*/{name}/*")) + if (path / "skills").is_dir()] + match = [path for path in copies if skills_fingerprint(path) == wanted] + if not match: + raise EvalFailure(("installed_plugin_differs:" if copies else "plugin_not_installed:") + name) + found.append(match[-1]) + return found + + +def prepare_repo(repo, case): + repo.mkdir() + subprocess.run(["git", "init", "-q", str(repo)], check=True, capture_output=True) + for relative, content in case.get("files", {}).items(): + path = (repo / relative).resolve() + if not path.is_relative_to(repo.resolve()): + raise EvalFailure("fixture_path_escape") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + + +def host_command(host, repo, plugins, config, model, effort, prompt, auth_home=None, native_auth=False, budget=2): + env = os.environ.copy() + for key in ("ANTHROPIC_API_KEY", "OPENAI_API_KEY", "CLAUDECODE", "ANTHROPIC_AUTH_TOKEN"): + env.pop(key, None) + if host == "codex": + # The user's own Codex home: plugins load the way they are installed, not through repository links. + env["CODEX_HOME"] = str(auth_home) + command = ["codex", "exec", "--json", "--ephemeral", + "--sandbox", "read-only", "-C", str(repo), "-m", model] + if effort is not None: + command.extend(["-c", f'model_reasoning_effort="{effort}"']) + else: + if not native_auth: + env["CLAUDE_CONFIG_DIR"] = str(auth_home or config) + settings = config / "settings.json" + settings.write_text(json.dumps({"disableAllHooks": True, "enableAllProjectMcpServers": False})) + command = ["claude", "-p", "--verbose", "--output-format", "stream-json", + "--no-session-persistence", "--setting-sources", "", "--settings", str(settings), + "--strict-mcp-config", "--mcp-config", '{"mcpServers":{}}'] + for plugin in plugins: + command.extend(["--plugin-dir", str(plugin)]) + command.extend(["--model", model, "--allowedTools", READ_ONLY_TOOLS, + "--max-budget-usd", str(budget), "--no-chrome"]) + if effort is not None: + command.extend(["--effort", effort]) + command.append(prompt) + return command, env + + +def run_once(host, case, output, label, timeout, sources, auth_home=None, model=None, native_auth=False): + with tempfile.TemporaryDirectory(prefix="method-natural-eval-") as directory: + temporary = Path(directory) + repo, config = temporary / "repo", temporary / "config" + config.mkdir() + if host == "codex": + staged = installed_codex_plugins(sources, Path(auth_home)) + else: + staged = [harness.stage_plugin(source, temporary / f"plugin-{index}") + for index, source in enumerate(sources)] + prepare_repo(repo, case) + coordinator = harness.model_policy(host, sources[0])["coordinator"] + effort = coordinator["effort"] if model in (None, coordinator["model"]) \ + else harness.model_effort(host, model, sources[0]) + model = model or coordinator["model"] + command, env = host_command(host, repo, staged, config, model, effort, case["prompt"], + auth_home, native_auth) + stdout = "" + try: + stdout = harness.execute(command, repo, env, timeout) + return loaded_skills(host, stdout, staged), model + except EvalFailure as exc: + stdout = exc.transcript or stdout + raise + finally: + harness.private_transcript(output / f"{label}-raw.jsonl", stdout) + + +def run_case(host, case, expected, output, runs, min_hits, timeout, sources, **options): + before = [harness.fingerprint(source) for source in sources] + observations, hits, model = [], 0, None + for index in range(1, runs + 1): + try: + loaded, model = run_once(host, case, output, f'{case["id"]}-{index}', timeout, sources, **options) + hit = judge(loaded, expected) + observations.append({"run": index, "loaded": loaded, "hit": hit}) + hits += hit + except (EvalFailure, OSError, ValueError) as exc: + observations.append({"run": index, "hit": False, + "failure": str(exc) if isinstance(exc, EvalFailure) else type(exc).__name__}) + changed = before != [harness.fingerprint(source) for source in sources] + result = {"case_id": case["id"], "host": host, "model": model, "runs": runs, "hits": hits, + "expected_any": expected.get("any", []), "prohibited": expected.get("prohibited", []), + "observations": observations, + "status": "passed" if hits >= min_hits and not changed else "failed"} + if changed: + result["failure"] = "plugin_changed_during_evaluation" + return result + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--host", required=True, choices=["codex", "claude"]) + parser.add_argument("--case") + parser.add_argument("--runs", type=int, default=1, help="repetitions per case") + parser.add_argument("--min-hits", type=int, help="hits a case needs to pass; default: every run") + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--timeout", type=float, default=420, help="maximum seconds per run") + auth = parser.add_mutually_exclusive_group() + auth.add_argument("--auth-home", type=Path) + auth.add_argument("--native-auth", action="store_true", + help="Claude only: preserve the calling terminal native authentication environment") + parser.add_argument("--model", help="default: the coordinator model of the host adapter") + parser.add_argument("--plugin-root", type=Path, default=ROOT, + help="plugin whose evals/natural cases are run") + parser.add_argument("--with-plugin", type=Path, action="append", default=[], + help="another plugin root to load beside it; repeatable") + args = parser.parse_args(argv) + if args.native_auth and args.host != "claude": + parser.error("--native-auth is supported only for Claude") + if args.host == "codex" and args.auth_home is None: + parser.error("Codex needs --auth-home: the Codex home in which the plugins are installed") + if args.runs < 1: + parser.error("--runs must be at least 1") + min_hits = args.runs if args.min_hits is None else args.min_hits + if not 0 <= min_hits <= args.runs: + parser.error("--min-hits must lie between 0 and --runs") + sources = [args.plugin_root.resolve(), *[path.resolve() for path in args.with_plugin]] + for source in sources: + if not (source / "skills").is_dir(): + parser.error(f"not a plugin root: {source}") + output = args.output_dir.resolve() + if any(output.is_relative_to(source) for source in sources): + parser.error("output must not be written into a plugin source") + natural = sources[0] / "evals/natural" + expected_file = natural / "expected.json" + cases = [json.loads(path.read_text()) for path in sorted((natural / "cases").glob("*.json"))] + if args.case: + cases = [case for case in cases if case["id"] == args.case] + if not cases or not expected_file.is_file(): + parser.error("no natural evaluation cases selected") + expected = json.loads(expected_file.read_text()) + if args.host == "codex": + try: + installed_codex_plugins(sources, args.auth_home) + except EvalFailure as exc: + parser.error(f"{exc}: reinstall the plugin in that Codex home, then run again") + output.mkdir(parents=True, exist_ok=True) + os.chmod(output, 0o700) + results = [] + for case in cases: + result = run_case(args.host, case, expected[case["id"]], output, args.runs, min_hits, args.timeout, + sources, auth_home=args.auth_home, model=args.model, native_auth=args.native_auth) + results.append(result) + (output / f'{case["id"]}.json').write_text(json.dumps(result, indent=2, ensure_ascii=False) + "\n") + print(json.dumps({key: result[key] for key in ("case_id", "status", "hits", "runs")} + | {"loaded": [o.get("loaded", o.get("failure")) for o in result["observations"]]}, + ensure_ascii=False), flush=True) + commit = subprocess.run(["git", "rev-parse", "HEAD"], cwd=sources[0], capture_output=True, text=True) + summary = {"status": "passed" if all(r["status"] == "passed" for r in results) else "failed", + "host": args.host, "native_auth": args.native_auth, "runs": args.runs, "min_hits": min_hits, + "source_commit": commit.stdout.strip(), + "plugins": [{"root": source.name, "sha256": harness.fingerprint(source)} for source in sources], + "completed_at": datetime.now(timezone.utc).isoformat(), + "hits": sum(r["hits"] for r in results), "total_runs": args.runs * len(results), "results": results} + (output / "summary.json").write_text(json.dumps(summary, indent=2, ensure_ascii=False) + "\n") + return 0 if summary["status"] == "passed" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_natural_evals.py b/tests/test_natural_evals.py new file mode 100644 index 0000000..a016125 --- /dev/null +++ b/tests/test_natural_evals.py @@ -0,0 +1,200 @@ +"""Unscripted routing evaluations count loads, not mentions, and never steer the host.""" + +from __future__ import annotations + +import io +import json +import subprocess +import tempfile +import unittest +from contextlib import redirect_stderr +from pathlib import Path +from unittest import mock + +from scripts import run_host_evals as harness +from scripts import run_natural_evals as natural + + +def lines(*events): + return "\n".join(json.dumps(event) for event in events) + + +def claude_call(identity, name, arguments, content="ok", error=False): + return [ + {"type": "assistant", "message": {"content": [ + {"type": "tool_use", "id": identity, "name": name, "input": arguments}]}}, + {"type": "user", "message": {"content": [ + {"type": "tool_result", "tool_use_id": identity, "content": content, "is_error": error}]}}, + ] + + +DONE = {"type": "result", "subtype": "success", "is_error": False, "result": "done"} + + +class LoadedSkillTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.plugin = Path(self.directory.name) / "plugin" + skill = self.plugin / "skills/alpha-skill" + skill.mkdir(parents=True) + self.content = "---\nname: alpha-skill\ndescription: Use when testing.\n---\n\n# Alpha\n\nBody.\n" + (skill / "SKILL.md").write_text(self.content) + + def tearDown(self): + self.directory.cleanup() + + def test_claude_counts_successful_skill_calls_and_skill_file_reads(self): + text = lines( + *claude_call("a", "Skill", {"skill": "writing-method:story-drafting"}), + *claude_call("b", "Read", {"file_path": "/x/skills/alpha-skill/SKILL.md"}), + *claude_call("c", "Skill", {"skill": "writing-method:story-drafting"}), + DONE) + self.assertEqual(["story-drafting", "alpha-skill"], natural.loaded_skills("claude", text, [self.plugin])) + + def test_claude_ignores_failed_calls_and_paths_inside_file_content(self): + text = lines( + *claude_call("a", "Skill", {"skill": "beta-skill"}, error=True), + *claude_call("b", "Read", {"file_path": "/x/policy.md"}, + content="see skills/alpha-skill/SKILL.md for details"), + {"type": "assistant", "message": {"content": [ + {"type": "text", "text": "I would use skills/alpha-skill/SKILL.md"}]}}, + DONE) + self.assertEqual([], natural.loaded_skills("claude", text, [self.plugin])) + + def test_codex_counts_a_complete_read_only(self): + def command(text, output, code=0): + return {"type": "item.completed", "item": {"type": "command_execution", "command": text, + "aggregated_output": output, "exit_code": code}} + path = ".agents/skills/alpha-skill/SKILL.md" + complete = lines(command(f"/bin/zsh -lc 'cat {path}'", self.content), {"type": "turn.completed"}) + self.assertEqual(["alpha-skill"], natural.loaded_skills("codex", complete, [self.plugin])) + compound = lines(command(f"/bin/zsh -lc 'pwd; rg --files -g AGENTS.md; cat {path}'", "/repo\n" + self.content), + {"type": "turn.completed"}) + self.assertEqual(["alpha-skill"], natural.loaded_skills("codex", compound, [self.plugin])) + listing = lines(command("/bin/zsh -lc 'rg --files .agents/skills'", path + "\n"), + command(f"/bin/zsh -lc 'cat {path}'", self.content, code=1), + command(f"/bin/zsh -lc 'head -2 {path}'", "---\nname: alpha-skill\n"), + {"type": "turn.completed"}) + self.assertEqual([], natural.loaded_skills("codex", listing, [self.plugin])) + + def test_unfinished_or_failed_transcripts_are_failures(self): + with self.assertRaisesRegex(harness.EvalFailure, "incomplete_transcript"): + natural.loaded_skills("claude", lines(*claude_call("a", "Skill", {"skill": "x"})), [self.plugin]) + with self.assertRaisesRegex(harness.EvalFailure, "host_error"): + natural.loaded_skills("codex", lines({"type": "turn.failed"}), [self.plugin]) + with self.assertRaisesRegex(harness.EvalFailure, "malformed_jsonl"): + natural.loaded_skills("claude", "{not json", [self.plugin]) + + def test_claude_counts_a_complete_shell_read(self): + path = str(self.plugin / "skills/alpha-skill/SKILL.md") + text = lines(*claude_call("a", "Bash", {"command": f"cat {path}"}, content=self.content), DONE) + self.assertEqual(["alpha-skill"], natural.loaded_skills("claude", text, [self.plugin])) + + def test_judgement(self): + expected = {"any": ["a", "b"], "prohibited": ["z"]} + self.assertTrue(natural.judge(["b", "other"], expected)) + self.assertFalse(natural.judge(["other"], expected)) + self.assertFalse(natural.judge(["a", "z"], expected)) + # A case that expects no skill hits only when nothing was loaded. + self.assertTrue(natural.judge([], {"any": []})) + self.assertFalse(natural.judge(["a"], {"any": []})) + + +class CommandTests(unittest.TestCase): + def test_prompt_is_sent_unchanged_with_every_plugin_and_read_only_tools(self): + with tempfile.TemporaryDirectory() as directory: + config = Path(directory) + plugins = [Path("/p/one"), Path("/p/two")] + command, env = natural.host_command("claude", Path("/repo"), plugins, config, "sonnet", "medium", + "Schreib bitte Kapitel 12.", native_auth=True) + self.assertEqual("Schreib bitte Kapitel 12.", command[-1]) + self.assertEqual(["/p/one", "/p/two"], + [command[i + 1] for i, word in enumerate(command) if word == "--plugin-dir"]) + self.assertEqual(natural.READ_ONLY_TOOLS, command[command.index("--allowedTools") + 1]) + self.assertNotIn("--json-schema", command) + self.assertNotIn("ANTHROPIC_API_KEY", env) + command, env = natural.host_command("codex", Path("/repo"), plugins, config, "m", None, + "Plain request.", auth_home=Path("/auth")) + self.assertEqual("Plain request.", command[-1]) + self.assertEqual("read-only", command[command.index("--sandbox") + 1]) + self.assertNotIn("--output-schema", command) + # The plugins load as installed, so the user's configuration stays in effect. + self.assertNotIn("--ignore-user-config", command) + self.assertEqual("/auth", env["CODEX_HOME"]) + for instruction in (harness.ROUTING_INSTRUCTION, harness.SKILL_DISCOVERY_INSTRUCTION): + self.assertNotIn(instruction.strip()[:40], " ".join(command)) + + def test_repository_holds_the_fixture_and_no_skill_links(self): + with tempfile.TemporaryDirectory() as directory: + base = Path(directory) + natural.prepare_repo(base / "repo", {"files": {"docs/a.md": "x"}}) + self.assertEqual("x", (base / "repo/docs/a.md").read_text()) + self.assertFalse((base / "repo/.agents").exists()) + with self.assertRaisesRegex(harness.EvalFailure, "fixture_path_escape"): + natural.prepare_repo(base / "repo3", {"files": {"../out.md": "x"}}) + + def test_codex_uses_the_installed_copy_only_when_its_skills_equal_the_source(self): + with tempfile.TemporaryDirectory() as directory: + base = Path(directory) + source, home = base / "source", base / "home" + (source / "skills/alpha").mkdir(parents=True) + (source / "skills/alpha/SKILL.md").write_text("current") + (source / ".codex-plugin").mkdir() + (source / ".codex-plugin/plugin.json").write_text(json.dumps({"name": "demo"})) + with self.assertRaisesRegex(harness.EvalFailure, "plugin_not_installed:demo"): + natural.installed_codex_plugins([source], home) + installed = home / "plugins/cache/market/demo/1.0.0" + (installed / "skills/alpha").mkdir(parents=True) + (installed / "skills/alpha/SKILL.md").write_text("older") + with self.assertRaisesRegex(harness.EvalFailure, "installed_plugin_differs:demo"): + natural.installed_codex_plugins([source], home) + (installed / "skills/alpha/SKILL.md").write_text("current") + self.assertEqual([installed], natural.installed_codex_plugins([source], home)) + + +class RunTests(unittest.TestCase): + def test_a_case_reports_hits_over_runs_and_never_stops_early(self): + observed = iter([(["a"], "m"), harness.EvalFailure("timeout"), (["other"], "m")]) + + def once(*args, **kwargs): + value = next(observed) + if isinstance(value, Exception): + raise value + return value + + with tempfile.TemporaryDirectory() as directory, \ + mock.patch.object(natural, "run_once", side_effect=once), \ + mock.patch.object(harness, "fingerprint", return_value="same"): + result = natural.run_case("claude", {"id": "c"}, {"any": ["a"]}, Path(directory), 3, 1, 1.0, + [Path(directory)]) + self.assertEqual((3, 1, "passed"), (result["runs"], result["hits"], result["status"])) + self.assertEqual([True, False, False], [o["hit"] for o in result["observations"]]) + self.assertEqual("timeout", result["observations"][1]["failure"]) + + def test_arguments_are_checked_before_any_host_runs(self): + with tempfile.TemporaryDirectory() as directory: + plugin = Path(directory) / "plugin" + (plugin / "skills").mkdir(parents=True) + base = ["--host", "claude", "--plugin-root", str(plugin)] + for extra in (["--output-dir", str(plugin / "out")], + ["--output-dir", str(Path(directory) / "out")], + ["--output-dir", str(Path(directory) / "out"), "--runs", "0"], + ["--output-dir", str(Path(directory) / "out"), "--runs", "2", "--min-hits", "3"], + ["--output-dir", str(Path(directory) / "out"), "--with-plugin", directory]): + with self.subTest(extra=extra), redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + natural.main(base + extra) + with redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + natural.main(["--host", "codex", "--native-auth", "--output-dir", str(Path(directory) / "o"), + "--plugin-root", str(plugin)]) + + +class HostProcessTests(unittest.TestCase): + def test_host_process_never_inherits_standard_input(self): + # An inherited open stdin makes a host wait for more input: the run then times out with no event. + output = harness.execute(["python3", "-c", "import sys; print(repr(sys.stdin.read()))"], + Path.cwd(), dict(__import__("os").environ), 20) + self.assertEqual("''", output.strip()) + + +if __name__ == "__main__": + unittest.main()