feat(evals): unscripted multi-plugin routing runner; close the host's stdin

README.md

@@ -61,4 +61,19 @@ python3 -m unittest discover -s tests -t . -v
 python3 scripts/validate-plugin
 ```
 
+## Host evaluations
+
+Two runners are exported to consumers. Both call a real host and cost usage; neither is part of the test suite.
+
+| Runner | What it sends | What it records |
+|---|---|---|
+| `scripts/run_host_evals.py` | A case prompt plus an instruction to select a workflow and write `decision.md`, with one plugin | The selected primary and supporting skills against `evals/<host>/expected.json` |
+| `scripts/run_natural_evals.py` | The case prompt exactly as written, with read-only tools and every plugin given by `--plugin-root` and `--with-plugin` | Which skills the host loaded on its own, over `--runs` repetitions, against `evals/natural/expected.json` |
+
+A natural case is a file `evals/natural/cases/<id>.json` with `id`, `prompt` and `files`. `evals/natural/expected.json` maps each ID to `any` (a run hits when one of these skills was loaded) and `prohibited` (a run misses when one of these was). A load is a successful skill call or a complete read of a `SKILL.md`; a skill path that only appears inside another file does not count. Write the prompt in a user's words, not in the description's vocabulary.
+
+```sh
+python3 scripts/run_natural_evals.py --host claude --native-auth --runs 3 --with-plugin ../other-method --output-dir /tmp/natural
+```
+
 Tests exercise real local Git/file operations, rollback after an injected write failure, existing state/continuity contracts, archive identity and OpenCode loader coexistence. They do not establish live model routing or provider availability. License/source metadata is retained in [third-party notices](THIRD_PARTY_NOTICES.md); original code uses the [MIT License](LICENSE).

method-core.json

@@ -26,6 +26,7 @@
     "scripts/project-state",
     "scripts/refresh-issue-cache",
     "scripts/run_host_evals.py",
+    "scripts/run_natural_evals.py",
     "scripts/sync-core",
     "scripts/validate-plugin",
     "scripts/validate_plugin.py",
@@ -68,6 +69,7 @@
     "tests/test_metadata.py",
     "tests/test_method_identity.py",
     "tests/test_models.py",
+    "tests/test_natural_evals.py",
     "tests/test_opencode_integration.py",
     "tests/test_packaging.py",
     "tests/test_platform_adapters.py",

scripts/run_host_evals.py

@@ -148,7 +148,8 @@ def native_failure_diagnostics(stdout, stderr):
 
 def execute(command, cwd, env, timeout):
     try:
-        process = subprocess.Popen(command, cwd=cwd, env=env, stdout=subprocess.PIPE,
+        # A host that inherits an open stdin waits for more input and never starts the turn.
+        process = subprocess.Popen(command, cwd=cwd, env=env, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE, text=True, start_new_session=True)
     except FileNotFoundError as exc:
         raise EvalFailure("missing_command") from exc

scripts/run_natural_evals.py

@@ -0,0 +1,262 @@
+#!/usr/bin/env python3
+"""Unscripted routing evaluations: plain requests, several plugins, no routing instruction.
+
+The routing harness tells a host to select a workflow. This one sends the case
+prompt exactly as written, with every given plugin loaded, and records which
+skills the host loaded on its own. Expectations never enter prompts. A case is
+repeated `--runs` times and reported as a count, because loading is statistical.
+"""
+from __future__ import annotations
+import argparse
+import hashlib
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+from datetime import datetime, timezone
+
+try:
+    from scripts import run_host_evals as harness
+except ImportError:  # executed as a script: the scripts directory is on the path
+    import run_host_evals as harness
+
+ROOT = harness.ROOT
+EvalFailure = harness.EvalFailure
+READ_ONLY_TOOLS = "Read,Glob,Grep,Skill"
+
+
+def bare(name):
+    """`plugin:skill` and `skill` name the same skill here: several plugins are loaded."""
+    return name.rsplit(":", 1)[-1] if isinstance(name, str) else ""
+
+
+def events(text):
+    found = []
+    for line in text.splitlines():
+        if not line.strip():
+            continue
+        try:
+            event = json.loads(line)
+        except ValueError as exc:
+            raise EvalFailure("malformed_jsonl", text) from exc
+        if isinstance(event, dict):
+            found.append(event)
+    return found
+
+
+def loaded_skills(host, text, plugins):
+    """Skills the host loaded through a successful tool call, in first-load order.
+
+    A skill path that only appears inside some file's content is not a load.
+    """
+    order, terminal, pending = [], False, {}
+
+    def add(names):
+        for name in names:
+            if name and name not in order:
+                order.append(name)
+
+    for event in events(text):
+        kind = event.get("type")
+        if host == "codex":
+            if kind in ("error", "turn.failed"):
+                raise EvalFailure("host_error", text)
+            terminal |= kind == "turn.completed"
+            item = event.get("item")
+            if kind == "item.completed" and isinstance(item, dict) \
+                    and item.get("type") == "command_execution" and item.get("exit_code") == 0:
+                command, output = item.get("command"), item.get("aggregated_output", "")
+                if isinstance(command, str) and isinstance(output, str):
+                    for plugin in plugins:
+                        add(sorted(harness.observed_read_skills(command, output, plugin)))
+            continue
+        if kind == "result":
+            if event.get("is_error") or event.get("subtype") != "success":
+                raise EvalFailure("host_error", text)
+            terminal = True
+        message = event.get("message")
+        content = message.get("content") if isinstance(message, dict) else None
+        for block in content if isinstance(content, list) else []:
+            if not isinstance(block, dict):
+                continue
+            if block.get("type") == "tool_use" and isinstance(block.get("id"), str):
+                pending[block["id"]] = block
+            if block.get("type") == "tool_result" and not block.get("is_error") and block.get("content"):
+                call = pending.get(block.get("tool_use_id"), {})
+                arguments = call.get("input") if isinstance(call.get("input"), dict) else {}
+                if call.get("name") == "Skill":
+                    add([bare(arguments.get("skill"))])
+                if call.get("name") == "Read":
+                    add(harness.SKILL_PATH.findall(str(arguments.get("file_path", ""))))
+    if not terminal:
+        raise EvalFailure("incomplete_transcript", text)
+    return order
+
+
+def judge(loaded, expected):
+    """A run hits when an expected skill was loaded and no prohibited one was."""
+    if set(loaded) & set(expected.get("prohibited", [])):
+        return False
+    wanted = expected.get("any", [])
+    return bool(set(loaded) & set(wanted)) if wanted else not loaded
+
+
+def prepare_repo(repo, case, plugins, host):
+    repo.mkdir()
+    subprocess.run(["git", "init", "-q", str(repo)], check=True, capture_output=True)
+    for relative, content in case.get("files", {}).items():
+        path = (repo / relative).resolve()
+        if not path.is_relative_to(repo.resolve()):
+            raise EvalFailure("fixture_path_escape")
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(content)
+    if host == "codex":
+        # Codex discovers repository skills here; a name two plugins share is linked once.
+        skills = repo / ".agents/skills"
+        skills.mkdir(parents=True)
+        for plugin in plugins:
+            for path in sorted((plugin / "skills").iterdir()):
+                if path.is_dir() and not (skills / path.name).exists():
+                    (skills / path.name).symlink_to(path.resolve(), target_is_directory=True)
+
+
+def host_command(host, repo, plugins, config, model, effort, prompt, auth_home=None, native_auth=False, budget=2):
+    env = os.environ.copy()
+    for key in ("ANTHROPIC_API_KEY", "OPENAI_API_KEY", "CLAUDECODE", "ANTHROPIC_AUTH_TOKEN"):
+        env.pop(key, None)
+    if host == "codex":
+        env["CODEX_HOME"] = str(auth_home or config)
+        command = ["codex", "exec", "--json", "--ephemeral", "--ignore-user-config",
+                   "--sandbox", "read-only", "-C", str(repo), "-m", model]
+        if effort is not None:
+            command.extend(["-c", f'model_reasoning_effort="{effort}"'])
+    else:
+        if not native_auth:
+            env["CLAUDE_CONFIG_DIR"] = str(auth_home or config)
+        settings = config / "settings.json"
+        settings.write_text(json.dumps({"disableAllHooks": True, "enableAllProjectMcpServers": False}))
+        command = ["claude", "-p", "--verbose", "--output-format", "stream-json",
+                   "--no-session-persistence", "--setting-sources", "", "--settings", str(settings),
+                   "--strict-mcp-config", "--mcp-config", '{"mcpServers":{}}']
+        for plugin in plugins:
+            command.extend(["--plugin-dir", str(plugin)])
+        command.extend(["--model", model, "--allowedTools", READ_ONLY_TOOLS,
+                        "--max-budget-usd", str(budget), "--no-chrome"])
+        if effort is not None:
+            command.extend(["--effort", effort])
+    command.append(prompt)
+    return command, env
+
+
+def run_once(host, case, output, label, timeout, sources, auth_home=None, model=None, native_auth=False):
+    with tempfile.TemporaryDirectory(prefix="method-natural-eval-") as directory:
+        temporary = Path(directory)
+        repo, config = temporary / "repo", temporary / "config"
+        config.mkdir()
+        staged = [harness.stage_plugin(source, temporary / f"plugin-{index}")
+                  for index, source in enumerate(sources)]
+        prepare_repo(repo, case, staged, host)
+        policy = harness.model_policy(host, sources[0])
+        model = model or policy["roles"][policy["evaluation_roles"]["routing"]][0]["model"]
+        effort = harness.model_effort(host, model, sources[0])
+        command, env = host_command(host, repo, staged, config, model, effort, case["prompt"],
+                                    auth_home, native_auth)
+        stdout = ""
+        try:
+            stdout = harness.execute(command, repo, env, timeout)
+            return loaded_skills(host, stdout, staged), model
+        except EvalFailure as exc:
+            stdout = exc.transcript or stdout
+            raise
+        finally:
+            harness.private_transcript(output / f"{label}-raw.jsonl", stdout)
+
+
+def run_case(host, case, expected, output, runs, min_hits, timeout, sources, **options):
+    before = [harness.fingerprint(source) for source in sources]
+    observations, hits, model = [], 0, None
+    for index in range(1, runs + 1):
+        try:
+            loaded, model = run_once(host, case, output, f'{case["id"]}-{index}', timeout, sources, **options)
+            hit = judge(loaded, expected)
+            observations.append({"run": index, "loaded": loaded, "hit": hit})
+            hits += hit
+        except (EvalFailure, OSError, ValueError) as exc:
+            observations.append({"run": index, "hit": False,
+                                 "failure": str(exc) if isinstance(exc, EvalFailure) else type(exc).__name__})
+    changed = before != [harness.fingerprint(source) for source in sources]
+    result = {"case_id": case["id"], "host": host, "model": model, "runs": runs, "hits": hits,
+              "expected_any": expected.get("any", []), "prohibited": expected.get("prohibited", []),
+              "observations": observations,
+              "status": "passed" if hits >= min_hits and not changed else "failed"}
+    if changed:
+        result["failure"] = "plugin_changed_during_evaluation"
+    return result
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--host", required=True, choices=["codex", "claude"])
+    parser.add_argument("--case")
+    parser.add_argument("--runs", type=int, default=1, help="repetitions per case")
+    parser.add_argument("--min-hits", type=int, help="hits a case needs to pass; default: every run")
+    parser.add_argument("--output-dir", type=Path, required=True)
+    parser.add_argument("--timeout", type=float, default=420, help="maximum seconds per run")
+    auth = parser.add_mutually_exclusive_group()
+    auth.add_argument("--auth-home", type=Path)
+    auth.add_argument("--native-auth", action="store_true",
+                      help="Claude only: preserve the calling terminal native authentication environment")
+    parser.add_argument("--model")
+    parser.add_argument("--plugin-root", type=Path, default=ROOT,
+                        help="plugin whose evals/natural cases are run")
+    parser.add_argument("--with-plugin", type=Path, action="append", default=[],
+                        help="another plugin root to load beside it; repeatable")
+    args = parser.parse_args(argv)
+    if args.native_auth and args.host != "claude":
+        parser.error("--native-auth is supported only for Claude")
+    if args.runs < 1:
+        parser.error("--runs must be at least 1")
+    min_hits = args.runs if args.min_hits is None else args.min_hits
+    if not 0 <= min_hits <= args.runs:
+        parser.error("--min-hits must lie between 0 and --runs")
+    sources = [args.plugin_root.resolve(), *[path.resolve() for path in args.with_plugin]]
+    for source in sources:
+        if not (source / "skills").is_dir():
+            parser.error(f"not a plugin root: {source}")
+    output = args.output_dir.resolve()
+    if any(output.is_relative_to(source) for source in sources):
+        parser.error("output must not be written into a plugin source")
+    natural = sources[0] / "evals/natural"
+    expected_file = natural / "expected.json"
+    cases = [json.loads(path.read_text()) for path in sorted((natural / "cases").glob("*.json"))]
+    if args.case:
+        cases = [case for case in cases if case["id"] == args.case]
+    if not cases or not expected_file.is_file():
+        parser.error("no natural evaluation cases selected")
+    expected = json.loads(expected_file.read_text())
+    output.mkdir(parents=True, exist_ok=True)
+    os.chmod(output, 0o700)
+    results = []
+    for case in cases:
+        result = run_case(args.host, case, expected[case["id"]], output, args.runs, min_hits, args.timeout,
+                          sources, auth_home=args.auth_home, model=args.model, native_auth=args.native_auth)
+        results.append(result)
+        (output / f'{case["id"]}.json').write_text(json.dumps(result, indent=2, ensure_ascii=False) + "\n")
+        print(json.dumps({key: result[key] for key in ("case_id", "status", "hits", "runs")}
+                         | {"loaded": [o.get("loaded", o.get("failure")) for o in result["observations"]]},
+                         ensure_ascii=False), flush=True)
+    commit = subprocess.run(["git", "rev-parse", "HEAD"], cwd=sources[0], capture_output=True, text=True)
+    summary = {"status": "passed" if all(r["status"] == "passed" for r in results) else "failed",
+               "host": args.host, "native_auth": args.native_auth, "runs": args.runs, "min_hits": min_hits,
+               "source_commit": commit.stdout.strip(),
+               "plugins": [{"root": source.name, "sha256": harness.fingerprint(source)} for source in sources],
+               "completed_at": datetime.now(timezone.utc).isoformat(),
+               "hits": sum(r["hits"] for r in results), "total_runs": args.runs * len(results), "results": results}
+    (output / "summary.json").write_text(json.dumps(summary, indent=2, ensure_ascii=False) + "\n")
+    return 0 if summary["status"] == "passed" else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())