diff --git a/README.md b/README.md index 4d8650f..b966a8d 100644 --- a/README.md +++ b/README.md @@ -61,4 +61,19 @@ python3 -m unittest discover -s tests -t . -v python3 scripts/validate-plugin ``` +## Host evaluations + +Two runners are exported to consumers. Both call a real host and cost usage; neither is part of the test suite. + +| Runner | What it sends | What it records | +|---|---|---| +| `scripts/run_host_evals.py` | A case prompt plus an instruction to select a workflow and write `decision.md`, with one plugin | The selected primary and supporting skills against `evals//expected.json` | +| `scripts/run_natural_evals.py` | The case prompt exactly as written, with read-only tools and every plugin given by `--plugin-root` and `--with-plugin` | Which skills the host loaded on its own, over `--runs` repetitions, against `evals/natural/expected.json` | + +A natural case is a file `evals/natural/cases/.json` with `id`, `prompt` and `files`. `evals/natural/expected.json` maps each ID to `any` (a run hits when one of these skills was loaded) and `prohibited` (a run misses when one of these was). A load is a successful skill call or a complete read of a `SKILL.md`; a skill path that only appears inside another file does not count. Write the prompt in a user's words, not in the description's vocabulary. + +```sh +python3 scripts/run_natural_evals.py --host claude --native-auth --runs 3 --with-plugin ../other-method --output-dir /tmp/natural +``` + Tests exercise real local Git/file operations, rollback after an injected write failure, existing state/continuity contracts, archive identity and OpenCode loader coexistence. They do not establish live model routing or provider availability. License/source metadata is retained in [third-party notices](THIRD_PARTY_NOTICES.md); original code uses the [MIT License](LICENSE). diff --git a/method-core.json b/method-core.json index affa138..9e0482d 100644 --- a/method-core.json +++ b/method-core.json @@ -26,6 +26,7 @@ "scripts/project-state", "scripts/refresh-issue-cache", "scripts/run_host_evals.py", + "scripts/run_natural_evals.py", "scripts/sync-core", "scripts/validate-plugin", "scripts/validate_plugin.py", @@ -68,6 +69,7 @@ "tests/test_metadata.py", "tests/test_method_identity.py", "tests/test_models.py", + "tests/test_natural_evals.py", "tests/test_opencode_integration.py", "tests/test_packaging.py", "tests/test_platform_adapters.py", diff --git a/scripts/run_host_evals.py b/scripts/run_host_evals.py index a6d613a..2d30d53 100644 --- a/scripts/run_host_evals.py +++ b/scripts/run_host_evals.py @@ -148,7 +148,8 @@ def native_failure_diagnostics(stdout, stderr): def execute(command, cwd, env, timeout): try: - process = subprocess.Popen(command, cwd=cwd, env=env, stdout=subprocess.PIPE, + # A host that inherits an open stdin waits for more input and never starts the turn. + process = subprocess.Popen(command, cwd=cwd, env=env, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, start_new_session=True) except FileNotFoundError as exc: raise EvalFailure("missing_command") from exc diff --git a/scripts/run_natural_evals.py b/scripts/run_natural_evals.py new file mode 100644 index 0000000..f2e1393 --- /dev/null +++ b/scripts/run_natural_evals.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 +"""Unscripted routing evaluations: plain requests, several plugins, no routing instruction. + +The routing harness tells a host to select a workflow. This one sends the case +prompt exactly as written, with every given plugin loaded, and records which +skills the host loaded on its own. Expectations never enter prompts. A case is +repeated `--runs` times and reported as a count, because loading is statistical. +""" +from __future__ import annotations +import argparse +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +from datetime import datetime, timezone + +try: + from scripts import run_host_evals as harness +except ImportError: # executed as a script: the scripts directory is on the path + import run_host_evals as harness + +ROOT = harness.ROOT +EvalFailure = harness.EvalFailure +READ_ONLY_TOOLS = "Read,Glob,Grep,Skill" + + +def bare(name): + """`plugin:skill` and `skill` name the same skill here: several plugins are loaded.""" + return name.rsplit(":", 1)[-1] if isinstance(name, str) else "" + + +def events(text): + found = [] + for line in text.splitlines(): + if not line.strip(): + continue + try: + event = json.loads(line) + except ValueError as exc: + raise EvalFailure("malformed_jsonl", text) from exc + if isinstance(event, dict): + found.append(event) + return found + + +def loaded_skills(host, text, plugins): + """Skills the host loaded through a successful tool call, in first-load order. + + A skill path that only appears inside some file's content is not a load. + """ + order, terminal, pending = [], False, {} + + def add(names): + for name in names: + if name and name not in order: + order.append(name) + + for event in events(text): + kind = event.get("type") + if host == "codex": + if kind in ("error", "turn.failed"): + raise EvalFailure("host_error", text) + terminal |= kind == "turn.completed" + item = event.get("item") + if kind == "item.completed" and isinstance(item, dict) \ + and item.get("type") == "command_execution" and item.get("exit_code") == 0: + command, output = item.get("command"), item.get("aggregated_output", "") + if isinstance(command, str) and isinstance(output, str): + for plugin in plugins: + add(sorted(harness.observed_read_skills(command, output, plugin))) + continue + if kind == "result": + if event.get("is_error") or event.get("subtype") != "success": + raise EvalFailure("host_error", text) + terminal = True + message = event.get("message") + content = message.get("content") if isinstance(message, dict) else None + for block in content if isinstance(content, list) else []: + if not isinstance(block, dict): + continue + if block.get("type") == "tool_use" and isinstance(block.get("id"), str): + pending[block["id"]] = block + if block.get("type") == "tool_result" and not block.get("is_error") and block.get("content"): + call = pending.get(block.get("tool_use_id"), {}) + arguments = call.get("input") if isinstance(call.get("input"), dict) else {} + if call.get("name") == "Skill": + add([bare(arguments.get("skill"))]) + if call.get("name") == "Read": + add(harness.SKILL_PATH.findall(str(arguments.get("file_path", "")))) + if not terminal: + raise EvalFailure("incomplete_transcript", text) + return order + + +def judge(loaded, expected): + """A run hits when an expected skill was loaded and no prohibited one was.""" + if set(loaded) & set(expected.get("prohibited", [])): + return False + wanted = expected.get("any", []) + return bool(set(loaded) & set(wanted)) if wanted else not loaded + + +def prepare_repo(repo, case, plugins, host): + repo.mkdir() + subprocess.run(["git", "init", "-q", str(repo)], check=True, capture_output=True) + for relative, content in case.get("files", {}).items(): + path = (repo / relative).resolve() + if not path.is_relative_to(repo.resolve()): + raise EvalFailure("fixture_path_escape") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + if host == "codex": + # Codex discovers repository skills here; a name two plugins share is linked once. + skills = repo / ".agents/skills" + skills.mkdir(parents=True) + for plugin in plugins: + for path in sorted((plugin / "skills").iterdir()): + if path.is_dir() and not (skills / path.name).exists(): + (skills / path.name).symlink_to(path.resolve(), target_is_directory=True) + + +def host_command(host, repo, plugins, config, model, effort, prompt, auth_home=None, native_auth=False, budget=2): + env = os.environ.copy() + for key in ("ANTHROPIC_API_KEY", "OPENAI_API_KEY", "CLAUDECODE", "ANTHROPIC_AUTH_TOKEN"): + env.pop(key, None) + if host == "codex": + env["CODEX_HOME"] = str(auth_home or config) + command = ["codex", "exec", "--json", "--ephemeral", "--ignore-user-config", + "--sandbox", "read-only", "-C", str(repo), "-m", model] + if effort is not None: + command.extend(["-c", f'model_reasoning_effort="{effort}"']) + else: + if not native_auth: + env["CLAUDE_CONFIG_DIR"] = str(auth_home or config) + settings = config / "settings.json" + settings.write_text(json.dumps({"disableAllHooks": True, "enableAllProjectMcpServers": False})) + command = ["claude", "-p", "--verbose", "--output-format", "stream-json", + "--no-session-persistence", "--setting-sources", "", "--settings", str(settings), + "--strict-mcp-config", "--mcp-config", '{"mcpServers":{}}'] + for plugin in plugins: + command.extend(["--plugin-dir", str(plugin)]) + command.extend(["--model", model, "--allowedTools", READ_ONLY_TOOLS, + "--max-budget-usd", str(budget), "--no-chrome"]) + if effort is not None: + command.extend(["--effort", effort]) + command.append(prompt) + return command, env + + +def run_once(host, case, output, label, timeout, sources, auth_home=None, model=None, native_auth=False): + with tempfile.TemporaryDirectory(prefix="method-natural-eval-") as directory: + temporary = Path(directory) + repo, config = temporary / "repo", temporary / "config" + config.mkdir() + staged = [harness.stage_plugin(source, temporary / f"plugin-{index}") + for index, source in enumerate(sources)] + prepare_repo(repo, case, staged, host) + policy = harness.model_policy(host, sources[0]) + model = model or policy["roles"][policy["evaluation_roles"]["routing"]][0]["model"] + effort = harness.model_effort(host, model, sources[0]) + command, env = host_command(host, repo, staged, config, model, effort, case["prompt"], + auth_home, native_auth) + stdout = "" + try: + stdout = harness.execute(command, repo, env, timeout) + return loaded_skills(host, stdout, staged), model + except EvalFailure as exc: + stdout = exc.transcript or stdout + raise + finally: + harness.private_transcript(output / f"{label}-raw.jsonl", stdout) + + +def run_case(host, case, expected, output, runs, min_hits, timeout, sources, **options): + before = [harness.fingerprint(source) for source in sources] + observations, hits, model = [], 0, None + for index in range(1, runs + 1): + try: + loaded, model = run_once(host, case, output, f'{case["id"]}-{index}', timeout, sources, **options) + hit = judge(loaded, expected) + observations.append({"run": index, "loaded": loaded, "hit": hit}) + hits += hit + except (EvalFailure, OSError, ValueError) as exc: + observations.append({"run": index, "hit": False, + "failure": str(exc) if isinstance(exc, EvalFailure) else type(exc).__name__}) + changed = before != [harness.fingerprint(source) for source in sources] + result = {"case_id": case["id"], "host": host, "model": model, "runs": runs, "hits": hits, + "expected_any": expected.get("any", []), "prohibited": expected.get("prohibited", []), + "observations": observations, + "status": "passed" if hits >= min_hits and not changed else "failed"} + if changed: + result["failure"] = "plugin_changed_during_evaluation" + return result + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--host", required=True, choices=["codex", "claude"]) + parser.add_argument("--case") + parser.add_argument("--runs", type=int, default=1, help="repetitions per case") + parser.add_argument("--min-hits", type=int, help="hits a case needs to pass; default: every run") + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--timeout", type=float, default=420, help="maximum seconds per run") + auth = parser.add_mutually_exclusive_group() + auth.add_argument("--auth-home", type=Path) + auth.add_argument("--native-auth", action="store_true", + help="Claude only: preserve the calling terminal native authentication environment") + parser.add_argument("--model") + parser.add_argument("--plugin-root", type=Path, default=ROOT, + help="plugin whose evals/natural cases are run") + parser.add_argument("--with-plugin", type=Path, action="append", default=[], + help="another plugin root to load beside it; repeatable") + args = parser.parse_args(argv) + if args.native_auth and args.host != "claude": + parser.error("--native-auth is supported only for Claude") + if args.runs < 1: + parser.error("--runs must be at least 1") + min_hits = args.runs if args.min_hits is None else args.min_hits + if not 0 <= min_hits <= args.runs: + parser.error("--min-hits must lie between 0 and --runs") + sources = [args.plugin_root.resolve(), *[path.resolve() for path in args.with_plugin]] + for source in sources: + if not (source / "skills").is_dir(): + parser.error(f"not a plugin root: {source}") + output = args.output_dir.resolve() + if any(output.is_relative_to(source) for source in sources): + parser.error("output must not be written into a plugin source") + natural = sources[0] / "evals/natural" + expected_file = natural / "expected.json" + cases = [json.loads(path.read_text()) for path in sorted((natural / "cases").glob("*.json"))] + if args.case: + cases = [case for case in cases if case["id"] == args.case] + if not cases or not expected_file.is_file(): + parser.error("no natural evaluation cases selected") + expected = json.loads(expected_file.read_text()) + output.mkdir(parents=True, exist_ok=True) + os.chmod(output, 0o700) + results = [] + for case in cases: + result = run_case(args.host, case, expected[case["id"]], output, args.runs, min_hits, args.timeout, + sources, auth_home=args.auth_home, model=args.model, native_auth=args.native_auth) + results.append(result) + (output / f'{case["id"]}.json').write_text(json.dumps(result, indent=2, ensure_ascii=False) + "\n") + print(json.dumps({key: result[key] for key in ("case_id", "status", "hits", "runs")} + | {"loaded": [o.get("loaded", o.get("failure")) for o in result["observations"]]}, + ensure_ascii=False), flush=True) + commit = subprocess.run(["git", "rev-parse", "HEAD"], cwd=sources[0], capture_output=True, text=True) + summary = {"status": "passed" if all(r["status"] == "passed" for r in results) else "failed", + "host": args.host, "native_auth": args.native_auth, "runs": args.runs, "min_hits": min_hits, + "source_commit": commit.stdout.strip(), + "plugins": [{"root": source.name, "sha256": harness.fingerprint(source)} for source in sources], + "completed_at": datetime.now(timezone.utc).isoformat(), + "hits": sum(r["hits"] for r in results), "total_runs": args.runs * len(results), "results": results} + (output / "summary.json").write_text(json.dumps(summary, indent=2, ensure_ascii=False) + "\n") + return 0 if summary["status"] == "passed" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_natural_evals.py b/tests/test_natural_evals.py new file mode 100644 index 0000000..7fc5b46 --- /dev/null +++ b/tests/test_natural_evals.py @@ -0,0 +1,179 @@ +"""Unscripted routing evaluations count loads, not mentions, and never steer the host.""" + +from __future__ import annotations + +import io +import json +import subprocess +import tempfile +import unittest +from contextlib import redirect_stderr +from pathlib import Path +from unittest import mock + +from scripts import run_host_evals as harness +from scripts import run_natural_evals as natural + + +def lines(*events): + return "\n".join(json.dumps(event) for event in events) + + +def claude_call(identity, name, arguments, content="ok", error=False): + return [ + {"type": "assistant", "message": {"content": [ + {"type": "tool_use", "id": identity, "name": name, "input": arguments}]}}, + {"type": "user", "message": {"content": [ + {"type": "tool_result", "tool_use_id": identity, "content": content, "is_error": error}]}}, + ] + + +DONE = {"type": "result", "subtype": "success", "is_error": False, "result": "done"} + + +class LoadedSkillTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.plugin = Path(self.directory.name) / "plugin" + skill = self.plugin / "skills/alpha-skill" + skill.mkdir(parents=True) + self.content = "---\nname: alpha-skill\ndescription: Use when testing.\n---\n\n# Alpha\n\nBody.\n" + (skill / "SKILL.md").write_text(self.content) + + def tearDown(self): + self.directory.cleanup() + + def test_claude_counts_successful_skill_calls_and_skill_file_reads(self): + text = lines( + *claude_call("a", "Skill", {"skill": "writing-method:story-drafting"}), + *claude_call("b", "Read", {"file_path": "/x/skills/alpha-skill/SKILL.md"}), + *claude_call("c", "Skill", {"skill": "writing-method:story-drafting"}), + DONE) + self.assertEqual(["story-drafting", "alpha-skill"], natural.loaded_skills("claude", text, [self.plugin])) + + def test_claude_ignores_failed_calls_and_paths_inside_file_content(self): + text = lines( + *claude_call("a", "Skill", {"skill": "beta-skill"}, error=True), + *claude_call("b", "Read", {"file_path": "/x/policy.md"}, + content="see skills/alpha-skill/SKILL.md for details"), + {"type": "assistant", "message": {"content": [ + {"type": "text", "text": "I would use skills/alpha-skill/SKILL.md"}]}}, + DONE) + self.assertEqual([], natural.loaded_skills("claude", text, [self.plugin])) + + def test_codex_counts_a_complete_read_only(self): + def command(text, output, code=0): + return {"type": "item.completed", "item": {"type": "command_execution", "command": text, + "aggregated_output": output, "exit_code": code}} + path = ".agents/skills/alpha-skill/SKILL.md" + complete = lines(command(f"/bin/zsh -lc 'cat {path}'", self.content), {"type": "turn.completed"}) + self.assertEqual(["alpha-skill"], natural.loaded_skills("codex", complete, [self.plugin])) + listing = lines(command("/bin/zsh -lc 'rg --files .agents/skills'", path + "\n"), + command(f"/bin/zsh -lc 'cat {path}'", self.content, code=1), + command(f"/bin/zsh -lc 'head -2 {path}'", "---\nname: alpha-skill\n"), + {"type": "turn.completed"}) + self.assertEqual([], natural.loaded_skills("codex", listing, [self.plugin])) + + def test_unfinished_or_failed_transcripts_are_failures(self): + with self.assertRaisesRegex(harness.EvalFailure, "incomplete_transcript"): + natural.loaded_skills("claude", lines(*claude_call("a", "Skill", {"skill": "x"})), [self.plugin]) + with self.assertRaisesRegex(harness.EvalFailure, "host_error"): + natural.loaded_skills("codex", lines({"type": "turn.failed"}), [self.plugin]) + with self.assertRaisesRegex(harness.EvalFailure, "malformed_jsonl"): + natural.loaded_skills("claude", "{not json", [self.plugin]) + + def test_judgement(self): + expected = {"any": ["a", "b"], "prohibited": ["z"]} + self.assertTrue(natural.judge(["b", "other"], expected)) + self.assertFalse(natural.judge(["other"], expected)) + self.assertFalse(natural.judge(["a", "z"], expected)) + # A case that expects no skill hits only when nothing was loaded. + self.assertTrue(natural.judge([], {"any": []})) + self.assertFalse(natural.judge(["a"], {"any": []})) + + +class CommandTests(unittest.TestCase): + def test_prompt_is_sent_unchanged_with_every_plugin_and_read_only_tools(self): + with tempfile.TemporaryDirectory() as directory: + config = Path(directory) + plugins = [Path("/p/one"), Path("/p/two")] + command, env = natural.host_command("claude", Path("/repo"), plugins, config, "sonnet", "medium", + "Schreib bitte Kapitel 12.", native_auth=True) + self.assertEqual("Schreib bitte Kapitel 12.", command[-1]) + self.assertEqual(["/p/one", "/p/two"], + [command[i + 1] for i, word in enumerate(command) if word == "--plugin-dir"]) + self.assertEqual(natural.READ_ONLY_TOOLS, command[command.index("--allowedTools") + 1]) + self.assertNotIn("--json-schema", command) + self.assertNotIn("ANTHROPIC_API_KEY", env) + command, env = natural.host_command("codex", Path("/repo"), plugins, config, "m", None, + "Plain request.", auth_home=Path("/auth")) + self.assertEqual("Plain request.", command[-1]) + self.assertEqual("read-only", command[command.index("--sandbox") + 1]) + self.assertNotIn("--output-schema", command) + self.assertEqual("/auth", env["CODEX_HOME"]) + for instruction in (harness.ROUTING_INSTRUCTION, harness.SKILL_DISCOVERY_INSTRUCTION): + self.assertNotIn(instruction.strip()[:40], " ".join(command)) + + def test_codex_repository_links_each_skill_name_once(self): + with tempfile.TemporaryDirectory() as directory: + base = Path(directory) + for plugin, names in (("one", ["shared", "first"]), ("two", ["shared", "second"])): + for name in names: + (base / plugin / "skills" / name).mkdir(parents=True) + (base / plugin / "skills" / name / "SKILL.md").write_text(plugin) + natural.prepare_repo(base / "repo", {"files": {"docs/a.md": "x"}}, [base / "one", base / "two"], "codex") + skills = base / "repo/.agents/skills" + self.assertEqual({"shared", "first", "second"}, {path.name for path in skills.iterdir()}) + self.assertEqual("one", (skills / "shared/SKILL.md").read_text()) + natural.prepare_repo(base / "repo2", {"files": {}}, [base / "one"], "claude") + self.assertFalse((base / "repo2/.agents").exists()) + with self.assertRaisesRegex(harness.EvalFailure, "fixture_path_escape"): + natural.prepare_repo(base / "repo3", {"files": {"../out.md": "x"}}, [base / "one"], "claude") + + +class RunTests(unittest.TestCase): + def test_a_case_reports_hits_over_runs_and_never_stops_early(self): + observed = iter([(["a"], "m"), harness.EvalFailure("timeout"), (["other"], "m")]) + + def once(*args, **kwargs): + value = next(observed) + if isinstance(value, Exception): + raise value + return value + + with tempfile.TemporaryDirectory() as directory, \ + mock.patch.object(natural, "run_once", side_effect=once), \ + mock.patch.object(harness, "fingerprint", return_value="same"): + result = natural.run_case("claude", {"id": "c"}, {"any": ["a"]}, Path(directory), 3, 1, 1.0, + [Path(directory)]) + self.assertEqual((3, 1, "passed"), (result["runs"], result["hits"], result["status"])) + self.assertEqual([True, False, False], [o["hit"] for o in result["observations"]]) + self.assertEqual("timeout", result["observations"][1]["failure"]) + + def test_arguments_are_checked_before_any_host_runs(self): + with tempfile.TemporaryDirectory() as directory: + plugin = Path(directory) / "plugin" + (plugin / "skills").mkdir(parents=True) + base = ["--host", "claude", "--plugin-root", str(plugin)] + for extra in (["--output-dir", str(plugin / "out")], + ["--output-dir", str(Path(directory) / "out")], + ["--output-dir", str(Path(directory) / "out"), "--runs", "0"], + ["--output-dir", str(Path(directory) / "out"), "--runs", "2", "--min-hits", "3"], + ["--output-dir", str(Path(directory) / "out"), "--with-plugin", directory]): + with self.subTest(extra=extra), redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + natural.main(base + extra) + with redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + natural.main(["--host", "codex", "--native-auth", "--output-dir", str(Path(directory) / "o"), + "--plugin-root", str(plugin)]) + + +class HostProcessTests(unittest.TestCase): + def test_host_process_never_inherits_standard_input(self): + # An inherited open stdin makes a host wait for more input: the run then times out with no event. + output = harness.execute(["python3", "-c", "import sys; print(repr(sys.stdin.read()))"], + Path.cwd(), dict(__import__("os").environ), 20) + self.assertEqual("''", output.strip()) + + +if __name__ == "__main__": + unittest.main()