From 6498f3c5e1ea7370d03c0af91c2437f377e79b94 Mon Sep 17 00:00:00 2001 From: Peter Hense Date: Fri, 13 Nov 2026 20:12:31 +0100 Subject: [PATCH] fix(evals): run unscripted evaluations as the user runs the host - default model is the adapter's coordinator model, not the routing role - Codex uses the plugins installed in the given Codex home and stops when an installed copy's skills differ from the source; repository skill links made Codex look for paths that do not exist - a complete shell read of a SKILL.md counts on Claude as well Co-Authored-By: Claude Fable 5.2 --- README.md | 2 +- scripts/run_natural_evals.py | 74 +++++++++++++++++++++++++++--------- tests/test_natural_evals.py | 42 ++++++++++++++------ 3 files changed, 87 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index b966a8d..b638fe2 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ Two runners are exported to consumers. Both call a real host and cost usage; nei | Runner | What it sends | What it records | |---|---|---| | `scripts/run_host_evals.py` | A case prompt plus an instruction to select a workflow and write `decision.md`, with one plugin | The selected primary and supporting skills against `evals//expected.json` | -| `scripts/run_natural_evals.py` | The case prompt exactly as written, with read-only tools and every plugin given by `--plugin-root` and `--with-plugin` | Which skills the host loaded on its own, over `--runs` repetitions, against `evals/natural/expected.json` | +| `scripts/run_natural_evals.py` | The case prompt exactly as written, with the adapter's coordinator model and every plugin given by `--plugin-root` and `--with-plugin`. Claude loads staged copies; Codex uses the plugins installed in `--auth-home` and stops when an installed copy's skills differ from the source | Which skills the host loaded on its own, over `--runs` repetitions, against `evals/natural/expected.json` | A natural case is a file `evals/natural/cases/.json` with `id`, `prompt` and `files`. `evals/natural/expected.json` maps each ID to `any` (a run hits when one of these skills was loaded) and `prohibited` (a run misses when one of these was). A load is a successful skill call or a complete read of a `SKILL.md`; a skill path that only appears inside another file does not count. Write the prompt in a user's words, not in the description's vocabulary. diff --git a/scripts/run_natural_evals.py b/scripts/run_natural_evals.py index f2e1393..c91ed29 100644 --- a/scripts/run_natural_evals.py +++ b/scripts/run_natural_evals.py @@ -5,6 +5,10 @@ prompt exactly as written, with every given plugin loaded, and records which skills the host loaded on its own. Expectations never enter prompts. A case is repeated `--runs` times and reported as a count, because loading is statistical. + +The host runs as its user runs it: with the adapter's coordinator model, and on +Codex with the plugins as they are installed in the given Codex home. An +installed copy whose skills differ from the source stops the run. """ from __future__ import annotations import argparse @@ -90,6 +94,12 @@ def add(names): add([bare(arguments.get("skill"))]) if call.get("name") == "Read": add(harness.SKILL_PATH.findall(str(arguments.get("file_path", "")))) + if call.get("name") == "Bash": + output = block["content"] + if isinstance(output, list): + output = "\n".join(str(part.get("text", "")) for part in output if isinstance(part, dict)) + for plugin in plugins: + add(sorted(harness.observed_read_skills(str(arguments.get("command", "")), str(output), plugin))) if not terminal: raise EvalFailure("incomplete_transcript", text) return order @@ -103,7 +113,31 @@ def judge(loaded, expected): return bool(set(loaded) & set(wanted)) if wanted else not loaded -def prepare_repo(repo, case, plugins, host): +def skills_fingerprint(plugin): + """Content of every skill file of a plugin, independent of where the plugin lies.""" + return harness.fingerprint(plugin / "skills") + + +def plugin_name(plugin): + manifest = plugin / ".codex-plugin/plugin.json" + return json.loads(manifest.read_text())["name"] if manifest.is_file() else plugin.name + + +def installed_codex_plugins(sources, codex_home): + """The installed copy of each source plugin; its skills must equal the source.""" + found = [] + for source in sources: + name, wanted = plugin_name(source), skills_fingerprint(source) + copies = [path for path in sorted((codex_home / "plugins/cache").glob(f"*/{name}/*")) + if (path / "skills").is_dir()] + match = [path for path in copies if skills_fingerprint(path) == wanted] + if not match: + raise EvalFailure(("installed_plugin_differs:" if copies else "plugin_not_installed:") + name) + found.append(match[-1]) + return found + + +def prepare_repo(repo, case): repo.mkdir() subprocess.run(["git", "init", "-q", str(repo)], check=True, capture_output=True) for relative, content in case.get("files", {}).items(): @@ -112,14 +146,6 @@ def prepare_repo(repo, case, plugins, host): raise EvalFailure("fixture_path_escape") path.parent.mkdir(parents=True, exist_ok=True) path.write_text(content) - if host == "codex": - # Codex discovers repository skills here; a name two plugins share is linked once. - skills = repo / ".agents/skills" - skills.mkdir(parents=True) - for plugin in plugins: - for path in sorted((plugin / "skills").iterdir()): - if path.is_dir() and not (skills / path.name).exists(): - (skills / path.name).symlink_to(path.resolve(), target_is_directory=True) def host_command(host, repo, plugins, config, model, effort, prompt, auth_home=None, native_auth=False, budget=2): @@ -127,8 +153,9 @@ def host_command(host, repo, plugins, config, model, effort, prompt, auth_home=N for key in ("ANTHROPIC_API_KEY", "OPENAI_API_KEY", "CLAUDECODE", "ANTHROPIC_AUTH_TOKEN"): env.pop(key, None) if host == "codex": - env["CODEX_HOME"] = str(auth_home or config) - command = ["codex", "exec", "--json", "--ephemeral", "--ignore-user-config", + # The user's own Codex home: plugins load the way they are installed, not through repository links. + env["CODEX_HOME"] = str(auth_home) + command = ["codex", "exec", "--json", "--ephemeral", "--sandbox", "read-only", "-C", str(repo), "-m", model] if effort is not None: command.extend(["-c", f'model_reasoning_effort="{effort}"']) @@ -155,12 +182,16 @@ def run_once(host, case, output, label, timeout, sources, auth_home=None, model= temporary = Path(directory) repo, config = temporary / "repo", temporary / "config" config.mkdir() - staged = [harness.stage_plugin(source, temporary / f"plugin-{index}") - for index, source in enumerate(sources)] - prepare_repo(repo, case, staged, host) - policy = harness.model_policy(host, sources[0]) - model = model or policy["roles"][policy["evaluation_roles"]["routing"]][0]["model"] - effort = harness.model_effort(host, model, sources[0]) + if host == "codex": + staged = installed_codex_plugins(sources, Path(auth_home)) + else: + staged = [harness.stage_plugin(source, temporary / f"plugin-{index}") + for index, source in enumerate(sources)] + prepare_repo(repo, case) + coordinator = harness.model_policy(host, sources[0])["coordinator"] + effort = coordinator["effort"] if model in (None, coordinator["model"]) \ + else harness.model_effort(host, model, sources[0]) + model = model or coordinator["model"] command, env = host_command(host, repo, staged, config, model, effort, case["prompt"], auth_home, native_auth) stdout = "" @@ -208,7 +239,7 @@ def main(argv=None): auth.add_argument("--auth-home", type=Path) auth.add_argument("--native-auth", action="store_true", help="Claude only: preserve the calling terminal native authentication environment") - parser.add_argument("--model") + parser.add_argument("--model", help="default: the coordinator model of the host adapter") parser.add_argument("--plugin-root", type=Path, default=ROOT, help="plugin whose evals/natural cases are run") parser.add_argument("--with-plugin", type=Path, action="append", default=[], @@ -216,6 +247,8 @@ def main(argv=None): args = parser.parse_args(argv) if args.native_auth and args.host != "claude": parser.error("--native-auth is supported only for Claude") + if args.host == "codex" and args.auth_home is None: + parser.error("Codex needs --auth-home: the Codex home in which the plugins are installed") if args.runs < 1: parser.error("--runs must be at least 1") min_hits = args.runs if args.min_hits is None else args.min_hits @@ -236,6 +269,11 @@ def main(argv=None): if not cases or not expected_file.is_file(): parser.error("no natural evaluation cases selected") expected = json.loads(expected_file.read_text()) + if args.host == "codex": + try: + installed_codex_plugins(sources, args.auth_home) + except EvalFailure as exc: + parser.error(f"{exc}: reinstall the plugin in that Codex home, then run again") output.mkdir(parents=True, exist_ok=True) os.chmod(output, 0o700) results = [] diff --git a/tests/test_natural_evals.py b/tests/test_natural_evals.py index 7fc5b46..d1b27a7 100644 --- a/tests/test_natural_evals.py +++ b/tests/test_natural_evals.py @@ -82,6 +82,11 @@ def test_unfinished_or_failed_transcripts_are_failures(self): with self.assertRaisesRegex(harness.EvalFailure, "malformed_jsonl"): natural.loaded_skills("claude", "{not json", [self.plugin]) + def test_claude_counts_a_complete_shell_read(self): + path = str(self.plugin / "skills/alpha-skill/SKILL.md") + text = lines(*claude_call("a", "Bash", {"command": f"cat {path}"}, content=self.content), DONE) + self.assertEqual(["alpha-skill"], natural.loaded_skills("claude", text, [self.plugin])) + def test_judgement(self): expected = {"any": ["a", "b"], "prohibited": ["z"]} self.assertTrue(natural.judge(["b", "other"], expected)) @@ -110,25 +115,38 @@ def test_prompt_is_sent_unchanged_with_every_plugin_and_read_only_tools(self): self.assertEqual("Plain request.", command[-1]) self.assertEqual("read-only", command[command.index("--sandbox") + 1]) self.assertNotIn("--output-schema", command) + # The plugins load as installed, so the user's configuration stays in effect. + self.assertNotIn("--ignore-user-config", command) self.assertEqual("/auth", env["CODEX_HOME"]) for instruction in (harness.ROUTING_INSTRUCTION, harness.SKILL_DISCOVERY_INSTRUCTION): self.assertNotIn(instruction.strip()[:40], " ".join(command)) - def test_codex_repository_links_each_skill_name_once(self): + def test_repository_holds_the_fixture_and_no_skill_links(self): with tempfile.TemporaryDirectory() as directory: base = Path(directory) - for plugin, names in (("one", ["shared", "first"]), ("two", ["shared", "second"])): - for name in names: - (base / plugin / "skills" / name).mkdir(parents=True) - (base / plugin / "skills" / name / "SKILL.md").write_text(plugin) - natural.prepare_repo(base / "repo", {"files": {"docs/a.md": "x"}}, [base / "one", base / "two"], "codex") - skills = base / "repo/.agents/skills" - self.assertEqual({"shared", "first", "second"}, {path.name for path in skills.iterdir()}) - self.assertEqual("one", (skills / "shared/SKILL.md").read_text()) - natural.prepare_repo(base / "repo2", {"files": {}}, [base / "one"], "claude") - self.assertFalse((base / "repo2/.agents").exists()) + natural.prepare_repo(base / "repo", {"files": {"docs/a.md": "x"}}) + self.assertEqual("x", (base / "repo/docs/a.md").read_text()) + self.assertFalse((base / "repo/.agents").exists()) with self.assertRaisesRegex(harness.EvalFailure, "fixture_path_escape"): - natural.prepare_repo(base / "repo3", {"files": {"../out.md": "x"}}, [base / "one"], "claude") + natural.prepare_repo(base / "repo3", {"files": {"../out.md": "x"}}) + + def test_codex_uses_the_installed_copy_only_when_its_skills_equal_the_source(self): + with tempfile.TemporaryDirectory() as directory: + base = Path(directory) + source, home = base / "source", base / "home" + (source / "skills/alpha").mkdir(parents=True) + (source / "skills/alpha/SKILL.md").write_text("current") + (source / ".codex-plugin").mkdir() + (source / ".codex-plugin/plugin.json").write_text(json.dumps({"name": "demo"})) + with self.assertRaisesRegex(harness.EvalFailure, "plugin_not_installed:demo"): + natural.installed_codex_plugins([source], home) + installed = home / "plugins/cache/market/demo/1.0.0" + (installed / "skills/alpha").mkdir(parents=True) + (installed / "skills/alpha/SKILL.md").write_text("older") + with self.assertRaisesRegex(harness.EvalFailure, "installed_plugin_differs:demo"): + natural.installed_codex_plugins([source], home) + (installed / "skills/alpha/SKILL.md").write_text("current") + self.assertEqual([installed], natural.installed_codex_plugins([source], home)) class RunTests(unittest.TestCase):