fix(evals): run unscripted evaluations as the user runs the host

README.md

@@ -68,7 +68,7 @@ Two runners are exported to consumers. Both call a real host and cost usage; nei
 | Runner | What it sends | What it records |
 |---|---|---|
 | `scripts/run_host_evals.py` | A case prompt plus an instruction to select a workflow and write `decision.md`, with one plugin | The selected primary and supporting skills against `evals/<host>/expected.json` |
-| `scripts/run_natural_evals.py` | The case prompt exactly as written, with read-only tools and every plugin given by `--plugin-root` and `--with-plugin` | Which skills the host loaded on its own, over `--runs` repetitions, against `evals/natural/expected.json` |
+| `scripts/run_natural_evals.py` | The case prompt exactly as written, with the adapter's coordinator model and every plugin given by `--plugin-root` and `--with-plugin`. Claude loads staged copies; Codex uses the plugins installed in `--auth-home` and stops when an installed copy's skills differ from the source | Which skills the host loaded on its own, over `--runs` repetitions, against `evals/natural/expected.json` |
 
 A natural case is a file `evals/natural/cases/<id>.json` with `id`, `prompt` and `files`. `evals/natural/expected.json` maps each ID to `any` (a run hits when one of these skills was loaded) and `prohibited` (a run misses when one of these was). A load is a successful skill call or a complete read of a `SKILL.md`; a skill path that only appears inside another file does not count. Write the prompt in a user's words, not in the description's vocabulary.
 

scripts/run_natural_evals.py

@@ -5,6 +5,10 @@
 prompt exactly as written, with every given plugin loaded, and records which
 skills the host loaded on its own. Expectations never enter prompts. A case is
 repeated `--runs` times and reported as a count, because loading is statistical.
+
+The host runs as its user runs it: with the adapter's coordinator model, and on
+Codex with the plugins as they are installed in the given Codex home. An
+installed copy whose skills differ from the source stops the run.
 """
 from __future__ import annotations
 import argparse
@@ -90,6 +94,12 @@ def add(names):
                     add([bare(arguments.get("skill"))])
                 if call.get("name") == "Read":
                     add(harness.SKILL_PATH.findall(str(arguments.get("file_path", ""))))
+                if call.get("name") == "Bash":
+                    output = block["content"]
+                    if isinstance(output, list):
+                        output = "\n".join(str(part.get("text", "")) for part in output if isinstance(part, dict))
+                    for plugin in plugins:
+                        add(sorted(harness.observed_read_skills(str(arguments.get("command", "")), str(output), plugin)))
     if not terminal:
         raise EvalFailure("incomplete_transcript", text)
     return order
@@ -103,7 +113,31 @@ def judge(loaded, expected):
     return bool(set(loaded) & set(wanted)) if wanted else not loaded
 
 
-def prepare_repo(repo, case, plugins, host):
+def skills_fingerprint(plugin):
+    """Content of every skill file of a plugin, independent of where the plugin lies."""
+    return harness.fingerprint(plugin / "skills")
+
+
+def plugin_name(plugin):
+    manifest = plugin / ".codex-plugin/plugin.json"
+    return json.loads(manifest.read_text())["name"] if manifest.is_file() else plugin.name
+
+
+def installed_codex_plugins(sources, codex_home):
+    """The installed copy of each source plugin; its skills must equal the source."""
+    found = []
+    for source in sources:
+        name, wanted = plugin_name(source), skills_fingerprint(source)
+        copies = [path for path in sorted((codex_home / "plugins/cache").glob(f"*/{name}/*"))
+                  if (path / "skills").is_dir()]
+        match = [path for path in copies if skills_fingerprint(path) == wanted]
+        if not match:
+            raise EvalFailure(("installed_plugin_differs:" if copies else "plugin_not_installed:") + name)
+        found.append(match[-1])
+    return found
+
+
+def prepare_repo(repo, case):
     repo.mkdir()
     subprocess.run(["git", "init", "-q", str(repo)], check=True, capture_output=True)
     for relative, content in case.get("files", {}).items():
@@ -112,23 +146,16 @@ def prepare_repo(repo, case, plugins, host):
             raise EvalFailure("fixture_path_escape")
         path.parent.mkdir(parents=True, exist_ok=True)
         path.write_text(content)
-    if host == "codex":
-        # Codex discovers repository skills here; a name two plugins share is linked once.
-        skills = repo / ".agents/skills"
-        skills.mkdir(parents=True)
-        for plugin in plugins:
-            for path in sorted((plugin / "skills").iterdir()):
-                if path.is_dir() and not (skills / path.name).exists():
-                    (skills / path.name).symlink_to(path.resolve(), target_is_directory=True)
 
 
 def host_command(host, repo, plugins, config, model, effort, prompt, auth_home=None, native_auth=False, budget=2):
     env = os.environ.copy()
     for key in ("ANTHROPIC_API_KEY", "OPENAI_API_KEY", "CLAUDECODE", "ANTHROPIC_AUTH_TOKEN"):
         env.pop(key, None)
     if host == "codex":
-        env["CODEX_HOME"] = str(auth_home or config)
-        command = ["codex", "exec", "--json", "--ephemeral", "--ignore-user-config",
+        # The user's own Codex home: plugins load the way they are installed, not through repository links.
+        env["CODEX_HOME"] = str(auth_home)
+        command = ["codex", "exec", "--json", "--ephemeral",
                    "--sandbox", "read-only", "-C", str(repo), "-m", model]
         if effort is not None:
             command.extend(["-c", f'model_reasoning_effort="{effort}"'])
@@ -155,12 +182,16 @@ def run_once(host, case, output, label, timeout, sources, auth_home=None, model=
         temporary = Path(directory)
         repo, config = temporary / "repo", temporary / "config"
         config.mkdir()
-        staged = [harness.stage_plugin(source, temporary / f"plugin-{index}")
-                  for index, source in enumerate(sources)]
-        prepare_repo(repo, case, staged, host)
-        policy = harness.model_policy(host, sources[0])
-        model = model or policy["roles"][policy["evaluation_roles"]["routing"]][0]["model"]
-        effort = harness.model_effort(host, model, sources[0])
+        if host == "codex":
+            staged = installed_codex_plugins(sources, Path(auth_home))
+        else:
+            staged = [harness.stage_plugin(source, temporary / f"plugin-{index}")
+                      for index, source in enumerate(sources)]
+        prepare_repo(repo, case)
+        coordinator = harness.model_policy(host, sources[0])["coordinator"]
+        effort = coordinator["effort"] if model in (None, coordinator["model"]) \
+            else harness.model_effort(host, model, sources[0])
+        model = model or coordinator["model"]
         command, env = host_command(host, repo, staged, config, model, effort, case["prompt"],
                                     auth_home, native_auth)
         stdout = ""
@@ -208,14 +239,16 @@ def main(argv=None):
     auth.add_argument("--auth-home", type=Path)
     auth.add_argument("--native-auth", action="store_true",
                       help="Claude only: preserve the calling terminal native authentication environment")
-    parser.add_argument("--model")
+    parser.add_argument("--model", help="default: the coordinator model of the host adapter")
     parser.add_argument("--plugin-root", type=Path, default=ROOT,
                         help="plugin whose evals/natural cases are run")
     parser.add_argument("--with-plugin", type=Path, action="append", default=[],
                         help="another plugin root to load beside it; repeatable")
     args = parser.parse_args(argv)
     if args.native_auth and args.host != "claude":
         parser.error("--native-auth is supported only for Claude")
+    if args.host == "codex" and args.auth_home is None:
+        parser.error("Codex needs --auth-home: the Codex home in which the plugins are installed")
     if args.runs < 1:
         parser.error("--runs must be at least 1")
     min_hits = args.runs if args.min_hits is None else args.min_hits
@@ -236,6 +269,11 @@ def main(argv=None):
     if not cases or not expected_file.is_file():
         parser.error("no natural evaluation cases selected")
     expected = json.loads(expected_file.read_text())
+    if args.host == "codex":
+        try:
+            installed_codex_plugins(sources, args.auth_home)
+        except EvalFailure as exc:
+            parser.error(f"{exc}: reinstall the plugin in that Codex home, then run again")
     output.mkdir(parents=True, exist_ok=True)
     os.chmod(output, 0o700)
     results = []

tests/test_natural_evals.py

@@ -82,6 +82,11 @@ def test_unfinished_or_failed_transcripts_are_failures(self):
         with self.assertRaisesRegex(harness.EvalFailure, "malformed_jsonl"):
             natural.loaded_skills("claude", "{not json", [self.plugin])
 
+    def test_claude_counts_a_complete_shell_read(self):
+        path = str(self.plugin / "skills/alpha-skill/SKILL.md")
+        text = lines(*claude_call("a", "Bash", {"command": f"cat {path}"}, content=self.content), DONE)
+        self.assertEqual(["alpha-skill"], natural.loaded_skills("claude", text, [self.plugin]))
+
     def test_judgement(self):
         expected = {"any": ["a", "b"], "prohibited": ["z"]}
         self.assertTrue(natural.judge(["b", "other"], expected))
@@ -110,25 +115,38 @@ def test_prompt_is_sent_unchanged_with_every_plugin_and_read_only_tools(self):
             self.assertEqual("Plain request.", command[-1])
             self.assertEqual("read-only", command[command.index("--sandbox") + 1])
             self.assertNotIn("--output-schema", command)
+            # The plugins load as installed, so the user's configuration stays in effect.
+            self.assertNotIn("--ignore-user-config", command)
             self.assertEqual("/auth", env["CODEX_HOME"])
         for instruction in (harness.ROUTING_INSTRUCTION, harness.SKILL_DISCOVERY_INSTRUCTION):
             self.assertNotIn(instruction.strip()[:40], " ".join(command))
 
-    def test_codex_repository_links_each_skill_name_once(self):
+    def test_repository_holds_the_fixture_and_no_skill_links(self):
         with tempfile.TemporaryDirectory() as directory:
             base = Path(directory)
-            for plugin, names in (("one", ["shared", "first"]), ("two", ["shared", "second"])):
-                for name in names:
-                    (base / plugin / "skills" / name).mkdir(parents=True)
-                    (base / plugin / "skills" / name / "SKILL.md").write_text(plugin)
-            natural.prepare_repo(base / "repo", {"files": {"docs/a.md": "x"}}, [base / "one", base / "two"], "codex")
-            skills = base / "repo/.agents/skills"
-            self.assertEqual({"shared", "first", "second"}, {path.name for path in skills.iterdir()})
-            self.assertEqual("one", (skills / "shared/SKILL.md").read_text())
-            natural.prepare_repo(base / "repo2", {"files": {}}, [base / "one"], "claude")
-            self.assertFalse((base / "repo2/.agents").exists())
+            natural.prepare_repo(base / "repo", {"files": {"docs/a.md": "x"}})
+            self.assertEqual("x", (base / "repo/docs/a.md").read_text())
+            self.assertFalse((base / "repo/.agents").exists())
             with self.assertRaisesRegex(harness.EvalFailure, "fixture_path_escape"):
-                natural.prepare_repo(base / "repo3", {"files": {"../out.md": "x"}}, [base / "one"], "claude")
+                natural.prepare_repo(base / "repo3", {"files": {"../out.md": "x"}})
+
+    def test_codex_uses_the_installed_copy_only_when_its_skills_equal_the_source(self):
+        with tempfile.TemporaryDirectory() as directory:
+            base = Path(directory)
+            source, home = base / "source", base / "home"
+            (source / "skills/alpha").mkdir(parents=True)
+            (source / "skills/alpha/SKILL.md").write_text("current")
+            (source / ".codex-plugin").mkdir()
+            (source / ".codex-plugin/plugin.json").write_text(json.dumps({"name": "demo"}))
+            with self.assertRaisesRegex(harness.EvalFailure, "plugin_not_installed:demo"):
+                natural.installed_codex_plugins([source], home)
+            installed = home / "plugins/cache/market/demo/1.0.0"
+            (installed / "skills/alpha").mkdir(parents=True)
+            (installed / "skills/alpha/SKILL.md").write_text("older")
+            with self.assertRaisesRegex(harness.EvalFailure, "installed_plugin_differs:demo"):
+                natural.installed_codex_plugins([source], home)
+            (installed / "skills/alpha/SKILL.md").write_text("current")
+            self.assertEqual([installed], natural.installed_codex_plugins([source], home))
 
 
 class RunTests(unittest.TestCase):