From 6c7d3128f1c1a4135ed79804cf25c2bd61a9a499 Mon Sep 17 00:00:00 2001 From: Peter Hense Date: Fri, 13 Nov 2026 20:43:30 +0100 Subject: [PATCH] fix(evals): count a skill read inside a compound command Codex reads a skill as part of commands such as `pwd; rg ...; cat `. The unscripted runner now counts a skill when the complete SKILL.md is in a successful command's output, whatever the command looked like. Re-scoring the saved transcripts of 36 Codex runs turned 8 apparent misses into loads. Co-Authored-By: Claude Fable 5.2 --- scripts/run_natural_evals.py | 21 +++++++++++++++++---- tests/test_natural_evals.py | 3 +++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/scripts/run_natural_evals.py b/scripts/run_natural_evals.py index c91ed29..4556816 100644 --- a/scripts/run_natural_evals.py +++ b/scripts/run_natural_evals.py @@ -36,6 +36,21 @@ def bare(name): return name.rsplit(":", 1)[-1] if isinstance(name, str) else "" +def skills_in_output(output, plugins): + """Skills whose complete SKILL.md is in a command's output, whatever the command looked like. + + Hosts read a skill inside compound commands (`pwd; rg ...; cat `). The + whole file content proves the read; a path or a partial read does not. + """ + found = [] + for plugin in plugins: + for path in sorted((plugin / "skills").glob("*/SKILL.md")): + content = path.read_text().rstrip("\n") + if content.strip() and content in output and path.parent.name not in found: + found.append(path.parent.name) + return found + + def events(text): found = [] for line in text.splitlines(): @@ -73,8 +88,7 @@ def add(names): and item.get("type") == "command_execution" and item.get("exit_code") == 0: command, output = item.get("command"), item.get("aggregated_output", "") if isinstance(command, str) and isinstance(output, str): - for plugin in plugins: - add(sorted(harness.observed_read_skills(command, output, plugin))) + add(skills_in_output(output, plugins)) continue if kind == "result": if event.get("is_error") or event.get("subtype") != "success": @@ -98,8 +112,7 @@ def add(names): output = block["content"] if isinstance(output, list): output = "\n".join(str(part.get("text", "")) for part in output if isinstance(part, dict)) - for plugin in plugins: - add(sorted(harness.observed_read_skills(str(arguments.get("command", "")), str(output), plugin))) + add(skills_in_output(str(output), plugins)) if not terminal: raise EvalFailure("incomplete_transcript", text) return order diff --git a/tests/test_natural_evals.py b/tests/test_natural_evals.py index d1b27a7..a016125 100644 --- a/tests/test_natural_evals.py +++ b/tests/test_natural_evals.py @@ -68,6 +68,9 @@ def command(text, output, code=0): path = ".agents/skills/alpha-skill/SKILL.md" complete = lines(command(f"/bin/zsh -lc 'cat {path}'", self.content), {"type": "turn.completed"}) self.assertEqual(["alpha-skill"], natural.loaded_skills("codex", complete, [self.plugin])) + compound = lines(command(f"/bin/zsh -lc 'pwd; rg --files -g AGENTS.md; cat {path}'", "/repo\n" + self.content), + {"type": "turn.completed"}) + self.assertEqual(["alpha-skill"], natural.loaded_skills("codex", compound, [self.plugin])) listing = lines(command("/bin/zsh -lc 'rg --files .agents/skills'", path + "\n"), command(f"/bin/zsh -lc 'cat {path}'", self.content, code=1), command(f"/bin/zsh -lc 'head -2 {path}'", "---\nname: alpha-skill\n"),