fix(evals): count a skill read inside a compound command

scripts/run_natural_evals.py

@@ -36,6 +36,21 @@ def bare(name):
     return name.rsplit(":", 1)[-1] if isinstance(name, str) else ""
 
 
+def skills_in_output(output, plugins):
+    """Skills whose complete SKILL.md is in a command's output, whatever the command looked like.
+
+    Hosts read a skill inside compound commands (`pwd; rg ...; cat <skill>`). The
+    whole file content proves the read; a path or a partial read does not.
+    """
+    found = []
+    for plugin in plugins:
+        for path in sorted((plugin / "skills").glob("*/SKILL.md")):
+            content = path.read_text().rstrip("\n")
+            if content.strip() and content in output and path.parent.name not in found:
+                found.append(path.parent.name)
+    return found
+
+
 def events(text):
     found = []
     for line in text.splitlines():
@@ -73,8 +88,7 @@ def add(names):
                     and item.get("type") == "command_execution" and item.get("exit_code") == 0:
                 command, output = item.get("command"), item.get("aggregated_output", "")
                 if isinstance(command, str) and isinstance(output, str):
-                    for plugin in plugins:
-                        add(sorted(harness.observed_read_skills(command, output, plugin)))
+                    add(skills_in_output(output, plugins))
             continue
         if kind == "result":
             if event.get("is_error") or event.get("subtype") != "success":
@@ -98,8 +112,7 @@ def add(names):
                     output = block["content"]
                     if isinstance(output, list):
                         output = "\n".join(str(part.get("text", "")) for part in output if isinstance(part, dict))
-                    for plugin in plugins:
-                        add(sorted(harness.observed_read_skills(str(arguments.get("command", "")), str(output), plugin)))
+                    add(skills_in_output(str(output), plugins))
     if not terminal:
         raise EvalFailure("incomplete_transcript", text)
     return order

tests/test_natural_evals.py

@@ -68,6 +68,9 @@ def command(text, output, code=0):
         path = ".agents/skills/alpha-skill/SKILL.md"
         complete = lines(command(f"/bin/zsh -lc 'cat {path}'", self.content), {"type": "turn.completed"})
         self.assertEqual(["alpha-skill"], natural.loaded_skills("codex", complete, [self.plugin]))
+        compound = lines(command(f"/bin/zsh -lc 'pwd; rg --files -g AGENTS.md; cat {path}'", "/repo\n" + self.content),
+                         {"type": "turn.completed"})
+        self.assertEqual(["alpha-skill"], natural.loaded_skills("codex", compound, [self.plugin]))
         listing = lines(command("/bin/zsh -lc 'rg --files .agents/skills'", path + "\n"),
                         command(f"/bin/zsh -lc 'cat {path}'", self.content, code=1),
                         command(f"/bin/zsh -lc 'head -2 {path}'", "---\nname: alpha-skill\n"),