From f4ce883951644e8fa1d17e8a5c0bdce0b67dd5ce Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 23 Oct 2026 22:18:57 -0700 Subject: [PATCH 01/13] feat(tasks): detect eval-task formats with `tasks inspect` A probe registry (dokimi.task_formats) names the format of a file, a task directory or a tree, and reports kind, item count, present/missing parts, lint problems and which dokimi backends consume it today. Probes: harbor and task-md (wrapping harnesses.tasks), terminal-bench v1, SWE-bench, Inspect AI samples and @task files (text scan, never imported), OpenAI evals, skill-creator evals.json and trigger sets, skill-reach QuerySet, promptfoo and dokimi questions. Unknown paths get a best-effort structure report; malformed files are lint problems. Detection only: nothing new runs. Signed-off-by: Nikolay Petrov --- ...1023_120000_napetrov_task-format-detect.md | 11 + dokimi/docs/architecture.md | 2 +- dokimi/docs/coding-harnesses.md | 29 ++ dokimi/dokimi/__init__.py | 3 +- dokimi/dokimi/commands/tasks.py | 45 ++ dokimi/dokimi/task_formats/__init__.py | 16 + dokimi/dokimi/task_formats/configs.py | 100 +++++ dokimi/dokimi/task_formats/datasets.py | 336 +++++++++++++++ dokimi/dokimi/task_formats/detect.py | 158 +++++++ dokimi/dokimi/task_formats/dirs.py | 155 +++++++ dokimi/dokimi/task_formats/model.py | 91 ++++ dokimi/dokimi/task_formats/source.py | 113 +++++ dokimi/tests/test_import_architecture.py | 2 +- dokimi/tests/test_task_format_detect.py | 387 ++++++++++++++++++ 14 files changed, 1445 insertions(+), 3 deletions(-) create mode 100644 dokimi/changelog.d/20261023_120000_napetrov_task-format-detect.md create mode 100644 dokimi/dokimi/task_formats/__init__.py create mode 100644 dokimi/dokimi/task_formats/configs.py create mode 100644 dokimi/dokimi/task_formats/datasets.py create mode 100644 dokimi/dokimi/task_formats/detect.py create mode 100644 dokimi/dokimi/task_formats/dirs.py create mode 100644 dokimi/dokimi/task_formats/model.py create mode 100644 dokimi/dokimi/task_formats/source.py create mode 100644 dokimi/tests/test_task_format_detect.py diff --git a/dokimi/changelog.d/20261023_120000_napetrov_task-format-detect.md b/dokimi/changelog.d/20261023_120000_napetrov_task-format-detect.md new file mode 100644 index 000000000..deafaf0d7 --- /dev/null +++ b/dokimi/changelog.d/20261023_120000_napetrov_task-format-detect.md @@ -0,0 +1,11 @@ +### Added + +- **`tasks inspect [--json]`** — detects the eval-task format of a file, + a task directory or a whole tree and reports its structure: kind + (executable, answer-only, trigger), item count, present and missing parts, + lint problems and which dokimi backends consume it today. Recognises Harbor, + BenchFlow `task.md`, terminal-bench v1, SWE-bench instances, Inspect AI + samples and `@task` files (scanned as text, never imported), OpenAI evals + JSONL, skill-creator `evals.json` and trigger sets, skill-reach query sets, + promptfoo configs and dokimi question sets; anything else is reported as + `unknown` with its files. Detection only: no new format runs yet. diff --git a/dokimi/docs/architecture.md b/dokimi/docs/architecture.md index 4a37c85da..64d8fcc9c 100644 --- a/dokimi/docs/architecture.md +++ b/dokimi/docs/architecture.md @@ -70,7 +70,7 @@ points up: | Layer | Modules | | --- | --- | | front end | `cli`, `commands`, `__main__` (the argparse front end; no layer below imports it) | -| tracks | `runner`, `report`, `subjects`, `novelty`, `harnesses`, `context`, `repos`, `explore`, `tool_preference`, `external`, `dashboard`, `orchestrator`, `datasets_export`, `coverage`, and the checkers `config_check`, `schema_check`, `schema_compat` | +| tracks | `runner`, `report`, `subjects`, `novelty`, `harnesses`, `context`, `repos`, `explore`, `tool_preference`, `external`, `dashboard`, `orchestrator`, `datasets_export`, `coverage`, `task_formats`, and the checkers `config_check`, `schema_check`, `schema_compat` | | evaluation | `eval`, `personas`, `questions`, `treatments`, `plugins` | | components | `ingest`, `metrics`, `gate`, `mcp`, `skills`, `agent_profiles` | | foundation | `__init__` (the `dokimi` package itself), `_version`, `artifacts`, `records`, `llm`, `process`, `caching`, `config`, `registry`, `intents`, `defaults`, `console`, `utils`, `metrics/usage.py` | diff --git a/dokimi/docs/coding-harnesses.md b/dokimi/docs/coding-harnesses.md index 91cb3a876..a482d1231 100644 --- a/dokimi/docs/coding-harnesses.md +++ b/dokimi/docs/coding-harnesses.md @@ -78,6 +78,35 @@ refused (a few hundred bytes of nested aliases expand without bound), and frontmatter nested too deeply to parse is a parse error. YAML and TOML dates and times become ISO strings, and YAML sets sorted lists. +### Other eval formats: `tasks inspect` + +`python cli.py tasks inspect [--json]` names the eval-task format of a +file, a task directory, or every item under a tree, and reports its structure: +kind, item count, which parts are present or missing (instruction, +environment, oracle, verifier, metadata), lint problems, and what in dokimi +can consume it today. It only reads: nothing is run or converted, a Python +file is scanned as text and never imported, and a file that does not parse is +a problem on its report, not an error. A path no format claims is reported as +`unknown` with its files (or keys) and likely prompt and verifier files; when +two formats claim a path about equally, the report lists the other as +`ambiguous` with its evidence. + +| Format | Recognised by | Kind | Consumed by | +|---|---|---|---| +| `harbor` | directory with `task.toml` | executable | Harbor-backed harnesses (unless Harbor would refuse it), `docker-*` | +| `task-md` | directory with a BenchFlow `task.md` | executable | `benchflow*`, `docker-*` | +| `terminal-bench-v1` | directory with `task.yaml` + `run-tests.sh` / `Dockerfile` / `solution.sh` | executable | none yet | +| `swe-bench` | JSON/JSONL rows with `instance_id`, `repo`, `base_commit`, `problem_statement` (`FAIL_TO_PASS`) | executable | none yet | +| `inspect-samples` | rows with `input` and `target` | answer-only | none yet | +| `inspect-task` | `.py` importing `inspect_ai` with a `@task` function | answer-only (executable with a `sandbox=`) | none yet | +| `openai-evals` | JSONL rows with `input` (chat messages) and `ideal` | answer-only | none yet | +| `skill-creator-evals` | `evals.json` with `skill_name` and `evals[].prompt` | answer-only | none yet | +| `skill-creator-trigger` | rows with `query` and `should_trigger` | trigger | none yet | +| `skill-reach` | `QuerySet` (`queries[]` with `query_id`/`id`, `text`, `expected_skill`) | trigger | none yet | +| `dokimi-questions` | questions.v1/v2 JSON | answer-only (trigger with `expected_skill`) | questions track | + +`tasks list` and `tasks run` still read only the first two. + The `docker-*` harnesses mount the oracle at `/solution` and `/oracle` and the verifier at `/tests` and `/verifier`, because a script written for one layout refers to its siblings by that layout's path. The directory's contents are diff --git a/dokimi/dokimi/__init__.py b/dokimi/dokimi/__init__.py index 81f93aa55..eabdfaaaa 100644 --- a/dokimi/dokimi/__init__.py +++ b/dokimi/dokimi/__init__.py @@ -22,7 +22,8 @@ [`dashboard`][dokimi.dashboard] aggregates results across libraries and [`orchestrator`][dokimi.orchestrator] runs the one-command pipeline. - Executable tasks: [`harnesses`][dokimi.harnesses] runs coding agents on task - directories. + directories; [`task_formats`][dokimi.task_formats] names the format of other + eval-task files and directories (detection only). - [`context`][dokimi.context] reads what a run's context window held. - Content novelty: [`novelty`][dokimi.novelty]. Repositories as subjects: [`repos`][dokimi.repos]. Subject scorecards, one per run: diff --git a/dokimi/dokimi/commands/tasks.py b/dokimi/dokimi/commands/tasks.py index 0564c8fc0..797795621 100644 --- a/dokimi/dokimi/commands/tasks.py +++ b/dokimi/dokimi/commands/tasks.py @@ -47,6 +47,43 @@ def cmd_tasks_list(args: argparse.Namespace) -> None: print(f"{task.name}\t{difficulty}\t{','.join(tags)}") +def _inspect_line(report: dict) -> str: + n = report["items"] + items = "" if n is None else f"\t{n} item{'' if n == 1 else 's'}" + return f"{report['path']}\t{report['format']}\t{report['kind']}{items}\t-> {', '.join(report['consumers'])}" + + +def _print_inspect(reports: list[dict]) -> None: + for r in reports: + print(_inspect_line(r)) + if r["present"] or r["missing"]: + print(f" parts: {', '.join(r['present']) or '-'}; missing: {', '.join(r['missing']) or '-'}") + for alt in r["alternatives"]: + print(f" ambiguous: also {alt['format']} ({alt['confidence']:.2f}: {', '.join(alt['evidence'])})") + for problem in r["problems"]: + print(f" problem: {problem}") + by_format: dict[str, int] = {} + for r in reports: + by_format[r["format"]] = by_format.get(r["format"], 0) + 1 + print(f"{len(reports)} item{'' if len(reports) == 1 else 's'}: " + ", ".join(f"{f} {n}" for f, n in sorted(by_format.items()))) + + +def cmd_tasks_inspect(args: argparse.Namespace) -> None: + """Detect the eval-task format of a path (or of everything under it) and report its structure.""" + import json + + from dokimi.task_formats import inspect_path + + try: + reports = [r.as_dict() for r in inspect_path(Path(args.path))] + except FileNotFoundError as exc: + raise SystemExit(f"error: {exc}") from None + if args.json: + print(json.dumps({"root": str(args.path), "items": reports}, indent=2, default=str)) + else: + _print_inspect(reports) + + def cmd_tasks_harnesses(args: argparse.Namespace) -> None: from dokimi.harnesses import known_harnesses @@ -185,6 +222,14 @@ def register(sub, positive_int) -> None: list_p.add_argument("--tasks-root", default=None, help=_TASKS_ROOT_HELP) list_p.set_defaults(func=cmd_tasks_list) + inspect_p = tasks_sub.add_parser( + "inspect", + help="Detect the eval-task format of a file or directory tree and report its structure (no run)", + ) + inspect_p.add_argument("path", help="A task directory, a dataset file, or a tree holding either") + inspect_p.add_argument("--json", action="store_true", help="Print the reports as JSON") + inspect_p.set_defaults(func=cmd_tasks_inspect) + harnesses_p = tasks_sub.add_parser("harnesses", help="List known coding harness aliases") harnesses_p.set_defaults(func=cmd_tasks_harnesses) diff --git a/dokimi/dokimi/task_formats/__init__.py b/dokimi/dokimi/task_formats/__init__.py new file mode 100644 index 000000000..12f9ab34d --- /dev/null +++ b/dokimi/dokimi/task_formats/__init__.py @@ -0,0 +1,16 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Eval-task format detection: which format is a path, and what does it hold? + +Detection and a structure report only: nothing here runs or converts a task. +``dokimi tasks inspect `` is the command line; :func:`inspect_path` the API. +""" + +from dokimi.task_formats.detect import DIR_PROBES, FILE_PROBES, PROBES, detect, inspect_path, unknown +from dokimi.task_formats.model import Fingerprint, Probe, Report + +__all__ = [ + "DIR_PROBES", "FILE_PROBES", "PROBES", "Fingerprint", "Probe", "Report", "detect", + "inspect_path", "unknown", +] diff --git a/dokimi/dokimi/task_formats/configs.py b/dokimi/dokimi/task_formats/configs.py new file mode 100644 index 000000000..75ff306d5 --- /dev/null +++ b/dokimi/dokimi/task_formats/configs.py @@ -0,0 +1,100 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Probes for eval definitions that are config or code: promptfoo, Inspect AI tasks. + +An Inspect ``@task`` file is Python: it is scanned as text with regular +expressions and never imported or run. +""" + +from __future__ import annotations + +import re +from typing import Any + +from dokimi.task_formats.model import KIND_ANSWER_ONLY, KIND_EXECUTABLE, Fingerprint, Report +from dokimi.task_formats.source import FileSource + +# --- promptfoo (promptfoo.dev/docs/configuration/reference) ------------------- + +_PROMPTFOO_NAMES = ("promptfooconfig.yaml", "promptfooconfig.yml", "promptfooconfig.json") + + +class PromptfooProbe: + name = "promptfoo" + target = "file" + + def fingerprint(self, source: FileSource) -> Fingerprint | None: + if source.suffix not in (".yaml", ".yml", ".json"): + return None + named = source.path.name in _PROMPTFOO_NAMES + data = source.data + if not isinstance(data, dict): + return Fingerprint(0.6, ("file name",)) if named else None + keys = [k for k in ("prompts", "providers", "targets", "tests", "scenarios") if k in data] + has_target = "providers" in data or "targets" in data + if not named and not ("prompts" in data and has_target and ("tests" in data or "scenarios" in data)): + return None + return Fingerprint(min(1.0, 0.5 + 0.1 * named + 0.1 * len(keys)), (("file name",) if named else ()) + tuple(keys)) + + def describe(self, source: FileSource) -> Report: + data: dict[str, Any] = source.data if isinstance(source.data, dict) else {} + tests = data.get("tests") + report = Report(source.path, self.name, KIND_ANSWER_ONLY, + items=len(tests) if isinstance(tests, list) else None) + report.problems.extend(source.problems) + if source.data is not None and not data: + report.problems.append("not a mapping") + asserts = isinstance(data.get("defaultTest"), dict) and "assert" in data["defaultTest"] + asserts = asserts or any(isinstance(t, dict) and t.get("assert") for t in tests or [] if isinstance(tests, list)) + report.set_parts({ + "instruction": bool(data.get("prompts")), + "environment": bool(data.get("providers") or data.get("targets")), + "verifier": asserts, + "metadata": bool(data.get("description")), + }) + if isinstance(tests, str): + report.details["tests_from"] = tests + elif data and tests is None and "scenarios" not in data: + report.problems.append("no tests") + if data and not data.get("prompts"): + report.problems.append("no prompts") + return report + + +# --- Inspect AI @task files ---------------------------------------------------- + +_INSPECT_IMPORT = re.compile(r"^\s*(?:from\s+inspect_ai(?:\.\w+)*\s+import|import\s+inspect_ai)\b", re.M) +_TASK_DECORATOR = re.compile(r"^\s*@(?:inspect_ai\.)?task\b", re.M) +_SANDBOX = re.compile(r"\bsandbox\s*=") +_SCORER = re.compile(r"\bscorer\s*=") +_DATASET = re.compile(r"\bdataset\s*=|\bSample\s*\(|\b\w+_dataset\s*\(") +_SOLVER = re.compile(r"\b(?:solver|plan)\s*=") + + +class InspectTaskProbe: + name = "inspect-task" + target = "file" + + def fingerprint(self, source: FileSource) -> Fingerprint | None: + if source.suffix != ".py" or not source.text: + return None + if not _INSPECT_IMPORT.search(source.text) or not _TASK_DECORATOR.search(source.text): + return None + return Fingerprint(0.9, ("imports inspect_ai", "@task")) + + def describe(self, source: FileSource) -> Report: + text = source.text or "" + sandboxed = bool(_SANDBOX.search(text)) + report = Report(source.path, self.name, KIND_EXECUTABLE if sandboxed else KIND_ANSWER_ONLY, + items=len(_TASK_DECORATOR.findall(text))) + report.problems.extend(source.problems) + report.set_parts({ + "instruction": bool(_DATASET.search(text)), + "environment": sandboxed, + "verifier": bool(_SCORER.search(text)), + "metadata": False, + }) + report.details["solver"] = bool(_SOLVER.search(text)) + report.details["scan"] = "static text scan; the file was not imported" + return report diff --git a/dokimi/dokimi/task_formats/datasets.py b/dokimi/dokimi/task_formats/datasets.py new file mode 100644 index 000000000..e433ff1d0 --- /dev/null +++ b/dokimi/dokimi/task_formats/datasets.py @@ -0,0 +1,336 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Probes for dataset files: one file, many items. + +Field names follow each upstream's own definition: + +- SWE-bench: ``swebench/types.py`` ``SWEbenchInstance`` (SWE-bench/SWE-bench). +- Inspect AI: ``Sample`` / ``FieldSpec`` defaults in ``inspect_ai/dataset/_dataset.py``. +- OpenAI evals: ``docs/build-eval.md`` (``input`` as chat messages, ``ideal``). +- anthropics/skills skill-creator: ``references/schemas.md`` (``evals.json``) and + the trigger eval set of its ``SKILL.md`` (``[{query, should_trigger}]``). +- google/skill-reach: ``Query`` / ``QuerySet`` in ``src/reach/models.py`` and + ``src/reach/queries.py`` (``id`` is the legacy spelling of ``query_id``). +""" + +from __future__ import annotations + +import json +from collections import Counter +from typing import Any + +from dokimi.task_formats.model import ( + BY_QUESTIONS, KIND_ANSWER_ONLY, KIND_EXECUTABLE, KIND_TRIGGER, Fingerprint, Report, +) +from dokimi.task_formats.source import FileSource, first_rows, key_share + +_ROW_SUFFIXES = (".json", ".jsonl", ".yaml", ".yml") + + +def _records(source: FileSource, marker: str) -> list[dict[str, Any]] | None: + """The file's rows; a single object holding *marker* is a one-row file.""" + if source.suffix not in _ROW_SUFFIXES: + return None + if source.rows is not None: + return source.rows + data = source.data + return [data] if isinstance(data, dict) and marker in data else None + + +def _without(rows: list[dict[str, Any]], key: str) -> int: + return sum(1 for r in rows if not r.get(key)) + + +def _lint_missing(report: Report, rows: list[dict[str, Any]], keys: tuple[str, ...]) -> None: + for key in keys: + n = _without(rows, key) + if n: + report.problems.append(f"{n} of {len(rows)} items have no {key}") + + +def _lint_duplicate_ids(report: Report, rows: list[dict[str, Any]], *keys: str) -> None: + ids = Counter(str(next((r[k] for k in keys if k in r), "")) for r in rows) + dupes = sorted(i for i, n in ids.items() if i and n > 1) + if dupes: + report.problems.append(f"duplicate ids: {', '.join(dupes[:5])}") + + +def _base(source: FileSource, name: str, kind: str, rows: list[dict[str, Any]]) -> Report: + report = Report(source.path, name, kind, items=len(rows)) + report.problems.extend(source.problems) + return report + + +# --- SWE-bench --------------------------------------------------------------- + +_SWE_REQUIRED = ("instance_id", "repo", "base_commit", "problem_statement") + + +def _test_list(value: Any) -> list[Any] | None: + """FAIL_TO_PASS / PASS_TO_PASS: a list, or (as the HF datasets ship it) its JSON string.""" + if isinstance(value, list): + return value + if isinstance(value, str): + try: + parsed = json.loads(value) + except ValueError: + return None + return parsed if isinstance(parsed, list) else None + return None + + +class SweBenchProbe: + name = "swe-bench" + target = "file" + + def fingerprint(self, source: FileSource) -> Fingerprint | None: + rows = first_rows(_records(source, "instance_id")) + share = key_share(rows, *_SWE_REQUIRED) + if share < 0.5: + return None + tests = key_share(rows, "FAIL_TO_PASS", "PASS_TO_PASS") + return Fingerprint(0.6 * share + 0.4 * tests, (*_SWE_REQUIRED, "FAIL_TO_PASS/PASS_TO_PASS" if tests else "")) + + def describe(self, source: FileSource) -> Report: + rows = _records(source, "instance_id") or [] + report = _base(source, self.name, KIND_EXECUTABLE, rows) + report.set_parts({ + "instruction": key_share(rows, "problem_statement") == 1, + "environment": key_share(rows, "repo", "base_commit") == 1, + "oracle": key_share(rows, "patch") == 1, + "verifier": key_share(rows, "FAIL_TO_PASS") == 1, + "metadata": key_share(rows, "version") == 1, + }) + _lint_missing(report, rows, (*_SWE_REQUIRED, "FAIL_TO_PASS")) + bad = sum(1 for r in rows if "FAIL_TO_PASS" in r and _test_list(r["FAIL_TO_PASS"]) is None) + if bad: + report.problems.append(f"{bad} items have a FAIL_TO_PASS that is not a list of test ids") + _lint_duplicate_ids(report, rows, "instance_id") + report.details["repos"] = sorted({str(r.get("repo")) for r in rows if r.get("repo")})[:20] + return report + + +# --- Inspect AI samples and OpenAI evals -------------------------------------- + + +def _is_chat(value: Any) -> bool: + return isinstance(value, list) and bool(value) and all( + isinstance(m, dict) and "role" in m and "content" in m for m in value) + + +class InspectSamplesProbe: + name = "inspect-samples" + target = "file" + + def fingerprint(self, source: FileSource) -> Fingerprint | None: + rows = first_rows(_records(source, "input")) + with_target = key_share(rows, "input", "target") + if with_target >= 0.5: + return Fingerprint(0.85 * with_target, ("input", "target")) + if key_share(rows, "input") >= 0.8 and not key_share(rows, "ideal"): + return Fingerprint(0.55, ("input (no target)",)) + return None + + def describe(self, source: FileSource) -> Report: + rows = _records(source, "input") or [] + report = _base(source, self.name, KIND_ANSWER_ONLY, rows) + report.set_parts({ + "instruction": key_share(rows, "input") == 1, + "environment": any(k in r for r in rows for k in ("sandbox", "files", "setup")), + "oracle": key_share(rows, "target") == 1, + # The scorer is code in the @task file, not in the samples. + "verifier": False, + "metadata": key_share(rows, "metadata") == 1, + }) + _lint_missing(report, rows, ("input",)) + bad = sum(1 for r in rows if "input" in r and not isinstance(r["input"], str) and not _is_chat(r["input"])) + if bad: + report.problems.append(f"{bad} items have an input that is neither text nor chat messages") + _lint_duplicate_ids(report, rows, "id") + report.details["with_choices"] = sum(1 for r in rows if "choices" in r) + return report + + +class OpenAIEvalsProbe: + name = "openai-evals" + target = "file" + + def fingerprint(self, source: FileSource) -> Fingerprint | None: + if source.suffix not in (".jsonl", ".json"): + return None + rows = first_rows(_records(source, "ideal")) + share = key_share(rows, "input", "ideal") + if share < 0.5: + return None + chat = sum(_is_chat(r.get("input")) for r in rows) / len(rows) + return Fingerprint(0.7 * share + 0.25 * chat, ("input", "ideal", "chat messages" if chat else "")) + + def describe(self, source: FileSource) -> Report: + rows = _records(source, "ideal") or [] + report = _base(source, self.name, KIND_ANSWER_ONLY, rows) + report.set_parts({ + "instruction": key_share(rows, "input") == 1, + "oracle": key_share(rows, "ideal") == 1, + }) + _lint_missing(report, rows, ("input", "ideal")) + chat = sum(1 for r in rows if _is_chat(r.get("input"))) + report.details["chat_inputs"] = chat + return report + + +# --- skill-creator ----------------------------------------------------------- + + +class SkillCreatorEvalsProbe: + name = "skill-creator-evals" + target = "file" + + def fingerprint(self, source: FileSource) -> Fingerprint | None: + data = source.data if source.suffix == ".json" else None + if not isinstance(data, dict) or not isinstance(data.get("evals"), list): + return None + rows = [r for r in data["evals"] if isinstance(r, dict)] + score = 0.5 + 0.2 * ("skill_name" in data) + 0.3 * key_share(first_rows(rows), "prompt") + return Fingerprint(score, ("evals[]", "skill_name" if "skill_name" in data else "", "prompt")) + + def describe(self, source: FileSource) -> Report: + data = source.data if isinstance(source.data, dict) else {} + rows = [r for r in data.get("evals", []) if isinstance(r, dict)] + report = _base(source, self.name, KIND_ANSWER_ONLY, rows) + report.set_parts({ + "instruction": key_share(rows, "prompt") == 1 and bool(rows), + "environment": any(r.get("files") for r in rows), + "oracle": key_share(rows, "expected_output") == 1 and bool(rows), + "verifier": any(r.get("expectations") for r in rows), + "metadata": bool(data.get("skill_name")), + }) + _lint_missing(report, rows, ("prompt",)) + bad = sum(1 for r in rows if not isinstance(r.get("expectations", []), list)) + if bad: + report.problems.append(f"{bad} items have expectations that are not a list") + _lint_duplicate_ids(report, rows, "id") + report.details["skill_name"] = data.get("skill_name") + return report + + +class SkillCreatorTriggerProbe: + name = "skill-creator-trigger" + target = "file" + + def fingerprint(self, source: FileSource) -> Fingerprint | None: + rows = first_rows(source.rows if source.suffix in (".json", ".jsonl") else None) + share = key_share(rows, "query", "should_trigger") + return Fingerprint(0.95 * share, ("query", "should_trigger")) if share >= 0.5 else None + + def describe(self, source: FileSource) -> Report: + rows = source.rows or [] + report = _base(source, self.name, KIND_TRIGGER, rows) + report.set_parts({"instruction": key_share(rows, "query") == 1, + "oracle": key_share(rows, "should_trigger") == 1}) + _lint_missing(report, rows, ("query",)) + bad = sum(1 for r in rows if not isinstance(r.get("should_trigger"), bool)) + if bad: + report.problems.append(f"{bad} items have a should_trigger that is not true/false") + report.details["should_trigger"] = sum(1 for r in rows if r.get("should_trigger") is True) + report.details["should_not_trigger"] = sum(1 for r in rows if r.get("should_trigger") is False) + return report + + +# --- skill-reach ------------------------------------------------------------- + +_REACH_QUERY_KEYS = frozenset({"query_id", "id", "text", "kind", "expected_skill", "acceptable_skills", "notes"}) +_REACH_SET_KEYS = frozenset({"catalog_id", "notes", "provenance", "queries"}) + + +def _reach_rows(source: FileSource) -> list[dict[str, Any]] | None: + data = source.data + if isinstance(data, dict) and isinstance(data.get("queries"), list): + return [q for q in data["queries"] if isinstance(q, dict)] + return source.rows + + +def _lint_reach_query(q: dict[str, Any]) -> str | None: + qid = q.get("query_id", q.get("id", "?")) + unknown = sorted(set(q) - _REACH_QUERY_KEYS) + if unknown: + return f"{qid}: keys skill-reach refuses: {', '.join(unknown)}" + if (q.get("kind") == "out_of_scope") != (q.get("expected_skill") is None): + return f"{qid}: kind={q.get('kind')!r} disagrees with expected_skill={q.get('expected_skill')!r}" + return None + + +class SkillReachProbe: + name = "skill-reach" + target = "file" + + def fingerprint(self, source: FileSource) -> Fingerprint | None: + if source.suffix not in _ROW_SUFFIXES: + return None + rows = first_rows(_reach_rows(source)) + texts = key_share(rows, "text", "expected_skill") + if texts < 0.5: + return None + ids = max(key_share(rows, "query_id"), key_share(rows, "id")) + wrapped = isinstance(source.data, dict) and "queries" in source.data + evidence = ("queries[]" if wrapped else "rows", "text", "expected_skill", "query_id/id" if ids else "") + return Fingerprint(0.5 * texts + 0.3 * ids + 0.2 * wrapped, evidence) + + def describe(self, source: FileSource) -> Report: + rows = _reach_rows(source) or [] + report = _base(source, self.name, KIND_TRIGGER, rows) + data = source.data if isinstance(source.data, dict) else {} + report.set_parts({"instruction": key_share(rows, "text") == 1, + "oracle": key_share(rows, "expected_skill") == 1, + "metadata": bool(data.get("catalog_id") or data.get("provenance"))}) + _lint_missing(report, rows, ("text",)) + extra = sorted(set(data) - _REACH_SET_KEYS) + if extra: + report.problems.append(f"QuerySet keys skill-reach refuses: {', '.join(extra)}") + report.problems.extend(p for p in map(_lint_reach_query, rows) if p) + _lint_duplicate_ids(report, rows, "query_id", "id") + report.details["out_of_scope"] = sum(1 for r in rows if r.get("expected_skill") is None) + return report + + +# --- dokimi questions -------------------------------------------------------- + +_QUESTIONS_STAMPS = ("questions.v1", "questions.v2") + + +class QuestionsProbe: + name = "dokimi-questions" + target = "file" + + def fingerprint(self, source: FileSource) -> Fingerprint | None: + data = source.data if source.suffix == ".json" else None + if isinstance(data, dict) and data.get("schema_version") in _QUESTIONS_STAMPS: + return Fingerprint(1.0, (f"schema_version {data['schema_version']}",)) + if not isinstance(data, dict) or not isinstance(data.get("questions"), list): + return None + rows = first_rows([q for q in data["questions"] if isinstance(q, dict)]) + share = max(key_share(rows, "question"), key_share(rows, "text")) + return Fingerprint(0.4 + 0.4 * share, ("questions[]", "question/text")) if share else None + + def describe(self, source: FileSource) -> Report: + from dokimi.questions.artifact import load_questions + + data = source.data if isinstance(source.data, dict) else {} + rows = [q for q in data.get("questions", []) if isinstance(q, dict)] + kind = KIND_TRIGGER if any("expected_skill" in q for q in rows) else KIND_ANSWER_ONLY + report = _base(source, self.name, kind, rows) + has_text = all(q.get("question") or q.get("text") for q in rows) and bool(rows) + report.set_parts({ + "instruction": has_text, + "oracle": any(q.get("ground_truth_chunk") or q.get("expected_skill") for q in rows), + "verifier": any(q.get("expected_topics") for q in rows), + "metadata": bool(data.get("product") or data.get("library")), + }) + try: + load_questions(source.path) + except Exception as exc: # noqa: BLE001 -- any refusal is a lint finding, the scan goes on + report.problems.append(" ".join(str(exc).split())[:300]) + else: + report.consumers.append(BY_QUESTIONS) + report.details["schema_version"] = data.get("schema_version") + return report diff --git a/dokimi/dokimi/task_formats/detect.py b/dokimi/dokimi/task_formats/detect.py new file mode 100644 index 000000000..f373d7d7a --- /dev/null +++ b/dokimi/dokimi/task_formats/detect.py @@ -0,0 +1,158 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The probe registry and the scan: which eval-task format is each path? + +A directory a directory probe claims (a task) is one item and is not descended +into; any other directory is walked, and each data file in it is offered to the +file probes. When two probes claim a path about equally, the best is reported +and the others are listed as alternatives with their evidence. A path nothing +claims is reported as ``unknown`` with what it holds, never raised. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterator, Sequence +from pathlib import Path +from typing import Any + +from dokimi.task_formats.configs import InspectTaskProbe, PromptfooProbe +from dokimi.task_formats.datasets import ( + InspectSamplesProbe, OpenAIEvalsProbe, QuestionsProbe, SkillCreatorEvalsProbe, + SkillCreatorTriggerProbe, SkillReachProbe, SweBenchProbe, +) +from dokimi.task_formats.dirs import HarborProbe, TaskMdProbe, TerminalBenchV1Probe +from dokimi.task_formats.model import ( + AMBIGUITY_MARGIN, FORMAT_UNKNOWN, KIND_UNKNOWN, MATCH_THRESHOLD, Fingerprint, Probe, Report, +) +from dokimi.task_formats.source import FileSource + +DIR_PROBES: tuple[Probe, ...] = (HarborProbe(), TaskMdProbe(), TerminalBenchV1Probe()) +FILE_PROBES: tuple[Probe, ...] = ( + SweBenchProbe(), InspectSamplesProbe(), OpenAIEvalsProbe(), SkillCreatorEvalsProbe(), + SkillCreatorTriggerProbe(), SkillReachProbe(), QuestionsProbe(), PromptfooProbe(), + InspectTaskProbe(), +) +PROBES: tuple[Probe, ...] = DIR_PROBES + FILE_PROBES + +#: Directories a scan never enters. +SKIP_DIRS = frozenset({".git", ".hg", ".svn", "node_modules", "__pycache__", ".venv", "venv", + ".tox", ".mypy_cache", ".pytest_cache", ".ruff_cache"}) +#: Suffixes a scan offers to the file probes. +SCAN_SUFFIXES = frozenset({".json", ".jsonl", ".yaml", ".yml", ".py"}) +#: A scan stops after this many files and says so. +MAX_SCAN_FILES = 20000 + + +def _claims(probes: Sequence[Probe], subject: Any) -> list[tuple[Fingerprint, Probe]]: + found = [] + for probe in probes: + fp = probe.fingerprint(subject) + if fp is not None and fp.confidence >= MATCH_THRESHOLD: + found.append((fp, probe)) + return sorted(found, key=lambda c: -c[0].confidence) + + +def _evidence(fp: Fingerprint) -> list[str]: + return [e for e in fp.evidence if e] + + +def detect(path: Path, probes: Sequence[Probe] | None = None) -> Report | None: + """The report of the best probe claiming *path*, or ``None`` if none does.""" + path = Path(path) + if probes is None: + probes = DIR_PROBES if path.is_dir() else FILE_PROBES + subject: Any = path if path.is_dir() else FileSource(path) + claims = _claims(probes, subject) + if not claims: + return None + (best_fp, best), rest = claims[0], claims[1:] + report = best.describe(subject) + report.confidence = best_fp.confidence + report.evidence = _evidence(best_fp) + report.alternatives = [ + {"format": p.name, "confidence": round(fp.confidence, 2), "evidence": _evidence(fp)} + for fp, p in rest if round(best_fp.confidence - fp.confidence, 6) <= AMBIGUITY_MARGIN + ] + return report + + +def inspect_path(path: Path) -> list[Report]: + """Every item under *path* (a file, a task directory, or a tree), or one ``unknown``.""" + path = Path(path) + if not path.exists(): + raise FileNotFoundError(f"no such file or directory: {path}") + if path.is_file(): + return [detect(path) or unknown(path)] + scan = _Scan() + reports = list(scan.walk(path)) + if not reports: + reports = [unknown(path)] + if scan.truncated: + reports[-1].problems.append(f"scan stopped after {MAX_SCAN_FILES} files") + return reports + + +class _Scan: + def __init__(self) -> None: + self.files = 0 + self.truncated = False + + def walk(self, directory: Path) -> Iterator[Report]: + task = detect(directory, DIR_PROBES) + if task is not None: + yield task + return + try: + children = sorted(directory.iterdir()) + except OSError: + return + for child in children: + if self.truncated: + return + yield from self._child(child) + + def _child(self, child: Path) -> Iterator[Report]: + if child.is_symlink(): + return + if child.is_dir(): + if child.name not in SKIP_DIRS and not child.name.startswith("."): + yield from self.walk(child) + return + if child.suffix.lower() not in SCAN_SUFFIXES: + return + self.files += 1 + if self.files > MAX_SCAN_FILES: + self.truncated = True + return + report = detect(child, FILE_PROBES) + if report is not None: + yield report + + +# --- unknown ----------------------------------------------------------------- + +_PROMPT_NAMES = re.compile(r"^(instruction|prompt|task|problem|question)s?[\w-]*\.(md|txt|ya?ml|json)$", re.I) +_VERIFIER_NAMES = re.compile(r"^(test|tests|verify|verifier|run-tests|check|grade|grader)[\w-]*(\.\w+)?$", re.I) +_LISTED = 30 + + +def unknown(path: Path) -> Report: + """Best-effort structure of a path no probe claims.""" + report = Report(path, FORMAT_UNKNOWN, KIND_UNKNOWN) + if path.is_dir(): + names = sorted(p.name + ("/" if p.is_dir() else "") for p in path.iterdir()) + report.details["files"] = names[:_LISTED] + report.details["prompt_candidates"] = [n for n in names if _PROMPT_NAMES.match(n)] + report.details["verifier_candidates"] = [n for n in names if _VERIFIER_NAMES.match(n.rstrip("/"))] + return report + source = FileSource(path) + data = source.data + report.problems.extend(source.problems) + if isinstance(data, dict): + report.details["keys"] = sorted(map(str, data))[:_LISTED] + elif source.rows: + report.items = len(source.rows) + report.details["row_keys"] = sorted(map(str, source.rows[0]))[:_LISTED] + return report diff --git a/dokimi/dokimi/task_formats/dirs.py b/dokimi/dokimi/task_formats/dirs.py new file mode 100644 index 000000000..0f2e2ff35 --- /dev/null +++ b/dokimi/dokimi/task_formats/dirs.py @@ -0,0 +1,155 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Probes for task directories: Harbor, BenchFlow task.md, terminal-bench v1. + +Harbor and task-md recognition is :func:`dokimi.harnesses.tasks.task_format` and +their checks are :func:`~dokimi.harnesses.tasks.load_task` and +:func:`~dokimi.harnesses.tasks.harbor_task_problem`: the probe reports what the +runners would do, it does not decide again. +""" + +from __future__ import annotations + +import warnings +from pathlib import Path +from typing import Any + +from dokimi.harnesses import tasks as task_layouts +from dokimi.task_formats.model import ( + BY_BENCHFLOW, BY_DOCKER, BY_HARBOR, KIND_EXECUTABLE, Fingerprint, Report, +) +from dokimi.task_formats.source import FileSource + + +def _any_file(path: Path, *names: str) -> bool: + return any((path / n).is_file() for n in names) + + +def _any_dir(path: Path, *names: str) -> bool: + return any((path / n).is_dir() for n in names) + + +def _load(path: Path, report: Report) -> task_layouts.TaskSpec | None: + """load_task, with its exception (and any warning it raises) as lint problems.""" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + try: + spec = task_layouts.load_task(path) + except (ValueError, OSError) as exc: + report.problems.append(str(exc)) + spec = None + report.problems.extend(str(w.message) for w in caught) + return spec + + +class HarborProbe: + name = task_layouts.FORMAT_HARBOR + target = "dir" + + def fingerprint(self, path: Path) -> Fingerprint | None: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + fmt = task_layouts.task_format(path) + if fmt != task_layouts.FORMAT_HARBOR: + return None + return Fingerprint(1.0, ("task.toml",)) + + def describe(self, path: Path) -> Report: + report = Report(path, self.name, KIND_EXECUTABLE) + spec = _load(path, report) + report.set_parts({ + "instruction": (path / "instruction.md").is_file(), + "environment": (path / "environment").is_dir(), + "oracle": _any_dir(path, "solution", "oracle"), + "verifier": _any_dir(path, "tests", "verifier"), + "metadata": bool(spec and spec.metadata), + }) + if spec is None: + return report + report.details["metadata"] = _summary(spec.metadata) + problem = task_layouts.harbor_task_problem(spec) + if problem: + report.problems.append(f"Harbor would refuse it: {problem}") + else: + report.consumers.append(BY_HARBOR) + if _any_dir(path, "tests", "verifier"): + report.consumers.append(BY_DOCKER) + return report + + +class TaskMdProbe: + name = task_layouts.FORMAT_TASK_MD + target = "dir" + + def fingerprint(self, path: Path) -> Fingerprint | None: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + fmt = task_layouts.task_format(path) + if fmt != task_layouts.FORMAT_TASK_MD: + return None + return Fingerprint(1.0, ("task.md with BenchFlow frontmatter",)) + + def describe(self, path: Path) -> Report: + report = Report(path, self.name, KIND_EXECUTABLE) + spec = _load(path, report) + report.set_parts({ + "instruction": bool(spec and spec.instruction.strip()), + "environment": (path / "environment").is_dir(), + "oracle": _any_dir(path, "oracle", "solution"), + "verifier": _any_dir(path, "verifier", "tests"), + "metadata": bool(spec and spec.metadata), + }) + if spec is None: + return report + report.details["metadata"] = _summary(spec.metadata) + report.details["schema_version"] = spec.schema_version + report.consumers.append(BY_BENCHFLOW) + if _any_dir(path, "verifier", "tests"): + report.consumers.append(BY_DOCKER) + return report + + +# terminal-bench 0.x (laude-institute/terminal-bench, original-tasks/): task.yaml +# holds the instruction and metadata; the container is Dockerfile + +# docker-compose.yaml, the oracle solution.sh (or solution.yaml), the verifier +# run-tests.sh running tests/. +_TB1_SIGNALS = ("run-tests.sh", "docker-compose.yaml", "Dockerfile", "solution.sh", "solution.yaml") + + +class TerminalBenchV1Probe: + name = "terminal-bench-v1" + target = "dir" + + def fingerprint(self, path: Path) -> Fingerprint | None: + if not (path / "task.yaml").is_file(): + return None + seen = tuple(n for n in _TB1_SIGNALS if (path / n).is_file()) + if not seen: + return None + return Fingerprint(min(1.0, 0.5 + 0.1 * len(seen)), ("task.yaml", *seen)) + + def describe(self, path: Path) -> Report: + report = Report(path, self.name, KIND_EXECUTABLE) + source = FileSource(path / "task.yaml") + data = source.data + report.problems.extend(f"task.yaml: {p}" for p in source.problems) + config: dict[str, Any] = data if isinstance(data, dict) else {} + if data is not None and not config: + report.problems.append("task.yaml: not a mapping") + instruction = config.get("instruction") or config.get("descriptions") + report.set_parts({ + "instruction": bool(instruction), + "environment": _any_file(path, "Dockerfile", "docker-compose.yaml"), + "oracle": _any_file(path, "solution.sh", "solution.yaml"), + "verifier": (path / "run-tests.sh").is_file(), + "metadata": any(k in config for k in ("difficulty", "category", "tags")), + }) + if config and not instruction: + report.problems.append("task.yaml: no instruction") + report.details["metadata"] = _summary(config, ("difficulty", "category", "tags", "parser_name")) + return report + + +def _summary(table: dict[str, Any], keys: tuple[str, ...] = ("difficulty", "category", "tags")) -> dict[str, Any]: + return {k: table[k] for k in keys if k in table} diff --git a/dokimi/dokimi/task_formats/model.py b/dokimi/dokimi/task_formats/model.py new file mode 100644 index 000000000..7843aa06f --- /dev/null +++ b/dokimi/dokimi/task_formats/model.py @@ -0,0 +1,91 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""What a format probe returns: a fingerprint and a structure report.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Protocol + +KIND_EXECUTABLE = "executable" # has an environment and a verifier +KIND_ANSWER_ONLY = "answer-only" # prompt/target samples +KIND_TRIGGER = "trigger" # skill-selection queries +KIND_UNKNOWN = "unknown" + +FORMAT_UNKNOWN = "unknown" + +#: The parts a report says are present or missing, in display order. +PARTS = ("instruction", "environment", "oracle", "verifier", "metadata") + +# Labels for what can consume a format today. +BY_HARBOR = "harbor" +BY_DOCKER = "docker-*" +BY_BENCHFLOW = "benchflow" +BY_QUESTIONS = "questions track" +NONE_YET = "none-yet" + +#: A probe at or above this confidence claims the path. +MATCH_THRESHOLD = 0.5 +#: Other claims within this margin of the best one are reported as ambiguous. +AMBIGUITY_MARGIN = 0.15 + + +@dataclass(frozen=True) +class Fingerprint: + """How sure a probe is that a path is its format, and why.""" + + confidence: float + evidence: tuple[str, ...] = () + + +@dataclass +class Report: + """Structure of one detected item (a task directory or a dataset file).""" + + path: Path + format: str + kind: str + confidence: float = 0.0 + evidence: list[str] = field(default_factory=list) + items: int | None = None + present: list[str] = field(default_factory=list) + missing: list[str] = field(default_factory=list) + problems: list[str] = field(default_factory=list) + consumers: list[str] = field(default_factory=list) + alternatives: list[dict[str, Any]] = field(default_factory=list) + details: dict[str, Any] = field(default_factory=dict) + + def set_parts(self, parts: dict[str, bool]) -> None: + """Record which of :data:`PARTS` are present; a part not named is missing.""" + self.present = [p for p in PARTS if parts.get(p)] + self.missing = [p for p in PARTS if not parts.get(p)] + + def as_dict(self) -> dict[str, Any]: + return { + "path": str(self.path), + "format": self.format, + "kind": self.kind, + "confidence": round(self.confidence, 2), + "evidence": list(self.evidence), + "items": self.items, + "present": list(self.present), + "missing": list(self.missing), + "problems": list(self.problems), + "consumers": list(self.consumers) or [NONE_YET], + "alternatives": list(self.alternatives), + "details": dict(self.details), + } + + +class Probe(Protocol): + """One format: recognise it (:meth:`fingerprint`) and describe it (:meth:`describe`).""" + + name: str + #: ``"dir"`` probes look at directories, ``"file"`` probes at files. + target: str + + def fingerprint(self, source: Any) -> Fingerprint | None: ... + + def describe(self, source: Any) -> Report: ... diff --git a/dokimi/dokimi/task_formats/source.py b/dokimi/dokimi/task_formats/source.py new file mode 100644 index 000000000..d239dfa61 --- /dev/null +++ b/dokimi/dokimi/task_formats/source.py @@ -0,0 +1,113 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""A file read once for every probe: parsed data, rows, and what failed to parse. + +Files are data here, never code: JSON, JSONL, YAML and TOML are parsed with safe +loaders (YAML refuses aliases), and a ``.py`` file is only read as text. A parse +failure is a lint problem on the report, not an exception. +""" + +from __future__ import annotations + +import json +import tomllib +from functools import cached_property +from pathlib import Path +from typing import Any + +import yaml + +from dokimi.harnesses.tasks import _load_yaml + +#: Files larger than this are not parsed (their structure is reported as unread). +MAX_BYTES = 64 * 1024 * 1024 + + +class FileSource: + """One candidate file. Parsing is lazy and happens at most once.""" + + def __init__(self, path: Path) -> None: + self.path = Path(path) + self.problems: list[str] = [] + + @property + def suffix(self) -> str: + return self.path.suffix.lower() + + @cached_property + def text(self) -> str | None: + try: + if self.path.stat().st_size > MAX_BYTES: + self.problems.append(f"larger than {MAX_BYTES // (1024 * 1024)} MiB: not read") + return None + return self.path.read_text(encoding="utf-8", errors="replace") + except OSError as exc: + self.problems.append(f"cannot read: {exc}") + return None + + @cached_property + def data(self) -> Any: + """The parsed document (a JSONL file parses to its list of rows), or ``None``.""" + text = self.text + if text is None or self.suffix == ".py": + return None + try: + return _parse(self.suffix, text, self.problems) + except (ValueError, yaml.YAMLError, RecursionError) as exc: + self.problems.append(f"not valid {_LANG.get(self.suffix, 'data')}: {_one_line(exc)}") + return None + + @cached_property + def rows(self) -> list[dict[str, Any]] | None: + """The records of a list-shaped document (JSONL, a JSON/YAML list), else ``None``.""" + data = self.data + if not isinstance(data, list): + return None + return [r for r in data if isinstance(r, dict)] + + +_LANG = {".json": "JSON", ".jsonl": "JSONL", ".yaml": "YAML", ".yml": "YAML", ".toml": "TOML"} + + +def _one_line(exc: BaseException) -> str: + return " ".join(str(exc).split())[:200] + + +def _parse(suffix: str, text: str, problems: list[str]) -> Any: + if suffix == ".jsonl": + return _parse_jsonl(text, problems) + if suffix == ".json": + return json.loads(text) + if suffix == ".toml": + return tomllib.loads(text) + return _load_yaml(text) + + +def _parse_jsonl(text: str, problems: list[str]) -> list[Any]: + rows: list[Any] = [] + bad = 0 + for number, line in enumerate(text.splitlines(), 1): + if not line.strip(): + continue + try: + rows.append(json.loads(line)) + except ValueError as exc: + bad += 1 + if bad <= 3: + problems.append(f"line {number}: not valid JSON: {_one_line(exc)}") + if bad > 3: + problems.append(f"{bad - 3} more lines are not valid JSON") + return rows + + +def first_rows(rows: list[dict[str, Any]] | None, n: int = 50) -> list[dict[str, Any]]: + """The rows a fingerprint looks at: the first *n*, so a large file costs no more.""" + return (rows or [])[:n] + + +def key_share(rows: list[dict[str, Any]], *keys: str) -> float: + """Share of *rows* that hold every one of *keys*.""" + if not rows: + return 0.0 + return sum(all(k in r for k in keys) for r in rows) / len(rows) diff --git a/dokimi/tests/test_import_architecture.py b/dokimi/tests/test_import_architecture.py index 5df0aac9f..978567bbc 100644 --- a/dokimi/tests/test_import_architecture.py +++ b/dokimi/tests/test_import_architecture.py @@ -52,7 +52,7 @@ "dokimi.datasets_export", "dokimi.explore", "dokimi.external", "dokimi.harnesses", "dokimi.novelty", "dokimi.orchestrator", "dokimi.report", "dokimi.repos", "dokimi.runner", "dokimi.schema_check", "dokimi.schema_compat", "dokimi.subjects", - "dokimi.tool_preference", + "dokimi.task_formats", "dokimi.tool_preference", )), # The LLM evaluation engine: answering, judging, panels, arms, questions, personas. ("evaluation", ( diff --git a/dokimi/tests/test_task_format_detect.py b/dokimi/tests/test_task_format_detect.py new file mode 100644 index 000000000..c5c3d77f7 --- /dev/null +++ b/dokimi/tests/test_task_format_detect.py @@ -0,0 +1,387 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for eval-task format detection (``dokimi.task_formats``, ``tasks inspect``). + +Each fixture is hand-written from the upstream shape it imitates: + +- terminal-bench v1: laude-institute/terminal-bench ``original-tasks/hello-world``. +- SWE-bench: ``SWEbenchInstance`` in SWE-bench/SWE-bench ``swebench/types.py`` + (``FAIL_TO_PASS`` as the JSON string the HF datasets ship). +- Inspect AI: ``Sample`` fields (``input``, ``target``, ``id``, ``sandbox``) and an + ``examples/hello_world.py``-style ``@task`` file. +- OpenAI evals: ``evals/registry/data/coqa/match.jsonl`` (chat ``input``, ``ideal``). +- skill-creator: ``references/schemas.md`` ``evals.json`` and the ``SKILL.md`` trigger set. +- skill-reach: ``QuerySet`` in google/skill-reach ``src/reach/queries.py``. +- promptfoo: ``examples/*/promptfooconfig.yaml``. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from dokimi.cli import build_parser +from dokimi.external.trigger_set import TriggerQuery, to_reach, to_skill_creator +from dokimi.harnesses import discover_tasks +from dokimi.task_formats import detect, inspect_path + + +def _write(path: Path, text: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + +def _jsonl(path: Path, rows: list) -> Path: + return _write(path, "".join(json.dumps(r) + "\n" for r in rows)) + + +def _harbor(root: Path, name: str = "hb") -> Path: + task = root / name + _write(task / "task.toml", '[metadata]\ndifficulty = "easy"\ntags = ["x"]\n') + _write(task / "instruction.md", "Fix it.") + (task / "environment").mkdir() + _write(task / "solution" / "solve.sh", "#!/bin/bash\n") + _write(task / "tests" / "test.sh", "#!/bin/bash\n") + return task + + +def _task_md(root: Path, name: str = "tm") -> Path: + task = root / name + _write(task / "task.md", "---\nschema_version: '1.3'\nmetadata:\n difficulty: medium\n" + "verifier:\n timeout_sec: 60\n---\n\nDo the thing.\n") + _write(task / "oracle" / "solve.sh", "#!/bin/bash\n") + _write(task / "verifier" / "test.sh", "#!/bin/bash\n") + return task + + +def _tb1(root: Path, name: str = "hello-world") -> Path: + task = root / name + _write(task / "task.yaml", "instruction: |\n Create hello.txt with 'Hello, world!'.\n" + "author_name: a\ndifficulty: easy\ncategory: file-operations\n" + "tags:\n - file-operations\nparser_name: pytest\n") + _write(task / "Dockerfile", "FROM ubuntu:24.04\nWORKDIR /app\n") + _write(task / "docker-compose.yaml", "services: {}\n") + _write(task / "run-tests.sh", "#!/bin/bash\npytest $TEST_DIR/test_outputs.py\n") + _write(task / "solution.sh", "echo 'Hello, world!' > hello.txt\n") + _write(task / "tests" / "test_outputs.py", "def test_hello():\n pass\n") + return task + + +_SWE_ROW = { + "repo": "astropy/astropy", "instance_id": "astropy__astropy-12907", + "base_commit": "d16bfe05a744909de4b27f5875fe0d4ed41ce607", "patch": "diff --git a/x b/x\n", + "test_patch": "diff --git a/t b/t\n", "problem_statement": "separability_matrix is wrong", + "hints_text": "", "created_at": "2022-03-03T15:14:54Z", "version": "4.3", + "FAIL_TO_PASS": '["astropy/modeling/tests/test_separable.py::test_separable[compound_model6]"]', + "PASS_TO_PASS": "[]", "environment_setup_commit": "298ccb478e6bf092953bca67a3d29dc6c35f6752", +} +_CHAT = [{"role": "system", "content": "Answer briefly."}, {"role": "user", "content": "2+2?"}] + + +def _one(path: Path): + reports = inspect_path(path) + assert len(reports) == 1, [r.as_dict() for r in reports] + return reports[0] + + +# ── one probe per format ────────────────────────────────────────────────────── + + +def test_harbor_task_is_executable_and_consumable_by_harbor_and_docker(tmp_path): + r = _one(_harbor(tmp_path)) + assert (r.format, r.kind, r.consumers) == ("harbor", "executable", ["harbor", "docker-*"]) + assert r.missing == [] and r.problems == [] + + +def test_harbor_task_without_environment_reports_why_harbor_refuses_it(tmp_path): + task = _harbor(tmp_path) + (task / "environment").rmdir() + r = _one(task) + assert "environment" in r.missing + assert r.consumers == ["docker-*"] + assert any("Harbor would refuse it: no environment/" in p for p in r.problems) + + +def test_task_md_task_is_consumable_by_benchflow(tmp_path): + r = _one(_task_md(tmp_path)) + assert (r.format, r.kind) == ("task-md", "executable") + assert r.consumers == ["benchflow", "docker-*"] + assert r.details["schema_version"] == "1.3" + + +def test_terminal_bench_v1_task_has_every_part_and_no_consumer_yet(tmp_path): + r = _one(_tb1(tmp_path)) + assert (r.format, r.kind, r.missing) == ("terminal-bench-v1", "executable", []) + assert r.as_dict()["consumers"] == ["none-yet"] + assert r.details["metadata"]["difficulty"] == "easy" + + +def test_swe_bench_jsonl_counts_instances_and_parses_string_test_lists(tmp_path): + other = {**_SWE_ROW, "instance_id": "astropy__astropy-14182"} + r = _one(_jsonl(tmp_path / "lite.jsonl", [_SWE_ROW, other])) + assert (r.format, r.kind, r.items) == ("swe-bench", "executable", 2) + assert r.missing == [] and r.problems == [] + assert r.details["repos"] == ["astropy/astropy"] + + +def test_swe_bench_lints_missing_fields_and_duplicate_ids(tmp_path): + broken = {k: v for k, v in _SWE_ROW.items() if k != "FAIL_TO_PASS"} + r = _one(_jsonl(tmp_path / "bad.jsonl", [_SWE_ROW, broken, {**_SWE_ROW, "FAIL_TO_PASS": "nope"}])) + assert r.format == "swe-bench" + assert "1 of 3 items have no FAIL_TO_PASS" in r.problems + assert "1 items have a FAIL_TO_PASS that is not a list of test ids" in r.problems + assert "duplicate ids: astropy__astropy-12907" in r.problems + + +def test_inspect_samples_jsonl_is_answer_only(tmp_path): + rows = [{"id": 1, "input": "Capital of France?", "target": "Paris"}, + {"id": 2, "input": _CHAT, "target": "4", "sandbox": "docker"}] + r = _one(_jsonl(tmp_path / "samples.jsonl", rows)) + assert (r.format, r.kind, r.items) == ("inspect-samples", "answer-only", 2) + assert "environment" in r.present and "verifier" in r.missing + + +def test_inspect_task_file_is_scanned_not_imported(tmp_path): + marker = tmp_path / "imported" + src = ( + "from inspect_ai import Task, task\nfrom inspect_ai.dataset import Sample\n" + f"open({str(marker)!r}, 'w').write('ran')\n\n" + "@task\ndef hello():\n return Task(dataset=[Sample(input='hi', target='hi')],\n" + " solver=[generate()], scorer=exact(), sandbox='docker')\n" + ) + r = _one(_write(tmp_path / "hello.py", src)) + assert (r.format, r.kind, r.items) == ("inspect-task", "executable", 1) + assert {"instruction", "environment", "verifier"} <= set(r.present) + assert not marker.exists() + + +def test_python_without_inspect_import_is_not_an_inspect_task(tmp_path): + r = _one(_write(tmp_path / "x.py", "@task\ndef f():\n pass\n")) + assert r.format == "unknown" + + +def test_openai_evals_chat_input_with_ideal(tmp_path): + r = _one(_jsonl(tmp_path / "match.jsonl", [{"input": _CHAT, "ideal": "4"}] * 3)) + assert (r.format, r.kind, r.items) == ("openai-evals", "answer-only", 3) + assert r.details["chat_inputs"] == 3 + assert r.alternatives == [] + + +def test_skill_creator_evals_json(tmp_path): + data = {"skill_name": "pdf", "evals": [ + {"id": 1, "prompt": "Extract the tables", "expected_output": "a CSV", + "files": ["evals/files/a.pdf"], "expectations": ["The output includes X"]}, + {"id": 1, "prompt": "Split it", "expectations": "not a list"}, + ]} + r = _one(_write(tmp_path / "evals" / "evals.json", json.dumps(data))) + assert (r.format, r.kind, r.items) == ("skill-creator-evals", "answer-only", 2) + assert r.details["skill_name"] == "pdf" + assert "1 items have expectations that are not a list" in r.problems + assert "duplicate ids: 1" in r.problems + + +def _queries() -> list[TriggerQuery]: + return [TriggerQuery(id="q1", text="load my pdf", expected_skill="pdf"), + TriggerQuery(id="q2", text="what's the weather", expected_skill=None)] + + +def test_skill_creator_trigger_set_written_by_dokimi_is_detected(tmp_path): + path = _write(tmp_path / "trigger.json", json.dumps(to_skill_creator(_queries(), "pdf"))) + r = _one(path) + assert (r.format, r.kind, r.items) == ("skill-creator-trigger", "trigger", 2) + assert (r.details["should_trigger"], r.details["should_not_trigger"]) == (1, 1) + + +def test_skill_reach_queryset_written_by_dokimi_is_detected_and_clean(tmp_path): + path = _write(tmp_path / "queries.json", json.dumps(to_reach(_queries(), "cat"))) + r = _one(path) + assert (r.format, r.kind, r.items) == ("skill-reach", "trigger", 2) + assert r.problems == [] + assert r.details["out_of_scope"] == 1 + + +def test_skill_reach_lints_what_reach_would_refuse(tmp_path): + data = {"queries": [{"query_id": "a", "text": "t", "expected_skill": None}, + {"query_id": "a", "text": "u", "expected_skill": "s", "extra": 1}], + "owner": "me"} + r = _one(_write(tmp_path / "q.json", json.dumps(data))) + assert r.format == "skill-reach" + assert "QuerySet keys skill-reach refuses: owner" in r.problems + assert "a: kind=None disagrees with expected_skill=None" in r.problems + assert "a: keys skill-reach refuses: extra" in r.problems + assert "duplicate ids: a" in r.problems + + +def test_promptfoo_config(tmp_path): + cfg = ("description: suite\nprompts:\n - 'Answer: {{query}}'\nproviders:\n - openai:gpt-4o\n" + "tests:\n - vars: {query: hi}\n assert:\n - type: contains\n value: hi\n" + " - vars: {query: yo}\n") + r = _one(_write(tmp_path / "promptfooconfig.yaml", cfg)) + assert (r.format, r.kind, r.items) == ("promptfoo", "answer-only", 2) + assert r.present == ["instruction", "environment", "verifier", "metadata"] + + +def test_promptfoo_tests_from_a_file_reference(tmp_path): + cfg = "prompts: [p.txt]\nproviders: [echo]\ntests: file://tests.csv\n" + r = _one(_write(tmp_path / "eval.yaml", cfg)) + assert (r.format, r.items, r.details["tests_from"]) == ("promptfoo", None, "file://tests.csv") + + +def test_dokimi_questions_v2_is_consumable_by_the_questions_track(tmp_path): + data = {"schema_version": "questions.v2", "product": "p", + "questions": [{"id": "q1", "question": "How?", "expected_topics": ["a"]}]} + r = _one(_write(tmp_path / "golden.json", json.dumps(data))) + assert (r.format, r.kind, r.items, r.consumers) == ("dokimi-questions", "answer-only", 1, ["questions track"]) + assert r.problems == [] + + +def test_dokimi_questions_with_expected_skill_are_trigger_and_v1_loads(tmp_path): + data = {"questions": [{"id": "q1", "text": "How?", "expected_skill": "pdf"}]} + r = _one(_write(tmp_path / "v1.json", json.dumps(data))) + assert (r.format, r.kind, r.consumers) == ("dokimi-questions", "trigger", ["questions track"]) + + +def test_dokimi_questions_the_loader_refuses_is_a_problem_not_a_consumer(tmp_path): + data = {"schema_version": "questions.v2", "questions": [{"id": "q1", "question": ""}]} + r = _one(_write(tmp_path / "bad.json", json.dumps(data))) + assert r.format == "dokimi-questions" + assert r.consumers == [] and r.problems + + +# ── ambiguity, unknown, malformed ───────────────────────────────────────────── + + +def test_rows_with_target_and_ideal_list_the_other_format_as_ambiguous(tmp_path): + rows = [{"input": _CHAT, "target": "4", "ideal": "4"}] * 2 + r = _one(_jsonl(tmp_path / "both.jsonl", rows)) + assert r.format == "openai-evals" + assert [a["format"] for a in r.alternatives] == ["inspect-samples"] + assert r.alternatives[0]["evidence"] == ["input", "target"] + + +def test_directory_with_task_toml_and_task_yaml_lists_terminal_bench_as_alternative(tmp_path): + task = _harbor(tmp_path) + _write(task / "task.yaml", "instruction: hi\n") + _write(task / "run-tests.sh", "#!/bin/bash\n") + r = _one(task) + assert r.format == "harbor" + assert [a["format"] for a in r.alternatives] == [] # 0.6 is not within the margin of 1.0 + assert detect(task).format == "harbor" + + +def test_unknown_directory_lists_prompt_and_verifier_candidates(tmp_path): + _write(tmp_path / "prompt.md", "do x") + _write(tmp_path / "check.sh", "exit 0") + _write(tmp_path / "notes.txt", "n") + r = _one(tmp_path) + assert (r.format, r.kind) == ("unknown", "unknown") + assert r.details["prompt_candidates"] == ["prompt.md"] + assert r.details["verifier_candidates"] == ["check.sh"] + + +def test_unknown_jsonl_reports_row_keys(tmp_path): + r = _one(_jsonl(tmp_path / "qa.jsonl", [{"question": "q", "answer": "a"}])) + assert (r.format, r.items, r.details["row_keys"]) == ("unknown", 1, ["answer", "question"]) + + +@pytest.mark.parametrize("name, text, problem", [ + ("bad.json", "{not json", "not valid JSON"), + ("bad.yaml", "a: [1, 2\n", "not valid YAML"), + ("alias.yaml", "a: &x [1]\nb: *x\n", "not valid YAML"), + ("bad.jsonl", '{"input": "a", "target": "b"}\n{oops\n', "line 2: not valid JSON"), +]) +def test_malformed_data_file_is_a_lint_problem_not_a_crash(tmp_path, name, text, problem): + r = _one(_write(tmp_path / name, text)) + assert any(problem in p for p in r.problems), r.problems + + +def test_malformed_jsonl_keeps_its_good_rows(tmp_path): + r = _one(_write(tmp_path / "s.jsonl", '{"input": "a", "target": "b"}\n{oops\n')) + assert (r.format, r.items) == ("inspect-samples", 1) + + +def test_malformed_task_toml_is_a_problem(tmp_path): + task = _harbor(tmp_path) + _write(task / "task.toml", "[metadata\n") + r = _one(task) + assert r.format == "harbor" + assert any("not valid TOML" in p for p in r.problems) + + +def test_malformed_tb1_task_yaml_is_a_problem(tmp_path): + task = _tb1(tmp_path) + _write(task / "task.yaml", "instruction: [\n") + r = _one(task) + assert r.format == "terminal-bench-v1" + assert any(p.startswith("task.yaml: not valid YAML") for p in r.problems) + + +# ── scan ────────────────────────────────────────────────────────────────────── + + +def test_scan_of_a_mixed_tree_reports_each_item_once(tmp_path): + _harbor(tmp_path / "tasks", "a") + _task_md(tmp_path / "tasks", "b") + _tb1(tmp_path / "tb") + _jsonl(tmp_path / "data" / "swe.jsonl", [_SWE_ROW]) + _write(tmp_path / "data" / "package.json", '{"name": "x"}') # not an eval: skipped + _write(tmp_path / ".git" / "q.json", json.dumps(to_reach(_queries()))) # never entered + found = {(r.path.relative_to(tmp_path).as_posix(), r.format) for r in inspect_path(tmp_path)} + assert found == {("tasks/a", "harbor"), ("tasks/b", "task-md"), + ("tb/hello-world", "terminal-bench-v1"), ("data/swe.jsonl", "swe-bench")} + + +def test_inspect_missing_path_raises(tmp_path): + with pytest.raises(FileNotFoundError): + inspect_path(tmp_path / "nope") + + +def test_existing_discovery_is_unchanged_by_new_formats_in_the_root(tmp_path): + _harbor(tmp_path, "a") + _task_md(tmp_path, "b") + _tb1(tmp_path, "c") + assert [(t.name, t.format) for t in discover_tasks(tmp_path)] == [("a", "harbor"), ("b", "task-md")] + + +# ── CLI ─────────────────────────────────────────────────────────────────────── + + +def _run(argv: list[str]) -> None: + args = build_parser().parse_args(argv) + args.func(args) + + +def test_cli_inspect_prints_a_table_line_per_item(tmp_path, capsys): + _harbor(tmp_path) + _jsonl(tmp_path / "m.jsonl", [{"input": _CHAT, "ideal": "4"}]) + _run(["tasks", "inspect", str(tmp_path)]) + out = capsys.readouterr().out.splitlines() + assert out[0] == f"{tmp_path / 'hb'}\tharbor\texecutable\t-> harbor, docker-*" + assert f"{tmp_path / 'm.jsonl'}\topenai-evals\tanswer-only\t1 item\t-> none-yet" in out + assert out[-1] == "2 items: harbor 1, openai-evals 1" + + +def test_cli_inspect_json(tmp_path, capsys): + _tb1(tmp_path) + _run(["tasks", "inspect", str(tmp_path), "--json"]) + payload = json.loads(capsys.readouterr().out) + assert [i["format"] for i in payload["items"]] == ["terminal-bench-v1"] + assert payload["items"][0]["consumers"] == ["none-yet"] + + +def test_cli_inspect_shows_ambiguity_and_problems(tmp_path, capsys): + _write(tmp_path / "x.jsonl", '{"input": "a", "target": "b", "ideal": "b"}\n{bad\n') + _run(["tasks", "inspect", str(tmp_path / "x.jsonl")]) + out = capsys.readouterr().out + assert " ambiguous: also openai-evals" in out + assert " problem: line 2: not valid JSON" in out + + +def test_cli_inspect_missing_path_exits_with_a_message(tmp_path): + with pytest.raises(SystemExit) as excinfo: + _run(["tasks", "inspect", str(tmp_path / "nope")]) + assert "no such file or directory" in str(excinfo.value) From f7abf7c192b9fe51705c30b7c69187939a693a86 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 24 Oct 2026 09:54:55 -0700 Subject: [PATCH 02/13] feat(catalog): read trigger sets back and list skill package formats Readers for the three trigger-set formats dokimi already writes (skill-reach QuerySet, skill-creator / agentskills.io eval queries, questions with expected_skill) into TriggerQuery, held to the writers by round-trip tests. catalog trigger-set --trigger-set relints an imported set against the catalog and hands it to the skill-catalog: arm; arms run points external sets at it. catalog formats names the skill/rule packages of other ecosystems (Cursor rules, Copilot instructions, Gemini extensions, Claude plugins) next to agentskills.io SKILL.md, with spec lint and the selecting mechanism. Signed-off-by: Nikolay Petrov --- ...0261024_120000_napetrov_trigger_formats.md | 4 + dokimi/dokimi/commands/arms.py | 25 ++ dokimi/dokimi/commands/catalog.py | 111 ++++++++- dokimi/dokimi/external/trigger_set.py | 147 +++++++++++ dokimi/dokimi/skills/formats.py | 228 ++++++++++++++++++ dokimi/dokimi/task_formats/datasets.py | 6 +- dokimi/dokimi/task_formats/model.py | 2 + dokimi/scripts/growth_baseline.json | 2 +- dokimi/tests/test_skill_formats.py | 107 ++++++++ dokimi/tests/test_trigger_set_readers.py | 169 +++++++++++++ 10 files changed, 786 insertions(+), 15 deletions(-) create mode 100644 dokimi/changelog.d/20261024_120000_napetrov_trigger_formats.md create mode 100644 dokimi/dokimi/skills/formats.py create mode 100644 dokimi/tests/test_skill_formats.py create mode 100644 dokimi/tests/test_trigger_set_readers.py diff --git a/dokimi/changelog.d/20261024_120000_napetrov_trigger_formats.md b/dokimi/changelog.d/20261024_120000_napetrov_trigger_formats.md new file mode 100644 index 000000000..80294970b --- /dev/null +++ b/dokimi/changelog.d/20261024_120000_napetrov_trigger_formats.md @@ -0,0 +1,4 @@ +### Added + +- **Read trigger sets back** — `dokimi catalog trigger-set --trigger-set PATH[=TARGET]` reads a google/skill-reach QuerySet, an anthropics skill-creator / agentskills.io `[{query, should_trigger}]` set (TARGET names its skill), or a questions set with `expected_skill`, relints it against `--catalog`, and writes any of the three formats, so an imported set runs on the `skill-catalog:` arm. `arms run` given a skill-reach or skill-creator file names the conversion command; `tasks inspect` lists `catalog trigger-set` as the consumer of both formats. +- **Skill package formats** — `dokimi catalog formats --root DIR [--json]` lists Agent Skills `SKILL.md` packages (with the host read from the path, the agentskills.io name/description limits linted, and Codex `agents/openai.yaml` noted), Cursor `.mdc` rules, Copilot `*.instructions.md`, Gemini CLI `gemini-extension.json` and Claude Code `.claude-plugin/plugin.json` bundles, with how each is selected (description, globs, always, install). diff --git a/dokimi/dokimi/commands/arms.py b/dokimi/dokimi/commands/arms.py index e07da1f84..e540b8274 100644 --- a/dokimi/dokimi/commands/arms.py +++ b/dokimi/dokimi/commands/arms.py @@ -450,6 +450,7 @@ def _load_questions(path: str) -> list[dict]: """ from dokimi.questions.artifact import load_questions + _refuse_external_trigger_set(path) questions = load_questions(path).questions ids = [question_id(q) for q in questions] for i, qid in enumerate(ids): @@ -461,6 +462,30 @@ def _load_questions(path: str) -> list[dict]: return questions +def _refuse_external_trigger_set(path: str) -> None: + """Point a skill-reach or skill-creator trigger set at the command that reads it. + + Those sets are read through ``catalog trigger-set --trigger-set``, which also + lints them against the catalog the arm will offer; a questions loader that + guessed at them would drop that lint and, for skill-creator, the target skill. + """ + import json + + from dokimi.external.trigger_set import FORMAT_QUESTIONS, detect_trigger_format + + try: + data = json.loads(Path(path).read_text(encoding="utf-8")) + except (OSError, ValueError): + return # the questions loader reports it + fmt = detect_trigger_format(data) + if fmt and fmt != FORMAT_QUESTIONS: + target = "=" if fmt == "skill-creator" else "" + raise ValueError( + f"{path} is a {fmt} trigger set, not a questions set; convert it with " + f"`dokimi catalog trigger-set --catalog --trigger-set {path}{target} " + f"--format questions --out q.json`, then run the skill-catalog: arm on q.json") + + def _refuse_duplicate_ids(ids: list, path: str) -> None: """Raise on a repeated id.""" counts = Counter(qid for qid in ids if qid is not None) diff --git a/dokimi/dokimi/commands/catalog.py b/dokimi/dokimi/commands/catalog.py index a365cb210..75fdcef44 100644 --- a/dokimi/dokimi/commands/catalog.py +++ b/dokimi/dokimi/commands/catalog.py @@ -67,7 +67,6 @@ def cmd_catalog_trigger_set(args: argparse.Namespace) -> None: from dokimi.external import trigger_set as ts from dokimi.skills import discover_skills - from dokimi.utils import read_json_input catalog_root = Path(args.catalog) if not catalog_root.is_dir(): @@ -80,16 +79,11 @@ def cmd_catalog_trigger_set(args: argparse.Namespace) -> None: if args.target not in {s.name for s in catalog}: raise SystemExit(f"--target {args.target!r} is not a skill in --catalog " f"(have: {', '.join(sorted(s.name for s in catalog))})") - sources = [] - for spec in args.questions: - path, _, rule = spec.rpartition("=") # labels never contain '=', paths may - if not path or not rule: - raise SystemExit(f"--questions needs PATH=LABEL (a skill name, 'none' or '@field'): {spec}") - data = read_json_input(path, flag="--questions") - questions = data.get("questions", data) if isinstance(data, dict) else data - if not isinstance(questions, list) or not all(isinstance(q, dict) for q in questions): - raise SystemExit(f"{path}: expected a list of question objects") - sources.append((questions, rule)) + sources = _question_sources(args.questions or []) + sources += _trigger_set_sources(args.trigger_set or [], catalog) + if not sources: + raise SystemExit("give --questions PATH=LABEL or --trigger-set PATH[=TARGET]") + specs = list(args.questions or []) + list(args.trigger_set or []) rewrite = None if args.rewrite: @@ -107,7 +101,7 @@ def rewrite(text: str) -> str: # An @field rule that resolves for no question in its source is a typo in the # path, not a set of negatives; refuse it rather than write "nothing loads". start = 0 - for (qs, rule), spec in zip(sources, args.questions, strict=True): + for (qs, rule), spec in zip(sources, specs, strict=True): chunk = queries[start:start + len(qs)] start += len(qs) if rule.startswith("@") and chunk and not any(q.labelled for q in chunk): @@ -143,6 +137,82 @@ def rewrite(text: str) -> str: print(f"wrote {args.out} ({args.format}, {n_out} queries, catalog of {len(catalog)})") +def cmd_catalog_formats(args: argparse.Namespace) -> None: + import json + + from dokimi.skills import formats + + root = Path(args.root) + if not root.is_dir(): + raise SystemExit(f"--root is not a directory: {root}") + packages = formats.detect_skill_packages(root) + summary = formats.summarize(packages) + if args.json: + print(json.dumps({"root": str(root), "summary": summary, + "packages": [p.to_dict(root) for p in packages]}, indent=2)) + return + for p in packages: + problems = "; ".join(p.problems) or "ok" + print(f"{p.format}\t{p.name or '-'}\t{p.selected_by}\t{p.host or '-'}\t" + f"{p.path.relative_to(root).as_posix()}\t{problems}") + print() + for fmt, row in summary.items(): + print(f"# {fmt}: {row['n']} ({row['loadable']} loadable as skills, " + f"{row['with_problems']} with problems); selected by {row['selected_by']}") + + +def _question_sources(specs: list) -> list: + """Each ``--questions PATH=LABEL`` as a (questions, rule) source.""" + from dokimi.utils import read_json_input + + sources = [] + for spec in specs: + path, _, rule = spec.rpartition("=") # labels never contain '=', paths may + if not path or not rule: + raise SystemExit(f"--questions needs PATH=LABEL (a skill name, 'none' or '@field'): {spec}") + data = read_json_input(path, flag="--questions") + questions = data.get("questions", data) if isinstance(data, dict) else data + if not isinstance(questions, list) or not all(isinstance(q, dict) for q in questions): + raise SystemExit(f"{path}: expected a list of question objects") + sources.append((questions, rule)) + return sources + + +def _trigger_set_sources(specs: list, catalog) -> list: + """Each ``--trigger-set PATH[=TARGET]`` read back as a (questions, rule) source. + + The set is read into the canonical trigger model and handed on as questions + labelled by their own ``expected_skill``, so it is linted (and rewritten) like + any other source. TARGET names the skill a skill-creator set was written for. + """ + from dokimi.external import trigger_set as ts + from dokimi.utils import read_json_input + + names = {s.name for s in catalog} + out = [] + for spec in specs: + path, target = spec, None + if not Path(spec).exists() and "=" in spec: + path, _, target = spec.rpartition("=") + data = read_json_input(path, flag="--trigger-set") + try: + fmt, queries = ts.read_trigger_set(data, target=target) + except ValueError as exc: + raise SystemExit(f"{path}: {exc}") from exc + if fmt == ts.FORMAT_SKILL_CREATOR: + if target not in names: + raise SystemExit(f"{spec}: skill-creator target {target!r} is not a skill in " + f"--catalog (have: {', '.join(sorted(names))})") + if len(names) > 1: + print(f"note: {path}: skill-creator negatives are read as 'no skill loads'; in a " + f"catalog of {len(names)} a rival load counts as a false trigger", file=sys.stderr) + elif target: + raise SystemExit(f"{spec}: =TARGET applies to skill-creator sets only, this is {fmt}") + print(f"read {path} as {fmt} ({len(queries)} queries)", file=sys.stderr) + out.append((ts.to_questions(queries)["questions"], "@expected_skill")) + return out + + def register(sub, positive_int) -> None: """Add the `catalog` subcommand group.""" @@ -167,16 +237,31 @@ def register(sub, positive_int) -> None: ) scan_p.set_defaults(func=cmd_catalog_scan) + fm_p = catalog_sub.add_parser( + "formats", + help="List the skill/rule packages under --root by ecosystem (Agent Skills, Cursor " + "rules, Copilot instructions, Gemini extensions, Claude plugins)", + ) + fm_p.add_argument("--root", required=True, help="Local checkout to scan") + fm_p.add_argument("--json", action="store_true", help="Print JSON instead of a table") + fm_p.set_defaults(func=cmd_catalog_formats) + ts_p = catalog_sub.add_parser( "trigger-set", help="Build a skill-selection query set (labels + leakage/framing lint) from question sets", ) ts_p.add_argument("--catalog", required=True, help="Directory of the skills the agent will see") ts_p.add_argument( - "--questions", action="append", required=True, metavar="PATH=LABEL", + "--questions", action="append", metavar="PATH=LABEL", help="Question file and its label: a skill name, 'none' (no skill should load), or " "'@field.path' read per question (e.g. @metadata.skill_under_test). Repeatable.", ) + ts_p.add_argument( + "--trigger-set", action="append", dest="trigger_set", metavar="PATH[=TARGET]", + help="An existing trigger set to read back: a skill-reach QuerySet, a skill-creator / " + "agentskills.io [{query, should_trigger}] set (=TARGET names its skill), or questions " + "with expected_skill. Repeatable; combines with --questions.", + ) ts_p.add_argument("--format", choices=["reach", "skill-creator", "questions"], default="reach", help="reach: google/skill-reach QuerySet; skill-creator: one target's " "eval set; questions: a questions artifact for the skill-catalog: arm") diff --git a/dokimi/dokimi/external/trigger_set.py b/dokimi/dokimi/external/trigger_set.py index 839fdfe7d..441c6d197 100644 --- a/dokimi/dokimi/external/trigger_set.py +++ b/dokimi/dokimi/external/trigger_set.py @@ -24,6 +24,10 @@ (``QuerySet`` JSON), anthropics/skills skill-creator (``[{query, should_trigger}]`` for one target skill), and this repo's questions artifact (for the ``skill-catalog:`` arm; saved with :func:`~dokimi.questions.artifact.save_questions_artifact`). + +Readers take the same three formats back to :class:`TriggerQuery` (the +canonical model), so a set authored for one tool can be linted against a +catalog and run by the ``skill-catalog:`` arm, or re-written for another tool. """ from __future__ import annotations @@ -257,3 +261,146 @@ def to_questions(queries: Sequence[TriggerQuery]) -> Dict[str, Any]: row["source_text"] = q.source_text rows.append(row) return {"questions": rows} + + +# --- readers ----------------------------------------------------------------- + +FORMAT_REACH = "reach" +FORMAT_SKILL_CREATOR = "skill-creator" +FORMAT_QUESTIONS = "questions" +TRIGGER_FORMATS = (FORMAT_REACH, FORMAT_SKILL_CREATOR, FORMAT_QUESTIONS) + + +def detect_trigger_format(data: Any) -> Optional[str]: + """Which trigger-set format *data* (parsed JSON) is, or ``None``. + + By shape only: a ``{"queries": [...]}`` object is a skill-reach QuerySet, a list + of ``{query, should_trigger}`` rows is a skill-creator (and agentskills.io + ``eval_queries.json``) set, and ``{"questions": [...]}`` or a bare list of + ``question`` rows is a dokimi questions set. + """ + if isinstance(data, dict): + if isinstance(data.get("queries"), list): + return FORMAT_REACH + if isinstance(data.get("questions"), list): + return FORMAT_QUESTIONS + return None + if isinstance(data, list) and data and all(isinstance(r, dict) for r in data): + if all("query" in r and "should_trigger" in r for r in data): + return FORMAT_SKILL_CREATOR + if all("question" in r or "text" in r for r in data): + return FORMAT_QUESTIONS + return None + + +def _label(value: Any, where: str) -> ExpectedSkill: + if value is None: + return None + if isinstance(value, str) and value: + return value + if isinstance(value, list) and all(isinstance(v, str) and v for v in value): + return list(value) or None + raise ValueError(f"{where}: expected_skill must be a skill name, a list of names, or null; " + f"got {value!r}") + + +def _text(row: Dict[str, Any], keys: Sequence[str], where: str) -> str: + for key in keys: + value = row.get(key) + if isinstance(value, str) and value.strip(): + return value + raise ValueError(f"{where}: no query text (looked in {', '.join(keys)})") + + +def from_reach(data: Dict[str, Any]) -> List[TriggerQuery]: + """A google/skill-reach ``QuerySet``; ``acceptable_skills`` join ``expected_skill``. + + skill-reach credits only ``expected_skill`` and treats ``acceptable_skills`` as + neutral; dokimi's selection score credits any of them, so a query with + acceptable siblings scores at least as high here. An id-less query gets + skill-reach's own default, ``q-001`` by position. + """ + rows = data.get("queries") + if not isinstance(rows, list): + raise ValueError("skill-reach QuerySet has no queries list") + out = [] + for i, q in enumerate(rows, start=1): + if not isinstance(q, dict): + raise ValueError(f"queries[{i - 1}] is not an object") + qid = str(q.get("id") or q.get("query_id") or f"q-{i:03d}") + first = _label(q.get("expected_skill"), qid) + if isinstance(first, list): + raise ValueError(f"{qid}: skill-reach expected_skill is one name; use acceptable_skills") + extra = q.get("acceptable_skills") or [] + if not isinstance(extra, list) or not all(isinstance(v, str) and v for v in extra): + raise ValueError(f"{qid}: acceptable_skills must be a list of skill names") + if first is None and extra: + raise ValueError(f"{qid}: acceptable_skills without an expected_skill") + names = [first, *[v for v in extra if v != first]] if first else [] + label: ExpectedSkill = names if len(names) > 1 else (first or None) + out.append(TriggerQuery(id=qid, text=_text(q, ("text",), qid), expected_skill=label)) + return out + + +def from_skill_creator(rows: List[Dict[str, Any]], target: str) -> List[TriggerQuery]: + """An anthropics skill-creator / agentskills.io set ``[{query, should_trigger}]`` for *target*. + + The format has no ids, so they are positional (``sc-001``). A negative is + read as "no skill should load": exact for a catalog of the target alone, + stricter than the source in a larger one, where loading a rival was not + something the set's author ruled out. + """ + if not target: + raise ValueError("a skill-creator trigger set needs the target skill it was written for") + out = [] + for i, r in enumerate(rows, start=1): + qid = f"sc-{i:03d}" + if not isinstance(r, dict): + raise ValueError(f"{qid}: row is not an object") + flag = r.get("should_trigger") + if not isinstance(flag, bool): + raise ValueError(f"{qid}: should_trigger must be true or false, got {flag!r}") + out.append(TriggerQuery(id=qid, text=_text(r, ("query",), qid), + expected_skill=target if flag else None)) + return out + + +def from_questions(data: Any) -> List[TriggerQuery]: + """A dokimi questions set; a question without ``expected_skill`` stays unlabelled.""" + rows = data.get("questions") if isinstance(data, dict) else data + if not isinstance(rows, list): + raise ValueError("questions set has no questions list") + out = [] + for i, q in enumerate(rows): + if not isinstance(q, dict): + raise ValueError(f"questions[{i}] is not an object") + raw = question_id(q) + qid = str(f"q{i + 1}" if raw is None else raw) + text = _text(q, ("question", "text", "prompt", "question_text"), qid) + tq = TriggerQuery(id=qid, text=text, expected_skill=None, labelled="expected_skill" in q) + if tq.labelled: + tq.expected_skill = _label(q["expected_skill"], qid) + if isinstance(q.get("source_text"), str): + tq.source_text = q["source_text"] + if isinstance(q.get("trigger_lint"), dict): + tq.lint = dict(q["trigger_lint"]) + out.append(tq) + return out + + +def read_trigger_set(data: Any, target: Optional[str] = None, + fmt: Optional[str] = None) -> tuple: + """Read a parsed trigger set as ``(format, [TriggerQuery, ...])``. + + *fmt* overrides detection; *target* is required for a skill-creator set, + which names no skill of its own. + """ + fmt = fmt or detect_trigger_format(data) + if fmt == FORMAT_REACH: + return fmt, from_reach(data) + if fmt == FORMAT_SKILL_CREATOR: + return fmt, from_skill_creator(data, target or "") + if fmt == FORMAT_QUESTIONS: + return fmt, from_questions(data) + raise ValueError("not a trigger set this dokimi reads (skill-reach QuerySet, " + "skill-creator [{query, should_trigger}], or questions with expected_skill)") diff --git a/dokimi/dokimi/skills/formats.py b/dokimi/dokimi/skills/formats.py new file mode 100644 index 000000000..b1ee03444 --- /dev/null +++ b/dokimi/dokimi/skills/formats.py @@ -0,0 +1,228 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Detect the skill and rule package formats of other agent ecosystems. + +The loader reads one format, the Agent Skills ``SKILL.md`` package +(agentskills.io), which Claude Code, OpenAI Codex, Gemini CLI and GitHub Copilot +all read. Other ecosystems ship guidance in shapes the loader does not load; +this module names them so a catalog scan can say what a checkout holds and +which part of it dokimi can evaluate. Nothing here loads or runs a package. + +Formats, and what selects each one at run time: + +- ``agent-skill`` -- ``SKILL.md`` with ``name``/``description`` frontmatter; the + description triggers it. Linted against the agentskills.io field limits. The + host is read from the path (``.claude/skills``, ``.agents/skills``, ...); a + Codex skill may add ``agents/openai.yaml`` (display metadata only); +- ``cursor-rule`` -- ``.cursor/rules/*.mdc``; ``alwaysApply``, ``globs`` or (an + "agent requested" rule) its ``description`` selects it; +- ``copilot-instructions`` -- ``*.instructions.md`` with ``applyTo`` globs, and + the repo-wide ``.github/copilot-instructions.md``; file globs select them; +- ``gemini-extension`` -- ``gemini-extension.json``: a bundle (context file, MCP + servers, commands, and skills under ``skills/``), installed, not triggered; +- ``claude-plugin`` -- ``.claude-plugin/plugin.json``: a bundle whose + ``skills/`` hold agent skills; installed, not triggered. + +Only ``agent-skill`` is loadable (``discover_skills``); the other rows are +reported so a reader sees what a description-trigger eval does not cover. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional + +from dokimi.utils import parse_frontmatter + +FORMAT_AGENT_SKILL = "agent-skill" +FORMAT_CURSOR_RULE = "cursor-rule" +FORMAT_COPILOT = "copilot-instructions" +FORMAT_GEMINI_EXTENSION = "gemini-extension" +FORMAT_CLAUDE_PLUGIN = "claude-plugin" + +# agentskills.io/specification: name 1-64 chars of [a-z0-9-], no leading, +# trailing or doubled hyphen, equal to the directory name; description 1-1024. +_SPEC_NAME = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$") +SPEC_NAME_MAX = 64 +SPEC_DESCRIPTION_MAX = 1024 + +# Path markers of the hosts that read SKILL.md, first match wins. +_HOSTS = ( + (".claude/skills", "claude-code"), + (".codex/skills", "codex"), + (".gemini/skills", "gemini-cli"), + (".github/skills", "copilot"), + (".agents/skills", "agents"), # the cross-tool alias Codex and Gemini CLI read + ("environment/skills", "task-bundled"), # a SkillsBench / Harbor task's own skills +) +_SKIP_DIRS = frozenset({".git", "node_modules", "__pycache__", ".venv"}) + + +@dataclass +class SkillPackage: + """One package found under a root.""" + + path: Path + format: str + name: str = "" + description: str = "" + selected_by: str = "" # description | globs | always | install + loadable: bool = False + host: Optional[str] = None + problems: List[str] = field(default_factory=list) + details: Dict[str, Any] = field(default_factory=dict) + + def to_dict(self, root: Optional[Path] = None) -> Dict[str, Any]: + path = self.path.relative_to(root) if root else self.path + return {"path": path.as_posix(), "format": self.format, "name": self.name, + "selected_by": self.selected_by, "loadable": self.loadable, "host": self.host, + "description_chars": len(self.description), "problems": list(self.problems), + "details": dict(self.details)} + + +def _frontmatter(text: str) -> Dict[str, Any]: + """Frontmatter, or flat ``key: value`` lines where YAML refuses it. + + Cursor rules write ``globs: **/*``, which YAML reads as an alias and rejects. + """ + meta, _ = parse_frontmatter(text) + if meta or not text.startswith("---"): + return meta + flat: Dict[str, Any] = {} + for line in text.splitlines()[1:]: + if line.strip() == "---": + break + key, sep, value = line.partition(":") + if sep and key.strip() and not key.startswith(" "): + flat[key.strip()] = value.strip().strip("'\"") + return flat + + +def _host(path: Path) -> Optional[str]: + posix = path.as_posix() + for marker, host in _HOSTS: + if f"{marker}/" in posix: + return host + return None + + +def lint_agent_skill(meta: Dict[str, Any], skill_file: Path) -> List[str]: + """Where a SKILL.md breaks the agentskills.io field limits.""" + problems = [] + name = str(meta.get("name") or "") + desc = str(meta.get("description") or "") + if not name: + problems.append("no name") + elif len(name) > SPEC_NAME_MAX or not _SPEC_NAME.match(name): + problems.append(f"name {name!r} is not 1-{SPEC_NAME_MAX} chars of a-z, 0-9, single hyphens") + elif name != skill_file.parent.name: + problems.append(f"name {name!r} differs from its directory {skill_file.parent.name!r}") + if not desc.strip(): + problems.append("no description") + elif len(desc) > SPEC_DESCRIPTION_MAX: + problems.append(f"description is {len(desc)} chars, over {SPEC_DESCRIPTION_MAX}") + return problems + + +def _agent_skill(path: Path) -> SkillPackage: + meta = _frontmatter(path.read_text(encoding="utf-8", errors="replace")) + pkg = SkillPackage(path, FORMAT_AGENT_SKILL, str(meta.get("name") or path.parent.name), + str(meta.get("description") or ""), "description", + loadable=bool(str(meta.get("description") or "").strip()), host=_host(path)) + pkg.problems = lint_agent_skill(meta, path) + if (path.parent / "agents" / "openai.yaml").is_file(): + pkg.details["codex_metadata"] = "agents/openai.yaml" + return pkg + + +def _cursor_rule(path: Path) -> SkillPackage: + meta = _frontmatter(path.read_text(encoding="utf-8", errors="replace")) + always = str(meta.get("alwaysApply", "")).lower() == "true" + globs = meta.get("globs") + desc = str(meta.get("description") or "") + selected = "always" if always else "globs" if globs else "description" if desc else "manual" + pkg = SkillPackage(path, FORMAT_CURSOR_RULE, path.stem, desc, selected) + if globs: + pkg.details["globs"] = globs if isinstance(globs, (str, list)) else str(globs) + return pkg + + +def _copilot(path: Path) -> SkillPackage: + meta = _frontmatter(path.read_text(encoding="utf-8", errors="replace")) + name = path.name.removesuffix(".instructions.md").removesuffix(".md") + pkg = SkillPackage(path, FORMAT_COPILOT, name, str(meta.get("description") or ""), + "globs" if meta.get("applyTo") else "always") + if meta.get("applyTo"): + pkg.details["applyTo"] = str(meta["applyTo"]) + return pkg + + +def _manifest(path: Path, fmt: str) -> SkillPackage: + pkg = SkillPackage(path, fmt, selected_by="install") + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + pkg.problems.append(f"unreadable JSON: {exc}") + return pkg + if not isinstance(data, dict): + pkg.problems.append("manifest is not an object") + return pkg + pkg.name, pkg.description = str(data.get("name") or ""), str(data.get("description") or "") + if not pkg.name: + pkg.problems.append("no name") + bundle = path.parent if fmt == FORMAT_GEMINI_EXTENSION else path.parent.parent + pkg.details["skills"] = len(list((bundle / "skills").glob("*/SKILL.md"))) + if fmt == FORMAT_GEMINI_EXTENSION: + pkg.details["context_file"] = data.get("contextFileName") or ( + "GEMINI.md" if (bundle / "GEMINI.md").is_file() else None) + pkg.details["mcp_servers"] = len(data.get("mcpServers") or {}) + return pkg + + +def classify(path: Path) -> Optional[SkillPackage]: + """The package *path* is, or ``None``.""" + name = path.name + if name == "SKILL.md": + return _agent_skill(path) + if name.endswith(".mdc"): # Cursor's own extension + return _cursor_rule(path) + if name.endswith(".instructions.md") or ( + name == "copilot-instructions.md" and path.parent.name == ".github"): + return _copilot(path) + if name == "gemini-extension.json": + return _manifest(path, FORMAT_GEMINI_EXTENSION) + if name == "plugin.json" and path.parent.name == ".claude-plugin": + return _manifest(path, FORMAT_CLAUDE_PLUGIN) + return None + + +def detect_skill_packages(root) -> List[SkillPackage]: + """Every package under *root*, in path order (VCS and dependency dirs skipped).""" + root = Path(root) + found = [] + for path in sorted(root.rglob("*")): + if not path.is_file() or _SKIP_DIRS.intersection(path.relative_to(root).parts): + continue + pkg = classify(path) + if pkg is not None: + found.append(pkg) + return found + + +def summarize(packages: List[SkillPackage]) -> Dict[str, Any]: + """Counts per format, how each is selected, and spec problems.""" + out: Dict[str, Any] = {} + for p in packages: + row = out.setdefault(p.format, {"n": 0, "loadable": 0, "with_problems": 0, + "selected_by": {}, "hosts": {}}) + row["n"] += 1 + row["loadable"] += p.loadable + row["with_problems"] += bool(p.problems) + row["selected_by"][p.selected_by] = row["selected_by"].get(p.selected_by, 0) + 1 + if p.host: + row["hosts"][p.host] = row["hosts"].get(p.host, 0) + 1 + return out diff --git a/dokimi/dokimi/task_formats/datasets.py b/dokimi/dokimi/task_formats/datasets.py index e433ff1d0..951bf33bf 100644 --- a/dokimi/dokimi/task_formats/datasets.py +++ b/dokimi/dokimi/task_formats/datasets.py @@ -21,7 +21,7 @@ from typing import Any from dokimi.task_formats.model import ( - BY_QUESTIONS, KIND_ANSWER_ONLY, KIND_EXECUTABLE, KIND_TRIGGER, Fingerprint, Report, + BY_QUESTIONS, BY_TRIGGER_SET, KIND_ANSWER_ONLY, KIND_EXECUTABLE, KIND_TRIGGER, Fingerprint, Report, ) from dokimi.task_formats.source import FileSource, first_rows, key_share @@ -234,6 +234,8 @@ def describe(self, source: FileSource) -> Report: report.problems.append(f"{bad} items have a should_trigger that is not true/false") report.details["should_trigger"] = sum(1 for r in rows if r.get("should_trigger") is True) report.details["should_not_trigger"] = sum(1 for r in rows if r.get("should_trigger") is False) + if not report.problems: + report.consumers.append(BY_TRIGGER_SET) return report @@ -290,6 +292,8 @@ def describe(self, source: FileSource) -> Report: report.problems.extend(p for p in map(_lint_reach_query, rows) if p) _lint_duplicate_ids(report, rows, "query_id", "id") report.details["out_of_scope"] = sum(1 for r in rows if r.get("expected_skill") is None) + if not report.problems: + report.consumers.append(BY_TRIGGER_SET) return report diff --git a/dokimi/dokimi/task_formats/model.py b/dokimi/dokimi/task_formats/model.py index 7843aa06f..c02bec383 100644 --- a/dokimi/dokimi/task_formats/model.py +++ b/dokimi/dokimi/task_formats/model.py @@ -24,6 +24,8 @@ BY_DOCKER = "docker-*" BY_BENCHFLOW = "benchflow" BY_QUESTIONS = "questions track" +# Read back by `catalog trigger-set --trigger-set`, then run by the skill-catalog: arm. +BY_TRIGGER_SET = "catalog trigger-set" NONE_YET = "none-yet" #: A probe at or above this confidence claims the path. diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index 13f916747..7023a3b71 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -21,7 +21,7 @@ "dokimi/commands/benchmark.py::cmd_benchmark_batch": {"complexity": 11}, "dokimi/commands/benchmark.py::cmd_benchmark_preflight": {"complexity": 22}, "dokimi/commands/catalog.py::cmd_catalog_scan": {"complexity": 12}, - "dokimi/commands/catalog.py::cmd_catalog_trigger_set": {"complexity": 18}, + "dokimi/commands/catalog.py::cmd_catalog_trigger_set": {"complexity": 16}, "dokimi/commands/evaluate.py::cmd_eval_judge_validity": {"complexity": 29, "rationale": "grew on main in #73 (judge-validity --context stored) from 25 to 29 before the ratchet landed; split it in its own PR"}, "dokimi/commands/judge_reliability.py::cmd_eval_cross_judge": {"complexity": 32, "rationale": "pre-existing when the gate was introduced"}, "dokimi/commands/novelty.py::cmd_novelty_routing": {"complexity": 12}, diff --git a/dokimi/tests/test_skill_formats.py b/dokimi/tests/test_skill_formats.py new file mode 100644 index 000000000..bbea2f360 --- /dev/null +++ b/dokimi/tests/test_skill_formats.py @@ -0,0 +1,107 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Skill/rule package detection across agent ecosystems (``dokimi.skills.formats``). + +The fixtures copy the shapes of public samples: openai/skills (``agents/openai.yaml``), +google-gemini/gemini-cli (``gemini-extension.json``, ``.gemini/skills``), +PatrickJS/awesome-cursorrules (``.mdc`` with ``globs: **/*``), github/awesome-copilot +(``*.instructions.md`` with ``applyTo``), anthropics/claude-plugins-official +(``.claude-plugin/plugin.json``) and benchflow-ai/skillsbench (``environment/skills``). +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from dokimi.cli import build_parser +from dokimi.skills import discover_skills +from dokimi.skills import formats as sf + + +def _w(path: Path, text: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + +def _skill(d: Path, name: str, desc: str = "Use when profiling.") -> Path: + return _w(d / "SKILL.md", f"---\nname: {name}\ndescription: {desc}\n---\nbody\n") + + +def _tree(root: Path) -> Path: + _skill(root / ".codex/skills/imagegen", "imagegen") + _w(root / ".codex/skills/imagegen/agents/openai.yaml", "interface:\n display_name: Image Gen\n") + _skill(root / ".gemini/skills/code_explorer", "code_explorer") + _skill(root / "tasks/t1/environment/skills/xlsx", "xlsx", "x" * 1100) + _w(root / ".cursor/rules/ts.mdc", + "---\ndescription: \"TypeScript rules\"\nglobs: **/*.ts\nalwaysApply: false\n---\n# TS\n") + _w(root / ".cursor/rules/always.mdc", "---\nalwaysApply: true\n---\nbe terse\n") + _w(root / ".github/instructions/swift.instructions.md", + "---\ndescription: 'Swift MCP'\napplyTo: \"**/*.swift\"\n---\n# Swift\n") + _w(root / ".github/copilot-instructions.md", "Repo-wide rules\n") + ext = root / "ext/policy" + _w(ext / "gemini-extension.json", + json.dumps({"name": "policy", "version": "1", "mcpServers": {"s": {}}})) + _w(ext / "GEMINI.md", "context\n") + _skill(ext / "skills/audit", "audit") + plugin = root / "plugins/hookify" + _w(plugin / ".claude-plugin/plugin.json", json.dumps({"name": "hookify", "description": "d"})) + _skill(plugin / "skills/writing-rules", "writing-hookify-rules") + _w(root / "node_modules/x/SKILL.md", "---\nname: x\ndescription: d\n---\n") + return root + + +def test_each_ecosystem_is_named_with_how_it_is_selected(tmp_path): + root = _tree(tmp_path) + by_path = {p.path.relative_to(root).as_posix(): p for p in sf.detect_skill_packages(root)} + assert "node_modules/x/SKILL.md" not in by_path + codex = by_path[".codex/skills/imagegen/SKILL.md"] + assert (codex.format, codex.host, codex.loadable, codex.problems) == ( + "agent-skill", "codex", True, []) + assert codex.details == {"codex_metadata": "agents/openai.yaml"} + assert by_path[".gemini/skills/code_explorer/SKILL.md"].problems[0].startswith( + "name 'code_explorer' is not") + bundled = by_path["tasks/t1/environment/skills/xlsx/SKILL.md"] + assert bundled.host == "task-bundled" and "over 1024" in bundled.problems[0] + ts_rule = by_path[".cursor/rules/ts.mdc"] + assert (ts_rule.format, ts_rule.selected_by, ts_rule.description) == ( + "cursor-rule", "globs", "TypeScript rules") # YAML refuses `**/*.ts`; read anyway + assert by_path[".cursor/rules/always.mdc"].selected_by == "always" + assert by_path[".github/instructions/swift.instructions.md"].details == {"applyTo": "**/*.swift"} + assert by_path[".github/copilot-instructions.md"].selected_by == "always" + ext = by_path["ext/policy/gemini-extension.json"] + assert (ext.selected_by, ext.details) == ( + "install", {"skills": 1, "context_file": "GEMINI.md", "mcp_servers": 1}) + plugin = by_path["plugins/hookify/.claude-plugin/plugin.json"] + assert (plugin.format, plugin.details["skills"]) == ("claude-plugin", 1) + assert "differs from its directory" in by_path[ + "plugins/hookify/skills/writing-rules/SKILL.md"].problems[0] + + +def test_every_loadable_package_is_one_the_skill_loader_loads(tmp_path): + root = _tree(tmp_path) + loadable = {p.name for p in sf.detect_skill_packages(root) if p.loadable} + # The loader does not skip node_modules/ (a vendored copy loads as a skill); the scan does. + assert loadable == {s.name for s in discover_skills(root)} - {"x"} + + +def test_spec_lint_rules(): + f = Path("a-b/SKILL.md") + assert sf.lint_agent_skill({"name": "a-b", "description": "d"}, f) == [] + for name in ("A-b", "-ab", "a--b", "a" * 65): + assert "is not" in sf.lint_agent_skill({"name": name, "description": "d"}, Path(f"{name}/SKILL.md"))[0] + assert sf.lint_agent_skill({}, f) == ["no name", "no description"] + + +def test_catalog_formats_cli(tmp_path, capsys): + root = _tree(tmp_path / "r") + args = build_parser().parse_args(["catalog", "formats", "--root", str(root), "--json"]) + args.func(args) + data = json.loads(capsys.readouterr().out) + assert data["summary"]["agent-skill"]["n"] == 5 + assert data["summary"]["cursor-rule"]["selected_by"] == {"globs": 1, "always": 1} + args = build_parser().parse_args(["catalog", "formats", "--root", str(root)]) + args.func(args) + assert "# gemini-extension: 1" in capsys.readouterr().out diff --git a/dokimi/tests/test_trigger_set_readers.py b/dokimi/tests/test_trigger_set_readers.py new file mode 100644 index 000000000..6e70f6a2c --- /dev/null +++ b/dokimi/tests/test_trigger_set_readers.py @@ -0,0 +1,169 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Reading trigger sets back: skill-reach, skill-creator / agentskills.io, dokimi questions. + +Each reader is held to the writer of the same format in ``dokimi.external.trigger_set``: +what dokimi writes, it reads back to the same labels. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from dokimi.cli import build_parser +from dokimi.external import trigger_set as ts +from dokimi.questions.artifact import load_questions +from dokimi.skills import discover_skills +from dokimi.task_formats import detect + + +def _write_skill(root: Path, name: str, description: str) -> None: + d = root / name + d.mkdir(parents=True) + (d / "SKILL.md").write_text(f"---\nname: {name}\ndescription: \"{description}\"\n---\n# {name}\n", + encoding="utf-8") + + +@pytest.fixture +def catalog(tmp_path: Path) -> Path: + root = tmp_path / "skills" + _write_skill(root, "linux-perf", "Profile Linux programs with perf: hotspots, IPC, cache misses.") + _write_skill(root, "dpnp-io", "Read and write dpnp arrays to files on Intel GPUs.") + _write_skill(root, "dpnp-memory", "Place dpnp arrays on devices and size memory.") + return root + + +@pytest.fixture +def queries(catalog): + return ts.build_trigger_set([ + ([{"id": "p", "text": "my program is slow, where is the time going"}], "linux-perf"), + ([{"id": "f", "text": "save this array to disk", "skills": ["dpnp-io", "dpnp-memory"]}], + "@skills"), + ([{"id": "n", "text": "write a haiku"}], "none"), + ([{"id": "u", "text": "unlabelled", "metadata": {}}], "@metadata.skill"), + ], discover_skills(catalog)) + + +def _labels(qs): + return [(q.id, q.text, q.expected_skill, q.labelled) for q in qs] + + +def test_reach_round_trip_keeps_ids_texts_and_acceptable_sets(queries): + back = ts.from_reach(ts.to_reach(queries, catalog_id="c")) + assert _labels(back) == _labels(ts.labelled(queries)) + assert ts.detect_trigger_format(ts.to_reach(queries)) == ts.FORMAT_REACH + + +def test_skill_creator_round_trip_is_identity_on_its_own_rows(queries): + rows = ts.to_skill_creator(queries, "linux-perf") + back = ts.from_skill_creator(rows, "linux-perf") + assert ts.to_skill_creator(back, "linux-perf") == rows + assert [q.expected_skill for q in back] == ["linux-perf", None, None] + assert [q.id for q in back] == ["sc-001", "sc-002", "sc-003"] + assert ts.detect_trigger_format(rows) == ts.FORMAT_SKILL_CREATOR + + +def test_questions_round_trip_keeps_unlabelled_and_lint(queries, tmp_path): + payload = ts.to_questions(queries) + back = ts.from_questions(payload) + assert _labels(back) == _labels(queries) + assert back[0].lint == queries[0].lint + # Through the stamped artifact the arms command loads, too. + from dokimi.questions.artifact import save_questions_artifact + + path = tmp_path / "q.json" + save_questions_artifact(payload, str(path)) + stamped = json.loads(path.read_text(encoding="utf-8")) + assert _labels(ts.from_questions(stamped)) == _labels(queries) + assert len(load_questions(path).questions) == 4 + + +def test_reach_reader_matches_skill_reach_defaults_and_refuses_bad_rows(): + qs = ts.from_reach({"queries": [ + {"text": "a", "expected_skill": "x", "acceptable_skills": ["y", "x"]}, + {"query_id": "k", "text": "b", "expected_skill": None, "kind": "out_of_scope"}, + ]}) + assert [(q.id, q.expected_skill) for q in qs] == [("q-001", ["x", "y"]), ("k", None)] + for bad, msg in [({"text": "a", "expected_skill": None, "acceptable_skills": ["y"]}, "without"), + ({"text": "", "expected_skill": "x"}, "no query text"), + ({"text": "a", "expected_skill": 3}, "skill name"), + ({"text": "a", "expected_skill": ["x"]}, "one name")]: + with pytest.raises(ValueError, match=msg): + ts.from_reach({"queries": [bad]}) + + +def test_skill_creator_reader_needs_a_target_and_booleans(): + with pytest.raises(ValueError, match="target"): + ts.read_trigger_set([{"query": "a", "should_trigger": True}]) + with pytest.raises(ValueError, match="true or false"): + ts.from_skill_creator([{"query": "a", "should_trigger": "yes"}], "x") + + +def test_detect_refuses_what_is_not_a_trigger_set(): + for data in ({"evals": []}, [], [{"prompt": "a"}], "text", {"queries": "x"}): + assert ts.detect_trigger_format(data) is None + with pytest.raises(ValueError, match="not a trigger set"): + ts.read_trigger_set({"evals": []}) + + +def _run(argv): + args = build_parser().parse_args(argv) + args.func(args) + + +def test_cli_reads_each_format_back_and_relints_against_the_catalog(tmp_path, catalog, queries, + capsys): + reach = tmp_path / "reach.json" + reach.write_text(json.dumps(ts.to_reach(queries)), encoding="utf-8") + sc = tmp_path / "sc.json" + sc.write_text(json.dumps([{"query": "use linux-perf on it", "should_trigger": True}]), + encoding="utf-8") + out = tmp_path / "q.json" + _run(["catalog", "trigger-set", "--catalog", str(catalog), "--format", "questions", + "--trigger-set", str(reach), "--trigger-set", f"{sc}=linux-perf", "--out", str(out)]) + rows = load_questions(out).questions + assert [r.get("expected_skill") for r in rows] == [ + "linux-perf", ["dpnp-io", "dpnp-memory"], None, "linux-perf"] + assert rows[3]["trigger_lint"]["names_skill"] == ["linux-perf"] # linted on import + err = capsys.readouterr().err + assert "as reach (3 queries)" in err and "rival load counts as a false trigger" in err + + +def test_cli_refuses_bad_trigger_set_specs(tmp_path, catalog): + sc = tmp_path / "sc.json" + sc.write_text(json.dumps([{"query": "a", "should_trigger": True}]), encoding="utf-8") + reach = tmp_path / "r.json" + reach.write_text(json.dumps({"queries": [{"id": "1", "text": "a", "expected_skill": None}]}), + encoding="utf-8") + base = ["catalog", "trigger-set", "--catalog", str(catalog), "--out", str(tmp_path / "o.json")] + with pytest.raises(SystemExit, match="not a skill in --catalog"): + _run(base + ["--trigger-set", f"{sc}=nope"]) + with pytest.raises(SystemExit, match="needs the target"): + _run(base + ["--trigger-set", str(sc)]) + with pytest.raises(SystemExit, match="skill-creator sets only"): + _run(base + ["--trigger-set", f"{reach}=linux-perf"]) + with pytest.raises(SystemExit, match="--questions PATH=LABEL or --trigger-set"): + _run(base) + + +def test_arms_points_an_external_trigger_set_at_the_converter(tmp_path): + from dokimi.commands.arms import _load_questions + + sc = tmp_path / "sc.json" + sc.write_text(json.dumps([{"query": "a", "should_trigger": True}]), encoding="utf-8") + with pytest.raises(ValueError, match=r"skill-creator trigger set.*--trigger-set .*="): + _load_questions(str(sc)) + + +def test_task_formats_name_the_reader_as_consumer(tmp_path, queries): + reach = tmp_path / "reach.json" + reach.write_text(json.dumps(ts.to_reach(queries)), encoding="utf-8") + sc = tmp_path / "sc.json" + sc.write_text(json.dumps(ts.to_skill_creator(queries, "linux-perf")), encoding="utf-8") + for path in (reach, sc): + report = detect(path) + assert report.consumers == ["catalog trigger-set"], report From 19822bab06088e77f5070399651318f467f4ceef Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 24 Oct 2026 10:57:20 -0700 Subject: [PATCH 03/13] feat(arms): precision in skill_selection_summary Hits over the questions where the arm loaded any skill, strict on multi-load like recall; rendered in the arms report's Skill selection table. Replaces the ad-hoc calculation used for the #211 evidence table. Signed-off-by: Nikolay Petrov --- .../20261024_120000_napetrov_trigger_formats.md | 1 + dokimi/dokimi/eval/arm_summary.py | 5 ++++- dokimi/dokimi/report/arms_report.py | 6 +++--- dokimi/tests/test_skill_selection.py | 14 ++++++++++++-- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/dokimi/changelog.d/20261024_120000_napetrov_trigger_formats.md b/dokimi/changelog.d/20261024_120000_napetrov_trigger_formats.md index 80294970b..1590aaa31 100644 --- a/dokimi/changelog.d/20261024_120000_napetrov_trigger_formats.md +++ b/dokimi/changelog.d/20261024_120000_napetrov_trigger_formats.md @@ -2,3 +2,4 @@ - **Read trigger sets back** — `dokimi catalog trigger-set --trigger-set PATH[=TARGET]` reads a google/skill-reach QuerySet, an anthropics skill-creator / agentskills.io `[{query, should_trigger}]` set (TARGET names its skill), or a questions set with `expected_skill`, relints it against `--catalog`, and writes any of the three formats, so an imported set runs on the `skill-catalog:` arm. `arms run` given a skill-reach or skill-creator file names the conversion command; `tasks inspect` lists `catalog trigger-set` as the consumer of both formats. - **Skill package formats** — `dokimi catalog formats --root DIR [--json]` lists Agent Skills `SKILL.md` packages (with the host read from the path, the agentskills.io name/description limits linted, and Codex `agents/openai.yaml` noted), Cursor `.mdc` rules, Copilot `*.instructions.md`, Gemini CLI `gemini-extension.json` and Claude Code `.claude-plugin/plugin.json` bundles, with how each is selected (description, globs, always, install). +- **Skill selection precision** — `skill_selection_summary` and the arms report's *Skill selection* table carry `precision`: hits over the questions where the arm loaded any skill, so misroutes and false triggers count against it. diff --git a/dokimi/dokimi/eval/arm_summary.py b/dokimi/dokimi/eval/arm_summary.py index 968cbf2b9..cf109ae53 100644 --- a/dokimi/dokimi/eval/arm_summary.py +++ b/dokimi/dokimi/eval/arm_summary.py @@ -130,7 +130,7 @@ def summarize_skill_selection(records: Iterable[RecordLike]) -> Dict[str, Any]: or ``None`` — no skill should load. Arms that offered no skill are skipped. The skills loaded in the first turn that loaded any are one choice. ``recall`` - is strict: a hit needs every one of them acceptable. ``recall_any`` (one + and ``precision`` (hits over questions where any skill loaded) are strict: a hit needs every one of them acceptable. ``recall_any`` (one acceptable skill among them), ``multi_load_rate`` and ``mean_first_turn_loads`` show how far the strict figure depends on multi-loading. """ @@ -192,6 +192,9 @@ def rate(a: int, b: int) -> Any: "n_failed": s["failed"], "selection_accuracy": rate(s["correct"], s["n"]), "recall": rate(s["hit"], s["pos"]), + # Of the questions where the arm loaded a skill, the share where every + # loaded skill was acceptable (misroutes and false triggers count against). + "precision": rate(s["hit"], s["loaded_n"]), # Lenient: the right skill was among those loaded in the first turn. "recall_any": rate(s["hit_any"], s["pos"]), "multi_load_rate": rate(s["multi"], s["loaded_n"]), diff --git a/dokimi/dokimi/report/arms_report.py b/dokimi/dokimi/report/arms_report.py index 702bf6de6..da298f0ea 100644 --- a/dokimi/dokimi/report/arms_report.py +++ b/dokimi/dokimi/report/arms_report.py @@ -265,13 +265,13 @@ def pct(v: Any) -> str: "harness does." ) lines.append("") - lines.append("| Arm | Accuracy | Recall | Recall (any) | Multi-load | Misroute | False trigger " + lines.append("| Arm | Accuracy | Recall | Precision | Recall (any) | Multi-load | Misroute | False trigger " "| +/− | n | Failed |") - lines.append("| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |") + lines.append("| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |") for arm_name, s in selection.items(): lines.append( f"| `{arm_name}` | {pct(s.get('selection_accuracy'))} | {pct(s.get('recall'))} | " - f"{pct(s.get('recall_any'))} | {pct(s.get('multi_load_rate'))} | " + f"{pct(s.get('precision'))} | {pct(s.get('recall_any'))} | {pct(s.get('multi_load_rate'))} | " f"{pct(s.get('misroute_rate'))} | {pct(s.get('false_trigger_rate'))} | " f"{s.get('n_positive', 0)}/{s.get('n_negative', 0)} | {s.get('n', 0)} | " f"{s.get('n_failed', 0)} |" diff --git a/dokimi/tests/test_skill_selection.py b/dokimi/tests/test_skill_selection.py index ae704b4af..eceed7207 100644 --- a/dokimi/tests/test_skill_selection.py +++ b/dokimi/tests/test_skill_selection.py @@ -101,6 +101,8 @@ def test_selection_summary_scores_hits_misroutes_and_false_triggers(): assert s["recall"] == 0.5 assert s["misroute_rate"] == 0.25 assert s["false_trigger_rate"] == 0.5 + # Loaded on 4 of 6: two hits, one misroute, one false trigger. + assert s["precision"] == 0.5 assert s["selection_accuracy"] == round(3 / 6, 3) assert s["confusion"]["linux-perf"] == {"linux-perf": 1, "onetbb-quickstart": 1, "(none)": 1} assert s["confusion"]["(none)"] == {"(none)": 1, "linux-perf": 1} @@ -117,7 +119,7 @@ def test_report_renders_selection_table(): "skill_selection_summary": summarize_skill_selection([_rec("a", "a"), _rec(None, "a")]), }) assert "## Skill selection" in md - assert "| `cat` | 50% | 100% | 100% | 0% | 0% | 100% | 1/1 | 2 | 0 |" in md + assert "| `cat` | 50% | 100% | 50% | 100% | 0% | 0% | 100% | 1/1 | 2 | 0 |" in md # ── trigger set ────────────────────────────────────────────────────────── @@ -428,7 +430,7 @@ def test_combined_arm_rejects_duplicate_tools(catalog): def test_report_renders_dash_for_missing_rates(): md = render_arms_report({"arms": ["cat"], "answers": [], "skill_selection_summary": summarize_skill_selection([_rec(None, None)])}) - assert "| `cat` | 100% | — | — | — | — | 0% | 0/1 | 1 | 0 |" in md + assert "| `cat` | 100% | — | — | — | — | — | 0% | 0/1 | 1 | 0 |" in md def _qfile(tmp_path, name, rows): @@ -726,3 +728,11 @@ def test_loading_the_whole_catalog_in_one_turn_is_not_a_hit(catalog): assert (s["recall"], s["misroute_rate"], s["recall_any"]) == (0.0, 1.0, 1.0) assert s["mean_first_turn_loads"] == 3.0 assert s["confusion"] == {"linux-perf": {"dpnp-io+linux-perf+onetbb-quickstart": 1}} + + +def test_selection_precision_is_none_when_nothing_loaded_and_strict_on_multi_load(): + assert summarize_skill_selection([_rec("a", None), _rec(None, None)])["cat"]["precision"] is None + rec = _rec("a", "a") + rec["arms"]["cat"]["tool_usage"]["skill"]["first_turn_loaded"] = ["a", "b"] + s = summarize_skill_selection([rec, _rec("a", "a")])["cat"] + assert (s["precision"], s["recall_any"]) == (0.5, 1.0) From 5de519914b11aff56b2c2b6bda113b4c8e3b60ca Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 24 Oct 2026 11:13:22 -0700 Subject: [PATCH 04/13] feat(catalog): trigger-probe runs each query in real Claude Code; payload leak audit catalog trigger-probe runs every trigger-set query in a fresh claude -p: a new git repo with the catalog committed in .claude/skills, scratch HOME and config, the query verbatim, bundled skills on. Skill calls are scored by the same selection summary as the skill-catalog arm, one column per trial. payload_audit reports dokimi artifacts (arm names, label keys, question ids, grading words, task paths) in a captured request, argv or env, with operator content blanked and the stock harness's own text subtracted. Signed-off-by: Nikolay Petrov --- ...261024_120000_napetrov_trigger-fidelity.md | 10 + dokimi/docs/evaluating-treatments.md | 13 ++ dokimi/dokimi/commands/catalog.py | 4 + dokimi/dokimi/commands/catalog_probe.py | 93 ++++++++ dokimi/dokimi/harnesses/payload_audit.py | 134 +++++++++++ dokimi/dokimi/harnesses/trigger_probe.py | 214 ++++++++++++++++++ dokimi/tests/test_payload_audit.py | 117 ++++++++++ dokimi/tests/test_trigger_probe.py | 131 +++++++++++ 8 files changed, 716 insertions(+) create mode 100644 dokimi/changelog.d/20261024_120000_napetrov_trigger-fidelity.md create mode 100644 dokimi/dokimi/commands/catalog_probe.py create mode 100644 dokimi/dokimi/harnesses/payload_audit.py create mode 100644 dokimi/dokimi/harnesses/trigger_probe.py create mode 100644 dokimi/tests/test_payload_audit.py create mode 100644 dokimi/tests/test_trigger_probe.py diff --git a/dokimi/changelog.d/20261024_120000_napetrov_trigger-fidelity.md b/dokimi/changelog.d/20261024_120000_napetrov_trigger-fidelity.md new file mode 100644 index 000000000..061b10536 --- /dev/null +++ b/dokimi/changelog.d/20261024_120000_napetrov_trigger-fidelity.md @@ -0,0 +1,10 @@ +### Added + +- **`catalog trigger-probe`** — runs each trigger-set query in a fresh, unannounced + `claude -p` session with the catalog committed as project skills, a scratch `HOME` + and config, and the query sent verbatim; scores the `Skill` calls with the same + selection summary as the `skill-catalog:` arm, one column per trial. +- **`dokimi.harnesses.payload_audit`** — `audit_payload` / `new_leaks` report dokimi + artifacts (arm names, label keys, question ids, grading words, task paths) in a + captured request, argv or env, with operator content blanked and the stock harness's + own text subtracted. diff --git a/dokimi/docs/evaluating-treatments.md b/dokimi/docs/evaluating-treatments.md index 493bd782f..e817e393a 100644 --- a/dokimi/docs/evaluating-treatments.md +++ b/dokimi/docs/evaluating-treatments.md @@ -582,6 +582,19 @@ uv run reach eval --skills /skills --queries trigger_set.json \ uv run reach diff --vary description control.json treatment.json # A/B vs noise floor ``` +**Real harness, in dokimi: `catalog trigger-probe`.** Runs each query in a fresh +`claude -p`: a new git repository with the catalog committed in `.claude/skills/`, a +scratch `HOME` and `CLAUDE_CONFIG_DIR` (your own skills, settings and `CLAUDE.md` never +reach it), the query sent verbatim with no notice or task framing, and bundled skills +left on as a user has them (`--sealed` turns them off). The skill chosen is read from the +transcript's `Skill` calls, and the output's `skill_selection_summary` is scored by the +same code as the `skill-catalog:` arm, one column per trial: + +```bash +python cli.py catalog trigger-probe --catalog /skills \ + --trigger-set trigger_set.json --model --trials 3 --out probe.json +``` + Two things decide whether such a number means anything: - **Listing budget.** Claude Code shows skill descriptions only up to a budget (a diff --git a/dokimi/dokimi/commands/catalog.py b/dokimi/dokimi/commands/catalog.py index 75fdcef44..6419430c9 100644 --- a/dokimi/dokimi/commands/catalog.py +++ b/dokimi/dokimi/commands/catalog.py @@ -274,3 +274,7 @@ def register(sub, positive_int) -> None: help="Exit 1 if any query names a skill, keeps exam framing, or has an unresolved @field label") ts_p.add_argument("--out", required=True, help="Output JSON path") ts_p.set_defaults(func=cmd_catalog_trigger_set) + + from dokimi.commands import catalog_probe + + catalog_probe.register(catalog_sub, positive_int) diff --git a/dokimi/dokimi/commands/catalog_probe.py b/dokimi/dokimi/commands/catalog_probe.py new file mode 100644 index 000000000..d55177e84 --- /dev/null +++ b/dokimi/dokimi/commands/catalog_probe.py @@ -0,0 +1,93 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""``catalog trigger-probe``: run a trigger set through real Claude Code, one fresh run per query.""" + +from __future__ import annotations + +import argparse +import json +import shutil +import sys +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + + +def cmd_catalog_trigger_probe(args: argparse.Namespace) -> None: + from dokimi.eval.arm_summary import summarize_skill_selection + from dokimi.external import trigger_set as ts + from dokimi.harnesses import trigger_probe as tp + + catalog = Path(args.catalog) + if not catalog.is_dir(): + raise SystemExit(f"--catalog is not a directory: {catalog}") + if shutil.which(args.claude) is None: + raise SystemExit(f"{args.claude!r} is not on PATH") + try: + data = json.loads(Path(args.trigger_set).read_text(encoding="utf-8")) + _fmt, queries = ts.read_trigger_set(data, target=args.target) + except (OSError, ValueError) as exc: + raise SystemExit(f"--trigger-set: {exc}") from exc + queries = [q for q in queries if q.labelled][: args.limit or None] + cfg = tp.ProbeConfig(model=args.model, max_turns=args.max_turns, timeout_sec=args.timeout, + stock=not args.sealed, listing_budget_fraction=args.listing_budget_fraction, + claude=args.claude) + + def one(q): + rec = tp.probe_trigger_set([q], catalog, cfg, trials=args.trials)[0] + print(f"{q.id}: " + ", ".join( + f"{a}={'+'.join(v['tool_usage']['skill']['first_turn_loaded']) or '-'}" + + (f" ({v['error']})" if v["error"] else "") for a, v in rec["arms"].items()), + file=sys.stderr) + return rec + + with ThreadPoolExecutor(max(1, args.concurrency)) as pool: + records = list(pool.map(one, queries)) + cost = sum(v["metrics"]["cost_usd"] or 0 for r in records for v in r["arms"].values()) + out = { + "harness": "claude-code", + "mode": "sealed" if args.sealed else "stock", + "model": args.model, + "catalog": str(catalog), + "trials": args.trials, + "max_turns": args.max_turns, + "listing_budget_fraction": args.listing_budget_fraction, + "claude_code_version": next((v["tool_usage"]["skill"]["claude_code_version"] for r in records + for v in r["arms"].values() + if v["tool_usage"]["skill"]["claude_code_version"]), None), + "cost_usd": round(cost, 4), + "answers": records, + "skill_selection_summary": summarize_skill_selection(records), + } + Path(args.out).write_text(json.dumps(out, indent=2) + "\n", encoding="utf-8") + print(json.dumps({"cost_usd": out["cost_usd"], "claude_code_version": out["claude_code_version"], + "summary": {a: {k: s[k] for k in ("recall", "false_trigger_rate", "misroute_rate", "n_failed")} + for a, s in out["skill_selection_summary"].items()}}, indent=2)) + print(f"wrote {args.out}") + + +def register(catalog_sub, positive_int) -> None: + """Add ``catalog trigger-probe`` to the ``catalog`` group.""" + p = catalog_sub.add_parser( + "trigger-probe", + help="Run each trigger-set query in a fresh, unannounced Claude Code session with the " + "catalog installed as project skills; score which skill it invokes", + ) + p.add_argument("--catalog", required=True, help="Directory of skills, installed as .claude/skills") + p.add_argument("--trigger-set", required=True, dest="trigger_set", + help="skill-reach QuerySet, skill-creator set (with --target) or questions with expected_skill") + p.add_argument("--target", help="Target skill of a skill-creator set") + p.add_argument("--model", help="--model passed to claude (default: the harness default)") + p.add_argument("--trials", type=positive_int, default=1, help="Fresh runs per query (default: 1)") + p.add_argument("--max-turns", type=positive_int, default=3, dest="max_turns", + help="claude --max-turns (default: 3)") + p.add_argument("--timeout", type=float, default=300.0, help="Seconds per run (default: 300)") + p.add_argument("--concurrency", type=positive_int, default=4, help="Queries run at once (default: 4)") + p.add_argument("--limit", type=positive_int, help="Only the first N labelled queries") + p.add_argument("--listing-budget-fraction", type=float, dest="listing_budget_fraction", + help="skillListingBudgetFraction for the run (default: the harness's own, 0.01)") + p.add_argument("--sealed", action="store_true", + help="Switch off bundled skills as the docker-claude arms do (default: stock harness)") + p.add_argument("--claude", default="claude", help="claude executable (default: claude)") + p.add_argument("--out", required=True, help="Output JSON path") + p.set_defaults(func=cmd_catalog_trigger_probe) diff --git a/dokimi/dokimi/harnesses/payload_audit.py b/dokimi/dokimi/harnesses/payload_audit.py new file mode 100644 index 000000000..d83c1e4e3 --- /dev/null +++ b/dokimi/dokimi/harnesses/payload_audit.py @@ -0,0 +1,134 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Leak audit: does what an agent is sent carry dokimi's own artifacts? + +A measured run is meant to reproduce what a user's agent sees. Anything dokimi adds +that names the experiment (an arm name, the ``expected_skill`` key, a question id, +judge or rubric text, a task path that says what is graded) can change what the model +does, and makes the number measure dokimi rather than the harness. + +:func:`audit_payload` walks a payload (a captured request body, an argv, an env map: +any JSON-like value) and reports every marker it finds. Text the operator supplied +on purpose (the query, the skills' own descriptions and bodies) is passed as +``allowed`` and blanked out before matching: a skill whose description says +"baseline" is content, not a leak. What is left is what dokimi wrote. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Any, Iterable, Iterator, List, Optional, Sequence, Tuple + +#: Words and keys that name an experiment rather than a user's task. Matched whole-word, +#: case-insensitive, after the allowed texts are blanked. +DEFAULT_MARKERS: Tuple[str, ...] = ( + "dokimi", "expected_skill", "acceptable_skills", "should_trigger", "trigger_set", + "skill_catalog", "skill-catalog", "skill_selection", "treatment", "baseline", + "with_docs", "without_docs", "judge", "rubric", "golden", "grader", "verifier", + "reward", "oracle", "eval", "evals", "evaluation", "benchmark", "arm_kind", + "abskill", "test_outputs", +) + + +@dataclass(frozen=True) +class Leak: + """One marker found: where (a JSON path), which marker, and the text around it.""" + + path: str + marker: str + excerpt: str + + def as_dict(self) -> dict: + return {"path": self.path, "marker": self.marker, "excerpt": self.excerpt} + + +def markers_for(*, question_ids: Iterable[str] = (), arm_names: Iterable[str] = (), + paths: Iterable[str] = (), extra: Iterable[str] = (), + base: Sequence[str] = DEFAULT_MARKERS) -> List[str]: + """The default markers plus this run's own identifiers. + + *question_ids* and *arm_names* come from the run; *paths* are host paths that + reveal intent (a task directory, an output dir). Empty values are dropped. + """ + out = list(base) + for group in (question_ids, arm_names, paths, extra): + out += [str(v) for v in group if str(v).strip()] + return out + + +def _strings(value: Any, path: str = "$") -> Iterator[Tuple[str, str]]: + """Every string in *value* (keys included) with its JSON path.""" + if isinstance(value, str): + yield path, value + elif isinstance(value, dict): + for k, v in value.items(): + yield f"{path}.{k}#key", str(k) + yield from _strings(v, f"{path}.{k}") + elif isinstance(value, (list, tuple)): + for i, v in enumerate(value): + yield from _strings(v, f"{path}[{i}]") + + +def _pattern(marker: str) -> "re.Pattern[str]": + # Identifier-ish markers match between non-alphanumerics, so `dokimi-run-x` and + # `eval_queries` are caught while `evaluate` is not; anything else is a substring. + if re.fullmatch(r"[\w-]+", marker): + return re.compile(rf"(? str: + for a in allowed: + if a and a in text: + text = text.replace(a, " " * len(a)) + return text + + +def audit_payload(payload: Any, markers: Optional[Iterable[str]] = None, + allowed: Iterable[str] = ()) -> List[Leak]: + """Markers found in *payload* outside the *allowed* texts, one :class:`Leak` each. + + *allowed* is longest-first blanked, so a skill body containing its description + blanks as a whole. An empty result means the payload carries none of the markers. + """ + pats = [(m, _pattern(m)) for m in (DEFAULT_MARKERS if markers is None else markers) if m] + keep = sorted({a for a in allowed if a and a.strip()}, key=len, reverse=True) + leaks: List[Leak] = [] + for path, text in _strings(payload): + visible = _blank(text, keep) + for marker, pat in pats: + for m in pat.finditer(visible): + lo, hi = max(0, m.start() - 40), min(len(text), m.end() + 40) + leaks.append(Leak(path, marker, text[lo:hi].replace("\n", " "))) + return leaks + + +def added_framing(prompt: str, query: str) -> str: + """What a harness put in the prompt around the user's *query* (``""`` if verbatim). + + The framing is not a leak by the marker test, but it is an influence: an + out-of-the-box run sends the query alone. + """ + if prompt == query: + return "" + if query and query in prompt: + head, _, tail = prompt.partition(query) + return (head + tail).strip() + return prompt + + +def new_leaks(payload: Any, reference: Any, markers: Optional[Iterable[str]] = None, + allowed: Iterable[str] = ()) -> List[Leak]: + """Leaks in *payload* that a *reference* capture does not have. + + The reference is the same harness run out of the box (no dokimi in the loop): its + own system prompt says "judge edge cases" and "golden path", which is the + harness, not the experiment. A marker counts as new when its excerpt is not + among the reference's. + """ + marks = list(DEFAULT_MARKERS if markers is None else markers) + allowed = list(allowed) + seen = {(lk.marker, lk.excerpt) for lk in audit_payload(reference, marks, allowed)} + return [lk for lk in audit_payload(payload, marks, allowed) if (lk.marker, lk.excerpt) not in seen] diff --git a/dokimi/dokimi/harnesses/trigger_probe.py b/dokimi/dokimi/harnesses/trigger_probe.py new file mode 100644 index 000000000..bce1b0053 --- /dev/null +++ b/dokimi/dokimi/harnesses/trigger_probe.py @@ -0,0 +1,214 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Real-harness skill triggering: one fresh ``claude -p`` per query, no notice. + +The ``skill-catalog:`` arm is a tool-calling emulation: one tool per skill, every +description in full, no system prompt, no other tools. Claude Code instead sends one +``Skill`` tool, a listing it cuts to a budget, its own system prompt and a dozen other +tools, and the model may look around before it loads anything. This module measures +that harness as shipped: + +* each query runs in a new git repository under a temp dir (no ancestor ``CLAUDE.md``, + no checkout to walk), with the catalog committed in ``.claude/skills/``, where + Claude Code looks for project skills, so ``git status`` shows nothing unusual; +* ``HOME`` and ``CLAUDE_CONFIG_DIR`` are fresh and the env is the sandbox allowlist, + so the operator's skills, settings and ``CLAUDE.md`` never reach the run; +* the prompt is the query verbatim: no treatment notice, no task framing; +* in ``stock`` mode (the default) the bundled skills and tools stay on, as a user + has them; ``sealed`` switches them off like the docker-claude arms do. + +The skills invoked are read from the ``stream-json`` transcript (``Skill`` tool uses); +the first assistant turn that invokes any is the choice, as for the emulated arm. +Records come out in the answers shape :func:`dokimi.eval.arm_summary.summarize_skill_selection` +reads, so both arms are scored by the same code. +""" + +from __future__ import annotations + +import json +import shutil +import tempfile +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence + +from dokimi import process +from dokimi.harnesses.agent_sandbox import _SEAL_ENV, AgentWorkspace, agent_env, operator_source_env +from dokimi.harnesses.claude_json import result_event + +#: Fixed identity and dates for the staging commit, so the git context the harness +#: shows the model is the same on every run. +_GIT_IDENT = { + "GIT_AUTHOR_NAME": "user", "GIT_AUTHOR_EMAIL": "user@example.com", + "GIT_COMMITTER_NAME": "user", "GIT_COMMITTER_EMAIL": "user@example.com", + "GIT_AUTHOR_DATE": "2026-01-01T00:00:00Z", "GIT_COMMITTER_DATE": "2026-01-01T00:00:00Z", +} +_COMMIT_MESSAGE = "Initial commit" + + +@dataclass +class ProbeConfig: + """How each query is run. ``listing_budget_fraction`` ``None`` keeps the harness default.""" + + model: Optional[str] = None + max_turns: int = 3 + timeout_sec: float = 300.0 + stock: bool = True + listing_budget_fraction: Optional[float] = None + claude: str = "claude" + extra_args: List[str] = field(default_factory=list) + + +#: Prefix of each run's temp root. Claude Code shows the model its working directory, +#: so the name must not say what the run is (``AgentWorkspace.create`` uses ``dokimi-run-``). +WORKSPACE_PREFIX = "project-" + + +def new_workspace() -> AgentWorkspace: + """A scratch ``{work,home,config}`` root whose path names no experiment.""" + root = Path(tempfile.mkdtemp(prefix=WORKSPACE_PREFIX)) + for sub in ("work", "home", "config"): + (root / sub).mkdir() + return AgentWorkspace(root=root) + + +def stage_workspace(catalog: Path, ws: AgentWorkspace, runner: process.Runner = process.run) -> Path: + """Copy *catalog* to ``work/.claude/skills`` and commit it; return the work dir.""" + work = ws.work + shutil.copytree(catalog, work / ".claude" / "skills", symlinks=False) + env = process.Env.only(("PATH",), {**_GIT_IDENT, "HOME": str(ws.root / "home"), + "GIT_CONFIG_NOSYSTEM": "1", "GIT_TERMINAL_PROMPT": "0"}) + for args in (("init", "-q", "-b", "main"), ("add", "-A"), + ("commit", "-q", "--no-gpg-sign", "-m", _COMMIT_MESSAGE)): + done = process.git(*args, cwd=work, env=env, runner=runner) + if done.returncode != 0: + raise RuntimeError(f"git {args[0]} failed staging the catalog: {process.git_failure(done)}") + return work + + +def probe_env(ws: AgentWorkspace, cfg: ProbeConfig, source: Optional[Dict[str, str]] = None) -> Dict[str, str]: + """The allowlisted env, scratch HOME/config; the bundled-skill seal only when not ``stock``.""" + env = agent_env(operator_source_env() if source is None else source) + if cfg.stock: + for k in _SEAL_ENV: + env.pop(k, None) + env["HOME"] = str(ws.root / "home") + env["CLAUDE_CONFIG_DIR"] = str(ws.root / "config") + env["TMPDIR"] = str(ws.root / "home") + return env + + +def probe_argv(query: str, cfg: ProbeConfig) -> List[str]: + """``claude`` argv: the query verbatim, stream-json out, and nothing that steers the model.""" + argv = [cfg.claude, "-p", query, "--output-format", "stream-json", "--verbose", + "--max-turns", str(cfg.max_turns), "--no-session-persistence"] + if cfg.model: + argv += ["--model", cfg.model] + if cfg.listing_budget_fraction is not None: + argv += ["--settings", json.dumps({"skillListingBudgetFraction": cfg.listing_budget_fraction})] + return argv + list(cfg.extra_args) + + +def _events(stream: str) -> Iterable[Dict[str, Any]]: + for line in stream.splitlines(): + line = line.strip() + if not line.startswith("{"): + continue + try: + obj = json.loads(line) + except ValueError: + continue + if isinstance(obj, dict): + yield obj + + +def _skill_name(raw: Any) -> str: + # A plugin skill is invoked as `plugin:skill`; the catalog names the skill alone. + return str(raw or "").rsplit(":", 1)[-1] + + +def parse_stream(stream: str) -> Dict[str, Any]: + """Skills listed and invoked, per the transcript; ``first_turn_loaded`` is the choice.""" + listed: List[str] = [] + version = None + turns: List[List[str]] = [] + other_tools: List[str] = [] + for ev in _events(stream): + if ev.get("type") == "system" and ev.get("subtype") == "init": + listed = [str(s) for s in ev.get("skills") or []] + version = ev.get("claude_code_version") + if ev.get("type") != "assistant": + continue + content = (ev.get("message") or {}).get("content") or [] + uses = [c for c in content if isinstance(c, dict) and c.get("type") == "tool_use"] + skills = [_skill_name((c.get("input") or {}).get("skill")) for c in uses if c.get("name") == "Skill"] + other_tools += [str(c.get("name")) for c in uses if c.get("name") != "Skill"] + if skills: + turns.append(skills) + first = turns[0] if turns else [] + return { + "offered": True, + "listed": listed, + "selected": first[0] if first else None, + "first_turn_loaded": first, + "loaded": [s for t in turns for s in t], + "tools_before_choice": other_tools, + "claude_code_version": version, + } + + +def run_query(query: str, catalog: Path, cfg: ProbeConfig, *, runner: process.Runner = process.run, + source_env: Optional[Dict[str, str]] = None) -> Dict[str, Any]: + """One fresh harness run of *query*; the parsed skill block plus cost and any error.""" + ws = new_workspace() + try: + work = stage_workspace(catalog, ws, runner=runner) + argv = probe_argv(query, cfg) + done = runner(argv, cwd=work, timeout=cfg.timeout_sec, + env=process.Env.only((), probe_env(ws, cfg, source_env)), input="", + capture_limit=None) + stream = done.stdout + skill = parse_stream(stream) + result = result_event(stream) or {} + error = None + if done.timed_out: + error = "timeout" + elif not result: + error = f"no result event (exit {done.returncode})" + # Out of turns without a Skill call is an observation (nothing triggered), not a failure. + elif result.get("is_error") and result.get("subtype") != "error_max_turns" \ + and not skill["first_turn_loaded"]: + error = str(result.get("subtype") or "error") + return {"skill": skill, "cost_usd": result.get("total_cost_usd"), + "num_turns": result.get("num_turns"), "error": error} + finally: + ws.cleanup() + + +def probe_trigger_set(queries: Sequence[Any], catalog: Path, cfg: ProbeConfig, *, trials: int = 1, + arm_name: str = "claude_code", runner: process.Runner = process.run, + on_record: Optional[Callable[[Dict[str, Any]], None]] = None) -> List[Dict[str, Any]]: + """Run every labelled query *trials* times; answers-shaped records for the selection summary. + + *queries* are :class:`dokimi.external.trigger_set.TriggerQuery`. Each trial is its own + arm (``#``), so the summary reports trials side by side. + """ + out: List[Dict[str, Any]] = [] + for q in queries: + if not getattr(q, "labelled", True): + continue + arms = {} + for k in range(trials): + r = run_query(q.text, catalog, cfg, runner=runner) + arms[f"{arm_name}#{k + 1}"] = { + "answer": None, + "error": r["error"], + "tool_usage": {"skill": r["skill"]}, + "metrics": {"cost_usd": r["cost_usd"], "num_turns": r["num_turns"]}, + } + rec = {"question_id": q.id, "question_text": q.text, "expected_skill": q.expected_skill, "arms": arms} + out.append(rec) + if on_record: + on_record(rec) + return out diff --git a/dokimi/tests/test_payload_audit.py b/dokimi/tests/test_payload_audit.py new file mode 100644 index 000000000..74f85fcde --- /dev/null +++ b/dokimi/tests/test_payload_audit.py @@ -0,0 +1,117 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Leak audit: markers in what an agent is sent, applied to the skill-catalog and docker-claude paths.""" + +from __future__ import annotations + +from pathlib import Path + +import dokimi.eval.agent_runner as ar_mod +from dokimi.eval.agent_runner import run_agent_loop +from dokimi.harnesses import load_task +from dokimi.harnesses.docker_solver import DockerSolverHarness +from dokimi.harnesses.payload_audit import added_framing, audit_payload, markers_for, new_leaks +from dokimi.harnesses.skill_treatment import TREATMENT_NOTICE, Treatment +from dokimi.skills import load_skill +from dokimi.treatments.arms import SkillCatalogTreatment + + +def _skill(root: Path, name: str, description: str, body: str) -> Path: + d = root / name + d.mkdir(parents=True) + (d / "SKILL.md").write_text(f"---\nname: {name}\ndescription: {description}\n---\n{body}\n", + encoding="utf-8") + return d / "SKILL.md" + + +def test_planted_markers_are_found_with_their_path(): + payload = {"messages": [{"role": "user", "content": "Q-17: how do I sum?"}], + "meta": {"expected_skill": "x"}, "cwd": "/srv/dokimi-run-ab12/work"} + leaks = audit_payload(payload, markers_for(question_ids=["Q-17"])) + found = {(lk.marker, lk.path) for lk in leaks} + assert ("Q-17", "$.messages[0].content") in found + assert ("expected_skill", "$.meta.expected_skill#key") in found + # A hyphenated path still names the tool: `dokimi-run-` is a leak. + assert ("dokimi", "$.cwd") in found + + +def test_marker_inside_a_longer_word_is_not_a_leak(): + assert audit_payload({"t": "evaluate the baseline_x reevaluation"}, ["eval"]) == [] + assert [lk.marker for lk in audit_payload({"t": "see eval_queries.json"}, ["eval"])] == ["eval"] + + +def test_operator_content_is_blanked_but_dokimi_text_around_it_is_not(): + desc = "Compare against a baseline run before tuning." + payload = {"tools": [{"description": f"Load skill. {desc}"}], + "system": "You are in the treatment arm."} + leaks = audit_payload(payload, allowed=[desc]) + assert [(lk.marker, lk.path) for lk in leaks] == [("treatment", "$.system")] + # Without the allowance the skill's own word is reported too. + assert {lk.marker for lk in audit_payload(payload)} == {"baseline", "treatment"} + + +def test_new_leaks_ignore_what_the_stock_harness_already_says(): + reference = {"system": "Test the golden path and judge edge cases."} + measured = {"system": "Test the golden path and judge edge cases.", "user": "graded by the judge"} + assert [(lk.path, lk.marker) for lk in new_leaks(measured, reference)] == [("$.user", "judge")] + + +def test_skill_catalog_request_carries_no_dokimi_artifacts(tmp_path, monkeypatch): + # The request the skill-catalog arm actually sends (messages + tool schemas), captured + # at the LLM chokepoint, holds the query and the skills' own text and nothing else of ours. + paths = [_skill(tmp_path / "cat", "sum-fast", "Use when summing arrays; beats the baseline loop.", + "# Sum\nUse a reduction."), + _skill(tmp_path / "cat", "io-read", "Use when reading files.", "# IO\nRead in chunks.")] + skills = [load_skill(p) for p in paths] + arm = SkillCatalogTreatment(skills) + question = "How do I sum a big array quickly?" + cfg = arm.prepare(question, "lib") + sent = [] + + class _Msg: + content, tool_calls = "Use a reduction.", None + + class _Resp: + choices = [type("C", (), {"message": _Msg()})()] + usage = type("U", (), {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2})() + + def fake(messages, model, tools=None, **kw): + sent.append({"messages": [dict(m) for m in messages], "tools": tools}) + return _Resp() + + monkeypatch.setattr(ar_mod, "chat_completion", fake) + run_agent_loop(question, cfg.tools, model="m", system_prompt=cfg.system_prompt) + allowed = [question] + [s.description for s in skills] + [s.name for s in skills] + markers = markers_for(question_ids=["q-001"], arm_names=[arm.name], paths=[str(tmp_path)]) + assert sent and audit_payload(sent, markers, allowed) == [] + # The audit is live on this payload: the arm's own name, planted, is caught. + sent[0]["messages"].append({"role": "user", "content": f"arm {arm.name}"}) + assert {lk.marker for lk in audit_payload(sent, markers, allowed)} == {"skill_catalog", arm.name} + + +def _task(tmp_path: Path) -> Path: + task = tmp_path / "content" / "tasks" / "demo-task" + (task / "tests").mkdir(parents=True) + (task / "solution").mkdir() + (task / "environment").mkdir() + (task / "instruction.md").write_text("Make sum() faster.", encoding="utf-8") + (task / "task.toml").write_text("[environment]\n", encoding="utf-8") + return task + + +def test_docker_claude_discover_prompt_is_task_framing_only(tmp_path): + task = load_task(_task(tmp_path)) + treatment = Treatment(name="sum-fast", plugin="sum-fast") + out = tmp_path / "out" + out.mkdir() + prompts = {} + for notice in (True, False): + h = DockerSolverHarness(harness_id="docker-claude", mode="claude", skill_notice=notice) + argv = h._claude_args(task, treatment, "/plugins/sum-fast", "haiku", out) + prompt = argv[argv.index("-p") + 1] + prompts[notice] = added_framing(prompt, task.instruction) + assert audit_payload(prompt, markers_for(paths=[str(task.path)])) == [] + # Discover keeps dokimi's task framing but drops the notice that names the skill. + assert "sum-fast" not in prompts[False] and prompts[False].startswith("Solve this coding task.") + assert prompts[True].startswith(TREATMENT_NOTICE.format(skill_id=treatment.skill_id).strip()) diff --git a/dokimi/tests/test_trigger_probe.py b/dokimi/tests/test_trigger_probe.py new file mode 100644 index 000000000..59015edf3 --- /dev/null +++ b/dokimi/tests/test_trigger_probe.py @@ -0,0 +1,131 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Real-harness trigger probe: staging, argv/env hygiene, transcript parsing, scoring.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from dokimi import process +from dokimi.eval.arm_summary import summarize_skill_selection +from dokimi.external.trigger_set import TriggerQuery +from dokimi.harnesses import trigger_probe as tp +from dokimi.harnesses.payload_audit import audit_payload, markers_for + + +def _catalog(tmp_path: Path) -> Path: + for name in ("sum-fast", "io-read"): + d = tmp_path / "cat" / name + d.mkdir(parents=True) + (d / "SKILL.md").write_text(f"---\nname: {name}\ndescription: Use for {name}.\n---\nbody\n", + encoding="utf-8") + return tmp_path / "cat" + + +def _stream(*skills_per_turn, other: str = "", version: str = "2.1.304") -> str: + lines = [{"type": "system", "subtype": "init", "skills": ["sum-fast", "io-read", "dataviz"], + "claude_code_version": version}] + if other: + lines.append({"type": "assistant", "message": {"content": [ + {"type": "tool_use", "name": other, "input": {}}]}}) + for turn in skills_per_turn: + lines.append({"type": "assistant", "message": {"content": [ + {"type": "text", "text": "ok"}] + [ + {"type": "tool_use", "name": "Skill", "input": {"skill": s}} for s in turn]}}) + lines.append({"type": "result", "subtype": "success", "is_error": False, + "total_cost_usd": 0.01, "num_turns": 2}) + return "\n".join(json.dumps(x) for x in lines) + "\n" + + +def test_parse_stream_takes_the_first_skill_turn_as_the_choice(): + s = tp.parse_stream(_stream(["plug:sum-fast", "io-read"], ["io-read"], other="Bash")) + assert s["first_turn_loaded"] == ["sum-fast", "io-read"] + assert s["selected"] == "sum-fast" + assert s["loaded"] == ["sum-fast", "io-read", "io-read"] + assert s["tools_before_choice"] == ["Bash"] + assert s["listed"] == ["sum-fast", "io-read", "dataviz"] and s["claude_code_version"] == "2.1.304" + assert tp.parse_stream(_stream())["selected"] is None + + +def test_argv_is_the_query_verbatim_and_pins_only_what_is_asked(): + cfg = tp.ProbeConfig(model="haiku", max_turns=2) + argv = tp.probe_argv("How do I sum?", cfg) + assert argv[argv.index("-p") + 1] == "How do I sum?" + assert "--settings" not in argv and "--tools" not in argv and "--plugin-dir" not in argv + pinned = tp.probe_argv("q", tp.ProbeConfig(listing_budget_fraction=0.03)) + assert json.loads(pinned[pinned.index("--settings") + 1]) == {"skillListingBudgetFraction": 0.03} + + +def test_env_keeps_operator_config_out_and_bundled_skills_on_in_stock_mode(tmp_path): + ws = tp.new_workspace() + try: + source = {"PATH": "/bin", "AWS_REGION": "us-east-2", "CLAUDECODE": "1", + "CLAUDE_CONFIG_DIR": "/home/op/.claude", "GITHUB_TOKEN": "x"} + stock = tp.probe_env(ws, tp.ProbeConfig(), source) + sealed = tp.probe_env(ws, tp.ProbeConfig(stock=False), source) + assert stock["CLAUDE_CONFIG_DIR"] == str(ws.root / "config") and stock["HOME"] == str(ws.root / "home") + assert "GITHUB_TOKEN" not in stock and "CLAUDECODE" not in stock + assert "CLAUDE_CODE_DISABLE_BUNDLED_SKILLS" not in stock + assert sealed["CLAUDE_CODE_DISABLE_BUNDLED_SKILLS"] == "1" + finally: + ws.cleanup() + + +def test_run_query_stages_a_clean_repo_whose_path_names_no_experiment(tmp_path): + seen = {} + + def handler(call: process.Call): + if call.argv[0] == "git": + return None if call.argv[1] != "status" else process.reply("") + seen["cwd"] = call.cwd + seen["skills"] = sorted(p.name for p in (call.cwd / ".claude" / "skills").iterdir()) + seen["env"] = call.env + return process.reply(_stream(["io-read"])) + + fake = process.FakeRunner(handler) + # Real git for staging (it is the clean-repo claim under test), fake claude. + def runner(argv, **kw): + return process.run(argv, **kw) if argv[0] == "git" else fake(argv, **kw) + + r = tp.run_query("read a csv", _catalog(tmp_path), tp.ProbeConfig(), runner=runner, + source_env={"PATH": "/usr/bin:/bin"}) + assert r["skill"]["first_turn_loaded"] == ["io-read"] and r["error"] is None and r["cost_usd"] == 0.01 + assert seen["skills"] == ["io-read", "sum-fast"] + assert audit_payload(str(seen["cwd"]), markers_for()) == [] + assert not seen["cwd"].exists() # cleaned up + + +def test_out_of_turns_without_a_skill_is_no_trigger_not_a_failure(tmp_path): + stream = _stream().replace('"subtype": "success", "is_error": false', + '"subtype": "error_max_turns", "is_error": true') + + def runner(argv, **kw): + return process.run(argv, **kw) if argv[0] == "git" else process.reply(stream) + + r = tp.run_query("q", _catalog(tmp_path), tp.ProbeConfig(), runner=runner, source_env={"PATH": "/usr/bin:/bin"}) + assert r["error"] is None and r["skill"]["first_turn_loaded"] == [] + broken = tp.run_query("q", _catalog(tmp_path / "b"), tp.ProbeConfig(), + runner=lambda a, **kw: process.run(a, **kw) if a[0] == "git" else process.reply("", returncode=1), + source_env={"PATH": "/usr/bin:/bin"}) + assert broken["error"] == "no result event (exit 1)" + + +def test_records_score_with_the_same_summary_as_the_emulated_arm(tmp_path): + answers = {"sum it": ["sum-fast"], "read it": ["sum-fast"], "hello": []} + + def runner(argv, **kw): + if argv[0] == "git": + return process.run(argv, **kw) + q = argv[list(argv).index("-p") + 1] + return process.reply(_stream(answers[q]) if answers[q] else _stream()) + + queries = [TriggerQuery("a", "sum it", "sum-fast"), TriggerQuery("b", "read it", "io-read"), + TriggerQuery("c", "hello", None), TriggerQuery("d", "unlabelled", None, labelled=False)] + recs = tp.probe_trigger_set(queries, _catalog(tmp_path), tp.ProbeConfig(), trials=2, runner=runner) + assert [r["question_id"] for r in recs] == ["a", "b", "c"] + s = summarize_skill_selection(recs) + assert set(s) == {"claude_code#1", "claude_code#2"} + assert s["claude_code#1"]["recall"] == 0.5 and s["claude_code#1"]["misroute_rate"] == 0.5 + assert s["claude_code#1"]["false_trigger_rate"] == 0.0 From 63ea270c60cb3c8d02d917fea412bd4a8d5a0ac0 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Sat, 24 Oct 2026 22:27:44 -0700 Subject: [PATCH 05/13] fix(catalog): group trigger-probe Skill calls by message.id Claude Code streams one assistant event per content block, so two Skill calls in one turn were scored as two turns and the strict first-turn rule, multi_load_rate and mean_first_turn_loads never fired. Group by message.id as agent_runner groups by iteration, and skip subagent events. Signed-off-by: Alexander Rybkin --- dokimi/dokimi/harnesses/trigger_probe.py | 16 ++++++++++------ dokimi/tests/test_trigger_probe.py | 21 +++++++++++++++++++++ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/dokimi/dokimi/harnesses/trigger_probe.py b/dokimi/dokimi/harnesses/trigger_probe.py index bce1b0053..01e845e9a 100644 --- a/dokimi/dokimi/harnesses/trigger_probe.py +++ b/dokimi/dokimi/harnesses/trigger_probe.py @@ -132,20 +132,24 @@ def parse_stream(stream: str) -> Dict[str, Any]: """Skills listed and invoked, per the transcript; ``first_turn_loaded`` is the choice.""" listed: List[str] = [] version = None - turns: List[List[str]] = [] + # Claude Code emits one assistant event per content block; the blocks of one turn + # share message.id. Subagent events (parent_tool_use_id set) are not the choice. + by_turn: Dict[str, List[str]] = {} other_tools: List[str] = [] - for ev in _events(stream): + for n, ev in enumerate(_events(stream)): if ev.get("type") == "system" and ev.get("subtype") == "init": listed = [str(s) for s in ev.get("skills") or []] version = ev.get("claude_code_version") - if ev.get("type") != "assistant": + if ev.get("type") != "assistant" or ev.get("parent_tool_use_id"): continue - content = (ev.get("message") or {}).get("content") or [] - uses = [c for c in content if isinstance(c, dict) and c.get("type") == "tool_use"] + message = ev.get("message") or {} + uses = [c for c in message.get("content") or [] if isinstance(c, dict) and c.get("type") == "tool_use"] skills = [_skill_name((c.get("input") or {}).get("skill")) for c in uses if c.get("name") == "Skill"] other_tools += [str(c.get("name")) for c in uses if c.get("name") != "Skill"] if skills: - turns.append(skills) + turn = by_turn.setdefault(str(message.get("id") or f"#{n}"), []) + turn += [s for s in dict.fromkeys(skills) if s not in turn] + turns = list(by_turn.values()) first = turns[0] if turns else [] return { "offered": True, diff --git a/dokimi/tests/test_trigger_probe.py b/dokimi/tests/test_trigger_probe.py index 59015edf3..5dab7d426 100644 --- a/dokimi/tests/test_trigger_probe.py +++ b/dokimi/tests/test_trigger_probe.py @@ -49,6 +49,27 @@ def test_parse_stream_takes_the_first_skill_turn_as_the_choice(): assert tp.parse_stream(_stream())["selected"] is None +def test_parse_stream_groups_per_block_events_by_message_id(): + # Claude Code streams each content block as its own event; one turn shares message.id. + def block(mid, block, parent=None): + return {"type": "assistant", "parent_tool_use_id": parent, "message": {"id": mid, "content": [block]}} + + def skill(s): + return {"type": "tool_use", "name": "Skill", "input": {"skill": s}} + + events = [block("m1", {"type": "thinking", "thinking": "..."}), block("m1", skill("sum-fast")), + block("m1", skill("io-read")), {"type": "user", "message": {"content": []}}, + block("m2", skill("dataviz"), parent="toolu_1"), block("m3", skill("sum-fast")), + {"type": "result", "subtype": "success", "is_error": False}] + s = tp.parse_stream("\n".join(json.dumps(e) for e in events)) + assert s["first_turn_loaded"] == ["sum-fast", "io-read"] + assert s["loaded"] == ["sum-fast", "io-read", "sum-fast"] + rec = {"question_id": "a", "expected_skill": "sum-fast", + "arms": {"cc": {"answer": None, "error": None, "tool_usage": {"skill": s}}}} + summary = summarize_skill_selection([rec])["cc"] + assert summary["recall"] == 0.0 and summary["multi_load_rate"] == 1.0 + + def test_argv_is_the_query_verbatim_and_pins_only_what_is_asked(): cfg = tp.ProbeConfig(model="haiku", max_turns=2) argv = tp.probe_argv("How do I sum?", cfg) From d99619088c2584906863e030e348534e1f260a0c Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Sat, 24 Oct 2026 22:28:16 -0700 Subject: [PATCH 06/13] fix(catalog): score a trigger-probe choice made before a timeout run_query checked timed_out first, so a query whose Skill call was already in the transcript, or whose result event was read before the CLI hung on a backgrounded Bash call, was dropped as n_failed. A Skill call seen is now the choice, and a result event decides the rest. Signed-off-by: Alexander Rybkin --- dokimi/dokimi/harnesses/trigger_probe.py | 11 ++++++----- dokimi/tests/test_trigger_probe.py | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/dokimi/dokimi/harnesses/trigger_probe.py b/dokimi/dokimi/harnesses/trigger_probe.py index 01e845e9a..b4a0b7f0c 100644 --- a/dokimi/dokimi/harnesses/trigger_probe.py +++ b/dokimi/dokimi/harnesses/trigger_probe.py @@ -176,13 +176,14 @@ def run_query(query: str, catalog: Path, cfg: ProbeConfig, *, runner: process.Ru skill = parse_stream(stream) result = result_event(stream) or {} error = None - if done.timed_out: - error = "timeout" + # A Skill call seen is the choice, whatever came after it (a hang, a timeout, an + # error). A result event read decides the rest, even if the CLI then hung. + if skill["first_turn_loaded"]: + pass elif not result: - error = f"no result event (exit {done.returncode})" + error = "timeout" if done.timed_out else f"no result event (exit {done.returncode})" # Out of turns without a Skill call is an observation (nothing triggered), not a failure. - elif result.get("is_error") and result.get("subtype") != "error_max_turns" \ - and not skill["first_turn_loaded"]: + elif result.get("is_error") and result.get("subtype") != "error_max_turns": error = str(result.get("subtype") or "error") return {"skill": skill, "cost_usd": result.get("total_cost_usd"), "num_turns": result.get("num_turns"), "error": error} diff --git a/dokimi/tests/test_trigger_probe.py b/dokimi/tests/test_trigger_probe.py index 5dab7d426..06e0933bc 100644 --- a/dokimi/tests/test_trigger_probe.py +++ b/dokimi/tests/test_trigger_probe.py @@ -133,6 +133,22 @@ def runner(argv, **kw): assert broken["error"] == "no result event (exit 1)" +def test_a_hang_after_the_choice_is_scored_not_dropped(tmp_path): + # A stock-tool Bash call can outlive the turn; the CLI is then killed at --timeout. + with_result = _stream(["sum-fast"]) + no_result = with_result.rsplit('{"type": "result"', 1)[0] + cases = {"after-result": (with_result, None, ["sum-fast"]), "no-result": (no_result, None, ["sum-fast"]), + "no-skill-after-result": (_stream(), None, []), "nothing": ("", "timeout", [])} + for name, (stream, error, loaded) in cases.items(): + def runner(argv, stream=stream, **kw): + return process.run(argv, **kw) if argv[0] == "git" else process.reply(stream, timed_out=True) + + r = tp.run_query("q", _catalog(tmp_path / name), tp.ProbeConfig(), runner=runner, + source_env={"PATH": "/usr/bin:/bin"}) + assert r["error"] == error, name + assert r["skill"]["first_turn_loaded"] == loaded, name + + def test_records_score_with_the_same_summary_as_the_emulated_arm(tmp_path): answers = {"sum it": ["sum-fast"], "read it": ["sum-fast"], "hello": []} From 25c992bd31afafd78b26d22cfa25d0081d834df8 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Sat, 24 Oct 2026 22:28:45 -0700 Subject: [PATCH 07/13] fix(catalog): apply the arms' plugin seal in trigger-probe --sealed --sealed set only the env seal, so the built-in plugin skills the docker-claude arms switch off via --settings stayed listed. Sealed argv now carries the same enabledPlugins, merged with --listing-budget-fraction into one --settings. Signed-off-by: Alexander Rybkin --- dokimi/dokimi/harnesses/trigger_probe.py | 10 ++++++++-- dokimi/tests/test_trigger_probe.py | 9 +++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/harnesses/trigger_probe.py b/dokimi/dokimi/harnesses/trigger_probe.py index b4a0b7f0c..04b134735 100644 --- a/dokimi/dokimi/harnesses/trigger_probe.py +++ b/dokimi/dokimi/harnesses/trigger_probe.py @@ -34,7 +34,9 @@ from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence from dokimi import process -from dokimi.harnesses.agent_sandbox import _SEAL_ENV, AgentWorkspace, agent_env, operator_source_env +from dokimi.harnesses.agent_sandbox import ( + _SEAL_ENV, _SEAL_SETTINGS, AgentWorkspace, agent_env, operator_source_env, +) from dokimi.harnesses.claude_json import result_event #: Fixed identity and dates for the staging commit, so the git context the harness @@ -105,8 +107,12 @@ def probe_argv(query: str, cfg: ProbeConfig) -> List[str]: "--max-turns", str(cfg.max_turns), "--no-session-persistence"] if cfg.model: argv += ["--model", cfg.model] + # One --settings: sealed carries the arms' plugin seal, merged with any listing budget. + settings: Dict[str, Any] = {} if cfg.stock else json.loads(json.dumps(_SEAL_SETTINGS)) if cfg.listing_budget_fraction is not None: - argv += ["--settings", json.dumps({"skillListingBudgetFraction": cfg.listing_budget_fraction})] + settings["skillListingBudgetFraction"] = cfg.listing_budget_fraction + if settings: + argv += ["--settings", json.dumps(settings)] return argv + list(cfg.extra_args) diff --git a/dokimi/tests/test_trigger_probe.py b/dokimi/tests/test_trigger_probe.py index 06e0933bc..5dadfe758 100644 --- a/dokimi/tests/test_trigger_probe.py +++ b/dokimi/tests/test_trigger_probe.py @@ -12,6 +12,7 @@ from dokimi.eval.arm_summary import summarize_skill_selection from dokimi.external.trigger_set import TriggerQuery from dokimi.harnesses import trigger_probe as tp +from dokimi.harnesses.agent_sandbox import solver_cli_flags from dokimi.harnesses.payload_audit import audit_payload, markers_for @@ -79,6 +80,14 @@ def test_argv_is_the_query_verbatim_and_pins_only_what_is_asked(): assert json.loads(pinned[pinned.index("--settings") + 1]) == {"skillListingBudgetFraction": 0.03} +def test_sealed_argv_switches_off_the_builtin_plugins_as_the_arms_do(): + seal = json.loads(solver_cli_flags()[solver_cli_flags().index("--settings") + 1]) + for frac, extra in ((None, {}), (0.05, {"skillListingBudgetFraction": 0.05})): + argv = tp.probe_argv("q", tp.ProbeConfig(stock=False, listing_budget_fraction=frac)) + assert argv.count("--settings") == 1 + assert json.loads(argv[argv.index("--settings") + 1]) == {**seal, **extra} + + def test_env_keeps_operator_config_out_and_bundled_skills_on_in_stock_mode(tmp_path): ws = tp.new_workspace() try: From 0d834120b8d1665fd08fb1ab4918c18bb53e3cd0 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Sat, 24 Oct 2026 22:29:40 -0700 Subject: [PATCH 08/13] fix(catalog): keep trigger-probe results when one run fails A relative --claude passed the which() preflight but was spawned from the scratch work dir, and any OSError/RuntimeError in one run escaped pool.map and lost every paid result before --out was written. Resolve --claude to an absolute path and record a failed run as that trial's error. Signed-off-by: Alexander Rybkin --- dokimi/dokimi/commands/catalog_probe.py | 7 +++-- dokimi/dokimi/harnesses/trigger_probe.py | 7 ++++- dokimi/tests/test_trigger_probe.py | 40 ++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/commands/catalog_probe.py b/dokimi/dokimi/commands/catalog_probe.py index d55177e84..42b86d3c3 100644 --- a/dokimi/dokimi/commands/catalog_probe.py +++ b/dokimi/dokimi/commands/catalog_probe.py @@ -7,6 +7,7 @@ import argparse import json +import os import shutil import sys from concurrent.futures import ThreadPoolExecutor @@ -21,7 +22,9 @@ def cmd_catalog_trigger_probe(args: argparse.Namespace) -> None: catalog = Path(args.catalog) if not catalog.is_dir(): raise SystemExit(f"--catalog is not a directory: {catalog}") - if shutil.which(args.claude) is None: + # Absolute: each run's cwd is a scratch dir, where a relative path names nothing. + claude = shutil.which(args.claude) + if claude is None: raise SystemExit(f"{args.claude!r} is not on PATH") try: data = json.loads(Path(args.trigger_set).read_text(encoding="utf-8")) @@ -31,7 +34,7 @@ def cmd_catalog_trigger_probe(args: argparse.Namespace) -> None: queries = [q for q in queries if q.labelled][: args.limit or None] cfg = tp.ProbeConfig(model=args.model, max_turns=args.max_turns, timeout_sec=args.timeout, stock=not args.sealed, listing_budget_fraction=args.listing_budget_fraction, - claude=args.claude) + claude=os.path.abspath(claude)) def one(q): rec = tp.probe_trigger_set([q], catalog, cfg, trials=args.trials)[0] diff --git a/dokimi/dokimi/harnesses/trigger_probe.py b/dokimi/dokimi/harnesses/trigger_probe.py index 04b134735..f52e6c071 100644 --- a/dokimi/dokimi/harnesses/trigger_probe.py +++ b/dokimi/dokimi/harnesses/trigger_probe.py @@ -211,7 +211,12 @@ def probe_trigger_set(queries: Sequence[Any], catalog: Path, cfg: ProbeConfig, * continue arms = {} for k in range(trials): - r = run_query(q.text, catalog, cfg, runner=runner) + try: + r = run_query(q.text, catalog, cfg, runner=runner) + except (OSError, RuntimeError) as exc: + # One failed run (staging, spawn) is that trial's error, not the lost set. + r = {"skill": parse_stream(""), "cost_usd": None, "num_turns": None, + "error": f"{type(exc).__name__}: {exc}"} arms[f"{arm_name}#{k + 1}"] = { "answer": None, "error": r["error"], diff --git a/dokimi/tests/test_trigger_probe.py b/dokimi/tests/test_trigger_probe.py index 5dadfe758..05f7afa11 100644 --- a/dokimi/tests/test_trigger_probe.py +++ b/dokimi/tests/test_trigger_probe.py @@ -6,9 +6,13 @@ from __future__ import annotations import json +import os from pathlib import Path +import pytest + from dokimi import process +from dokimi.cli import build_parser from dokimi.eval.arm_summary import summarize_skill_selection from dokimi.external.trigger_set import TriggerQuery from dokimi.harnesses import trigger_probe as tp @@ -175,3 +179,39 @@ def runner(argv, **kw): assert set(s) == {"claude_code#1", "claude_code#2"} assert s["claude_code#1"]["recall"] == 0.5 and s["claude_code#1"]["misroute_rate"] == 0.5 assert s["claude_code#1"]["false_trigger_rate"] == 0.0 + + +def test_one_failed_run_is_that_trials_error_not_the_lost_set(tmp_path): + def runner(argv, **kw): + if argv[0] == "git": + return process.run(argv, **kw) + if argv[list(argv).index("-p") + 1] == "read it": + raise FileNotFoundError(2, "No such file or directory", argv[0]) + return process.reply(_stream(["sum-fast"])) + + queries = [TriggerQuery("a", "sum it", "sum-fast"), TriggerQuery("b", "read it", "io-read")] + recs = tp.probe_trigger_set(queries, _catalog(tmp_path), tp.ProbeConfig(), runner=runner) + assert [r["question_id"] for r in recs] == ["a", "b"] + assert recs[1]["arms"]["claude_code#1"]["error"].startswith("FileNotFoundError") + s = summarize_skill_selection(recs)["claude_code#1"] + assert s["recall"] == 1.0 and s["n_failed"] == 1 + + +@pytest.mark.skipif(os.name == "nt", reason="a shell-script stand-in for claude") +def test_cli_runs_a_relative_claude_from_the_scratch_cwd(tmp_path, monkeypatch, capsys): + (tmp_path / "stream.jsonl").write_text(_stream(["sum-fast"]), encoding="utf-8") + fake = tmp_path / "bin" / "claude" + fake.parent.mkdir() + fake.write_text("#!/bin/sh\nexec cat \"$STREAM\"\n".replace("$STREAM", str(tmp_path / "stream.jsonl")), + encoding="utf-8") + fake.chmod(0o755) + (tmp_path / "ts.json").write_text(json.dumps({"questions": [ + {"id": "a", "question": "sum it", "expected_skill": "sum-fast"}]}), encoding="utf-8") + _catalog(tmp_path) + monkeypatch.chdir(tmp_path) + args = build_parser().parse_args(["catalog", "trigger-probe", "--catalog", "cat", "--trigger-set", + "ts.json", "--claude", "bin/claude", "--out", "probe.json"]) + args.func(args) + out = json.loads((tmp_path / "probe.json").read_text(encoding="utf-8")) + arm = out["answers"][0]["arms"]["claude_code#1"] + assert arm["error"] is None and arm["tool_usage"]["skill"]["first_turn_loaded"] == ["sum-fast"] From e27d2bde0656037151d4f5ef140bfcf2c0545641 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 26 Oct 2026 20:32:09 -0700 Subject: [PATCH 09/13] fix(catalog): address trigger-probe review on #217 - tools_before_choice stops at the first Skill turn. - --out directory and --target are checked before any paid run; records are written after each query, so an interrupt keeps what ran. - Fail when a catalog skill is missing from the init event's listing; report runs that ended on error_max_turns (max_turns_hit). - Kill processes left in the run's workspace (a backgrounded Bash job runs in its own session and escaped the group kill). - Docs: the operator's config is not loaded, but host paths stay readable. - Tests: every CLI flag reaches ProbeConfig, argv/env/timeout/stdin of a run, symlinked skills copied as files, payload-audit case, lookbehind, tuple and longest-first blanking. Signed-off-by: Nikolay Petrov --- dokimi/docs/evaluating-treatments.md | 5 +- dokimi/dokimi/commands/catalog_probe.py | 36 ++++++- dokimi/dokimi/harnesses/trigger_probe.py | 41 +++++++- dokimi/tests/test_payload_audit.py | 12 +++ dokimi/tests/test_trigger_probe.py | 121 +++++++++++++++++++++++ 5 files changed, 206 insertions(+), 9 deletions(-) diff --git a/dokimi/docs/evaluating-treatments.md b/dokimi/docs/evaluating-treatments.md index e817e393a..e652d8ef3 100644 --- a/dokimi/docs/evaluating-treatments.md +++ b/dokimi/docs/evaluating-treatments.md @@ -584,8 +584,9 @@ uv run reach diff --vary description control.json treatment.json # A/B vs nois **Real harness, in dokimi: `catalog trigger-probe`.** Runs each query in a fresh `claude -p`: a new git repository with the catalog committed in `.claude/skills/`, a -scratch `HOME` and `CLAUDE_CONFIG_DIR` (your own skills, settings and `CLAUDE.md` never -reach it), the query sent verbatim with no notice or task framing, and bundled skills +scratch `HOME` and `CLAUDE_CONFIG_DIR` (your own skills, settings and `CLAUDE.md` are not +loaded; the model can still read any host path it has access to, the trigger set and its +labels included), the query sent verbatim with no notice or task framing, and bundled skills left on as a user has them (`--sealed` turns them off). The skill chosen is read from the transcript's `Skill` calls, and the output's `skill_selection_summary` is scored by the same code as the `skill-catalog:` arm, one column per trial: diff --git a/dokimi/dokimi/commands/catalog_probe.py b/dokimi/dokimi/commands/catalog_probe.py index 42b86d3c3..3949afc28 100644 --- a/dokimi/dokimi/commands/catalog_probe.py +++ b/dokimi/dokimi/commands/catalog_probe.py @@ -10,6 +10,7 @@ import os import shutil import sys +import threading from concurrent.futures import ThreadPoolExecutor from pathlib import Path @@ -18,10 +19,19 @@ def cmd_catalog_trigger_probe(args: argparse.Namespace) -> None: from dokimi.eval.arm_summary import summarize_skill_selection from dokimi.external import trigger_set as ts from dokimi.harnesses import trigger_probe as tp + from dokimi.skills import discover_skills catalog = Path(args.catalog) if not catalog.is_dir(): raise SystemExit(f"--catalog is not a directory: {catalog}") + names = {s.name for s in discover_skills(catalog)} + if args.target and args.target not in names: + raise SystemExit(f"--target {args.target!r} is not a skill in --catalog " + f"(have: {', '.join(sorted(names))})") + # Checked before any paid run, so a bad --out costs nothing. + out_path = Path(args.out) + if not out_path.parent.is_dir(): + raise SystemExit(f"--out: directory does not exist: {out_path.parent}") # Absolute: each run's cwd is a scratch dir, where a relative path names nothing. claude = shutil.which(args.claude) if claude is None: @@ -36,8 +46,14 @@ def cmd_catalog_trigger_probe(args: argparse.Namespace) -> None: stock=not args.sealed, listing_budget_fraction=args.listing_budget_fraction, claude=os.path.abspath(claude)) + done: list = [] + lock = threading.Lock() + def one(q): rec = tp.probe_trigger_set([q], catalog, cfg, trials=args.trials)[0] + with lock: # what has run so far survives an interrupt or a later failure + done.append(rec) + _write(out_path, {"partial": True, "answers": done}) print(f"{q.id}: " + ", ".join( f"{a}={'+'.join(v['tool_usage']['skill']['first_turn_loaded']) or '-'}" + (f" ({v['error']})" if v["error"] else "") for a, v in rec["arms"].items()), @@ -46,6 +62,11 @@ def one(q): with ThreadPoolExecutor(max(1, args.concurrency)) as pool: records = list(pool.map(one, queries)) + runs = [v for r in records for v in r["arms"].values()] + listed = {s.rsplit(":", 1)[-1] for v in runs for s in v["tool_usage"]["skill"]["listed"]} + # A skill the harness never offered (a frontmatter name that is not its directory's, + # a nested layout) reads as a 0% recall; say so instead of scoring it. + not_listed = sorted(names - listed) if listed else [] cost = sum(v["metrics"]["cost_usd"] or 0 for r in records for v in r["arms"].values()) out = { "harness": "claude-code", @@ -59,14 +80,25 @@ def one(q): for v in r["arms"].values() if v["tool_usage"]["skill"]["claude_code_version"]), None), "cost_usd": round(cost, 4), + "max_turns_hit": sum(v["metrics"].get("stop") == "error_max_turns" for v in runs), + "not_listed": not_listed, "answers": records, "skill_selection_summary": summarize_skill_selection(records), } - Path(args.out).write_text(json.dumps(out, indent=2) + "\n", encoding="utf-8") + _write(out_path, out) print(json.dumps({"cost_usd": out["cost_usd"], "claude_code_version": out["claude_code_version"], + "max_turns_hit": out["max_turns_hit"], "not_listed": not_listed, "summary": {a: {k: s[k] for k in ("recall", "false_trigger_rate", "misroute_rate", "n_failed")} for a, s in out["skill_selection_summary"].items()}}, indent=2)) print(f"wrote {args.out}") + if not_listed: + raise SystemExit(f"catalog skills Claude Code did not list: {', '.join(not_listed)}") + + +def _write(path: Path, data: dict) -> None: + tmp = path.with_name(path.name + ".tmp") + tmp.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") + os.replace(tmp, path) def register(catalog_sub, positive_int) -> None: @@ -83,7 +115,7 @@ def register(catalog_sub, positive_int) -> None: p.add_argument("--model", help="--model passed to claude (default: the harness default)") p.add_argument("--trials", type=positive_int, default=1, help="Fresh runs per query (default: 1)") p.add_argument("--max-turns", type=positive_int, default=3, dest="max_turns", - help="claude --max-turns (default: 3)") + help="claude --max-turns (default: 3); runs that hit it are counted in max_turns_hit") p.add_argument("--timeout", type=float, default=300.0, help="Seconds per run (default: 300)") p.add_argument("--concurrency", type=positive_int, default=4, help="Queries run at once (default: 4)") p.add_argument("--limit", type=positive_int, help="Only the first N labelled queries") diff --git a/dokimi/dokimi/harnesses/trigger_probe.py b/dokimi/dokimi/harnesses/trigger_probe.py index f52e6c071..255eb2320 100644 --- a/dokimi/dokimi/harnesses/trigger_probe.py +++ b/dokimi/dokimi/harnesses/trigger_probe.py @@ -13,7 +13,8 @@ no checkout to walk), with the catalog committed in ``.claude/skills/``, where Claude Code looks for project skills, so ``git status`` shows nothing unusual; * ``HOME`` and ``CLAUDE_CONFIG_DIR`` are fresh and the env is the sandbox allowlist, - so the operator's skills, settings and ``CLAUDE.md`` never reach the run; + so the operator's skills, settings and ``CLAUDE.md`` are not loaded (the model can + still read any host path it is allowed to, the trigger set included); * the prompt is the query verbatim: no treatment notice, no task framing; * in ``stock`` mode (the default) the bundled skills and tools stay on, as a user has them; ``sealed`` switches them off like the docker-claude arms do. @@ -27,7 +28,9 @@ from __future__ import annotations import json +import os import shutil +import signal import tempfile from dataclasses import dataclass, field from pathlib import Path @@ -151,7 +154,8 @@ def parse_stream(stream: str) -> Dict[str, Any]: message = ev.get("message") or {} uses = [c for c in message.get("content") or [] if isinstance(c, dict) and c.get("type") == "tool_use"] skills = [_skill_name((c.get("input") or {}).get("skill")) for c in uses if c.get("name") == "Skill"] - other_tools += [str(c.get("name")) for c in uses if c.get("name") != "Skill"] + if not by_turn: # tools after the choice are not "before" it + other_tools += [str(c.get("name")) for c in uses if c.get("name") != "Skill"] if skills: turn = by_turn.setdefault(str(message.get("id") or f"#{n}"), []) turn += [s for s in dict.fromkeys(skills) if s not in turn] @@ -192,11 +196,38 @@ def run_query(query: str, catalog: Path, cfg: ProbeConfig, *, runner: process.Ru elif result.get("is_error") and result.get("subtype") != "error_max_turns": error = str(result.get("subtype") or "error") return {"skill": skill, "cost_usd": result.get("total_cost_usd"), - "num_turns": result.get("num_turns"), "error": error} + "num_turns": result.get("num_turns"), "stop": result.get("subtype"), "error": error} finally: + kill_strays(ws.root) ws.cleanup() +def kill_strays(root: Path) -> List[int]: + """SIGKILL every process whose cwd is under *root*; return their pids (Linux only). + + A command the model backgrounds in its Bash tool runs in a session of its own, so + the timeout's process-group kill misses it and it outlives the run and its workspace. + """ + if not Path("/proc").is_dir(): + return [] + prefix = str(Path(root).resolve()) + killed = [] + for entry in os.listdir("/proc"): + if not entry.isdigit() or int(entry) == os.getpid(): + continue + try: + cwd = os.readlink(f"/proc/{entry}/cwd") + except OSError: # exited meanwhile, or not ours to read + continue + if cwd == prefix or cwd.startswith(prefix + os.sep): + try: + os.kill(int(entry), signal.SIGKILL) + killed.append(int(entry)) + except OSError: + pass + return killed + + def probe_trigger_set(queries: Sequence[Any], catalog: Path, cfg: ProbeConfig, *, trials: int = 1, arm_name: str = "claude_code", runner: process.Runner = process.run, on_record: Optional[Callable[[Dict[str, Any]], None]] = None) -> List[Dict[str, Any]]: @@ -215,13 +246,13 @@ def probe_trigger_set(queries: Sequence[Any], catalog: Path, cfg: ProbeConfig, * r = run_query(q.text, catalog, cfg, runner=runner) except (OSError, RuntimeError) as exc: # One failed run (staging, spawn) is that trial's error, not the lost set. - r = {"skill": parse_stream(""), "cost_usd": None, "num_turns": None, + r = {"skill": parse_stream(""), "cost_usd": None, "num_turns": None, "stop": None, "error": f"{type(exc).__name__}: {exc}"} arms[f"{arm_name}#{k + 1}"] = { "answer": None, "error": r["error"], "tool_usage": {"skill": r["skill"]}, - "metrics": {"cost_usd": r["cost_usd"], "num_turns": r["num_turns"]}, + "metrics": {"cost_usd": r["cost_usd"], "num_turns": r["num_turns"], "stop": r.get("stop")}, } rec = {"question_id": q.id, "question_text": q.text, "expected_skill": q.expected_skill, "arms": arms} out.append(rec) diff --git a/dokimi/tests/test_payload_audit.py b/dokimi/tests/test_payload_audit.py index 74f85fcde..1d2d284b2 100644 --- a/dokimi/tests/test_payload_audit.py +++ b/dokimi/tests/test_payload_audit.py @@ -41,6 +41,18 @@ def test_marker_inside_a_longer_word_is_not_a_leak(): assert [lk.marker for lk in audit_payload({"t": "see eval_queries.json"}, ["eval"])] == ["eval"] +def test_markers_match_any_case_only_between_non_alphanumerics_and_inside_tuples(): + assert [lk.marker for lk in audit_payload({"t": "Run by DOKIMI"}, ["dokimi"])] == ["dokimi"] + assert audit_payload({"t": "xdokimi and dokimix"}, ["dokimi"]) == [] + assert [(lk.path, lk.marker) for lk in audit_payload({"t": ("ok", "the judge")}, ["judge"])] == [ + ("$.t[1]", "judge")] + + +def test_allowed_texts_blank_longest_first(): + # Shortest first would blank "skill" inside the longer allowance, leaving "eval" visible. + assert audit_payload({"t": "eval the skill"}, ["eval"], allowed=["skill", "eval the skill"]) == [] + + def test_operator_content_is_blanked_but_dokimi_text_around_it_is_not(): desc = "Compare against a baseline run before tuning." payload = {"tools": [{"description": f"Load skill. {desc}"}], diff --git a/dokimi/tests/test_trigger_probe.py b/dokimi/tests/test_trigger_probe.py index 05f7afa11..595b0529d 100644 --- a/dokimi/tests/test_trigger_probe.py +++ b/dokimi/tests/test_trigger_probe.py @@ -54,6 +54,15 @@ def test_parse_stream_takes_the_first_skill_turn_as_the_choice(): assert tp.parse_stream(_stream())["selected"] is None +def test_tools_after_the_choice_are_not_counted_before_it(): + events = [{"type": "assistant", "message": {"id": "m1", "content": [ + {"type": "tool_use", "name": "Skill", "input": {"skill": "sum-fast"}}]}}, + {"type": "assistant", "message": {"id": "m2", "content": [ + {"type": "tool_use", "name": "Bash", "input": {}}]}}] + s = tp.parse_stream("\n".join(json.dumps(e) for e in events)) + assert s["first_turn_loaded"] == ["sum-fast"] and s["tools_before_choice"] == [] + + def test_parse_stream_groups_per_block_events_by_message_id(): # Claude Code streams each content block as its own event; one turn shares message.id. def block(mid, block, parent=None): @@ -215,3 +224,115 @@ def test_cli_runs_a_relative_claude_from_the_scratch_cwd(tmp_path, monkeypatch, out = json.loads((tmp_path / "probe.json").read_text(encoding="utf-8")) arm = out["answers"][0]["arms"]["claude_code#1"] assert arm["error"] is None and arm["tool_usage"]["skill"]["first_turn_loaded"] == ["sum-fast"] + + +def test_run_query_passes_argv_env_timeout_and_empty_stdin(tmp_path): + seen = {} + + def runner(argv, **kw): + if argv[0] == "git": + return process.run(argv, **kw) + seen.update(kw, argv=list(argv)) + return process.reply(_stream(["sum-fast"])) + + cfg = tp.ProbeConfig(model="haiku", max_turns=7, timeout_sec=12.5, stock=False, + listing_budget_fraction=0.02, claude="/opt/claude") + tp.run_query("sum it", _catalog(tmp_path), cfg, runner=runner, source_env={"PATH": "/bin"}) + argv = seen["argv"] + assert argv[:3] == ["/opt/claude", "-p", "sum it"] + for flag, value in (("--output-format", "stream-json"), ("--max-turns", "7"), ("--model", "haiku")): + assert argv[argv.index(flag) + 1] == value + assert "--verbose" in argv and "--no-session-persistence" in argv + assert json.loads(argv[argv.index("--settings") + 1])["skillListingBudgetFraction"] == 0.02 + env = seen["env"].resolve() + root = Path(env["HOME"]).parent + assert env["TMPDIR"] == env["HOME"] == str(root / "home") and env["CLAUDE_CONFIG_DIR"] == str(root / "config") + assert env["CLAUDE_CODE_DISABLE_BUNDLED_SKILLS"] == "1" + assert seen["timeout"] == 12.5 and seen["input"] == "" and seen["cwd"] == root / "work" + + +def test_staging_copies_symlinked_skills_as_files(tmp_path): + cat = _catalog(tmp_path) + (tmp_path / "ext").mkdir() + (tmp_path / "ext" / "ref.md").write_text("ref", encoding="utf-8") + (cat / "io-read" / "ref.md").symlink_to(tmp_path / "ext" / "ref.md") + ws = tp.new_workspace() + try: + copied = tp.stage_workspace(cat, ws) / ".claude" / "skills" / "io-read" / "ref.md" + assert not copied.is_symlink() and copied.read_text(encoding="utf-8") == "ref" + finally: + ws.cleanup() + + +@pytest.mark.skipif(not Path("/proc/self/cwd").exists(), reason="reads /proc") +def test_kill_strays_ends_a_backgrounded_process_left_in_the_workspace(tmp_path): + import subprocess + + (tmp_path / "work").mkdir() + # Its own session, as the Bash tool's background job is: a group kill does not reach it. + proc = subprocess.Popen(["sleep", "60"], cwd=tmp_path / "work", start_new_session=True) + try: + assert tp.kill_strays(tmp_path) == [proc.pid] + assert proc.wait(timeout=5) == -9 + finally: + proc.kill() + + +def _cli(tmp_path, monkeypatch, *extra, init_skills=("sum-fast", "io-read")): + from dokimi.commands import catalog_probe + + seen = {} + + def fake_probe(queries, catalog, cfg, *, trials=1, **kw): + seen.update(cfg=cfg, trials=trials, catalog=catalog) + seen.setdefault("queries", []).extend(q.id for q in queries) + skill = tp.parse_stream(_stream(["sum-fast"]).replace( + '["sum-fast", "io-read", "dataviz"]', json.dumps(list(init_skills)))) + return [{"question_id": q.id, "question_text": q.text, "expected_skill": q.expected_skill, + "arms": {"claude_code#1": {"answer": None, "error": None, "tool_usage": {"skill": skill}, + "metrics": {"cost_usd": 0.01, "num_turns": 3, + "stop": "error_max_turns"}}}} for q in queries] + + class Pool(catalog_probe.ThreadPoolExecutor): + def __init__(self, n): + seen["concurrency"] = n + super().__init__(n) + + monkeypatch.setattr(tp, "probe_trigger_set", fake_probe) + monkeypatch.setattr(catalog_probe, "ThreadPoolExecutor", Pool) + (tmp_path / "ts.json").write_text(json.dumps({"questions": [ + {"id": q, "question": f"q {q}", "expected_skill": "sum-fast"} for q in ("a", "b", "c")]}), + encoding="utf-8") + monkeypatch.chdir(tmp_path) + if not (tmp_path / "cat").exists(): + _catalog(tmp_path) + args = build_parser().parse_args(["catalog", "trigger-probe", "--catalog", "cat", "--trigger-set", + "ts.json", "--claude", "sh", *extra]) + args.func(args) + return seen + + +def test_cli_passes_every_flag_to_the_probe(tmp_path, monkeypatch): + seen = _cli(tmp_path, monkeypatch, "--model", "haiku", "--limit", "2", "--trials", "3", + "--max-turns", "6", "--timeout", "9", "--listing-budget-fraction", "0.04", + "--concurrency", "5", "--sealed", "--out", "probe.json") + cfg = seen["cfg"] + assert (cfg.model, cfg.max_turns, cfg.timeout_sec, cfg.stock, cfg.listing_budget_fraction) == ( + "haiku", 6, 9.0, False, 0.04) + assert os.path.isabs(cfg.claude) and seen["trials"] == 3 and seen["concurrency"] == 5 + assert sorted(seen["queries"]) == ["a", "b"] + out = json.loads((tmp_path / "probe.json").read_text(encoding="utf-8")) + assert out["mode"] == "sealed" and out["max_turns_hit"] == 2 and out["not_listed"] == [] + + +def test_cli_refuses_a_bad_out_or_target_before_any_run(tmp_path, monkeypatch): + with pytest.raises(SystemExit, match="directory does not exist"): + _cli(tmp_path, monkeypatch, "--out", "nodir/probe.json") + with pytest.raises(SystemExit, match="not a skill in --catalog"): + _cli(tmp_path, monkeypatch, "--target", "sum-fsat", "--out", "probe.json") + + +def test_cli_fails_when_claude_code_did_not_list_a_catalog_skill(tmp_path, monkeypatch): + with pytest.raises(SystemExit, match="did not list: io-read"): + _cli(tmp_path, monkeypatch, "--out", "probe.json", init_skills=("sum-fast", "dataviz")) + assert json.loads((tmp_path / "probe.json").read_text(encoding="utf-8"))["not_listed"] == ["io-read"] From 6901cde9fb78540d2ac1cd193ae193d97b948507 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 26 Oct 2026 20:43:17 -0700 Subject: [PATCH 10/13] fix(catalog): trigger-probe self-review: partial file, input errors, result_subtype Keep in-progress records in .partial so a rerun does not clobber a finished result; refuse a non-writable --out and a malformed catalog as input errors (exit 2) before any run; name the field result_subtype as the rest of the codebase does; document max_turns_hit, not_listed and the partial file. Signed-off-by: Nikolay Petrov --- dokimi/docs/evaluating-treatments.md | 5 ++++- dokimi/dokimi/commands/catalog_probe.py | 22 ++++++++++++++-------- dokimi/dokimi/harnesses/trigger_probe.py | 6 +++--- dokimi/tests/test_trigger_probe.py | 9 ++++++--- 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/dokimi/docs/evaluating-treatments.md b/dokimi/docs/evaluating-treatments.md index e652d8ef3..ee3a71cf2 100644 --- a/dokimi/docs/evaluating-treatments.md +++ b/dokimi/docs/evaluating-treatments.md @@ -589,7 +589,10 @@ loaded; the model can still read any host path it has access to, the trigger set labels included), the query sent verbatim with no notice or task framing, and bundled skills left on as a user has them (`--sealed` turns them off). The skill chosen is read from the transcript's `Skill` calls, and the output's `skill_selection_summary` is scored by the -same code as the `skill-catalog:` arm, one column per trial: +same code as the `skill-catalog:` arm, one column per trial. `max_turns_hit` counts runs +that ended on `--max-turns` (scored as no load), and the command exits 1 when Claude Code +did not list a catalog skill (`not_listed`). Records are kept in `.partial` while the +probe runs: ```bash python cli.py catalog trigger-probe --catalog /skills \ diff --git a/dokimi/dokimi/commands/catalog_probe.py b/dokimi/dokimi/commands/catalog_probe.py index 3949afc28..6981a74f1 100644 --- a/dokimi/dokimi/commands/catalog_probe.py +++ b/dokimi/dokimi/commands/catalog_probe.py @@ -20,18 +20,23 @@ def cmd_catalog_trigger_probe(args: argparse.Namespace) -> None: from dokimi.external import trigger_set as ts from dokimi.harnesses import trigger_probe as tp from dokimi.skills import discover_skills + from dokimi.utils import InputError catalog = Path(args.catalog) if not catalog.is_dir(): - raise SystemExit(f"--catalog is not a directory: {catalog}") - names = {s.name for s in discover_skills(catalog)} + raise InputError(f"--catalog is not a directory: {catalog}") + try: + names = {s.name for s in discover_skills(catalog)} + except (OSError, ValueError) as exc: + raise InputError(f"--catalog: {exc}") from exc if args.target and args.target not in names: - raise SystemExit(f"--target {args.target!r} is not a skill in --catalog " + raise InputError(f"--target {args.target!r} is not a skill in --catalog " f"(have: {', '.join(sorted(names))})") # Checked before any paid run, so a bad --out costs nothing. out_path = Path(args.out) - if not out_path.parent.is_dir(): - raise SystemExit(f"--out: directory does not exist: {out_path.parent}") + partial = out_path.with_name(out_path.name + ".partial") + if out_path.is_dir() or not out_path.parent.is_dir() or not os.access(out_path.parent, os.W_OK): + raise InputError(f"--out: not a writable file path: {out_path}") # Absolute: each run's cwd is a scratch dir, where a relative path names nothing. claude = shutil.which(args.claude) if claude is None: @@ -51,9 +56,9 @@ def cmd_catalog_trigger_probe(args: argparse.Namespace) -> None: def one(q): rec = tp.probe_trigger_set([q], catalog, cfg, trials=args.trials)[0] - with lock: # what has run so far survives an interrupt or a later failure + with lock: # what has run so far survives a later failure or a kill done.append(rec) - _write(out_path, {"partial": True, "answers": done}) + _write(partial, {"partial": True, "answers": done}) print(f"{q.id}: " + ", ".join( f"{a}={'+'.join(v['tool_usage']['skill']['first_turn_loaded']) or '-'}" + (f" ({v['error']})" if v["error"] else "") for a, v in rec["arms"].items()), @@ -80,12 +85,13 @@ def one(q): for v in r["arms"].values() if v["tool_usage"]["skill"]["claude_code_version"]), None), "cost_usd": round(cost, 4), - "max_turns_hit": sum(v["metrics"].get("stop") == "error_max_turns" for v in runs), + "max_turns_hit": sum(v["metrics"].get("result_subtype") == "error_max_turns" for v in runs), "not_listed": not_listed, "answers": records, "skill_selection_summary": summarize_skill_selection(records), } _write(out_path, out) + partial.unlink(missing_ok=True) print(json.dumps({"cost_usd": out["cost_usd"], "claude_code_version": out["claude_code_version"], "max_turns_hit": out["max_turns_hit"], "not_listed": not_listed, "summary": {a: {k: s[k] for k in ("recall", "false_trigger_rate", "misroute_rate", "n_failed")} diff --git a/dokimi/dokimi/harnesses/trigger_probe.py b/dokimi/dokimi/harnesses/trigger_probe.py index 255eb2320..328078be1 100644 --- a/dokimi/dokimi/harnesses/trigger_probe.py +++ b/dokimi/dokimi/harnesses/trigger_probe.py @@ -196,7 +196,7 @@ def run_query(query: str, catalog: Path, cfg: ProbeConfig, *, runner: process.Ru elif result.get("is_error") and result.get("subtype") != "error_max_turns": error = str(result.get("subtype") or "error") return {"skill": skill, "cost_usd": result.get("total_cost_usd"), - "num_turns": result.get("num_turns"), "stop": result.get("subtype"), "error": error} + "num_turns": result.get("num_turns"), "result_subtype": result.get("subtype"), "error": error} finally: kill_strays(ws.root) ws.cleanup() @@ -246,13 +246,13 @@ def probe_trigger_set(queries: Sequence[Any], catalog: Path, cfg: ProbeConfig, * r = run_query(q.text, catalog, cfg, runner=runner) except (OSError, RuntimeError) as exc: # One failed run (staging, spawn) is that trial's error, not the lost set. - r = {"skill": parse_stream(""), "cost_usd": None, "num_turns": None, "stop": None, + r = {"skill": parse_stream(""), "cost_usd": None, "num_turns": None, "result_subtype": None, "error": f"{type(exc).__name__}: {exc}"} arms[f"{arm_name}#{k + 1}"] = { "answer": None, "error": r["error"], "tool_usage": {"skill": r["skill"]}, - "metrics": {"cost_usd": r["cost_usd"], "num_turns": r["num_turns"], "stop": r.get("stop")}, + "metrics": {"cost_usd": r["cost_usd"], "num_turns": r["num_turns"], "result_subtype": r.get("result_subtype")}, } rec = {"question_id": q.id, "question_text": q.text, "expected_skill": q.expected_skill, "arms": arms} out.append(rec) diff --git a/dokimi/tests/test_trigger_probe.py b/dokimi/tests/test_trigger_probe.py index 595b0529d..c417c1321 100644 --- a/dokimi/tests/test_trigger_probe.py +++ b/dokimi/tests/test_trigger_probe.py @@ -291,7 +291,7 @@ def fake_probe(queries, catalog, cfg, *, trials=1, **kw): return [{"question_id": q.id, "question_text": q.text, "expected_skill": q.expected_skill, "arms": {"claude_code#1": {"answer": None, "error": None, "tool_usage": {"skill": skill}, "metrics": {"cost_usd": 0.01, "num_turns": 3, - "stop": "error_max_turns"}}}} for q in queries] + "result_subtype": "error_max_turns"}}}} for q in queries] class Pool(catalog_probe.ThreadPoolExecutor): def __init__(self, n): @@ -322,13 +322,16 @@ def test_cli_passes_every_flag_to_the_probe(tmp_path, monkeypatch): assert os.path.isabs(cfg.claude) and seen["trials"] == 3 and seen["concurrency"] == 5 assert sorted(seen["queries"]) == ["a", "b"] out = json.loads((tmp_path / "probe.json").read_text(encoding="utf-8")) + assert not (tmp_path / "probe.json.partial").exists() assert out["mode"] == "sealed" and out["max_turns_hit"] == 2 and out["not_listed"] == [] def test_cli_refuses_a_bad_out_or_target_before_any_run(tmp_path, monkeypatch): - with pytest.raises(SystemExit, match="directory does not exist"): + from dokimi.utils import InputError + + with pytest.raises(InputError, match="not a writable file path"): _cli(tmp_path, monkeypatch, "--out", "nodir/probe.json") - with pytest.raises(SystemExit, match="not a skill in --catalog"): + with pytest.raises(InputError, match="not a skill in --catalog"): _cli(tmp_path, monkeypatch, "--target", "sum-fsat", "--out", "probe.json") From ea9d304ffbf565958da9217fd140dc415334b9ea Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 27 Oct 2026 11:06:28 +0000 Subject: [PATCH 11/13] Reject a non-positive --timeout before any run Signed-off-by: Anton Volkov --- dokimi/dokimi/commands/catalog_probe.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dokimi/dokimi/commands/catalog_probe.py b/dokimi/dokimi/commands/catalog_probe.py index 6981a74f1..5a8545cb6 100644 --- a/dokimi/dokimi/commands/catalog_probe.py +++ b/dokimi/dokimi/commands/catalog_probe.py @@ -37,6 +37,8 @@ def cmd_catalog_trigger_probe(args: argparse.Namespace) -> None: partial = out_path.with_name(out_path.name + ".partial") if out_path.is_dir() or not out_path.parent.is_dir() or not os.access(out_path.parent, os.W_OK): raise InputError(f"--out: not a writable file path: {out_path}") + if not args.timeout > 0: + raise InputError(f"--timeout must be > 0, got {args.timeout}") # Absolute: each run's cwd is a scratch dir, where a relative path names nothing. claude = shutil.which(args.claude) if claude is None: From 687f90fdd8e0746a09ce177fc9f57d5e710dad20 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Tue, 27 Oct 2026 05:14:34 -0700 Subject: [PATCH 12/13] test(catalog): pin trigger-probe's stray-process kill in run_query Only kill_strays itself was tested, so dropping its call from run_query or matching a sibling root's prefix left the suite green. Run a fake claude that leaves a job in its workspace and one in a same-prefix sibling root: the first must die, the second must survive. Signed-off-by: Alexander Rybkin --- dokimi/tests/test_trigger_probe.py | 33 ++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/dokimi/tests/test_trigger_probe.py b/dokimi/tests/test_trigger_probe.py index c417c1321..36fea16e9 100644 --- a/dokimi/tests/test_trigger_probe.py +++ b/dokimi/tests/test_trigger_probe.py @@ -278,6 +278,39 @@ def test_kill_strays_ends_a_backgrounded_process_left_in_the_workspace(tmp_path) proc.kill() +@pytest.mark.skipif(not Path("/proc/self/cwd").exists(), reason="reads /proc") +def test_run_query_kills_what_the_run_left_in_its_workspace_only(tmp_path): + import shutil + import subprocess + + procs = {} + + def runner(argv, **kw): + if argv[0] == "git": + return process.run(argv, **kw) + sibling = Path(str(kw["cwd"].parent) + "d") # a concurrent run's root, same prefix + sibling.mkdir() + procs["sibling_dir"] = sibling + # Own sessions, as the Bash tool's background jobs: the timeout's group kill misses them. + for name, cwd in (("left", kw["cwd"]), ("sibling", sibling)): + procs[name] = subprocess.Popen(["sleep", "60"], cwd=cwd, start_new_session=True) + return process.reply(_stream(["sum-fast"])) + + try: + r = tp.run_query("q", _catalog(tmp_path), tp.ProbeConfig(), runner=runner, + source_env={"PATH": "/usr/bin:/bin"}) + assert r["error"] is None + assert procs["left"].wait(timeout=5) == -9 + assert procs["sibling"].poll() is None + finally: + for name in ("left", "sibling"): + if name in procs: + procs[name].kill() + procs[name].wait() + if "sibling_dir" in procs: + shutil.rmtree(procs["sibling_dir"], ignore_errors=True) + + def _cli(tmp_path, monkeypatch, *extra, init_skills=("sum-fast", "io-read")): from dokimi.commands import catalog_probe From e4fbbf242538208c440af747cd1b8eb7d3599720 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Tue, 27 Oct 2026 05:14:54 -0700 Subject: [PATCH 13/13] fix(catalog): refuse a trigger-probe label that names no catalog skill A query whose expected_skill is not in --catalog can never be loaded, so it scored as a miss after the paid run: a typo read as recall 0. The skill-catalog: arm refuses such a label; trigger-probe now does too, before any run, as it already does for --target. Signed-off-by: Alexander Rybkin --- dokimi/dokimi/commands/catalog_probe.py | 27 ++++++++++++++++++------- dokimi/tests/test_trigger_probe.py | 13 ++++++++++-- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/dokimi/dokimi/commands/catalog_probe.py b/dokimi/dokimi/commands/catalog_probe.py index 5a8545cb6..b149613b5 100644 --- a/dokimi/dokimi/commands/catalog_probe.py +++ b/dokimi/dokimi/commands/catalog_probe.py @@ -17,7 +17,6 @@ def cmd_catalog_trigger_probe(args: argparse.Namespace) -> None: from dokimi.eval.arm_summary import summarize_skill_selection - from dokimi.external import trigger_set as ts from dokimi.harnesses import trigger_probe as tp from dokimi.skills import discover_skills from dokimi.utils import InputError @@ -43,12 +42,7 @@ def cmd_catalog_trigger_probe(args: argparse.Namespace) -> None: claude = shutil.which(args.claude) if claude is None: raise SystemExit(f"{args.claude!r} is not on PATH") - try: - data = json.loads(Path(args.trigger_set).read_text(encoding="utf-8")) - _fmt, queries = ts.read_trigger_set(data, target=args.target) - except (OSError, ValueError) as exc: - raise SystemExit(f"--trigger-set: {exc}") from exc - queries = [q for q in queries if q.labelled][: args.limit or None] + queries = _labelled_queries(args, names) cfg = tp.ProbeConfig(model=args.model, max_turns=args.max_turns, timeout_sec=args.timeout, stock=not args.sealed, listing_budget_fraction=args.listing_budget_fraction, claude=os.path.abspath(claude)) @@ -103,6 +97,25 @@ def one(q): raise SystemExit(f"catalog skills Claude Code did not list: {', '.join(not_listed)}") +def _labelled_queries(args: argparse.Namespace, names: set) -> list: + """The labelled queries to run; a label naming no *names* skill is an input error.""" + from dokimi.external import trigger_set as ts + from dokimi.utils import InputError + + try: + data = json.loads(Path(args.trigger_set).read_text(encoding="utf-8")) + _fmt, queries = ts.read_trigger_set(data, target=args.target) + except (OSError, ValueError) as exc: + raise SystemExit(f"--trigger-set: {exc}") from exc + queries = [q for q in queries if q.labelled][: args.limit or None] + # As the skill-catalog: arm refuses it: a label no run can load scores as a miss. + unknown = sorted({s for q in queries for s in q.acceptable} - names) + if unknown: + raise InputError(f"--trigger-set: expected_skill names no skill in --catalog: " + f"{', '.join(unknown)}") + return queries + + def _write(path: Path, data: dict) -> None: tmp = path.with_name(path.name + ".tmp") tmp.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") diff --git a/dokimi/tests/test_trigger_probe.py b/dokimi/tests/test_trigger_probe.py index 36fea16e9..041789890 100644 --- a/dokimi/tests/test_trigger_probe.py +++ b/dokimi/tests/test_trigger_probe.py @@ -311,7 +311,7 @@ def runner(argv, **kw): shutil.rmtree(procs["sibling_dir"], ignore_errors=True) -def _cli(tmp_path, monkeypatch, *extra, init_skills=("sum-fast", "io-read")): +def _cli(tmp_path, monkeypatch, *extra, init_skills=("sum-fast", "io-read"), expected="sum-fast"): from dokimi.commands import catalog_probe seen = {} @@ -334,7 +334,7 @@ def __init__(self, n): monkeypatch.setattr(tp, "probe_trigger_set", fake_probe) monkeypatch.setattr(catalog_probe, "ThreadPoolExecutor", Pool) (tmp_path / "ts.json").write_text(json.dumps({"questions": [ - {"id": q, "question": f"q {q}", "expected_skill": "sum-fast"} for q in ("a", "b", "c")]}), + {"id": q, "question": f"q {q}", "expected_skill": expected} for q in ("a", "b", "c")]}), encoding="utf-8") monkeypatch.chdir(tmp_path) if not (tmp_path / "cat").exists(): @@ -368,6 +368,15 @@ def test_cli_refuses_a_bad_out_or_target_before_any_run(tmp_path, monkeypatch): _cli(tmp_path, monkeypatch, "--target", "sum-fsat", "--out", "probe.json") +def test_cli_refuses_a_label_that_names_no_catalog_skill(tmp_path, monkeypatch): + from dokimi.utils import InputError + + with pytest.raises(InputError, match="names no skill in --catalog: sum-fsat"): + _cli(tmp_path, monkeypatch, "--out", "probe.json", expected="sum-fsat") + assert not (tmp_path / "probe.json.partial").exists() + _cli(tmp_path, monkeypatch, "--out", "probe.json", expected=["sum-fast", "io-read"]) + + def test_cli_fails_when_claude_code_did_not_list_a_catalog_skill(tmp_path, monkeypatch): with pytest.raises(SystemExit, match="did not list: io-read"): _cli(tmp_path, monkeypatch, "--out", "probe.json", init_skills=("sum-fast", "dataviz"))