diff --git a/dokimi/changelog.d/20261031_180000_napetrov_eval-coverage.md b/dokimi/changelog.d/20261031_180000_napetrov_eval-coverage.md new file mode 100644 index 000000000..ab188e799 --- /dev/null +++ b/dokimi/changelog.d/20261031_180000_napetrov_eval-coverage.md @@ -0,0 +1,24 @@ + + +### Added + +- **`eval coverage`: which needs of a subject an eval set grades, without a model call.** + `python cli.py eval coverage --graph need_graph.json` + reads a frozen need graph (new `need_graph.v1` schema; a novelty need graph is valid as + written) and the `needs` list on each `evals.json` row or in a task's `metadata.needs`. It + prints the weighted coverage, the coverage of `core` + `rare_critical` and of `rare_critical` + alone, the uncovered `core` / `rare_critical` nodes, and each eval's relevance + (`relevant_core`, `relevant_peripheral`, `out_of_scope`, `orphan`, `unlabelled`). `--out` + writes an `eval_coverage.v1` artifact. Exit 1 under a `--min [METRIC=]X` floor, exit 2 on an + unreadable input, a graph with duplicate node ids, or a `needs` label that is not a list of + ids the graph has. A row or task with no `needs` key is `unlabelled`; an explicit + `"needs": null` is malformed and exits 2. `--out` is refused when it would overwrite the + graph or any input file. A `--min` floor is judged on the unrounded share, not the rounded + figure shown in the report; a floor on a class the graph has no node of (n/a) also exits 1. + Known gap: a terminal-bench v1 task (`task.yaml`) has its `metadata.needs` ignored, so it counts + as `unlabelled`; `task.toml` and `task.md` tasks are read. Novelty's need graph now keeps + `confirmed_by` and `provenance`. + Building the graph, the user interview and drafting labels follow in #331. diff --git a/dokimi/data/skills/dokimi-design-evals/references/coverage-report.md b/dokimi/data/skills/dokimi-design-evals/references/coverage-report.md index ff1e2fcce..e97f083fe 100644 --- a/dokimi/data/skills/dokimi-design-evals/references/coverage-report.md +++ b/dokimi/data/skills/dokimi-design-evals/references/coverage-report.md @@ -1,9 +1,17 @@ # Coverage report -dokimi has no command that writes this report (`python -m dokimi.coverage` -prints a product x awareness/work task count, not scenario coverage). Write -it by hand, in the suite's README and in the PR that adds or changes the -suite. +Write this report by hand, in the suite's README and in the PR that adds or +changes the suite. `python -m dokimi.coverage` prints only a product x +awareness/work task count. + +`python cli.py eval coverage --graph need_graph.json` adds the part +a script can check. It needs a need graph (what users of the subject need, +built from its name and description only) and a `needs` list on each eval +that names the graph nodes it grades. From those two files alone it lists the +core and rare_critical nodes that no eval names, and the relevance of each +eval: `orphan` for `needs: []`, `unlabelled` for no `needs`. Paste its +Markdown under the table below. It does not replace the areas in the table, +and a node the graph never had is still a gap nobody looked at. ## Template diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index c9eb437d1..51d1c1e38 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -4017,6 +4017,94 @@ "subcommand_required": null, "subcommands": {} }, + "coverage": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "evals", + "kind": "argument", + "nargs": null, + "position": 0, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--graph", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_AppendAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--min", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.commands.eval_coverage._floor#bbd5737352a9" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--out", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi eval coverage", + "subcommand_required": null, + "subcommands": {} + }, "cross-judge": { "allow_abbrev": true, "params": [ @@ -12885,9 +12973,11 @@ "'arms'", "'cli_routes'", "'eval'", + "'eval_coverage'", "'explore_runs'", "'guidance_reach'", "'matrix_rollup'", + "'need_graph'", "'novelty'", "'plan'", "'plugin_delta'", diff --git a/dokimi/docs/user-journeys/journeys.md b/dokimi/docs/user-journeys/journeys.md index 1f26b9c5e..c49317040 100644 --- a/dokimi/docs/user-journeys/journeys.md +++ b/dokimi/docs/user-journeys/journeys.md @@ -492,7 +492,7 @@ down in the PR. #### UC-J07-004 State coverage gaps -- **AC-J07-007** An eval set's report lists the capabilities or areas it has no task for. *Proposed.* +- **AC-J07-007** An eval set's report lists the capabilities or areas it has no task for. *Partial: `eval coverage` lists the core and rare_critical nodes of a frozen need graph that no eval's `needs` names; building the graph, the user interview and drafting labels are #331.* --- diff --git a/dokimi/dokimi/artifacts.py b/dokimi/dokimi/artifacts.py index 0088ff5eb..55fd25cf6 100644 --- a/dokimi/dokimi/artifacts.py +++ b/dokimi/dokimi/artifacts.py @@ -12,7 +12,8 @@ what moves it), ``repo_environment`` (static inspection of what a repository offers an agent), ``plan`` (the cells a run promises to measure), ``result`` (one status per planned cell), ``repo_budget`` (what a repository puts -into an agent's context) and ``task_inspect`` (which eval-task format each +into an agent's context), ``need_graph`` and ``eval_coverage`` (a frozen need graph and an +eval set's coverage of it) and ``task_inspect`` (which eval-task format each path under a tree is). This module stamps each with a ``schema_version`` and validates it against the JSON Schema in ``dokimi/schemas/`` on save and load, so fixtures and baselines stay @@ -58,6 +59,8 @@ "repo_budget": "repo_budget.v1", "task_inspect": "task_inspect.v1", "cli_routes": "cli_routes.v1", + "need_graph": "need_graph.v1", + "eval_coverage": "eval_coverage.v1", } # kind -> superseded versions still accepted on *load*, newest first. An artifact written under an diff --git a/dokimi/dokimi/cli_routes.py b/dokimi/dokimi/cli_routes.py index 47008bcc9..8d57df1ed 100644 --- a/dokimi/dokimi/cli_routes.py +++ b/dokimi/dokimi/cli_routes.py @@ -142,6 +142,7 @@ class Tombstone: Leaf(("eval", "score"), PAID), Leaf(("eval", "ragas"), PAID), Leaf(("eval", "grounding"), FREE), + Leaf(("eval", "coverage"), FREE), Leaf(("eval", "judge-validity"), PAID), Leaf(("eval", "checklist"), PAID), Leaf(("eval", "judge-wording"), PAID), diff --git a/dokimi/dokimi/commands/eval_coverage.py b/dokimi/dokimi/commands/eval_coverage.py new file mode 100644 index 000000000..81e297486 --- /dev/null +++ b/dokimi/dokimi/commands/eval_coverage.py @@ -0,0 +1,116 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""``eval coverage``: coverage and relevance of an eval set against a frozen need graph (#331).""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from dokimi.exit_codes import EXIT_FAILURE, EXIT_USAGE + + +METRICS = ("weighted", "core_rare_critical", "rare_critical") + + +def _floor(value: str) -> tuple[str, float]: + metric, sep, number = value.rpartition("=") + metric = metric if sep else "weighted" + if metric not in METRICS: + raise argparse.ArgumentTypeError(f"metric must be one of {', '.join(METRICS)}, got {metric!r}") + try: + floor = float(number) + except ValueError: + raise argparse.ArgumentTypeError(f"expected a number, got {number!r}") from None + if not 0.0 <= floor <= 1.0: + raise argparse.ArgumentTypeError(f"must lie in 0-1, got {floor}") + return metric, floor + + +def _below(report: dict, floors: list[tuple[str, float]]) -> list[str]: + from dokimi.eval_coverage import exact_shares + + exact = exact_shares(report) + out = [] + for metric, floor in floors: + share = exact[metric] + if share is None: # nothing of that class in the graph: the floor cannot be met + out.append(f"{metric} coverage n/a < {floor:.6f}") + elif share < floor: + out.append(f"{metric} coverage {share:.6f} < {floor:.6f}") + return out + + +def _refuse_out_over_input(args: argparse.Namespace, root: Path) -> None: + """Exit 2 when ``--out`` cannot be resolved (a symlink loop) or would overwrite an input or the sibling.""" + from dokimi.eval_coverage import input_files + + try: + # save_artifact also removes .rejected after a valid save, so that sibling is an output too + out = Path(args.out) + outputs = (out, out.with_name(out.name + ".rejected")) + inputs = {Path(args.graph).resolve(), *input_files(root)} + clobbers = any(p.resolve() in inputs for p in outputs) + except (OSError, RuntimeError, ValueError) as exc: # a symlink loop: Python 3.11/3.12 raise RuntimeError + print(f"Error: cannot resolve {args.out}: {exc}", file=sys.stderr) + sys.exit(EXIT_USAGE) + if clobbers: + print(f"Error: --out would overwrite an input file: {args.out}", file=sys.stderr) + sys.exit(EXIT_USAGE) + + +def cmd_eval_coverage(args: argparse.Namespace) -> None: + """Print the report; exit 2 on an unreadable input or a bad ``needs`` label, 1 under a floor.""" + from dokimi.artifacts import ArtifactValidationError, save_artifact + from dokimi.eval_coverage import coverage_report, load_graph, read_labels, render_markdown + + root = Path(args.evals) + try: + if not root.exists(): + print(f"Error: {root} does not exist", file=sys.stderr) + sys.exit(EXIT_USAGE) + graph = load_graph(Path(args.graph)) + evals = read_labels(root) + except (OSError, ValueError, ArtifactValidationError) as exc: + print(f"Error: {exc}", file=sys.stderr) + sys.exit(EXIT_USAGE) + if not evals: + print(f"Error: no evals.json rows or tasks under {root}", file=sys.stderr) + sys.exit(EXIT_USAGE) + if args.out: + _refuse_out_over_input(args, root) + report = coverage_report(graph, evals, str(root)) + if args.out: + try: + report = save_artifact("eval_coverage", report, Path(args.out)) + except OSError as exc: + print(f"Error: cannot write {args.out}: {exc.strerror or exc}", file=sys.stderr) + sys.exit(EXIT_USAGE) + print(render_markdown(report)) + if report["problems"]: + print(f"Error: {len(report['problems'])} problem(s) with the evals' needs labels", + file=sys.stderr) + sys.exit(EXIT_USAGE) + failed = _below(report, args.min or []) + if failed: + print("FAIL: " + "; ".join(failed), file=sys.stderr) + sys.exit(EXIT_FAILURE) + + +def register(eval_sub) -> None: + """Add ``eval coverage`` to the ``eval`` group.""" + p = eval_sub.add_parser( + "coverage", + help="Coverage of a frozen need graph by an eval set's `needs` labels (no model call)", + ) + p.add_argument("evals", help="evals.json, a skill directory (evals/evals.json) or a tasks root") + p.add_argument("--graph", required=True, + help="need_graph.v1 file (a novelty need graph is valid as written)") + p.add_argument("--out", default=None, help="Write the eval_coverage.v1 artifact here") + p.add_argument("--min", type=_floor, action="append", metavar="[METRIC=]X", + help="Exit 1 when a coverage figure is below X (0-1) or n/a (the graph has no " + "node of that class); METRIC is weighted (default), core_rare_critical or " + "rare_critical. Repeatable") + p.set_defaults(func=cmd_eval_coverage) diff --git a/dokimi/dokimi/commands/evaluate.py b/dokimi/dokimi/commands/evaluate.py index 18f1da851..0ecf5f29e 100644 --- a/dokimi/dokimi/commands/evaluate.py +++ b/dokimi/dokimi/commands/evaluate.py @@ -11,6 +11,7 @@ import sys from pathlib import Path +from dokimi.commands import eval_coverage from dokimi.llm import ANSWER_PROVIDERS from dokimi.defaults import ( DEFAULT_CONCURRENCY, @@ -467,6 +468,7 @@ def register(sub, positive_int) -> None: # Eval subcommand group eval_p = sub.add_parser("eval", help="Evaluate answers") eval_sub = eval_p.add_subparsers(dest="eval_cmd", required=True) + eval_coverage.register(eval_sub) # eval score score_p = eval_sub.add_parser("score", help="Score answers using LLM-as-judge") diff --git a/dokimi/dokimi/eval_coverage.py b/dokimi/dokimi/eval_coverage.py new file mode 100644 index 000000000..0c91ac56b --- /dev/null +++ b/dokimi/dokimi/eval_coverage.py @@ -0,0 +1,257 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Eval coverage of a subject's problem space, from files only (issue #331, AC-J07-007). + +The problem space is a frozen need graph (``need_graph.v1``): the graph +:func:`dokimi.novelty.needs.build_need_graph` builds from a subject's name and description, +written once and kept next to the suite. Each eval names the nodes it grades in ``needs``: + +* a row of a skill-creator ``evals.json``: ``{"id": ..., "needs": ["n1", "n4"]}``; +* a task's ``task.toml`` / ``task.md``: ``metadata.needs`` (a list; a bare string is refused). + +:func:`coverage_report` is a pure function of the graph and those labels, so the same files +always give the same report. No model is called here; drafting labels is a separate, paid step. + +Relevance of one eval: ``relevant_core`` (names a core or rare_critical node), +``relevant_peripheral`` (only branch or supporting nodes), ``out_of_scope`` (only nodes the +graph marks out of scope), ``orphan`` (``needs: []``: it grades nothing the subject is for) +and ``unlabelled`` (no ``needs`` at all: not yet mapped, so it covers nothing). +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from dokimi.artifacts import load_artifact +from dokimi.harnesses import tbench_v1 +from dokimi.harnesses.tasks import FORMAT_TASK_MD, is_task_dir, read_task_config, task_format +from dokimi.novelty.needs import PATH_WEIGHTS + +CRITICAL = ("core", "rare_critical") +CLASSES = ("core", "rare_critical", "branch", "supporting", "out_of_scope") + + +@dataclass(frozen=True) +class LabelledEval: + """One eval and the need-node ids it says it grades (``None``: the row or task has no ``needs`` key).""" + + id: str + source: str + needs: tuple[str, ...] | None + malformed: bool = False # ``needs`` is there but is not a list of strings + + +def load_graph(path: Path) -> dict: + """Read and validate a ``need_graph.v1`` file (an unstamped ``save_graph`` file is valid). + + Raises ``ValueError`` when two nodes share an id: ids are what the ``needs`` labels join on. + """ + graph = load_artifact("need_graph", Path(path)) + ids = [n["id"] for n in graph["nodes"]] + dupes = sorted({i for i in ids if ids.count(i) > 1}) + if dupes: + raise ValueError(f"{path}: duplicate node ids {dupes}") + return graph + + +def graph_digest(graph: dict) -> str: + """sha256 of each node's id, text, type, path_class and confirmed_by, defaults filled in. + + Other keys and node order do not change it; a confirmation does. + """ + nodes = sorted(({"id": n["id"], "text": n["text"], "type": n.get("type", "intent"), + "path_class": n["path_class"], + "confirmed_by": n.get("confirmed_by", "hypothesis")} + for n in graph["nodes"]), key=lambda n: n["id"]) + blob = json.dumps(nodes, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return hashlib.sha256(blob.encode("utf-8")).hexdigest() + + +_ABSENT = object() # no ``needs`` key at all: not the same as ``"needs": null``, which is malformed + + +def _labelled(eid: str, source: str, value: Any) -> LabelledEval: + if value is _ABSENT: + return LabelledEval(eid, source, None) + if isinstance(value, list) and all(isinstance(v, str) for v in value): + return LabelledEval(eid, source, tuple(value)) + return LabelledEval(eid, source, None, malformed=True) + + +def _evals_files(root: Path) -> list[Path]: + if root.is_file(): + return [root] + return [p for p in (root / "evals.json", root / "evals" / "evals.json") if p.is_file()] + + +# every task file a task reader may parse: Harbor / BenchFlow (task.toml, task.md, instruction.md, which load_task +# runs), terminal-bench v1 (task.yaml, compose) +_TASK_FILES = ("task.toml", "task.md", "instruction.md", tbench_v1.CONFIG, tbench_v1.COMPOSE) + + +def _is_task(path: Path) -> bool: + """A task directory the way discover_tasks counts it: a lone task.md without frontmatter is an author's note.""" + if not is_task_dir(path): + return False + if task_format(path) != FORMAT_TASK_MD: + return True + with (path / "task.md").open(encoding="utf-8", errors="replace") as fh: + return fh.readline().strip() == "---" + + +def input_files(root: Path) -> list[Path]: + """The resolved files :func:`read_labels` reads under ``root``, so an output can be refused when it would + overwrite one of them.""" + files = _evals_files(root) + if root.is_dir(): + for task_dir in (p for p in root.iterdir() if _is_task(p)): + files += [task_dir / name for name in _TASK_FILES if (task_dir / name).is_file()] + return [p.resolve() for p in files] + + +def exact_shares(report: dict) -> dict[str, float | None]: + """The three coverage figures computed from the report's counts and weights, unrounded. A floor is judged + on these: the report rounds its own figures for display only.""" + by_class, weights = report["coverage"]["by_class"], report["coverage"]["weights"] + scope = [c for c in CLASSES if c != "out_of_scope"] + weight = sum(weights[c] * by_class[c]["total"] for c in scope) + + def share(classes: tuple[str, ...]) -> float | None: + total = sum(by_class[c]["total"] for c in classes) + return sum(by_class[c]["covered"] for c in classes) / total if total else None + + return { + "weighted": sum(weights[c] * by_class[c]["covered"] for c in scope) / weight if weight else None, + "core_rare_critical": share(CRITICAL), + "rare_critical": share(("rare_critical",)), + } + + +def read_labels(root: Path) -> list[LabelledEval]: + """Every eval under ``root`` with its ``needs``: skill-creator ``evals.json`` rows, then tasks. + + ``root`` is an ``evals.json`` file, a skill directory (its ``evals/evals.json``) or a tasks + root whose child directories are tasks. Raises ``ValueError`` on an unreadable ``evals.json`` + or task. + """ + out: list[LabelledEval] = [] + for path in _evals_files(root): + try: + rows = json.loads(path.read_text(encoding="utf-8")).get("evals", []) + except (OSError, ValueError, AttributeError) as exc: + raise ValueError(f"{path}: not a skill-creator evals.json ({exc})") from exc + if not isinstance(rows, list): + raise ValueError(f"{path}: \"evals\" is not a list") + for index, row in enumerate(rows): + if isinstance(row, dict): + out.append(_labelled(str(row.get("id", index)), path.name, row.get("needs", _ABSENT))) + if root.is_dir(): + for task_dir in sorted(p for p in root.iterdir() if _is_task(p)): + try: + meta = read_task_config(task_dir).get("metadata", {}) + except (ValueError, OSError) as exc: + raise ValueError(f"{task_dir}: unreadable task ({exc})") from exc + needs = meta.get("needs", _ABSENT) if isinstance(meta, dict) else _ABSENT + out.append(_labelled(task_dir.name, "task", needs)) + return out + + +def _relevance(needs: tuple[str, ...] | None, classes: dict[str, str]) -> str: + if needs is None: + return "unlabelled" + named = [classes[n] for n in needs if n in classes] + if not named: + return "orphan" + if any(c in CRITICAL for c in named): + return "relevant_core" + if any(c != "out_of_scope" for c in named): + return "relevant_peripheral" + return "out_of_scope" + + +def _share(nodes: list[dict], covered: set[str]) -> float | None: + return round(sum(n["id"] in covered for n in nodes) / len(nodes), 4) if nodes else None + + +def coverage_report(graph: dict, evals: list[LabelledEval], evals_root: str) -> dict: + """The ``eval_coverage.v1`` payload (unstamped) for ``graph`` and ``evals``.""" + classes = {n["id"]: n["path_class"] for n in graph["nodes"]} + problems = [{"eval": e.id, "source": e.source, "kind": "malformed_needs", "need": None} + for e in evals if e.malformed] + problems += [{"eval": e.id, "source": e.source, "kind": "unknown_need", "need": n} + for e in evals for n in (e.needs or ()) if n not in classes] + covered = {n for e in evals for n in (e.needs or ()) if n in classes} + scope = [n for n in graph["nodes"] if n["path_class"] != "out_of_scope"] + weight = sum(PATH_WEIGHTS[n["path_class"]] for n in scope) + critical = [n for n in scope if n["path_class"] in CRITICAL] + return { + "evals_root": evals_root, + "graph": { + "artifact_name": graph["artifact_name"], + "digest": graph_digest(graph), + "nodes": len(graph["nodes"]), + "in_scope": len(scope), + "hypotheses": sum(n.get("confirmed_by", "hypothesis") == "hypothesis" + for n in graph["nodes"]), + }, + "coverage": { + "weights": {c: PATH_WEIGHTS[c] for c in CLASSES}, + "weighted": round(sum(PATH_WEIGHTS[n["path_class"]] for n in scope + if n["id"] in covered) / weight, 4) if weight else None, + "core_rare_critical": _share(critical, covered), + "rare_critical": _share([n for n in scope if n["path_class"] == "rare_critical"], covered), + "by_class": {c: {"covered": sum(n["id"] in covered for n in graph["nodes"] + if n["path_class"] == c), + "total": sum(n["path_class"] == c for n in graph["nodes"])} + for c in CLASSES}, + }, + "gaps": [{"id": n["id"], "path_class": n["path_class"], "text": n["text"], + "confirmed_by": n.get("confirmed_by", "hypothesis")} + for n in critical if n["id"] not in covered], + "evals": [{"id": e.id, "source": e.source, "needs": list(e.needs or ()), + "relevance": _relevance(e.needs, classes)} for e in evals], + "problems": problems, + } + + +def _pct(value: float | None) -> str: + return "n/a" if value is None else f"{value:.0%}" + + +def _problem(p: dict) -> str: + what = "needs is not a list of node ids" if p["kind"] == "malformed_needs" \ + else f"unknown need id {p['need']!r}" + return f"- {p['eval']} ({p['source']}): {what}" + + +def _cell(text: str) -> str: + """A Markdown table cell: pipes escaped, line breaks flattened so the row stays one line.""" + return " ".join(text.replace("|", "\\|").splitlines()) + + +def render_markdown(report: dict) -> str: + """The report as Markdown, for a suite's README or the PR that changes the suite.""" + g, c = report["graph"], report["coverage"] + lines = [ + f"## Eval coverage: {g['artifact_name']} (graph {g['digest'][:12]})", + "", + f"Weighted coverage {_pct(c['weighted'])}; core + rare_critical " + f"{_pct(c['core_rare_critical'])}; rare_critical {_pct(c['rare_critical'])}. " + f"{g['hypotheses']} of {g['nodes']} nodes are unconfirmed hypotheses.", + "", + "| Eval | Source | Needs | Relevance |", + "|---|---|---|---|", + ] + lines += [f"| {_cell(e['id'])} | {_cell(e['source'])} | {_cell(', '.join(e['needs']) or '-')} | {e['relevance']} |" + for e in report["evals"]] + lines += ["", "Gaps (core and rare_critical nodes no eval names):"] + lines += [f"- {n['id']} [{n['path_class']}, {n['confirmed_by']}] {n['text']}" + for n in report["gaps"]] or ["- none"] + if report["problems"]: + lines += ["", "Problems:"] + [_problem(p) for p in report["problems"]] + return "\n".join(lines) + "\n" diff --git a/dokimi/dokimi/novelty/needs.py b/dokimi/dokimi/novelty/needs.py index 5e7918c09..9c3e0e9ab 100644 --- a/dokimi/dokimi/novelty/needs.py +++ b/dokimi/dokimi/novelty/needs.py @@ -81,6 +81,7 @@ class NeedNode: text: str type: str = "intent" path_class: str = "core" + confirmed_by: str = "hypothesis" # "user" once a person confirmed it (need_graph.v1) @property def weight(self) -> float: @@ -90,7 +91,10 @@ def weight(self) -> float: def as_dict(self) -> dict: """Serializable form.""" return {"id": self.id, "text": self.text, "type": self.type, - "path_class": self.path_class} + "path_class": self.path_class, "confirmed_by": self.confirmed_by} + + +_NODE_FIELDS = frozenset(NeedNode.__dataclass_fields__) @dataclass @@ -101,6 +105,7 @@ class NeedGraph: description: str nodes: list[NeedNode] = field(default_factory=list) edges: list[dict] = field(default_factory=list) + provenance: dict = field(default_factory=dict) def task_statement(self) -> str: """A task description for the value judge, built without the artifact body.""" @@ -126,6 +131,7 @@ def as_dict(self) -> dict: "description": self.description, "nodes": [n.as_dict() for n in self.nodes], "edges": list(self.edges), + **({"provenance": dict(self.provenance)} if self.provenance else {}), } @classmethod @@ -134,8 +140,10 @@ def from_dict(cls, data: dict) -> "NeedGraph": return cls( artifact_name=data.get("artifact_name", ""), description=data.get("description", ""), - nodes=[NeedNode(**row) for row in data.get("nodes", [])], + nodes=[NeedNode(**{k: v for k, v in row.items() if k in _NODE_FIELDS}) + for row in data.get("nodes", [])], edges=list(data.get("edges", [])), + provenance=dict(data.get("provenance") or {}), ) diff --git a/dokimi/dokimi/runs/kinds.py b/dokimi/dokimi/runs/kinds.py index cb8e49a39..fde7e7cfa 100644 --- a/dokimi/dokimi/runs/kinds.py +++ b/dokimi/dokimi/runs/kinds.py @@ -65,6 +65,8 @@ class KindFields: "repo_budget": KindFields(subject=("tree",), counts=("entries",)), "task_inspect": KindFields(subject=("root",), counts=("items",)), "cli_routes": KindFields(counts=("roots", "leaves", "tombstones")), + "need_graph": KindFields(subject=("artifact_name",), counts=("nodes", "edges")), + "eval_coverage": KindFields(subject=("graph.artifact_name",), counts=("gaps", "evals", "problems")), } #: Every schema_version an artifact may carry (current and superseded) -> its kind. diff --git a/dokimi/dokimi/schemas/eval_coverage.v1.json b/dokimi/dokimi/schemas/eval_coverage.v1.json new file mode 100644 index 000000000..c3e05218a --- /dev/null +++ b/dokimi/dokimi/schemas/eval_coverage.v1.json @@ -0,0 +1,89 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/intel/dokimi/schemas/eval_coverage.v1.json", + "title": "Eval coverage report v1", + "description": "What `eval coverage` computed from a frozen need graph and the `needs` labels of an eval set, with no model call: the same files give the same report. Coverage counts in-scope nodes some eval names; weights are dokimi.novelty.needs.PATH_WEIGHTS.", + "type": "object", + "required": ["schema_version", "evals_root", "graph", "coverage", "gaps", "evals", "problems"], + "additionalProperties": false, + "properties": { + "schema_version": {"const": "eval_coverage.v1"}, + "evals_root": {"type": "string"}, + "graph": { + "type": "object", + "required": ["artifact_name", "digest", "nodes", "in_scope", "hypotheses"], + "additionalProperties": false, + "properties": { + "artifact_name": {"type": "string"}, + "digest": {"type": "string", "description": "sha256 of each node's id, text, type, path_class and confirmed_by (defaults filled in), canonical JSON"}, + "nodes": {"type": "integer", "minimum": 0}, + "in_scope": {"type": "integer", "minimum": 0}, + "hypotheses": {"type": "integer", "minimum": 0, "description": "Nodes no person confirmed"} + } + }, + "coverage": { + "type": "object", + "required": ["weights", "weighted", "core_rare_critical", "rare_critical", "by_class"], + "additionalProperties": false, + "properties": { + "weights": {"type": "object", "description": "Weight per path class used for `weighted`", + "additionalProperties": {"type": "number"}}, + "weighted": {"type": ["number", "null"]}, + "core_rare_critical": {"type": ["number", "null"]}, + "rare_critical": {"type": ["number", "null"]}, + "by_class": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": ["covered", "total"], + "additionalProperties": false, + "properties": {"covered": {"type": "integer"}, "total": {"type": "integer"}} + } + } + } + }, + "gaps": { + "type": "array", + "description": "Uncovered core and rare_critical nodes", + "items": { + "type": "object", + "required": ["id", "path_class", "text", "confirmed_by"], + "additionalProperties": false, + "properties": { + "id": {"type": "string"}, + "path_class": {"type": "string"}, + "text": {"type": "string"}, + "confirmed_by": {"enum": ["user", "hypothesis"]} + } + } + }, + "evals": { + "type": "array", + "items": { + "type": "object", + "required": ["id", "source", "needs", "relevance"], + "additionalProperties": false, + "properties": { + "id": {"type": "string"}, + "source": {"type": "string"}, + "needs": {"type": "array", "items": {"type": "string"}}, + "relevance": {"enum": ["relevant_core", "relevant_peripheral", "out_of_scope", "orphan", "unlabelled"]} + } + } + }, + "problems": { + "type": "array", + "items": { + "type": "object", + "required": ["eval", "source", "kind", "need"], + "additionalProperties": false, + "properties": { + "eval": {"type": "string"}, + "source": {"type": "string"}, + "kind": {"enum": ["unknown_need", "malformed_needs"]}, + "need": {"type": ["string", "null"]} + } + } + } + } +} diff --git a/dokimi/dokimi/schemas/need_graph.v1.json b/dokimi/dokimi/schemas/need_graph.v1.json new file mode 100644 index 000000000..b5c67c0ba --- /dev/null +++ b/dokimi/dokimi/schemas/need_graph.v1.json @@ -0,0 +1,37 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/intel/dokimi/schemas/need_graph.v1.json", + "title": "Need graph v1", + "description": "What a user of a subject needs, built from the subject's name and description only (dokimi.novelty.needs), frozen in a file so that `eval coverage` reads it instead of regenerating it. The graph `dokimi.novelty.needs.save_graph` writes is valid unstamped. `confirmed_by` says whether a person confirmed a node (`user`) or it is a model's guess (`hypothesis`, the default).", + "type": "object", + "required": ["artifact_name", "nodes"], + "properties": { + "schema_version": {"const": "need_graph.v1"}, + "artifact_name": {"type": "string", "minLength": 1}, + "description": {"type": "string"}, + "nodes": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/node"}}, + "edges": {"type": "array", "items": {"type": "object"}}, + "provenance": { + "type": "object", + "description": "How the graph was made: model, prompt version, draws, and the coverage spread over the draws. Open: recorded, not interpreted.", + "properties": { + "model": {"type": "string"}, + "prompt_version": {"type": "string"}, + "draws": {"type": "integer", "minimum": 1} + } + } + }, + "$defs": { + "node": { + "type": "object", + "required": ["id", "text", "path_class"], + "properties": { + "id": {"type": "string", "minLength": 1}, + "text": {"type": "string", "minLength": 1}, + "type": {"enum": ["intent", "evidence", "action", "constraint", "decision"]}, + "path_class": {"enum": ["core", "branch", "rare_critical", "supporting", "out_of_scope"]}, + "confirmed_by": {"enum": ["user", "hypothesis"]} + } + } + } +} diff --git a/dokimi/tests/data/cli_flags_allowlist.txt b/dokimi/tests/data/cli_flags_allowlist.txt index f8e037fda..9b398b12e 100644 --- a/dokimi/tests/data/cli_flags_allowlist.txt +++ b/dokimi/tests/data/cli_flags_allowlist.txt @@ -222,3 +222,5 @@ --work-output-dir --work-repeats -o +--graph # eval coverage: the frozen need graph input, a file no registry concept names yet (#331) +--min # eval coverage gate floor, [METRIC=]X like --timeout [PHASE=]SEC (#331) diff --git a/dokimi/tests/fixtures/artifacts/eval_coverage.v1.json b/dokimi/tests/fixtures/artifacts/eval_coverage.v1.json new file mode 100644 index 000000000..52b788f7b --- /dev/null +++ b/dokimi/tests/fixtures/artifacts/eval_coverage.v1.json @@ -0,0 +1,70 @@ +{ + "schema_version": "eval_coverage.v1", + "evals_root": "skill", + "graph": { + "artifact_name": "dpnp-io", + "digest": "502b794faa561af3c1910c221a2d25db8db77563ec9fde3a05127121a8f12d67", + "nodes": 3, + "in_scope": 2, + "hypotheses": 2 + }, + "coverage": { + "weights": { + "core": 1.0, + "rare_critical": 1.0, + "branch": 0.7, + "supporting": 0.4, + "out_of_scope": 0.0 + }, + "weighted": 0.5, + "core_rare_critical": 0.5, + "rare_critical": 0.0, + "by_class": { + "core": { + "covered": 1, + "total": 1 + }, + "rare_critical": { + "covered": 0, + "total": 1 + }, + "branch": { + "covered": 0, + "total": 0 + }, + "supporting": { + "covered": 0, + "total": 0 + }, + "out_of_scope": { + "covered": 0, + "total": 1 + } + } + }, + "gaps": [ + { + "id": "n2", + "path_class": "rare_critical", + "text": "keep allow_pickle=False for untrusted files", + "confirmed_by": "hypothesis" + } + ], + "evals": [ + { + "id": "save", + "source": "evals.json", + "needs": [ + "n1" + ], + "relevance": "relevant_core" + }, + { + "id": "css", + "source": "evals.json", + "needs": [], + "relevance": "orphan" + } + ], + "problems": [] +} diff --git a/dokimi/tests/fixtures/artifacts/need_graph.v1.json b/dokimi/tests/fixtures/artifacts/need_graph.v1.json new file mode 100644 index 000000000..be765fdce --- /dev/null +++ b/dokimi/tests/fixtures/artifacts/need_graph.v1.json @@ -0,0 +1,38 @@ +{ + "schema_version": "need_graph.v1", + "artifact_name": "dpnp-io", + "description": "Save and load dpnp arrays.", + "nodes": [ + { + "id": "n1", + "text": "convert with dpnp.asnumpy before numpy.save", + "type": "action", + "path_class": "core", + "confirmed_by": "user" + }, + { + "id": "n2", + "text": "keep allow_pickle=False for untrusted files", + "type": "constraint", + "path_class": "rare_critical" + }, + { + "id": "n3", + "text": "plot the loaded array", + "type": "intent", + "path_class": "out_of_scope" + } + ], + "edges": [ + { + "from": "n2", + "to": "n1", + "kind": "requires" + } + ], + "provenance": { + "model": "claude-sonnet-5-5", + "prompt_version": "needs-v1", + "draws": 3 + } +} diff --git a/dokimi/tests/outcomes/journeys.yaml b/dokimi/tests/outcomes/journeys.yaml index 1a435da4a..14ce6780c 100644 --- a/dokimi/tests/outcomes/journeys.yaml +++ b/dokimi/tests/outcomes/journeys.yaml @@ -314,7 +314,14 @@ criteria: AC-J07-006: delivery: proposed AC-J07-007: - delivery: proposed + delivery: partial + enforce: true + capability: eval-coverage + surfaces: ["eval coverage --graph"] + evidence: + - {id: cjm-eval-coverage-gaps, kind: unit, stubs: []} + gaps: + - "the need graph is built by a model and read as given: building, interviewing and drafting labels are not commands yet (#331)" # ---------------------------------------------------------------- J11 AC-J11-001: diff --git a/dokimi/tests/test_eval_coverage.py b/dokimi/tests/test_eval_coverage.py new file mode 100644 index 000000000..19cc90c72 --- /dev/null +++ b/dokimi/tests/test_eval_coverage.py @@ -0,0 +1,472 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""`eval coverage`: coverage and relevance of an eval set against a frozen need graph (#331).""" + +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from dokimi.artifacts import ArtifactValidationError, load_artifact +from dokimi.cli import build_parser +from dokimi.eval_coverage import LabelledEval, coverage_report, exact_shares, graph_digest, load_graph, read_labels +from dokimi.novelty.needs import NeedGraph, NeedNode +from dokimi.novelty.needs import load_graph as novelty_load_graph +from dokimi.novelty.needs import save_graph + +NODES = [ + {"id": "n1", "text": "convert with asnumpy before numpy.save", "path_class": "core"}, + {"id": "n2", "text": "memory-map files larger than RAM", "path_class": "core", + "confirmed_by": "user"}, + {"id": "n3", "text": "keep allow_pickle=False for untrusted files", "path_class": "rare_critical"}, + {"id": "n4", "text": "choose HDF5 for chunked data", "path_class": "branch"}, + {"id": "n5", "text": "plot the array", "path_class": "out_of_scope"}, +] + + +def _graph(tmp_path: Path) -> Path: + path = tmp_path / "need_graph.json" + path.write_text(json.dumps({"artifact_name": "dpnp-io", "nodes": NODES})) + return path + + +def _skill(tmp_path: Path, rows: list[dict]) -> Path: + skill = tmp_path / "skill" + (skill / "evals").mkdir(parents=True) + (skill / "evals" / "evals.json").write_text(json.dumps({"skill_name": "dpnp-io", "evals": rows})) + return skill + + +ROWS = [ + {"id": "save", "prompt": "save a dpnp array", "needs": ["n1"]}, + {"id": "big-file", "prompt": "50 GB file", "needs": ["n2", "n4"]}, + {"id": "plot", "prompt": "plot it", "needs": ["n5"]}, + {"id": "css", "prompt": "fix my css", "needs": []}, + {"id": "new", "prompt": "not mapped yet"}, +] + + +def _run(capsys, *argv: str) -> SimpleNamespace: + args = build_parser().parse_args(["eval", "coverage", *argv]) + code = 0 + try: + args.func(args) + except SystemExit as exc: + code = int(exc.code or 0) + out = capsys.readouterr() + return SimpleNamespace(returncode=code, stdout=out.out, stderr=out.err) + + +@pytest.mark.outcome("cjm-eval-coverage-gaps") +def test_report_lists_uncovered_critical_nodes_and_classifies_each_eval(tmp_path): + report = coverage_report(load_graph(_graph(tmp_path)), read_labels(_skill(tmp_path, ROWS)), "skill") + + assert [g["id"] for g in report["gaps"]] == ["n3"] + assert report["gaps"][0]["confirmed_by"] == "hypothesis" + assert {e["id"]: e["relevance"] for e in report["evals"]} == { + "save": "relevant_core", "big-file": "relevant_core", "plot": "out_of_scope", + "css": "orphan", "new": "unlabelled", + } + # in scope: n1 1.0 + n2 1.0 + n3 1.0 + n4 0.7 = 3.7; covered n1, n2, n4 = 2.7 + assert report["coverage"]["weighted"] == pytest.approx(2.7 / 3.7, abs=1e-4) + assert report["coverage"]["core_rare_critical"] == pytest.approx(2 / 3, abs=1e-4) + assert report["coverage"]["rare_critical"] == 0.0 + assert (report["graph"]["hypotheses"], report["graph"]["in_scope"]) == (4, 4) + + +def test_peripheral_eval_and_task_metadata_needs(tmp_path): + tasks = tmp_path / "tasks" + task = tasks / "hdf5-write" + task.mkdir(parents=True) + (task / "task.toml").write_text('[metadata]\nneeds = ["n4"]\n') + (task / "instruction.md").write_text("write chunks") + + labels = read_labels(tasks) + report = coverage_report(load_graph(_graph(tmp_path)), labels, "tasks") + + assert [(e.id, e.source, e.needs) for e in labels] == [("hdf5-write", "task", ("n4",))] + assert report["evals"][0]["relevance"] == "relevant_peripheral" + assert report["coverage"]["by_class"]["branch"] == {"covered": 1, "total": 1} + + +def test_same_files_same_report_and_digest_ignores_node_order(tmp_path): + graph, skill = _graph(tmp_path), _skill(tmp_path, ROWS) + first = coverage_report(load_graph(graph), read_labels(skill), "skill") + second = coverage_report(load_graph(graph), read_labels(skill), "skill") + reordered = {"artifact_name": "dpnp-io", "nodes": list(reversed(NODES))} + + assert json.dumps(first, sort_keys=True) == json.dumps(second, sort_keys=True) + assert graph_digest(reordered) == first["graph"]["digest"] + assert graph_digest({"nodes": NODES[:4]}) != first["graph"]["digest"] + defaults = [{**n, "type": "intent", "confirmed_by": n.get("confirmed_by", "hypothesis"), + "note": "x"} for n in NODES] + assert graph_digest({"nodes": defaults}) == first["graph"]["digest"] + confirmed = [{**n, "confirmed_by": "user"} for n in NODES] + assert graph_digest({"nodes": confirmed}) != first["graph"]["digest"] + + +def test_novelty_save_graph_output_loads_as_need_graph(tmp_path): + path = tmp_path / "g.json" + node = NeedNode("n1", "a need", path_class="core", confirmed_by="user") + save_graph(NeedGraph("x", "d", nodes=[node], provenance={"draws": 3}), path) + + assert load_graph(path)["nodes"][0]["confirmed_by"] == "user" + assert novelty_load_graph(path).as_dict() == json.loads(path.read_text()) + + +def test_both_readers_agree_on_the_fixture(): + fixture = Path(__file__).parent / "fixtures" / "artifacts" / "need_graph.v1.json" + novelty = novelty_load_graph(fixture) + + assert [(n.id, n.path_class, n.confirmed_by) for n in novelty.nodes] == [ + (n["id"], n["path_class"], n.get("confirmed_by", "hypothesis")) + for n in load_graph(fixture)["nodes"]] + + +def test_duplicate_node_ids_are_rejected(tmp_path): + path = tmp_path / "g.json" + path.write_text(json.dumps({"artifact_name": "x", "nodes": [ + {"id": "n1", "text": "a", "path_class": "core"}, + {"id": "n1", "text": "b", "path_class": "out_of_scope"}]})) + + with pytest.raises(ValueError, match="duplicate node ids"): + load_graph(path) + + +def test_graph_with_unknown_path_class_is_rejected(tmp_path): + path = tmp_path / "g.json" + path.write_text(json.dumps({"artifact_name": "x", "nodes": [ + {"id": "n1", "text": "t", "path_class": "critical"}]})) + + with pytest.raises(ArtifactValidationError, match="path_class"): + load_graph(path) + + +def test_cli_writes_artifact_and_fails_under_floor(tmp_path, capsys): + out = tmp_path / "cov.json" + graph, skill = _graph(tmp_path), _skill(tmp_path, ROWS) + + ok = _run(capsys, str(skill), "--graph", str(graph), "--out", str(out)) + low = _run(capsys, str(skill), "--graph", str(graph), "--min", "0.5", + "--min", "rare_critical=0.5") + + assert ok.returncode == 0, ok.stderr + assert "n3 [rare_critical, hypothesis]" in ok.stdout + assert load_artifact("eval_coverage", out)["gaps"][0]["id"] == "n3" + assert low.returncode == 1 + assert "FAIL: rare_critical coverage 0.000000 < 0.500000" in low.stderr + assert load_artifact("eval_coverage", out)["coverage"]["weights"]["branch"] == 0.7 + + +def test_cli_unknown_need_id_and_bad_inputs_exit_2(tmp_path, capsys): + graph = _graph(tmp_path) + typo = _skill(tmp_path, [{"id": "save", "needs": ["n9"]}]) + broken = tmp_path / "evals.json" + broken.write_text("{not json") + + out = tmp_path / "cov.json" + unknown = _run(capsys, str(typo), "--graph", str(graph), "--out", str(out)) + bad = _run(capsys, str(broken), "--graph", str(graph)) + missing = _run(capsys, str(tmp_path / "nope"), "--graph", str(graph)) + no_graph = _run(capsys, str(typo), "--graph", str(tmp_path / "none.json")) + + assert unknown.returncode == 2 + assert load_artifact("eval_coverage", out)["problems"] == [ + {"eval": "save", "source": "evals.json", "kind": "unknown_need", "need": "n9"}] + assert (bad.returncode, missing.returncode, no_graph.returncode) == (2, 2, 2) + assert "not a skill-creator evals.json" in bad.stderr + + +def test_cli_rejects_a_floor_outside_0_1(capsys): + for value in ("1.5", "high", "recall=0.5"): + with pytest.raises(SystemExit) as exc: + build_parser().parse_args(["eval", "coverage", "x", "--graph", "g", "--min", value]) + assert exc.value.code == 2 + err = capsys.readouterr().err + assert "must lie in 0-1, got 1.5" in err and "got 'recall'" in err + + +def test_markdown_lists_problems_and_says_none_without_gaps(tmp_path, capsys): + rows = [{"id": "all", "needs": ["n1", "n2", "n3", "n9"]}] + res = _run(capsys, str(_skill(tmp_path, rows)), "--graph", str(_graph(tmp_path))) + + assert res.returncode == 2 + assert "- none" in res.stdout + assert "Problems:\n- all (evals.json): unknown need id 'n9'" in res.stdout + + +@pytest.mark.parametrize("needs", ["n1", {"n1": 1}, 3, [1], None]) +def test_needs_that_is_not_a_list_of_ids_is_a_problem(tmp_path, capsys, needs): + res = _run(capsys, str(_skill(tmp_path, [{"id": "e", "needs": needs}])), "--graph", + str(_graph(tmp_path))) + + assert res.returncode == 2 + assert "- e (evals.json): needs is not a list of node ids" in res.stdout + + +def test_rows_that_are_not_objects_are_ignored_and_an_unparsable_task_refused(tmp_path): + root = tmp_path / "suite" + root.mkdir() + (root / "evals.json").write_text(json.dumps({"evals": ["text row", {"needs": ["n1"]}]})) + assert read_labels(root) == [LabelledEval("1", "evals.json", ("n1",))] + + bad = root / "broken-task" + bad.mkdir() + (bad / "task.toml").write_text("[metadata\n") + (bad / "instruction.md").write_text("x") + with pytest.raises(ValueError, match="unreadable task"): + read_labels(root) + + +def test_cli_refuses_an_empty_suite_a_non_list_evals_and_out_over_graph(tmp_path, capsys): + graph, empty = _graph(tmp_path), tmp_path / "empty" + empty.mkdir() + (tmp_path / "evals.json").write_text(json.dumps({"evals": {"id": "x"}})) + + none = _run(capsys, str(empty), "--graph", str(graph)) + dict_rows = _run(capsys, str(tmp_path / "evals.json"), "--graph", str(graph)) + clobber = _run(capsys, str(_skill(tmp_path, ROWS)), "--graph", str(graph), "--out", str(graph)) + + assert (none.returncode, dict_rows.returncode, clobber.returncode) == (2, 2, 2) + assert "no evals.json rows or tasks" in none.stderr + assert '"evals" is not a list' in dict_rows.stderr + assert "--out would overwrite an input file" in clobber.stderr + assert json.loads(graph.read_text())["nodes"] == NODES + + +def test_graph_with_only_out_of_scope_nodes_has_no_coverage_figures(tmp_path): + graph = {"artifact_name": "x", "nodes": [{"id": "n1", "text": "t", "path_class": "out_of_scope"}]} + report = coverage_report(graph, [LabelledEval("e", "task", None)], "r") + + assert report["coverage"] == { + "weights": {"core": 1.0, "rare_critical": 1.0, "branch": 0.7, "supporting": 0.4, + "out_of_scope": 0.0}, + "weighted": None, "core_rare_critical": None, "rare_critical": None, + "by_class": {c: {"covered": 0, "total": int(c == "out_of_scope")} + for c in ("core", "rare_critical", "branch", "supporting", "out_of_scope")}, + } + + +@pytest.mark.parametrize("target", ["graph", "evals", "task", "instruction"]) +def test_out_never_overwrites_an_input_file(tmp_path, capsys, target): + graph, skill = _graph(tmp_path), _skill(tmp_path, ROWS) + tasks, task = tmp_path / "tasks", tmp_path / "tasks" / "t1" + task.mkdir(parents=True) + (task / "task.toml").write_text('[metadata]\nneeds = ["n1"]\n') + (task / "instruction.md").write_text("x") + victim = {"graph": graph, "evals": skill / "evals" / "evals.json", "task": task / "task.toml", + "instruction": task / "instruction.md"}[target] + root = tasks if target in ("task", "instruction") else skill + before = victim.read_bytes() + + res = _run(capsys, str(root), "--graph", str(graph), "--out", str(victim)) + + assert res.returncode == 2 + assert "--out would overwrite an input file" in res.stderr + assert victim.read_bytes() == before + + +@pytest.mark.parametrize("name", ["task.yaml", "docker-compose.yaml"]) +def test_out_never_overwrites_a_terminal_bench_task_file(tmp_path, capsys, name): + task = tmp_path / "tasks" / "tb1" + task.mkdir(parents=True) + (task / "task.yaml").write_text("instruction: write chunks\n") + (task / "docker-compose.yaml").write_text("services: {}\n") + victim = task / name + before = victim.read_bytes() + + res = _run(capsys, str(tmp_path / "tasks"), "--graph", str(_graph(tmp_path)), "--out", str(victim)) + + assert res.returncode == 2, res.stderr + assert "--out would overwrite an input file" in res.stderr + assert victim.read_bytes() == before + + +def test_a_task_md_without_frontmatter_is_a_note_not_a_task(tmp_path, capsys): + tasks = tmp_path / "tasks" + notes = tasks / "notes" + notes.mkdir(parents=True) + (notes / "task.md").write_text("# planning notes\n") + real = tasks / "t1" + real.mkdir() + (real / "task.toml").write_text('[metadata]\nneeds = ["n1"]\n') + + labels = read_labels(tasks) + res = _run(capsys, str(tasks), "--graph", str(_graph(tmp_path))) + + assert [e.id for e in labels] == ["t1"] + assert res.returncode == 0, res.stderr + + +def test_exact_shares_are_the_hand_computed_figures(): + # weighted = (core c1 + 3 branch at 0.7) / (core c1, c2 + rare r1 + 3 branch) = 3.1 / 5.1; out_of_scope is outside it + graph = {"artifact_name": "x", "nodes": [ + {"id": "c1", "text": "t", "path_class": "core"}, + {"id": "c2", "text": "t", "path_class": "core"}, + {"id": "r1", "text": "t", "path_class": "rare_critical"}, + {"id": "b1", "text": "t", "path_class": "branch"}, + {"id": "b2", "text": "t", "path_class": "branch"}, + {"id": "b3", "text": "t", "path_class": "branch"}, + {"id": "o1", "text": "t", "path_class": "out_of_scope"}, + ]} + evals = [LabelledEval("e1", "task", ("c1", "b1", "b2", "b3", "o1"))] + + figures = exact_shares(coverage_report(graph, evals, "r")) + + assert figures["weighted"] == pytest.approx(3.1 / 5.1, abs=1e-12) + assert figures["core_rare_critical"] == pytest.approx(1 / 3, abs=1e-12) + assert figures["rare_critical"] == 0.0 + + +def test_a_bare_floor_is_on_the_weighted_figure(tmp_path, capsys): + # weighted is 0.6078 and core_rare_critical 0.3333: a bare 0.5 passes only if the default metric is weighted + graph = tmp_path / "need_graph.json" + graph.write_text(json.dumps({"artifact_name": "x", "nodes": [ + {"id": "c1", "text": "t", "path_class": "core"}, + {"id": "c2", "text": "t", "path_class": "core"}, + {"id": "r1", "text": "t", "path_class": "rare_critical"}, + {"id": "b1", "text": "t", "path_class": "branch"}, + {"id": "b2", "text": "t", "path_class": "branch"}, + {"id": "b3", "text": "t", "path_class": "branch"}, + ]})) + skill = _skill(tmp_path, [{"id": "e", "needs": ["c1", "b1", "b2", "b3"]}]) + + res = _run(capsys, str(skill), "--graph", str(graph), "--min", "0.5") + + assert res.returncode == 0, res.stderr + + +def test_a_floor_equal_to_the_share_passes(tmp_path, capsys): + nodes = [{"id": f"c{i}", "text": "t", "path_class": "core"} for i in range(4)] + graph = tmp_path / "need_graph.json" + graph.write_text(json.dumps({"artifact_name": "x", "nodes": nodes})) + skill = _skill(tmp_path, [{"id": "e", "needs": ["c0", "c1"]}]) + + res = _run(capsys, str(skill), "--graph", str(graph), "--min", "core_rare_critical=0.5") + + assert res.returncode == 0, res.stderr + + +def test_an_unreadable_evals_directory_exits_2_not_a_traceback(tmp_path, capsys, monkeypatch): + skill = _skill(tmp_path, ROWS) + real_exists = Path.exists + + def exists(self, *args, **kwargs): + if self == skill / "evals" / "evals.json" or self == skill: + raise PermissionError(13, "Permission denied", str(self)) + return real_exists(self, *args, **kwargs) + + monkeypatch.setattr(Path, "exists", exists) + res = _run(capsys, str(skill), "--graph", str(_graph(tmp_path))) + + assert res.returncode == 2 + assert "Permission denied" in res.stderr + + +def test_out_whose_rejected_sibling_is_the_graph_is_refused(tmp_path, capsys): + # save_artifact removes .rejected after a valid save: an input at that path must not be deleted + graph = _graph(tmp_path) + sibling = graph.with_name("cov.json.rejected") + graph.rename(sibling) + before = sibling.read_bytes() + + res = _run(capsys, str(_skill(tmp_path, ROWS)), "--graph", str(sibling), "--out", str(tmp_path / "cov.json")) + + assert res.returncode == 2 + assert "--out would overwrite an input file" in res.stderr + assert sibling.read_bytes() == before + + +def test_a_floor_on_a_class_the_graph_lacks_is_n_a_and_fails(tmp_path, capsys): + nodes = [{"id": f"c{i}", "text": "t", "path_class": "core"} for i in range(2)] + graph = tmp_path / "need_graph.json" + graph.write_text(json.dumps({"artifact_name": "x", "nodes": nodes})) + skill = _skill(tmp_path, [{"id": "e", "needs": ["c0"]}]) + + res = _run(capsys, str(skill), "--graph", str(graph), "--min", "rare_critical=0.5") + + assert res.returncode == 1 + assert "FAIL: rare_critical coverage n/a < 0.500000" in res.stderr + + +def test_out_that_is_a_symlink_loop_exits_2_not_a_traceback(tmp_path, capsys, monkeypatch): + # Python 3.11/3.12 raise RuntimeError from Path.resolve() on a loop; 3.13+ do not, so raise it here + monkeypatch.chdir(tmp_path) + real_resolve = Path.resolve + + def resolve(self, *args, **kwargs): + if self.name == "loop": + raise RuntimeError("Symlink loop") + return real_resolve(self, *args, **kwargs) + + monkeypatch.setattr(Path, "resolve", resolve) + + res = _run(capsys, str(_skill(tmp_path, ROWS)), "--graph", str(_graph(tmp_path)), "--out", "loop") + + assert res.returncode == 2 + assert "cannot resolve" in res.stderr + + +def test_out_as_a_relative_path_to_an_input_is_refused(tmp_path, capsys, monkeypatch): + skill = _skill(tmp_path, ROWS) + work = tmp_path / "work" + work.mkdir() + monkeypatch.chdir(work) + victim = skill / "evals" / "evals.json" + before = victim.read_bytes() + + res = _run(capsys, "../skill", "--graph", str(_graph(tmp_path)), "--out", "../skill/evals/evals.json") + + assert res.returncode == 2 + assert "--out would overwrite an input file" in res.stderr + assert victim.read_bytes() == before + + +def test_a_null_task_needs_is_malformed_and_a_missing_key_is_unlabelled(tmp_path): + tasks = tmp_path / "tasks" + task = tasks / "t-null" + task.mkdir(parents=True) + (task / "task.md").write_text("---\nmetadata:\n needs:\n---\n\nwrite chunks\n") + missing = tasks / "t-missing" + missing.mkdir() + (missing / "task.toml").write_text("[metadata]\n") + (missing / "instruction.md").write_text("x") + + labels = {e.id: e for e in read_labels(tasks)} + + assert labels["t-null"].malformed and labels["t-null"].needs is None + assert not labels["t-missing"].malformed and labels["t-missing"].needs is None + + +@pytest.mark.parametrize(("covered", "floor", "code"), [(6, "0.85714", 0), (2, "0.66669", 1)]) +def test_a_floor_is_judged_on_the_unrounded_share(tmp_path, capsys, covered, floor, code): + # 6 of 7 core nodes is 0.857142... (rounds to 0.8571); 2 of 3 is 0.666666... (rounds to 0.6667) + cores = 7 if covered == 6 else 3 + nodes = [{"id": f"c{i}", "text": "t", "path_class": "core"} for i in range(cores)] + graph = tmp_path / "need_graph.json" + graph.write_text(json.dumps({"artifact_name": "x", "nodes": nodes})) + skill = _skill(tmp_path, [{"id": "e", "needs": [f"c{i}" for i in range(covered)]}]) + + res = _run(capsys, str(skill), "--graph", str(graph), "--min", f"core_rare_critical={floor}") + + assert res.returncode == code, res.stderr + + +def test_markdown_table_cells_escape_pipes_and_newlines(tmp_path, capsys): + rows = [{"id": "save|load", "needs": ["n1"]}, {"id": "two\nlines", "needs": ["n2", "n3"]}] + res = _run(capsys, str(_skill(tmp_path, rows)), "--graph", str(_graph(tmp_path))) + + assert res.returncode == 0, res.stderr + assert "| save\\|load | evals.json | n1 | " in res.stdout + assert "| two lines | evals.json | n2, n3 | " in res.stdout + + +def test_out_on_a_directory_is_a_controlled_error(tmp_path, capsys): + res = _run(capsys, str(_skill(tmp_path, ROWS)), "--graph", str(_graph(tmp_path)), "--out", str(tmp_path)) + + assert res.returncode == 2 + assert "Error: cannot write" in res.stderr and "Traceback" not in res.stderr diff --git a/dokimi/tests/test_import_architecture.py b/dokimi/tests/test_import_architecture.py index 6d1025d78..969eef92c 100644 --- a/dokimi/tests/test_import_architecture.py +++ b/dokimi/tests/test_import_architecture.py @@ -49,7 +49,7 @@ # repositories, subjects, reports, dashboards, the one-command pipeline. ("tracks", ( "dokimi.config_check", "dokimi.context", "dokimi.coverage", "dokimi.dashboard", - "dokimi.datasets_export", "dokimi.explore", "dokimi.external", "dokimi.harnesses", + "dokimi.datasets_export", "dokimi.eval_coverage", "dokimi.explore", "dokimi.external", "dokimi.harnesses", "dokimi.novelty", "dokimi.orchestrator", "dokimi.report", "dokimi.repos", "dokimi.runner", "dokimi.runs", "dokimi.schema_check", "dokimi.schema_compat", "dokimi.subjects", "dokimi.task_formats", "dokimi.tool_preference", diff --git a/dokimi/tests/test_runs_scan.py b/dokimi/tests/test_runs_scan.py index 9b5b9997d..b5bd54ee2 100644 --- a/dokimi/tests/test_runs_scan.py +++ b/dokimi/tests/test_runs_scan.py @@ -48,6 +48,8 @@ "repo_budget.v1": ({}, "toy", None, None), "task_inspect.v1": ({}, "tests/fixtures/task_formats", None, None), "cli_routes.v1": ({}, None, None, None), + "need_graph.v1": ({}, "dpnp-io", None, None), + "eval_coverage.v1": ({}, "dpnp-io", None, None), } diff --git a/dokimi/tests/test_runs_show.py b/dokimi/tests/test_runs_show.py index 07ad7a9f5..de76bb8d5 100644 --- a/dokimi/tests/test_runs_show.py +++ b/dokimi/tests/test_runs_show.py @@ -37,6 +37,8 @@ "result.v1": {"cells": 6}, "subject_scorecard.v2": {"cells": 0}, "cli_routes.v1": {"roots": 2, "leaves": 2, "tombstones": 1}, + "eval_coverage.v1": {"gaps": 1, "evals": 2, "problems": 0}, + "need_graph.v1": {"nodes": 3, "edges": 1}, "task_inspect.v1": {"items": 4}, "task_runs.v1": {"tasks": 6, "harnesses": 2, "results": 24}, "tool_preference.v1": {"items": 16, "results": 16},