From 2c41579392951d4016ae7a0c5a0c174ad781e044 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 20:16:39 -0700 Subject: [PATCH 1/9] feat(eval): eval coverage: coverage and relevance of an eval set against a frozen need graph (#331) need_graph.v1 and eval_coverage.v1 schemas; evals name graph nodes in 'needs' (evals.json rows or task metadata). Pure function of files, no model call; exit 1 under --min-coverage / --min-rare-critical, 2 on unknown ids. Signed-off-by: Nikolay Petrov --- .../20261031_180000_napetrov_eval-coverage.md | 17 ++ .../references/coverage-report.md | 16 +- dokimi/docs/reference/cli-interface.json | 125 ++++++++++++ dokimi/docs/user-journeys/journeys.md | 2 +- dokimi/dokimi/artifacts.py | 5 +- dokimi/dokimi/cli_routes.py | 1 + dokimi/dokimi/commands/eval_coverage.py | 81 ++++++++ dokimi/dokimi/commands/evaluate.py | 2 + dokimi/dokimi/eval_coverage.py | 179 +++++++++++++++++ dokimi/dokimi/runs/kinds.py | 2 + dokimi/dokimi/schemas/eval_coverage.v1.json | 74 +++++++ dokimi/dokimi/schemas/need_graph.v1.json | 37 ++++ dokimi/tests/data/cli_flags_allowlist.txt | 3 + .../fixtures/artifacts/eval_coverage.v1.json | 63 ++++++ .../fixtures/artifacts/need_graph.v1.json | 38 ++++ dokimi/tests/outcomes/journeys.yaml | 9 +- dokimi/tests/test_eval_coverage.py | 188 ++++++++++++++++++ dokimi/tests/test_import_architecture.py | 2 +- dokimi/tests/test_runs_scan.py | 2 + dokimi/tests/test_runs_show.py | 2 + 20 files changed, 840 insertions(+), 8 deletions(-) create mode 100644 dokimi/changelog.d/20261031_180000_napetrov_eval-coverage.md create mode 100644 dokimi/dokimi/commands/eval_coverage.py create mode 100644 dokimi/dokimi/eval_coverage.py create mode 100644 dokimi/dokimi/schemas/eval_coverage.v1.json create mode 100644 dokimi/dokimi/schemas/need_graph.v1.json create mode 100644 dokimi/tests/fixtures/artifacts/eval_coverage.v1.json create mode 100644 dokimi/tests/fixtures/artifacts/need_graph.v1.json create mode 100644 dokimi/tests/test_eval_coverage.py diff --git a/dokimi/changelog.d/20261031_180000_napetrov_eval-coverage.md b/dokimi/changelog.d/20261031_180000_napetrov_eval-coverage.md new file mode 100644 index 000000000..815f6c0a8 --- /dev/null +++ b/dokimi/changelog.d/20261031_180000_napetrov_eval-coverage.md @@ -0,0 +1,17 @@ + + +### Added + +- **`eval coverage`: which needs of a subject an eval set grades, without a model call.** + `python cli.py eval coverage --graph need_graph.json` + reads a frozen need graph (new `need_graph.v1` schema; a novelty need graph is valid as + written) and the `needs` list on each `evals.json` row or in a task's `metadata.needs`. It + prints the weighted coverage, the coverage of `core` + `rare_critical` and of `rare_critical` + alone, the uncovered `core` / `rare_critical` nodes, and each eval's relevance + (`relevant_core`, `relevant_peripheral`, `out_of_scope`, `orphan`, `unlabelled`). `--out` + writes an `eval_coverage.v1` artifact. Exit 1 under `--min-coverage` / `--min-rare-critical`, + exit 2 on an unreadable input or a `needs` id the graph lacks. Building the graph, the user + interview and drafting labels follow in #331. diff --git a/dokimi/data/skills/dokimi-design-evals/references/coverage-report.md b/dokimi/data/skills/dokimi-design-evals/references/coverage-report.md index ff1e2fcce..e97f083fe 100644 --- a/dokimi/data/skills/dokimi-design-evals/references/coverage-report.md +++ b/dokimi/data/skills/dokimi-design-evals/references/coverage-report.md @@ -1,9 +1,17 @@ # Coverage report -dokimi has no command that writes this report (`python -m dokimi.coverage` -prints a product x awareness/work task count, not scenario coverage). Write -it by hand, in the suite's README and in the PR that adds or changes the -suite. +Write this report by hand, in the suite's README and in the PR that adds or +changes the suite. `python -m dokimi.coverage` prints only a product x +awareness/work task count. + +`python cli.py eval coverage --graph need_graph.json` adds the part +a script can check. It needs a need graph (what users of the subject need, +built from its name and description only) and a `needs` list on each eval +that names the graph nodes it grades. From those two files alone it lists the +core and rare_critical nodes that no eval names, and the relevance of each +eval: `orphan` for `needs: []`, `unlabelled` for no `needs`. Paste its +Markdown under the table below. It does not replace the areas in the table, +and a node the graph never had is still a gap nobody looked at. ## Template diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index c9eb437d1..5e8c80e4f 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -4017,6 +4017,129 @@ "subcommand_required": null, "subcommands": {} }, + "coverage": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "evals", + "kind": "argument", + "nargs": null, + "position": 0, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'json'", + "'markdown'" + ], + "const": null, + "default": "markdown", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--format", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--graph", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--min-coverage", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.commands.eval_coverage._floor#e9bf1357f2fa" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--min-rare-critical", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.commands.eval_coverage._floor#e9bf1357f2fa" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--out", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi eval coverage", + "subcommand_required": null, + "subcommands": {} + }, "cross-judge": { "allow_abbrev": true, "params": [ @@ -12885,9 +13008,11 @@ "'arms'", "'cli_routes'", "'eval'", + "'eval_coverage'", "'explore_runs'", "'guidance_reach'", "'matrix_rollup'", + "'need_graph'", "'novelty'", "'plan'", "'plugin_delta'", diff --git a/dokimi/docs/user-journeys/journeys.md b/dokimi/docs/user-journeys/journeys.md index 1f26b9c5e..c49317040 100644 --- a/dokimi/docs/user-journeys/journeys.md +++ b/dokimi/docs/user-journeys/journeys.md @@ -492,7 +492,7 @@ down in the PR. #### UC-J07-004 State coverage gaps -- **AC-J07-007** An eval set's report lists the capabilities or areas it has no task for. *Proposed.* +- **AC-J07-007** An eval set's report lists the capabilities or areas it has no task for. *Partial: `eval coverage` lists the core and rare_critical nodes of a frozen need graph that no eval's `needs` names; building the graph, the user interview and drafting labels are #331.* --- diff --git a/dokimi/dokimi/artifacts.py b/dokimi/dokimi/artifacts.py index 0088ff5eb..55fd25cf6 100644 --- a/dokimi/dokimi/artifacts.py +++ b/dokimi/dokimi/artifacts.py @@ -12,7 +12,8 @@ what moves it), ``repo_environment`` (static inspection of what a repository offers an agent), ``plan`` (the cells a run promises to measure), ``result`` (one status per planned cell), ``repo_budget`` (what a repository puts -into an agent's context) and ``task_inspect`` (which eval-task format each +into an agent's context), ``need_graph`` and ``eval_coverage`` (a frozen need graph and an +eval set's coverage of it) and ``task_inspect`` (which eval-task format each path under a tree is). This module stamps each with a ``schema_version`` and validates it against the JSON Schema in ``dokimi/schemas/`` on save and load, so fixtures and baselines stay @@ -58,6 +59,8 @@ "repo_budget": "repo_budget.v1", "task_inspect": "task_inspect.v1", "cli_routes": "cli_routes.v1", + "need_graph": "need_graph.v1", + "eval_coverage": "eval_coverage.v1", } # kind -> superseded versions still accepted on *load*, newest first. An artifact written under an diff --git a/dokimi/dokimi/cli_routes.py b/dokimi/dokimi/cli_routes.py index 47008bcc9..8d57df1ed 100644 --- a/dokimi/dokimi/cli_routes.py +++ b/dokimi/dokimi/cli_routes.py @@ -142,6 +142,7 @@ class Tombstone: Leaf(("eval", "score"), PAID), Leaf(("eval", "ragas"), PAID), Leaf(("eval", "grounding"), FREE), + Leaf(("eval", "coverage"), FREE), Leaf(("eval", "judge-validity"), PAID), Leaf(("eval", "checklist"), PAID), Leaf(("eval", "judge-wording"), PAID), diff --git a/dokimi/dokimi/commands/eval_coverage.py b/dokimi/dokimi/commands/eval_coverage.py new file mode 100644 index 000000000..58845b9f3 --- /dev/null +++ b/dokimi/dokimi/commands/eval_coverage.py @@ -0,0 +1,81 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""``eval coverage``: coverage and relevance of an eval set against a frozen need graph (#331).""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from dokimi.exit_codes import EXIT_FAILURE, EXIT_USAGE + + +def _floor(value: str) -> float: + try: + number = float(value) + except ValueError: + raise argparse.ArgumentTypeError(f"expected a number, got {value!r}") from None + if not 0.0 <= number <= 1.0: + raise argparse.ArgumentTypeError(f"must lie in 0-1, got {number}") + return number + + +def _below(report: dict, args: argparse.Namespace) -> list[str]: + cov = report["coverage"] + failed = [] + for name, floor, value in (("weighted", args.min_coverage, cov["weighted"]), + ("rare_critical", args.min_rare_critical, cov["rare_critical"])): + if floor is not None and value is not None and value < floor: + failed.append(f"{name} coverage {value:.2f} < {floor:.2f}") + return failed + + +def cmd_eval_coverage(args: argparse.Namespace) -> None: + """Print the report; exit 2 on an unreadable input or unknown need ids, 1 under a floor.""" + from dokimi.artifacts import ArtifactValidationError, save_artifact + from dokimi.eval_coverage import coverage_report, load_graph, read_labels, render_markdown + + root = Path(args.evals) + if not root.exists(): + print(f"Error: {root} does not exist", file=sys.stderr) + sys.exit(EXIT_USAGE) + try: + graph = load_graph(Path(args.graph)) + evals = read_labels(root) + except (OSError, ValueError, ArtifactValidationError) as exc: + print(f"Error: {exc}", file=sys.stderr) + sys.exit(EXIT_USAGE) + report = coverage_report(graph, evals, str(root)) + if args.out: + report = save_artifact("eval_coverage", report, Path(args.out)) + print(json.dumps(report, indent=2) if args.format == "json" else render_markdown(report)) + if report["problems"]: + print(f"Error: {len(report['problems'])} eval(s) name need ids the graph does not have", + file=sys.stderr) + sys.exit(EXIT_USAGE) + failed = _below(report, args) + if failed: + print("FAIL: " + "; ".join(failed), file=sys.stderr) + sys.exit(EXIT_FAILURE) + + +def register(eval_sub) -> None: + """Add ``eval coverage`` to the ``eval`` group.""" + p = eval_sub.add_parser( + "coverage", + help="Coverage of a frozen need graph by an eval set's `needs` labels (no model call)", + ) + p.add_argument("evals", help="evals.json, a skill directory (evals/evals.json) or a tasks root") + p.add_argument("--graph", required=True, + help="need_graph.v1 file (a novelty need graph is valid as written)") + p.add_argument("--out", default=None, help="Write the eval_coverage.v1 artifact here") + p.add_argument("--format", choices=["markdown", "json"], default="markdown", + help="What stdout shows (default: markdown)") + p.add_argument("--min-coverage", type=_floor, default=None, dest="min_coverage", + help="Exit 1 when weighted coverage is below this (0-1)") + p.add_argument("--min-rare-critical", type=_floor, default=None, dest="min_rare_critical", + help="Exit 1 when rare_critical coverage is below this (0-1)") + p.set_defaults(func=cmd_eval_coverage) diff --git a/dokimi/dokimi/commands/evaluate.py b/dokimi/dokimi/commands/evaluate.py index 18f1da851..0ecf5f29e 100644 --- a/dokimi/dokimi/commands/evaluate.py +++ b/dokimi/dokimi/commands/evaluate.py @@ -11,6 +11,7 @@ import sys from pathlib import Path +from dokimi.commands import eval_coverage from dokimi.llm import ANSWER_PROVIDERS from dokimi.defaults import ( DEFAULT_CONCURRENCY, @@ -467,6 +468,7 @@ def register(sub, positive_int) -> None: # Eval subcommand group eval_p = sub.add_parser("eval", help="Evaluate answers") eval_sub = eval_p.add_subparsers(dest="eval_cmd", required=True) + eval_coverage.register(eval_sub) # eval score score_p = eval_sub.add_parser("score", help="Score answers using LLM-as-judge") diff --git a/dokimi/dokimi/eval_coverage.py b/dokimi/dokimi/eval_coverage.py new file mode 100644 index 000000000..8e5951c9a --- /dev/null +++ b/dokimi/dokimi/eval_coverage.py @@ -0,0 +1,179 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Eval coverage of a subject's problem space, from files only (issue #331, AC-J07-007). + +The problem space is a frozen need graph (``need_graph.v1``): the graph +:func:`dokimi.novelty.needs.build_need_graph` builds from a subject's name and description, +written once and kept next to the suite. Each eval names the nodes it grades in ``needs``: + +* a row of a skill-creator ``evals.json``: ``{"id": ..., "needs": ["n1", "n4"]}``; +* a task's ``task.toml`` / ``task.md``: ``metadata.needs``. + +:func:`coverage_report` is a pure function of the graph and those labels, so the same files +always give the same report. No model is called here; drafting labels is a separate, paid step. + +Relevance of one eval: ``relevant_core`` (names a core or rare_critical node), +``relevant_peripheral`` (only branch or supporting nodes), ``out_of_scope`` (only nodes the +graph marks out of scope), ``orphan`` (``needs: []``: it grades nothing the subject is for) +and ``unlabelled`` (no ``needs`` at all: not yet mapped, so it covers nothing). +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from dokimi.artifacts import load_artifact +from dokimi.harnesses.tasks import is_task_dir, read_task_config +from dokimi.novelty.needs import PATH_WEIGHTS + +CRITICAL = ("core", "rare_critical") +CLASSES = ("core", "rare_critical", "branch", "supporting", "out_of_scope") + + +@dataclass(frozen=True) +class LabelledEval: + """One eval and the need-node ids it says it grades (``None``: it has no ``needs``).""" + + id: str + source: str + needs: tuple[str, ...] | None + + +def load_graph(path: Path) -> dict: + """Read and validate a ``need_graph.v1`` file (an unstamped ``save_graph`` file is valid).""" + return load_artifact("need_graph", Path(path)) + + +def graph_digest(graph: dict) -> str: + """sha256 of the nodes in canonical JSON: the identity a report and a plan can record.""" + nodes = sorted(graph["nodes"], key=lambda n: n["id"]) + blob = json.dumps(nodes, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return hashlib.sha256(blob.encode("utf-8")).hexdigest() + + +def _needs(value: Any) -> tuple[str, ...] | None: + if value is None: + return None + if isinstance(value, str): # one id, not a sequence of letters + return (value,) + return tuple(str(v) for v in value) if isinstance(value, list) else None + + +def _evals_files(root: Path) -> list[Path]: + if root.is_file(): + return [root] + return [p for p in (root / "evals.json", root / "evals" / "evals.json") if p.is_file()] + + +def read_labels(root: Path) -> list[LabelledEval]: + """Every eval under ``root`` with its ``needs``: skill-creator ``evals.json`` rows, then tasks. + + ``root`` is an ``evals.json`` file, a skill directory (its ``evals/evals.json``) or a tasks + root whose child directories are tasks. Raises ``ValueError`` on an unreadable ``evals.json``. + """ + out: list[LabelledEval] = [] + for path in _evals_files(root): + try: + rows = json.loads(path.read_text(encoding="utf-8")).get("evals", []) + except (OSError, ValueError, AttributeError) as exc: + raise ValueError(f"{path}: not a skill-creator evals.json ({exc})") from exc + for index, row in enumerate(rows if isinstance(rows, list) else []): + if isinstance(row, dict): + eid = str(row.get("id", index)) + out.append(LabelledEval(eid, path.name, _needs(row.get("needs")))) + if root.is_dir(): + for task_dir in sorted(p for p in root.iterdir() if is_task_dir(p)): + try: + meta = read_task_config(task_dir).get("metadata", {}) + except (ValueError, OSError): + continue + needs = _needs(meta.get("needs")) if isinstance(meta, dict) else None + out.append(LabelledEval(task_dir.name, "task", needs)) + return out + + +def _relevance(needs: tuple[str, ...] | None, classes: dict[str, str]) -> str: + if needs is None: + return "unlabelled" + named = [classes[n] for n in needs if n in classes] + if not named: + return "orphan" + if any(c in CRITICAL for c in named): + return "relevant_core" + if any(c != "out_of_scope" for c in named): + return "relevant_peripheral" + return "out_of_scope" + + +def _share(nodes: list[dict], covered: set[str]) -> float | None: + return round(sum(n["id"] in covered for n in nodes) / len(nodes), 4) if nodes else None + + +def coverage_report(graph: dict, evals: list[LabelledEval], evals_root: str) -> dict: + """The ``eval_coverage.v1`` payload (unstamped) for ``graph`` and ``evals``.""" + classes = {n["id"]: n["path_class"] for n in graph["nodes"]} + problems = [f"{e.id}: unknown need id {n!r}" + for e in evals for n in (e.needs or ()) if n not in classes] + covered = {n for e in evals for n in (e.needs or ()) if n in classes} + scope = [n for n in graph["nodes"] if n["path_class"] != "out_of_scope"] + weight = sum(PATH_WEIGHTS[n["path_class"]] for n in scope) + critical = [n for n in scope if n["path_class"] in CRITICAL] + return { + "evals_root": evals_root, + "graph": { + "artifact_name": graph["artifact_name"], + "digest": graph_digest(graph), + "nodes": len(graph["nodes"]), + "in_scope": len(scope), + "hypotheses": sum(n.get("confirmed_by", "hypothesis") == "hypothesis" + for n in graph["nodes"]), + }, + "coverage": { + "weighted": round(sum(PATH_WEIGHTS[n["path_class"]] for n in scope + if n["id"] in covered) / weight, 4) if weight else None, + "core_rare_critical": _share(critical, covered), + "rare_critical": _share([n for n in scope if n["path_class"] == "rare_critical"], covered), + "by_class": {c: {"covered": sum(n["id"] in covered for n in graph["nodes"] + if n["path_class"] == c), + "total": sum(n["path_class"] == c for n in graph["nodes"])} + for c in CLASSES}, + }, + "gaps": [{"id": n["id"], "path_class": n["path_class"], "text": n["text"], + "confirmed_by": n.get("confirmed_by", "hypothesis")} + for n in critical if n["id"] not in covered], + "evals": [{"id": e.id, "source": e.source, "needs": list(e.needs or ()), + "relevance": _relevance(e.needs, classes)} for e in evals], + "problems": problems, + } + + +def _pct(value: float | None) -> str: + return "n/a" if value is None else f"{value:.0%}" + + +def render_markdown(report: dict) -> str: + """The report as Markdown, for a suite's README or the PR that changes the suite.""" + g, c = report["graph"], report["coverage"] + lines = [ + f"## Eval coverage: {g['artifact_name']} (graph {g['digest'][:12]})", + "", + f"Weighted coverage {_pct(c['weighted'])}; core + rare_critical " + f"{_pct(c['core_rare_critical'])}; rare_critical {_pct(c['rare_critical'])}. " + f"{g['hypotheses']} of {g['nodes']} nodes are unconfirmed hypotheses.", + "", + "| Eval | Source | Needs | Relevance |", + "|---|---|---|---|", + ] + lines += [f"| {e['id']} | {e['source']} | {', '.join(e['needs']) or '-'} | {e['relevance']} |" + for e in report["evals"]] + lines += ["", "Gaps (core and rare_critical nodes no eval names):"] + lines += [f"- {n['id']} [{n['path_class']}, {n['confirmed_by']}] {n['text']}" + for n in report["gaps"]] or ["- none"] + if report["problems"]: + lines += ["", "Problems:"] + [f"- {p}" for p in report["problems"]] + return "\n".join(lines) + "\n" diff --git a/dokimi/dokimi/runs/kinds.py b/dokimi/dokimi/runs/kinds.py index cb8e49a39..fde7e7cfa 100644 --- a/dokimi/dokimi/runs/kinds.py +++ b/dokimi/dokimi/runs/kinds.py @@ -65,6 +65,8 @@ class KindFields: "repo_budget": KindFields(subject=("tree",), counts=("entries",)), "task_inspect": KindFields(subject=("root",), counts=("items",)), "cli_routes": KindFields(counts=("roots", "leaves", "tombstones")), + "need_graph": KindFields(subject=("artifact_name",), counts=("nodes", "edges")), + "eval_coverage": KindFields(subject=("graph.artifact_name",), counts=("gaps", "evals", "problems")), } #: Every schema_version an artifact may carry (current and superseded) -> its kind. diff --git a/dokimi/dokimi/schemas/eval_coverage.v1.json b/dokimi/dokimi/schemas/eval_coverage.v1.json new file mode 100644 index 000000000..f199dd765 --- /dev/null +++ b/dokimi/dokimi/schemas/eval_coverage.v1.json @@ -0,0 +1,74 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/intel/dokimi/schemas/eval_coverage.v1.json", + "title": "Eval coverage report v1", + "description": "What `eval coverage` computed from a frozen need graph and the `needs` labels of an eval set, with no model call: the same files give the same report. Coverage counts in-scope nodes some eval names; weights are dokimi.novelty.needs.PATH_WEIGHTS.", + "type": "object", + "required": ["schema_version", "evals_root", "graph", "coverage", "gaps", "evals", "problems"], + "additionalProperties": false, + "properties": { + "schema_version": {"const": "eval_coverage.v1"}, + "evals_root": {"type": "string"}, + "graph": { + "type": "object", + "required": ["artifact_name", "digest", "nodes", "in_scope", "hypotheses"], + "additionalProperties": false, + "properties": { + "artifact_name": {"type": "string"}, + "digest": {"type": "string", "description": "sha256 of the graph's nodes, canonical JSON"}, + "nodes": {"type": "integer", "minimum": 0}, + "in_scope": {"type": "integer", "minimum": 0}, + "hypotheses": {"type": "integer", "minimum": 0, "description": "Nodes no person confirmed"} + } + }, + "coverage": { + "type": "object", + "required": ["weighted", "core_rare_critical", "rare_critical", "by_class"], + "additionalProperties": false, + "properties": { + "weighted": {"type": ["number", "null"]}, + "core_rare_critical": {"type": ["number", "null"]}, + "rare_critical": {"type": ["number", "null"]}, + "by_class": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": ["covered", "total"], + "additionalProperties": false, + "properties": {"covered": {"type": "integer"}, "total": {"type": "integer"}} + } + } + } + }, + "gaps": { + "type": "array", + "description": "Uncovered core and rare_critical nodes", + "items": { + "type": "object", + "required": ["id", "path_class", "text", "confirmed_by"], + "additionalProperties": false, + "properties": { + "id": {"type": "string"}, + "path_class": {"type": "string"}, + "text": {"type": "string"}, + "confirmed_by": {"enum": ["user", "hypothesis"]} + } + } + }, + "evals": { + "type": "array", + "items": { + "type": "object", + "required": ["id", "source", "needs", "relevance"], + "additionalProperties": false, + "properties": { + "id": {"type": "string"}, + "source": {"type": "string"}, + "needs": {"type": "array", "items": {"type": "string"}}, + "relevance": {"enum": ["relevant_core", "relevant_peripheral", "out_of_scope", "orphan", "unlabelled"]} + } + } + }, + "problems": {"type": "array", "items": {"type": "string"}} + } +} diff --git a/dokimi/dokimi/schemas/need_graph.v1.json b/dokimi/dokimi/schemas/need_graph.v1.json new file mode 100644 index 000000000..b5c67c0ba --- /dev/null +++ b/dokimi/dokimi/schemas/need_graph.v1.json @@ -0,0 +1,37 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/intel/dokimi/schemas/need_graph.v1.json", + "title": "Need graph v1", + "description": "What a user of a subject needs, built from the subject's name and description only (dokimi.novelty.needs), frozen in a file so that `eval coverage` reads it instead of regenerating it. The graph `dokimi.novelty.needs.save_graph` writes is valid unstamped. `confirmed_by` says whether a person confirmed a node (`user`) or it is a model's guess (`hypothesis`, the default).", + "type": "object", + "required": ["artifact_name", "nodes"], + "properties": { + "schema_version": {"const": "need_graph.v1"}, + "artifact_name": {"type": "string", "minLength": 1}, + "description": {"type": "string"}, + "nodes": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/node"}}, + "edges": {"type": "array", "items": {"type": "object"}}, + "provenance": { + "type": "object", + "description": "How the graph was made: model, prompt version, draws, and the coverage spread over the draws. Open: recorded, not interpreted.", + "properties": { + "model": {"type": "string"}, + "prompt_version": {"type": "string"}, + "draws": {"type": "integer", "minimum": 1} + } + } + }, + "$defs": { + "node": { + "type": "object", + "required": ["id", "text", "path_class"], + "properties": { + "id": {"type": "string", "minLength": 1}, + "text": {"type": "string", "minLength": 1}, + "type": {"enum": ["intent", "evidence", "action", "constraint", "decision"]}, + "path_class": {"enum": ["core", "branch", "rare_critical", "supporting", "out_of_scope"]}, + "confirmed_by": {"enum": ["user", "hypothesis"]} + } + } + } +} diff --git a/dokimi/tests/data/cli_flags_allowlist.txt b/dokimi/tests/data/cli_flags_allowlist.txt index f8e037fda..2a117a739 100644 --- a/dokimi/tests/data/cli_flags_allowlist.txt +++ b/dokimi/tests/data/cli_flags_allowlist.txt @@ -222,3 +222,6 @@ --work-output-dir --work-repeats -o +--graph # eval coverage: the frozen need graph input, a file no registry concept names yet (#331) +--min-coverage # eval coverage gate floor; joins a gate-floor concept with --min-accuracy (#331) +--min-rare-critical # eval coverage gate floor, same concept as --min-coverage (#331) diff --git a/dokimi/tests/fixtures/artifacts/eval_coverage.v1.json b/dokimi/tests/fixtures/artifacts/eval_coverage.v1.json new file mode 100644 index 000000000..78595cb0c --- /dev/null +++ b/dokimi/tests/fixtures/artifacts/eval_coverage.v1.json @@ -0,0 +1,63 @@ +{ + "schema_version": "eval_coverage.v1", + "evals_root": "skill", + "graph": { + "artifact_name": "dpnp-io", + "digest": "502b794faa561af3c1910c221a2d25db8db77563ec9fde3a05127121a8f12d67", + "nodes": 3, + "in_scope": 2, + "hypotheses": 2 + }, + "coverage": { + "weighted": 0.5, + "core_rare_critical": 0.5, + "rare_critical": 0.0, + "by_class": { + "core": { + "covered": 1, + "total": 1 + }, + "rare_critical": { + "covered": 0, + "total": 1 + }, + "branch": { + "covered": 0, + "total": 0 + }, + "supporting": { + "covered": 0, + "total": 0 + }, + "out_of_scope": { + "covered": 0, + "total": 1 + } + } + }, + "gaps": [ + { + "id": "n2", + "path_class": "rare_critical", + "text": "keep allow_pickle=False for untrusted files", + "confirmed_by": "hypothesis" + } + ], + "evals": [ + { + "id": "save", + "source": "evals.json", + "needs": [ + "n1" + ], + "relevance": "relevant_core" + }, + { + "id": "css", + "source": "evals.json", + "needs": [], + "relevance": "orphan" + } + ], + "problems": [] +} diff --git a/dokimi/tests/fixtures/artifacts/need_graph.v1.json b/dokimi/tests/fixtures/artifacts/need_graph.v1.json new file mode 100644 index 000000000..be765fdce --- /dev/null +++ b/dokimi/tests/fixtures/artifacts/need_graph.v1.json @@ -0,0 +1,38 @@ +{ + "schema_version": "need_graph.v1", + "artifact_name": "dpnp-io", + "description": "Save and load dpnp arrays.", + "nodes": [ + { + "id": "n1", + "text": "convert with dpnp.asnumpy before numpy.save", + "type": "action", + "path_class": "core", + "confirmed_by": "user" + }, + { + "id": "n2", + "text": "keep allow_pickle=False for untrusted files", + "type": "constraint", + "path_class": "rare_critical" + }, + { + "id": "n3", + "text": "plot the loaded array", + "type": "intent", + "path_class": "out_of_scope" + } + ], + "edges": [ + { + "from": "n2", + "to": "n1", + "kind": "requires" + } + ], + "provenance": { + "model": "claude-sonnet-5-5", + "prompt_version": "needs-v1", + "draws": 3 + } +} diff --git a/dokimi/tests/outcomes/journeys.yaml b/dokimi/tests/outcomes/journeys.yaml index 1a435da4a..14ce6780c 100644 --- a/dokimi/tests/outcomes/journeys.yaml +++ b/dokimi/tests/outcomes/journeys.yaml @@ -314,7 +314,14 @@ criteria: AC-J07-006: delivery: proposed AC-J07-007: - delivery: proposed + delivery: partial + enforce: true + capability: eval-coverage + surfaces: ["eval coverage --graph"] + evidence: + - {id: cjm-eval-coverage-gaps, kind: unit, stubs: []} + gaps: + - "the need graph is built by a model and read as given: building, interviewing and drafting labels are not commands yet (#331)" # ---------------------------------------------------------------- J11 AC-J11-001: diff --git a/dokimi/tests/test_eval_coverage.py b/dokimi/tests/test_eval_coverage.py new file mode 100644 index 000000000..3bad386ff --- /dev/null +++ b/dokimi/tests/test_eval_coverage.py @@ -0,0 +1,188 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""`eval coverage`: coverage and relevance of an eval set against a frozen need graph (#331).""" + +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from dokimi.artifacts import ArtifactValidationError, load_artifact +from dokimi.cli import build_parser +from dokimi.eval_coverage import LabelledEval, coverage_report, graph_digest, load_graph, read_labels +from dokimi.novelty.needs import NeedGraph, NeedNode, save_graph + +NODES = [ + {"id": "n1", "text": "convert with asnumpy before numpy.save", "path_class": "core"}, + {"id": "n2", "text": "memory-map files larger than RAM", "path_class": "core", + "confirmed_by": "user"}, + {"id": "n3", "text": "keep allow_pickle=False for untrusted files", "path_class": "rare_critical"}, + {"id": "n4", "text": "choose HDF5 for chunked data", "path_class": "branch"}, + {"id": "n5", "text": "plot the array", "path_class": "out_of_scope"}, +] + + +def _graph(tmp_path: Path) -> Path: + path = tmp_path / "need_graph.json" + path.write_text(json.dumps({"artifact_name": "dpnp-io", "nodes": NODES})) + return path + + +def _skill(tmp_path: Path, rows: list[dict]) -> Path: + skill = tmp_path / "skill" + (skill / "evals").mkdir(parents=True) + (skill / "evals" / "evals.json").write_text(json.dumps({"skill_name": "dpnp-io", "evals": rows})) + return skill + + +ROWS = [ + {"id": "save", "prompt": "save a dpnp array", "needs": ["n1"]}, + {"id": "big-file", "prompt": "50 GB file", "needs": ["n2", "n4"]}, + {"id": "plot", "prompt": "plot it", "needs": ["n5"]}, + {"id": "css", "prompt": "fix my css", "needs": []}, + {"id": "new", "prompt": "not mapped yet"}, +] + + +def _run(capsys, *argv: str) -> SimpleNamespace: + args = build_parser().parse_args(["eval", "coverage", *argv]) + code = 0 + try: + args.func(args) + except SystemExit as exc: + code = int(exc.code or 0) + out = capsys.readouterr() + return SimpleNamespace(returncode=code, stdout=out.out, stderr=out.err) + + +@pytest.mark.outcome("cjm-eval-coverage-gaps") +def test_report_lists_uncovered_critical_nodes_and_classifies_each_eval(tmp_path): + report = coverage_report(load_graph(_graph(tmp_path)), read_labels(_skill(tmp_path, ROWS)), "skill") + + assert [g["id"] for g in report["gaps"]] == ["n3"] + assert report["gaps"][0]["confirmed_by"] == "hypothesis" + assert {e["id"]: e["relevance"] for e in report["evals"]} == { + "save": "relevant_core", "big-file": "relevant_core", "plot": "out_of_scope", + "css": "orphan", "new": "unlabelled", + } + # in scope: n1 1.0 + n2 1.0 + n3 1.0 + n4 0.7 = 3.7; covered n1, n2, n4 = 2.7 + assert report["coverage"]["weighted"] == pytest.approx(2.7 / 3.7, abs=1e-4) + assert report["coverage"]["core_rare_critical"] == pytest.approx(2 / 3, abs=1e-4) + assert report["coverage"]["rare_critical"] == 0.0 + assert (report["graph"]["hypotheses"], report["graph"]["in_scope"]) == (4, 4) + + +def test_peripheral_eval_and_task_metadata_needs(tmp_path): + tasks = tmp_path / "tasks" + task = tasks / "hdf5-write" + task.mkdir(parents=True) + (task / "task.toml").write_text('[metadata]\nneeds = "n4"\n') + (task / "instruction.md").write_text("write chunks") + + labels = read_labels(tasks) + report = coverage_report(load_graph(_graph(tmp_path)), labels, "tasks") + + assert [(e.id, e.source, e.needs) for e in labels] == [("hdf5-write", "task", ("n4",))] + assert report["evals"][0]["relevance"] == "relevant_peripheral" + assert report["coverage"]["by_class"]["branch"] == {"covered": 1, "total": 1} + + +def test_same_files_same_report_and_digest_ignores_node_order(tmp_path): + graph, skill = _graph(tmp_path), _skill(tmp_path, ROWS) + first = coverage_report(load_graph(graph), read_labels(skill), "skill") + second = coverage_report(load_graph(graph), read_labels(skill), "skill") + reordered = {"artifact_name": "dpnp-io", "nodes": list(reversed(NODES))} + + assert json.dumps(first, sort_keys=True) == json.dumps(second, sort_keys=True) + assert graph_digest(reordered) == first["graph"]["digest"] + assert graph_digest({"nodes": NODES[:4]}) != first["graph"]["digest"] + + +def test_novelty_save_graph_output_loads_as_need_graph(tmp_path): + path = tmp_path / "g.json" + save_graph(NeedGraph("x", "d", nodes=[NeedNode("n1", "a need", path_class="core")]), path) + + assert load_graph(path)["nodes"][0]["path_class"] == "core" + + +def test_graph_with_unknown_path_class_is_rejected(tmp_path): + path = tmp_path / "g.json" + path.write_text(json.dumps({"artifact_name": "x", "nodes": [ + {"id": "n1", "text": "t", "path_class": "critical"}]})) + + with pytest.raises(ArtifactValidationError, match="path_class"): + load_graph(path) + + +def test_cli_writes_artifact_and_fails_under_floor(tmp_path, capsys): + out = tmp_path / "cov.json" + graph, skill = _graph(tmp_path), _skill(tmp_path, ROWS) + + ok = _run(capsys, str(skill), "--graph", str(graph), "--out", str(out)) + low = _run(capsys, str(skill), "--graph", str(graph), "--min-rare-critical", "0.5") + + assert ok.returncode == 0, ok.stderr + assert "n3 [rare_critical, hypothesis]" in ok.stdout + assert load_artifact("eval_coverage", out)["gaps"][0]["id"] == "n3" + assert low.returncode == 1 + assert "rare_critical coverage 0.00 < 0.50" in low.stderr + + +def test_cli_unknown_need_id_and_bad_inputs_exit_2(tmp_path, capsys): + graph = _graph(tmp_path) + typo = _skill(tmp_path, [{"id": "save", "needs": ["n9"]}]) + broken = tmp_path / "evals.json" + broken.write_text("{not json") + + unknown = _run(capsys, str(typo), "--graph", str(graph), "--format", "json") + bad = _run(capsys, str(broken), "--graph", str(graph)) + missing = _run(capsys, str(tmp_path / "nope"), "--graph", str(graph)) + no_graph = _run(capsys, str(typo), "--graph", str(tmp_path / "none.json")) + + assert unknown.returncode == 2 + assert json.loads(unknown.stdout)["problems"] == ["save: unknown need id 'n9'"] + assert (bad.returncode, missing.returncode, no_graph.returncode) == (2, 2, 2) + assert "not a skill-creator evals.json" in bad.stderr + + +def test_cli_rejects_a_floor_outside_0_1(capsys): + for value in ("1.5", "high"): + with pytest.raises(SystemExit) as exc: + build_parser().parse_args(["eval", "coverage", "x", "--graph", "g", "--min-coverage", value]) + assert exc.value.code == 2 + assert "must lie in 0-1, got 1.5" in capsys.readouterr().err + + +def test_markdown_lists_problems_and_says_none_without_gaps(tmp_path, capsys): + rows = [{"id": "all", "needs": ["n1", "n2", "n3", "n9"]}] + res = _run(capsys, str(_skill(tmp_path, rows)), "--graph", str(_graph(tmp_path))) + + assert res.returncode == 2 + assert "- none" in res.stdout + assert "Problems:\n- all: unknown need id 'n9'" in res.stdout + + +def test_unparsable_task_is_skipped_and_rows_that_are_not_objects_ignored(tmp_path): + root = tmp_path / "suite" + bad = root / "broken-task" + bad.mkdir(parents=True) + (bad / "task.toml").write_text("[metadata\n") + (bad / "instruction.md").write_text("x") + (root / "evals.json").write_text(json.dumps({"evals": ["text row", {"needs": "n1"}]})) + + assert read_labels(root) == [LabelledEval("1", "evals.json", ("n1",))] + + +def test_graph_with_only_out_of_scope_nodes_has_no_coverage_figures(tmp_path): + graph = {"artifact_name": "x", "nodes": [{"id": "n1", "text": "t", "path_class": "out_of_scope"}]} + report = coverage_report(graph, [LabelledEval("e", "task", None)], "r") + + assert report["coverage"] == { + "weighted": None, "core_rare_critical": None, "rare_critical": None, + "by_class": {c: {"covered": 0, "total": int(c == "out_of_scope")} + for c in ("core", "rare_critical", "branch", "supporting", "out_of_scope")}, + } diff --git a/dokimi/tests/test_import_architecture.py b/dokimi/tests/test_import_architecture.py index 6d1025d78..969eef92c 100644 --- a/dokimi/tests/test_import_architecture.py +++ b/dokimi/tests/test_import_architecture.py @@ -49,7 +49,7 @@ # repositories, subjects, reports, dashboards, the one-command pipeline. ("tracks", ( "dokimi.config_check", "dokimi.context", "dokimi.coverage", "dokimi.dashboard", - "dokimi.datasets_export", "dokimi.explore", "dokimi.external", "dokimi.harnesses", + "dokimi.datasets_export", "dokimi.eval_coverage", "dokimi.explore", "dokimi.external", "dokimi.harnesses", "dokimi.novelty", "dokimi.orchestrator", "dokimi.report", "dokimi.repos", "dokimi.runner", "dokimi.runs", "dokimi.schema_check", "dokimi.schema_compat", "dokimi.subjects", "dokimi.task_formats", "dokimi.tool_preference", diff --git a/dokimi/tests/test_runs_scan.py b/dokimi/tests/test_runs_scan.py index 9b5b9997d..b5bd54ee2 100644 --- a/dokimi/tests/test_runs_scan.py +++ b/dokimi/tests/test_runs_scan.py @@ -48,6 +48,8 @@ "repo_budget.v1": ({}, "toy", None, None), "task_inspect.v1": ({}, "tests/fixtures/task_formats", None, None), "cli_routes.v1": ({}, None, None, None), + "need_graph.v1": ({}, "dpnp-io", None, None), + "eval_coverage.v1": ({}, "dpnp-io", None, None), } diff --git a/dokimi/tests/test_runs_show.py b/dokimi/tests/test_runs_show.py index 07ad7a9f5..de76bb8d5 100644 --- a/dokimi/tests/test_runs_show.py +++ b/dokimi/tests/test_runs_show.py @@ -37,6 +37,8 @@ "result.v1": {"cells": 6}, "subject_scorecard.v2": {"cells": 0}, "cli_routes.v1": {"roots": 2, "leaves": 2, "tombstones": 1}, + "eval_coverage.v1": {"gaps": 1, "evals": 2, "problems": 0}, + "need_graph.v1": {"nodes": 3, "edges": 1}, "task_inspect.v1": {"items": 4}, "task_runs.v1": {"tasks": 6, "harnesses": 2, "results": 24}, "tool_preference.v1": {"items": 16, "results": 16}, From af6e66f7667446ab12d566bb1fd5ea6b05541b48 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 20:27:44 -0700 Subject: [PATCH 2/9] fix(eval): coverage review pass 1: one need-graph format, --min [METRIC=]X, strict needs Novelty's NeedNode keeps confirmed_by and NeedGraph keeps provenance, so both readers accept the same need_graph.v1 file. Duplicate node ids, a needs value that is not a list of ids, an unreadable task, a non-list evals and an empty suite exit 2. Problems are structured; the report records the weights; the digest covers only identity fields. Signed-off-by: Nikolay Petrov --- .../20261031_180000_napetrov_eval-coverage.md | 5 +- dokimi/docs/reference/cli-interface.json | 41 +------- dokimi/dokimi/commands/eval_coverage.py | 53 ++++++----- dokimi/dokimi/eval_coverage.py | 68 +++++++++---- dokimi/dokimi/novelty/needs.py | 12 ++- dokimi/dokimi/schemas/eval_coverage.v1.json | 21 +++- dokimi/tests/data/cli_flags_allowlist.txt | 3 +- .../fixtures/artifacts/eval_coverage.v1.json | 7 ++ dokimi/tests/test_eval_coverage.py | 95 +++++++++++++++---- 9 files changed, 197 insertions(+), 108 deletions(-) diff --git a/dokimi/changelog.d/20261031_180000_napetrov_eval-coverage.md b/dokimi/changelog.d/20261031_180000_napetrov_eval-coverage.md index 815f6c0a8..ad3c2d9f6 100644 --- a/dokimi/changelog.d/20261031_180000_napetrov_eval-coverage.md +++ b/dokimi/changelog.d/20261031_180000_napetrov_eval-coverage.md @@ -12,6 +12,7 @@ SPDX-License-Identifier: Apache-2.0 prints the weighted coverage, the coverage of `core` + `rare_critical` and of `rare_critical` alone, the uncovered `core` / `rare_critical` nodes, and each eval's relevance (`relevant_core`, `relevant_peripheral`, `out_of_scope`, `orphan`, `unlabelled`). `--out` - writes an `eval_coverage.v1` artifact. Exit 1 under `--min-coverage` / `--min-rare-critical`, - exit 2 on an unreadable input or a `needs` id the graph lacks. Building the graph, the user + writes an `eval_coverage.v1` artifact. Exit 1 under a `--min [METRIC=]X` floor, exit 2 on an + unreadable input, a graph with duplicate node ids, or a `needs` label that is not a list of + ids the graph has. Novelty's need graph now keeps `confirmed_by` and `provenance`. Building the graph, the user interview and drafting labels follow in #331. diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index 5e8c80e4f..51d1c1e38 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -4036,25 +4036,6 @@ "required": true, "type": null }, - { - "action": "_StoreAction", - "choices": [ - "'json'", - "'markdown'" - ], - "const": null, - "default": "markdown", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--format", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, { "action": "_StoreAction", "choices": null, @@ -4088,23 +4069,7 @@ "type": null }, { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--min-coverage", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.commands.eval_coverage._floor#e9bf1357f2fa" - }, - { - "action": "_StoreAction", + "action": "_AppendAction", "choices": null, "const": null, "default": null, @@ -4112,12 +4077,12 @@ "group_required": null, "hidden": false, "is_flag": false, - "key": "--min-rare-critical", + "key": "--min", "kind": "option", "nargs": null, "position": null, "required": false, - "type": "dokimi.commands.eval_coverage._floor#e9bf1357f2fa" + "type": "dokimi.commands.eval_coverage._floor#bbd5737352a9" }, { "action": "_StoreAction", diff --git a/dokimi/dokimi/commands/eval_coverage.py b/dokimi/dokimi/commands/eval_coverage.py index 58845b9f3..d33607d50 100644 --- a/dokimi/dokimi/commands/eval_coverage.py +++ b/dokimi/dokimi/commands/eval_coverage.py @@ -6,35 +6,37 @@ from __future__ import annotations import argparse -import json import sys from pathlib import Path from dokimi.exit_codes import EXIT_FAILURE, EXIT_USAGE -def _floor(value: str) -> float: +METRICS = ("weighted", "core_rare_critical", "rare_critical") + + +def _floor(value: str) -> tuple[str, float]: + metric, sep, number = value.rpartition("=") + metric = metric if sep else "weighted" + if metric not in METRICS: + raise argparse.ArgumentTypeError(f"metric must be one of {', '.join(METRICS)}, got {metric!r}") try: - number = float(value) + floor = float(number) except ValueError: - raise argparse.ArgumentTypeError(f"expected a number, got {value!r}") from None - if not 0.0 <= number <= 1.0: - raise argparse.ArgumentTypeError(f"must lie in 0-1, got {number}") - return number + raise argparse.ArgumentTypeError(f"expected a number, got {number!r}") from None + if not 0.0 <= floor <= 1.0: + raise argparse.ArgumentTypeError(f"must lie in 0-1, got {floor}") + return metric, floor -def _below(report: dict, args: argparse.Namespace) -> list[str]: +def _below(report: dict, floors: list[tuple[str, float]]) -> list[str]: cov = report["coverage"] - failed = [] - for name, floor, value in (("weighted", args.min_coverage, cov["weighted"]), - ("rare_critical", args.min_rare_critical, cov["rare_critical"])): - if floor is not None and value is not None and value < floor: - failed.append(f"{name} coverage {value:.2f} < {floor:.2f}") - return failed + return [f"{metric} coverage {cov[metric]:.2f} < {floor:.2f}" for metric, floor in floors + if cov[metric] is not None and cov[metric] < floor] def cmd_eval_coverage(args: argparse.Namespace) -> None: - """Print the report; exit 2 on an unreadable input or unknown need ids, 1 under a floor.""" + """Print the report; exit 2 on an unreadable input or a bad ``needs`` label, 1 under a floor.""" from dokimi.artifacts import ArtifactValidationError, save_artifact from dokimi.eval_coverage import coverage_report, load_graph, read_labels, render_markdown @@ -48,15 +50,21 @@ def cmd_eval_coverage(args: argparse.Namespace) -> None: except (OSError, ValueError, ArtifactValidationError) as exc: print(f"Error: {exc}", file=sys.stderr) sys.exit(EXIT_USAGE) + if not evals: + print(f"Error: no evals.json rows or tasks under {root}", file=sys.stderr) + sys.exit(EXIT_USAGE) + if args.out and Path(args.out).resolve() == Path(args.graph).resolve(): + print("Error: --out would overwrite --graph", file=sys.stderr) + sys.exit(EXIT_USAGE) report = coverage_report(graph, evals, str(root)) if args.out: report = save_artifact("eval_coverage", report, Path(args.out)) - print(json.dumps(report, indent=2) if args.format == "json" else render_markdown(report)) + print(render_markdown(report)) if report["problems"]: - print(f"Error: {len(report['problems'])} eval(s) name need ids the graph does not have", + print(f"Error: {len(report['problems'])} problem(s) with the evals' needs labels", file=sys.stderr) sys.exit(EXIT_USAGE) - failed = _below(report, args) + failed = _below(report, args.min or []) if failed: print("FAIL: " + "; ".join(failed), file=sys.stderr) sys.exit(EXIT_FAILURE) @@ -72,10 +80,7 @@ def register(eval_sub) -> None: p.add_argument("--graph", required=True, help="need_graph.v1 file (a novelty need graph is valid as written)") p.add_argument("--out", default=None, help="Write the eval_coverage.v1 artifact here") - p.add_argument("--format", choices=["markdown", "json"], default="markdown", - help="What stdout shows (default: markdown)") - p.add_argument("--min-coverage", type=_floor, default=None, dest="min_coverage", - help="Exit 1 when weighted coverage is below this (0-1)") - p.add_argument("--min-rare-critical", type=_floor, default=None, dest="min_rare_critical", - help="Exit 1 when rare_critical coverage is below this (0-1)") + p.add_argument("--min", type=_floor, action="append", metavar="[METRIC=]X", + help="Exit 1 when a coverage figure is below X (0-1); METRIC is weighted " + "(default), core_rare_critical or rare_critical. Repeatable") p.set_defaults(func=cmd_eval_coverage) diff --git a/dokimi/dokimi/eval_coverage.py b/dokimi/dokimi/eval_coverage.py index 8e5951c9a..e34e8ef03 100644 --- a/dokimi/dokimi/eval_coverage.py +++ b/dokimi/dokimi/eval_coverage.py @@ -8,7 +8,7 @@ written once and kept next to the suite. Each eval names the nodes it grades in ``needs``: * a row of a skill-creator ``evals.json``: ``{"id": ..., "needs": ["n1", "n4"]}``; -* a task's ``task.toml`` / ``task.md``: ``metadata.needs``. +* a task's ``task.toml`` / ``task.md``: ``metadata.needs`` (a list; a bare string is refused). :func:`coverage_report` is a pure function of the graph and those labels, so the same files always give the same report. No model is called here; drafting labels is a separate, paid step. @@ -42,26 +42,41 @@ class LabelledEval: id: str source: str needs: tuple[str, ...] | None + malformed: bool = False # ``needs`` is there but is not a list of strings def load_graph(path: Path) -> dict: - """Read and validate a ``need_graph.v1`` file (an unstamped ``save_graph`` file is valid).""" - return load_artifact("need_graph", Path(path)) + """Read and validate a ``need_graph.v1`` file (an unstamped ``save_graph`` file is valid). + + Raises ``ValueError`` when two nodes share an id: ids are what the ``needs`` labels join on. + """ + graph = load_artifact("need_graph", Path(path)) + ids = [n["id"] for n in graph["nodes"]] + dupes = sorted({i for i in ids if ids.count(i) > 1}) + if dupes: + raise ValueError(f"{path}: duplicate node ids {dupes}") + return graph def graph_digest(graph: dict) -> str: - """sha256 of the nodes in canonical JSON: the identity a report and a plan can record.""" - nodes = sorted(graph["nodes"], key=lambda n: n["id"]) + """sha256 of each node's id, text, type, path_class and confirmed_by, defaults filled in. + + Other keys and node order do not change it; a confirmation does. + """ + nodes = sorted(({"id": n["id"], "text": n["text"], "type": n.get("type", "intent"), + "path_class": n["path_class"], + "confirmed_by": n.get("confirmed_by", "hypothesis")} + for n in graph["nodes"]), key=lambda n: n["id"]) blob = json.dumps(nodes, sort_keys=True, separators=(",", ":"), ensure_ascii=False) return hashlib.sha256(blob.encode("utf-8")).hexdigest() -def _needs(value: Any) -> tuple[str, ...] | None: +def _labelled(eid: str, source: str, value: Any) -> LabelledEval: if value is None: - return None - if isinstance(value, str): # one id, not a sequence of letters - return (value,) - return tuple(str(v) for v in value) if isinstance(value, list) else None + return LabelledEval(eid, source, None) + if isinstance(value, list) and all(isinstance(v, str) for v in value): + return LabelledEval(eid, source, tuple(value)) + return LabelledEval(eid, source, None, malformed=True) def _evals_files(root: Path) -> list[Path]: @@ -74,7 +89,8 @@ def read_labels(root: Path) -> list[LabelledEval]: """Every eval under ``root`` with its ``needs``: skill-creator ``evals.json`` rows, then tasks. ``root`` is an ``evals.json`` file, a skill directory (its ``evals/evals.json``) or a tasks - root whose child directories are tasks. Raises ``ValueError`` on an unreadable ``evals.json``. + root whose child directories are tasks. Raises ``ValueError`` on an unreadable ``evals.json`` + or task. """ out: list[LabelledEval] = [] for path in _evals_files(root): @@ -82,18 +98,19 @@ def read_labels(root: Path) -> list[LabelledEval]: rows = json.loads(path.read_text(encoding="utf-8")).get("evals", []) except (OSError, ValueError, AttributeError) as exc: raise ValueError(f"{path}: not a skill-creator evals.json ({exc})") from exc - for index, row in enumerate(rows if isinstance(rows, list) else []): + if not isinstance(rows, list): + raise ValueError(f"{path}: \"evals\" is not a list") + for index, row in enumerate(rows): if isinstance(row, dict): - eid = str(row.get("id", index)) - out.append(LabelledEval(eid, path.name, _needs(row.get("needs")))) + out.append(_labelled(str(row.get("id", index)), path.name, row.get("needs"))) if root.is_dir(): for task_dir in sorted(p for p in root.iterdir() if is_task_dir(p)): try: meta = read_task_config(task_dir).get("metadata", {}) - except (ValueError, OSError): - continue - needs = _needs(meta.get("needs")) if isinstance(meta, dict) else None - out.append(LabelledEval(task_dir.name, "task", needs)) + except (ValueError, OSError) as exc: + raise ValueError(f"{task_dir}: unreadable task ({exc})") from exc + needs = meta.get("needs") if isinstance(meta, dict) else None + out.append(_labelled(task_dir.name, "task", needs)) return out @@ -117,8 +134,10 @@ def _share(nodes: list[dict], covered: set[str]) -> float | None: def coverage_report(graph: dict, evals: list[LabelledEval], evals_root: str) -> dict: """The ``eval_coverage.v1`` payload (unstamped) for ``graph`` and ``evals``.""" classes = {n["id"]: n["path_class"] for n in graph["nodes"]} - problems = [f"{e.id}: unknown need id {n!r}" - for e in evals for n in (e.needs or ()) if n not in classes] + problems = [{"eval": e.id, "source": e.source, "kind": "malformed_needs", "need": None} + for e in evals if e.malformed] + problems += [{"eval": e.id, "source": e.source, "kind": "unknown_need", "need": n} + for e in evals for n in (e.needs or ()) if n not in classes] covered = {n for e in evals for n in (e.needs or ()) if n in classes} scope = [n for n in graph["nodes"] if n["path_class"] != "out_of_scope"] weight = sum(PATH_WEIGHTS[n["path_class"]] for n in scope) @@ -134,6 +153,7 @@ def coverage_report(graph: dict, evals: list[LabelledEval], evals_root: str) -> for n in graph["nodes"]), }, "coverage": { + "weights": {c: PATH_WEIGHTS[c] for c in CLASSES}, "weighted": round(sum(PATH_WEIGHTS[n["path_class"]] for n in scope if n["id"] in covered) / weight, 4) if weight else None, "core_rare_critical": _share(critical, covered), @@ -156,6 +176,12 @@ def _pct(value: float | None) -> str: return "n/a" if value is None else f"{value:.0%}" +def _problem(p: dict) -> str: + what = "needs is not a list of node ids" if p["kind"] == "malformed_needs" \ + else f"unknown need id {p['need']!r}" + return f"- {p['eval']} ({p['source']}): {what}" + + def render_markdown(report: dict) -> str: """The report as Markdown, for a suite's README or the PR that changes the suite.""" g, c = report["graph"], report["coverage"] @@ -175,5 +201,5 @@ def render_markdown(report: dict) -> str: lines += [f"- {n['id']} [{n['path_class']}, {n['confirmed_by']}] {n['text']}" for n in report["gaps"]] or ["- none"] if report["problems"]: - lines += ["", "Problems:"] + [f"- {p}" for p in report["problems"]] + lines += ["", "Problems:"] + [_problem(p) for p in report["problems"]] return "\n".join(lines) + "\n" diff --git a/dokimi/dokimi/novelty/needs.py b/dokimi/dokimi/novelty/needs.py index 5e7918c09..9c3e0e9ab 100644 --- a/dokimi/dokimi/novelty/needs.py +++ b/dokimi/dokimi/novelty/needs.py @@ -81,6 +81,7 @@ class NeedNode: text: str type: str = "intent" path_class: str = "core" + confirmed_by: str = "hypothesis" # "user" once a person confirmed it (need_graph.v1) @property def weight(self) -> float: @@ -90,7 +91,10 @@ def weight(self) -> float: def as_dict(self) -> dict: """Serializable form.""" return {"id": self.id, "text": self.text, "type": self.type, - "path_class": self.path_class} + "path_class": self.path_class, "confirmed_by": self.confirmed_by} + + +_NODE_FIELDS = frozenset(NeedNode.__dataclass_fields__) @dataclass @@ -101,6 +105,7 @@ class NeedGraph: description: str nodes: list[NeedNode] = field(default_factory=list) edges: list[dict] = field(default_factory=list) + provenance: dict = field(default_factory=dict) def task_statement(self) -> str: """A task description for the value judge, built without the artifact body.""" @@ -126,6 +131,7 @@ def as_dict(self) -> dict: "description": self.description, "nodes": [n.as_dict() for n in self.nodes], "edges": list(self.edges), + **({"provenance": dict(self.provenance)} if self.provenance else {}), } @classmethod @@ -134,8 +140,10 @@ def from_dict(cls, data: dict) -> "NeedGraph": return cls( artifact_name=data.get("artifact_name", ""), description=data.get("description", ""), - nodes=[NeedNode(**row) for row in data.get("nodes", [])], + nodes=[NeedNode(**{k: v for k, v in row.items() if k in _NODE_FIELDS}) + for row in data.get("nodes", [])], edges=list(data.get("edges", [])), + provenance=dict(data.get("provenance") or {}), ) diff --git a/dokimi/dokimi/schemas/eval_coverage.v1.json b/dokimi/dokimi/schemas/eval_coverage.v1.json index f199dd765..c3e05218a 100644 --- a/dokimi/dokimi/schemas/eval_coverage.v1.json +++ b/dokimi/dokimi/schemas/eval_coverage.v1.json @@ -15,7 +15,7 @@ "additionalProperties": false, "properties": { "artifact_name": {"type": "string"}, - "digest": {"type": "string", "description": "sha256 of the graph's nodes, canonical JSON"}, + "digest": {"type": "string", "description": "sha256 of each node's id, text, type, path_class and confirmed_by (defaults filled in), canonical JSON"}, "nodes": {"type": "integer", "minimum": 0}, "in_scope": {"type": "integer", "minimum": 0}, "hypotheses": {"type": "integer", "minimum": 0, "description": "Nodes no person confirmed"} @@ -23,9 +23,11 @@ }, "coverage": { "type": "object", - "required": ["weighted", "core_rare_critical", "rare_critical", "by_class"], + "required": ["weights", "weighted", "core_rare_critical", "rare_critical", "by_class"], "additionalProperties": false, "properties": { + "weights": {"type": "object", "description": "Weight per path class used for `weighted`", + "additionalProperties": {"type": "number"}}, "weighted": {"type": ["number", "null"]}, "core_rare_critical": {"type": ["number", "null"]}, "rare_critical": {"type": ["number", "null"]}, @@ -69,6 +71,19 @@ } } }, - "problems": {"type": "array", "items": {"type": "string"}} + "problems": { + "type": "array", + "items": { + "type": "object", + "required": ["eval", "source", "kind", "need"], + "additionalProperties": false, + "properties": { + "eval": {"type": "string"}, + "source": {"type": "string"}, + "kind": {"enum": ["unknown_need", "malformed_needs"]}, + "need": {"type": ["string", "null"]} + } + } + } } } diff --git a/dokimi/tests/data/cli_flags_allowlist.txt b/dokimi/tests/data/cli_flags_allowlist.txt index 2a117a739..9b398b12e 100644 --- a/dokimi/tests/data/cli_flags_allowlist.txt +++ b/dokimi/tests/data/cli_flags_allowlist.txt @@ -223,5 +223,4 @@ --work-repeats -o --graph # eval coverage: the frozen need graph input, a file no registry concept names yet (#331) ---min-coverage # eval coverage gate floor; joins a gate-floor concept with --min-accuracy (#331) ---min-rare-critical # eval coverage gate floor, same concept as --min-coverage (#331) +--min # eval coverage gate floor, [METRIC=]X like --timeout [PHASE=]SEC (#331) diff --git a/dokimi/tests/fixtures/artifacts/eval_coverage.v1.json b/dokimi/tests/fixtures/artifacts/eval_coverage.v1.json index 78595cb0c..52b788f7b 100644 --- a/dokimi/tests/fixtures/artifacts/eval_coverage.v1.json +++ b/dokimi/tests/fixtures/artifacts/eval_coverage.v1.json @@ -9,6 +9,13 @@ "hypotheses": 2 }, "coverage": { + "weights": { + "core": 1.0, + "rare_critical": 1.0, + "branch": 0.7, + "supporting": 0.4, + "out_of_scope": 0.0 + }, "weighted": 0.5, "core_rare_critical": 0.5, "rare_critical": 0.0, diff --git a/dokimi/tests/test_eval_coverage.py b/dokimi/tests/test_eval_coverage.py index 3bad386ff..f5b8c121e 100644 --- a/dokimi/tests/test_eval_coverage.py +++ b/dokimi/tests/test_eval_coverage.py @@ -14,7 +14,9 @@ from dokimi.artifacts import ArtifactValidationError, load_artifact from dokimi.cli import build_parser from dokimi.eval_coverage import LabelledEval, coverage_report, graph_digest, load_graph, read_labels -from dokimi.novelty.needs import NeedGraph, NeedNode, save_graph +from dokimi.novelty.needs import NeedGraph, NeedNode +from dokimi.novelty.needs import load_graph as novelty_load_graph +from dokimi.novelty.needs import save_graph NODES = [ {"id": "n1", "text": "convert with asnumpy before numpy.save", "path_class": "core"}, @@ -80,7 +82,7 @@ def test_peripheral_eval_and_task_metadata_needs(tmp_path): tasks = tmp_path / "tasks" task = tasks / "hdf5-write" task.mkdir(parents=True) - (task / "task.toml").write_text('[metadata]\nneeds = "n4"\n') + (task / "task.toml").write_text('[metadata]\nneeds = ["n4"]\n') (task / "instruction.md").write_text("write chunks") labels = read_labels(tasks) @@ -100,13 +102,39 @@ def test_same_files_same_report_and_digest_ignores_node_order(tmp_path): assert json.dumps(first, sort_keys=True) == json.dumps(second, sort_keys=True) assert graph_digest(reordered) == first["graph"]["digest"] assert graph_digest({"nodes": NODES[:4]}) != first["graph"]["digest"] + defaults = [{**n, "type": "intent", "confirmed_by": n.get("confirmed_by", "hypothesis"), + "note": "x"} for n in NODES] + assert graph_digest({"nodes": defaults}) == first["graph"]["digest"] + confirmed = [{**n, "confirmed_by": "user"} for n in NODES] + assert graph_digest({"nodes": confirmed}) != first["graph"]["digest"] def test_novelty_save_graph_output_loads_as_need_graph(tmp_path): path = tmp_path / "g.json" - save_graph(NeedGraph("x", "d", nodes=[NeedNode("n1", "a need", path_class="core")]), path) + node = NeedNode("n1", "a need", path_class="core", confirmed_by="user") + save_graph(NeedGraph("x", "d", nodes=[node], provenance={"draws": 3}), path) - assert load_graph(path)["nodes"][0]["path_class"] == "core" + assert load_graph(path)["nodes"][0]["confirmed_by"] == "user" + assert novelty_load_graph(path).as_dict() == json.loads(path.read_text()) + + +def test_both_readers_agree_on_the_fixture(): + fixture = Path(__file__).parent / "fixtures" / "artifacts" / "need_graph.v1.json" + novelty = novelty_load_graph(fixture) + + assert [(n.id, n.path_class, n.confirmed_by) for n in novelty.nodes] == [ + (n["id"], n["path_class"], n.get("confirmed_by", "hypothesis")) + for n in load_graph(fixture)["nodes"]] + + +def test_duplicate_node_ids_are_rejected(tmp_path): + path = tmp_path / "g.json" + path.write_text(json.dumps({"artifact_name": "x", "nodes": [ + {"id": "n1", "text": "a", "path_class": "core"}, + {"id": "n1", "text": "b", "path_class": "out_of_scope"}]})) + + with pytest.raises(ValueError, match="duplicate node ids"): + load_graph(path) def test_graph_with_unknown_path_class_is_rejected(tmp_path): @@ -123,13 +151,15 @@ def test_cli_writes_artifact_and_fails_under_floor(tmp_path, capsys): graph, skill = _graph(tmp_path), _skill(tmp_path, ROWS) ok = _run(capsys, str(skill), "--graph", str(graph), "--out", str(out)) - low = _run(capsys, str(skill), "--graph", str(graph), "--min-rare-critical", "0.5") + low = _run(capsys, str(skill), "--graph", str(graph), "--min", "0.5", + "--min", "rare_critical=0.5") assert ok.returncode == 0, ok.stderr assert "n3 [rare_critical, hypothesis]" in ok.stdout assert load_artifact("eval_coverage", out)["gaps"][0]["id"] == "n3" assert low.returncode == 1 - assert "rare_critical coverage 0.00 < 0.50" in low.stderr + assert "FAIL: rare_critical coverage 0.00 < 0.50" in low.stderr + assert load_artifact("eval_coverage", out)["coverage"]["weights"]["branch"] == 0.7 def test_cli_unknown_need_id_and_bad_inputs_exit_2(tmp_path, capsys): @@ -138,23 +168,26 @@ def test_cli_unknown_need_id_and_bad_inputs_exit_2(tmp_path, capsys): broken = tmp_path / "evals.json" broken.write_text("{not json") - unknown = _run(capsys, str(typo), "--graph", str(graph), "--format", "json") + out = tmp_path / "cov.json" + unknown = _run(capsys, str(typo), "--graph", str(graph), "--out", str(out)) bad = _run(capsys, str(broken), "--graph", str(graph)) missing = _run(capsys, str(tmp_path / "nope"), "--graph", str(graph)) no_graph = _run(capsys, str(typo), "--graph", str(tmp_path / "none.json")) assert unknown.returncode == 2 - assert json.loads(unknown.stdout)["problems"] == ["save: unknown need id 'n9'"] + assert load_artifact("eval_coverage", out)["problems"] == [ + {"eval": "save", "source": "evals.json", "kind": "unknown_need", "need": "n9"}] assert (bad.returncode, missing.returncode, no_graph.returncode) == (2, 2, 2) assert "not a skill-creator evals.json" in bad.stderr def test_cli_rejects_a_floor_outside_0_1(capsys): - for value in ("1.5", "high"): + for value in ("1.5", "high", "recall=0.5"): with pytest.raises(SystemExit) as exc: - build_parser().parse_args(["eval", "coverage", "x", "--graph", "g", "--min-coverage", value]) + build_parser().parse_args(["eval", "coverage", "x", "--graph", "g", "--min", value]) assert exc.value.code == 2 - assert "must lie in 0-1, got 1.5" in capsys.readouterr().err + err = capsys.readouterr().err + assert "must lie in 0-1, got 1.5" in err and "got 'recall'" in err def test_markdown_lists_problems_and_says_none_without_gaps(tmp_path, capsys): @@ -163,18 +196,46 @@ def test_markdown_lists_problems_and_says_none_without_gaps(tmp_path, capsys): assert res.returncode == 2 assert "- none" in res.stdout - assert "Problems:\n- all: unknown need id 'n9'" in res.stdout + assert "Problems:\n- all (evals.json): unknown need id 'n9'" in res.stdout -def test_unparsable_task_is_skipped_and_rows_that_are_not_objects_ignored(tmp_path): +@pytest.mark.parametrize("needs", ["n1", {"n1": 1}, 3, [1]]) +def test_needs_that_is_not_a_list_of_ids_is_a_problem(tmp_path, capsys, needs): + res = _run(capsys, str(_skill(tmp_path, [{"id": "e", "needs": needs}])), "--graph", + str(_graph(tmp_path))) + + assert res.returncode == 2 + assert "- e (evals.json): needs is not a list of node ids" in res.stdout + + +def test_rows_that_are_not_objects_are_ignored_and_an_unparsable_task_refused(tmp_path): root = tmp_path / "suite" + root.mkdir() + (root / "evals.json").write_text(json.dumps({"evals": ["text row", {"needs": ["n1"]}]})) + assert read_labels(root) == [LabelledEval("1", "evals.json", ("n1",))] + bad = root / "broken-task" - bad.mkdir(parents=True) + bad.mkdir() (bad / "task.toml").write_text("[metadata\n") (bad / "instruction.md").write_text("x") - (root / "evals.json").write_text(json.dumps({"evals": ["text row", {"needs": "n1"}]})) + with pytest.raises(ValueError, match="unreadable task"): + read_labels(root) - assert read_labels(root) == [LabelledEval("1", "evals.json", ("n1",))] + +def test_cli_refuses_an_empty_suite_a_non_list_evals_and_out_over_graph(tmp_path, capsys): + graph, empty = _graph(tmp_path), tmp_path / "empty" + empty.mkdir() + (tmp_path / "evals.json").write_text(json.dumps({"evals": {"id": "x"}})) + + none = _run(capsys, str(empty), "--graph", str(graph)) + dict_rows = _run(capsys, str(tmp_path / "evals.json"), "--graph", str(graph)) + clobber = _run(capsys, str(_skill(tmp_path, ROWS)), "--graph", str(graph), "--out", str(graph)) + + assert (none.returncode, dict_rows.returncode, clobber.returncode) == (2, 2, 2) + assert "no evals.json rows or tasks" in none.stderr + assert '"evals" is not a list' in dict_rows.stderr + assert "--out would overwrite --graph" in clobber.stderr + assert json.loads(graph.read_text())["nodes"] == NODES def test_graph_with_only_out_of_scope_nodes_has_no_coverage_figures(tmp_path): @@ -182,6 +243,8 @@ def test_graph_with_only_out_of_scope_nodes_has_no_coverage_figures(tmp_path): report = coverage_report(graph, [LabelledEval("e", "task", None)], "r") assert report["coverage"] == { + "weights": {"core": 1.0, "rare_critical": 1.0, "branch": 0.7, "supporting": 0.4, + "out_of_scope": 0.0}, "weighted": None, "core_rare_critical": None, "rare_critical": None, "by_class": {c: {"covered": 0, "total": int(c == "out_of_scope")} for c in ("core", "rare_critical", "branch", "supporting", "out_of_scope")}, From 91d982b833e1499d528027fd36e50533f1c6dd05 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 22:45:52 -0700 Subject: [PATCH 3/9] fix(eval): coverage round 2: refuse --out over every input, unrounded floors, needs null is malformed --out is refused when it would overwrite the graph or any file a read consumes, including a tbench task.yaml and docker-compose.yaml. A --min floor is judged on the unrounded share, so 6 of 7 passes 0.85714 and 2 of 3 fails 0.66669. An explicit needs: null is malformed (exit 2); a missing needs key stays unlabelled. Red tests first. Signed-off-by: Nikolay Petrov --- .../20261031_180000_napetrov_eval-coverage.md | 7 +- dokimi/dokimi/commands/eval_coverage.py | 14 ++-- dokimi/dokimi/eval_coverage.py | 44 ++++++++++-- dokimi/tests/test_eval_coverage.py | 70 ++++++++++++++++++- 4 files changed, 120 insertions(+), 15 deletions(-) diff --git a/dokimi/changelog.d/20261031_180000_napetrov_eval-coverage.md b/dokimi/changelog.d/20261031_180000_napetrov_eval-coverage.md index ad3c2d9f6..d8fd19c95 100644 --- a/dokimi/changelog.d/20261031_180000_napetrov_eval-coverage.md +++ b/dokimi/changelog.d/20261031_180000_napetrov_eval-coverage.md @@ -14,5 +14,8 @@ SPDX-License-Identifier: Apache-2.0 (`relevant_core`, `relevant_peripheral`, `out_of_scope`, `orphan`, `unlabelled`). `--out` writes an `eval_coverage.v1` artifact. Exit 1 under a `--min [METRIC=]X` floor, exit 2 on an unreadable input, a graph with duplicate node ids, or a `needs` label that is not a list of - ids the graph has. Novelty's need graph now keeps `confirmed_by` and `provenance`. Building the graph, the user - interview and drafting labels follow in #331. + ids the graph has. A row or task with no `needs` key is `unlabelled`; an explicit + `"needs": null` is malformed and exits 2. `--out` is refused when it would overwrite the + graph or any input file. A `--min` floor is judged on the unrounded share, not the rounded + figure shown in the report. Novelty's need graph now keeps `confirmed_by` and `provenance`. + Building the graph, the user interview and drafting labels follow in #331. diff --git a/dokimi/dokimi/commands/eval_coverage.py b/dokimi/dokimi/commands/eval_coverage.py index d33607d50..ec8daba7d 100644 --- a/dokimi/dokimi/commands/eval_coverage.py +++ b/dokimi/dokimi/commands/eval_coverage.py @@ -30,15 +30,17 @@ def _floor(value: str) -> tuple[str, float]: def _below(report: dict, floors: list[tuple[str, float]]) -> list[str]: - cov = report["coverage"] - return [f"{metric} coverage {cov[metric]:.2f} < {floor:.2f}" for metric, floor in floors - if cov[metric] is not None and cov[metric] < floor] + from dokimi.eval_coverage import exact_shares + + exact = exact_shares(report) + return [f"{metric} coverage {exact[metric]:.6f} < {floor:.6f}" for metric, floor in floors + if exact[metric] is not None and exact[metric] < floor] def cmd_eval_coverage(args: argparse.Namespace) -> None: """Print the report; exit 2 on an unreadable input or a bad ``needs`` label, 1 under a floor.""" from dokimi.artifacts import ArtifactValidationError, save_artifact - from dokimi.eval_coverage import coverage_report, load_graph, read_labels, render_markdown + from dokimi.eval_coverage import coverage_report, input_files, load_graph, read_labels, render_markdown root = Path(args.evals) if not root.exists(): @@ -53,8 +55,8 @@ def cmd_eval_coverage(args: argparse.Namespace) -> None: if not evals: print(f"Error: no evals.json rows or tasks under {root}", file=sys.stderr) sys.exit(EXIT_USAGE) - if args.out and Path(args.out).resolve() == Path(args.graph).resolve(): - print("Error: --out would overwrite --graph", file=sys.stderr) + if args.out and Path(args.out).resolve() in {Path(args.graph).resolve(), *input_files(root)}: + print(f"Error: --out would overwrite an input file: {args.out}", file=sys.stderr) sys.exit(EXIT_USAGE) report = coverage_report(graph, evals, str(root)) if args.out: diff --git a/dokimi/dokimi/eval_coverage.py b/dokimi/dokimi/eval_coverage.py index e34e8ef03..00937febc 100644 --- a/dokimi/dokimi/eval_coverage.py +++ b/dokimi/dokimi/eval_coverage.py @@ -28,6 +28,7 @@ from typing import Any from dokimi.artifacts import load_artifact +from dokimi.harnesses import tbench_v1 from dokimi.harnesses.tasks import is_task_dir, read_task_config from dokimi.novelty.needs import PATH_WEIGHTS @@ -37,7 +38,7 @@ @dataclass(frozen=True) class LabelledEval: - """One eval and the need-node ids it says it grades (``None``: it has no ``needs``).""" + """One eval and the need-node ids it says it grades (``None``: the row or task has no ``needs`` key).""" id: str source: str @@ -71,8 +72,11 @@ def graph_digest(graph: dict) -> str: return hashlib.sha256(blob.encode("utf-8")).hexdigest() +_ABSENT = object() # no ``needs`` key at all: not the same as ``"needs": null``, which is malformed + + def _labelled(eid: str, source: str, value: Any) -> LabelledEval: - if value is None: + if value is _ABSENT: return LabelledEval(eid, source, None) if isinstance(value, list) and all(isinstance(v, str) for v in value): return LabelledEval(eid, source, tuple(value)) @@ -85,6 +89,38 @@ def _evals_files(root: Path) -> list[Path]: return [p for p in (root / "evals.json", root / "evals" / "evals.json") if p.is_file()] +# every task file a task reader may parse: Harbor / BenchFlow (task.toml, task.md), terminal-bench v1 (task.yaml, compose) +_TASK_FILES = ("task.toml", "task.md", tbench_v1.CONFIG, tbench_v1.COMPOSE) + + +def input_files(root: Path) -> list[Path]: + """The resolved files :func:`read_labels` reads under ``root``, so an output can be refused when it would + overwrite one of them.""" + files = _evals_files(root) + if root.is_dir(): + for task_dir in (p for p in root.iterdir() if is_task_dir(p)): + files += [task_dir / name for name in _TASK_FILES if (task_dir / name).is_file()] + return [p.resolve() for p in files] + + +def exact_shares(report: dict) -> dict[str, float | None]: + """The three coverage figures computed from the report's counts and weights, unrounded. A floor is judged + on these: the report rounds its own figures for display only.""" + by_class, weights = report["coverage"]["by_class"], report["coverage"]["weights"] + scope = [c for c in CLASSES if c != "out_of_scope"] + weight = sum(weights[c] * by_class[c]["total"] for c in scope) + + def share(classes: tuple[str, ...]) -> float | None: + total = sum(by_class[c]["total"] for c in classes) + return sum(by_class[c]["covered"] for c in classes) / total if total else None + + return { + "weighted": sum(weights[c] * by_class[c]["covered"] for c in scope) / weight if weight else None, + "core_rare_critical": share(CRITICAL), + "rare_critical": share(("rare_critical",)), + } + + def read_labels(root: Path) -> list[LabelledEval]: """Every eval under ``root`` with its ``needs``: skill-creator ``evals.json`` rows, then tasks. @@ -102,14 +138,14 @@ def read_labels(root: Path) -> list[LabelledEval]: raise ValueError(f"{path}: \"evals\" is not a list") for index, row in enumerate(rows): if isinstance(row, dict): - out.append(_labelled(str(row.get("id", index)), path.name, row.get("needs"))) + out.append(_labelled(str(row.get("id", index)), path.name, row.get("needs", _ABSENT))) if root.is_dir(): for task_dir in sorted(p for p in root.iterdir() if is_task_dir(p)): try: meta = read_task_config(task_dir).get("metadata", {}) except (ValueError, OSError) as exc: raise ValueError(f"{task_dir}: unreadable task ({exc})") from exc - needs = meta.get("needs") if isinstance(meta, dict) else None + needs = meta.get("needs", _ABSENT) if isinstance(meta, dict) else _ABSENT out.append(_labelled(task_dir.name, "task", needs)) return out diff --git a/dokimi/tests/test_eval_coverage.py b/dokimi/tests/test_eval_coverage.py index f5b8c121e..16d003294 100644 --- a/dokimi/tests/test_eval_coverage.py +++ b/dokimi/tests/test_eval_coverage.py @@ -158,7 +158,7 @@ def test_cli_writes_artifact_and_fails_under_floor(tmp_path, capsys): assert "n3 [rare_critical, hypothesis]" in ok.stdout assert load_artifact("eval_coverage", out)["gaps"][0]["id"] == "n3" assert low.returncode == 1 - assert "FAIL: rare_critical coverage 0.00 < 0.50" in low.stderr + assert "FAIL: rare_critical coverage 0.000000 < 0.500000" in low.stderr assert load_artifact("eval_coverage", out)["coverage"]["weights"]["branch"] == 0.7 @@ -199,7 +199,7 @@ def test_markdown_lists_problems_and_says_none_without_gaps(tmp_path, capsys): assert "Problems:\n- all (evals.json): unknown need id 'n9'" in res.stdout -@pytest.mark.parametrize("needs", ["n1", {"n1": 1}, 3, [1]]) +@pytest.mark.parametrize("needs", ["n1", {"n1": 1}, 3, [1], None]) def test_needs_that_is_not_a_list_of_ids_is_a_problem(tmp_path, capsys, needs): res = _run(capsys, str(_skill(tmp_path, [{"id": "e", "needs": needs}])), "--graph", str(_graph(tmp_path))) @@ -234,7 +234,7 @@ def test_cli_refuses_an_empty_suite_a_non_list_evals_and_out_over_graph(tmp_path assert (none.returncode, dict_rows.returncode, clobber.returncode) == (2, 2, 2) assert "no evals.json rows or tasks" in none.stderr assert '"evals" is not a list' in dict_rows.stderr - assert "--out would overwrite --graph" in clobber.stderr + assert "--out would overwrite an input file" in clobber.stderr assert json.loads(graph.read_text())["nodes"] == NODES @@ -249,3 +249,67 @@ def test_graph_with_only_out_of_scope_nodes_has_no_coverage_figures(tmp_path): "by_class": {c: {"covered": 0, "total": int(c == "out_of_scope")} for c in ("core", "rare_critical", "branch", "supporting", "out_of_scope")}, } + + +@pytest.mark.parametrize("target", ["graph", "evals", "task"]) +def test_out_never_overwrites_an_input_file(tmp_path, capsys, target): + graph, skill = _graph(tmp_path), _skill(tmp_path, ROWS) + tasks, task = tmp_path / "tasks", tmp_path / "tasks" / "t1" + task.mkdir(parents=True) + (task / "task.toml").write_text('[metadata]\nneeds = ["n1"]\n') + (task / "instruction.md").write_text("x") + victim = {"graph": graph, "evals": skill / "evals" / "evals.json", "task": task / "task.toml"}[target] + root = tasks if target == "task" else skill + before = victim.read_bytes() + + res = _run(capsys, str(root), "--graph", str(graph), "--out", str(victim)) + + assert res.returncode == 2 + assert "--out would overwrite an input file" in res.stderr + assert victim.read_bytes() == before + + +@pytest.mark.parametrize("name", ["task.yaml", "docker-compose.yaml"]) +def test_out_never_overwrites_a_terminal_bench_task_file(tmp_path, capsys, name): + task = tmp_path / "tasks" / "tb1" + task.mkdir(parents=True) + (task / "task.yaml").write_text("instruction: write chunks\n") + (task / "docker-compose.yaml").write_text("services: {}\n") + victim = task / name + before = victim.read_bytes() + + res = _run(capsys, str(tmp_path / "tasks"), "--graph", str(_graph(tmp_path)), "--out", str(victim)) + + assert res.returncode == 2, res.stderr + assert "--out would overwrite an input file" in res.stderr + assert victim.read_bytes() == before + + +def test_a_null_task_needs_is_malformed_and_a_missing_key_is_unlabelled(tmp_path): + tasks = tmp_path / "tasks" + task = tasks / "t-null" + task.mkdir(parents=True) + (task / "task.md").write_text("---\nmetadata:\n needs:\n---\n\nwrite chunks\n") + missing = tasks / "t-missing" + missing.mkdir() + (missing / "task.toml").write_text("[metadata]\n") + (missing / "instruction.md").write_text("x") + + labels = {e.id: e for e in read_labels(tasks)} + + assert labels["t-null"].malformed and labels["t-null"].needs is None + assert not labels["t-missing"].malformed and labels["t-missing"].needs is None + + +@pytest.mark.parametrize(("covered", "floor", "code"), [(6, "0.85714", 0), (2, "0.66669", 1)]) +def test_a_floor_is_judged_on_the_unrounded_share(tmp_path, capsys, covered, floor, code): + # 6 of 7 core nodes is 0.857142... (rounds to 0.8571); 2 of 3 is 0.666666... (rounds to 0.6667) + cores = 7 if covered == 6 else 3 + nodes = [{"id": f"c{i}", "text": "t", "path_class": "core"} for i in range(cores)] + graph = tmp_path / "need_graph.json" + graph.write_text(json.dumps({"artifact_name": "x", "nodes": nodes})) + skill = _skill(tmp_path, [{"id": "e", "needs": [f"c{i}" for i in range(covered)]}]) + + res = _run(capsys, str(skill), "--graph", str(graph), "--min", f"core_rare_critical={floor}") + + assert res.returncode == code, res.stderr From 3fa238b9a83b5d1031f67a97d5a08d9493db1fec Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 23:36:11 -0700 Subject: [PATCH 4/9] fix(eval): coverage refuses an unwritable --out; table cells escape pipes and newlines; mypy - _below reads the share into a local, so mypy narrows it (the CI mypy failure). - A failing --out write (a directory, no permission) is a usage error, not a traceback. - Markdown table cells escape "|" and flatten line breaks, so an id cannot add a column or split a row. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/eval_coverage.py | 14 +++++++++++--- dokimi/dokimi/eval_coverage.py | 7 ++++++- dokimi/tests/test_eval_coverage.py | 16 ++++++++++++++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/dokimi/dokimi/commands/eval_coverage.py b/dokimi/dokimi/commands/eval_coverage.py index ec8daba7d..a5384aaaa 100644 --- a/dokimi/dokimi/commands/eval_coverage.py +++ b/dokimi/dokimi/commands/eval_coverage.py @@ -33,8 +33,12 @@ def _below(report: dict, floors: list[tuple[str, float]]) -> list[str]: from dokimi.eval_coverage import exact_shares exact = exact_shares(report) - return [f"{metric} coverage {exact[metric]:.6f} < {floor:.6f}" for metric, floor in floors - if exact[metric] is not None and exact[metric] < floor] + out = [] + for metric, floor in floors: + share = exact[metric] + if share is not None and share < floor: + out.append(f"{metric} coverage {share:.6f} < {floor:.6f}") + return out def cmd_eval_coverage(args: argparse.Namespace) -> None: @@ -60,7 +64,11 @@ def cmd_eval_coverage(args: argparse.Namespace) -> None: sys.exit(EXIT_USAGE) report = coverage_report(graph, evals, str(root)) if args.out: - report = save_artifact("eval_coverage", report, Path(args.out)) + try: + report = save_artifact("eval_coverage", report, Path(args.out)) + except OSError as exc: + print(f"Error: cannot write {args.out}: {exc.strerror or exc}", file=sys.stderr) + sys.exit(EXIT_USAGE) print(render_markdown(report)) if report["problems"]: print(f"Error: {len(report['problems'])} problem(s) with the evals' needs labels", diff --git a/dokimi/dokimi/eval_coverage.py b/dokimi/dokimi/eval_coverage.py index 00937febc..cbcca36bd 100644 --- a/dokimi/dokimi/eval_coverage.py +++ b/dokimi/dokimi/eval_coverage.py @@ -218,6 +218,11 @@ def _problem(p: dict) -> str: return f"- {p['eval']} ({p['source']}): {what}" +def _cell(text: str) -> str: + """A Markdown table cell: pipes escaped, line breaks flattened so the row stays one line.""" + return " ".join(text.replace("|", "\\|").splitlines()) + + def render_markdown(report: dict) -> str: """The report as Markdown, for a suite's README or the PR that changes the suite.""" g, c = report["graph"], report["coverage"] @@ -231,7 +236,7 @@ def render_markdown(report: dict) -> str: "| Eval | Source | Needs | Relevance |", "|---|---|---|---|", ] - lines += [f"| {e['id']} | {e['source']} | {', '.join(e['needs']) or '-'} | {e['relevance']} |" + lines += [f"| {_cell(e['id'])} | {_cell(e['source'])} | {_cell(', '.join(e['needs']) or '-')} | {e['relevance']} |" for e in report["evals"]] lines += ["", "Gaps (core and rare_critical nodes no eval names):"] lines += [f"- {n['id']} [{n['path_class']}, {n['confirmed_by']}] {n['text']}" diff --git a/dokimi/tests/test_eval_coverage.py b/dokimi/tests/test_eval_coverage.py index 16d003294..6183696aa 100644 --- a/dokimi/tests/test_eval_coverage.py +++ b/dokimi/tests/test_eval_coverage.py @@ -313,3 +313,19 @@ def test_a_floor_is_judged_on_the_unrounded_share(tmp_path, capsys, covered, flo res = _run(capsys, str(skill), "--graph", str(graph), "--min", f"core_rare_critical={floor}") assert res.returncode == code, res.stderr + + +def test_markdown_table_cells_escape_pipes_and_newlines(tmp_path, capsys): + rows = [{"id": "save|load", "needs": ["n1"]}, {"id": "two\nlines", "needs": ["n2", "n3"]}] + res = _run(capsys, str(_skill(tmp_path, rows)), "--graph", str(_graph(tmp_path))) + + assert res.returncode == 0, res.stderr + assert "| save\\|load | evals.json | n1 | " in res.stdout + assert "| two lines | evals.json | n2, n3 | " in res.stdout + + +def test_out_on_a_directory_is_a_controlled_error(tmp_path, capsys): + res = _run(capsys, str(_skill(tmp_path, ROWS)), "--graph", str(_graph(tmp_path)), "--out", str(tmp_path)) + + assert res.returncode == 2 + assert "Error: cannot write" in res.stderr and "Traceback" not in res.stderr From db9ef3eaadb1d3adf288f8c111b4fa2a7d60dbdd Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 09:42:17 -0800 Subject: [PATCH 5/9] fix(eval): coverage refuses to overwrite instruction.md and skips task.md notes --out could replace a task's instruction.md, which load_task runs, because the overwrite guard listed task.toml and task.md but not instruction.md. A task root with a planning-notes task.md (no frontmatter) aborted the report: the directory counted as a task and read_task_config raised. Both now follow discover_tasks: instruction.md is a protected input, and a lone task.md without frontmatter is not a task. Tests: the instruction.md overwrite (red before), the notes directory (red before), a floor equal to the share, a bare --min on the weighted figure, a hand-computed exact_shares case, and a relative --out to an input. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/eval_coverage.py | 21 +++++-- dokimi/tests/test_eval_coverage.py | 89 ++++++++++++++++++++++++++++-- 2 files changed, 101 insertions(+), 9 deletions(-) diff --git a/dokimi/dokimi/eval_coverage.py b/dokimi/dokimi/eval_coverage.py index cbcca36bd..0c91ac56b 100644 --- a/dokimi/dokimi/eval_coverage.py +++ b/dokimi/dokimi/eval_coverage.py @@ -29,7 +29,7 @@ from dokimi.artifacts import load_artifact from dokimi.harnesses import tbench_v1 -from dokimi.harnesses.tasks import is_task_dir, read_task_config +from dokimi.harnesses.tasks import FORMAT_TASK_MD, is_task_dir, read_task_config, task_format from dokimi.novelty.needs import PATH_WEIGHTS CRITICAL = ("core", "rare_critical") @@ -89,8 +89,19 @@ def _evals_files(root: Path) -> list[Path]: return [p for p in (root / "evals.json", root / "evals" / "evals.json") if p.is_file()] -# every task file a task reader may parse: Harbor / BenchFlow (task.toml, task.md), terminal-bench v1 (task.yaml, compose) -_TASK_FILES = ("task.toml", "task.md", tbench_v1.CONFIG, tbench_v1.COMPOSE) +# every task file a task reader may parse: Harbor / BenchFlow (task.toml, task.md, instruction.md, which load_task +# runs), terminal-bench v1 (task.yaml, compose) +_TASK_FILES = ("task.toml", "task.md", "instruction.md", tbench_v1.CONFIG, tbench_v1.COMPOSE) + + +def _is_task(path: Path) -> bool: + """A task directory the way discover_tasks counts it: a lone task.md without frontmatter is an author's note.""" + if not is_task_dir(path): + return False + if task_format(path) != FORMAT_TASK_MD: + return True + with (path / "task.md").open(encoding="utf-8", errors="replace") as fh: + return fh.readline().strip() == "---" def input_files(root: Path) -> list[Path]: @@ -98,7 +109,7 @@ def input_files(root: Path) -> list[Path]: overwrite one of them.""" files = _evals_files(root) if root.is_dir(): - for task_dir in (p for p in root.iterdir() if is_task_dir(p)): + for task_dir in (p for p in root.iterdir() if _is_task(p)): files += [task_dir / name for name in _TASK_FILES if (task_dir / name).is_file()] return [p.resolve() for p in files] @@ -140,7 +151,7 @@ def read_labels(root: Path) -> list[LabelledEval]: if isinstance(row, dict): out.append(_labelled(str(row.get("id", index)), path.name, row.get("needs", _ABSENT))) if root.is_dir(): - for task_dir in sorted(p for p in root.iterdir() if is_task_dir(p)): + for task_dir in sorted(p for p in root.iterdir() if _is_task(p)): try: meta = read_task_config(task_dir).get("metadata", {}) except (ValueError, OSError) as exc: diff --git a/dokimi/tests/test_eval_coverage.py b/dokimi/tests/test_eval_coverage.py index 6183696aa..3044bc164 100644 --- a/dokimi/tests/test_eval_coverage.py +++ b/dokimi/tests/test_eval_coverage.py @@ -13,7 +13,7 @@ from dokimi.artifacts import ArtifactValidationError, load_artifact from dokimi.cli import build_parser -from dokimi.eval_coverage import LabelledEval, coverage_report, graph_digest, load_graph, read_labels +from dokimi.eval_coverage import LabelledEval, coverage_report, exact_shares, graph_digest, load_graph, read_labels from dokimi.novelty.needs import NeedGraph, NeedNode from dokimi.novelty.needs import load_graph as novelty_load_graph from dokimi.novelty.needs import save_graph @@ -251,15 +251,16 @@ def test_graph_with_only_out_of_scope_nodes_has_no_coverage_figures(tmp_path): } -@pytest.mark.parametrize("target", ["graph", "evals", "task"]) +@pytest.mark.parametrize("target", ["graph", "evals", "task", "instruction"]) def test_out_never_overwrites_an_input_file(tmp_path, capsys, target): graph, skill = _graph(tmp_path), _skill(tmp_path, ROWS) tasks, task = tmp_path / "tasks", tmp_path / "tasks" / "t1" task.mkdir(parents=True) (task / "task.toml").write_text('[metadata]\nneeds = ["n1"]\n') (task / "instruction.md").write_text("x") - victim = {"graph": graph, "evals": skill / "evals" / "evals.json", "task": task / "task.toml"}[target] - root = tasks if target == "task" else skill + victim = {"graph": graph, "evals": skill / "evals" / "evals.json", "task": task / "task.toml", + "instruction": task / "instruction.md"}[target] + root = tasks if target in ("task", "instruction") else skill before = victim.read_bytes() res = _run(capsys, str(root), "--graph", str(graph), "--out", str(victim)) @@ -285,6 +286,86 @@ def test_out_never_overwrites_a_terminal_bench_task_file(tmp_path, capsys, name) assert victim.read_bytes() == before +def test_a_task_md_without_frontmatter_is_a_note_not_a_task(tmp_path, capsys): + tasks = tmp_path / "tasks" + notes = tasks / "notes" + notes.mkdir(parents=True) + (notes / "task.md").write_text("# planning notes\n") + real = tasks / "t1" + real.mkdir() + (real / "task.toml").write_text('[metadata]\nneeds = ["n1"]\n') + + labels = read_labels(tasks) + res = _run(capsys, str(tasks), "--graph", str(_graph(tmp_path))) + + assert [e.id for e in labels] == ["t1"] + assert res.returncode == 0, res.stderr + + +def test_exact_shares_are_the_hand_computed_figures(): + # weighted = (core c1 + 3 branch at 0.7) / (core c1, c2 + rare r1 + 3 branch) = 3.1 / 5.1; out_of_scope is outside it + graph = {"artifact_name": "x", "nodes": [ + {"id": "c1", "text": "t", "path_class": "core"}, + {"id": "c2", "text": "t", "path_class": "core"}, + {"id": "r1", "text": "t", "path_class": "rare_critical"}, + {"id": "b1", "text": "t", "path_class": "branch"}, + {"id": "b2", "text": "t", "path_class": "branch"}, + {"id": "b3", "text": "t", "path_class": "branch"}, + {"id": "o1", "text": "t", "path_class": "out_of_scope"}, + ]} + evals = [LabelledEval("e1", "task", ("c1", "b1", "b2", "b3", "o1"))] + + figures = exact_shares(coverage_report(graph, evals, "r")) + + assert figures["weighted"] == pytest.approx(3.1 / 5.1, abs=1e-12) + assert figures["core_rare_critical"] == pytest.approx(1 / 3, abs=1e-12) + assert figures["rare_critical"] == 0.0 + + +def test_a_bare_floor_is_on_the_weighted_figure(tmp_path, capsys): + # weighted is 0.6078 and core_rare_critical 0.3333: a bare 0.5 passes only if the default metric is weighted + graph = tmp_path / "need_graph.json" + graph.write_text(json.dumps({"artifact_name": "x", "nodes": [ + {"id": "c1", "text": "t", "path_class": "core"}, + {"id": "c2", "text": "t", "path_class": "core"}, + {"id": "r1", "text": "t", "path_class": "rare_critical"}, + {"id": "b1", "text": "t", "path_class": "branch"}, + {"id": "b2", "text": "t", "path_class": "branch"}, + {"id": "b3", "text": "t", "path_class": "branch"}, + ]})) + skill = _skill(tmp_path, [{"id": "e", "needs": ["c1", "b1", "b2", "b3"]}]) + + res = _run(capsys, str(skill), "--graph", str(graph), "--min", "0.5") + + assert res.returncode == 0, res.stderr + + +def test_a_floor_equal_to_the_share_passes(tmp_path, capsys): + nodes = [{"id": f"c{i}", "text": "t", "path_class": "core"} for i in range(4)] + graph = tmp_path / "need_graph.json" + graph.write_text(json.dumps({"artifact_name": "x", "nodes": nodes})) + skill = _skill(tmp_path, [{"id": "e", "needs": ["c0", "c1"]}]) + + res = _run(capsys, str(skill), "--graph", str(graph), "--min", "core_rare_critical=0.5") + + assert res.returncode == 0, res.stderr + + +def test_out_as_a_relative_path_to_an_input_is_refused(tmp_path, capsys, monkeypatch): + skill = _skill(tmp_path, ROWS) + work = tmp_path / "work" + work.mkdir() + monkeypatch.chdir(work) + victim = skill / "evals" / "evals.json" + before = victim.read_bytes() + + res = _run(capsys, "../skill", "--graph", str(_graph(tmp_path)), "--out", "../skill/evals/evals.json") + + assert res.returncode == 2 + assert "--out would overwrite an input file" in res.stderr + assert victim.read_bytes() == before + + def test_a_null_task_needs_is_malformed_and_a_missing_key_is_unlabelled(tmp_path): tasks = tmp_path / "tasks" task = tasks / "t-null" From 6351ffe19144bc4f2133cf176eb191415253578f Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 09:53:22 -0800 Subject: [PATCH 6/9] fix(eval): coverage --min fails on a class the graph lacks A floor on a coverage figure with no node behind it (n/a) used to pass silently. It now exits 1 with " coverage n/a < X", as the changelog and the --min help say. The changelog also records the known gap: a terminal-bench v1 task.yaml has its metadata.needs ignored. Signed-off-by: Nikolay Petrov --- .../20261031_180000_napetrov_eval-coverage.md | 5 ++++- dokimi/dokimi/commands/eval_coverage.py | 9 ++++++--- dokimi/tests/test_eval_coverage.py | 12 ++++++++++++ 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/dokimi/changelog.d/20261031_180000_napetrov_eval-coverage.md b/dokimi/changelog.d/20261031_180000_napetrov_eval-coverage.md index d8fd19c95..ab188e799 100644 --- a/dokimi/changelog.d/20261031_180000_napetrov_eval-coverage.md +++ b/dokimi/changelog.d/20261031_180000_napetrov_eval-coverage.md @@ -17,5 +17,8 @@ SPDX-License-Identifier: Apache-2.0 ids the graph has. A row or task with no `needs` key is `unlabelled`; an explicit `"needs": null` is malformed and exits 2. `--out` is refused when it would overwrite the graph or any input file. A `--min` floor is judged on the unrounded share, not the rounded - figure shown in the report. Novelty's need graph now keeps `confirmed_by` and `provenance`. + figure shown in the report; a floor on a class the graph has no node of (n/a) also exits 1. + Known gap: a terminal-bench v1 task (`task.yaml`) has its `metadata.needs` ignored, so it counts + as `unlabelled`; `task.toml` and `task.md` tasks are read. Novelty's need graph now keeps + `confirmed_by` and `provenance`. Building the graph, the user interview and drafting labels follow in #331. diff --git a/dokimi/dokimi/commands/eval_coverage.py b/dokimi/dokimi/commands/eval_coverage.py index a5384aaaa..f5552d8ef 100644 --- a/dokimi/dokimi/commands/eval_coverage.py +++ b/dokimi/dokimi/commands/eval_coverage.py @@ -36,7 +36,9 @@ def _below(report: dict, floors: list[tuple[str, float]]) -> list[str]: out = [] for metric, floor in floors: share = exact[metric] - if share is not None and share < floor: + if share is None: # nothing of that class in the graph: the floor cannot be met + out.append(f"{metric} coverage n/a < {floor:.6f}") + elif share < floor: out.append(f"{metric} coverage {share:.6f} < {floor:.6f}") return out @@ -91,6 +93,7 @@ def register(eval_sub) -> None: help="need_graph.v1 file (a novelty need graph is valid as written)") p.add_argument("--out", default=None, help="Write the eval_coverage.v1 artifact here") p.add_argument("--min", type=_floor, action="append", metavar="[METRIC=]X", - help="Exit 1 when a coverage figure is below X (0-1); METRIC is weighted " - "(default), core_rare_critical or rare_critical. Repeatable") + help="Exit 1 when a coverage figure is below X (0-1) or n/a (the graph has no " + "node of that class); METRIC is weighted (default), core_rare_critical or " + "rare_critical. Repeatable") p.set_defaults(func=cmd_eval_coverage) diff --git a/dokimi/tests/test_eval_coverage.py b/dokimi/tests/test_eval_coverage.py index 3044bc164..7020ea940 100644 --- a/dokimi/tests/test_eval_coverage.py +++ b/dokimi/tests/test_eval_coverage.py @@ -351,6 +351,18 @@ def test_a_floor_equal_to_the_share_passes(tmp_path, capsys): assert res.returncode == 0, res.stderr +def test_a_floor_on_a_class_the_graph_lacks_is_n_a_and_fails(tmp_path, capsys): + nodes = [{"id": f"c{i}", "text": "t", "path_class": "core"} for i in range(2)] + graph = tmp_path / "need_graph.json" + graph.write_text(json.dumps({"artifact_name": "x", "nodes": nodes})) + skill = _skill(tmp_path, [{"id": "e", "needs": ["c0"]}]) + + res = _run(capsys, str(skill), "--graph", str(graph), "--min", "rare_critical=0.5") + + assert res.returncode == 1 + assert "FAIL: rare_critical coverage n/a < 0.500000" in res.stderr + + def test_out_as_a_relative_path_to_an_input_is_refused(tmp_path, capsys, monkeypatch): skill = _skill(tmp_path, ROWS) work = tmp_path / "work" From daa0a1e58efa9e38b31b832d103a8f8dfc81760e Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 09:54:47 -0800 Subject: [PATCH 7/9] fix(eval): coverage --out on a symlink loop exits 2, not a traceback Path.resolve() raises RuntimeError on a symlink loop on Python 3.11 and 3.12, and the output-path guard let it escape. The guard now treats OSError and RuntimeError as an unresolvable --out and exits 2. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/eval_coverage.py | 12 +++++++++--- dokimi/tests/test_eval_coverage.py | 11 +++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/commands/eval_coverage.py b/dokimi/dokimi/commands/eval_coverage.py index f5552d8ef..97c094ab6 100644 --- a/dokimi/dokimi/commands/eval_coverage.py +++ b/dokimi/dokimi/commands/eval_coverage.py @@ -61,9 +61,15 @@ def cmd_eval_coverage(args: argparse.Namespace) -> None: if not evals: print(f"Error: no evals.json rows or tasks under {root}", file=sys.stderr) sys.exit(EXIT_USAGE) - if args.out and Path(args.out).resolve() in {Path(args.graph).resolve(), *input_files(root)}: - print(f"Error: --out would overwrite an input file: {args.out}", file=sys.stderr) - sys.exit(EXIT_USAGE) + if args.out: + try: + clobbers = Path(args.out).resolve() in {Path(args.graph).resolve(), *input_files(root)} + except (OSError, RuntimeError) as exc: # a symlink loop: Python 3.11/3.12 raise RuntimeError + print(f"Error: cannot resolve {args.out}: {exc}", file=sys.stderr) + sys.exit(EXIT_USAGE) + if clobbers: + print(f"Error: --out would overwrite an input file: {args.out}", file=sys.stderr) + sys.exit(EXIT_USAGE) report = coverage_report(graph, evals, str(root)) if args.out: try: diff --git a/dokimi/tests/test_eval_coverage.py b/dokimi/tests/test_eval_coverage.py index 7020ea940..c972e9045 100644 --- a/dokimi/tests/test_eval_coverage.py +++ b/dokimi/tests/test_eval_coverage.py @@ -363,6 +363,17 @@ def test_a_floor_on_a_class_the_graph_lacks_is_n_a_and_fails(tmp_path, capsys): assert "FAIL: rare_critical coverage n/a < 0.500000" in res.stderr +def test_out_that_is_a_symlink_loop_exits_2_not_a_traceback(tmp_path, capsys, monkeypatch): + # Python 3.11/3.12 raise RuntimeError from Path.resolve() on a loop; 3.13+ raise nothing here + monkeypatch.chdir(tmp_path) + (tmp_path / "loop").symlink_to("loop") + + res = _run(capsys, str(_skill(tmp_path, ROWS)), "--graph", str(_graph(tmp_path)), "--out", "loop") + + assert res.returncode == 2 + assert "Error:" in res.stderr + + def test_out_as_a_relative_path_to_an_input_is_refused(tmp_path, capsys, monkeypatch): skill = _skill(tmp_path, ROWS) work = tmp_path / "work" From 095d2fe16b226ee617104f6501f3badc5e4fb5be Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 10:01:43 -0800 Subject: [PATCH 8/9] fix(eval): coverage exits 2 on an unreadable evals path; --out guards .rejected An unreadable parent directory made the existence check raise PermissionError outside the input error handler, so it printed a traceback. The check now sits inside that handler. save_artifact removes .rejected after a valid save, so an input at that path was deleted. The --out guard now treats it as an output too. The guard moves to a helper, which keeps the command under the complexity ratchet. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/eval_coverage.py | 35 ++++++++++++++++--------- dokimi/tests/test_eval_coverage.py | 30 +++++++++++++++++++++ 2 files changed, 53 insertions(+), 12 deletions(-) diff --git a/dokimi/dokimi/commands/eval_coverage.py b/dokimi/dokimi/commands/eval_coverage.py index 97c094ab6..81e297486 100644 --- a/dokimi/dokimi/commands/eval_coverage.py +++ b/dokimi/dokimi/commands/eval_coverage.py @@ -43,16 +43,34 @@ def _below(report: dict, floors: list[tuple[str, float]]) -> list[str]: return out +def _refuse_out_over_input(args: argparse.Namespace, root: Path) -> None: + """Exit 2 when ``--out`` cannot be resolved (a symlink loop) or would overwrite an input or the sibling.""" + from dokimi.eval_coverage import input_files + + try: + # save_artifact also removes .rejected after a valid save, so that sibling is an output too + out = Path(args.out) + outputs = (out, out.with_name(out.name + ".rejected")) + inputs = {Path(args.graph).resolve(), *input_files(root)} + clobbers = any(p.resolve() in inputs for p in outputs) + except (OSError, RuntimeError, ValueError) as exc: # a symlink loop: Python 3.11/3.12 raise RuntimeError + print(f"Error: cannot resolve {args.out}: {exc}", file=sys.stderr) + sys.exit(EXIT_USAGE) + if clobbers: + print(f"Error: --out would overwrite an input file: {args.out}", file=sys.stderr) + sys.exit(EXIT_USAGE) + + def cmd_eval_coverage(args: argparse.Namespace) -> None: """Print the report; exit 2 on an unreadable input or a bad ``needs`` label, 1 under a floor.""" from dokimi.artifacts import ArtifactValidationError, save_artifact - from dokimi.eval_coverage import coverage_report, input_files, load_graph, read_labels, render_markdown + from dokimi.eval_coverage import coverage_report, load_graph, read_labels, render_markdown root = Path(args.evals) - if not root.exists(): - print(f"Error: {root} does not exist", file=sys.stderr) - sys.exit(EXIT_USAGE) try: + if not root.exists(): + print(f"Error: {root} does not exist", file=sys.stderr) + sys.exit(EXIT_USAGE) graph = load_graph(Path(args.graph)) evals = read_labels(root) except (OSError, ValueError, ArtifactValidationError) as exc: @@ -62,14 +80,7 @@ def cmd_eval_coverage(args: argparse.Namespace) -> None: print(f"Error: no evals.json rows or tasks under {root}", file=sys.stderr) sys.exit(EXIT_USAGE) if args.out: - try: - clobbers = Path(args.out).resolve() in {Path(args.graph).resolve(), *input_files(root)} - except (OSError, RuntimeError) as exc: # a symlink loop: Python 3.11/3.12 raise RuntimeError - print(f"Error: cannot resolve {args.out}: {exc}", file=sys.stderr) - sys.exit(EXIT_USAGE) - if clobbers: - print(f"Error: --out would overwrite an input file: {args.out}", file=sys.stderr) - sys.exit(EXIT_USAGE) + _refuse_out_over_input(args, root) report = coverage_report(graph, evals, str(root)) if args.out: try: diff --git a/dokimi/tests/test_eval_coverage.py b/dokimi/tests/test_eval_coverage.py index c972e9045..113a82d0c 100644 --- a/dokimi/tests/test_eval_coverage.py +++ b/dokimi/tests/test_eval_coverage.py @@ -351,6 +351,36 @@ def test_a_floor_equal_to_the_share_passes(tmp_path, capsys): assert res.returncode == 0, res.stderr +def test_an_unreadable_evals_directory_exits_2_not_a_traceback(tmp_path, capsys, monkeypatch): + skill = _skill(tmp_path, ROWS) + real_exists = Path.exists + + def exists(self, *args, **kwargs): + if self == skill / "evals" / "evals.json" or self == skill: + raise PermissionError(13, "Permission denied", str(self)) + return real_exists(self, *args, **kwargs) + + monkeypatch.setattr(Path, "exists", exists) + res = _run(capsys, str(skill), "--graph", str(_graph(tmp_path))) + + assert res.returncode == 2 + assert "Permission denied" in res.stderr + + +def test_out_whose_rejected_sibling_is_the_graph_is_refused(tmp_path, capsys): + # save_artifact removes .rejected after a valid save: an input at that path must not be deleted + graph = _graph(tmp_path) + sibling = graph.with_name("cov.json.rejected") + graph.rename(sibling) + before = sibling.read_bytes() + + res = _run(capsys, str(_skill(tmp_path, ROWS)), "--graph", str(sibling), "--out", str(tmp_path / "cov.json")) + + assert res.returncode == 2 + assert "--out would overwrite an input file" in res.stderr + assert sibling.read_bytes() == before + + def test_a_floor_on_a_class_the_graph_lacks_is_n_a_and_fails(tmp_path, capsys): nodes = [{"id": f"c{i}", "text": "t", "path_class": "core"} for i in range(2)] graph = tmp_path / "need_graph.json" From 0f218cd6784a22f9055d029bdb6465652e1ea7d4 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sun, 1 Nov 2026 20:22:46 +0000 Subject: [PATCH 9/9] Patch Path.resolve in the symlink-loop test so it runs on 3.13+ Signed-off-by: Anton Volkov --- dokimi/tests/test_eval_coverage.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/dokimi/tests/test_eval_coverage.py b/dokimi/tests/test_eval_coverage.py index 113a82d0c..19cc90c72 100644 --- a/dokimi/tests/test_eval_coverage.py +++ b/dokimi/tests/test_eval_coverage.py @@ -394,14 +394,21 @@ def test_a_floor_on_a_class_the_graph_lacks_is_n_a_and_fails(tmp_path, capsys): def test_out_that_is_a_symlink_loop_exits_2_not_a_traceback(tmp_path, capsys, monkeypatch): - # Python 3.11/3.12 raise RuntimeError from Path.resolve() on a loop; 3.13+ raise nothing here + # Python 3.11/3.12 raise RuntimeError from Path.resolve() on a loop; 3.13+ do not, so raise it here monkeypatch.chdir(tmp_path) - (tmp_path / "loop").symlink_to("loop") + real_resolve = Path.resolve + + def resolve(self, *args, **kwargs): + if self.name == "loop": + raise RuntimeError("Symlink loop") + return real_resolve(self, *args, **kwargs) + + monkeypatch.setattr(Path, "resolve", resolve) res = _run(capsys, str(_skill(tmp_path, ROWS)), "--graph", str(_graph(tmp_path)), "--out", "loop") assert res.returncode == 2 - assert "Error:" in res.stderr + assert "cannot resolve" in res.stderr def test_out_as_a_relative_path_to_an_input_is_refused(tmp_path, capsys, monkeypatch):