feat(eval): eval coverage against a frozen need graph (#331)

dokimi/changelog.d/20261031_180000_napetrov_eval-coverage.md

@@ -0,0 +1,24 @@
+<!--
+Copyright (C) 2026 Intel Corporation
+SPDX-License-Identifier: Apache-2.0
+-->
+
+### Added
+
+- **`eval coverage`: which needs of a subject an eval set grades, without a model call.**
+  `python cli.py eval coverage <evals.json | skill dir | tasks root> --graph need_graph.json`
+  reads a frozen need graph (new `need_graph.v1` schema; a novelty need graph is valid as
+  written) and the `needs` list on each `evals.json` row or in a task's `metadata.needs`. It
+  prints the weighted coverage, the coverage of `core` + `rare_critical` and of `rare_critical`
+  alone, the uncovered `core` / `rare_critical` nodes, and each eval's relevance
+  (`relevant_core`, `relevant_peripheral`, `out_of_scope`, `orphan`, `unlabelled`). `--out`
+  writes an `eval_coverage.v1` artifact. Exit 1 under a `--min [METRIC=]X` floor, exit 2 on an
+  unreadable input, a graph with duplicate node ids, or a `needs` label that is not a list of
+  ids the graph has. A row or task with no `needs` key is `unlabelled`; an explicit
+  `"needs": null` is malformed and exits 2. `--out` is refused when it would overwrite the
+  graph or any input file. A `--min` floor is judged on the unrounded share, not the rounded
+  figure shown in the report; a floor on a class the graph has no node of (n/a) also exits 1.
+  Known gap: a terminal-bench v1 task (`task.yaml`) has its `metadata.needs` ignored, so it counts
+  as `unlabelled`; `task.toml` and `task.md` tasks are read. Novelty's need graph now keeps
+  `confirmed_by` and `provenance`.
+  Building the graph, the user interview and drafting labels follow in #331.

dokimi/data/skills/dokimi-design-evals/references/coverage-report.md

@@ -1,9 +1,17 @@
 # Coverage report
 
-dokimi has no command that writes this report (`python -m dokimi.coverage`
-prints a product x awareness/work task count, not scenario coverage). Write
-it by hand, in the suite's README and in the PR that adds or changes the
-suite.
+Write this report by hand, in the suite's README and in the PR that adds or
+changes the suite. `python -m dokimi.coverage` prints only a product x
+awareness/work task count.
+
+`python cli.py eval coverage <evals> --graph need_graph.json` adds the part
+a script can check. It needs a need graph (what users of the subject need,
+built from its name and description only) and a `needs` list on each eval
+that names the graph nodes it grades. From those two files alone it lists the
+core and rare_critical nodes that no eval names, and the relevance of each
+eval: `orphan` for `needs: []`, `unlabelled` for no `needs`. Paste its
+Markdown under the table below. It does not replace the areas in the table,
+and a node the graph never had is still a gap nobody looked at.
 
 ## Template
 

dokimi/docs/reference/cli-interface.json

@@ -4017,6 +4017,94 @@
           "subcommand_required": null,
           "subcommands": {}
         },
+        "coverage": {
+          "allow_abbrev": true,
+          "params": [
+            {
+              "action": "_StoreAction",
+              "choices": null,
+              "const": null,
+              "default": null,
+              "group_key": null,
+              "group_required": null,
+              "hidden": false,
+              "is_flag": false,
+              "key": "evals",
+              "kind": "argument",
+              "nargs": null,
+              "position": 0,
+              "required": true,
+              "type": null
+            },
+            {
+              "action": "_StoreAction",
+              "choices": null,
+              "const": null,
+              "default": null,
+              "group_key": null,
+              "group_required": null,
+              "hidden": false,
+              "is_flag": false,
+              "key": "--graph",
+              "kind": "option",
+              "nargs": null,
+              "position": null,
+              "required": true,
+              "type": null
+            },
+            {
+              "action": "_HelpAction",
+              "choices": null,
+              "const": null,
+              "default": "==SUPPRESS==",
+              "group_key": null,
+              "group_required": null,
+              "hidden": false,
+              "is_flag": true,
+              "key": "--help/-h",
+              "kind": "option",
+              "nargs": 0,
+              "position": null,
+              "required": false,
+              "type": null
+            },
+            {
+              "action": "_AppendAction",
+              "choices": null,
+              "const": null,
+              "default": null,
+              "group_key": null,
+              "group_required": null,
+              "hidden": false,
+              "is_flag": false,
+              "key": "--min",
+              "kind": "option",
+              "nargs": null,
+              "position": null,
+              "required": false,
+              "type": "dokimi.commands.eval_coverage._floor#bbd5737352a9"
+            },
+            {
+              "action": "_StoreAction",
+              "choices": null,
+              "const": null,
+              "default": null,
+              "group_key": null,
+              "group_required": null,
+              "hidden": false,
+              "is_flag": false,
+              "key": "--out",
+              "kind": "option",
+              "nargs": null,
+              "position": null,
+              "required": false,
+              "type": null
+            }
+          ],
+          "path": "dokimi eval coverage",
+          "subcommand_required": null,
+          "subcommands": {}
+        },
         "cross-judge": {
           "allow_abbrev": true,
           "params": [
@@ -12885,9 +12973,11 @@
                 "'arms'",
                 "'cli_routes'",
                 "'eval'",
+                "'eval_coverage'",
                 "'explore_runs'",
                 "'guidance_reach'",
                 "'matrix_rollup'",
+                "'need_graph'",
                 "'novelty'",
                 "'plan'",
                 "'plugin_delta'",

dokimi/docs/user-journeys/journeys.md

@@ -492,7 +492,7 @@ down in the PR.
 
 #### UC-J07-004 State coverage gaps
 
-- **AC-J07-007** An eval set's report lists the capabilities or areas it has no task for. *Proposed.*
+- **AC-J07-007** An eval set's report lists the capabilities or areas it has no task for. *Partial: `eval coverage` lists the core and rare_critical nodes of a frozen need graph that no eval's `needs` names; building the graph, the user interview and drafting labels are #331.*
 
 ---
 

dokimi/dokimi/artifacts.py

@@ -12,7 +12,8 @@
 what moves it), ``repo_environment`` (static inspection of what a
 repository offers an agent), ``plan`` (the cells a run promises to measure),
 ``result`` (one status per planned cell), ``repo_budget`` (what a repository puts
-into an agent's context) and ``task_inspect`` (which eval-task format each
+into an agent's context), ``need_graph`` and ``eval_coverage`` (a frozen need graph and an
+eval set's coverage of it) and ``task_inspect`` (which eval-task format each
 path under a tree is). This module stamps each with a
 ``schema_version`` and validates it against the JSON Schema in
 ``dokimi/schemas/`` on save and load, so fixtures and baselines stay
@@ -58,6 +59,8 @@
     "repo_budget": "repo_budget.v1",
     "task_inspect": "task_inspect.v1",
     "cli_routes": "cli_routes.v1",
+    "need_graph": "need_graph.v1",
+    "eval_coverage": "eval_coverage.v1",
 }
 
 # kind -> superseded versions still accepted on *load*, newest first. An artifact written under an

dokimi/dokimi/cli_routes.py

@@ -142,6 +142,7 @@ class Tombstone:
     Leaf(("eval", "score"), PAID),
     Leaf(("eval", "ragas"), PAID),
     Leaf(("eval", "grounding"), FREE),
+    Leaf(("eval", "coverage"), FREE),
     Leaf(("eval", "judge-validity"), PAID),
     Leaf(("eval", "checklist"), PAID),
     Leaf(("eval", "judge-wording"), PAID),

dokimi/dokimi/commands/eval_coverage.py

@@ -0,0 +1,116 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""``eval coverage``: coverage and relevance of an eval set against a frozen need graph (#331)."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from dokimi.exit_codes import EXIT_FAILURE, EXIT_USAGE
+
+
+METRICS = ("weighted", "core_rare_critical", "rare_critical")
+
+
+def _floor(value: str) -> tuple[str, float]:
+    metric, sep, number = value.rpartition("=")
+    metric = metric if sep else "weighted"
+    if metric not in METRICS:
+        raise argparse.ArgumentTypeError(f"metric must be one of {', '.join(METRICS)}, got {metric!r}")
+    try:
+        floor = float(number)
+    except ValueError:
+        raise argparse.ArgumentTypeError(f"expected a number, got {number!r}") from None
+    if not 0.0 <= floor <= 1.0:
+        raise argparse.ArgumentTypeError(f"must lie in 0-1, got {floor}")
+    return metric, floor
+
+
+def _below(report: dict, floors: list[tuple[str, float]]) -> list[str]:
+    from dokimi.eval_coverage import exact_shares
+
+    exact = exact_shares(report)
+    out = []
+    for metric, floor in floors:
+        share = exact[metric]
+        if share is None:  # nothing of that class in the graph: the floor cannot be met
+            out.append(f"{metric} coverage n/a < {floor:.6f}")
+        elif share < floor:
+            out.append(f"{metric} coverage {share:.6f} < {floor:.6f}")
+    return out
+
+
+def _refuse_out_over_input(args: argparse.Namespace, root: Path) -> None:
+    """Exit 2 when ``--out`` cannot be resolved (a symlink loop) or would overwrite an input or the sibling."""
+    from dokimi.eval_coverage import input_files
+
+    try:
+        # save_artifact also removes <out>.rejected after a valid save, so that sibling is an output too
+        out = Path(args.out)
+        outputs = (out, out.with_name(out.name + ".rejected"))
+        inputs = {Path(args.graph).resolve(), *input_files(root)}
+        clobbers = any(p.resolve() in inputs for p in outputs)
+    except (OSError, RuntimeError, ValueError) as exc:  # a symlink loop: Python 3.11/3.12 raise RuntimeError
+        print(f"Error: cannot resolve {args.out}: {exc}", file=sys.stderr)
+        sys.exit(EXIT_USAGE)
+    if clobbers:
+        print(f"Error: --out would overwrite an input file: {args.out}", file=sys.stderr)
+        sys.exit(EXIT_USAGE)
+
+
+def cmd_eval_coverage(args: argparse.Namespace) -> None:
+    """Print the report; exit 2 on an unreadable input or a bad ``needs`` label, 1 under a floor."""
+    from dokimi.artifacts import ArtifactValidationError, save_artifact
+    from dokimi.eval_coverage import coverage_report, load_graph, read_labels, render_markdown
+
+    root = Path(args.evals)
+    try:
+        if not root.exists():
+            print(f"Error: {root} does not exist", file=sys.stderr)
+            sys.exit(EXIT_USAGE)
+        graph = load_graph(Path(args.graph))
+        evals = read_labels(root)
+    except (OSError, ValueError, ArtifactValidationError) as exc:
+        print(f"Error: {exc}", file=sys.stderr)
+        sys.exit(EXIT_USAGE)
+    if not evals:
+        print(f"Error: no evals.json rows or tasks under {root}", file=sys.stderr)
+        sys.exit(EXIT_USAGE)
+    if args.out:
+        _refuse_out_over_input(args, root)
+    report = coverage_report(graph, evals, str(root))
+    if args.out:
+        try:
+            report = save_artifact("eval_coverage", report, Path(args.out))
+        except OSError as exc:
+            print(f"Error: cannot write {args.out}: {exc.strerror or exc}", file=sys.stderr)
+            sys.exit(EXIT_USAGE)
+    print(render_markdown(report))
+    if report["problems"]:
+        print(f"Error: {len(report['problems'])} problem(s) with the evals' needs labels",
+              file=sys.stderr)
+        sys.exit(EXIT_USAGE)
+    failed = _below(report, args.min or [])
+    if failed:
+        print("FAIL: " + "; ".join(failed), file=sys.stderr)
+        sys.exit(EXIT_FAILURE)
+
+
+def register(eval_sub) -> None:
+    """Add ``eval coverage`` to the ``eval`` group."""
+    p = eval_sub.add_parser(
+        "coverage",
+        help="Coverage of a frozen need graph by an eval set's `needs` labels (no model call)",
+    )
+    p.add_argument("evals", help="evals.json, a skill directory (evals/evals.json) or a tasks root")
+    p.add_argument("--graph", required=True,
+                   help="need_graph.v1 file (a novelty need graph is valid as written)")
+    p.add_argument("--out", default=None, help="Write the eval_coverage.v1 artifact here")
+    p.add_argument("--min", type=_floor, action="append", metavar="[METRIC=]X",
+                   help="Exit 1 when a coverage figure is below X (0-1) or n/a (the graph has no "
+                        "node of that class); METRIC is weighted (default), core_rare_critical or "
+                        "rare_critical. Repeatable")
+    p.set_defaults(func=cmd_eval_coverage)

dokimi/dokimi/commands/evaluate.py

@@ -11,6 +11,7 @@
 import sys
 from pathlib import Path
 
+from dokimi.commands import eval_coverage
 from dokimi.llm import ANSWER_PROVIDERS
 from dokimi.defaults import (
     DEFAULT_CONCURRENCY,
@@ -467,6 +468,7 @@ def register(sub, positive_int) -> None:
     # Eval subcommand group
     eval_p = sub.add_parser("eval", help="Evaluate answers")
     eval_sub = eval_p.add_subparsers(dest="eval_cmd", required=True)
+    eval_coverage.register(eval_sub)
 
     # eval score
     score_p = eval_sub.add_parser("score", help="Score answers using LLM-as-judge")