From 67975e9a6f46550fed0d150ff743e9818c661185 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 21 Oct 2026 22:14:38 -0700 Subject: [PATCH 01/17] feat(runner): plan.v1 / result.v1 and engine-computed completeness A plan lists every cell (arm x model x task x rep, stable cell_id) a run promises to measure; a result gives each cell one status and points back at the plan by digest. runner.plan.completeness computes planned minus completed: a run is complete only when every planned cell is pass, fail or inconclusive, and a result for another plan is invalid. check_smoke_runs.py now declares its planned cells and checks them through completeness, so a missing cell fails the smoke gate. Part of #121. Signed-off-by: Nikolay Petrov --- ...0261021_120000_napetrov_run-plan-result.md | 7 ++ dokimi/dokimi/artifacts.py | 7 +- dokimi/dokimi/runner/plan.py | 113 ++++++++++++++++++ dokimi/dokimi/schemas/plan.v1.json | 87 ++++++++++++++ dokimi/dokimi/schemas/result.v1.json | 28 +++++ dokimi/scripts/check_smoke_runs.py | 35 ++++++ dokimi/tests/fixtures/artifacts/plan.v1.json | 82 +++++++++++++ .../tests/fixtures/artifacts/result.v1.json | 44 +++++++ dokimi/tests/test_check_smoke_runs.py | 34 +++++- dokimi/tests/test_run_plan.py | 111 +++++++++++++++++ 10 files changed, 544 insertions(+), 4 deletions(-) create mode 100644 dokimi/changelog.d/20261021_120000_napetrov_run-plan-result.md create mode 100644 dokimi/dokimi/runner/plan.py create mode 100644 dokimi/dokimi/schemas/plan.v1.json create mode 100644 dokimi/dokimi/schemas/result.v1.json create mode 100644 dokimi/tests/fixtures/artifacts/plan.v1.json create mode 100644 dokimi/tests/fixtures/artifacts/result.v1.json create mode 100644 dokimi/tests/test_run_plan.py diff --git a/dokimi/changelog.d/20261021_120000_napetrov_run-plan-result.md b/dokimi/changelog.d/20261021_120000_napetrov_run-plan-result.md new file mode 100644 index 000000000..9ec71722c --- /dev/null +++ b/dokimi/changelog.d/20261021_120000_napetrov_run-plan-result.md @@ -0,0 +1,7 @@ +### Added + +- **Run plan and result artifacts** — two new artifact kinds, `plan.v1` and `result.v1`. A plan lists every cell a run will measure (arm × model × task × rep, each with a stable `cell_id`) and records the subjects, suites, models, harnesses, grader digest, budget estimate and the one `varied_variable` (a list for a 2×2 design). A result gives each cell one status: `pass`, `fail`, `inconclusive`, `invalid`, `infra_error` or `not_run`, and refers to its plan by `plan_digest`. `dokimi.runner.plan.completeness` computes planned minus completed cells. A run is `complete` only when every planned cell is `pass`, `fail` or `inconclusive`. A result for a different plan, or one with unplanned or duplicated cells, is `invalid`. + +### Changed + +- **External-catalog smoke gate** — `scripts/check_smoke_runs.py` now declares its planned cells and checks them with `completeness`. A planned cell missing from `runs.json` fails the gate even when every count in the summary is clean. diff --git a/dokimi/dokimi/artifacts.py b/dokimi/dokimi/artifacts.py index e954c7360..aa0b71ab0 100644 --- a/dokimi/dokimi/artifacts.py +++ b/dokimi/dokimi/artifacts.py @@ -9,8 +9,9 @@ localization results), ``novelty`` (what an artifact's content adds to one model), ``guidance_reach`` (L1 reach probes), ``repo_readiness`` (L0 claim checks), ``tool_preference`` (which of two equal tools a model picks, and -what moves it) and ``repo_environment`` (static inspection of what a -repository offers an agent). This module stamps each with a +what moves it), ``repo_environment`` (static inspection of what a +repository offers an agent), ``plan`` (the cells a run promises to measure) and +``result`` (one status per planned cell). This module stamps each with a ``schema_version`` and validates it against the JSON Schema in ``dokimi/schemas/`` on save and load, so fixtures and baselines stay forward-compatible and self-describing. @@ -50,6 +51,8 @@ "repo_readiness": "repo_readiness.v1", "tool_preference": "tool_preference.v1", "repo_environment": "repo_environment.v1", + "plan": "plan.v1", + "result": "result.v1", } # kind -> superseded versions still accepted on *load*, newest first. An artifact written under an diff --git a/dokimi/dokimi/runner/plan.py b/dokimi/dokimi/runner/plan.py new file mode 100644 index 000000000..ed9a32f09 --- /dev/null +++ b/dokimi/dokimi/runner/plan.py @@ -0,0 +1,113 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Planned cells versus completed cells: the ``plan.v1`` / ``result.v1`` pair. + +A plan lists every cell (arm x model x task x rep) a run promises to measure; a result +gives each cell one status. :func:`completeness` is the one place that decides whether +a run is complete, computed as planned minus completed. A run that finishes 2 of its 6 +planned cells is ``incomplete`` however well those two did, and a result that points at +another plan (digest mismatch) or names cells the plan does not have is ``invalid``. + +Everything here is pure: no file, model or process is touched. Reading and writing the +artifacts goes through :mod:`dokimi.artifacts` (kinds ``plan`` and ``result``). +""" + +from __future__ import annotations + +import hashlib +import itertools +import json +from collections import Counter +from dataclasses import dataclass, field +from typing import Any, Iterable, Mapping + +#: Statuses that are a measurement of the cell. ``invalid``, ``infra_error`` and +#: ``not_run`` are recorded, but the cell still counts as missing. +MEASURED = frozenset({"pass", "fail", "inconclusive"}) +STATUSES = ("pass", "fail", "inconclusive", "invalid", "infra_error", "not_run") + + +def cell_id(arm: str, model: str | None, task: str, rep: int) -> str: + """Stable id of one cell: the same four coordinates give the same id in every plan.""" + key = json.dumps([arm, model, task, int(rep)], separators=(",", ":"), ensure_ascii=True) + return "c-" + hashlib.sha256(key.encode("utf-8")).hexdigest()[:16] + + +def plan_cells(arms: Iterable[str], models: Iterable[str | None], tasks: Iterable[str], + reps: int = 1) -> list[dict[str, Any]]: + """The full arm x model x task x rep grid, in that nesting order, each with its id.""" + if reps < 1: + raise ValueError(f"reps must be at least 1, got {reps}") + return [ + {"cell_id": cell_id(arm, model, task, rep), "arm": arm, "model": model, "task": task, "rep": rep} + for arm, model, task, rep in itertools.product(list(arms), list(models), list(tasks), + range(1, reps + 1)) + ] + + +def plan_digest(plan: Mapping[str, Any]) -> str: + """``sha256:`` of the plan's canonical JSON, ``generated_at`` left out.""" + body = {k: v for k, v in plan.items() if k != "generated_at"} + text = json.dumps(body, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + return "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest() + + +@dataclass(frozen=True) +class Completeness: + """What :func:`completeness` found. ``state`` is ``complete``, ``incomplete`` or ``invalid``.""" + + state: str + planned: int + measured: int + missing: tuple[str, ...] + statuses: dict[str, int] + problems: tuple[str, ...] = field(default=()) + + @property + def complete(self) -> bool: + return self.state == "complete" + + def as_dict(self) -> dict[str, Any]: + return {"state": self.state, "planned": self.planned, "measured": self.measured, + "missing": list(self.missing), "statuses": dict(self.statuses), + "problems": list(self.problems)} + + +def completeness(plan: Mapping[str, Any], result: Mapping[str, Any]) -> Completeness: + """Planned minus completed cells of ``result`` against ``plan``. + + - ``invalid``: the result's ``plan_digest`` is not this plan's, the plan repeats a cell + id, or the result names a cell the plan lacks or gives one cell two statuses. + - ``incomplete``: some planned cell has no measured status (absent, ``not_run``, + ``infra_error`` or ``invalid``); those ids are in ``missing``, in plan order. + - ``complete``: every planned cell is ``pass``, ``fail`` or ``inconclusive``. Complete + says nothing about whether the run passed; that is a gate's call. + """ + planned = [c["cell_id"] for c in plan.get("cells", [])] + problems: list[str] = [] + expected, actual = plan_digest(plan), result.get("plan_digest") + if actual != expected: + problems.append(f"result is for plan {actual}, not {expected}") + dup_planned = sorted(k for k, n in Counter(planned).items() if n > 1) + if dup_planned: + problems.append(f"plan repeats cell ids: {dup_planned}") + seen = Counter(c["cell_id"] for c in result.get("cells", [])) + dup_result = sorted(k for k, n in seen.items() if n > 1) + if dup_result: + problems.append(f"result gives more than one status to cells: {dup_result}") + unplanned = sorted(set(seen) - set(planned)) + if unplanned: + problems.append(f"result has cells the plan does not: {unplanned}") + status = {c["cell_id"]: c["status"] for c in result.get("cells", [])} + counts = Counter(status.get(cid, "not_run") for cid in dict.fromkeys(planned)) + missing = tuple(cid for cid in dict.fromkeys(planned) if status.get(cid, "not_run") not in MEASURED) + if problems: + state = "invalid" + elif missing or not planned: + state = "incomplete" + else: + state = "complete" + return Completeness(state=state, planned=len(set(planned)), measured=len(set(planned)) - len(missing), + missing=missing, statuses={s: counts[s] for s in STATUSES if counts[s]}, + problems=tuple(problems)) diff --git a/dokimi/dokimi/schemas/plan.v1.json b/dokimi/dokimi/schemas/plan.v1.json new file mode 100644 index 000000000..361c4285f --- /dev/null +++ b/dokimi/dokimi/schemas/plan.v1.json @@ -0,0 +1,87 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/intel/dokimi/schemas/plan.v1.json", + "title": "Run plan v1", + "description": "What a run is going to measure, declared before any paid call: the subjects and suites it reads, the models, harnesses and grader it uses, the one variable its arms vary, and every cell (arm x model x task x rep) it promises to run. A result.v1 points back at a plan by plan_digest (dokimi.runner.plan.plan_digest); completeness is computed from the plan's cells, never from what happened to finish.", + "type": "object", + "required": ["schema_version", "subjects", "suites", "models", "harnesses", "grader", "varied_variable", "cells", + "budget"], + "additionalProperties": false, + "properties": { + "schema_version": {"const": "plan.v1"}, + "generated_at": {"type": "string", "description": "Left out of plan_digest: re-stamping a plan does not change it."}, + "subjects": { + "type": "array", + "description": "What is under test: a skill, repository, doc source or profile, pinned by commit and content.", + "items": { + "type": "object", + "required": ["id", "sha", "fingerprint"], + "properties": { + "id": {"type": "string", "minLength": 1}, + "kind": {"type": "string"}, + "sha": {"type": ["string", "null"], "description": "commit the subject was read at; null when it is not in git"}, + "fingerprint": {"type": ["string", "null"], "description": "digest of the subject's content as the run sees it"} + } + } + }, + "suites": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["id", "hash", "reasons"], + "properties": { + "id": {"type": "string", "minLength": 1}, + "hash": {"type": "string", "minLength": 1, "description": "digest of the suite's task/question content"}, + "reasons": {"type": "array", "items": {"type": "string"}, + "description": "why this suite was selected (changed file, label, policy)"} + } + } + }, + "models": {"type": "array", "minItems": 1, "items": {"type": ["string", "null"]}, + "description": "requested model ids; null for a cell that calls no model (an oracle or probe harness)"}, + "harnesses": {"type": "array", "minItems": 1, "items": {"type": "string", "minLength": 1}}, + "grader": { + "type": "object", + "required": ["id", "digest"], + "properties": { + "id": {"type": "string", "minLength": 1}, + "digest": {"type": ["string", "null"], "description": "digest of the verifier or judge prompt the run grades with"} + } + }, + "varied_variable": { + "description": "The one thing the arms differ in. A list names a factorial design (2x2: code and guidance), not several free variables.", + "oneOf": [ + {"type": "string", "minLength": 1}, + {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string", "minLength": 1}} + ] + }, + "cells": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["cell_id", "arm", "model", "task", "rep"], + "additionalProperties": false, + "properties": { + "cell_id": {"type": "string", "minLength": 1, + "description": "dokimi.runner.plan.cell_id(arm, model, task, rep): stable across re-plans"}, + "arm": {"type": "string", "minLength": 1}, + "model": {"type": ["string", "null"]}, + "task": {"type": "string", "minLength": 1}, + "rep": {"type": "integer", "minimum": 1} + } + } + }, + "budget": { + "type": "object", + "required": ["cells"], + "properties": { + "cells": {"type": "integer", "minimum": 0}, + "agent_calls": {"type": ["integer", "null"], "minimum": 0}, + "cost_usd": {"type": ["number", "null"], "minimum": 0}, + "wall_sec": {"type": ["number", "null"], "minimum": 0} + } + } + } +} diff --git a/dokimi/dokimi/schemas/result.v1.json b/dokimi/dokimi/schemas/result.v1.json new file mode 100644 index 000000000..495858cca --- /dev/null +++ b/dokimi/dokimi/schemas/result.v1.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/intel/dokimi/schemas/result.v1.json", + "title": "Run result v1", + "description": "One status per cell of the plan.v1 the run was given (plan_digest). A planned cell absent here is not_run; dokimi.runner.plan.completeness computes planned minus completed and refuses to call a run complete while any planned cell lacks a measured status (pass, fail or inconclusive).", + "type": "object", + "required": ["schema_version", "plan_digest", "cells"], + "additionalProperties": false, + "properties": { + "schema_version": {"const": "result.v1"}, + "generated_at": {"type": "string"}, + "plan_digest": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"}, + "execution_id": {"type": ["string", "null"], "description": "the run's execution id, when the producer has one"}, + "cells": { + "type": "array", + "items": { + "type": "object", + "required": ["cell_id", "status"], + "properties": { + "cell_id": {"type": "string", "minLength": 1}, + "status": {"enum": ["pass", "fail", "inconclusive", "invalid", "infra_error", "not_run"]}, + "model": {"type": ["string", "null"], "description": "the model that answered (resolved), which may differ from the planned id"}, + "detail": {"type": ["string", "null"]} + } + } + } + } +} diff --git a/dokimi/scripts/check_smoke_runs.py b/dokimi/scripts/check_smoke_runs.py index 509120488..48dfe09ce 100755 --- a/dokimi/scripts/check_smoke_runs.py +++ b/dokimi/scripts/check_smoke_runs.py @@ -10,6 +10,9 @@ - every cell ran and passed: ``n > 0``, ``passed == n`` and no errored cell. ``pass_rate`` alone is not enough, because an errored cell is left out of its denominator (one pass + one errored cell reads ``pass_rate == 1.0``); +- every planned cell (each harness on :data:`TASKS`, once) has a measured + status, computed by :func:`dokimi.runner.plan.completeness`: a cell missing + from the artifact fails the gate however well the others did; - each ``docker-probe`` row shows the egress proxy denying ``example.com`` both as a CONNECT (``:443``) and as a plain-HTTP GET (``:80``, a separate code path): "closed" alone also fits a sidecar nobody reached. @@ -22,9 +25,32 @@ import json import sys +from pathlib import Path from typing import Any +PROJECT_ROOT = Path(__file__).resolve().parents[1] +if str(PROJECT_ROOT) not in sys.path: # run as a script from CI, not installed + sys.path.insert(0, str(PROJECT_ROOT)) + +from dokimi.harnesses.runner import is_errored +from dokimi.runner.plan import cell_id, completeness, plan_cells, plan_digest + HARNESSES = ("docker-oracle", "docker-probe") +#: The task the workflow runs (``tasks run --tasks`` in agent-quality.yml). +TASKS = ("onetbb-histogram-local-aggregation",) + + +def planned() -> dict[str, Any]: + """The cells this gate expects; the oracle and probe call no model.""" + return {"cells": plan_cells(HARNESSES, [None], TASKS)} + + +def result_for(plan: dict[str, Any], data: dict[str, Any]) -> dict[str, Any]: + """The ``task_runs.v1`` rows as result cells of ``plan``.""" + cells = [{"cell_id": cell_id(row["harness"], None, row.get("task", ""), row.get("repeat", 1)), + "status": "infra_error" if is_errored(row) else "pass" if row.get("passed") else "fail"} + for row in data["results"]] + return {"plan_digest": plan_digest(plan), "cells": cells} def harness_ok(stats: dict[str, Any]) -> bool: @@ -39,6 +65,15 @@ def smoke_ok(data: dict[str, Any], out=sys.stdout) -> bool: print(f"{harness}: {stats['passed']}/{stats['n']} pass, " f"errored={stats.get('errored', 0)} (rate={stats.get('pass_rate')})", file=out) ok = harness_ok(stats) and ok + plan = planned() + found = completeness(plan, result_for(plan, data)) + print(f"cells: {found.measured}/{found.planned} measured ({found.state})", file=out) + names = {c["cell_id"]: f"{c['arm']} on {c['task']}" for c in plan["cells"]} + for missing in found.missing: + print(f" not measured: {names[missing]}", file=out) + for problem in found.problems: + print(f" {problem}", file=out) + ok = found.complete and ok for row in data["results"]: metrics = row.get("metrics") or {} egress = metrics.get("egress") or {} diff --git a/dokimi/tests/fixtures/artifacts/plan.v1.json b/dokimi/tests/fixtures/artifacts/plan.v1.json new file mode 100644 index 000000000..5b3087b46 --- /dev/null +++ b/dokimi/tests/fixtures/artifacts/plan.v1.json @@ -0,0 +1,82 @@ +{ + "schema_version": "plan.v1", + "generated_at": "2026-10-21T00:00:00Z", + "subjects": [ + { + "id": "skill:onetbb", + "kind": "skill", + "sha": "0123456789abcdef0123456789abcdef01234567", + "fingerprint": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ], + "suites": [ + { + "id": "onetbb-tasks", + "hash": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "reasons": [ + "SKILL.md changed" + ] + } + ], + "models": [ + "claude-sonnet-4-6" + ], + "harnesses": [ + "docker-claude" + ], + "grader": { + "id": "pytest-verifier", + "digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + }, + "varied_variable": "skill", + "cells": [ + { + "cell_id": "c-10d62354d75a76ad", + "arm": "no-skill", + "model": "claude-sonnet-4-6", + "task": "fix-build", + "rep": 1 + }, + { + "cell_id": "c-0dcdf544e6633536", + "arm": "no-skill", + "model": "claude-sonnet-4-6", + "task": "add-test", + "rep": 1 + }, + { + "cell_id": "c-76d4abb63bbedfa5", + "arm": "no-skill", + "model": "claude-sonnet-4-6", + "task": "review-pr", + "rep": 1 + }, + { + "cell_id": "c-c496602cb859e4b5", + "arm": "skill", + "model": "claude-sonnet-4-6", + "task": "fix-build", + "rep": 1 + }, + { + "cell_id": "c-c072b8552ce95b52", + "arm": "skill", + "model": "claude-sonnet-4-6", + "task": "add-test", + "rep": 1 + }, + { + "cell_id": "c-59a34a383f800169", + "arm": "skill", + "model": "claude-sonnet-4-6", + "task": "review-pr", + "rep": 1 + } + ], + "budget": { + "cells": 6, + "agent_calls": 6, + "cost_usd": 1.2, + "wall_sec": 1800 + } +} \ No newline at end of file diff --git a/dokimi/tests/fixtures/artifacts/result.v1.json b/dokimi/tests/fixtures/artifacts/result.v1.json new file mode 100644 index 000000000..0af2cdd18 --- /dev/null +++ b/dokimi/tests/fixtures/artifacts/result.v1.json @@ -0,0 +1,44 @@ +{ + "schema_version": "result.v1", + "generated_at": "2026-10-21T01:00:00Z", + "plan_digest": "sha256:6bbd76a2642039efd7b9d8fee6ffe85ec785ee3ec0985f4fc0fcd56c521f2348", + "execution_id": null, + "cells": [ + { + "cell_id": "c-10d62354d75a76ad", + "status": "pass", + "model": "claude-sonnet-4-6", + "detail": null + }, + { + "cell_id": "c-0dcdf544e6633536", + "status": "fail", + "model": "claude-sonnet-4-6", + "detail": null + }, + { + "cell_id": "c-76d4abb63bbedfa5", + "status": "inconclusive", + "model": "claude-sonnet-4-6", + "detail": null + }, + { + "cell_id": "c-c496602cb859e4b5", + "status": "invalid", + "model": "claude-sonnet-4-6", + "detail": null + }, + { + "cell_id": "c-c072b8552ce95b52", + "status": "infra_error", + "model": "claude-sonnet-4-6", + "detail": null + }, + { + "cell_id": "c-59a34a383f800169", + "status": "not_run", + "model": "claude-sonnet-4-6", + "detail": null + } + ] +} \ No newline at end of file diff --git a/dokimi/tests/test_check_smoke_runs.py b/dokimi/tests/test_check_smoke_runs.py index 10e28f6e6..8af214dc3 100644 --- a/dokimi/tests/test_check_smoke_runs.py +++ b/dokimi/tests/test_check_smoke_runs.py @@ -17,6 +17,9 @@ _spec.loader.exec_module(csr) +_TASK = "onetbb-histogram-local-aggregation" + + def _runs(oracle=None, probe=None, denied=("example.com:443", "example.com:80")) -> dict: clean = {"n": 1, "passed": 1, "errored": 0, "pass_rate": 1.0} return { @@ -25,8 +28,9 @@ def _runs(oracle=None, probe=None, denied=("example.com:443", "example.com:80")) "docker-probe": probe or clean, }}, "results": [ - {"harness": "docker-oracle", "metrics": {}}, - {"harness": "docker-probe", "metrics": {"egress": {"denied_entries": list(denied)}}}, + {"harness": "docker-oracle", "task": _TASK, "passed": True, "metrics": {}}, + {"harness": "docker-probe", "task": _TASK, "passed": True, + "metrics": {"egress": {"denied_entries": list(denied)}}}, ], } @@ -79,3 +83,29 @@ def test_an_unreadable_sidecar_log_fails_even_when_every_cell_passed(): def test_usage_without_a_path(): assert csr.main([]) == 2 + + +def test_a_planned_cell_missing_from_the_artifact_fails_with_every_count_clean(): + """The 2-of-6 gap: per-harness stats can be all-pass while a planned cell never ran.""" + data = _runs() + data["results"] = [r for r in data["results"] if r["harness"] != "docker-oracle"] + out = io.StringIO() + assert not csr.smoke_ok(data, out=out) + assert "1/2 measured (incomplete)" in out.getvalue() + assert "not measured: docker-oracle on " + _TASK in out.getvalue() + + +def test_a_row_for_a_task_nobody_planned_fails(): + data = _runs() + data["results"].append({"harness": "docker-oracle", "task": "other-task", "passed": True, "metrics": {}}) + out = io.StringIO() + assert not csr.smoke_ok(data, out=out) + assert "(invalid)" in out.getvalue() + + +def test_an_errored_row_is_an_unmeasured_cell(): + data = _runs() + data["results"][0]["metrics"] = {"error": "docker daemon gone"} + out = io.StringIO() + assert not csr.smoke_ok(data, out=out) + assert "1/2 measured (incomplete)" in out.getvalue() diff --git a/dokimi/tests/test_run_plan.py b/dokimi/tests/test_run_plan.py new file mode 100644 index 000000000..87278b625 --- /dev/null +++ b/dokimi/tests/test_run_plan.py @@ -0,0 +1,111 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""plan.v1 / result.v1 and the completeness check that refuses a partial run.""" + +from __future__ import annotations + +import copy + +import pytest + +from dokimi.artifacts import ArtifactValidationError, load_artifact, save_artifact +from dokimi.runner.plan import cell_id, completeness, plan_cells, plan_digest + + +def _plan(tmp_path, **over): + cells = plan_cells(["no-skill", "skill"], ["m1"], ["t1", "t2", "t3"]) + plan = {"subjects": [{"id": "skill:x", "sha": None, "fingerprint": None}], + "suites": [{"id": "s", "hash": "sha256:00", "reasons": ["SKILL.md changed"]}], + "models": ["m1"], "harnesses": ["docker-claude"], "grader": {"id": "pytest", "digest": None}, + "varied_variable": "skill", "cells": cells, "budget": {"cells": len(cells)}, **over} + return save_artifact("plan", plan, tmp_path / "plan.json") + + +def _result(plan, statuses, **over): + cells = [{"cell_id": c["cell_id"], "status": s} for c, s in zip(plan["cells"], statuses, strict=True) if s] + return {"plan_digest": plan_digest(plan), "cells": cells, **over} + + +def test_cell_ids_are_stable_and_distinct_per_coordinate(): + assert cell_id("a", "m", "t", 1) == cell_id("a", "m", "t", 1) + ids = {cell_id(*c) for c in [("a", "m", "t", 1), ("a", "m", "t", 2), ("a", None, "t", 1), + ("b", "m", "t", 1), ("a", "m", "u", 1)]} + assert len(ids) == 5 + + +def test_plan_cells_is_the_full_grid(): + cells = plan_cells(["a", "b"], ["m1", "m2"], ["t1", "t2", "t3"], reps=3) + assert len(cells) == 36 + assert len({c["cell_id"] for c in cells}) == 36 + assert cells[1] == {"cell_id": cell_id("a", "m1", "t1", 2), "arm": "a", "model": "m1", "task": "t1", "rep": 2} + with pytest.raises(ValueError): + plan_cells(["a"], ["m"], ["t"], reps=0) + + +def test_two_of_six_cells_is_incomplete_even_when_both_pass(tmp_path): + plan = _plan(tmp_path) + found = completeness(plan, _result(plan, ["pass", "pass", None, None, None, None])) + assert (found.state, found.planned, found.measured) == ("incomplete", 6, 2) + assert found.missing == tuple(c["cell_id"] for c in plan["cells"][2:]) + assert found.statuses == {"pass": 2, "not_run": 4} + assert not found.complete + + +@pytest.mark.parametrize("unmeasured", ["not_run", "infra_error", "invalid"]) +def test_a_recorded_but_unmeasured_cell_keeps_the_run_incomplete(tmp_path, unmeasured): + plan = _plan(tmp_path) + found = completeness(plan, _result(plan, ["pass"] * 5 + [unmeasured])) + assert found.state == "incomplete" + assert found.missing == (plan["cells"][5]["cell_id"],) + + +def test_every_cell_measured_is_complete_whatever_the_verdicts(tmp_path): + plan = _plan(tmp_path) + found = completeness(plan, _result(plan, ["pass", "fail", "inconclusive", "fail", "fail", "pass"])) + assert found.complete + assert found.as_dict()["statuses"] == {"pass": 2, "fail": 3, "inconclusive": 1} + + +def test_a_result_for_another_plan_is_invalid(tmp_path): + plan = _plan(tmp_path) + result = _result(plan, ["pass"] * 6) + edited = copy.deepcopy(plan) + edited["grader"]["digest"] = "sha256:ff" # the grader changed after the run + found = completeness(edited, result) + assert found.state == "invalid" + assert "not " + plan_digest(edited) in found.problems[0] + + +def test_the_digest_ignores_generated_at_only(tmp_path): + plan = _plan(tmp_path) + assert plan_digest({**plan, "generated_at": "later"}) == plan_digest(plan) + assert plan_digest({**plan, "varied_variable": ["code", "guidance"]}) != plan_digest(plan) + + +def test_unplanned_or_doubled_cells_are_invalid(tmp_path): + plan = _plan(tmp_path) + extra = _result(plan, ["pass"] * 6) + extra["cells"].append({"cell_id": cell_id("other", "m1", "t1", 1), "status": "pass"}) + assert completeness(plan, extra).state == "invalid" + doubled = _result(plan, ["pass"] * 6) + doubled["cells"].append({"cell_id": plan["cells"][0]["cell_id"], "status": "fail"}) + assert "more than one status" in completeness(plan, doubled).problems[0] + + +def test_artifacts_round_trip_and_reject_an_unknown_status(tmp_path): + plan = _plan(tmp_path) + assert load_artifact("plan", tmp_path / "plan.json") == plan + result = _result(plan, ["pass"] * 6, execution_id="exec-1") + save_artifact("result", result, tmp_path / "result.json") + assert completeness(load_artifact("plan", tmp_path / "plan.json"), + load_artifact("result", tmp_path / "result.json")).complete + result["cells"][0]["status"] = "skipped" + with pytest.raises(ArtifactValidationError): + save_artifact("result", result, tmp_path / "bad.json") + + +def test_a_plan_with_two_free_variables_needs_the_tuple_form(tmp_path): + _plan(tmp_path, varied_variable=["code", "guidance"]) + with pytest.raises(ArtifactValidationError): + _plan(tmp_path, varied_variable=[]) From 7852366285405de18b87a6a0afd6c605bdbfb31e Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Thu, 22 Oct 2026 12:59:34 +0000 Subject: [PATCH 02/17] Close subject, suite, grader, budget and result cell objects in schemas Signed-off-by: Anton Volkov --- dokimi/dokimi/schemas/plan.v1.json | 4 ++++ dokimi/dokimi/schemas/result.v1.json | 1 + 2 files changed, 5 insertions(+) diff --git a/dokimi/dokimi/schemas/plan.v1.json b/dokimi/dokimi/schemas/plan.v1.json index 361c4285f..57bd6c10c 100644 --- a/dokimi/dokimi/schemas/plan.v1.json +++ b/dokimi/dokimi/schemas/plan.v1.json @@ -16,6 +16,7 @@ "items": { "type": "object", "required": ["id", "sha", "fingerprint"], + "additionalProperties": false, "properties": { "id": {"type": "string", "minLength": 1}, "kind": {"type": "string"}, @@ -30,6 +31,7 @@ "items": { "type": "object", "required": ["id", "hash", "reasons"], + "additionalProperties": false, "properties": { "id": {"type": "string", "minLength": 1}, "hash": {"type": "string", "minLength": 1, "description": "digest of the suite's task/question content"}, @@ -44,6 +46,7 @@ "grader": { "type": "object", "required": ["id", "digest"], + "additionalProperties": false, "properties": { "id": {"type": "string", "minLength": 1}, "digest": {"type": ["string", "null"], "description": "digest of the verifier or judge prompt the run grades with"} @@ -76,6 +79,7 @@ "budget": { "type": "object", "required": ["cells"], + "additionalProperties": false, "properties": { "cells": {"type": "integer", "minimum": 0}, "agent_calls": {"type": ["integer", "null"], "minimum": 0}, diff --git a/dokimi/dokimi/schemas/result.v1.json b/dokimi/dokimi/schemas/result.v1.json index 495858cca..4c3db0c7e 100644 --- a/dokimi/dokimi/schemas/result.v1.json +++ b/dokimi/dokimi/schemas/result.v1.json @@ -16,6 +16,7 @@ "items": { "type": "object", "required": ["cell_id", "status"], + "additionalProperties": false, "properties": { "cell_id": {"type": "string", "minLength": 1}, "status": {"enum": ["pass", "fail", "inconclusive", "invalid", "infra_error", "not_run"]}, From eb69341417d2c5bdbacf196781f6631025d1dee8 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 22 Oct 2026 08:07:07 -0700 Subject: [PATCH 03/17] fix(runner): completeness reports malformed cells as invalid instead of raising Reads cells with .get, flags a missing cell_id or a status outside STATUSES, and reports measured=0 for an invalid result. Adds the empty-plan and repeated-cell tests and makes the unreadable-sidecar smoke test isolate the egress check again. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/runner/plan.py | 35 +++++++++++++++++--- dokimi/tests/test_check_smoke_runs.py | 5 +-- dokimi/tests/test_run_plan.py | 46 +++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 7 deletions(-) diff --git a/dokimi/dokimi/runner/plan.py b/dokimi/dokimi/runner/plan.py index ed9a32f09..c506c71cc 100644 --- a/dokimi/dokimi/runner/plan.py +++ b/dokimi/dokimi/runner/plan.py @@ -78,28 +78,52 @@ def completeness(plan: Mapping[str, Any], result: Mapping[str, Any]) -> Complete """Planned minus completed cells of ``result`` against ``plan``. - ``invalid``: the result's ``plan_digest`` is not this plan's, the plan repeats a cell - id, or the result names a cell the plan lacks or gives one cell two statuses. + id, a cell has no string ``cell_id`` or a result cell has a status outside + :data:`STATUSES`, or the result names a cell the plan lacks or gives one cell two + statuses. An ``invalid`` result reports ``measured=0``. - ``incomplete``: some planned cell has no measured status (absent, ``not_run``, ``infra_error`` or ``invalid``); those ids are in ``missing``, in plan order. - ``complete``: every planned cell is ``pass``, ``fail`` or ``inconclusive``. Complete says nothing about whether the run passed; that is a gate's call. """ - planned = [c["cell_id"] for c in plan.get("cells", [])] problems: list[str] = [] + + def _ids(side: str, cells: Any) -> list[str]: + """The string ``cell_id`` of each cell; a cell without one is a problem, not a crash.""" + ids: list[str] = [] + for cell in cells if isinstance(cells, list) else []: + cid = cell.get("cell_id") if isinstance(cell, Mapping) else None + if isinstance(cid, str): + ids.append(cid) + else: + problems.append(f"{side} has a cell without a string cell_id: {cell!r}") + return ids + + planned = _ids("plan", plan.get("cells", [])) expected, actual = plan_digest(plan), result.get("plan_digest") if actual != expected: problems.append(f"result is for plan {actual}, not {expected}") dup_planned = sorted(k for k, n in Counter(planned).items() if n > 1) if dup_planned: problems.append(f"plan repeats cell ids: {dup_planned}") - seen = Counter(c["cell_id"] for c in result.get("cells", [])) + result_cells = result.get("cells", []) + seen = Counter(_ids("result", result_cells)) dup_result = sorted(k for k, n in seen.items() if n > 1) if dup_result: problems.append(f"result gives more than one status to cells: {dup_result}") unplanned = sorted(set(seen) - set(planned)) if unplanned: problems.append(f"result has cells the plan does not: {unplanned}") - status = {c["cell_id"]: c["status"] for c in result.get("cells", [])} + status: dict[str, str] = {} + for cell in result_cells if isinstance(result_cells, list) else []: + cid = cell.get("cell_id") if isinstance(cell, Mapping) else None + st = cell.get("status") if isinstance(cell, Mapping) else None + if not isinstance(cid, str): + continue # reported above + if isinstance(st, str) and st in STATUSES: + status[cid] = st + else: + problems.append(f"result cell {cid} has an unknown status: {st!r}") counts = Counter(status.get(cid, "not_run") for cid in dict.fromkeys(planned)) missing = tuple(cid for cid in dict.fromkeys(planned) if status.get(cid, "not_run") not in MEASURED) if problems: @@ -108,6 +132,7 @@ def completeness(plan: Mapping[str, Any], result: Mapping[str, Any]) -> Complete state = "incomplete" else: state = "complete" - return Completeness(state=state, planned=len(set(planned)), measured=len(set(planned)) - len(missing), + measured = 0 if problems else len(set(planned)) - len(missing) + return Completeness(state=state, planned=len(set(planned)), measured=measured, missing=missing, statuses={s: counts[s] for s in STATUSES if counts[s]}, problems=tuple(problems)) diff --git a/dokimi/tests/test_check_smoke_runs.py b/dokimi/tests/test_check_smoke_runs.py index 8af214dc3..001bb8c9b 100644 --- a/dokimi/tests/test_check_smoke_runs.py +++ b/dokimi/tests/test_check_smoke_runs.py @@ -76,8 +76,9 @@ def test_a_cell_that_failed_without_erroring_fails(): def test_an_unreadable_sidecar_log_fails_even_when_every_cell_passed(): """An empty egress record from a timed-out docker logs is unverified, not clean.""" data = _runs() - data["results"].append({"harness": "docker-claude", "metrics": {"egress": { - "status": "unknown", "denied_entries": []}}}) + probe = next(r for r in data["results"] if r["harness"] == "docker-probe") + probe["metrics"]["egress"] = { + "status": "unknown", "denied_entries": ["example.com:443", "example.com:80"]} assert not csr.smoke_ok(data, out=io.StringIO()) diff --git a/dokimi/tests/test_run_plan.py b/dokimi/tests/test_run_plan.py index 87278b625..5847a53c0 100644 --- a/dokimi/tests/test_run_plan.py +++ b/dokimi/tests/test_run_plan.py @@ -109,3 +109,49 @@ def test_a_plan_with_two_free_variables_needs_the_tuple_form(tmp_path): _plan(tmp_path, varied_variable=["code", "guidance"]) with pytest.raises(ArtifactValidationError): _plan(tmp_path, varied_variable=[]) + + +def test_an_empty_plan_is_incomplete_not_complete(): + empty = {"cells": []} + got = completeness(empty, {"plan_digest": plan_digest(empty), "cells": []}) + assert got.state == "incomplete" + assert not got.complete + + +def test_a_plan_that_repeats_a_cell_is_invalid(tmp_path): + plan = _plan(tmp_path) + plan["cells"] = [*plan["cells"], plan["cells"][0]] + got = completeness(plan, _result(plan, ["pass"] * 7)) + assert got.state == "invalid" + assert "plan repeats" in got.problems[0] + + +def test_planned_counts_distinct_cells_when_the_plan_repeats_one(): + cell = {"cell_id": "c-1"} + plan = {"cells": [cell, cell]} + got = completeness(plan, {"plan_digest": plan_digest(plan), "cells": []}) + assert got.planned == 1 + + +@pytest.mark.parametrize("bad", [{"cell_id": "c-1"}, {"status": "pass"}, {"cell_id": "c-1", "status": ["pass"]}, + {"cell_id": "c-1", "status": "PASS"}, {"cell_id": 3, "status": "pass"}]) +def test_a_malformed_result_cell_is_invalid_not_an_exception(tmp_path, bad): + plan = _plan(tmp_path) + result = _result(plan, ["pass"] * 6) + result["cells"].append(bad) + got = completeness(plan, result) + assert got.state == "invalid" + assert got.problems + + +def test_a_plan_cell_without_an_id_is_invalid(): + plan = {"cells": [{"arm": "a"}]} + got = completeness(plan, {"plan_digest": plan_digest(plan), "cells": []}) + assert got.state == "invalid" + + +def test_an_invalid_result_reports_no_measured_cells(tmp_path): + plan = _plan(tmp_path) + got = completeness(plan, _result(plan, ["pass"] * 6, plan_digest="sha256:other")) + assert got.state == "invalid" + assert got.measured == 0 From 266a0aa09ab06082d2b7c57b9129e05a30cf6eba Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Fri, 23 Oct 2026 00:43:19 -0700 Subject: [PATCH 04/17] refactor(runner): move the result-status loop out of completeness Merged with current main, completeness() has complexity 13, over the growth ratchet's cap of 10 (tests/test_growth_ratchet.py fails). The status loop becomes _statuses(); behaviour and problem order are unchanged. Signed-off-by: Alexander Rybkin --- dokimi/dokimi/runner/plan.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/dokimi/dokimi/runner/plan.py b/dokimi/dokimi/runner/plan.py index c506c71cc..2a5b57290 100644 --- a/dokimi/dokimi/runner/plan.py +++ b/dokimi/dokimi/runner/plan.py @@ -74,6 +74,21 @@ def as_dict(self) -> dict[str, Any]: "problems": list(self.problems)} +def _statuses(result_cells: Any, problems: list[str]) -> dict[str, str]: + """``cell_id -> status`` of the result; an unknown status is appended to ``problems``.""" + status: dict[str, str] = {} + for cell in result_cells if isinstance(result_cells, list) else []: + cid = cell.get("cell_id") if isinstance(cell, Mapping) else None + st = cell.get("status") if isinstance(cell, Mapping) else None + if not isinstance(cid, str): + continue # reported by completeness + if isinstance(st, str) and st in STATUSES: + status[cid] = st + else: + problems.append(f"result cell {cid} has an unknown status: {st!r}") + return status + + def completeness(plan: Mapping[str, Any], result: Mapping[str, Any]) -> Completeness: """Planned minus completed cells of ``result`` against ``plan``. @@ -114,16 +129,7 @@ def _ids(side: str, cells: Any) -> list[str]: unplanned = sorted(set(seen) - set(planned)) if unplanned: problems.append(f"result has cells the plan does not: {unplanned}") - status: dict[str, str] = {} - for cell in result_cells if isinstance(result_cells, list) else []: - cid = cell.get("cell_id") if isinstance(cell, Mapping) else None - st = cell.get("status") if isinstance(cell, Mapping) else None - if not isinstance(cid, str): - continue # reported above - if isinstance(st, str) and st in STATUSES: - status[cid] = st - else: - problems.append(f"result cell {cid} has an unknown status: {st!r}") + status = _statuses(result_cells, problems) counts = Counter(status.get(cid, "not_run") for cid in dict.fromkeys(planned)) missing = tuple(cid for cid in dict.fromkeys(planned) if status.get(cid, "not_run") not in MEASURED) if problems: From 043179cad2fc74c5113f3ab5f2bc74e00a4e138f Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Fri, 23 Oct 2026 00:43:49 -0700 Subject: [PATCH 05/17] test(runner): pin the malformed-cell guards of completeness The existing malformed-cell cases use an unplanned id, so the unplanned check makes them invalid and reverting any guard keeps the suite green. Put the bad status on a planned cell, assert the problem text, and cover a non-object cell and a non-list cells value. Signed-off-by: Alexander Rybkin --- dokimi/tests/test_run_plan.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/dokimi/tests/test_run_plan.py b/dokimi/tests/test_run_plan.py index 5847a53c0..96e5fa010 100644 --- a/dokimi/tests/test_run_plan.py +++ b/dokimi/tests/test_run_plan.py @@ -155,3 +155,28 @@ def test_an_invalid_result_reports_no_measured_cells(tmp_path): got = completeness(plan, _result(plan, ["pass"] * 6, plan_digest="sha256:other")) assert got.state == "invalid" assert got.measured == 0 + + +@pytest.mark.parametrize("bad", [{"status": "PASS"}, {"status": ["pass"]}, {}]) +def test_a_bad_status_on_a_planned_cell_is_invalid(tmp_path, bad): + plan = _plan(tmp_path) + result = _result(plan, ["pass"] * 6) + result["cells"][0] = {"cell_id": plan["cells"][0]["cell_id"], **bad} + got = completeness(plan, result) + assert (got.state, got.measured) == ("invalid", 0) + assert any("unknown status" in p for p in got.problems) + + +@pytest.mark.parametrize("bad", ["junk", {"cell_id": 3, "status": "pass"}]) +def test_a_result_cell_without_a_string_id_is_reported(tmp_path, bad): + plan = _plan(tmp_path) + result = _result(plan, ["pass"] * 6) + result["cells"].append(bad) + got = completeness(plan, result) + assert got.state == "invalid" + assert any("without a string cell_id" in p for p in got.problems) + + +def test_result_cells_that_are_not_a_list_do_not_raise(tmp_path): + plan = _plan(tmp_path) + assert not completeness(plan, {"plan_digest": plan_digest(plan), "cells": None}).complete From d446c51b847c9115fcefd6ad9bae463798d57dfe Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 23 Oct 2026 12:38:50 -0700 Subject: [PATCH 06/17] test(runner): pin the list cell_id guard and the :443 half of the egress gate Signed-off-by: Nikolay Petrov --- dokimi/tests/test_check_smoke_runs.py | 5 +++++ dokimi/tests/test_run_plan.py | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/dokimi/tests/test_check_smoke_runs.py b/dokimi/tests/test_check_smoke_runs.py index 001bb8c9b..b4ba5cc56 100644 --- a/dokimi/tests/test_check_smoke_runs.py +++ b/dokimi/tests/test_check_smoke_runs.py @@ -68,6 +68,11 @@ def test_a_probe_whose_plain_http_get_was_not_denied_fails(): assert not csr.smoke_ok(_runs(denied=("example.com:443",)), out=io.StringIO()) +def test_a_probe_whose_connect_was_not_denied_fails(): + """A CONNECT path that skips the allowlist leaves no :443 denial.""" + assert not csr.smoke_ok(_runs(denied=("example.com:80",)), out=io.StringIO()) + + def test_a_cell_that_failed_without_erroring_fails(): assert not csr.smoke_ok(_runs(oracle={"n": 2, "passed": 1, "errored": 0, "pass_rate": 0.5}), out=io.StringIO()) diff --git a/dokimi/tests/test_run_plan.py b/dokimi/tests/test_run_plan.py index 96e5fa010..6c58b12d8 100644 --- a/dokimi/tests/test_run_plan.py +++ b/dokimi/tests/test_run_plan.py @@ -167,7 +167,8 @@ def test_a_bad_status_on_a_planned_cell_is_invalid(tmp_path, bad): assert any("unknown status" in p for p in got.problems) -@pytest.mark.parametrize("bad", ["junk", {"cell_id": 3, "status": "pass"}]) +@pytest.mark.parametrize("bad", ["junk", {"cell_id": 3, "status": "pass"}, + {"cell_id": ["c-1"], "status": "pass"}]) def test_a_result_cell_without_a_string_id_is_reported(tmp_path, bad): plan = _plan(tmp_path) result = _result(plan, ["pass"] * 6) From d9c464e8a9f7b760f6f8cafef07734be88e6b715 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 26 Oct 2026 13:12:21 +0000 Subject: [PATCH 07/17] Test that the plan digest covers the cells Signed-off-by: Anton Volkov --- dokimi/tests/test_run_plan.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dokimi/tests/test_run_plan.py b/dokimi/tests/test_run_plan.py index 6c58b12d8..6eba47f35 100644 --- a/dokimi/tests/test_run_plan.py +++ b/dokimi/tests/test_run_plan.py @@ -81,6 +81,7 @@ def test_the_digest_ignores_generated_at_only(tmp_path): plan = _plan(tmp_path) assert plan_digest({**plan, "generated_at": "later"}) == plan_digest(plan) assert plan_digest({**plan, "varied_variable": ["code", "guidance"]}) != plan_digest(plan) + assert plan_digest({**plan, "cells": plan["cells"][:-1]}) != plan_digest(plan) def test_unplanned_or_doubled_cells_are_invalid(tmp_path): From aafab7c250b3b068c8a613457f31ee504187d128 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 26 Oct 2026 13:12:23 +0000 Subject: [PATCH 08/17] Test that the smoke gate prints the completeness problems Signed-off-by: Anton Volkov --- dokimi/tests/test_check_smoke_runs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dokimi/tests/test_check_smoke_runs.py b/dokimi/tests/test_check_smoke_runs.py index b4ba5cc56..5218a24d6 100644 --- a/dokimi/tests/test_check_smoke_runs.py +++ b/dokimi/tests/test_check_smoke_runs.py @@ -107,6 +107,7 @@ def test_a_row_for_a_task_nobody_planned_fails(): out = io.StringIO() assert not csr.smoke_ok(data, out=out) assert "(invalid)" in out.getvalue() + assert "result has cells the plan does not" in out.getvalue() def test_an_errored_row_is_an_unmeasured_cell(): From 26426da074e0bf976915918a2c3eb6fe9ce7dc76 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 26 Oct 2026 11:40:08 -0700 Subject: [PATCH 09/17] Test that a misspelled key in a closed plan or result object is refused plan.v1 and result.v1 close the subject, suite, grader, budget and result cell objects (7852366). Nothing tested it: a typo such as fingerprnt on a subject was accepted and kept. The new test fails on the open schemas and passes on the closed ones. Signed-off-by: Nikolay Petrov --- dokimi/tests/test_run_plan.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/dokimi/tests/test_run_plan.py b/dokimi/tests/test_run_plan.py index 6eba47f35..1971b9bdf 100644 --- a/dokimi/tests/test_run_plan.py +++ b/dokimi/tests/test_run_plan.py @@ -106,6 +106,21 @@ def test_artifacts_round_trip_and_reject_an_unknown_status(tmp_path): save_artifact("result", result, tmp_path / "bad.json") +@pytest.mark.parametrize("kind, where", [ + ("plan", ("subjects", 0)), ("plan", ("suites", 0)), ("plan", ("grader",)), ("plan", ("budget",)), + ("result", ("cells", 0)), +]) +def test_a_misspelled_key_in_a_closed_object_is_refused(tmp_path, kind, where): + plan = _plan(tmp_path) + artifact = copy.deepcopy(plan if kind == "plan" else _result(plan, ["pass"] * 6)) + target = artifact + for part in where: + target = target[part] + target["fingerprnt"] = "typo" + with pytest.raises(ArtifactValidationError): + save_artifact(kind, artifact, tmp_path / "bad.json") + + def test_a_plan_with_two_free_variables_needs_the_tuple_form(tmp_path): _plan(tmp_path, varied_variable=["code", "guidance"]) with pytest.raises(ArtifactValidationError): From 294941918b94695b0428d9f2125c5fe9777d3508 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 26 Oct 2026 11:45:19 -0700 Subject: [PATCH 10/17] Report a plan or result whose cells is not a list as invalid A mapping or null in cells was read as zero cells: no problem was recorded, and every planned cell read not_run. A result written as {cell_id: status} gave a plain incomplete with no reason. Both sides now report a problem and the state is invalid. The cell-id helper moves to module level, as _statuses already is, to keep completeness under the complexity cap. Test: test_cells_that_are_not_a_list_are_invalid_not_empty (fails on the previous plan.py). Signed-off-by: Nikolay Petrov --- dokimi/dokimi/runner/plan.py | 37 +++++++++++++++++++---------------- dokimi/tests/test_run_plan.py | 13 ++++++++++++ 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/dokimi/dokimi/runner/plan.py b/dokimi/dokimi/runner/plan.py index 2a5b57290..8dc98ce8e 100644 --- a/dokimi/dokimi/runner/plan.py +++ b/dokimi/dokimi/runner/plan.py @@ -89,32 +89,35 @@ def _statuses(result_cells: Any, problems: list[str]) -> dict[str, str]: return status +def _ids(side: str, cells: Any, problems: list[str]) -> list[str]: + """The string ``cell_id`` of each cell; a cell without one is a problem, not a crash.""" + if not isinstance(cells, list): + problems.append(f"{side} cells is not a list: {cells!r}") + return [] + ids: list[str] = [] + for cell in cells: + cid = cell.get("cell_id") if isinstance(cell, Mapping) else None + if isinstance(cid, str): + ids.append(cid) + else: + problems.append(f"{side} has a cell without a string cell_id: {cell!r}") + return ids + + def completeness(plan: Mapping[str, Any], result: Mapping[str, Any]) -> Completeness: """Planned minus completed cells of ``result`` against ``plan``. - ``invalid``: the result's ``plan_digest`` is not this plan's, the plan repeats a cell - id, a cell has no string ``cell_id`` or a result cell has a status outside - :data:`STATUSES`, or the result names a cell the plan lacks or gives one cell two - statuses. An ``invalid`` result reports ``measured=0``. + id, ``cells`` is not a list, a cell has no string ``cell_id``, or a result cell has a + status outside :data:`STATUSES`; or the result names a cell the plan lacks or gives one + cell two statuses. An ``invalid`` result reports ``measured=0``. - ``incomplete``: some planned cell has no measured status (absent, ``not_run``, ``infra_error`` or ``invalid``); those ids are in ``missing``, in plan order. - ``complete``: every planned cell is ``pass``, ``fail`` or ``inconclusive``. Complete says nothing about whether the run passed; that is a gate's call. """ problems: list[str] = [] - - def _ids(side: str, cells: Any) -> list[str]: - """The string ``cell_id`` of each cell; a cell without one is a problem, not a crash.""" - ids: list[str] = [] - for cell in cells if isinstance(cells, list) else []: - cid = cell.get("cell_id") if isinstance(cell, Mapping) else None - if isinstance(cid, str): - ids.append(cid) - else: - problems.append(f"{side} has a cell without a string cell_id: {cell!r}") - return ids - - planned = _ids("plan", plan.get("cells", [])) + planned = _ids("plan", plan.get("cells", []), problems) expected, actual = plan_digest(plan), result.get("plan_digest") if actual != expected: problems.append(f"result is for plan {actual}, not {expected}") @@ -122,7 +125,7 @@ def _ids(side: str, cells: Any) -> list[str]: if dup_planned: problems.append(f"plan repeats cell ids: {dup_planned}") result_cells = result.get("cells", []) - seen = Counter(_ids("result", result_cells)) + seen = Counter(_ids("result", result_cells, problems)) dup_result = sorted(k for k, n in seen.items() if n > 1) if dup_result: problems.append(f"result gives more than one status to cells: {dup_result}") diff --git a/dokimi/tests/test_run_plan.py b/dokimi/tests/test_run_plan.py index 1971b9bdf..66926a073 100644 --- a/dokimi/tests/test_run_plan.py +++ b/dokimi/tests/test_run_plan.py @@ -197,3 +197,16 @@ def test_a_result_cell_without_a_string_id_is_reported(tmp_path, bad): def test_result_cells_that_are_not_a_list_do_not_raise(tmp_path): plan = _plan(tmp_path) assert not completeness(plan, {"plan_digest": plan_digest(plan), "cells": None}).complete + + +@pytest.mark.parametrize("cells", [{"c-1": "pass"}, None, "pass"]) +def test_cells_that_are_not_a_list_are_invalid_not_empty(tmp_path, cells): + # A mapping or null used to read as zero cells: no problem, and every planned cell "not_run". + plan = _plan(tmp_path) + got = completeness(plan, {"plan_digest": plan_digest(plan), "cells": cells}) + assert got.state == "invalid" + assert got.problems + bad_plan = {**plan, "cells": cells} + got = completeness(bad_plan, {"plan_digest": plan_digest(bad_plan), "cells": []}) + assert got.state == "invalid" + assert got.problems From 935f6c07fb37c0f14e882ed902304a56308bbee5 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 26 Oct 2026 21:52:23 -0700 Subject: [PATCH 11/17] fix(plan): digest as saved, refuse mismatched ids and budget, no statuses on invalid - plan_digest fills a missing schema_version, so a digest taken before save_artifact matches the saved plan. - completeness refuses a plan cell whose id is not cell_id of its coordinates, and a budget.cells that is not the distinct cell count. - an invalid result reports empty statuses and every planned cell missing. - plan.v1: arm carries the harness when a plan crosses harnesses. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/runner/plan.py | 46 +++++++++++++++++++++++++----- dokimi/dokimi/schemas/plan.v1.json | 8 ++++-- dokimi/tests/test_run_plan.py | 40 ++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 10 deletions(-) diff --git a/dokimi/dokimi/runner/plan.py b/dokimi/dokimi/runner/plan.py index 8dc98ce8e..cd7e9eeca 100644 --- a/dokimi/dokimi/runner/plan.py +++ b/dokimi/dokimi/runner/plan.py @@ -25,6 +25,8 @@ #: Statuses that are a measurement of the cell. ``invalid``, ``infra_error`` and #: ``not_run`` are recorded, but the cell still counts as missing. MEASURED = frozenset({"pass", "fail", "inconclusive"}) +#: ``SCHEMA_VERSIONS["plan"]`` in :mod:`dokimi.artifacts`, which ``save_artifact`` stamps. +_PLAN_SCHEMA = "plan.v1" STATUSES = ("pass", "fail", "inconclusive", "invalid", "infra_error", "not_run") @@ -47,8 +49,14 @@ def plan_cells(arms: Iterable[str], models: Iterable[str | None], tasks: Iterabl def plan_digest(plan: Mapping[str, Any]) -> str: - """``sha256:`` of the plan's canonical JSON, ``generated_at`` left out.""" + """``sha256:`` of the plan's canonical JSON, ``generated_at`` left out. + + The plan is hashed as :func:`dokimi.artifacts.save_artifact` writes it: a missing + ``schema_version`` is filled in, so the digest of a dict taken before saving matches + the digest of the same plan loaded back. + """ body = {k: v for k, v in plan.items() if k != "generated_at"} + body.setdefault("schema_version", _PLAN_SCHEMA) text = json.dumps(body, sort_keys=True, separators=(",", ":"), ensure_ascii=True) return "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest() @@ -104,13 +112,37 @@ def _ids(side: str, cells: Any, problems: list[str]) -> list[str]: return ids +def _plan_problems(plan: Mapping[str, Any], planned: list[str], problems: list[str]) -> None: + """A repeated id, an id that does not match its coordinates, or a ``budget.cells`` that is + not the number of distinct planned cells.""" + dup_planned = sorted(k for k, n in Counter(planned).items() if n > 1) + if dup_planned: + problems.append(f"plan repeats cell ids: {dup_planned}") + for cell in plan.get("cells") or []: + if not isinstance(cell, Mapping) or not isinstance(cell.get("cell_id"), str): + continue # reported by _ids + try: + want = cell_id(cell["arm"], cell["model"], cell["task"], cell["rep"]) + except (KeyError, TypeError, ValueError): + want = None + if cell["cell_id"] != want: + problems.append(f"plan cell {cell['cell_id']} does not match its coordinates " + f"(arm, model, task, rep); expected {want}") + budget = plan.get("budget") + if isinstance(budget, Mapping) and "cells" in budget and budget["cells"] != len(set(planned)): + problems.append(f"plan budget.cells is {budget['cells']!r}, " + f"but the plan has {len(set(planned))} distinct cells") + + def completeness(plan: Mapping[str, Any], result: Mapping[str, Any]) -> Completeness: """Planned minus completed cells of ``result`` against ``plan``. - ``invalid``: the result's ``plan_digest`` is not this plan's, the plan repeats a cell id, ``cells`` is not a list, a cell has no string ``cell_id``, or a result cell has a status outside :data:`STATUSES`; or the result names a cell the plan lacks or gives one - cell two statuses. An ``invalid`` result reports ``measured=0``. + cell two statuses; or a plan cell's id is not :func:`cell_id` of its coordinates, or + the plan's ``budget.cells`` is not its number of distinct cells. An ``invalid`` result + reports ``measured=0``, empty ``statuses`` and every planned cell in ``missing``. - ``incomplete``: some planned cell has no measured status (absent, ``not_run``, ``infra_error`` or ``invalid``); those ids are in ``missing``, in plan order. - ``complete``: every planned cell is ``pass``, ``fail`` or ``inconclusive``. Complete @@ -121,9 +153,7 @@ def completeness(plan: Mapping[str, Any], result: Mapping[str, Any]) -> Complete expected, actual = plan_digest(plan), result.get("plan_digest") if actual != expected: problems.append(f"result is for plan {actual}, not {expected}") - dup_planned = sorted(k for k, n in Counter(planned).items() if n > 1) - if dup_planned: - problems.append(f"plan repeats cell ids: {dup_planned}") + _plan_problems(plan, planned, problems) result_cells = result.get("cells", []) seen = Counter(_ids("result", result_cells, problems)) dup_result = sorted(k for k, n in seen.items() if n > 1) @@ -133,15 +163,17 @@ def completeness(plan: Mapping[str, Any], result: Mapping[str, Any]) -> Complete if unplanned: problems.append(f"result has cells the plan does not: {unplanned}") status = _statuses(result_cells, problems) + if problems: # a rejected result measures nothing: no statuses, every planned cell missing + status = {} counts = Counter(status.get(cid, "not_run") for cid in dict.fromkeys(planned)) missing = tuple(cid for cid in dict.fromkeys(planned) if status.get(cid, "not_run") not in MEASURED) if problems: - state = "invalid" + state, counts = "invalid", Counter() elif missing or not planned: state = "incomplete" else: state = "complete" - measured = 0 if problems else len(set(planned)) - len(missing) + measured = len(set(planned)) - len(missing) return Completeness(state=state, planned=len(set(planned)), measured=measured, missing=missing, statuses={s: counts[s] for s in STATUSES if counts[s]}, problems=tuple(problems)) diff --git a/dokimi/dokimi/schemas/plan.v1.json b/dokimi/dokimi/schemas/plan.v1.json index 57bd6c10c..8cd6cf66c 100644 --- a/dokimi/dokimi/schemas/plan.v1.json +++ b/dokimi/dokimi/schemas/plan.v1.json @@ -68,8 +68,9 @@ "additionalProperties": false, "properties": { "cell_id": {"type": "string", "minLength": 1, - "description": "dokimi.runner.plan.cell_id(arm, model, task, rep): stable across re-plans"}, - "arm": {"type": "string", "minLength": 1}, + "description": "dokimi.runner.plan.cell_id(arm, model, task, rep): stable across re-plans; completeness refuses an id that does not match the coordinates"}, + "arm": {"type": "string", "minLength": 1, + "description": "the full condition the cell runs under. When a plan crosses several harnesses, the harness is part of the arm (e.g. docker-claude/skill), so cell ids stay distinct; a cell has no separate harness key"}, "model": {"type": ["string", "null"]}, "task": {"type": "string", "minLength": 1}, "rep": {"type": "integer", "minimum": 1} @@ -81,7 +82,8 @@ "required": ["cells"], "additionalProperties": false, "properties": { - "cells": {"type": "integer", "minimum": 0}, + "cells": {"type": "integer", "minimum": 0, + "description": "number of distinct planned cells; completeness refuses a plan where it differs"}, "agent_calls": {"type": ["integer", "null"], "minimum": 0}, "cost_usd": {"type": ["number", "null"], "minimum": 0}, "wall_sec": {"type": ["number", "null"], "minimum": 0} diff --git a/dokimi/tests/test_run_plan.py b/dokimi/tests/test_run_plan.py index 66926a073..c2a107d6e 100644 --- a/dokimi/tests/test_run_plan.py +++ b/dokimi/tests/test_run_plan.py @@ -171,6 +171,46 @@ def test_an_invalid_result_reports_no_measured_cells(tmp_path): got = completeness(plan, _result(plan, ["pass"] * 6, plan_digest="sha256:other")) assert got.state == "invalid" assert got.measured == 0 + assert got.statuses == {} # no pass is reported from a rejected run + assert got.missing == tuple(c["cell_id"] for c in plan["cells"]) + + +def test_a_plan_digested_before_saving_matches_the_saved_plan(tmp_path): + raw = {"subjects": [], "suites": [{"id": "s", "hash": "h", "reasons": []}], "models": ["m1"], + "harnesses": ["h"], "grader": {"id": "g", "digest": None}, "varied_variable": "skill", + "cells": plan_cells(["a"], ["m1"], ["t1"]), "budget": {"cells": 1}} + digest = plan_digest(raw) + save_artifact("plan", copy.deepcopy(raw), tmp_path / "plan.json") + result = {"plan_digest": digest, "cells": [{"cell_id": raw["cells"][0]["cell_id"], "status": "pass"}]} + save_artifact("result", result, tmp_path / "result.json") + found = completeness(load_artifact("plan", tmp_path / "plan.json"), + load_artifact("result", tmp_path / "result.json")) + assert found.complete, found.problems + + +def test_the_plan_schema_constant_matches_artifacts(): + from dokimi.artifacts import SCHEMA_VERSIONS + from dokimi.runner import plan as plan_module + + assert plan_module._PLAN_SCHEMA == SCHEMA_VERSIONS["plan"] + + +def test_a_plan_cell_whose_id_does_not_match_its_coordinates_is_invalid(tmp_path): + plan = _plan(tmp_path) + edited = copy.deepcopy(plan) + edited["cells"][0]["arm"] = "renamed" # old id kept + found = completeness(edited, _result(edited, ["pass"] * 6)) + assert found.state == "invalid" + assert any("does not match its coordinates" in p for p in found.problems) + edited["cells"][0]["rep"] = "x" # coordinates that cell_id cannot hash + assert completeness(edited, _result(edited, ["pass"] * 6)).state == "invalid" + + +def test_a_budget_that_miscounts_the_cells_is_invalid(tmp_path): + plan = _plan(tmp_path, budget={"cells": 5}) + found = completeness(plan, _result(plan, ["pass"] * 6)) + assert found.state == "invalid" + assert any("budget.cells is 5" in p for p in found.problems) @pytest.mark.parametrize("bad", [{"status": "PASS"}, {"status": ["pass"]}, {}]) From 586031d21d13300fd2bb4492291f2a0a860bbc53 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 26 Oct 2026 21:52:45 -0700 Subject: [PATCH 12/17] docs(changelog): plan budget.cells and invalid-result reporting Signed-off-by: Nikolay Petrov --- dokimi/changelog.d/20261021_120000_napetrov_run-plan-result.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/changelog.d/20261021_120000_napetrov_run-plan-result.md b/dokimi/changelog.d/20261021_120000_napetrov_run-plan-result.md index 9ec71722c..4683088a9 100644 --- a/dokimi/changelog.d/20261021_120000_napetrov_run-plan-result.md +++ b/dokimi/changelog.d/20261021_120000_napetrov_run-plan-result.md @@ -1,6 +1,6 @@ ### Added -- **Run plan and result artifacts** — two new artifact kinds, `plan.v1` and `result.v1`. A plan lists every cell a run will measure (arm × model × task × rep, each with a stable `cell_id`) and records the subjects, suites, models, harnesses, grader digest, budget estimate and the one `varied_variable` (a list for a 2×2 design). A result gives each cell one status: `pass`, `fail`, `inconclusive`, `invalid`, `infra_error` or `not_run`, and refers to its plan by `plan_digest`. `dokimi.runner.plan.completeness` computes planned minus completed cells. A run is `complete` only when every planned cell is `pass`, `fail` or `inconclusive`. A result for a different plan, or one with unplanned or duplicated cells, is `invalid`. +- **Run plan and result artifacts** — two new artifact kinds, `plan.v1` and `result.v1`. A plan lists every cell a run will measure (arm × model × task × rep, each with a stable `cell_id`) and records the subjects, suites, models, harnesses, grader digest, budget (whose `cells` must equal the number of planned cells) and the one `varied_variable` (a list for a 2×2 design). A result gives each cell one status: `pass`, `fail`, `inconclusive`, `invalid`, `infra_error` or `not_run`, and refers to its plan by `plan_digest`. `dokimi.runner.plan.completeness` computes planned minus completed cells. A run is `complete` only when every planned cell is `pass`, `fail` or `inconclusive`. A result for a different plan, or one with unplanned or duplicated cells, is `invalid`, as is a plan whose cell ids do not match their coordinates; an `invalid` result reports no statuses. When a plan crosses harnesses, the harness is part of the arm. ### Changed From 021ad8f422e0ef457dcdc254d242cacf97fabe55 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 27 Oct 2026 11:47:38 +0000 Subject: [PATCH 13/17] test(plan): pin planned, measured and missing of an invalid result Signed-off-by: Anton Volkov --- dokimi/tests/test_run_plan.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dokimi/tests/test_run_plan.py b/dokimi/tests/test_run_plan.py index c2a107d6e..3790e1e8a 100644 --- a/dokimi/tests/test_run_plan.py +++ b/dokimi/tests/test_run_plan.py @@ -139,6 +139,8 @@ def test_a_plan_that_repeats_a_cell_is_invalid(tmp_path): plan["cells"] = [*plan["cells"], plan["cells"][0]] got = completeness(plan, _result(plan, ["pass"] * 7)) assert got.state == "invalid" + assert (got.planned, got.measured) == (6, 0) + assert got.missing == tuple(dict.fromkeys(c["cell_id"] for c in plan["cells"])) assert "plan repeats" in got.problems[0] From 3535d6259ec98cbbac86c1dbc7b3e3adcff13969 Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Tue, 27 Oct 2026 04:51:02 -0700 Subject: [PATCH 14/17] test(plan): keep the result schema status enum in step with STATUSES --- dokimi/tests/test_run_plan.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dokimi/tests/test_run_plan.py b/dokimi/tests/test_run_plan.py index 3790e1e8a..0c70d77c8 100644 --- a/dokimi/tests/test_run_plan.py +++ b/dokimi/tests/test_run_plan.py @@ -197,6 +197,16 @@ def test_the_plan_schema_constant_matches_artifacts(): assert plan_module._PLAN_SCHEMA == SCHEMA_VERSIONS["plan"] +def test_the_result_schema_status_enum_matches_statuses(): + import json + + from dokimi.artifacts import _schema_path + from dokimi.runner.plan import STATUSES + + schema = json.loads(_schema_path("result").read_text(encoding="utf-8")) + assert tuple(schema["properties"]["cells"]["items"]["properties"]["status"]["enum"]) == STATUSES + + def test_a_plan_cell_whose_id_does_not_match_its_coordinates_is_invalid(tmp_path): plan = _plan(tmp_path) edited = copy.deepcopy(plan) From 29b27c8d00e8de96e4d55e8dbee30043c66850e6 Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Tue, 27 Oct 2026 04:51:30 -0700 Subject: [PATCH 15/17] test(smoke): cover an unknown egress on a non-probe row --- dokimi/tests/test_check_smoke_runs.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dokimi/tests/test_check_smoke_runs.py b/dokimi/tests/test_check_smoke_runs.py index 5218a24d6..8c2ef28d9 100644 --- a/dokimi/tests/test_check_smoke_runs.py +++ b/dokimi/tests/test_check_smoke_runs.py @@ -85,6 +85,12 @@ def test_an_unreadable_sidecar_log_fails_even_when_every_cell_passed(): probe["metrics"]["egress"] = { "status": "unknown", "denied_entries": ["example.com:443", "example.com:80"]} assert not csr.smoke_ok(data, out=io.StringIO()) + data = _runs() # the check is for every row, not only the probe + oracle = next(r for r in data["results"] if r["harness"] == "docker-oracle") + oracle["metrics"]["egress"] = {"status": "unknown"} + out = io.StringIO() + assert not csr.smoke_ok(data, out=out) + assert "docker-oracle: egress unknown" in out.getvalue() def test_usage_without_a_path(): From 6390266b87f6c2686ccbb2335d97077b4df46ca3 Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Tue, 27 Oct 2026 04:51:53 -0700 Subject: [PATCH 16/17] fix(plan): a plan whose cells is a number is invalid, not a TypeError --- dokimi/dokimi/runner/plan.py | 3 ++- dokimi/tests/test_run_plan.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/runner/plan.py b/dokimi/dokimi/runner/plan.py index cd7e9eeca..6d230a6ff 100644 --- a/dokimi/dokimi/runner/plan.py +++ b/dokimi/dokimi/runner/plan.py @@ -118,7 +118,8 @@ def _plan_problems(plan: Mapping[str, Any], planned: list[str], problems: list[s dup_planned = sorted(k for k, n in Counter(planned).items() if n > 1) if dup_planned: problems.append(f"plan repeats cell ids: {dup_planned}") - for cell in plan.get("cells") or []: + cells = plan.get("cells") + for cell in cells if isinstance(cells, list) else []: # a non-list is reported by _ids if not isinstance(cell, Mapping) or not isinstance(cell.get("cell_id"), str): continue # reported by _ids try: diff --git a/dokimi/tests/test_run_plan.py b/dokimi/tests/test_run_plan.py index 0c70d77c8..f3c0d4364 100644 --- a/dokimi/tests/test_run_plan.py +++ b/dokimi/tests/test_run_plan.py @@ -251,7 +251,7 @@ def test_result_cells_that_are_not_a_list_do_not_raise(tmp_path): assert not completeness(plan, {"plan_digest": plan_digest(plan), "cells": None}).complete -@pytest.mark.parametrize("cells", [{"c-1": "pass"}, None, "pass"]) +@pytest.mark.parametrize("cells", [{"c-1": "pass"}, None, "pass", 5, 1.5, True]) def test_cells_that_are_not_a_list_are_invalid_not_empty(tmp_path, cells): # A mapping or null used to read as zero cells: no problem, and every planned cell "not_run". plan = _plan(tmp_path) From 5bd6906caa963c6b8169b656227dd59f75012ec8 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Tue, 27 Oct 2026 05:47:33 -0700 Subject: [PATCH 17/17] test(plan): pin the rep coordinate of the cell id check Every plan in the tests had reps=1, so the coordinate check could drop rep and still pass. A cell renumbered to rep 2 with its old id kept must be invalid. Signed-off-by: Alexander Rybkin --- dokimi/tests/test_run_plan.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dokimi/tests/test_run_plan.py b/dokimi/tests/test_run_plan.py index f3c0d4364..d055f0eab 100644 --- a/dokimi/tests/test_run_plan.py +++ b/dokimi/tests/test_run_plan.py @@ -214,6 +214,9 @@ def test_a_plan_cell_whose_id_does_not_match_its_coordinates_is_invalid(tmp_path found = completeness(edited, _result(edited, ["pass"] * 6)) assert found.state == "invalid" assert any("does not match its coordinates" in p for p in found.problems) + renumbered = copy.deepcopy(plan) + renumbered["cells"][0]["rep"] = 2 # old id kept + assert completeness(renumbered, _result(renumbered, ["pass"] * 6)).state == "invalid" edited["cells"][0]["rep"] = "x" # coordinates that cell_id cannot hash assert completeness(edited, _result(edited, ["pass"] * 6)).state == "invalid"