diff --git a/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-novelty.md b/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-novelty.md new file mode 100644 index 00000000..0e95d07a --- /dev/null +++ b/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-novelty.md @@ -0,0 +1,3 @@ +### Fixed + +- **`novelty` subcommands** exit with a message on an unreadable or invalid `--placebo-text`, `--probes`, `--atoms`, `--novelty`, `--queries` or `--artifact`, or an `--atoms` file whose `bundle` is not an object, instead of a traceback. diff --git a/dokimi/changelog.d/20261026_000000_vchamarthi_atoms-bundle-values.md b/dokimi/changelog.d/20261026_000000_vchamarthi_atoms-bundle-values.md new file mode 100644 index 00000000..380b4c69 --- /dev/null +++ b/dokimi/changelog.d/20261026_000000_vchamarthi_atoms-bundle-values.md @@ -0,0 +1,3 @@ +### Fixed + +- **`novelty probe` and `novelty shorten`** exit with a message on an `--atoms` file whose spans, atoms, `extraction` or `bundle` have a missing or wrong-type field or whose JSON is nested too deeply to parse, and `novelty view` does the same for deep nesting, instead of a traceback. diff --git a/dokimi/dokimi/commands/inputs.py b/dokimi/dokimi/commands/inputs.py new file mode 100644 index 00000000..c39f0d8c --- /dev/null +++ b/dokimi/dokimi/commands/inputs.py @@ -0,0 +1,38 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Reading a file a command was pointed at, with an exit that names it.""" + +from __future__ import annotations + +from pathlib import Path + + +def read_text_or_exit(path: str | None, flag: str) -> str | None: + """The text of the file `flag` names, None when the flag is unset. + + A file that cannot be read stops the command with a message naming the flag and the + path, instead of a traceback. + """ + if not path: + return None + try: + return Path(path).read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as exc: + raise SystemExit(f"cannot read {flag} {path}: {exc}") from None + + +def placebo_text_for_arms_or_exit(path: str | None, arms: tuple[str, ...]) -> str | None: + """The ``--placebo-text`` of an analyze run, checked against its arms before any model call. + + A typo in --arms or --placebo-text would otherwise cost the extraction calls that run + before the measurement checks the same thing. + """ + from dokimi.novelty.measure import check_arms + + placebo_text = read_text_or_exit(path, "--placebo-text") + try: + check_arms(arms, placebo_text) + except ValueError as exc: + raise SystemExit(str(exc)) from None + return placebo_text diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index 1ed02cae..24a701c4 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -19,6 +19,7 @@ from pathlib import Path from typing import TYPE_CHECKING, NoReturn +from dokimi.commands.inputs import placebo_text_for_arms_or_exit, read_text_or_exit from dokimi.defaults import ( DEFAULT_ANSWER_MODEL, DEFAULT_ANSWER_PROVIDER, @@ -94,6 +95,7 @@ def cmd_novelty_analyze(args: argparse.Namespace) -> None: ) return + placebo_text = placebo_text_for_arms_or_exit(args.placebo_text, _arms(args.arms)) extractor, answerer, judge = _clients(args) config = AnalyzeConfig( artifact_ref=args.artifact, @@ -105,10 +107,7 @@ def cmd_novelty_analyze(args: argparse.Namespace) -> None: budgets=tuple(args.budgets), concurrency=args.concurrency, allow_self_judge=args.allow_self_judge, - placebo_text=( - Path(args.placebo_text).read_text(encoding="utf-8") - if args.placebo_text else None - ), + placebo_text=placebo_text, cache_dir=Path(args.cache_dir) if args.cache_dir else None, seed=args.seed, include_resources=args.include_resources, @@ -119,7 +118,7 @@ def cmd_novelty_analyze(args: argparse.Namespace) -> None: result = analyze(artifact_bundle.text, config, extractor=extractor, answerer=answerer, judge=judge, artifact_bundle=artifact_bundle) - except (SelfJudgeError, ExtractionError) as exc: + except (SelfJudgeError, ExtractionError, ValueError) as exc: raise SystemExit(str(exc)) from None out_dir = Path(args.out_dir) if args.out_dir else _default_dir(args.artifact) @@ -173,21 +172,21 @@ def cmd_novelty_atoms(args: argparse.Namespace) -> None: def cmd_novelty_probe(args: argparse.Namespace) -> None: """Run the paired probes against a saved atom set and print the 2×2.""" - from dokimi.novelty.atoms import load_atoms + from dokimi.novelty.atoms import load_atoms_or_exit from dokimi.novelty.bundle import check_answer_key from dokimi.novelty.measure import SelfJudgeError, run_measurement from dokimi.novelty.probes import build_probe_set from dokimi.novelty.report import build_probe_record - atoms, saved_spans = load_atoms(args.atoms) + atoms, saved_spans, saved = load_atoms_or_exit(args.atoms) _warn_unread(saved_spans, "the atoms probed here leave out whatever those spans assert") - saved = json.loads(Path(args.atoms).read_text(encoding="utf-8")) verdict = check_answer_key(args.artifact, saved.get("bundle"), source=args.atoms) if not verdict.ok: raise SystemExit(verdict.message) for note in verdict.notes: print(note, file=sys.stderr) settings = _settings(args) + placebo_text = read_text_or_exit(args.placebo_text, "--placebo-text") extractor, answerer, judge = _clients(args) if saved.get("extractor") == answerer.model: print(f"WARNING: {answerer.model} extracted these atoms, so the model under measurement " @@ -209,9 +208,7 @@ def cmd_novelty_probe(args: argparse.Namespace) -> None: atoms, probe_set.probes, verdict.text, answerer=answerer, judge=judge, arms=_arms(args.arms), concurrency=args.concurrency, allow_self_judge=args.allow_self_judge, seed=args.seed, - placebo_text=( - Path(args.placebo_text).read_text(encoding="utf-8") if args.placebo_text else None - ), + placebo_text=placebo_text, ) except (SelfJudgeError, ValueError) as exc: raise SystemExit(str(exc)) from None @@ -320,7 +317,7 @@ def card(ref: str) -> tuple[Candidate, str]: file=sys.stderr) print(f"--catalog {args.catalog}: {len(catalog)} competitors", file=sys.stderr) competitors += catalog - except (FileNotFoundError, ValueError) as exc: + except (OSError, ValueError, RecursionError) as exc: raise SystemExit(f"cannot load an artifact: {exc}") from None # Checked before the query file is read, so the message is about the real problem. try: @@ -436,15 +433,14 @@ def _refuse_no_queries(args: argparse.Namespace, queries: list[str], self_derive def cmd_novelty_shorten(args: argparse.Namespace) -> None: """Score shortening candidates on the five axes and optionally write the best one.""" - from dokimi.novelty.analyze import load_artifact_text - from dokimi.novelty.atoms import load_atoms + from dokimi.novelty.artifact import load_artifact_or_exit + from dokimi.novelty.atoms import load_atoms_or_exit from dokimi.novelty.bundle import entry_point_only from dokimi.novelty.retention import RetentionOracle, error_warning from dokimi.novelty.shorten import build_candidates, pick_best - atoms, spans = load_atoms(args.atoms) - _kind, _name, _description, frontmatter, _body = load_artifact_text(args.artifact) - saved = json.loads(Path(args.atoms).read_text(encoding="utf-8")) + atoms, spans, saved = load_atoms_or_exit(args.atoms) + _kind, _name, _description, frontmatter, _body = load_artifact_or_exit(args.artifact) try: spans, atoms, note = entry_point_only(saved.get("bundle"), spans, atoms) except ValueError as exc: @@ -500,13 +496,11 @@ def cmd_novelty_shorten(args: argparse.Namespace) -> None: def cmd_novelty_view(args: argparse.Namespace) -> None: """Re-render the annotated view from a saved novelty artifact plus its atoms.""" - from dokimi.artifacts import load_artifact - from dokimi.novelty.atoms import load_atoms + from dokimi.commands.novelty_view_inputs import load_view_inputs from dokimi.novelty.attribute import attribute_spans from dokimi.novelty.report import render_annotated_html, render_annotated_markdown - artifact = load_artifact("novelty", Path(args.novelty)) - atoms, spans = load_atoms(args.atoms) + artifact, atoms, spans = load_view_inputs(args.novelty, args.atoms) _warn_unread(spans, "they render as `unread`, not as text that carries nothing") # Re-rendering needs per-atom outcomes; the artifact carries aggregates, so a view # rebuilt from it alone would colour by category only. Say so instead of implying the @@ -550,7 +544,7 @@ def _bundle_from_args(args: argparse.Namespace): max_file_words=args.max_file_words, max_total_words=args.max_resource_words, ) - except (FileNotFoundError, ValueError) as exc: + except (OSError, ValueError, RecursionError) as exc: raise SystemExit(f"cannot load --artifact {args.artifact}: {exc}") from None note = artifact_bundle.coverage_note() if note: diff --git a/dokimi/dokimi/commands/novelty_view_inputs.py b/dokimi/dokimi/commands/novelty_view_inputs.py new file mode 100644 index 00000000..21855ef4 --- /dev/null +++ b/dokimi/dokimi/commands/novelty_view_inputs.py @@ -0,0 +1,35 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""`novelty view`: read the saved novelty artifact and atoms file, or exit 1 naming the bad one.""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import NoReturn + + +def _fail(message: str) -> NoReturn: + print(f"Error: {message}", file=sys.stderr) + raise SystemExit(1) + + +def load_view_inputs(novelty_path: str, atoms_path: str): + """``(artifact, atoms, spans)`` from the two files; an unreadable or invalid one exits 1.""" + from dokimi.artifacts import load_artifact + from dokimi.novelty.atoms import load_atoms + + try: + artifact = load_artifact("novelty", Path(novelty_path)) + except OSError as exc: + _fail(f"cannot read {novelty_path}: {exc}") + except (ValueError, RecursionError) as exc: # not UTF-8, not JSON, or not a novelty artifact + _fail(f"{novelty_path} is not a valid novelty artifact: {exc}") + try: + atoms, spans = load_atoms(atoms_path) + except OSError as exc: + _fail(f"cannot read {atoms_path}: {exc}") + except (ValueError, RecursionError) as exc: # not UTF-8, not JSON, or not an atoms record + _fail(f"{atoms_path} is not a valid atoms file: {exc}") + return artifact, atoms, spans diff --git a/dokimi/dokimi/novelty/artifact.py b/dokimi/dokimi/novelty/artifact.py index c864196c..7b110e93 100644 --- a/dokimi/dokimi/novelty/artifact.py +++ b/dokimi/dokimi/novelty/artifact.py @@ -28,10 +28,20 @@ def load_artifact_text(ref: str) -> tuple[str, str, str, str, str]: skill = load_skill(ref.removeprefix("skill:")) # Verbatim, as shorten.render promises: a rebuilt `description: a: b` is not YAML. lines = skill.path.read_text(encoding="utf-8").lstrip("").splitlines() - end = next(i for i in range(1, len(lines)) if lines[i].strip() == "---") + end = next((i for i in range(1, len(lines)) if lines[i].strip() == "---"), None) + if end is None: + raise ValueError(f"{skill.path} has no closing '---' after its frontmatter") frontmatter = "\n".join(lines[: end + 1]) return "skill", skill.name, skill.description, frontmatter, skill.body path = Path(ref.removeprefix("text:")) if not path.is_file(): raise FileNotFoundError(f"artifact not found: {path}") return "text", path.stem, "", "", path.read_text(encoding="utf-8") + + +def load_artifact_or_exit(ref: str) -> tuple[str, str, str, str, str]: + """:func:`load_artifact_text`, or exit naming ``ref`` when it is missing, unreadable or not UTF-8.""" + try: + return load_artifact_text(ref) + except (OSError, ValueError, RecursionError) as exc: + raise SystemExit(f"cannot load --artifact {ref}: {exc}") from None diff --git a/dokimi/dokimi/novelty/atoms.py b/dokimi/dokimi/novelty/atoms.py index 2eee848b..b858d288 100644 --- a/dokimi/dokimi/novelty/atoms.py +++ b/dokimi/dokimi/novelty/atoms.py @@ -520,7 +520,68 @@ def load_atoms(path) -> tuple[list[Atom], list[Span]]: """ from pathlib import Path - data = json.loads(Path(path).read_text(encoding="utf-8")) + return _atoms_from_record(json.loads(Path(path).read_text(encoding="utf-8"))) + + +def _is_int(value) -> bool: + return isinstance(value, int) and not isinstance(value, bool) + + +def _is_str_list(value) -> bool: + return isinstance(value, list) and all(isinstance(item, str) for item in value) + + +_STR = (lambda v: isinstance(v, str), "a string") +_INT = (_is_int, "an integer") +_STR_LIST = (_is_str_list, "a list of strings") +_OBJ = (lambda v: isinstance(v, dict), "an object") +_STR_OR_NULL = (lambda v: v is None or isinstance(v, str), "a string or null") + +# Every field a reader takes from a row: (type check, what it should be), and the fields +# a row must have. +_SPAN_FIELDS = {"id": _STR, "kind": _STR, "text": _STR, "start": _INT, "end": _INT, + "heading_path": _STR_LIST, "protection": _STR_LIST, "atom_ids": _STR_LIST} +_SPAN_REQUIRED = ("id", "kind", "text", "start", "end") +_ATOM_FIELDS = {"id": _STR, "statement": _STR, "question": _STR, "answer": _STR, + "kind": _STR, "faithfulness": _STR, "tags": _STR_LIST, + "span_ids": _STR_LIST, "meta": _OBJ} +_ATOM_REQUIRED = ("statement",) +_FILE_FIELDS = {"path": _STR, "role": _STR, "start": _INT, "end": _INT, "words": _INT, + "sha256": _STR_OR_NULL} +_FILE_REQUIRED = ("path",) +_EXCLUDED_FIELDS = {"path": _STR, "role": _STR, "reason": _STR, "words": _INT} + + +def _check_rows(where: str, rows, fields: dict, required: tuple = ()) -> None: + """Raise ValueError unless ``rows`` is a list of objects whose fields have their types.""" + if not (isinstance(rows, list) and all(isinstance(row, dict) for row in rows)): + raise ValueError(f"'{where}' is not a list of objects") + for row in rows: + for key in required: + if key not in row: + raise ValueError(f"'{where}' has a row without '{key}'") + for key, (ok, expected) in fields.items(): + if key in row and not ok(row[key]): + raise ValueError(f"'{where}' has a row whose '{key}' is not {expected}") + + +def _check_record_fields(data: dict) -> None: + """Raise ValueError unless the spans, atoms and extraction record have their types.""" + _check_rows("spans", data.get("spans", []), _SPAN_FIELDS, _SPAN_REQUIRED) + _check_rows("atoms", data.get("atoms", []), _ATOM_FIELDS, _ATOM_REQUIRED) + extraction = data.get("extraction") + if extraction is not None and not isinstance(extraction, dict): + raise ValueError("'extraction' is not an object") + unread = (extraction or {}).get("unread_span_ids") + if unread is not None and not _is_str_list(unread): + raise ValueError("'extraction.unread_span_ids' is not a list of strings") + + +def _atoms_from_record(data) -> tuple[list[Atom], list[Span]]: + """The atoms and spans of a parsed atoms record; ValueError when it is not one.""" + if not isinstance(data, dict): + raise ValueError(f"the file holds {type(data).__name__}, not an object") + _check_record_fields(data) atoms = [Atom.from_dict(row) for row in data.get("atoms", [])] unread = set((data.get("extraction") or {}).get("unread_span_ids") or ()) spans = [ @@ -534,3 +595,39 @@ def load_atoms(path) -> tuple[list[Atom], list[Span]]: for row in data.get("spans", []) ] return atoms, spans + + +def _check_bundle_shape(bundle) -> None: + """Raise ValueError unless a ``bundle`` record has the types its readers rely on.""" + if bundle is None: + return + if not isinstance(bundle, dict): + raise ValueError("'bundle' is not an object") + for key in ("max_file_words", "max_total_words", "n_files", "resource_words_measured"): + if bundle.get(key) is not None and not _is_int(bundle[key]): + raise ValueError(f"'bundle.{key}' is not an integer") + if bundle.get("artifact_name") is not None and not isinstance(bundle["artifact_name"], str): + raise ValueError("'bundle.artifact_name' is not a string") + if bundle.get("resource_roles") is not None and not _is_str_list(bundle["resource_roles"]): + raise ValueError("'bundle.resource_roles' is not a list of strings") + if bundle.get("files") is not None: + _check_rows("bundle.files", bundle["files"], _FILE_FIELDS, _FILE_REQUIRED) + if bundle.get("excluded") is not None: + _check_rows("bundle.excluded", bundle["excluded"], _EXCLUDED_FIELDS) + + +def load_atoms_or_exit(path) -> tuple[list[Atom], list[Span], dict]: + """:func:`load_atoms` plus the parsed record, for a command. + + Exits with the file named when it cannot be read or is not an atom set, including a + ``bundle`` whose fields do not have the types its readers use. + """ + from pathlib import Path + + try: + data = json.loads(Path(path).read_text(encoding="utf-8")) + atoms, spans = _atoms_from_record(data) + _check_bundle_shape(data.get("bundle")) + except (OSError, ValueError, RecursionError) as exc: + raise SystemExit(f"cannot read atoms from {path}: {exc}") from None + return atoms, spans, data diff --git a/dokimi/dokimi/novelty/bundle.py b/dokimi/dokimi/novelty/bundle.py index 5ec74aa9..c290c8d9 100644 --- a/dokimi/dokimi/novelty/bundle.py +++ b/dokimi/dokimi/novelty/bundle.py @@ -517,7 +517,7 @@ def check_answer_key(ref: str, record: dict | None, *, if not record: try: body = load_artifact_text(ref)[4] - except (FileNotFoundError, ValueError) as exc: + except (OSError, ValueError, RecursionError) as exc: return AnswerKeyVerdict(problem="unloadable", message=f"cannot load {ref}: {exc}") return AnswerKeyVerdict(text=body, notes=( f"WARNING: {source} predates bundle provenance; assuming the entry point only. " @@ -530,7 +530,7 @@ def check_answer_key(ref: str, record: dict | None, *, "rather than probing against an answer key whose provenance cannot be checked")) try: now = load_bundle(ref, **_replay_selection(record)) - except (FileNotFoundError, ValueError) as exc: + except (OSError, ValueError, RecursionError) as exc: return AnswerKeyVerdict(problem="unloadable", message=f"cannot load {ref}: {exc}") # An answer key is for one artifact. Without this, skill A's atoms were accepted diff --git a/dokimi/dokimi/novelty/items.py b/dokimi/dokimi/novelty/items.py index 27c662a2..b913f780 100644 --- a/dokimi/dokimi/novelty/items.py +++ b/dokimi/dokimi/novelty/items.py @@ -228,6 +228,37 @@ class ProbeFileError(ValueError): """A ``novelty probe`` output that item analysis cannot use, with the reason.""" +def _read_probe_file(raw: str): + """The parsed JSON of one probe file; a file that cannot be read says which.""" + try: + return json.loads(Path(raw).read_text(encoding="utf-8")) + except (OSError, ValueError, RecursionError) as exc: # bad JSON, non-UTF-8 bytes, or too deep + raise ProbeFileError(f"{raw}: {exc}") from None + + +def _check_probe_strings(raw: str, data: dict) -> None: + """Refuse a probe file whose model, artifact or ref_digest is not a string.""" + for key in ("model", "artifact", "ref_digest"): + value = data.get(key) + if value is None and key != "model": # older output records no artifact or digest + continue + if key in data and not isinstance(value, str): + raise ProbeFileError( + f"{raw} has `{key}` as {type(value).__name__}, not a string; " + "it is not `novelty probe` output" + ) + + +def _check_run_strings(raw: str, row: dict) -> None: + """Refuse a probe run whose id, kind, arm or verdict is not a string.""" + for key in ("atom_id", "probe_kind", "arm", "verdict"): + if key in row and not isinstance(row[key], str): + raise ProbeFileError( + f"{raw} has a probe run whose `{key}` is {type(row[key]).__name__}, " + "not a string; it is not `novelty probe` output" + ) + + def load_probe_cells(paths: list[str]) -> tuple[list[dict], list[str]]: """Read ``novelty probe`` outputs into item-analysis cells, one per file. @@ -242,12 +273,12 @@ def load_probe_cells(paths: list[str]) -> tuple[list[dict], list[str]]: "ref_digest"}`` as :func:`discrimination` reads it. Raises: - ProbeFileError: A file that is not probe output, or files that measured different - texts. A file that cannot be read or is not JSON raises as reading it does. + ProbeFileError: A file that cannot be read, is not JSON, is not probe output, or + files that measured different texts. """ cells = [] for raw in paths: - data = json.loads(Path(raw).read_text(encoding="utf-8")) + data = _read_probe_file(raw) if not isinstance(data, dict): raise ProbeFileError( f"{raw} is {type(data).__name__}, not an object; " @@ -259,6 +290,7 @@ def load_probe_cells(paths: list[str]) -> tuple[list[dict], list[str]]: f"{raw} has `runs` as {type(runs_value).__name__}, not a list; " "it is not `novelty probe` output" ) + _check_probe_strings(raw, data) runs = [] for row in runs_value: if not isinstance(row, dict): @@ -266,6 +298,7 @@ def load_probe_cells(paths: list[str]) -> tuple[list[dict], list[str]]: f"{raw} has a run entry as {type(row).__name__}, not an object; " "it is not `novelty probe` output" ) + _check_run_strings(raw, row) try: runs.append(ProbeRun( atom_id=row["atom_id"], probe_kind=row["probe_kind"], arm=row["arm"], diff --git a/dokimi/dokimi/novelty/measure.py b/dokimi/dokimi/novelty/measure.py index 344c1ba9..6ef577f3 100644 --- a/dokimi/dokimi/novelty/measure.py +++ b/dokimi/dokimi/novelty/measure.py @@ -210,6 +210,21 @@ def parse_arms(spec: str) -> tuple[str, ...]: return arms +def check_arms(arms: tuple[str, ...], placebo_text: str | None) -> None: + """Refuse an arm set that cannot be run, before any model is called. + + Raises: + ValueError: An unknown arm, a set without ``full``, or ``placebo`` with no text. + """ + unknown = [arm for arm in arms if arm not in ARMS] + if unknown: + raise ValueError(f"unknown novelty arms: {unknown}; known: {list(ARMS)}") + if "full" not in arms: + raise ValueError("the arms must include full: the headline 2x2 pairs it with no_artifact") + if "placebo" in arms and not placebo_text: + raise ValueError("the placebo arm needs --placebo-text (domain-mismatched text)") + + def run_measurement( atoms: list[Atom], probes: list[Probe], @@ -246,13 +261,7 @@ def run_measurement( "the model that produced it inflates the score. Pass a different --judge-model, " "or --allow-self-judge to do it deliberately." ) - unknown = [arm for arm in arms if arm not in ARMS] - if unknown: - raise ValueError(f"unknown novelty arms: {unknown}; known: {list(ARMS)}") - if "full" not in arms: - raise ValueError("the arms must include full: the headline 2x2 pairs it with no_artifact") - if "placebo" in arms and not placebo_text: - raise ValueError("the placebo arm needs --placebo-text (domain-mismatched text)") + check_arms(arms, placebo_text) contexts = { BASELINE_ARM: None, diff --git a/dokimi/dokimi/novelty/routing.py b/dokimi/dokimi/novelty/routing.py index 341c17d1..e18c0424 100644 --- a/dokimi/dokimi/novelty/routing.py +++ b/dokimi/dokimi/novelty/routing.py @@ -493,13 +493,16 @@ def load_queries(path: str) -> list[str]: Blank queries are dropped, so the result can be empty. Raises: - RoutingInputError: The file holds something other than a list of strings. A file - that cannot be read raises as reading it does. + RoutingInputError: The file cannot be read (missing, unreadable or not UTF-8), or + holds something other than a list of strings. """ - raw = Path(path).read_text(encoding="utf-8-sig") + try: + raw = Path(path).read_text(encoding="utf-8-sig") + except (OSError, UnicodeDecodeError) as exc: + raise RoutingInputError(f"cannot read --queries {path}: {exc}") from None try: data = json.loads(raw) - except json.JSONDecodeError: + except (ValueError, RecursionError): # JSONDecodeError is a ValueError data = None if not isinstance(data, (list, dict)): # one-per-line, e.g. "[urgent] build fails" data = raw.splitlines() diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index 72aeb637..97ae1f01 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -1,6 +1,6 @@ { "files": { - "dokimi/commands/novelty.py": {"lines": 984, "rationale": "estimate-only projection (expected_usd, arm scaling, extrapolation flag) in novelty CLI, on top of main's W-series and #173 lines; file split tracked separately"}, + "dokimi/commands/novelty.py": {"lines": 978, "rationale": "estimate-only projection (expected_usd, arm scaling, extrapolation flag) in novelty CLI, on top of main's W-series and #173 lines; file split tracked separately"}, "dokimi/harnesses/benchflow.py": {"lines": 869}, "dokimi/harnesses/docker_solver.py": {"lines": 900, "rationale": "pre-existing when the gate was introduced"}, "dokimi/llm.py": {"lines": 1118, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"}, diff --git a/dokimi/tests/test_novelty_atoms_input.py b/dokimi/tests/test_novelty_atoms_input.py new file mode 100644 index 00000000..04ea04ae --- /dev/null +++ b/dokimi/tests/test_novelty_atoms_input.py @@ -0,0 +1,229 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""`novelty probe` and `novelty shorten` on an --atoms file they cannot read.""" + +from __future__ import annotations + +import json + +import pytest + +from dokimi.cli import build_parser + + +def _bundle(extra, end="5"): + row = '{"role": "entry", "path": "SKILL.md", "end": %s}' % end + inner = '"include_resources": true, "files": [%s]' % row + return '{"atoms": [], "spans": [], "bundle": {%s%s}}' % (inner, ", " + extra if extra else "") + + +_SPAN = {"id": "s0001", "kind": "sentence", "text": "Text.", "start": 0, "end": 5} +_ATOM = {"id": "a1", "statement": "Text.", "question": "Q?", "span_ids": ["s0001"]} + + +def _record(span=None, atom=None): + """An atoms record of one span and one atom, with the given fields replaced.""" + return json.dumps({"atoms": [{**_ATOM, **(atom or {})}], + "spans": [{**_SPAN, **(span or {})}]}) + + +_WRONG_SHAPES = { + "top_level_list": "[]", + "top_level_null": "null", + "top_level_number": "5", + "wrong_shape": '{"atoms": [{"unexpected": 1}], "spans": []}', + "atoms_not_a_list": '{"atoms": "x"}', + "bundle_is_a_list": '{"atoms": [], "spans": [], "bundle": [1]}', + "bundle_is_a_number": '{"atoms": [], "spans": [], "bundle": 5}', + "bundle_files_not_objects": + '{"atoms": [], "spans": [], "bundle": {"include_resources": true, "files": [1]}}', + "bundle_files_not_a_list": '{"atoms": [], "spans": [], "bundle": {"files": "x"}}', + "bundle_max_file_words_is_a_list": _bundle('"max_file_words": [1]'), + "bundle_max_total_words_is_a_list": _bundle('"max_total_words": [1]'), + "span_text_is_a_number": _record(span={"text": 5}), + "span_start_is_a_string": _record(span={"start": "a"}), + "span_start_is_a_bool": _record(span={"start": True}), + "span_end_is_null": _record(span={"end": None}), + "atom_id_is_a_list": _record(atom={"id": [1]}), + "atom_question_is_a_number": _record(atom={"question": 5}), + "atom_statement_is_null": _record(atom={"statement": None}), + "atom_span_ids_has_a_list": _record(atom={"span_ids": [[1]]}), + "atom_span_ids_is_a_string": _record(atom={"span_ids": "s0001"}), + "span_id_is_a_list": _record(span={"id": [1]}), + "span_kind_is_a_number": _record(span={"kind": 5}), + "span_heading_path_is_a_string": _record(span={"heading_path": "x"}), + "span_protection_has_a_number": _record(span={"protection": [1]}), + "spans_not_a_list": json.dumps({"atoms": [], "spans": "x"}), + "spans_has_a_string": json.dumps({"atoms": [], "spans": ["x"]}), + "atoms_has_a_number": json.dumps({"atoms": [5], "spans": []}), + "bundle_max_file_words_is_a_bool": _bundle('"max_file_words": true'), + "bundle_max_total_words_is_a_bool": _bundle('"max_total_words": true'), + "bundle_roles_is_a_number": _bundle('"resource_roles": 5'), + "bundle_roles_is_a_string": _bundle('"resource_roles": "entry"'), + "bundle_roles_has_a_number": _bundle('"resource_roles": ["a", 1]'), + "bundle_excluded_not_objects": _bundle('"excluded": ["x"]'), + "bundle_excluded_path_is_a_list": _bundle('"excluded": [{"path": ["x"]}]'), + "bundle_excluded_path_is_an_object": _bundle('"excluded": [{"path": {}}]'), + "bundle_file_end_is_a_string": _bundle('', end='"x"'), + "bundle_file_end_is_a_bool": _bundle('', end='true'), + "bundle_file_words_is_a_string": _bundle('', end='5, "words": "x"'), + "bundle_file_role_is_a_list": _bundle('', end='5, "role": ["entry"]'), + "bundle_file_path_is_a_number": _bundle('', end='5, "path": 5'), + **{f"span_has_no_{key}": json.dumps({"atoms": [], "spans": [ + {k: v for k, v in _SPAN.items() if k != key}]}) + for key in ("id", "kind", "text", "start", "end")}, + "atoms_is_a_number": '{"atoms": 5, "spans": []}', + "spans_is_null": '{"atoms": [], "spans": null}', + **{f"bundle_{key}_is_falsy": json.dumps({"atoms": [], "spans": [], "bundle": {key: value}}) + for key, value in (("max_file_words", False), ("max_total_words", False), + ("n_files", False), ("resource_words_measured", False), + ("artifact_name", 0), ("resource_roles", ""), + ("files", 0), ("excluded", 0))}, + "extraction_is_falsy": json.dumps({"atoms": [], "spans": [], "extraction": 0}), + "extraction_unread_is_falsy": json.dumps( + {"atoms": [], "spans": [], "extraction": {"unread_span_ids": ""}}), + "span_atom_ids_has_a_number": _record(span={"atom_ids": [1]}), + "atom_answer_is_a_number": _record(atom={"answer": 5}), + "atom_kind_is_a_list": _record(atom={"kind": ["x"]}), + "atom_faithfulness_is_a_number": _record(atom={"faithfulness": 5}), + "atom_tags_is_a_string": _record(atom={"tags": "x"}), + "atom_meta_is_a_list": _record(atom={"meta": [1]}), + "extraction_is_a_list": json.dumps({"atoms": [], "spans": [], "extraction": [1]}), + "extraction_unread_has_a_list": json.dumps( + {"atoms": [], "spans": [], "extraction": {"unread_span_ids": [[1]]}}), + "bundle_file_has_no_path": '{"atoms": [], "spans": [], "bundle": {"include_resources": ' + 'true, "files": [{"role": "entry", "end": 5}]}}', + "bundle_file_start_is_a_string": _bundle('', end='5, "start": "x"'), + "bundle_file_sha256_is_a_number": _bundle('', end='5, "sha256": 5'), + "bundle_artifact_name_is_a_list": _bundle('"artifact_name": ["s"]'), + "bundle_n_files_is_a_string": _bundle('"n_files": "1"'), + "bundle_resource_words_is_a_list": _bundle('"resource_words_measured": [1]'), + "bundle_excluded_role_is_a_number": _bundle('"excluded": [{"path": "a", "role": 5}]'), + "bundle_excluded_words_is_a_string": _bundle('"excluded": [{"path": "a", "words": "x"}]'), + "nested_deeply": "[" * 100000 + "]" * 100000, + "atoms_nested_deeply": '{"atoms": ' + "[" * 100000 + "]" * 100000 + "}", +} + + +def _atoms_file(tmp_path, kind): + path = tmp_path / "atoms.json" + if kind == "invalid_json": + path.write_text("{not json", encoding="utf-8") + elif kind == "not_utf8": + path.write_bytes(b"\xff\xfe\x00") + elif kind == "directory": + path.mkdir() + elif kind in _WRONG_SHAPES: + path.write_text(_WRONG_SHAPES[kind], encoding="utf-8") + return path + + +@pytest.mark.parametrize("sub", ["probe", "shorten"]) +@pytest.mark.parametrize("kind", ["missing", "invalid_json", "not_utf8", "directory", *_WRONG_SHAPES]) +def test_an_unreadable_atoms_file_exits_with_a_message_naming_it(tmp_path, sub, kind): + skill = tmp_path / "SKILL.md" + skill.write_text("---\nname: s\ndescription: d\n---\n\n# H\n\nText.\n", encoding="utf-8") + atoms = _atoms_file(tmp_path, kind) + args = build_parser().parse_args( + ["novelty", sub, "--artifact", f"skill:{skill}", "--atoms", str(atoms)] + ) + with pytest.raises(SystemExit) as excinfo: + args.func(args) + message = str(excinfo.value) + assert message.startswith("cannot read atoms from ") + assert str(atoms) in message + assert "has no attribute" not in message + + +@pytest.mark.parametrize("bundle", ['null', '{}', '{"files": []}', + '{"files": [{"path": "SKILL.md", "sha256": null}]}']) +def test_a_bundle_block_of_the_right_shape_is_not_refused(tmp_path, bundle): + from dokimi.novelty.atoms import load_atoms_or_exit + + path = tmp_path / "atoms.json" + path.write_text('{"atoms": [], "spans": [], "bundle": %s}' % bundle, encoding="utf-8") + atoms, spans, record = load_atoms_or_exit(path) + assert (atoms, spans, record["bundle"]) == ([], [], json.loads(bundle)) + + +def test_an_atom_with_only_a_statement_is_accepted(tmp_path): + from dokimi.novelty.atoms import load_atoms_or_exit + + path = tmp_path / "atoms.json" + span = {"id": "s0001", "kind": "sentence", "text": "S.", "start": 0, "end": 2} + path.write_text(json.dumps({"atoms": [{"statement": "S."}], "spans": [span]}), encoding="utf-8") + atoms, spans, _ = load_atoms_or_exit(path) + assert [(a.statement, a.question, a.span_ids) for a in atoms] == [("S.", "", ())] + assert [(s.id, s.heading_path, s.protection) for s in spans] == [("s0001", (), ())] + + +@pytest.mark.parametrize("record, fragment", [ + pytest.param(_record(span={"text": 5}), "'text' is not a string", id="span_text"), + pytest.param(_record(span={"start": True}), "'start' is not an integer", + id="span_start_bool"), + pytest.param(_record(span={"end": None}), "'end' is not an integer", + id="span_end_null"), + pytest.param(_record(span={"id": [1]}), "'spans' has a row whose 'id' is not a string", id="span_id"), + pytest.param(_record(span={"kind": 5}), "'kind' is not a string", id="span_kind"), + pytest.param(_record(span={"protection": [1]}), "'protection' is not a list", + id="span_protection"), + pytest.param(_record(span={"heading_path": "x"}), "'heading_path' is not a list", + id="span_heading_path"), + pytest.param(_record(atom={"id": [1]}), "'atoms' has a row whose 'id' is not a string", + id="atom_id"), + pytest.param(_record(atom={"question": 5}), "'question' is not a string", + id="atom_question"), + pytest.param(_record(atom={"statement": None}), + "'statement' is not a string", id="atom_statement"), + pytest.param(_record(atom={"span_ids": "s0001"}), "'span_ids' is not a list of strings", + id="atom_span_ids"), + pytest.param(json.dumps({"atoms": [], "spans": "x"}), "'spans' is not a list of objects", + id="spans_not_a_list"), + pytest.param(json.dumps({"atoms": [5], "spans": []}), "'atoms' is not a list of objects", + id="atoms_has_a_number"), +]) +def test_a_wrong_span_or_atom_field_is_named_in_the_refusal(tmp_path, record, fragment): + from dokimi.novelty.atoms import load_atoms_or_exit + + path = tmp_path / "atoms.json" + path.write_text(record, encoding="utf-8") + with pytest.raises(SystemExit) as exc: + load_atoms_or_exit(path) + assert fragment in str(exc.value) + + +@pytest.mark.parametrize("kind, fragment", [ + ("span_has_no_id", "'spans' has a row without 'id'"), + ("span_atom_ids_has_a_number", "'atom_ids' is not a list of strings"), + ("atom_answer_is_a_number", "'answer' is not a string"), + ("atom_kind_is_a_list", "'atoms' has a row whose 'kind' is not a string"), + ("atom_faithfulness_is_a_number", "'faithfulness' is not a string"), + ("atom_tags_is_a_string", "'tags' is not a list of strings"), + ("atom_meta_is_a_list", "'meta' is not an object"), + ("extraction_is_a_list", "'extraction' is not an object"), + ("extraction_unread_has_a_list", "'extraction.unread_span_ids' is not a list of strings"), + ("bundle_file_has_no_path", "'bundle.files' has a row without 'path'"), + ("bundle_file_start_is_a_string", "'start' is not an integer"), + ("bundle_file_sha256_is_a_number", "'sha256' is not a string or null"), + ("bundle_artifact_name_is_a_list", "'bundle.artifact_name' is not a string"), + ("bundle_n_files_is_a_string", "'bundle.n_files' is not an integer"), + ("bundle_resource_words_is_a_list", "'bundle.resource_words_measured' is not an integer"), + ("bundle_excluded_role_is_a_number", "'bundle.excluded' has a row whose 'role'"), + ("bundle_excluded_words_is_a_string", "'bundle.excluded' has a row whose 'words'"), +]) +def test_every_field_a_reader_uses_is_checked(tmp_path, kind, fragment): + from dokimi.novelty.atoms import load_atoms_or_exit + + with pytest.raises(SystemExit) as exc: + load_atoms_or_exit(_atoms_file(tmp_path, kind)) + assert fragment in str(exc.value) + + +def test_a_file_row_without_a_role_is_left_to_the_answer_key_check(tmp_path): + from dokimi.novelty.atoms import load_atoms_or_exit + + path = tmp_path / "atoms.json" + path.write_text(json.dumps({"atoms": [], "spans": [], "bundle": { + "files": [{"path": "SKILL.md", "end": 5}]}}), encoding="utf-8") + assert load_atoms_or_exit(path)[2]["bundle"]["files"][0]["path"] == "SKILL.md" diff --git a/dokimi/tests/test_novelty_bundle_key.py b/dokimi/tests/test_novelty_bundle_key.py index 32193192..daca296a 100644 --- a/dokimi/tests/test_novelty_bundle_key.py +++ b/dokimi/tests/test_novelty_bundle_key.py @@ -73,6 +73,33 @@ def test_a_key_for_an_artifact_that_cannot_be_loaded_is_refused(tmp_path): assert (verdict.ok, verdict.problem) == (False, "unloadable") + +def test_a_key_with_a_recorded_bundle_whose_skill_is_nested_too_deeply_is_refused(tmp_path): + from dokimi.novelty.bundle import check_answer_key + + root = tmp_path / "deep" + root.mkdir() + (root / "SKILL.md").write_text( + "---\nname: deep\ndescription: " + "[" * 50000 + "]" * 50000 + "\n---\n\nBody.\n", + encoding="utf-8") + record = {"files": [{"path": "SKILL.md", "role": "entry_point", "words": 3, "end": 5}], + "resource_roles": []} + verdict = check_answer_key(f"skill:{root}", record, source="a.json") + assert (verdict.ok, verdict.problem) == (False, "unloadable") + assert "maximum recursion" in verdict.message + + +def test_a_key_for_an_entry_point_that_cannot_be_read_is_refused(tmp_path): + from dokimi.novelty.bundle import check_answer_key + + # An entry point that exists but is not readable (here: a directory) is an OSError + # other than FileNotFoundError; it must come back as a verdict, not a traceback. + root = tmp_path / "unreadable" + (root / "SKILL.md").mkdir(parents=True) + for record in ({}, {"files": [{"path": "SKILL.md", "role": "entry_point"}]}): + verdict = check_answer_key(f"skill:{root}", record) + assert (verdict.ok, verdict.problem) == (False, "unloadable") + def test_a_record_without_usable_file_rows_is_refused(tmp_path): root = _key_skill(tmp_path) for broken in ({"include_resources": True, "files": []}, diff --git a/dokimi/tests/test_novelty_cli.py b/dokimi/tests/test_novelty_cli.py index a314a53f..8eb6e478 100644 --- a/dokimi/tests/test_novelty_cli.py +++ b/dokimi/tests/test_novelty_cli.py @@ -121,6 +121,77 @@ def test_a_missing_artifact_exits_with_a_message(capsys): +def _non_utf8_file(tmp_path): + path = tmp_path / "bin.txt" + path.write_bytes(b"\xff\xfe\x00") + return path + + +@pytest.mark.parametrize("make_path", [ + lambda tmp_path: tmp_path / "missing.txt", + lambda tmp_path: tmp_path, + _non_utf8_file, +], ids=["missing", "directory", "not-utf8"]) +def test_an_unreadable_placebo_text_exits_with_a_message(tmp_path, make_path): + skill = tmp_path / "skill" + skill.mkdir() + (skill / "SKILL.md").write_text("---\nname: s\ndescription: d\n---\n\n# S\n\nText.\n", + encoding="utf-8") + placebo = make_path(tmp_path) + args = _parse("novelty", "analyze", "--artifact", f"skill:{skill}", + "--placebo-text", str(placebo)) + with pytest.raises(SystemExit, match=f"^cannot read --placebo-text {re.escape(str(placebo))}"): + args.func(args) + + +@pytest.mark.parametrize("sub", ["analyze", "probe"]) +def test_placebo_text_is_read_before_a_model_client_is_built(tmp_path, monkeypatch, sub): + skill = tmp_path / "skill" + skill.mkdir() + (skill / "SKILL.md").write_text("---\nname: s\ndescription: d\n---\n\n# S\n\nText.\n", + encoding="utf-8") + argv = ["novelty", sub, "--artifact", f"skill:{skill}", + "--placebo-text", str(tmp_path / "missing.txt")] + if sub == "probe": + atoms = tmp_path / "atoms.json" + atoms.write_text('{"atoms": [], "spans": [], "bundle": {}}', encoding="utf-8") + argv += ["--atoms", str(atoms)] + + def built(*_args, **_kwargs): + raise AssertionError("a model client was built before --placebo-text was read") + + monkeypatch.setattr("dokimi.commands.novelty._clients", built) + args = _parse(*argv) + with pytest.raises(SystemExit, match="^cannot read --placebo-text"): + args.func(args) + + +@pytest.mark.parametrize(("arms", "placebo", "message"), [ + ("no_artifact,full,placebo", "", "^the placebo arm needs --placebo-text"), + ("no_artifact,full,placebo", None, "^the placebo arm needs --placebo-text"), + ("no_artifact,full,bogus", None, r"^unknown novelty arms: \['bogus'\]"), +]) +def test_analyze_exits_with_the_message_when_the_arms_cannot_run( + tmp_path, monkeypatch, arms, placebo, message): + from tests.test_novelty_analyze import SKILL + + skill = tmp_path / "skill" + skill.mkdir() + (skill / "SKILL.md").write_text(SKILL, encoding="utf-8") + argv = ["novelty", "analyze", "--artifact", f"skill:{skill}", "--arms", arms] + if placebo is not None: + (tmp_path / "placebo.txt").write_text(placebo, encoding="utf-8") + argv += ["--placebo-text", str(tmp_path / "placebo.txt")] + + def built(*_args, **_kwargs): + raise AssertionError("a model client was built for arms that cannot run") + + monkeypatch.setattr("dokimi.commands.novelty._clients", built) + args = _parse(*argv) + with pytest.raises(SystemExit, match=message): + args.func(args) + + def test_coverage_note_printed_to_stderr_for_atoms(tmp_path, monkeypatch, capsys): """The coverage note is printed to stderr on entry-point run.""" skill = tmp_path / "s" / "SKILL.md" @@ -395,6 +466,21 @@ def test_probe_refuses_a_stale_answer_key_before_building_a_client(tmp_path): args.func(args) +def test_probe_exits_with_a_message_on_an_unreadable_placebo_text(tmp_path): + from dokimi.novelty.atoms import save_atoms + from dokimi.novelty.bundle import load_bundle + + root = _bundle_skill(tmp_path) + bundle = load_bundle(f"skill:{root}", include_resources=True) + atoms = tmp_path / "atoms.json" + save_atoms([], [], atoms, extractor="e", cap=None, bundle=bundle.summary()) + missing = tmp_path / "missing.txt" + args = _parse("novelty", "probe", "--artifact", f"skill:{root}", "--atoms", str(atoms), + "--placebo-text", str(missing)) + with pytest.raises(SystemExit, match=f"^cannot read --placebo-text {re.escape(str(missing))}"): + args.func(args) + + def test_shorten_restricts_a_bundle_answer_key_to_the_entry_point(tmp_path, capsys): from dokimi.novelty.atoms import save_atoms from dokimi.novelty.bundle import load_bundle @@ -439,6 +525,42 @@ def test_item_analysis_refuses_a_file_that_carries_no_probe_runs(tmp_path): args.func(args) +@pytest.mark.parametrize("content", [ + None, "{not json", b"\xff\xfe\x00bad", + pytest.param("[" * 100000 + "]" * 100000, id="nested_deeply"), +]) +def test_item_analysis_exits_with_a_message_on_an_unreadable_probes_file(tmp_path, content): + path = tmp_path / "novelty.json" + if isinstance(content, str): + path.write_text(content, encoding="utf-8") + elif content is not None: + path.write_bytes(content) + args = _parse("novelty", "items", "--probes", str(path)) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert str(exc.value).startswith(str(path)) # 3.14 parses deep nesting, then refuses the list + + +@pytest.mark.parametrize("top, run", [ + ({"model": ["m"]}, {}), + ({"model": None}, {}), + ({"artifact": {"a": 1}}, {}), + ({"ref_digest": [1]}, {}), + ({}, {"atom_id": ["a"]}), + ({}, {"probe_kind": ["x"]}), + ({}, {"arm": {}}), + ({}, {"verdict": ["x"]}), +], ids=["model", "model_null", "artifact", "ref_digest", "atom_id", "probe_kind", "arm", "verdict"]) +def test_item_analysis_refuses_a_probe_field_that_is_not_a_string(tmp_path, top, run): + path = tmp_path / "novelty.json" + body = {"model": "m", "runs": [{"atom_id": "a1", "probe_kind": "direct", "arm": "full", + "verdict": "correct", **run}], **top} + path.write_text(json.dumps(body), encoding="utf-8") + args = _parse("novelty", "items", "--probes", str(path)) + with pytest.raises(SystemExit, match="not a string"): + args.func(args) + + def test_item_analysis_over_three_models_reports_discrimination(tmp_path, capsys): def probes_file(name, outcomes): path = tmp_path / name @@ -722,3 +844,45 @@ def test_a_bundle_run_capped_by_the_budget_warns_without_advising_the_flag_it_pa assert "coverage: 25% of the artifact's readable words were measured\n" in out assert "WARNING: only 25%" in out assert "--include-resources" not in out + + +def _deep_skill(tmp_path, name="deep"): + root = tmp_path / name + root.mkdir() + (root / "SKILL.md").write_text( + f"---\nname: {name}\ndescription: " + "[" * 5000 + "\n---\n\nBody.\n", encoding="utf-8") + return f"skill:{root}" + + +def test_load_artifact_or_exit_names_a_frontmatter_nested_too_deeply(tmp_path): + from dokimi.novelty.artifact import load_artifact_or_exit + + with pytest.raises(SystemExit, match="^cannot load --artifact skill:.*maximum recursion"): + load_artifact_or_exit(_deep_skill(tmp_path)) + + +@pytest.mark.parametrize("sub", ["analyze", "atoms", "probe", "shorten", "routing"]) +def test_a_command_given_a_frontmatter_nested_too_deeply_exits_with_a_message(tmp_path, sub): + atoms = tmp_path / "atoms.json" + atoms.write_text('{"atoms": [], "spans": []}', encoding="utf-8") + extra = { + "probe": ["--atoms", str(atoms)], + "shorten": ["--atoms", str(atoms)], + "routing": ["--competitors", _deep_skill(tmp_path, "other")], + }.get(sub, []) + args = _parse("novelty", sub, "--artifact", _deep_skill(tmp_path), *extra) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert "maximum recursion" in str(exc.value) + assert "RecursionError" not in str(exc.value) + + +def test_item_analysis_accepts_a_probe_file_that_records_no_digest(tmp_path): + from dokimi.novelty.items import load_probe_cells + + path = tmp_path / "novelty.json" + path.write_text(json.dumps({"model": "m", "artifact": "skill:x", "ref_digest": None, "runs": [ + {"atom_id": "a1", "probe_kind": "direct", "arm": "full", "verdict": "correct"}]}), + encoding="utf-8") + cells, _ = load_probe_cells([str(path)]) + assert (cells[0]["artifact"], cells[0]["ref_digest"]) == ("skill:x", None) diff --git a/dokimi/tests/test_novelty_routing.py b/dokimi/tests/test_novelty_routing.py index 128ff3a1..f209122b 100644 --- a/dokimi/tests/test_novelty_routing.py +++ b/dokimi/tests/test_novelty_routing.py @@ -260,6 +260,18 @@ def test_a_competitor_that_is_also_in_the_catalog_is_counted_once(tmp_path): assert json.loads(out.read_text(encoding="utf-8"))["n_candidates"] == 3 # perf, xpu once, db +@pytest.mark.parametrize("content", [None, b"\xff\xfe queries"]) +def test_an_unreadable_queries_file_exits_with_a_message(tmp_path, content): + queries = tmp_path / "q.txt" + if content is not None: + queries.write_bytes(content) + args = _parse("novelty", "routing", "--artifact", _skill(tmp_path, "a", "d"), + "--competitors", _skill(tmp_path, "b", "e"), "--queries", str(queries)) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert str(exc.value).startswith(f"cannot read --queries {queries}: ") + + def test_a_catalog_that_is_not_a_directory_is_refused(tmp_path): args = _parse("novelty", "routing", "--artifact", _skill(tmp_path, "a", "d"), "--catalog", str(tmp_path / "nope"), "--queries-from-headings") @@ -470,13 +482,22 @@ def test_a_query_file_is_a_list_a_queries_key_or_one_per_line(tmp_path, content, assert load_queries(str(path)) == queries -def test_a_query_file_that_is_not_strings_is_refused_but_a_missing_one_is_not_masked( - tmp_path): +@pytest.mark.parametrize("deep", [ + pytest.param("[" * 100000 + "]" * 100000, id="recursion"), + pytest.param("[" + "1" * 6000 + "]", id="integer_digits"), +]) +def test_a_query_file_nested_too_deeply_to_parse_is_read_one_query_per_line(tmp_path, deep): + path = tmp_path / "q.json" + path.write_text(deep + "\nprofile a binary\n", encoding="utf-8") + assert load_queries(str(path)) == [deep, "profile a binary"] + + +def test_a_query_file_that_is_not_strings_or_is_missing_is_refused(tmp_path): path = tmp_path / "q.json" path.write_text(json.dumps({"queries": "a b"}), encoding="utf-8") with pytest.raises(RoutingInputError, match="list of strings"): load_queries(str(path)) - with pytest.raises(FileNotFoundError): + with pytest.raises(RoutingInputError, match="cannot read --queries .*missing.json"): load_queries(str(tmp_path / "missing.json")) diff --git a/dokimi/tests/test_novelty_settings_parity.py b/dokimi/tests/test_novelty_settings_parity.py index 2a704453..03fea006 100644 --- a/dokimi/tests/test_novelty_settings_parity.py +++ b/dokimi/tests/test_novelty_settings_parity.py @@ -239,6 +239,10 @@ def _kept_estimate(step: dict) -> dict: return {**step, "stdout": payload, "stderr": stderr} +# Runs main refuses only after the extraction calls it had already made. +REFUSED_BEFORE_ANY_CALL = {"run/arms-without-full"} + + @pytest.mark.parametrize("name", list(CASES)) def test_no_setting_bill_or_tier_record_differs_from_main(name, tmp_path): golden = json.loads(GOLDEN.read_text(encoding="utf-8")) @@ -251,7 +255,12 @@ def test_no_setting_bill_or_tier_record_differs_from_main(name, tmp_path): assert [_kept_estimate(step) for step in steps] == [ _kept_estimate(step) for step in golden[name]] else: - assert steps == golden[name] + expected = golden[name] + if name in REFUSED_BEFORE_ANY_CALL: + # Intended change: the arm and placebo checks now run before the clients are + # built, so the refusal makes no model call. Same status and output as on main. + expected = [dict(step, calls=[]) for step in expected] + assert steps == expected def test_the_golden_cases_are_the_ones_this_module_runs(): diff --git a/dokimi/tests/test_novelty_shorten_artifact.py b/dokimi/tests/test_novelty_shorten_artifact.py new file mode 100644 index 00000000..cb553cbc --- /dev/null +++ b/dokimi/tests/test_novelty_shorten_artifact.py @@ -0,0 +1,72 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""`novelty shorten`: an artifact that cannot be loaded exits with a message, not a traceback.""" + +from __future__ import annotations + +import pytest + +from dokimi.cli import build_parser +from dokimi.novelty.atoms import save_atoms +from dokimi.novelty.spans import segment + + +def _shorten(tmp_path, artifact): + atoms = tmp_path / "atoms.json" + save_atoms([], segment("# H\n\nSome text.\n"), atoms, extractor="e", cap=None) + args = build_parser().parse_args([ + "novelty", "shorten", "--artifact", artifact, "--atoms", str(atoms), "--lexical-only", + ]) + args.func(args) + + +def test_a_missing_text_artifact_exits_naming_it(tmp_path): + missing = tmp_path / "missing.md" + with pytest.raises(SystemExit) as exc: + _shorten(tmp_path, f"text:{missing}") + assert str(exc.value) == f"cannot load --artifact text:{missing}: artifact not found: {missing}" + + +def test_a_non_utf8_text_artifact_exits_naming_it(tmp_path): + binary = tmp_path / "bin.md" + binary.write_bytes(bytes([0xFF, 0xFE, 0x00])) + with pytest.raises(SystemExit) as exc: + _shorten(tmp_path, f"text:{binary}") + assert str(exc.value).startswith(f"cannot load --artifact text:{binary}: ") + + +def test_a_skill_file_without_a_closing_frontmatter_fence_exits_naming_it(tmp_path, monkeypatch): + from types import SimpleNamespace + + entry = tmp_path / "SKILL.md" + entry.write_text("---\nname: s\ndescription: d\n\n# H\n", encoding="utf-8") + monkeypatch.setattr("dokimi.skills.loader.load_skill", lambda _p: SimpleNamespace( + path=entry, name="s", description="d", body="# H\n")) + with pytest.raises(SystemExit) as exc: + _shorten(tmp_path, f"skill:{tmp_path}") + assert str(exc.value) == (f"cannot load --artifact skill:{tmp_path}: " + f"{entry} has no closing '---' after its frontmatter") + + +def _unreadable_artifact(tmp_path, kind): + if kind == "text_directory": + target = tmp_path / "dir.md" + target.mkdir() + return f"text:{target}" + skill = tmp_path / "skill" + (skill / "SKILL.md").mkdir(parents=True) + return f"skill:{skill}" + + +@pytest.mark.parametrize("kind", ["text_directory", "skill_md_is_directory"]) +@pytest.mark.parametrize("command", ["analyze", "atoms", "routing"]) +def test_an_unreadable_artifact_exits_without_a_traceback(tmp_path, command, kind): + artifact = _unreadable_artifact(tmp_path, kind) + argv = ["novelty", command, "--artifact", artifact] + if command == "routing": + argv += ["--queries-from-headings"] + args = build_parser().parse_args(argv) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert str(exc.value).startswith("cannot load ") diff --git a/dokimi/tests/test_novelty_view_inputs.py b/dokimi/tests/test_novelty_view_inputs.py new file mode 100644 index 00000000..609068c1 --- /dev/null +++ b/dokimi/tests/test_novelty_view_inputs.py @@ -0,0 +1,92 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""`novelty view`: an unreadable or invalid --novelty or --atoms file exits 1 with a message.""" + +from __future__ import annotations + +import pytest + +from dokimi.cli import build_parser +from tests.test_novelty_atoms_input import _WRONG_SHAPES + + +def _view(novelty, atoms, out): + args = build_parser().parse_args([ + "novelty", "view", "--novelty", str(novelty), "--atoms", str(atoms), + "--format", "md", "--out", str(out), + ]) + args.func(args) + + +def _err(tmp_path, novelty, atoms, capsys): + out = tmp_path / "annotated.md" + with pytest.raises(SystemExit) as exc: + _view(novelty, atoms, out) + assert exc.value.code == 1 + assert not out.exists() + return capsys.readouterr().err + + +def test_a_missing_novelty_file_is_reported_as_unreadable(tmp_path, capsys): + missing = tmp_path / "missing.json" + assert f"Error: cannot read {missing}" in _err(tmp_path, missing, tmp_path / "a.json", capsys) + + +def test_a_directory_as_novelty_is_reported_as_unreadable(tmp_path, capsys): + assert f"Error: cannot read {tmp_path}" in _err(tmp_path, tmp_path, tmp_path / "a.json", capsys) + + +@pytest.mark.parametrize("payload", [b"", b"{not json", bytes([0xFF, 0xFE]), b"[1, 2]", b"{}", pytest.param(b"[" * 100000 + b"]" * 100000, id="nested_deeply")]) +def test_a_novelty_file_that_is_not_a_novelty_artifact_is_reported_as_invalid(tmp_path, capsys, payload): + bad = tmp_path / "bad.json" + bad.write_bytes(payload) + assert f"Error: {bad} is not a valid novelty artifact" in _err(tmp_path, bad, tmp_path / "a.json", capsys) + + +def test_a_missing_atoms_file_is_reported_as_unreadable(tmp_path, capsys): + from dokimi.artifacts import save_artifact + from tests.test_novelty_cli import _minimal_artifact + + novelty = tmp_path / "novelty.json" + save_artifact("novelty", _minimal_artifact(), novelty) + missing = tmp_path / "missing-atoms.json" + assert f"Error: cannot read {missing}" in _err(tmp_path, novelty, missing, capsys) + + +@pytest.mark.parametrize("payload", [ + b"", b"{not json", bytes([0xFF, 0xFE]), b"[1, 2]", pytest.param(b"[" * 100000 + b"]" * 100000, id="nested_deeply"), + b'{"atoms": [{"unexpected": 1}]}', b'{"atoms": "x"}', b'{"spans": [1]}', +]) +def test_an_atoms_file_that_is_not_an_atoms_record_is_reported_as_invalid(tmp_path, capsys, payload): + from dokimi.artifacts import save_artifact + from tests.test_novelty_cli import _minimal_artifact + + novelty = tmp_path / "novelty.json" + save_artifact("novelty", _minimal_artifact(), novelty) + bad = tmp_path / "atoms.json" + bad.write_bytes(payload) + assert f"Error: {bad} is not a valid atoms file" in _err(tmp_path, novelty, bad, capsys) + + +def test_an_atoms_error_names_the_exception_without_its_repr_quotes(tmp_path, capsys): + from dokimi.artifacts import save_artifact + from tests.test_novelty_cli import _minimal_artifact + + novelty = tmp_path / "novelty.json" + save_artifact("novelty", _minimal_artifact(), novelty) + bad = tmp_path / "atoms.json" + bad.write_text('{"atoms": [{"unexpected": 1}]}', encoding="utf-8") + assert f"Error: {bad} is not a valid atoms file: " in _err(tmp_path, novelty, bad, capsys) + + +@pytest.mark.parametrize("kind", [k for k in _WRONG_SHAPES if not k.startswith("bundle")]) +def test_an_atoms_file_of_a_wrong_shape_is_reported_as_invalid_by_view(tmp_path, capsys, kind): + from dokimi.artifacts import save_artifact + from tests.test_novelty_cli import _minimal_artifact + + novelty = tmp_path / "novelty.json" + save_artifact("novelty", _minimal_artifact(), novelty) + bad = tmp_path / "atoms.json" + bad.write_text(_WRONG_SHAPES[kind], encoding="utf-8") + assert f"Error: {bad} is not a valid atoms file" in _err(tmp_path, novelty, bad, capsys)