fix(cli): novelty subcommands exit with a message on unreadable or invalid input

dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-novelty.md

@@ -0,0 +1,3 @@
+### Fixed
+
+- **`novelty` subcommands** exit with a message on an unreadable or invalid `--placebo-text`, `--probes`, `--atoms`, `--novelty`, `--queries` or `--artifact`, or an `--atoms` file whose `bundle` is not an object, instead of a traceback.

dokimi/changelog.d/20261026_000000_vchamarthi_atoms-bundle-values.md

@@ -0,0 +1,3 @@
+### Fixed
+
+- **`novelty probe` and `novelty shorten`** exit with a message on an `--atoms` file whose spans, atoms, `extraction` or `bundle` have a missing or wrong-type field or whose JSON is nested too deeply to parse, and `novelty view` does the same for deep nesting, instead of a traceback.

dokimi/dokimi/commands/inputs.py

@@ -0,0 +1,38 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""Reading a file a command was pointed at, with an exit that names it."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+
+def read_text_or_exit(path: str | None, flag: str) -> str | None:
+    """The text of the file `flag` names, None when the flag is unset.
+
+    A file that cannot be read stops the command with a message naming the flag and the
+    path, instead of a traceback.
+    """
+    if not path:
+        return None
+    try:
+        return Path(path).read_text(encoding="utf-8")
+    except (OSError, UnicodeDecodeError) as exc:
+        raise SystemExit(f"cannot read {flag} {path}: {exc}") from None
+
+
+def placebo_text_for_arms_or_exit(path: str | None, arms: tuple[str, ...]) -> str | None:
+    """The ``--placebo-text`` of an analyze run, checked against its arms before any model call.
+
+    A typo in --arms or --placebo-text would otherwise cost the extraction calls that run
+    before the measurement checks the same thing.
+    """
+    from dokimi.novelty.measure import check_arms
+
+    placebo_text = read_text_or_exit(path, "--placebo-text")
+    try:
+        check_arms(arms, placebo_text)
+    except ValueError as exc:
+        raise SystemExit(str(exc)) from None
+    return placebo_text

dokimi/dokimi/commands/novelty.py

@@ -19,6 +19,7 @@
 from pathlib import Path
 from typing import TYPE_CHECKING, NoReturn
 
+from dokimi.commands.inputs import placebo_text_for_arms_or_exit, read_text_or_exit
 from dokimi.defaults import (
     DEFAULT_ANSWER_MODEL,
     DEFAULT_ANSWER_PROVIDER,
@@ -94,6 +95,7 @@ def cmd_novelty_analyze(args: argparse.Namespace) -> None:
             )
         return
 
+    placebo_text = placebo_text_for_arms_or_exit(args.placebo_text, _arms(args.arms))
     extractor, answerer, judge = _clients(args)
     config = AnalyzeConfig(
         artifact_ref=args.artifact,
@@ -105,10 +107,7 @@ def cmd_novelty_analyze(args: argparse.Namespace) -> None:
         budgets=tuple(args.budgets),
         concurrency=args.concurrency,
         allow_self_judge=args.allow_self_judge,
-        placebo_text=(
-            Path(args.placebo_text).read_text(encoding="utf-8")
-            if args.placebo_text else None
-        ),
+        placebo_text=placebo_text,
         cache_dir=Path(args.cache_dir) if args.cache_dir else None,
         seed=args.seed,
         include_resources=args.include_resources,
@@ -119,7 +118,7 @@ def cmd_novelty_analyze(args: argparse.Namespace) -> None:
         result = analyze(artifact_bundle.text, config,
                          extractor=extractor, answerer=answerer, judge=judge,
                          artifact_bundle=artifact_bundle)
-    except (SelfJudgeError, ExtractionError) as exc:
+    except (SelfJudgeError, ExtractionError, ValueError) as exc:
         raise SystemExit(str(exc)) from None
 
     out_dir = Path(args.out_dir) if args.out_dir else _default_dir(args.artifact)
@@ -173,21 +172,21 @@ def cmd_novelty_atoms(args: argparse.Namespace) -> None:
 
 def cmd_novelty_probe(args: argparse.Namespace) -> None:
     """Run the paired probes against a saved atom set and print the 2×2."""
-    from dokimi.novelty.atoms import load_atoms
+    from dokimi.novelty.atoms import load_atoms_or_exit
     from dokimi.novelty.bundle import check_answer_key
     from dokimi.novelty.measure import SelfJudgeError, run_measurement
     from dokimi.novelty.probes import build_probe_set
     from dokimi.novelty.report import build_probe_record
 
-    atoms, saved_spans = load_atoms(args.atoms)
+    atoms, saved_spans, saved = load_atoms_or_exit(args.atoms)
     _warn_unread(saved_spans, "the atoms probed here leave out whatever those spans assert")
-    saved = json.loads(Path(args.atoms).read_text(encoding="utf-8"))
     verdict = check_answer_key(args.artifact, saved.get("bundle"), source=args.atoms)
     if not verdict.ok:
         raise SystemExit(verdict.message)
     for note in verdict.notes:
         print(note, file=sys.stderr)
     settings = _settings(args)
+    placebo_text = read_text_or_exit(args.placebo_text, "--placebo-text")
     extractor, answerer, judge = _clients(args)
     if saved.get("extractor") == answerer.model:
         print(f"WARNING: {answerer.model} extracted these atoms, so the model under measurement "
@@ -209,9 +208,7 @@ def cmd_novelty_probe(args: argparse.Namespace) -> None:
             atoms, probe_set.probes, verdict.text, answerer=answerer, judge=judge,
             arms=_arms(args.arms), concurrency=args.concurrency,
             allow_self_judge=args.allow_self_judge, seed=args.seed,
-            placebo_text=(
-                Path(args.placebo_text).read_text(encoding="utf-8") if args.placebo_text else None
-            ),
+            placebo_text=placebo_text,
         )
     except (SelfJudgeError, ValueError) as exc:
         raise SystemExit(str(exc)) from None
@@ -320,7 +317,7 @@ def card(ref: str) -> tuple[Candidate, str]:
                       file=sys.stderr)
             print(f"--catalog {args.catalog}: {len(catalog)} competitors", file=sys.stderr)
             competitors += catalog
-    except (FileNotFoundError, ValueError) as exc:
+    except (OSError, ValueError, RecursionError) as exc:
         raise SystemExit(f"cannot load an artifact: {exc}") from None
     # Checked before the query file is read, so the message is about the real problem.
     try:
@@ -436,15 +433,14 @@ def _refuse_no_queries(args: argparse.Namespace, queries: list[str], self_derive
 
 def cmd_novelty_shorten(args: argparse.Namespace) -> None:
     """Score shortening candidates on the five axes and optionally write the best one."""
-    from dokimi.novelty.analyze import load_artifact_text
-    from dokimi.novelty.atoms import load_atoms
+    from dokimi.novelty.artifact import load_artifact_or_exit
+    from dokimi.novelty.atoms import load_atoms_or_exit
     from dokimi.novelty.bundle import entry_point_only
     from dokimi.novelty.retention import RetentionOracle, error_warning
     from dokimi.novelty.shorten import build_candidates, pick_best
 
-    atoms, spans = load_atoms(args.atoms)
-    _kind, _name, _description, frontmatter, _body = load_artifact_text(args.artifact)
-    saved = json.loads(Path(args.atoms).read_text(encoding="utf-8"))
+    atoms, spans, saved = load_atoms_or_exit(args.atoms)
+    _kind, _name, _description, frontmatter, _body = load_artifact_or_exit(args.artifact)
     try:
         spans, atoms, note = entry_point_only(saved.get("bundle"), spans, atoms)
     except ValueError as exc:
@@ -500,13 +496,11 @@ def cmd_novelty_shorten(args: argparse.Namespace) -> None:
 
 def cmd_novelty_view(args: argparse.Namespace) -> None:
     """Re-render the annotated view from a saved novelty artifact plus its atoms."""
-    from dokimi.artifacts import load_artifact
-    from dokimi.novelty.atoms import load_atoms
+    from dokimi.commands.novelty_view_inputs import load_view_inputs
     from dokimi.novelty.attribute import attribute_spans
     from dokimi.novelty.report import render_annotated_html, render_annotated_markdown
 
-    artifact = load_artifact("novelty", Path(args.novelty))
-    atoms, spans = load_atoms(args.atoms)
+    artifact, atoms, spans = load_view_inputs(args.novelty, args.atoms)
     _warn_unread(spans, "they render as `unread`, not as text that carries nothing")
     # Re-rendering needs per-atom outcomes; the artifact carries aggregates, so a view
     # rebuilt from it alone would colour by category only. Say so instead of implying the
@@ -550,7 +544,7 @@ def _bundle_from_args(args: argparse.Namespace):
             max_file_words=args.max_file_words,
             max_total_words=args.max_resource_words,
         )
-    except (FileNotFoundError, ValueError) as exc:
+    except (OSError, ValueError, RecursionError) as exc:
         raise SystemExit(f"cannot load --artifact {args.artifact}: {exc}") from None
     note = artifact_bundle.coverage_note()
     if note:

dokimi/dokimi/commands/novelty_view_inputs.py

@@ -0,0 +1,35 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""`novelty view`: read the saved novelty artifact and atoms file, or exit 1 naming the bad one."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from typing import NoReturn
+
+
+def _fail(message: str) -> NoReturn:
+    print(f"Error: {message}", file=sys.stderr)
+    raise SystemExit(1)
+
+
+def load_view_inputs(novelty_path: str, atoms_path: str):
+    """``(artifact, atoms, spans)`` from the two files; an unreadable or invalid one exits 1."""
+    from dokimi.artifacts import load_artifact
+    from dokimi.novelty.atoms import load_atoms
+
+    try:
+        artifact = load_artifact("novelty", Path(novelty_path))
+    except OSError as exc:
+        _fail(f"cannot read {novelty_path}: {exc}")
+    except (ValueError, RecursionError) as exc:  # not UTF-8, not JSON, or not a novelty artifact
+        _fail(f"{novelty_path} is not a valid novelty artifact: {exc}")
+    try:
+        atoms, spans = load_atoms(atoms_path)
+    except OSError as exc:
+        _fail(f"cannot read {atoms_path}: {exc}")
+    except (ValueError, RecursionError) as exc:  # not UTF-8, not JSON, or not an atoms record
+        _fail(f"{atoms_path} is not a valid atoms file: {exc}")
+    return artifact, atoms, spans

dokimi/dokimi/novelty/artifact.py

@@ -28,10 +28,20 @@ def load_artifact_text(ref: str) -> tuple[str, str, str, str, str]:
         skill = load_skill(ref.removeprefix("skill:"))
         # Verbatim, as shorten.render promises: a rebuilt `description: a: b` is not YAML.
         lines = skill.path.read_text(encoding="utf-8").lstrip("").splitlines()
-        end = next(i for i in range(1, len(lines)) if lines[i].strip() == "---")
+        end = next((i for i in range(1, len(lines)) if lines[i].strip() == "---"), None)
+        if end is None:
+            raise ValueError(f"{skill.path} has no closing '---' after its frontmatter")
         frontmatter = "\n".join(lines[: end + 1])
         return "skill", skill.name, skill.description, frontmatter, skill.body
     path = Path(ref.removeprefix("text:"))
     if not path.is_file():
         raise FileNotFoundError(f"artifact not found: {path}")
     return "text", path.stem, "", "", path.read_text(encoding="utf-8")
+
+
+def load_artifact_or_exit(ref: str) -> tuple[str, str, str, str, str]:
+    """:func:`load_artifact_text`, or exit naming ``ref`` when it is missing, unreadable or not UTF-8."""
+    try:
+        return load_artifact_text(ref)
+    except (OSError, ValueError, RecursionError) as exc:
+        raise SystemExit(f"cannot load --artifact {ref}: {exc}") from None

dokimi/dokimi/novelty/atoms.py

@@ -520,7 +520,68 @@ def load_atoms(path) -> tuple[list[Atom], list[Span]]:
     """
     from pathlib import Path
 
-    data = json.loads(Path(path).read_text(encoding="utf-8"))
+    return _atoms_from_record(json.loads(Path(path).read_text(encoding="utf-8")))
+
+
+def _is_int(value) -> bool:
+    return isinstance(value, int) and not isinstance(value, bool)
+
+
+def _is_str_list(value) -> bool:
+    return isinstance(value, list) and all(isinstance(item, str) for item in value)
+
+
+_STR = (lambda v: isinstance(v, str), "a string")
+_INT = (_is_int, "an integer")
+_STR_LIST = (_is_str_list, "a list of strings")
+_OBJ = (lambda v: isinstance(v, dict), "an object")
+_STR_OR_NULL = (lambda v: v is None or isinstance(v, str), "a string or null")
+
+# Every field a reader takes from a row: (type check, what it should be), and the fields
+# a row must have.
+_SPAN_FIELDS = {"id": _STR, "kind": _STR, "text": _STR, "start": _INT, "end": _INT,
+                "heading_path": _STR_LIST, "protection": _STR_LIST, "atom_ids": _STR_LIST}
+_SPAN_REQUIRED = ("id", "kind", "text", "start", "end")
+_ATOM_FIELDS = {"id": _STR, "statement": _STR, "question": _STR, "answer": _STR,
+                "kind": _STR, "faithfulness": _STR, "tags": _STR_LIST,
+                "span_ids": _STR_LIST, "meta": _OBJ}
+_ATOM_REQUIRED = ("statement",)
+_FILE_FIELDS = {"path": _STR, "role": _STR, "start": _INT, "end": _INT, "words": _INT,
+                "sha256": _STR_OR_NULL}
+_FILE_REQUIRED = ("path",)
+_EXCLUDED_FIELDS = {"path": _STR, "role": _STR, "reason": _STR, "words": _INT}
+
+
+def _check_rows(where: str, rows, fields: dict, required: tuple = ()) -> None:
+    """Raise ValueError unless ``rows`` is a list of objects whose fields have their types."""
+    if not (isinstance(rows, list) and all(isinstance(row, dict) for row in rows)):
+        raise ValueError(f"'{where}' is not a list of objects")
+    for row in rows:
+        for key in required:
+            if key not in row:
+                raise ValueError(f"'{where}' has a row without '{key}'")
+        for key, (ok, expected) in fields.items():
+            if key in row and not ok(row[key]):
+                raise ValueError(f"'{where}' has a row whose '{key}' is not {expected}")
+
+
+def _check_record_fields(data: dict) -> None:
+    """Raise ValueError unless the spans, atoms and extraction record have their types."""
+    _check_rows("spans", data.get("spans", []), _SPAN_FIELDS, _SPAN_REQUIRED)
+    _check_rows("atoms", data.get("atoms", []), _ATOM_FIELDS, _ATOM_REQUIRED)
+    extraction = data.get("extraction")
+    if extraction is not None and not isinstance(extraction, dict):
+        raise ValueError("'extraction' is not an object")
+    unread = (extraction or {}).get("unread_span_ids")
+    if unread is not None and not _is_str_list(unread):
+        raise ValueError("'extraction.unread_span_ids' is not a list of strings")
+
+
+def _atoms_from_record(data) -> tuple[list[Atom], list[Span]]:
+    """The atoms and spans of a parsed atoms record; ValueError when it is not one."""
+    if not isinstance(data, dict):
+        raise ValueError(f"the file holds {type(data).__name__}, not an object")
+    _check_record_fields(data)
     atoms = [Atom.from_dict(row) for row in data.get("atoms", [])]
     unread = set((data.get("extraction") or {}).get("unread_span_ids") or ())
     spans = [
@@ -534,3 +595,39 @@ def load_atoms(path) -> tuple[list[Atom], list[Span]]:
         for row in data.get("spans", [])
     ]
     return atoms, spans
+
+
+def _check_bundle_shape(bundle) -> None:
+    """Raise ValueError unless a ``bundle`` record has the types its readers rely on."""
+    if bundle is None:
+        return
+    if not isinstance(bundle, dict):
+        raise ValueError("'bundle' is not an object")
+    for key in ("max_file_words", "max_total_words", "n_files", "resource_words_measured"):
+        if bundle.get(key) is not None and not _is_int(bundle[key]):
+            raise ValueError(f"'bundle.{key}' is not an integer")
+    if bundle.get("artifact_name") is not None and not isinstance(bundle["artifact_name"], str):
+        raise ValueError("'bundle.artifact_name' is not a string")
+    if bundle.get("resource_roles") is not None and not _is_str_list(bundle["resource_roles"]):
+        raise ValueError("'bundle.resource_roles' is not a list of strings")
+    if bundle.get("files") is not None:
+        _check_rows("bundle.files", bundle["files"], _FILE_FIELDS, _FILE_REQUIRED)
+    if bundle.get("excluded") is not None:
+        _check_rows("bundle.excluded", bundle["excluded"], _EXCLUDED_FIELDS)
+
+
+def load_atoms_or_exit(path) -> tuple[list[Atom], list[Span], dict]:
+    """:func:`load_atoms` plus the parsed record, for a command.
+
+    Exits with the file named when it cannot be read or is not an atom set, including a
+    ``bundle`` whose fields do not have the types its readers use.
+    """
+    from pathlib import Path
+
+    try:
+        data = json.loads(Path(path).read_text(encoding="utf-8"))
+        atoms, spans = _atoms_from_record(data)
+        _check_bundle_shape(data.get("bundle"))
+    except (OSError, ValueError, RecursionError) as exc:
+        raise SystemExit(f"cannot read atoms from {path}: {exc}") from None
+    return atoms, spans, data

dokimi/dokimi/novelty/bundle.py

@@ -517,7 +517,7 @@ def check_answer_key(ref: str, record: dict | None, *,
     if not record:
         try:
             body = load_artifact_text(ref)[4]
-        except (FileNotFoundError, ValueError) as exc:
+        except (OSError, ValueError, RecursionError) as exc:
             return AnswerKeyVerdict(problem="unloadable", message=f"cannot load {ref}: {exc}")
         return AnswerKeyVerdict(text=body, notes=(
             f"WARNING: {source} predates bundle provenance; assuming the entry point only. "
@@ -530,7 +530,7 @@ def check_answer_key(ref: str, record: dict | None, *,
             "rather than probing against an answer key whose provenance cannot be checked"))
     try:
         now = load_bundle(ref, **_replay_selection(record))
-    except (FileNotFoundError, ValueError) as exc:
+    except (OSError, ValueError, RecursionError) as exc:
         return AnswerKeyVerdict(problem="unloadable", message=f"cannot load {ref}: {exc}")
 
     # An answer key is for one artifact. Without this, skill A's atoms were accepted