From b4229b7fa4fcc4353cc290d33b4103b5dadd123f Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Sun, 25 Oct 2026 16:29:40 -0500
Subject: [PATCH 01/18] fix(eval): checklist and cross-judge --grounded exit 2
 on an unreadable --questions file

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/commands/judge_reliability.py   | 18 ++++++--
 .../tests/test_eval_questions_file_errors.py  | 41 +++++++++++++++++++
 2 files changed, 55 insertions(+), 4 deletions(-)
 create mode 100644 dokimi/tests/test_eval_questions_file_errors.py

diff --git a/dokimi/dokimi/commands/judge_reliability.py b/dokimi/dokimi/commands/judge_reliability.py
index 8e2e86cec..38ca9c7a6 100644
--- a/dokimi/dokimi/commands/judge_reliability.py
+++ b/dokimi/dokimi/commands/judge_reliability.py
@@ -64,6 +64,17 @@ def _load_answers(path: str):
     return data, answers
 
 
+def _load_points(path: str):
+    """question_id -> required points from a questions file, or exit 2 naming the file."""
+    from dokimi.eval.checklist import load_points
+
+    try:
+        return load_points(path)
+    except (OSError, ValueError) as exc:  # ValueError covers bad JSON, bad UTF-8 and a refused set
+        print(f"Error: cannot read questions {path}: {exc}", file=sys.stderr)
+        sys.exit(2)
+
+
 def _require_arms(path: str, records, command: str) -> None:
     """Exit 2, before any judge call, when no answers record holds ``arms``.
 
@@ -169,11 +180,11 @@ def cmd_eval_pairwise(args: argparse.Namespace) -> None:
 
 def cmd_eval_checklist(args: argparse.Namespace) -> None:
     """Required-point coverage of every arm, against the questions' expected_topics."""
-    from dokimi.eval.checklist import checklist_jobs, load_points, run_checklist
+    from dokimi.eval.checklist import checklist_jobs, run_checklist
     from dokimi.eval.judge import Judge
 
     data, records = _load_answers(args.answers)
-    points = load_points(args.questions)
+    points = _load_points(args.questions)
     baseline = args.baseline_arm or (data.get("baseline_arm") if isinstance(data, dict) else None) or "baseline"
     arms = [a.strip() for a in args.arms.split(",")] if args.arms else None
     _require_arms(args.answers, records, "checklist")
@@ -264,7 +275,6 @@ def cmd_eval_cross_judge(args: argparse.Namespace) -> None:
         stored_measure, unreproducible_arms,
     )
     from dokimi.eval.arm_runner import _ground_truth_from_topics
-    from dokimi.eval.checklist import load_points
     from dokimi.eval.judge import Judge
 
     data, records = _load_answers(args.answers)
@@ -274,7 +284,7 @@ def cmd_eval_cross_judge(args: argparse.Namespace) -> None:
             print("Error: --grounded needs --questions (their expected_topics are the ground truth)",
                   file=sys.stderr)
             sys.exit(2)
-        truth = {q: t for q, pts in load_points(args.questions).items()
+        truth = {q: t for q, pts in _load_points(args.questions).items()
                  if (t := _ground_truth_from_topics({"expected_topics": pts}))}
 
     def scorer_of(judge):
diff --git a/dokimi/tests/test_eval_questions_file_errors.py b/dokimi/tests/test_eval_questions_file_errors.py
new file mode 100644
index 000000000..808295c25
--- /dev/null
+++ b/dokimi/tests/test_eval_questions_file_errors.py
@@ -0,0 +1,41 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""eval checklist / cross-judge --grounded: an unreadable --questions file is bad input (exit 2)."""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+
+
+def _arms_file(tmp_path):
+    path = tmp_path / "arms.json"
+    path.write_text(json.dumps({"answers": [{"question_id": "q1", "arms": {"skill": {"answer": "s"}}}]}),
+                    encoding="utf-8")
+    return path
+
+
+@pytest.mark.parametrize("command", [
+    ["checklist"],
+    ["cross-judge", "--grounded", "--no-stored"],
+], ids=["checklist", "cross-judge-grounded"])
+@pytest.mark.parametrize("content", [None, "{not json", "\xff\xfe"],
+                         ids=["missing", "malformed", "not-utf8"])
+def test_an_unreadable_questions_file_exits_2_with_a_message(tmp_path, monkeypatch, capsys, command, content):
+    from dokimi import cli
+
+    questions = tmp_path / "questions.json"
+    if content == "\xff\xfe":
+        questions.write_bytes(b"\xff\xfe\x00")
+    elif content is not None:
+        questions.write_text(content, encoding="utf-8")
+    monkeypatch.setattr("sys.argv", ["dokimi", "eval", *command, "--answers", str(_arms_file(tmp_path)),
+                                     "--questions", str(questions), "--output", str(tmp_path / "out.json")])
+    with pytest.raises(SystemExit) as exc:
+        cli.main()
+    assert exc.value.code == 2  # bad input; 1 is the verdict code of the gates
+    err = capsys.readouterr().err
+    assert err.startswith("Error: ") and str(questions) in err
+    assert not (tmp_path / "out.json").exists()

From f1df18676c2a8fd436e41c86d2d051109580474c Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Sun, 25 Oct 2026 16:29:40 -0500
Subject: [PATCH 02/18] fix(report): eval exits 1 with a message on an
 unreadable or invalid eval file

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/report/eval_report.py           | 17 +++++++--
 dokimi/tests/test_report_eval_input_errors.py | 36 +++++++++++++++++++
 2 files changed, 51 insertions(+), 2 deletions(-)
 create mode 100644 dokimi/tests/test_report_eval_input_errors.py

diff --git a/dokimi/dokimi/report/eval_report.py b/dokimi/dokimi/report/eval_report.py
index 5af77dfdd..17cdf1478 100755
--- a/dokimi/dokimi/report/eval_report.py
+++ b/dokimi/dokimi/report/eval_report.py
@@ -173,9 +173,22 @@ def _answers_by_id(answers_path: str) -> dict[str, Record]:
     return out
 
 
+def _load_eval(eval_path: str) -> dict:
+    """The eval file's JSON object, or a RecordError naming the file."""
+    try:
+        with open(eval_path, encoding="utf-8") as f:
+            data = json.load(f)
+    except OSError as exc:
+        raise RecordError(f"cannot read {eval_path}: {exc}") from exc
+    except ValueError as exc:  # not JSON, or not UTF-8
+        raise RecordError(f"{eval_path} is not valid UTF-8 JSON: {exc}") from exc
+    if not isinstance(data, dict) or not isinstance(data.get("evaluations"), list):
+        raise RecordError(f"{eval_path} is not a JSON object with an `evaluations` list")
+    return data
+
+
 def generate_report(eval_path: str, out_path: str, qa_json_out: str | None = None):
-    with open(eval_path, encoding="utf-8") as f:
-        data = json.load(f)
+    data = _load_eval(eval_path)
 
     evals = records(data["evaluations"], source=eval_path, kind="evaluations")
     meta = data.get("run_metadata", {})
diff --git a/dokimi/tests/test_report_eval_input_errors.py b/dokimi/tests/test_report_eval_input_errors.py
new file mode 100644
index 000000000..db9cbf34d
--- /dev/null
+++ b/dokimi/tests/test_report_eval_input_errors.py
@@ -0,0 +1,36 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""report eval: an unreadable --eval file is bad input (exit 1 with a message), not a traceback."""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+
+
+@pytest.mark.parametrize("content, reason", [
+    (None, "cannot read"),
+    (b"{not json", "is not valid UTF-8 JSON"),
+    (b"\xff\xfe\x00", "is not valid UTF-8 JSON"),
+    (b"[1, 2]", "`evaluations` list"),
+    (b"{}", "`evaluations` list"),
+    (json.dumps({"evaluations": {}}).encode(), "`evaluations` list"),
+], ids=["missing", "malformed", "not-utf8", "not-an-object", "no-evaluations",
+        "evaluations-not-a-list"])
+def test_report_eval_exits_1_with_a_message_on_an_unreadable_eval_file(tmp_path, monkeypatch, capsys,
+                                                                       content, reason):
+    from dokimi import cli
+
+    eval_file, out = tmp_path / "eval.json", tmp_path / "r.md"
+    if content is not None:
+        eval_file.write_bytes(content)
+    monkeypatch.setattr("sys.argv", ["dokimi", "report", "eval", "--product", "P", "--eval", str(eval_file),
+                                     "--out", str(out)])
+    with pytest.raises(SystemExit) as exc:
+        cli.main()
+    assert exc.value.code == 1
+    err = capsys.readouterr().err
+    assert err.startswith("Error: ") and str(eval_file) in err and reason in err
+    assert not out.exists()

From b5fde53e3fa85d57f988c97386b94086d37a0922 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Sun, 25 Oct 2026 16:29:40 -0500
Subject: [PATCH 03/18] fix(answers): generate exits 1 with a message on an
 unreadable or non-UTF-8 runtime config

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/commands/answers.py    |  3 +++
 dokimi/tests/test_answers_command.py | 20 ++++++++++++++++++++
 2 files changed, 23 insertions(+)

diff --git a/dokimi/dokimi/commands/answers.py b/dokimi/dokimi/commands/answers.py
index 46383d4f6..80313d009 100644
--- a/dokimi/dokimi/commands/answers.py
+++ b/dokimi/dokimi/commands/answers.py
@@ -34,6 +34,9 @@ def _load_runtime_config() -> dict:
     except yaml.YAMLError as exc:
         print(f"Error: invalid YAML in {config_path}: {exc}", file=sys.stderr)
         raise SystemExit(1) from exc
+    except (OSError, UnicodeDecodeError) as exc:  # unreadable or not UTF-8
+        print(f"Error: cannot read {config_path}: {exc}", file=sys.stderr)
+        raise SystemExit(1) from exc
     if loaded is not None and not isinstance(loaded, dict):
         print(f"Error: expected a mapping in {config_path}", file=sys.stderr)
         raise SystemExit(1)
diff --git a/dokimi/tests/test_answers_command.py b/dokimi/tests/test_answers_command.py
index 0fab43167..052ac18d1 100644
--- a/dokimi/tests/test_answers_command.py
+++ b/dokimi/tests/test_answers_command.py
@@ -63,6 +63,26 @@ def test_invalid_yaml_config_exits(tmp_path, monkeypatch, capsys):
     assert "invalid YAML" in capsys.readouterr().err
 
 
+def test_non_utf8_config_exits_with_message(tmp_path, monkeypatch, capsys):
+    path = tmp_path / "products.yaml"
+    path.write_bytes(b"retrieval:\n  top_k: \xff\xfe\n")
+    _point_config(monkeypatch, path)
+    with pytest.raises(SystemExit) as exc:
+        answers._load_runtime_config()
+    assert exc.value.code == 1
+    assert f"cannot read {path}" in capsys.readouterr().err
+
+
+def test_unreadable_config_exits_with_message(tmp_path, monkeypatch, capsys):
+    path = tmp_path / "products.yaml"
+    path.mkdir()  # a directory where the file should be: read_text raises OSError
+    _point_config(monkeypatch, path)
+    with pytest.raises(SystemExit) as exc:
+        answers._load_runtime_config()
+    assert exc.value.code == 1
+    assert f"cannot read {path}" in capsys.readouterr().err
+
+
 def test_non_mapping_config_exits(tmp_path, monkeypatch, capsys):
     path = tmp_path / "products.yaml"
     path.write_text("- a\n- b\n", encoding="utf-8")

From 8325400df3e8dab70ab85de0b7532a323641ade8 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Sun, 25 Oct 2026 16:29:40 -0500
Subject: [PATCH 04/18] fix(report): model-compare exits with a message on a
 non-UTF-8 or unreadable run file

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/report/model_compare.py |  9 ++++-----
 dokimi/tests/test_model_compare.py    | 12 ++++++++++++
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/dokimi/dokimi/report/model_compare.py b/dokimi/dokimi/report/model_compare.py
index cec86bf7d..efa6e5267 100644
--- a/dokimi/dokimi/report/model_compare.py
+++ b/dokimi/dokimi/report/model_compare.py
@@ -63,13 +63,12 @@
 def load_run(path: str) -> dict[str, Any]:
     """Load a judged arms JSON file.  Raises SystemExit with a clear message on failure."""
     try:
-        with open(path, encoding="utf-8") as f:
-            data = json.load(f)
+        data = json.loads(Path(path).read_text(encoding="utf-8"))
     except FileNotFoundError as exc:
         raise SystemExit(f"Error: File not found: {path}") from exc
-    except PermissionError as exc:
-        raise SystemExit(f"Error: Permission denied reading {path}") from exc
-    except json.JSONDecodeError as exc:
+    except OSError as exc:  # permission denied, a directory, an I/O error
+        raise SystemExit(f"Error: cannot read {path}: {exc.strerror or exc}") from exc
+    except ValueError as exc:  # JSONDecodeError, or bytes that are not UTF-8
         raise SystemExit(f"Error: Invalid JSON in {path}: {exc}") from exc
     if isinstance(data, dict):
         for section in ("answers", "evaluations"):
diff --git a/dokimi/tests/test_model_compare.py b/dokimi/tests/test_model_compare.py
index 1f66ca9cc..9a776a424 100644
--- a/dokimi/tests/test_model_compare.py
+++ b/dokimi/tests/test_model_compare.py
@@ -83,6 +83,18 @@ def test_load_run_invalid_json(tmp_path):
         load_run(str(p))
 
 
+def test_load_run_non_utf8_file_names_the_file(tmp_path):
+    p = tmp_path / "latin1.json"
+    p.write_bytes(b'{"note": "caf\xe9"}')
+    with pytest.raises(SystemExit, match=r"Error: Invalid JSON in .*latin1\.json: .*utf-8"):
+        load_run(str(p))
+
+
+def test_load_run_directory_is_an_error_message_not_a_traceback(tmp_path):
+    with pytest.raises(SystemExit, match=r"Error: cannot read .*Is a directory"):
+        load_run(str(tmp_path))
+
+
 # ---------------------------------------------------------------------------
 # check_run_consistency
 # ---------------------------------------------------------------------------

From cfde4bedf2d7d866f75aabf6a2a1dc7a15037033 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Sun, 25 Oct 2026 16:29:40 -0500
Subject: [PATCH 05/18] fix(benchmark): run exits 1 with a message on a missing
 or invalid --questions-from

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/commands/benchmark.py           | 33 ++++++++++++++-----
 .../test_benchmark_command_question_set.py    | 30 +++++++++++++++++
 2 files changed, 54 insertions(+), 9 deletions(-)

diff --git a/dokimi/dokimi/commands/benchmark.py b/dokimi/dokimi/commands/benchmark.py
index b4b1f129b..2c50c724e 100644
--- a/dokimi/dokimi/commands/benchmark.py
+++ b/dokimi/dokimi/commands/benchmark.py
@@ -110,9 +110,22 @@ def _print_plan(plan: dict) -> None:
             print(f"  {key}: {value}")
 
 
+def _check_questions_from(questions_from: str, key: str) -> tuple[Path, list, str | None]:
+    """The resolved --questions-from file, its records, and a message when it is unusable."""
+    from dokimi.orchestrator.pipeline import load_questions_payload, resolve_questions_from_path
+
+    qf = resolve_questions_from_path(Path(questions_from), key)
+    if not qf.exists():
+        return qf, [], f"questions source not found: {qf}"
+    try:
+        return qf, load_questions_payload(qf), None
+    except Exception as exc:
+        return qf, [], f"questions source invalid: {qf} ({exc})"
+
+
 def cmd_benchmark_preflight(args: argparse.Namespace) -> None:
     """Validate the resolved benchmark plan without running LLM calls."""
-    from dokimi.orchestrator.pipeline import load_questions_payload, resolve_questions_from_path
+    from dokimi.orchestrator.pipeline import load_questions_payload
     from dokimi.mcp.factory import create_doc_source_client
 
     registry = _load_registry(args)
@@ -149,15 +162,11 @@ def cmd_benchmark_preflight(args: argparse.Namespace) -> None:
         errors.append(f"doc source invalid: {plan['doc_source']} ({exc})")
 
     if plan["questions_from"]:
-        qf = resolve_questions_from_path(Path(plan["questions_from"]), entry.key)
-        if not qf.exists():
-            errors.append(f"questions source not found: {qf}")
+        qf, questions, problem = _check_questions_from(plan["questions_from"], entry.key)
+        if problem:
+            errors.append(problem)
         else:
-            try:
-                questions = load_questions_payload(qf)
-                print(f"  questions: {qf} ({len(questions)} items)")
-            except Exception as exc:
-                errors.append(f"questions source invalid: {qf} ({exc})")
+            print(f"  questions: {qf} ({len(questions)} items)")
 
     # Check golden set only if no external questions source (--questions-from takes precedence)
     if plan["questions_root"] and not plan["questions_from"]:
@@ -218,6 +227,12 @@ def cmd_benchmark_run(args: argparse.Namespace) -> None:
     questions_from = getattr(args, "questions_from", None)
     questions_root = getattr(args, "questions_root", None)
 
+    if questions_from:
+        _, _, problem = _check_questions_from(questions_from, entry.key)
+        if problem:
+            print(f"Error: {problem}", file=sys.stderr)
+            sys.exit(1)
+
     if n_runs == 1:
         _run_single_library(
             entry,
diff --git a/dokimi/tests/test_benchmark_command_question_set.py b/dokimi/tests/test_benchmark_command_question_set.py
index a02b4548f..30c4c32ba 100644
--- a/dokimi/tests/test_benchmark_command_question_set.py
+++ b/dokimi/tests/test_benchmark_command_question_set.py
@@ -369,6 +369,36 @@ def test_preflight_rejects_an_unwritable_output_dir(monkeypatch, tmp_path, capsy
     assert "output dir not writable" in capsys.readouterr().err
 
 
+@pytest.mark.parametrize(
+    "make_source, message",
+    [
+        (lambda p: p / "nowhere", "questions source not found"),
+        (lambda p: p / "bad.json", "questions source invalid"),
+    ],
+    ids=["missing", "invalid"],
+)
+def test_run_rejects_an_unusable_questions_from_before_any_call(
+    monkeypatch, tmp_path, capsys, make_source, message
+):
+    """benchmark run exits 1 with a message, not a traceback, and starts no pipeline."""
+    calls: list[object] = []
+    (tmp_path / "bad.json").write_text(json.dumps({"a": 1}), encoding="utf-8")
+    monkeypatch.setattr(benchmark, "_load_registry", lambda args: _Registry())
+    monkeypatch.setattr(benchmark, "_run_single_library", lambda *a, **k: calls.append(k))
+
+    args = build_parser().parse_args([
+        "benchmark", "run", "--library", "onetbb",
+        "--output-dir", str(tmp_path / "out"),
+        "--questions-from", str(make_source(tmp_path)),
+    ])
+    with pytest.raises(SystemExit) as exc:
+        benchmark.cmd_benchmark_run(args)
+
+    assert exc.value.code == 1
+    assert message in capsys.readouterr().err
+    assert calls == []
+
+
 @pytest.mark.outcome("pinned-questions-reuse")
 def test_multi_run_forwards_questions_root_to_every_run(monkeypatch, tmp_path):
     """Every run in a multi-run forwards the same --questions-root."""

From 1a865252343964465d4f4af563398d0ad696e723 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Sun, 25 Oct 2026 16:29:40 -0500
Subject: [PATCH 06/18] fix(evaluate): exit 1 with a message on a missing
 --custom-questions file or unusable --output-dir

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/commands/orchestrate.py    | 48 +++++++++++++-----------
 dokimi/tests/test_evaluate_cli_inputs.py | 36 ++++++++++++++++++
 2 files changed, 62 insertions(+), 22 deletions(-)
 create mode 100644 dokimi/tests/test_evaluate_cli_inputs.py

diff --git a/dokimi/dokimi/commands/orchestrate.py b/dokimi/dokimi/commands/orchestrate.py
index 2f86b55f6..f31b9bbac 100644
--- a/dokimi/dokimi/commands/orchestrate.py
+++ b/dokimi/dokimi/commands/orchestrate.py
@@ -46,28 +46,32 @@ def cmd_evaluate(args: argparse.Namespace) -> None:
         context="evaluate",
     )
 
-    # Create pipeline
-    pipeline = EvaluationPipeline(
-        product=args.product,
-        repo=getattr(args, "repo", None),
-        description=getattr(args, "description", None),
-        output_dir=Path(args.output_dir),
-        custom_questions_path=Path(args.custom_questions) if args.custom_questions else None,
-        questions_root=Path(args.questions_root) if getattr(args, "questions_root", None) else None,
-        model=args.model,
-        provider=args.provider,
-        judge_model=args.judge_model,
-        judge_provider=args.judge_provider,
-        judge_temperature=args.judge_temperature,
-        personas_count=args.personas_count,
-        questions_per_topic=args.questions_per_topic,
-        # Only pass top_k when explicitly set; otherwise let pipeline use config default
-        **({"top_k": args.top_k} if args.top_k is not None else {}),
-        rerank_threshold=args.rerank_threshold,
-        debug_retrieval=args.debug_retrieval,
-        doc_source=getattr(args, "doc_source", "context7"),
-        context7_id=getattr(args, "context7_id", None),
-    )
+    # Create pipeline; it checks --custom-questions and creates the output directories
+    try:
+        pipeline = EvaluationPipeline(
+            product=args.product,
+            repo=getattr(args, "repo", None),
+            description=getattr(args, "description", None),
+            output_dir=Path(args.output_dir),
+            custom_questions_path=Path(args.custom_questions) if args.custom_questions else None,
+            questions_root=Path(args.questions_root) if getattr(args, "questions_root", None) else None,
+            model=args.model,
+            provider=args.provider,
+            judge_model=args.judge_model,
+            judge_provider=args.judge_provider,
+            judge_temperature=args.judge_temperature,
+            personas_count=args.personas_count,
+            questions_per_topic=args.questions_per_topic,
+            # Only pass top_k when explicitly set; otherwise let pipeline use config default
+            **({"top_k": args.top_k} if args.top_k is not None else {}),
+            rerank_threshold=args.rerank_threshold,
+            debug_retrieval=args.debug_retrieval,
+            doc_source=getattr(args, "doc_source", "context7"),
+            context7_id=getattr(args, "context7_id", None),
+        )
+    except OSError as exc:  # a missing --custom-questions file, an unusable --output-dir
+        print(f"Error: {exc}", file=sys.stderr)
+        sys.exit(1)
 
     # Run pipeline
     try:
diff --git a/dokimi/tests/test_evaluate_cli_inputs.py b/dokimi/tests/test_evaluate_cli_inputs.py
new file mode 100644
index 000000000..9510c6c99
--- /dev/null
+++ b/dokimi/tests/test_evaluate_cli_inputs.py
@@ -0,0 +1,36 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""`evaluate`: a bad --custom-questions path or --output-dir exits 1 with a message."""
+
+from __future__ import annotations
+
+import pytest
+
+from dokimi.cli import build_parser
+
+
+def _evaluate(*extra: str):
+    args = build_parser().parse_args(["evaluate", "--product", "p", "--description", "d", *extra])
+    args.func(args)
+
+
+def test_a_missing_custom_questions_file_exits_1_with_a_message(tmp_path, capsys):
+    missing = tmp_path / "missing.json"
+    with pytest.raises(SystemExit) as exc:
+        _evaluate("--output-dir", str(tmp_path / "out"), "--custom-questions", str(missing))
+    assert exc.value.code == 1
+    err = capsys.readouterr().err
+    assert f"Error: --custom-questions path does not exist: {missing}" in err
+    assert "Traceback" not in err
+
+
+def test_an_unwritable_output_dir_exits_1_with_a_message(tmp_path, capsys):
+    blocker = tmp_path / "file"
+    blocker.write_text("x", encoding="utf-8")
+    with pytest.raises(SystemExit) as exc:
+        _evaluate("--output-dir", str(blocker))
+    assert exc.value.code == 1
+    err = capsys.readouterr().err
+    assert "Error: " in err and "Not a directory" in err
+    assert "Traceback" not in err

From ecc3b86afc504d60fdfccb4f8358d6fb2dd923a3 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Sun, 25 Oct 2026 16:29:40 -0500
Subject: [PATCH 07/18] fix(eval): grounding and judge-validity exit 1 with a
 message when --output cannot be written

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/commands/evaluate.py            | 19 +++--
 .../test_eval_grounding_output_errors.py      | 75 +++++++++++++++++++
 2 files changed, 87 insertions(+), 7 deletions(-)
 create mode 100644 dokimi/tests/test_eval_grounding_output_errors.py

diff --git a/dokimi/dokimi/commands/evaluate.py b/dokimi/dokimi/commands/evaluate.py
index 0618fcf63..4acda4aa2 100644
--- a/dokimi/dokimi/commands/evaluate.py
+++ b/dokimi/dokimi/commands/evaluate.py
@@ -53,6 +53,16 @@ def _load_answers(path: str):
         sys.exit(2)
 
 
+def _write_json_or_exit(path: Path, payload: dict) -> None:
+    """Write ``payload`` as JSON to ``path``, or exit 1 naming ``path`` when it cannot be written."""
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
+    except OSError as exc:
+        print(f"Error: cannot write {path}: {exc}", file=sys.stderr)
+        sys.exit(1)
+
+
 def _require_ragas() -> None:
     """Exit before any model call when the ragas extra is not installed."""
     if importlib.util.find_spec("ragas") is None:
@@ -243,9 +253,6 @@ class _Args:
 
 def cmd_eval_grounding(args: argparse.Namespace) -> None:
     """Compute reference-free grounding/citation metrics (no LLM calls)."""
-    import json
-    from pathlib import Path
-
     from dokimi.eval.grounding import evaluate_grounding
 
     answers = _load_answers(args.answers).records
@@ -260,8 +267,7 @@ def cmd_eval_grounding(args: argparse.Namespace) -> None:
 
     if args.output:
         out = Path(args.output)
-        out.parent.mkdir(parents=True, exist_ok=True)
-        out.write_text(json.dumps(result, indent=2), encoding="utf-8")
+        _write_json_or_exit(out, result)
         print(f"\n✅ Grounding report saved to {out}")
 
 
@@ -405,14 +411,13 @@ def progress(done: int, total: int) -> None:
     # Written before anything is printed: a console that cannot encode the summary must not lose the run.
     if args.output:
         out = Path(args.output)
-        out.parent.mkdir(parents=True, exist_ok=True)
         result = {**report.to_dict(), "judge_temperature": args.judge_temperature}
         # Which answers the gate was measured on: a judge that passes on one
         # answer model's output is not thereby validated on another's.
         result["summary"]["answers_model"] = run.meta.get("model")
         if typed is not None:
             result["summary"]["typed_arm"] = typed.report_fields()
-        out.write_text(json.dumps(result, indent=2), encoding="utf-8")
+        _write_json_or_exit(out, result)
 
     print()
     print(report.format_summary())
diff --git a/dokimi/tests/test_eval_grounding_output_errors.py b/dokimi/tests/test_eval_grounding_output_errors.py
new file mode 100644
index 000000000..055db51f8
--- /dev/null
+++ b/dokimi/tests/test_eval_grounding_output_errors.py
@@ -0,0 +1,75 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""`eval grounding` and `eval judge-validity`: an unwritable --output exits 1 with a message."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from dokimi.cli import build_parser
+
+
+def _grounding(tmp_path: Path, output: Path) -> None:
+    answers = tmp_path / "answers.json"
+    answers.write_text(json.dumps({"answers": [
+        {"question_id": "q1", "question_text": "How do I loop?",
+         "with_docs": {"answer": "use parallel_for",
+                       "retrieved_docs": [{"snippet": "parallel_for runs tasks"}]}},
+    ]}), encoding="utf-8")
+    args = build_parser().parse_args(["eval", "grounding", "--answers", str(answers), "--output", str(output)])
+    args.func(args)
+
+
+def test_grounding_report_is_written(tmp_path):
+    out = tmp_path / "nested" / "grounding.json"
+    _grounding(tmp_path, out)
+    assert json.loads(out.read_text(encoding="utf-8"))["schema_version"] == "grounding.v1"
+
+
+def test_output_below_a_file_exits_1_naming_the_path(tmp_path, capsys):
+    blocker = tmp_path / "blocker"
+    blocker.write_text("x", encoding="utf-8")  # a regular file: nothing can be created below it
+    target = blocker / "grounding.json"
+    with pytest.raises(SystemExit) as exc:
+        _grounding(tmp_path, target)
+    assert exc.value.code == 1
+    err = capsys.readouterr().err
+    assert f"Error: cannot write {target}" in err
+    assert "Traceback" not in err
+
+
+def test_judge_validity_output_below_a_file_exits_1_naming_the_path(tmp_path, monkeypatch, capsys):
+    import dokimi.eval.judge as judge_mod
+    import dokimi.eval.judge_validity as jv
+
+    answer = ("Use perf stat to collect IPC, then narrow the hotspot with VTune. "
+              "Enable AVX-512 in the build and expect roughly 1.5x on 8 threads. ") * 12
+    answers = tmp_path / "answers.json"
+    answers.write_text(json.dumps([
+        {"question_id": f"Q{i}", "question_text": "q", "arms": {"a": {"answer": f"{i}. " + answer}}}
+        for i in range(20)
+    ]), encoding="utf-8")
+    monkeypatch.setattr(judge_mod, "Judge", lambda **kw: None)
+    monkeypatch.setattr(jv, "judge_scorer", lambda judge: lambda lib, q, a: float(len(a)))
+    blocker = tmp_path / "blocker"
+    blocker.write_text("x", encoding="utf-8")
+    target = blocker / "validity.json"
+    args = build_parser().parse_args(["eval", "judge-validity", "--answers", str(answers), "--bases", "20",
+                                      "--variants", "truncate_20", "--output", str(target)])
+    with pytest.raises(SystemExit) as exc:
+        args.func(args)
+    assert exc.value.code == 1
+    assert f"Error: cannot write {target}" in capsys.readouterr().err
+
+
+def test_output_that_is_a_directory_exits_1_naming_the_path(tmp_path, capsys):
+    target = tmp_path / "is-a-dir"
+    target.mkdir()
+    with pytest.raises(SystemExit) as exc:
+        _grounding(tmp_path, target)
+    assert exc.value.code == 1
+    assert f"Error: cannot write {target}" in capsys.readouterr().err

From 865b81734e1d35369e8811c9c28f111209e06ef5 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Sun, 25 Oct 2026 16:29:40 -0500
Subject: [PATCH 08/18] fix(report): generate exits 1 with a message when
 --output cannot be written

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/commands/report.py       |  8 ++++++--
 dokimi/tests/test_report_cli_inputs.py | 23 +++++++++++++++++++++++
 2 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/dokimi/dokimi/commands/report.py b/dokimi/dokimi/commands/report.py
index f52f1c321..7366c13f6 100644
--- a/dokimi/dokimi/commands/report.py
+++ b/dokimi/dokimi/commands/report.py
@@ -150,8 +150,12 @@ def cmd_report_generate(args: argparse.Namespace) -> None:
 
     # Save output
     output_path = Path(args.output) if args.output else Path(f"reports/{args.product}.{args.format}")
-    output_path.parent.mkdir(parents=True, exist_ok=True)
-    output_path.write_text(report, encoding="utf-8")
+    try:
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        output_path.write_text(report, encoding="utf-8")
+    except OSError as exc:
+        print(f"Error: cannot write {output_path}: {exc}", file=sys.stderr)
+        raise SystemExit(1) from None
 
     print(f"\n✅ Generated report: {output_path}")
 
diff --git a/dokimi/tests/test_report_cli_inputs.py b/dokimi/tests/test_report_cli_inputs.py
index 0fad80339..dc417bf38 100644
--- a/dokimi/tests/test_report_cli_inputs.py
+++ b/dokimi/tests/test_report_cli_inputs.py
@@ -56,6 +56,29 @@ def test_eval_json_that_is_not_an_object_is_rejected(tmp_path, capsys, text):
     assert f"Error: {bad} must hold a JSON object" in _err(tmp_path, bad, capsys)
 
 
+@pytest.mark.parametrize("target", ["is_a_directory", "parent_is_a_file"])
+def test_an_unwritable_output_exits_1_naming_the_path(tmp_path, capsys, monkeypatch, target):
+    monkeypatch.setattr("dokimi.report.ReportGenerator.generate_report", lambda self, **kw: "report")
+    ev = tmp_path / "ev.json"
+    ev.write_text('{"evaluations": []}', encoding="utf-8")
+    questions = tmp_path / "questions.json"
+    questions.write_text('{"questions": []}', encoding="utf-8")
+    if target == "is_a_directory":
+        out = tmp_path / "out.md"
+        out.mkdir()
+    else:
+        (tmp_path / "blocker").write_text("x", encoding="utf-8")
+        out = tmp_path / "blocker" / "out.md"
+    args = build_parser().parse_args([
+        "report", "generate", "--product", "p", "--eval", str(ev),
+        "--questions", str(questions), "--output", str(out),
+    ])
+    with pytest.raises(SystemExit) as exc:
+        args.func(args)
+    assert exc.value.code == 1
+    assert f"Error: cannot write {out}" in capsys.readouterr().err
+
+
 @pytest.mark.parametrize("evaluations", ["null", "5"])
 def test_non_list_evaluations_exit_1_with_its_path(tmp_path, capsys, evaluations):
     bad = tmp_path / "evd.json"

From cbb1fc16b14b45435516f2f6308f960f5dc52848 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Sun, 25 Oct 2026 22:37:25 -0500
Subject: [PATCH 09/18] chore: changelog fragment and growth baseline

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 .../20261025_120000_vchamarthi_cli-input-eval-report.md       | 3 +++
 dokimi/scripts/growth_baseline.json                           | 4 ++--
 2 files changed, 5 insertions(+), 2 deletions(-)
 create mode 100644 dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-eval-report.md

diff --git a/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-eval-report.md b/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-eval-report.md
new file mode 100644
index 000000000..3d975eb64
--- /dev/null
+++ b/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-eval-report.md
@@ -0,0 +1,3 @@
+### Fixed
+
+- **`eval`, `report`, `answers`, `benchmark` and `evaluate`** exit with a message on an unreadable or invalid input file or an unwritable `--output`, instead of a traceback.
diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json
index d0d35275e..2c4d7eda2 100644
--- a/dokimi/scripts/growth_baseline.json
+++ b/dokimi/scripts/growth_baseline.json
@@ -4,7 +4,7 @@
     "dokimi/harnesses/benchflow.py": {"lines": 869},
     "dokimi/harnesses/docker_solver.py": {"lines": 900, "rationale": "pre-existing when the gate was introduced"},
     "dokimi/llm.py": {"lines": 1118, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"},
-    "dokimi/report/model_compare.py": {"lines": 1210, "rationale": "grew on main in #73 (judge-validity --context stored) before the ratchet landed; split it in its own PR"},
+    "dokimi/report/model_compare.py": {"lines": 1209, "rationale": "grew on main in #73 (judge-validity --context stored) before the ratchet landed; split it in its own PR"},
     "tests/test_arm_runner.py": {"lines": 1176, "rationale": "pre-existing when the gate was introduced"},
     "tests/test_benchflow_harness.py": {"lines": 1716, "rationale": "pre-existing when the gate was introduced"},
     "tests/test_container_agent.py": {"lines": 1251, "rationale": "pre-existing when the gate was introduced"},
@@ -19,7 +19,7 @@
     "dokimi/commands/arms.py::_build_arms_output": {"complexity": 13},
     "dokimi/commands/arms.py::_validate_expected_skills": {"complexity": 11},
     "dokimi/commands/benchmark.py::cmd_benchmark_batch": {"complexity": 11},
-    "dokimi/commands/benchmark.py::cmd_benchmark_preflight": {"complexity": 22},
+    "dokimi/commands/benchmark.py::cmd_benchmark_preflight": {"complexity": 21},
     "dokimi/commands/catalog.py::cmd_catalog_trigger_set": {"complexity": 16},
     "dokimi/commands/evaluate.py::cmd_eval_judge_validity": {"complexity": 29, "rationale": "grew on main in #73 (judge-validity --context stored) from 25 to 29 before the ratchet landed; split it in its own PR"},
     "dokimi/commands/judge_reliability.py::cmd_eval_cross_judge": {"complexity": 32, "rationale": "pre-existing when the gate was introduced"},

From 0644d19d8bc5714369653e7dd9cd466dbbeaf15f Mon Sep 17 00:00:00 2001
From: Anton Volkov <antonwolfy@gmail.com>
Date: Mon, 26 Oct 2026 13:25:36 +0000
Subject: [PATCH 10/18] test(evaluate): do not depend on the OS message for an
 unwritable dir

Signed-off-by: Anton Volkov <antonwolfy@gmail.com>
---
 dokimi/tests/test_evaluate_cli_inputs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dokimi/tests/test_evaluate_cli_inputs.py b/dokimi/tests/test_evaluate_cli_inputs.py
index 9510c6c99..ef9902c50 100644
--- a/dokimi/tests/test_evaluate_cli_inputs.py
+++ b/dokimi/tests/test_evaluate_cli_inputs.py
@@ -32,5 +32,5 @@ def test_an_unwritable_output_dir_exits_1_with_a_message(tmp_path, capsys):
         _evaluate("--output-dir", str(blocker))
     assert exc.value.code == 1
     err = capsys.readouterr().err
-    assert "Error: " in err and "Not a directory" in err
+    assert err.startswith("Error: ") and str(blocker) in err
     assert "Traceback" not in err

From 8137a3c95996c8ed7d48549c2daa9ae7e6e00218 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Mon, 26 Oct 2026 11:04:01 -0500
Subject: [PATCH 11/18] fix(report,eval): exit with a message on unreadable
 questions and unwritable report output

report generate, report eval and report model-compare catch OSError on the
questions read and the report write. eval judge-validity exits 2 when
--output cannot be written, matching its documented malformed-input code.
Pin the benchmark run questions-from pre-check on the multi-run path and
for a directory source, and the report eval directory input.

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/commands/evaluate.py            |  8 +--
 dokimi/dokimi/commands/report.py              | 30 ++++++----
 .../test_benchmark_command_question_set.py    |  9 ++-
 .../test_eval_grounding_output_errors.py      |  6 +-
 dokimi/tests/test_report_cli_inputs.py        | 57 +++++++++++++++++++
 dokimi/tests/test_report_eval_input_errors.py | 14 +++++
 6 files changed, 104 insertions(+), 20 deletions(-)

diff --git a/dokimi/dokimi/commands/evaluate.py b/dokimi/dokimi/commands/evaluate.py
index 4acda4aa2..3230fa179 100644
--- a/dokimi/dokimi/commands/evaluate.py
+++ b/dokimi/dokimi/commands/evaluate.py
@@ -53,14 +53,14 @@ def _load_answers(path: str):
         sys.exit(2)
 
 
-def _write_json_or_exit(path: Path, payload: dict) -> None:
-    """Write ``payload`` as JSON to ``path``, or exit 1 naming ``path`` when it cannot be written."""
+def _write_json_or_exit(path: Path, payload: dict, code: int = 1) -> None:
+    """Write ``payload`` as JSON to ``path``, or exit ``code`` naming ``path`` when it cannot be written."""
     try:
         path.parent.mkdir(parents=True, exist_ok=True)
         path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
     except OSError as exc:
         print(f"Error: cannot write {path}: {exc}", file=sys.stderr)
-        sys.exit(1)
+        sys.exit(code)
 
 
 def _require_ragas() -> None:
@@ -417,7 +417,7 @@ def progress(done: int, total: int) -> None:
         result["summary"]["answers_model"] = run.meta.get("model")
         if typed is not None:
             result["summary"]["typed_arm"] = typed.report_fields()
-        _write_json_or_exit(out, result)
+        _write_json_or_exit(out, result, code=2)
 
     print()
     print(report.format_summary())
diff --git a/dokimi/dokimi/commands/report.py b/dokimi/dokimi/commands/report.py
index 7366c13f6..a98861d48 100644
--- a/dokimi/dokimi/commands/report.py
+++ b/dokimi/dokimi/commands/report.py
@@ -67,16 +67,20 @@ def cmd_report_model_compare(args: argparse.Namespace) -> None:
     for w in all_warnings:
         print(w, file=sys.stderr)
 
-    generate_combined_report(
-        regular_runs,
-        golden_runs,
-        run_ids,
-        args.out,
-        treatment_arm=pinned_arm,
-        correction=getattr(args, "stats_correction", "holm"),
-        pool_replicates=pool_replicates,
-        warnings=all_warnings,
-    )
+    try:
+        generate_combined_report(
+            regular_runs,
+            golden_runs,
+            run_ids,
+            args.out,
+            treatment_arm=pinned_arm,
+            correction=getattr(args, "stats_correction", "holm"),
+            pool_replicates=pool_replicates,
+            warnings=all_warnings,
+        )
+    except OSError as exc:
+        print(f"Error: cannot write {args.out}: {exc}", file=sys.stderr)
+        raise SystemExit(1) from None
 
 
 def cmd_report(args: argparse.Namespace) -> None:
@@ -106,6 +110,9 @@ def cmd_report_eval(args: argparse.Namespace) -> None:
     except RecordError as exc:
         print(f"Error: {exc}", file=sys.stderr)
         raise SystemExit(1) from None
+    except OSError as exc:
+        print(f"Error: cannot write {out}: {exc}", file=sys.stderr)
+        raise SystemExit(1) from None
 
 
 def cmd_report_generate(args: argparse.Namespace) -> None:
@@ -129,6 +136,9 @@ def cmd_report_generate(args: argparse.Namespace) -> None:
 
     try:
         questions_data = load_questions(args.questions).artifact
+    except OSError as exc:
+        print(f"Error: cannot read {args.questions}: {exc}", file=sys.stderr)
+        raise SystemExit(1) from None
     except ValueError as exc:  # ArtifactValidationError: names the record and the fix
         print(f"Error: {exc}", file=sys.stderr)
         raise SystemExit(1) from exc
diff --git a/dokimi/tests/test_benchmark_command_question_set.py b/dokimi/tests/test_benchmark_command_question_set.py
index 30c4c32ba..70cc65579 100644
--- a/dokimi/tests/test_benchmark_command_question_set.py
+++ b/dokimi/tests/test_benchmark_command_question_set.py
@@ -374,20 +374,23 @@ def test_preflight_rejects_an_unwritable_output_dir(monkeypatch, tmp_path, capsy
     [
         (lambda p: p / "nowhere", "questions source not found"),
         (lambda p: p / "bad.json", "questions source invalid"),
+        (lambda p: p / "adir", "questions source invalid"),
     ],
-    ids=["missing", "invalid"],
+    ids=["missing", "invalid", "directory"],
 )
+@pytest.mark.parametrize("multi_run", [1, 3])
 def test_run_rejects_an_unusable_questions_from_before_any_call(
-    monkeypatch, tmp_path, capsys, make_source, message
+    monkeypatch, tmp_path, capsys, make_source, message, multi_run
 ):
     """benchmark run exits 1 with a message, not a traceback, and starts no pipeline."""
     calls: list[object] = []
     (tmp_path / "bad.json").write_text(json.dumps({"a": 1}), encoding="utf-8")
+    (tmp_path / "adir" / "questions" / "onetbb.json").mkdir(parents=True)
     monkeypatch.setattr(benchmark, "_load_registry", lambda args: _Registry())
     monkeypatch.setattr(benchmark, "_run_single_library", lambda *a, **k: calls.append(k))
 
     args = build_parser().parse_args([
-        "benchmark", "run", "--library", "onetbb",
+        "benchmark", "run", "--library", "onetbb", "--multi-run", str(multi_run),
         "--output-dir", str(tmp_path / "out"),
         "--questions-from", str(make_source(tmp_path)),
     ])
diff --git a/dokimi/tests/test_eval_grounding_output_errors.py b/dokimi/tests/test_eval_grounding_output_errors.py
index 055db51f8..7531d78c6 100644
--- a/dokimi/tests/test_eval_grounding_output_errors.py
+++ b/dokimi/tests/test_eval_grounding_output_errors.py
@@ -1,7 +1,7 @@
 # Copyright (C) 2026 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-"""`eval grounding` and `eval judge-validity`: an unwritable --output exits 1 with a message."""
+"""`eval grounding` and `eval grounding` exits 1, `eval judge-validity` exits 2 (malformed input), when --output cannot be written."""
 
 from __future__ import annotations
 
@@ -42,7 +42,7 @@ def test_output_below_a_file_exits_1_naming_the_path(tmp_path, capsys):
     assert "Traceback" not in err
 
 
-def test_judge_validity_output_below_a_file_exits_1_naming_the_path(tmp_path, monkeypatch, capsys):
+def test_judge_validity_output_below_a_file_exits_2_naming_the_path(tmp_path, monkeypatch, capsys):
     import dokimi.eval.judge as judge_mod
     import dokimi.eval.judge_validity as jv
 
@@ -62,7 +62,7 @@ def test_judge_validity_output_below_a_file_exits_1_naming_the_path(tmp_path, mo
                                       "--variants", "truncate_20", "--output", str(target)])
     with pytest.raises(SystemExit) as exc:
         args.func(args)
-    assert exc.value.code == 1
+    assert exc.value.code == 2
     assert f"Error: cannot write {target}" in capsys.readouterr().err
 
 
diff --git a/dokimi/tests/test_report_cli_inputs.py b/dokimi/tests/test_report_cli_inputs.py
index dc417bf38..170aab645 100644
--- a/dokimi/tests/test_report_cli_inputs.py
+++ b/dokimi/tests/test_report_cli_inputs.py
@@ -5,6 +5,8 @@
 
 from __future__ import annotations
 
+import json
+
 import pytest
 
 from dokimi.cli import build_parser
@@ -84,3 +86,58 @@ def test_non_list_evaluations_exit_1_with_its_path(tmp_path, capsys, evaluations
     bad = tmp_path / "evd.json"
     bad.write_text(f'{{"evaluations": {evaluations}}}', encoding="utf-8")
     assert f"{bad} evaluations: expected a list of records" in _err(tmp_path, bad, capsys)
+
+
+@pytest.mark.parametrize("kind", ["missing", "directory"])
+def test_generate_unreadable_questions_exits_1_naming_the_path(tmp_path, capsys, kind):
+    ev = tmp_path / "ev.json"
+    ev.write_text('{"evaluations": []}', encoding="utf-8")
+    qs = tmp_path / "nope.json"
+    if kind == "directory":
+        qs.mkdir()
+    args = build_parser().parse_args([
+        "report", "generate", "--product", "p", "--eval", str(ev),
+        "--questions", str(qs), "--output", str(tmp_path / "r.md"),
+    ])
+    with pytest.raises(SystemExit) as exc:
+        args.func(args)
+    assert exc.value.code == 1
+    err = capsys.readouterr().err
+    assert err.startswith("Error: cannot read ") and str(qs) in err
+    assert "Traceback" not in err
+
+
+def test_report_eval_unwritable_out_exits_1_naming_the_path(tmp_path, capsys):
+    ev = tmp_path / "ev.json"
+    ev.write_text('{"evaluations": []}', encoding="utf-8")
+    blocker = tmp_path / "blocker"
+    blocker.write_text("x", encoding="utf-8")
+    out = blocker / "r.md"
+    args = build_parser().parse_args([
+        "report", "eval", "--product", "P", "--eval", str(ev), "--out", str(out),
+    ])
+    with pytest.raises(SystemExit) as exc:
+        args.func(args)
+    assert exc.value.code == 1
+    err = capsys.readouterr().err
+    assert err.startswith("Error: cannot write ") and str(out) in err
+
+
+def test_model_compare_unwritable_out_exits_1_naming_the_path(tmp_path, capsys):
+    run = tmp_path / "run.json"
+    run.write_text(json.dumps({"baseline_arm": "baseline", "evaluations": [
+        {"question_id": f"q{i}", "question_text": "q", "category": "", "difficulty": "easy",
+         "persona": "", "scores": {"baseline": {"aggregate": 50.0}, "docs": {"aggregate": 60.0}}}
+        for i in range(3)
+    ]}), encoding="utf-8")
+    blocker = tmp_path / "blocker"
+    blocker.write_text("x", encoding="utf-8")
+    out = blocker / "x.md"
+    args = build_parser().parse_args([
+        "report", "model-compare", "--regular-runs", str(run), "--run-ids", "a", "--out", str(out),
+    ])
+    with pytest.raises(SystemExit) as exc:
+        args.func(args)
+    assert exc.value.code == 1
+    err = capsys.readouterr().err
+    assert err.startswith("Error: cannot write ") and str(out) in err
diff --git a/dokimi/tests/test_report_eval_input_errors.py b/dokimi/tests/test_report_eval_input_errors.py
index db9cbf34d..0368742a8 100644
--- a/dokimi/tests/test_report_eval_input_errors.py
+++ b/dokimi/tests/test_report_eval_input_errors.py
@@ -34,3 +34,17 @@ def test_report_eval_exits_1_with_a_message_on_an_unreadable_eval_file(tmp_path,
     err = capsys.readouterr().err
     assert err.startswith("Error: ") and str(eval_file) in err and reason in err
     assert not out.exists()
+
+
+def test_report_eval_exits_1_with_a_message_when_the_eval_path_is_a_directory(tmp_path, monkeypatch, capsys):
+    from dokimi import cli
+
+    out = tmp_path / "r.md"
+    monkeypatch.setattr("sys.argv", ["dokimi", "report", "eval", "--product", "P", "--eval", str(tmp_path),
+                                     "--out", str(out)])
+    with pytest.raises(SystemExit) as exc:
+        cli.main()
+    assert exc.value.code == 1
+    err = capsys.readouterr().err
+    assert err.startswith("Error: cannot read ") and str(tmp_path) in err
+    assert not out.exists()

From 11f9e77ba75bac1f47945e0424cc397884b6e333 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Mon, 26 Oct 2026 11:27:47 -0500
Subject: [PATCH 12/18] refactor(report): move model-compare report write into
 a helper

Keeps cmd_report_model_compare within its complexity baseline.

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/commands/report.py | 36 +++++++++++++++++++-------------
 1 file changed, 21 insertions(+), 15 deletions(-)

diff --git a/dokimi/dokimi/commands/report.py b/dokimi/dokimi/commands/report.py
index a98861d48..58d47934a 100644
--- a/dokimi/dokimi/commands/report.py
+++ b/dokimi/dokimi/commands/report.py
@@ -13,11 +13,30 @@
 from dokimi.report.markdown_report import write_compare_report, write_run_report
 
 
+def _write_combined_report(args, regular_runs, golden_runs, run_ids, warnings) -> None:
+    """Write the combined model-compare report, exiting cleanly if the output is unwritable."""
+    from dokimi.report.model_compare import generate_combined_report
+
+    try:
+        generate_combined_report(
+            regular_runs,
+            golden_runs,
+            run_ids,
+            args.out,
+            treatment_arm=getattr(args, "treatment_arm", None),
+            correction=getattr(args, "stats_correction", "holm"),
+            pool_replicates=getattr(args, "pool_replicates", False),
+            warnings=warnings,
+        )
+    except OSError as exc:
+        print(f"Error: cannot write {args.out}: {exc}", file=sys.stderr)
+        raise SystemExit(1) from None
+
+
 def cmd_report_model_compare(args: argparse.Namespace) -> None:
     """Compare multiple model runs and write a Markdown report."""
     from dokimi.report.model_compare import (
         check_run_consistency,
-        generate_combined_report,
         load_run,
     )
 
@@ -67,20 +86,7 @@ def cmd_report_model_compare(args: argparse.Namespace) -> None:
     for w in all_warnings:
         print(w, file=sys.stderr)
 
-    try:
-        generate_combined_report(
-            regular_runs,
-            golden_runs,
-            run_ids,
-            args.out,
-            treatment_arm=pinned_arm,
-            correction=getattr(args, "stats_correction", "holm"),
-            pool_replicates=pool_replicates,
-            warnings=all_warnings,
-        )
-    except OSError as exc:
-        print(f"Error: cannot write {args.out}: {exc}", file=sys.stderr)
-        raise SystemExit(1) from None
+    _write_combined_report(args, regular_runs, golden_runs, run_ids, all_warnings)
 
 
 def cmd_report(args: argparse.Namespace) -> None:

From 33319493b88946bf25cd752f1ca295c00578eafd Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Mon, 26 Oct 2026 12:18:52 -0500
Subject: [PATCH 13/18] fix(eval,report): guard every eval output write and
 name the failing report sidecar

Route the judge-reliability, score, panel-score and ragas writers through one
exit-with-message helper, exit 2 for an unwritable grounding output like
judge-validity, name the Q&A sidecar when it cannot be written, and stop the
sidecar from overwriting an --out that has no .md suffix. Catch ValueError in
the answers runtime-config read to match the sibling change.

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/commands/answers.py             |  2 +-
 dokimi/dokimi/commands/evaluate.py            | 35 ++++----
 dokimi/dokimi/commands/judge_reliability.py   | 26 +++---
 dokimi/dokimi/report/eval_report.py           | 28 +++---
 dokimi/dokimi/utils.py                        | 11 +++
 .../test_eval_grounding_output_errors.py      | 10 +--
 dokimi/tests/test_eval_output_write_errors.py | 90 +++++++++++++++++++
 7 files changed, 153 insertions(+), 49 deletions(-)
 create mode 100644 dokimi/tests/test_eval_output_write_errors.py

diff --git a/dokimi/dokimi/commands/answers.py b/dokimi/dokimi/commands/answers.py
index 80313d009..8128af314 100644
--- a/dokimi/dokimi/commands/answers.py
+++ b/dokimi/dokimi/commands/answers.py
@@ -34,7 +34,7 @@ def _load_runtime_config() -> dict:
     except yaml.YAMLError as exc:
         print(f"Error: invalid YAML in {config_path}: {exc}", file=sys.stderr)
         raise SystemExit(1) from exc
-    except (OSError, UnicodeDecodeError) as exc:  # unreadable or not UTF-8
+    except (OSError, ValueError) as exc:  # unreadable or not UTF-8
         print(f"Error: cannot read {config_path}: {exc}", file=sys.stderr)
         raise SystemExit(1) from exc
     if loaded is not None and not isinstance(loaded, dict):
diff --git a/dokimi/dokimi/commands/evaluate.py b/dokimi/dokimi/commands/evaluate.py
index 3230fa179..46cd8564c 100644
--- a/dokimi/dokimi/commands/evaluate.py
+++ b/dokimi/dokimi/commands/evaluate.py
@@ -20,7 +20,7 @@
 )
 from dokimi.eval.judge import JUDGE_PROVIDERS
 from dokimi.eval.judge_validity import DEFAULT_MIN_ACCURACY, NoUsablePairs
-from dokimi.utils import normalize_model_ref
+from dokimi.utils import normalize_model_ref, write_text_or_exit
 
 
 def _warn_judge_independence(
@@ -55,12 +55,7 @@ def _load_answers(path: str):
 
 def _write_json_or_exit(path: Path, payload: dict, code: int = 1) -> None:
     """Write ``payload`` as JSON to ``path``, or exit ``code`` naming ``path`` when it cannot be written."""
-    try:
-        path.parent.mkdir(parents=True, exist_ok=True)
-        path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
-    except OSError as exc:
-        print(f"Error: cannot write {path}: {exc}", file=sys.stderr)
-        sys.exit(code)
+    write_text_or_exit(path, json.dumps(payload, indent=2), code)
 
 
 def _require_ragas() -> None:
@@ -86,10 +81,8 @@ def _run_ragas_eval(answers, output_path, args) -> None:
         result = evaluator.evaluate(answers, include_without_docs=True)
 
         # Save ragas results alongside answers
-        import json
         ragas_path = output_path.with_name(output_path.stem + "_ragas.json")
-        ragas_path.parent.mkdir(parents=True, exist_ok=True)
-        ragas_path.write_text(json.dumps(result.to_dict(), indent=2), encoding="utf-8")
+        _write_json_or_exit(ragas_path, result.to_dict(), code=2)
 
         print(result.format_summary())
         print(f"✅ Saved RAGAS results to {ragas_path}")
@@ -156,11 +149,15 @@ def cmd_eval_panel_score(args: argparse.Namespace) -> None:
     print(f"Roles: {', '.join(roles)}\nModel: {args.provider}/{args.model}\n")
 
     output_path = Path(args.output) if args.output else Path(f"eval/{args.product}_panel.json")
-    results = panel.evaluate_answers(answers, library_name=args.product,
-                                     output_path=output_path,
-                                     limit=getattr(args, "limit", None),
-                                     run_metadata={"answer_model": answer_model,
-                                                   "answer_provider": answer_provider})
+    try:
+        results = panel.evaluate_answers(answers, library_name=args.product,
+                                         output_path=output_path,
+                                         limit=getattr(args, "limit", None),
+                                         run_metadata={"answer_model": answer_model,
+                                                       "answer_provider": answer_provider})
+    except OSError as exc:
+        print(f"Error: cannot write {exc.filename or output_path}: {exc.strerror or exc}", file=sys.stderr)
+        sys.exit(2)
 
     # Summary: the context arm's own quality, not a comparison.
     from dokimi.records import CONTEXT, records
@@ -229,11 +226,13 @@ def cmd_eval_score(args: argparse.Namespace) -> None:
     try:
         evaluations = judge.evaluate_answers(args.product, answers, output_path=output_path,
                                              concurrency=args.concurrency)
+        judge.save_evaluations(evaluations, output_path)
     except RecordError as exc:
         print(f"Error: {args.answers}: {exc}", file=sys.stderr)
         sys.exit(2)
-
-    judge.save_evaluations(evaluations, output_path)
+    except OSError as exc:
+        print(f"Error: cannot write {exc.filename or output_path}: {exc.strerror or exc}", file=sys.stderr)
+        sys.exit(2)
     print(f"\n✅ Saved evaluations to {output_path}")
 
 
@@ -267,7 +266,7 @@ def cmd_eval_grounding(args: argparse.Namespace) -> None:
 
     if args.output:
         out = Path(args.output)
-        _write_json_or_exit(out, result)
+        _write_json_or_exit(out, result, code=2)
         print(f"\n✅ Grounding report saved to {out}")
 
 
diff --git a/dokimi/dokimi/commands/judge_reliability.py b/dokimi/dokimi/commands/judge_reliability.py
index 38ca9c7a6..ffca9a0fa 100644
--- a/dokimi/dokimi/commands/judge_reliability.py
+++ b/dokimi/dokimi/commands/judge_reliability.py
@@ -19,6 +19,7 @@
 )
 from dokimi.eval.judge import JUDGE_PROVIDERS
 from dokimi.eval.judge_validity import DEFAULT_MIN_ACCURACY
+from dokimi.utils import write_text_or_exit
 
 
 def _read_json(path: str):
@@ -165,10 +166,9 @@ def cmd_eval_pairwise(args: argparse.Namespace) -> None:
         sys.exit(2)
     md = report.format_markdown()
     out = Path(args.output or Path(args.answers).with_name(Path(args.answers).stem + "_pairwise.json"))
-    out.parent.mkdir(parents=True, exist_ok=True)
-    out.write_text(json.dumps({**report.to_dict(), "judge_temperature": getattr(args, "judge_temperature", None)},
-                              indent=2), encoding="utf-8")
-    out.with_suffix(".md").write_text(md, encoding="utf-8")
+    write_text_or_exit(out, json.dumps({**report.to_dict(), "judge_temperature": getattr(args, "judge_temperature", None)},
+                                       indent=2), code=2)
+    write_text_or_exit(out.with_suffix(".md"), md, code=2)
     if not report.verdicts:
         # Written first, so the reasons are not lost with the run.
         print(f"Error: all {len(report.errors)} comparison(s) failed; see `errors` in {out}", file=sys.stderr)
@@ -204,12 +204,10 @@ def cmd_eval_checklist(args: argparse.Namespace) -> None:
         print("Error: no question in the answers file has expected_topics in --questions", file=sys.stderr)
         sys.exit(2)
     out = Path(args.output or Path(args.answers).with_name(Path(args.answers).stem + "_checklist.json"))
-    out.parent.mkdir(parents=True, exist_ok=True)
     # The JSON first: the paid calls are in it, and a rendering error must not lose them.
-    out.write_text(json.dumps({**report.to_dict(), "judge_temperature": args.judge_temperature}, indent=2),
-                   encoding="utf-8")
+    write_text_or_exit(out, json.dumps({**report.to_dict(), "judge_temperature": args.judge_temperature}, indent=2), code=2)
     md = report.format_markdown()
-    out.with_suffix(".md").write_text(md, encoding="utf-8")
+    write_text_or_exit(out.with_suffix(".md"), md, code=2)
     print()
     print(md)
     print(f"✅ Coverage report saved to {out} (+ .md)")
@@ -252,9 +250,8 @@ def progress(cond: str, done: int, total: int) -> None:
     )
     if args.output:
         out = Path(args.output)
-        out.parent.mkdir(parents=True, exist_ok=True)
-        out.write_text(json.dumps({**report.to_dict(), "judge_temperature": args.judge_temperature}, indent=2),
-                       encoding="utf-8")
+        write_text_or_exit(out, json.dumps({**report.to_dict(), "judge_temperature": args.judge_temperature}, indent=2),
+                           code=2)
     print()
     print(report.format_summary())
     if args.output:
@@ -417,10 +414,9 @@ def score(library, question, answer, ground_truth, context):
         sys.exit(2)
     md = report.format_markdown()
     out = Path(args.output or Path(args.answers).with_name(Path(args.answers).stem + "_crossjudge.json"))
-    out.parent.mkdir(parents=True, exist_ok=True)
-    out.write_text(json.dumps({**report.to_dict(), "judge_temperature": args.judge_temperature}, indent=2),
-                   encoding="utf-8")
-    out.with_suffix(".md").write_text(md, encoding="utf-8")
+    write_text_or_exit(out, json.dumps({**report.to_dict(), "judge_temperature": args.judge_temperature}, indent=2),
+                       code=2)
+    write_text_or_exit(out.with_suffix(".md"), md, code=2)
     print()
     print(md)
     print(f"✅ Cross-judge report saved to {out} (+ .md)")
diff --git a/dokimi/dokimi/report/eval_report.py b/dokimi/dokimi/report/eval_report.py
index 17cdf1478..ebb8c657f 100755
--- a/dokimi/dokimi/report/eval_report.py
+++ b/dokimi/dokimi/report/eval_report.py
@@ -187,6 +187,15 @@ def _load_eval(eval_path: str) -> dict:
     return data
 
 
+def _write_qa_json(path: str, payload: dict) -> None:
+    """Write the Q&A sidecar, or raise a RecordError naming ``path``."""
+    try:
+        with open(path, "w", encoding="utf-8") as jf:
+            json.dump(payload, jf, indent=2, ensure_ascii=False)
+    except OSError as exc:
+        raise RecordError(f"cannot write {path}: {exc.strerror or exc}") from exc
+
+
 def generate_report(eval_path: str, out_path: str, qa_json_out: str | None = None):
     data = _load_eval(eval_path)
 
@@ -462,7 +471,7 @@ def _val(key):
 
     # Sidecar JSON: questions + answers + scores
     if qa_json_out is None:
-        qa_json_out = out_path.replace(".md", ".qa.json")
+        qa_json_out = os.path.splitext(out_path)[0] + ".qa.json"
 
     answers_by_id = _answers_by_id(eval_path.replace("/eval/", "/answers/"))
 
@@ -485,15 +494,14 @@ def _answer_text(a: Record | None, condition: str):
             "without_docs_answer": _answer_text(a, BASELINE),
         })
 
-    with open(qa_json_out, "w", encoding="utf-8") as jf:
-        json.dump({
-            "eval_path": eval_path,
-            "report_path": out_path,
-            "generated_at": datetime.now(timezone.utc).isoformat(),
-            "run_metadata": meta,
-            "count": len(qa_records),
-            "records": qa_records,
-        }, jf, indent=2, ensure_ascii=False)
+    _write_qa_json(qa_json_out, {
+        "eval_path": eval_path,
+        "report_path": out_path,
+        "generated_at": datetime.now(timezone.utc).isoformat(),
+        "run_metadata": meta,
+        "count": len(qa_records),
+        "records": qa_records,
+    })
 
     print(f"✅ Report written to: {out_path}")
     print(f"✅ Q&A JSON written to: {qa_json_out}")
diff --git a/dokimi/dokimi/utils.py b/dokimi/dokimi/utils.py
index 9c0ffeda9..cda2ff68c 100644
--- a/dokimi/dokimi/utils.py
+++ b/dokimi/dokimi/utils.py
@@ -213,3 +213,14 @@ def read_json_input(path: str | os.PathLike[str], *, flag: str) -> Any:
     except (OSError, ValueError, RecursionError) as exc:  # JSONDecodeError and bad UTF-8 are ValueErrors
         reason = exc.strerror if isinstance(exc, OSError) and exc.strerror else exc
         raise InputError(f"{flag} {path}: {reason}") from exc
+
+
+def write_text_or_exit(path: Path, text: str, code: int = 1) -> None:
+    """Write ``text`` to ``path`` (creating parents), or exit ``code`` naming ``path`` when it cannot be written."""
+    import sys
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(text, encoding="utf-8")
+    except OSError as exc:
+        print(f"Error: cannot write {path}: {exc}", file=sys.stderr)
+        sys.exit(code)
diff --git a/dokimi/tests/test_eval_grounding_output_errors.py b/dokimi/tests/test_eval_grounding_output_errors.py
index 7531d78c6..2b00baf58 100644
--- a/dokimi/tests/test_eval_grounding_output_errors.py
+++ b/dokimi/tests/test_eval_grounding_output_errors.py
@@ -1,7 +1,7 @@
 # Copyright (C) 2026 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-"""`eval grounding` and `eval grounding` exits 1, `eval judge-validity` exits 2 (malformed input), when --output cannot be written."""
+"""`eval grounding` and `eval judge-validity` exit 2 naming the path when --output cannot be written."""
 
 from __future__ import annotations
 
@@ -30,13 +30,13 @@ def test_grounding_report_is_written(tmp_path):
     assert json.loads(out.read_text(encoding="utf-8"))["schema_version"] == "grounding.v1"
 
 
-def test_output_below_a_file_exits_1_naming_the_path(tmp_path, capsys):
+def test_output_below_a_file_exits_2_naming_the_path(tmp_path, capsys):
     blocker = tmp_path / "blocker"
     blocker.write_text("x", encoding="utf-8")  # a regular file: nothing can be created below it
     target = blocker / "grounding.json"
     with pytest.raises(SystemExit) as exc:
         _grounding(tmp_path, target)
-    assert exc.value.code == 1
+    assert exc.value.code == 2
     err = capsys.readouterr().err
     assert f"Error: cannot write {target}" in err
     assert "Traceback" not in err
@@ -66,10 +66,10 @@ def test_judge_validity_output_below_a_file_exits_2_naming_the_path(tmp_path, mo
     assert f"Error: cannot write {target}" in capsys.readouterr().err
 
 
-def test_output_that_is_a_directory_exits_1_naming_the_path(tmp_path, capsys):
+def test_output_that_is_a_directory_exits_2_naming_the_path(tmp_path, capsys):
     target = tmp_path / "is-a-dir"
     target.mkdir()
     with pytest.raises(SystemExit) as exc:
         _grounding(tmp_path, target)
-    assert exc.value.code == 1
+    assert exc.value.code == 2
     assert f"Error: cannot write {target}" in capsys.readouterr().err
diff --git a/dokimi/tests/test_eval_output_write_errors.py b/dokimi/tests/test_eval_output_write_errors.py
new file mode 100644
index 000000000..7db6321de
--- /dev/null
+++ b/dokimi/tests/test_eval_output_write_errors.py
@@ -0,0 +1,90 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""eval writers and report eval: an unwritable output exits with a message naming the failing path."""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from dokimi.cli import build_parser
+from dokimi.commands import judge_reliability
+
+
+def _blocked(tmp_path):
+    blocker = tmp_path / "blocker"
+    blocker.write_text("x", encoding="utf-8")  # a regular file: nothing can be created below it
+    return blocker / "out.json"
+
+
+
+def test_write_text_or_exit_names_the_path_and_exits_with_the_code(tmp_path, capsys):
+    from dokimi.utils import write_text_or_exit
+
+    target = _blocked(tmp_path)
+    with pytest.raises(SystemExit) as exc:
+        write_text_or_exit(target, "{}", code=2)
+    assert exc.value.code == 2
+    assert f"Error: cannot write {target}" in capsys.readouterr().err
+
+
+def test_judge_reliability_writers_use_the_guarded_write():
+    import inspect
+
+    src = inspect.getsource(judge_reliability)
+    assert ".write_text(" not in src and "mkdir(" not in src
+
+
+def test_score_output_that_cannot_be_written_exits_2_naming_the_path(tmp_path, monkeypatch, capsys):
+    class Judge:
+        def evaluate_answers(self, *a, **k):
+            return []
+
+        def save_evaluations(self, evals, path):
+            raise PermissionError(13, "Permission denied", str(path))
+
+    answers = tmp_path / "answers.json"
+    answers.write_text(json.dumps({"answers": []}), encoding="utf-8")
+    import dokimi.eval as eval_mod
+
+    monkeypatch.setattr(eval_mod, "Judge", lambda **kw: Judge())
+    target = tmp_path / "e.json"
+    args = build_parser().parse_args(["eval", "score", "--product", "P", "--answers", str(answers),
+                                      "--output", str(target)])
+    with pytest.raises(SystemExit) as exc:
+        args.func(args)
+    assert exc.value.code == 2
+    assert f"Error: cannot write {target}" in capsys.readouterr().err
+
+
+def test_report_eval_names_the_sidecar_when_it_cannot_be_written(tmp_path, monkeypatch, capsys):
+    from dokimi import cli
+
+    eval_file, out = tmp_path / "eval.json", tmp_path / "r.md"
+    eval_file.write_text(json.dumps({"evaluations": [{
+        "question_id": "q1", "question_text": "Q?", "with_docs": {"aggregate": 80},
+        "without_docs": {"aggregate": 60}, "delta": 20}]}), encoding="utf-8")
+    (tmp_path / "r.qa.json").mkdir()
+    monkeypatch.setattr("sys.argv", ["dokimi", "report", "eval", "--product", "P", "--eval", str(eval_file),
+                                     "--out", str(out)])
+    with pytest.raises(SystemExit) as exc:
+        cli.main()
+    assert exc.value.code == 1
+    err = capsys.readouterr().err
+    assert str(tmp_path / "r.qa.json") in err and f"cannot write {out}" not in err
+
+
+def test_report_eval_sidecar_does_not_overwrite_an_out_without_md(tmp_path, monkeypatch):
+    from dokimi import cli
+
+    eval_file, out = tmp_path / "eval.json", tmp_path / "report"
+    eval_file.write_text(json.dumps({"evaluations": [{
+        "question_id": "q1", "question_text": "Q?", "with_docs": {"aggregate": 80},
+        "without_docs": {"aggregate": 60}, "delta": 20}]}), encoding="utf-8")
+    monkeypatch.setattr("sys.argv", ["dokimi", "report", "eval", "--product", "P", "--eval", str(eval_file),
+                                     "--out", str(out)])
+    cli.main()
+    assert not out.read_text(encoding="utf-8").lstrip().startswith("{")
+    assert (tmp_path / "report.qa.json").exists()

From 34f8e413bf6291780623d4c22868de56b5c25084 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Mon, 26 Oct 2026 13:00:58 -0500
Subject: [PATCH 14/18] test(eval): pin the eval output write guards with
 behavioural tests

Cover eval pairwise, checklist, judge-wording and cross-judge (JSON and .md
sidecar), the ragas results write and the panel-score OSError handler, each
asserting exit 2 and the failing path. Replaces the source-text check.

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/tests/test_eval_output_write_errors.py | 200 ++++++++++++++++--
 1 file changed, 183 insertions(+), 17 deletions(-)

diff --git a/dokimi/tests/test_eval_output_write_errors.py b/dokimi/tests/test_eval_output_write_errors.py
index 7db6321de..2b0df47e6 100644
--- a/dokimi/tests/test_eval_output_write_errors.py
+++ b/dokimi/tests/test_eval_output_write_errors.py
@@ -19,7 +19,6 @@ def _blocked(tmp_path):
     return blocker / "out.json"
 
 
-
 def test_write_text_or_exit_names_the_path_and_exits_with_the_code(tmp_path, capsys):
     from dokimi.utils import write_text_or_exit
 
@@ -30,11 +29,147 @@ def test_write_text_or_exit_names_the_path_and_exits_with_the_code(tmp_path, cap
     assert f"Error: cannot write {target}" in capsys.readouterr().err
 
 
-def test_judge_reliability_writers_use_the_guarded_write():
-    import inspect
+class _Report:
+    verdicts = [object()]
+    errors: list = []
+    measures = [object()]
+
+    def __init__(self, *a, **k):
+        pass
+
+    def to_dict(self):
+        return {}
+
+    def format_markdown(self):
+        return "md"
+
+    def apply_validity(self, validity):
+        return []
+
+    def summary(self):
+        return {"arms": {}, "measured": True, "fragile_variants": []}
+
+    def format_summary(self):
+        return ""
+
+
+def _stub_judge_commands(monkeypatch):
+    """Replace every judge-facing dependency of the four judge_reliability writers with a stub."""
+    import types
+
+    import dokimi.eval.checklist as checklist
+    import dokimi.eval.cross_judge as cross_judge
+    import dokimi.eval.judge as judge
+    import dokimi.eval.judge_validity as judge_validity
+    import dokimi.eval.judge_wording as judge_wording
+    import dokimi.eval.pairwise as pairwise
+
+    monkeypatch.setattr(judge_reliability, "_load_answers", lambda path: (None, []))
+    monkeypatch.setattr(judge_reliability, "_load_points", lambda path: {})
+    monkeypatch.setattr(judge_reliability, "_require_arms", lambda *a, **k: None)
+    monkeypatch.setattr(judge_reliability, "_check_judge_temperature", lambda *a, **k: None)
+    monkeypatch.setattr(judge_reliability, "pairwise_judge", lambda args: (None, "label"))
+    monkeypatch.setattr(pairwise, "pairwise_jobs", lambda *a, **k: None)
+    monkeypatch.setattr(pairwise, "run_arm_pairwise", lambda *a, **k: _Report())
+    monkeypatch.setattr(checklist, "checklist_jobs", lambda *a, **k: None)
+    monkeypatch.setattr(checklist, "run_checklist", lambda *a, **k: _Report())
+    monkeypatch.setattr(judge, "Judge", lambda **kw: types.SimpleNamespace(llm=None))
+    monkeypatch.setattr(judge_validity, "_build", lambda *a, **k: types.SimpleNamespace(pairs=[]))
+    monkeypatch.setattr(judge_wording, "run_wording", lambda *a, **k: _Report())
+    monkeypatch.setattr(cross_judge, "CrossJudgeReport", _Report)
+    monkeypatch.setattr(cross_judge, "scored_answers", lambda *a, **k: None)
+    monkeypatch.setattr(cross_judge, "score_measure", lambda *a, **k: object())
+
+
+_JUDGE_COMMANDS = {
+    "pairwise": ["eval", "pairwise"],
+    "checklist": ["eval", "checklist", "--questions", "q.json"],
+    "judge-wording": ["eval", "judge-wording"],
+    "cross-judge": ["eval", "cross-judge", "--judges", "openai/m", "--no-stored"],
+}
+_WRITES_MD = ["pairwise", "checklist", "cross-judge"]
+
+
+def _run(command, out):
+    args = build_parser().parse_args(
+        [*_JUDGE_COMMANDS[command], "--answers", "a.json", "--output", str(out)]
+    )
+    with pytest.raises(SystemExit) as exc:
+        args.func(args)
+    return exc.value.code
+
+
+@pytest.mark.parametrize("command", list(_JUDGE_COMMANDS))
+def test_judge_reliability_json_write_failure_exits_2_naming_the_path(
+    command, tmp_path, monkeypatch, capsys
+):
+    _stub_judge_commands(monkeypatch)
+    out = _blocked(tmp_path)
+    assert _run(command, out) == 2
+    assert f"Error: cannot write {out}" in capsys.readouterr().err
+
 
-    src = inspect.getsource(judge_reliability)
-    assert ".write_text(" not in src and "mkdir(" not in src
+@pytest.mark.parametrize("command", _WRITES_MD)
+def test_judge_reliability_md_sidecar_write_failure_exits_2_naming_the_path(
+    command, tmp_path, monkeypatch, capsys
+):
+    _stub_judge_commands(monkeypatch)
+    out = tmp_path / "out.json"
+    sidecar = out.with_suffix(".md")
+    sidecar.mkdir()
+    assert _run(command, out) == 2
+    assert f"Error: cannot write {sidecar}" in capsys.readouterr().err
+
+
+def test_ragas_results_that_cannot_be_written_exit_2_naming_the_path(tmp_path, monkeypatch, capsys):
+    import types
+
+    import dokimi.eval.ragas_eval as ragas_eval
+    from dokimi.commands.evaluate import _run_ragas_eval
+
+    class Evaluator:
+        def __init__(self, **kw):
+            pass
+
+        def evaluate(self, answers, include_without_docs=True):
+            return types.SimpleNamespace(to_dict=dict, format_summary=lambda: "")
+
+    monkeypatch.setattr(ragas_eval, "RagasEvaluator", Evaluator)
+    blocked = _blocked(tmp_path)
+    with pytest.raises(SystemExit) as exc:
+        _run_ragas_eval([], blocked, types.SimpleNamespace())
+    assert exc.value.code == 2
+    assert f"Error: cannot write {blocked.with_name('out_ragas.json')}" in capsys.readouterr().err
+
+
+def test_panel_score_output_that_cannot_be_written_exits_2_naming_the_path(
+    tmp_path, monkeypatch, capsys
+):
+    import types
+
+    import dokimi.eval.panel as panel_mod
+    from dokimi.commands import evaluate
+
+    target = tmp_path / "p.json"
+
+    class Panel:
+        def __init__(self, **kw):
+            pass
+
+        def evaluate_answers(self, *a, **k):
+            raise PermissionError(13, "denied", str(target))
+
+    monkeypatch.setattr(panel_mod, "JudgePanel", Panel)
+    monkeypatch.setattr(
+        evaluate, "_load_answers", lambda path: types.SimpleNamespace(records=[], meta={})
+    )
+    args = build_parser().parse_args(
+        ["eval", "panel-score", "--product", "P", "--answers", "a.json", "--output", str(target)]
+    )
+    with pytest.raises(SystemExit) as exc:
+        args.func(args)
+    assert exc.value.code == 2
+    assert f"Error: cannot write {target}" in capsys.readouterr().err
 
 
 def test_score_output_that_cannot_be_written_exits_2_naming_the_path(tmp_path, monkeypatch, capsys):
@@ -51,8 +186,9 @@ def save_evaluations(self, evals, path):
 
     monkeypatch.setattr(eval_mod, "Judge", lambda **kw: Judge())
     target = tmp_path / "e.json"
-    args = build_parser().parse_args(["eval", "score", "--product", "P", "--answers", str(answers),
-                                      "--output", str(target)])
+    args = build_parser().parse_args(
+        ["eval", "score", "--product", "P", "--answers", str(answers), "--output", str(target)]
+    )
     with pytest.raises(SystemExit) as exc:
         args.func(args)
     assert exc.value.code == 2
@@ -63,12 +199,27 @@ def test_report_eval_names_the_sidecar_when_it_cannot_be_written(tmp_path, monke
     from dokimi import cli
 
     eval_file, out = tmp_path / "eval.json", tmp_path / "r.md"
-    eval_file.write_text(json.dumps({"evaluations": [{
-        "question_id": "q1", "question_text": "Q?", "with_docs": {"aggregate": 80},
-        "without_docs": {"aggregate": 60}, "delta": 20}]}), encoding="utf-8")
+    eval_file.write_text(
+        json.dumps(
+            {
+                "evaluations": [
+                    {
+                        "question_id": "q1",
+                        "question_text": "Q?",
+                        "with_docs": {"aggregate": 80},
+                        "without_docs": {"aggregate": 60},
+                        "delta": 20,
+                    }
+                ]
+            }
+        ),
+        encoding="utf-8",
+    )
     (tmp_path / "r.qa.json").mkdir()
-    monkeypatch.setattr("sys.argv", ["dokimi", "report", "eval", "--product", "P", "--eval", str(eval_file),
-                                     "--out", str(out)])
+    monkeypatch.setattr(
+        "sys.argv",
+        ["dokimi", "report", "eval", "--product", "P", "--eval", str(eval_file), "--out", str(out)],
+    )
     with pytest.raises(SystemExit) as exc:
         cli.main()
     assert exc.value.code == 1
@@ -80,11 +231,26 @@ def test_report_eval_sidecar_does_not_overwrite_an_out_without_md(tmp_path, monk
     from dokimi import cli
 
     eval_file, out = tmp_path / "eval.json", tmp_path / "report"
-    eval_file.write_text(json.dumps({"evaluations": [{
-        "question_id": "q1", "question_text": "Q?", "with_docs": {"aggregate": 80},
-        "without_docs": {"aggregate": 60}, "delta": 20}]}), encoding="utf-8")
-    monkeypatch.setattr("sys.argv", ["dokimi", "report", "eval", "--product", "P", "--eval", str(eval_file),
-                                     "--out", str(out)])
+    eval_file.write_text(
+        json.dumps(
+            {
+                "evaluations": [
+                    {
+                        "question_id": "q1",
+                        "question_text": "Q?",
+                        "with_docs": {"aggregate": 80},
+                        "without_docs": {"aggregate": 60},
+                        "delta": 20,
+                    }
+                ]
+            }
+        ),
+        encoding="utf-8",
+    )
+    monkeypatch.setattr(
+        "sys.argv",
+        ["dokimi", "report", "eval", "--product", "P", "--eval", str(eval_file), "--out", str(out)],
+    )
     cli.main()
     assert not out.read_text(encoding="utf-8").lstrip().startswith("{")
     assert (tmp_path / "report.qa.json").exists()

From 5ebaecab466a1cc163394187b8574eec6c8dcbd1 Mon Sep 17 00:00:00 2001
From: Nikolay Petrov <nikolay.a.petrov@intel.com>
Date: Mon, 26 Oct 2026 11:08:06 -0700
Subject: [PATCH 15/18] fix(answers): match #237's comment on the
 runtime-config except so the two PRs merge cleanly

Signed-off-by: Nikolay Petrov <nikolay.a.petrov@intel.com>
---
 dokimi/dokimi/commands/answers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dokimi/dokimi/commands/answers.py b/dokimi/dokimi/commands/answers.py
index 8128af314..e40dcde8e 100644
--- a/dokimi/dokimi/commands/answers.py
+++ b/dokimi/dokimi/commands/answers.py
@@ -34,7 +34,7 @@ def _load_runtime_config() -> dict:
     except yaml.YAMLError as exc:
         print(f"Error: invalid YAML in {config_path}: {exc}", file=sys.stderr)
         raise SystemExit(1) from exc
-    except (OSError, ValueError) as exc:  # unreadable or not UTF-8
+    except (OSError, ValueError) as exc:  # ValueError covers bad UTF-8
         print(f"Error: cannot read {config_path}: {exc}", file=sys.stderr)
         raise SystemExit(1) from exc
     if loaded is not None and not isinstance(loaded, dict):

From 33bb01eecc62a9cb569f90db7b1603ea37a5ae04 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Mon, 26 Oct 2026 13:31:55 -0500
Subject: [PATCH 16/18] fix(answers): exit 2 naming the path when answers
 generate cannot write its output

generate_answers saves incrementally and save_answers saves at the end; an
unwritable --output now exits with 'Error: cannot write <path>' like eval score
instead of a traceback.

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/commands/answers.py    | 28 +++++++++-------
 dokimi/tests/test_answers_command.py | 50 ++++++++++++++++++++++++++++
 2 files changed, 66 insertions(+), 12 deletions(-)

diff --git a/dokimi/dokimi/commands/answers.py b/dokimi/dokimi/commands/answers.py
index e40dcde8e..28f66f98b 100644
--- a/dokimi/dokimi/commands/answers.py
+++ b/dokimi/dokimi/commands/answers.py
@@ -108,19 +108,23 @@ def cmd_answers_generate(args: argparse.Namespace) -> None:
     )
 
     output_path = _output_path(args)
-    answers = answerer.generate_answers(
-        library_name=args.product,
-        library_id=library_id,
-        questions=questions,
-        max_tokens_per_question=args.max_tokens,
-        output_path=output_path,
-        concurrency=args.concurrency,
-    )
+    try:
+        answers = answerer.generate_answers(
+            library_name=args.product,
+            library_id=library_id,
+            questions=questions,
+            max_tokens_per_question=args.max_tokens,
+            output_path=output_path,
+            concurrency=args.concurrency,
+        )
 
-    # generate_answers already saves incrementally to output_path after each
-    # question, so the final write here is a safety flush only (ensures the
-    # file reflects any last-second ordering fix).
-    answerer.save_answers(answers, output_path)
+        # generate_answers already saves incrementally to output_path after each
+        # question, so the final write here is a safety flush only (ensures the
+        # file reflects any last-second ordering fix).
+        answerer.save_answers(answers, output_path)
+    except OSError as exc:
+        print(f"Error: cannot write {exc.filename or output_path}: {exc.strerror or exc}", file=sys.stderr)
+        sys.exit(2)
     print(f"\n✅ Saved answers to {output_path}")
 
     # Optional RAGAS meta-evaluation (--ragas flag)
diff --git a/dokimi/tests/test_answers_command.py b/dokimi/tests/test_answers_command.py
index 052ac18d1..75b1e5fce 100644
--- a/dokimi/tests/test_answers_command.py
+++ b/dokimi/tests/test_answers_command.py
@@ -152,3 +152,53 @@ def save_answers(self, answers_, path):
     assert calls["generate"]["output_path"] == out
     assert calls["saved"] == (["a1"], out)
     assert f"Saved answers to {out}" in capsys.readouterr().out
+
+
+def _stub_generate(monkeypatch, tmp_path, answerer_cls):
+    _point_config(monkeypatch, tmp_path / "absent.yaml")
+    monkeypatch.setattr(dokimi.questions.artifact, "load_questions",
+                        lambda path: SimpleNamespace(questions=["q1"]))
+    monkeypatch.setattr(dokimi.mcp.factory, "create_doc_source_client", lambda source: "client")
+    monkeypatch.setattr(dokimi.mcp.factory, "resolve_library_id_for_product",
+                        lambda product, client, explicit_id=None: "/lib/id")
+    monkeypatch.setattr(dokimi.eval, "Answerer", answerer_cls)
+
+
+def test_generate_output_that_cannot_be_written_exits_2_naming_the_path(tmp_path, monkeypatch, capsys):
+    blocker = tmp_path / "blocker"
+    blocker.write_text("x", encoding="utf-8")  # a regular file: nothing can be created below it
+    target = blocker / "a.json"
+
+    class FakeAnswerer:
+        def __init__(self, **kwargs):
+            pass
+
+        def generate_answers(self, output_path, **kwargs):
+            output_path.parent.mkdir(parents=True, exist_ok=True)
+            return ["a1"]
+
+    _stub_generate(monkeypatch, tmp_path, FakeAnswerer)
+    with pytest.raises(SystemExit) as exc:
+        answers.cmd_answers_generate(_args(output=str(target)))
+    assert exc.value.code == 2
+    assert "Error: cannot write" in capsys.readouterr().err
+
+
+def test_generate_final_save_that_fails_exits_2_naming_the_path(tmp_path, monkeypatch, capsys):
+    target = tmp_path / "a.json"
+
+    class FakeAnswerer:
+        def __init__(self, **kwargs):
+            pass
+
+        def generate_answers(self, **kwargs):
+            return ["a1"]
+
+        def save_answers(self, answers_, path):
+            raise PermissionError(13, "Permission denied", str(path))
+
+    _stub_generate(monkeypatch, tmp_path, FakeAnswerer)
+    with pytest.raises(SystemExit) as exc:
+        answers.cmd_answers_generate(_args(output=str(target)))
+    assert exc.value.code == 2
+    assert f"Error: cannot write {target}: Permission denied" in capsys.readouterr().err

From fabe4bee57931b860ac792d348b2ffc603352935 Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Mon, 26 Oct 2026 13:58:56 -0500
Subject: [PATCH 17/18] fix(answers): name the --output path, not the temp
 file, when answers generate cannot write

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/commands/answers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dokimi/dokimi/commands/answers.py b/dokimi/dokimi/commands/answers.py
index 28f66f98b..ef4591926 100644
--- a/dokimi/dokimi/commands/answers.py
+++ b/dokimi/dokimi/commands/answers.py
@@ -123,7 +123,7 @@ def cmd_answers_generate(args: argparse.Namespace) -> None:
         # file reflects any last-second ordering fix).
         answerer.save_answers(answers, output_path)
     except OSError as exc:
-        print(f"Error: cannot write {exc.filename or output_path}: {exc.strerror or exc}", file=sys.stderr)
+        print(f"Error: cannot write {output_path}: {exc.strerror or exc}", file=sys.stderr)
         sys.exit(2)
     print(f"\n✅ Saved answers to {output_path}")
 

From eeebd1bd28b5c692243271cf9ff4220503428dfb Mon Sep 17 00:00:00 2001
From: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
Date: Mon, 26 Oct 2026 14:23:51 -0500
Subject: [PATCH 18/18] fix(answers): exit 1 when --questions cannot be read;
 assert real output path in write-error tests

_load_questions now catches OSError (missing file, directory) and exits 1
naming the path. eval score and panel-score report the --output path, not
the temp sibling. Tests use temp-named PermissionError filenames and a
real directory --output.

Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com>
---
 dokimi/dokimi/commands/answers.py             |  3 ++
 dokimi/dokimi/commands/evaluate.py            |  4 +-
 dokimi/tests/test_answers_command.py          | 46 ++++++++++++++++++-
 dokimi/tests/test_eval_output_write_errors.py | 10 ++--
 4 files changed, 54 insertions(+), 9 deletions(-)

diff --git a/dokimi/dokimi/commands/answers.py b/dokimi/dokimi/commands/answers.py
index ef4591926..a6bb9a013 100644
--- a/dokimi/dokimi/commands/answers.py
+++ b/dokimi/dokimi/commands/answers.py
@@ -51,6 +51,9 @@ def _load_questions(path: str) -> list:
     except ValueError as exc:  # ArtifactValidationError: names the record and the fix
         print(f"Error: {exc}", file=sys.stderr)
         raise SystemExit(1) from exc
+    except OSError as exc:
+        print(f"Error: cannot read {path}: {exc.strerror or exc}", file=sys.stderr)
+        raise SystemExit(1) from exc
 
 
 def _output_path(args: argparse.Namespace) -> Path:
diff --git a/dokimi/dokimi/commands/evaluate.py b/dokimi/dokimi/commands/evaluate.py
index 46cd8564c..73e713226 100644
--- a/dokimi/dokimi/commands/evaluate.py
+++ b/dokimi/dokimi/commands/evaluate.py
@@ -156,7 +156,7 @@ def cmd_eval_panel_score(args: argparse.Namespace) -> None:
                                          run_metadata={"answer_model": answer_model,
                                                        "answer_provider": answer_provider})
     except OSError as exc:
-        print(f"Error: cannot write {exc.filename or output_path}: {exc.strerror or exc}", file=sys.stderr)
+        print(f"Error: cannot write {output_path}: {exc.strerror or exc}", file=sys.stderr)
         sys.exit(2)
 
     # Summary: the context arm's own quality, not a comparison.
@@ -231,7 +231,7 @@ def cmd_eval_score(args: argparse.Namespace) -> None:
         print(f"Error: {args.answers}: {exc}", file=sys.stderr)
         sys.exit(2)
     except OSError as exc:
-        print(f"Error: cannot write {exc.filename or output_path}: {exc.strerror or exc}", file=sys.stderr)
+        print(f"Error: cannot write {output_path}: {exc.strerror or exc}", file=sys.stderr)
         sys.exit(2)
     print(f"\n✅ Saved evaluations to {output_path}")
 
diff --git a/dokimi/tests/test_answers_command.py b/dokimi/tests/test_answers_command.py
index 75b1e5fce..57501fdbf 100644
--- a/dokimi/tests/test_answers_command.py
+++ b/dokimi/tests/test_answers_command.py
@@ -181,7 +181,7 @@ def generate_answers(self, output_path, **kwargs):
     with pytest.raises(SystemExit) as exc:
         answers.cmd_answers_generate(_args(output=str(target)))
     assert exc.value.code == 2
-    assert "Error: cannot write" in capsys.readouterr().err
+    assert f"Error: cannot write {target}: " in capsys.readouterr().err
 
 
 def test_generate_final_save_that_fails_exits_2_naming_the_path(tmp_path, monkeypatch, capsys):
@@ -195,10 +195,52 @@ def generate_answers(self, **kwargs):
             return ["a1"]
 
         def save_answers(self, answers_, path):
-            raise PermissionError(13, "Permission denied", str(path))
+            raise PermissionError(13, "Permission denied", f"{path}.123.0.tmp")
 
     _stub_generate(monkeypatch, tmp_path, FakeAnswerer)
     with pytest.raises(SystemExit) as exc:
         answers.cmd_answers_generate(_args(output=str(target)))
     assert exc.value.code == 2
     assert f"Error: cannot write {target}: Permission denied" in capsys.readouterr().err
+
+
+def test_generate_output_that_is_a_directory_exits_2_naming_the_path(tmp_path, monkeypatch, capsys):
+    from dokimi.eval import Answerer as RealAnswerer
+
+    target = tmp_path / "outdir"
+    target.mkdir()
+
+    class FakeAnswerer:
+        def __init__(self, **kwargs):
+            pass
+
+        def generate_answers(self, **kwargs):
+            return [{"question_id": "q1"}]
+
+        model = "m"
+        provider = "p"
+
+        def _get_timestamp(self):
+            return "t"
+
+        _build_output = RealAnswerer._build_output
+
+        save_answers = RealAnswerer.save_answers
+
+    _stub_generate(monkeypatch, tmp_path, FakeAnswerer)
+    with pytest.raises(SystemExit) as exc:
+        answers.cmd_answers_generate(_args(output=str(target)))
+    assert exc.value.code == 2
+    assert f"Error: cannot write {target}: " in capsys.readouterr().err
+
+
+@pytest.mark.parametrize("kind", ["missing", "directory"])
+def test_generate_unreadable_questions_exits_1_naming_the_path(tmp_path, monkeypatch, capsys, kind):
+    _point_config(monkeypatch, tmp_path / "absent.yaml")
+    path = tmp_path / "q.json"
+    if kind == "directory":
+        path.mkdir()
+    with pytest.raises(SystemExit) as exc:
+        answers.cmd_answers_generate(_args(questions=str(path)))
+    assert exc.value.code == 1
+    assert f"Error: cannot read {path}" in capsys.readouterr().err
diff --git a/dokimi/tests/test_eval_output_write_errors.py b/dokimi/tests/test_eval_output_write_errors.py
index 2b0df47e6..07f145cd5 100644
--- a/dokimi/tests/test_eval_output_write_errors.py
+++ b/dokimi/tests/test_eval_output_write_errors.py
@@ -26,7 +26,7 @@ def test_write_text_or_exit_names_the_path_and_exits_with_the_code(tmp_path, cap
     with pytest.raises(SystemExit) as exc:
         write_text_or_exit(target, "{}", code=2)
     assert exc.value.code == 2
-    assert f"Error: cannot write {target}" in capsys.readouterr().err
+    assert f"Error: cannot write {target}: " in capsys.readouterr().err
 
 
 class _Report:
@@ -157,7 +157,7 @@ def __init__(self, **kw):
             pass
 
         def evaluate_answers(self, *a, **k):
-            raise PermissionError(13, "denied", str(target))
+            raise PermissionError(13, "denied", f"{target}.123.0.tmp")
 
     monkeypatch.setattr(panel_mod, "JudgePanel", Panel)
     monkeypatch.setattr(
@@ -169,7 +169,7 @@ def evaluate_answers(self, *a, **k):
     with pytest.raises(SystemExit) as exc:
         args.func(args)
     assert exc.value.code == 2
-    assert f"Error: cannot write {target}" in capsys.readouterr().err
+    assert f"Error: cannot write {target}: " in capsys.readouterr().err
 
 
 def test_score_output_that_cannot_be_written_exits_2_naming_the_path(tmp_path, monkeypatch, capsys):
@@ -178,7 +178,7 @@ def evaluate_answers(self, *a, **k):
             return []
 
         def save_evaluations(self, evals, path):
-            raise PermissionError(13, "Permission denied", str(path))
+            raise PermissionError(13, "Permission denied", f"{path}.123.0.tmp")
 
     answers = tmp_path / "answers.json"
     answers.write_text(json.dumps({"answers": []}), encoding="utf-8")
@@ -192,7 +192,7 @@ def save_evaluations(self, evals, path):
     with pytest.raises(SystemExit) as exc:
         args.func(args)
     assert exc.value.code == 2
-    assert f"Error: cannot write {target}" in capsys.readouterr().err
+    assert f"Error: cannot write {target}: " in capsys.readouterr().err
 
 
 def test_report_eval_names_the_sidecar_when_it_cannot_be_written(tmp_path, monkeypatch, capsys):