fix(cli): eval, report, answers, benchmark and evaluate exit with a message on bad input or output

dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-eval-report.md

@@ -0,0 +1,3 @@
+### Fixed
+
+- **`eval`, `report`, `answers`, `benchmark` and `evaluate`** exit with a message on an unreadable or invalid input file or an unwritable `--output`, instead of a traceback.

dokimi/dokimi/commands/answers.py

@@ -34,6 +34,9 @@ def _load_runtime_config() -> dict:
     except yaml.YAMLError as exc:
         print(f"Error: invalid YAML in {config_path}: {exc}", file=sys.stderr)
         raise SystemExit(1) from exc
+    except (OSError, ValueError) as exc:  # ValueError covers bad UTF-8
+        print(f"Error: cannot read {config_path}: {exc}", file=sys.stderr)
+        raise SystemExit(1) from exc
     if loaded is not None and not isinstance(loaded, dict):
         print(f"Error: expected a mapping in {config_path}", file=sys.stderr)
         raise SystemExit(1)
@@ -48,6 +51,9 @@ def _load_questions(path: str) -> list:
     except ValueError as exc:  # ArtifactValidationError: names the record and the fix
         print(f"Error: {exc}", file=sys.stderr)
         raise SystemExit(1) from exc
+    except OSError as exc:
+        print(f"Error: cannot read {path}: {exc.strerror or exc}", file=sys.stderr)
+        raise SystemExit(1) from exc
 
 
 def _output_path(args: argparse.Namespace) -> Path:
@@ -105,19 +111,23 @@ def cmd_answers_generate(args: argparse.Namespace) -> None:
     )
 
     output_path = _output_path(args)
-    answers = answerer.generate_answers(
-        library_name=args.product,
-        library_id=library_id,
-        questions=questions,
-        max_tokens_per_question=args.max_tokens,
-        output_path=output_path,
-        concurrency=args.concurrency,
-    )
+    try:
+        answers = answerer.generate_answers(
+            library_name=args.product,
+            library_id=library_id,
+            questions=questions,
+            max_tokens_per_question=args.max_tokens,
+            output_path=output_path,
+            concurrency=args.concurrency,
+        )
 
-    # generate_answers already saves incrementally to output_path after each
-    # question, so the final write here is a safety flush only (ensures the
-    # file reflects any last-second ordering fix).
-    answerer.save_answers(answers, output_path)
+        # generate_answers already saves incrementally to output_path after each
+        # question, so the final write here is a safety flush only (ensures the
+        # file reflects any last-second ordering fix).
+        answerer.save_answers(answers, output_path)
+    except OSError as exc:
+        print(f"Error: cannot write {output_path}: {exc.strerror or exc}", file=sys.stderr)
+        sys.exit(2)
     print(f"\n✅ Saved answers to {output_path}")
 
     # Optional RAGAS meta-evaluation (--ragas flag)

dokimi/dokimi/commands/benchmark.py

@@ -110,9 +110,22 @@ def _print_plan(plan: dict) -> None:
             print(f"  {key}: {value}")
 
 
+def _check_questions_from(questions_from: str, key: str) -> tuple[Path, list, str | None]:
+    """The resolved --questions-from file, its records, and a message when it is unusable."""
+    from dokimi.orchestrator.pipeline import load_questions_payload, resolve_questions_from_path
+
+    qf = resolve_questions_from_path(Path(questions_from), key)
+    if not qf.exists():
+        return qf, [], f"questions source not found: {qf}"
+    try:
+        return qf, load_questions_payload(qf), None
+    except Exception as exc:
+        return qf, [], f"questions source invalid: {qf} ({exc})"
+
+
 def cmd_benchmark_preflight(args: argparse.Namespace) -> None:
     """Validate the resolved benchmark plan without running LLM calls."""
-    from dokimi.orchestrator.pipeline import load_questions_payload, resolve_questions_from_path
+    from dokimi.orchestrator.pipeline import load_questions_payload
     from dokimi.mcp.factory import create_doc_source_client
 
     registry = _load_registry(args)
@@ -149,15 +162,11 @@ def cmd_benchmark_preflight(args: argparse.Namespace) -> None:
         errors.append(f"doc source invalid: {plan['doc_source']} ({exc})")
 
     if plan["questions_from"]:
-        qf = resolve_questions_from_path(Path(plan["questions_from"]), entry.key)
-        if not qf.exists():
-            errors.append(f"questions source not found: {qf}")
+        qf, questions, problem = _check_questions_from(plan["questions_from"], entry.key)
+        if problem:
+            errors.append(problem)
         else:
-            try:
-                questions = load_questions_payload(qf)
-                print(f"  questions: {qf} ({len(questions)} items)")
-            except Exception as exc:
-                errors.append(f"questions source invalid: {qf} ({exc})")
+            print(f"  questions: {qf} ({len(questions)} items)")
 
     # Check golden set only if no external questions source (--questions-from takes precedence)
     if plan["questions_root"] and not plan["questions_from"]:
@@ -218,6 +227,12 @@ def cmd_benchmark_run(args: argparse.Namespace) -> None:
     questions_from = getattr(args, "questions_from", None)
     questions_root = getattr(args, "questions_root", None)
 
+    if questions_from:
+        _, _, problem = _check_questions_from(questions_from, entry.key)
+        if problem:
+            print(f"Error: {problem}", file=sys.stderr)
+            sys.exit(1)
+
     if n_runs == 1:
         _run_single_library(
             entry,

dokimi/dokimi/commands/evaluate.py

@@ -20,7 +20,7 @@
 )
 from dokimi.eval.judge import JUDGE_PROVIDERS
 from dokimi.eval.judge_validity import DEFAULT_MIN_ACCURACY, NoUsablePairs
-from dokimi.utils import normalize_model_ref
+from dokimi.utils import normalize_model_ref, write_text_or_exit
 
 
 def _warn_judge_independence(
@@ -53,6 +53,11 @@ def _load_answers(path: str):
         sys.exit(2)
 
 
+def _write_json_or_exit(path: Path, payload: dict, code: int = 1) -> None:
+    """Write ``payload`` as JSON to ``path``, or exit ``code`` naming ``path`` when it cannot be written."""
+    write_text_or_exit(path, json.dumps(payload, indent=2), code)
+
+
 def _require_ragas() -> None:
     """Exit before any model call when the ragas extra is not installed."""
     if importlib.util.find_spec("ragas") is None:
@@ -76,10 +81,8 @@ def _run_ragas_eval(answers, output_path, args) -> None:
         result = evaluator.evaluate(answers, include_without_docs=True)
 
         # Save ragas results alongside answers
-        import json
         ragas_path = output_path.with_name(output_path.stem + "_ragas.json")
-        ragas_path.parent.mkdir(parents=True, exist_ok=True)
-        ragas_path.write_text(json.dumps(result.to_dict(), indent=2), encoding="utf-8")
+        _write_json_or_exit(ragas_path, result.to_dict(), code=2)
 
         print(result.format_summary())
         print(f"✅ Saved RAGAS results to {ragas_path}")
@@ -146,11 +149,15 @@ def cmd_eval_panel_score(args: argparse.Namespace) -> None:
     print(f"Roles: {', '.join(roles)}\nModel: {args.provider}/{args.model}\n")
 
     output_path = Path(args.output) if args.output else Path(f"eval/{args.product}_panel.json")
-    results = panel.evaluate_answers(answers, library_name=args.product,
-                                     output_path=output_path,
-                                     limit=getattr(args, "limit", None),
-                                     run_metadata={"answer_model": answer_model,
-                                                   "answer_provider": answer_provider})
+    try:
+        results = panel.evaluate_answers(answers, library_name=args.product,
+                                         output_path=output_path,
+                                         limit=getattr(args, "limit", None),
+                                         run_metadata={"answer_model": answer_model,
+                                                       "answer_provider": answer_provider})
+    except OSError as exc:
+        print(f"Error: cannot write {output_path}: {exc.strerror or exc}", file=sys.stderr)
+        sys.exit(2)
 
     # Summary: the context arm's own quality, not a comparison.
     from dokimi.records import CONTEXT, records
@@ -219,11 +226,13 @@ def cmd_eval_score(args: argparse.Namespace) -> None:
     try:
         evaluations = judge.evaluate_answers(args.product, answers, output_path=output_path,
                                              concurrency=args.concurrency)
+        judge.save_evaluations(evaluations, output_path)
     except RecordError as exc:
         print(f"Error: {args.answers}: {exc}", file=sys.stderr)
         sys.exit(2)
-
-    judge.save_evaluations(evaluations, output_path)
+    except OSError as exc:
+        print(f"Error: cannot write {output_path}: {exc.strerror or exc}", file=sys.stderr)
+        sys.exit(2)
     print(f"\n✅ Saved evaluations to {output_path}")
 
 
@@ -243,9 +252,6 @@ class _Args:
 
 def cmd_eval_grounding(args: argparse.Namespace) -> None:
     """Compute reference-free grounding/citation metrics (no LLM calls)."""
-    import json
-    from pathlib import Path
-
     from dokimi.eval.grounding import evaluate_grounding
 
     answers = _load_answers(args.answers).records
@@ -260,8 +266,7 @@ def cmd_eval_grounding(args: argparse.Namespace) -> None:
 
     if args.output:
         out = Path(args.output)
-        out.parent.mkdir(parents=True, exist_ok=True)
-        out.write_text(json.dumps(result, indent=2), encoding="utf-8")
+        _write_json_or_exit(out, result, code=2)
         print(f"\n✅ Grounding report saved to {out}")
 
 
@@ -405,14 +410,13 @@ def progress(done: int, total: int) -> None:
     # Written before anything is printed: a console that cannot encode the summary must not lose the run.
     if args.output:
         out = Path(args.output)
-        out.parent.mkdir(parents=True, exist_ok=True)
         result = {**report.to_dict(), "judge_temperature": args.judge_temperature}
         # Which answers the gate was measured on: a judge that passes on one
         # answer model's output is not thereby validated on another's.
         result["summary"]["answers_model"] = run.meta.get("model")
         if typed is not None:
             result["summary"]["typed_arm"] = typed.report_fields()
-        out.write_text(json.dumps(result, indent=2), encoding="utf-8")
+        _write_json_or_exit(out, result, code=2)
 
     print()
     print(report.format_summary())

dokimi/dokimi/commands/judge_reliability.py

@@ -19,6 +19,7 @@
 )
 from dokimi.eval.judge import JUDGE_PROVIDERS
 from dokimi.eval.judge_validity import DEFAULT_MIN_ACCURACY
+from dokimi.utils import write_text_or_exit
 
 
 def _read_json(path: str):
@@ -64,6 +65,17 @@ def _load_answers(path: str):
     return data, answers
 
 
+def _load_points(path: str):
+    """question_id -> required points from a questions file, or exit 2 naming the file."""
+    from dokimi.eval.checklist import load_points
+
+    try:
+        return load_points(path)
+    except (OSError, ValueError) as exc:  # ValueError covers bad JSON, bad UTF-8 and a refused set
+        print(f"Error: cannot read questions {path}: {exc}", file=sys.stderr)
+        sys.exit(2)
+
+
 def _require_arms(path: str, records, command: str) -> None:
     """Exit 2, before any judge call, when no answers record holds ``arms``.
 
@@ -154,10 +166,9 @@ def cmd_eval_pairwise(args: argparse.Namespace) -> None:
         sys.exit(2)
     md = report.format_markdown()
     out = Path(args.output or Path(args.answers).with_name(Path(args.answers).stem + "_pairwise.json"))
-    out.parent.mkdir(parents=True, exist_ok=True)
-    out.write_text(json.dumps({**report.to_dict(), "judge_temperature": getattr(args, "judge_temperature", None)},
-                              indent=2), encoding="utf-8")
-    out.with_suffix(".md").write_text(md, encoding="utf-8")
+    write_text_or_exit(out, json.dumps({**report.to_dict(), "judge_temperature": getattr(args, "judge_temperature", None)},
+                                       indent=2), code=2)
+    write_text_or_exit(out.with_suffix(".md"), md, code=2)
     if not report.verdicts:
         # Written first, so the reasons are not lost with the run.
         print(f"Error: all {len(report.errors)} comparison(s) failed; see `errors` in {out}", file=sys.stderr)
@@ -169,11 +180,11 @@ def cmd_eval_pairwise(args: argparse.Namespace) -> None:
 
 def cmd_eval_checklist(args: argparse.Namespace) -> None:
     """Required-point coverage of every arm, against the questions' expected_topics."""
-    from dokimi.eval.checklist import checklist_jobs, load_points, run_checklist
+    from dokimi.eval.checklist import checklist_jobs, run_checklist
     from dokimi.eval.judge import Judge
 
     data, records = _load_answers(args.answers)
-    points = load_points(args.questions)
+    points = _load_points(args.questions)
     baseline = args.baseline_arm or (data.get("baseline_arm") if isinstance(data, dict) else None) or "baseline"
     arms = [a.strip() for a in args.arms.split(",")] if args.arms else None
     _require_arms(args.answers, records, "checklist")
@@ -193,12 +204,10 @@ def cmd_eval_checklist(args: argparse.Namespace) -> None:
         print("Error: no question in the answers file has expected_topics in --questions", file=sys.stderr)
         sys.exit(2)
     out = Path(args.output or Path(args.answers).with_name(Path(args.answers).stem + "_checklist.json"))
-    out.parent.mkdir(parents=True, exist_ok=True)
     # The JSON first: the paid calls are in it, and a rendering error must not lose them.
-    out.write_text(json.dumps({**report.to_dict(), "judge_temperature": args.judge_temperature}, indent=2),
-                   encoding="utf-8")
+    write_text_or_exit(out, json.dumps({**report.to_dict(), "judge_temperature": args.judge_temperature}, indent=2), code=2)
     md = report.format_markdown()
-    out.with_suffix(".md").write_text(md, encoding="utf-8")
+    write_text_or_exit(out.with_suffix(".md"), md, code=2)
     print()
     print(md)
     print(f"✅ Coverage report saved to {out} (+ .md)")
@@ -241,9 +250,8 @@ def progress(cond: str, done: int, total: int) -> None:
     )
     if args.output:
         out = Path(args.output)
-        out.parent.mkdir(parents=True, exist_ok=True)
-        out.write_text(json.dumps({**report.to_dict(), "judge_temperature": args.judge_temperature}, indent=2),
-                       encoding="utf-8")
+        write_text_or_exit(out, json.dumps({**report.to_dict(), "judge_temperature": args.judge_temperature}, indent=2),
+                           code=2)
     print()
     print(report.format_summary())
     if args.output:
@@ -264,7 +272,6 @@ def cmd_eval_cross_judge(args: argparse.Namespace) -> None:
         stored_measure, unreproducible_arms,
     )
     from dokimi.eval.arm_runner import _ground_truth_from_topics
-    from dokimi.eval.checklist import load_points
     from dokimi.eval.judge import Judge
 
     data, records = _load_answers(args.answers)
@@ -274,7 +281,7 @@ def cmd_eval_cross_judge(args: argparse.Namespace) -> None:
             print("Error: --grounded needs --questions (their expected_topics are the ground truth)",
                   file=sys.stderr)
             sys.exit(2)
-        truth = {q: t for q, pts in load_points(args.questions).items()
+        truth = {q: t for q, pts in _load_points(args.questions).items()
                  if (t := _ground_truth_from_topics({"expected_topics": pts}))}
 
     def scorer_of(judge):
@@ -407,10 +414,9 @@ def score(library, question, answer, ground_truth, context):
         sys.exit(2)
     md = report.format_markdown()
     out = Path(args.output or Path(args.answers).with_name(Path(args.answers).stem + "_crossjudge.json"))
-    out.parent.mkdir(parents=True, exist_ok=True)
-    out.write_text(json.dumps({**report.to_dict(), "judge_temperature": args.judge_temperature}, indent=2),
-                   encoding="utf-8")
-    out.with_suffix(".md").write_text(md, encoding="utf-8")
+    write_text_or_exit(out, json.dumps({**report.to_dict(), "judge_temperature": args.judge_temperature}, indent=2),
+                       code=2)
+    write_text_or_exit(out.with_suffix(".md"), md, code=2)
     print()
     print(md)
     print(f"✅ Cross-judge report saved to {out} (+ .md)")

dokimi/dokimi/commands/orchestrate.py

@@ -46,28 +46,32 @@ def cmd_evaluate(args: argparse.Namespace) -> None:
         context="evaluate",
     )
 
-    # Create pipeline
-    pipeline = EvaluationPipeline(
-        product=args.product,
-        repo=getattr(args, "repo", None),
-        description=getattr(args, "description", None),
-        output_dir=Path(args.output_dir),
-        custom_questions_path=Path(args.custom_questions) if args.custom_questions else None,
-        questions_root=Path(args.questions_root) if getattr(args, "questions_root", None) else None,
-        model=args.model,
-        provider=args.provider,
-        judge_model=args.judge_model,
-        judge_provider=args.judge_provider,
-        judge_temperature=args.judge_temperature,
-        personas_count=args.personas_count,
-        questions_per_topic=args.questions_per_topic,
-        # Only pass top_k when explicitly set; otherwise let pipeline use config default
-        **({"top_k": args.top_k} if args.top_k is not None else {}),
-        rerank_threshold=args.rerank_threshold,
-        debug_retrieval=args.debug_retrieval,
-        doc_source=getattr(args, "doc_source", "context7"),
-        context7_id=getattr(args, "context7_id", None),
-    )
+    # Create pipeline; it checks --custom-questions and creates the output directories
+    try:
+        pipeline = EvaluationPipeline(
+            product=args.product,
+            repo=getattr(args, "repo", None),
+            description=getattr(args, "description", None),
+            output_dir=Path(args.output_dir),
+            custom_questions_path=Path(args.custom_questions) if args.custom_questions else None,
+            questions_root=Path(args.questions_root) if getattr(args, "questions_root", None) else None,
+            model=args.model,
+            provider=args.provider,
+            judge_model=args.judge_model,
+            judge_provider=args.judge_provider,
+            judge_temperature=args.judge_temperature,
+            personas_count=args.personas_count,
+            questions_per_topic=args.questions_per_topic,
+            # Only pass top_k when explicitly set; otherwise let pipeline use config default
+            **({"top_k": args.top_k} if args.top_k is not None else {}),
+            rerank_threshold=args.rerank_threshold,
+            debug_retrieval=args.debug_retrieval,
+            doc_source=getattr(args, "doc_source", "context7"),
+            context7_id=getattr(args, "context7_id", None),
+        )
+    except OSError as exc:  # a missing --custom-questions file, an unusable --output-dir
+        print(f"Error: {exc}", file=sys.stderr)
+        sys.exit(1)
 
     # Run pipeline
     try: