diff --git a/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-eval-report.md b/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-eval-report.md new file mode 100644 index 000000000..3d975eb64 --- /dev/null +++ b/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-eval-report.md @@ -0,0 +1,3 @@ +### Fixed + +- **`eval`, `report`, `answers`, `benchmark` and `evaluate`** exit with a message on an unreadable or invalid input file or an unwritable `--output`, instead of a traceback. diff --git a/dokimi/dokimi/commands/answers.py b/dokimi/dokimi/commands/answers.py index 46383d4f6..a6bb9a013 100644 --- a/dokimi/dokimi/commands/answers.py +++ b/dokimi/dokimi/commands/answers.py @@ -34,6 +34,9 @@ def _load_runtime_config() -> dict: except yaml.YAMLError as exc: print(f"Error: invalid YAML in {config_path}: {exc}", file=sys.stderr) raise SystemExit(1) from exc + except (OSError, ValueError) as exc: # ValueError covers bad UTF-8 + print(f"Error: cannot read {config_path}: {exc}", file=sys.stderr) + raise SystemExit(1) from exc if loaded is not None and not isinstance(loaded, dict): print(f"Error: expected a mapping in {config_path}", file=sys.stderr) raise SystemExit(1) @@ -48,6 +51,9 @@ def _load_questions(path: str) -> list: except ValueError as exc: # ArtifactValidationError: names the record and the fix print(f"Error: {exc}", file=sys.stderr) raise SystemExit(1) from exc + except OSError as exc: + print(f"Error: cannot read {path}: {exc.strerror or exc}", file=sys.stderr) + raise SystemExit(1) from exc def _output_path(args: argparse.Namespace) -> Path: @@ -105,19 +111,23 @@ def cmd_answers_generate(args: argparse.Namespace) -> None: ) output_path = _output_path(args) - answers = answerer.generate_answers( - library_name=args.product, - library_id=library_id, - questions=questions, - max_tokens_per_question=args.max_tokens, - output_path=output_path, - concurrency=args.concurrency, - ) + try: + answers = answerer.generate_answers( + library_name=args.product, + library_id=library_id, + questions=questions, + max_tokens_per_question=args.max_tokens, + output_path=output_path, + concurrency=args.concurrency, + ) - # generate_answers already saves incrementally to output_path after each - # question, so the final write here is a safety flush only (ensures the - # file reflects any last-second ordering fix). - answerer.save_answers(answers, output_path) + # generate_answers already saves incrementally to output_path after each + # question, so the final write here is a safety flush only (ensures the + # file reflects any last-second ordering fix). + answerer.save_answers(answers, output_path) + except OSError as exc: + print(f"Error: cannot write {output_path}: {exc.strerror or exc}", file=sys.stderr) + sys.exit(2) print(f"\n✅ Saved answers to {output_path}") # Optional RAGAS meta-evaluation (--ragas flag) diff --git a/dokimi/dokimi/commands/benchmark.py b/dokimi/dokimi/commands/benchmark.py index b4b1f129b..2c50c724e 100644 --- a/dokimi/dokimi/commands/benchmark.py +++ b/dokimi/dokimi/commands/benchmark.py @@ -110,9 +110,22 @@ def _print_plan(plan: dict) -> None: print(f" {key}: {value}") +def _check_questions_from(questions_from: str, key: str) -> tuple[Path, list, str | None]: + """The resolved --questions-from file, its records, and a message when it is unusable.""" + from dokimi.orchestrator.pipeline import load_questions_payload, resolve_questions_from_path + + qf = resolve_questions_from_path(Path(questions_from), key) + if not qf.exists(): + return qf, [], f"questions source not found: {qf}" + try: + return qf, load_questions_payload(qf), None + except Exception as exc: + return qf, [], f"questions source invalid: {qf} ({exc})" + + def cmd_benchmark_preflight(args: argparse.Namespace) -> None: """Validate the resolved benchmark plan without running LLM calls.""" - from dokimi.orchestrator.pipeline import load_questions_payload, resolve_questions_from_path + from dokimi.orchestrator.pipeline import load_questions_payload from dokimi.mcp.factory import create_doc_source_client registry = _load_registry(args) @@ -149,15 +162,11 @@ def cmd_benchmark_preflight(args: argparse.Namespace) -> None: errors.append(f"doc source invalid: {plan['doc_source']} ({exc})") if plan["questions_from"]: - qf = resolve_questions_from_path(Path(plan["questions_from"]), entry.key) - if not qf.exists(): - errors.append(f"questions source not found: {qf}") + qf, questions, problem = _check_questions_from(plan["questions_from"], entry.key) + if problem: + errors.append(problem) else: - try: - questions = load_questions_payload(qf) - print(f" questions: {qf} ({len(questions)} items)") - except Exception as exc: - errors.append(f"questions source invalid: {qf} ({exc})") + print(f" questions: {qf} ({len(questions)} items)") # Check golden set only if no external questions source (--questions-from takes precedence) if plan["questions_root"] and not plan["questions_from"]: @@ -218,6 +227,12 @@ def cmd_benchmark_run(args: argparse.Namespace) -> None: questions_from = getattr(args, "questions_from", None) questions_root = getattr(args, "questions_root", None) + if questions_from: + _, _, problem = _check_questions_from(questions_from, entry.key) + if problem: + print(f"Error: {problem}", file=sys.stderr) + sys.exit(1) + if n_runs == 1: _run_single_library( entry, diff --git a/dokimi/dokimi/commands/evaluate.py b/dokimi/dokimi/commands/evaluate.py index 0618fcf63..73e713226 100644 --- a/dokimi/dokimi/commands/evaluate.py +++ b/dokimi/dokimi/commands/evaluate.py @@ -20,7 +20,7 @@ ) from dokimi.eval.judge import JUDGE_PROVIDERS from dokimi.eval.judge_validity import DEFAULT_MIN_ACCURACY, NoUsablePairs -from dokimi.utils import normalize_model_ref +from dokimi.utils import normalize_model_ref, write_text_or_exit def _warn_judge_independence( @@ -53,6 +53,11 @@ def _load_answers(path: str): sys.exit(2) +def _write_json_or_exit(path: Path, payload: dict, code: int = 1) -> None: + """Write ``payload`` as JSON to ``path``, or exit ``code`` naming ``path`` when it cannot be written.""" + write_text_or_exit(path, json.dumps(payload, indent=2), code) + + def _require_ragas() -> None: """Exit before any model call when the ragas extra is not installed.""" if importlib.util.find_spec("ragas") is None: @@ -76,10 +81,8 @@ def _run_ragas_eval(answers, output_path, args) -> None: result = evaluator.evaluate(answers, include_without_docs=True) # Save ragas results alongside answers - import json ragas_path = output_path.with_name(output_path.stem + "_ragas.json") - ragas_path.parent.mkdir(parents=True, exist_ok=True) - ragas_path.write_text(json.dumps(result.to_dict(), indent=2), encoding="utf-8") + _write_json_or_exit(ragas_path, result.to_dict(), code=2) print(result.format_summary()) print(f"✅ Saved RAGAS results to {ragas_path}") @@ -146,11 +149,15 @@ def cmd_eval_panel_score(args: argparse.Namespace) -> None: print(f"Roles: {', '.join(roles)}\nModel: {args.provider}/{args.model}\n") output_path = Path(args.output) if args.output else Path(f"eval/{args.product}_panel.json") - results = panel.evaluate_answers(answers, library_name=args.product, - output_path=output_path, - limit=getattr(args, "limit", None), - run_metadata={"answer_model": answer_model, - "answer_provider": answer_provider}) + try: + results = panel.evaluate_answers(answers, library_name=args.product, + output_path=output_path, + limit=getattr(args, "limit", None), + run_metadata={"answer_model": answer_model, + "answer_provider": answer_provider}) + except OSError as exc: + print(f"Error: cannot write {output_path}: {exc.strerror or exc}", file=sys.stderr) + sys.exit(2) # Summary: the context arm's own quality, not a comparison. from dokimi.records import CONTEXT, records @@ -219,11 +226,13 @@ def cmd_eval_score(args: argparse.Namespace) -> None: try: evaluations = judge.evaluate_answers(args.product, answers, output_path=output_path, concurrency=args.concurrency) + judge.save_evaluations(evaluations, output_path) except RecordError as exc: print(f"Error: {args.answers}: {exc}", file=sys.stderr) sys.exit(2) - - judge.save_evaluations(evaluations, output_path) + except OSError as exc: + print(f"Error: cannot write {output_path}: {exc.strerror or exc}", file=sys.stderr) + sys.exit(2) print(f"\n✅ Saved evaluations to {output_path}") @@ -243,9 +252,6 @@ class _Args: def cmd_eval_grounding(args: argparse.Namespace) -> None: """Compute reference-free grounding/citation metrics (no LLM calls).""" - import json - from pathlib import Path - from dokimi.eval.grounding import evaluate_grounding answers = _load_answers(args.answers).records @@ -260,8 +266,7 @@ def cmd_eval_grounding(args: argparse.Namespace) -> None: if args.output: out = Path(args.output) - out.parent.mkdir(parents=True, exist_ok=True) - out.write_text(json.dumps(result, indent=2), encoding="utf-8") + _write_json_or_exit(out, result, code=2) print(f"\n✅ Grounding report saved to {out}") @@ -405,14 +410,13 @@ def progress(done: int, total: int) -> None: # Written before anything is printed: a console that cannot encode the summary must not lose the run. if args.output: out = Path(args.output) - out.parent.mkdir(parents=True, exist_ok=True) result = {**report.to_dict(), "judge_temperature": args.judge_temperature} # Which answers the gate was measured on: a judge that passes on one # answer model's output is not thereby validated on another's. result["summary"]["answers_model"] = run.meta.get("model") if typed is not None: result["summary"]["typed_arm"] = typed.report_fields() - out.write_text(json.dumps(result, indent=2), encoding="utf-8") + _write_json_or_exit(out, result, code=2) print() print(report.format_summary()) diff --git a/dokimi/dokimi/commands/judge_reliability.py b/dokimi/dokimi/commands/judge_reliability.py index 8e2e86cec..ffca9a0fa 100644 --- a/dokimi/dokimi/commands/judge_reliability.py +++ b/dokimi/dokimi/commands/judge_reliability.py @@ -19,6 +19,7 @@ ) from dokimi.eval.judge import JUDGE_PROVIDERS from dokimi.eval.judge_validity import DEFAULT_MIN_ACCURACY +from dokimi.utils import write_text_or_exit def _read_json(path: str): @@ -64,6 +65,17 @@ def _load_answers(path: str): return data, answers +def _load_points(path: str): + """question_id -> required points from a questions file, or exit 2 naming the file.""" + from dokimi.eval.checklist import load_points + + try: + return load_points(path) + except (OSError, ValueError) as exc: # ValueError covers bad JSON, bad UTF-8 and a refused set + print(f"Error: cannot read questions {path}: {exc}", file=sys.stderr) + sys.exit(2) + + def _require_arms(path: str, records, command: str) -> None: """Exit 2, before any judge call, when no answers record holds ``arms``. @@ -154,10 +166,9 @@ def cmd_eval_pairwise(args: argparse.Namespace) -> None: sys.exit(2) md = report.format_markdown() out = Path(args.output or Path(args.answers).with_name(Path(args.answers).stem + "_pairwise.json")) - out.parent.mkdir(parents=True, exist_ok=True) - out.write_text(json.dumps({**report.to_dict(), "judge_temperature": getattr(args, "judge_temperature", None)}, - indent=2), encoding="utf-8") - out.with_suffix(".md").write_text(md, encoding="utf-8") + write_text_or_exit(out, json.dumps({**report.to_dict(), "judge_temperature": getattr(args, "judge_temperature", None)}, + indent=2), code=2) + write_text_or_exit(out.with_suffix(".md"), md, code=2) if not report.verdicts: # Written first, so the reasons are not lost with the run. print(f"Error: all {len(report.errors)} comparison(s) failed; see `errors` in {out}", file=sys.stderr) @@ -169,11 +180,11 @@ def cmd_eval_pairwise(args: argparse.Namespace) -> None: def cmd_eval_checklist(args: argparse.Namespace) -> None: """Required-point coverage of every arm, against the questions' expected_topics.""" - from dokimi.eval.checklist import checklist_jobs, load_points, run_checklist + from dokimi.eval.checklist import checklist_jobs, run_checklist from dokimi.eval.judge import Judge data, records = _load_answers(args.answers) - points = load_points(args.questions) + points = _load_points(args.questions) baseline = args.baseline_arm or (data.get("baseline_arm") if isinstance(data, dict) else None) or "baseline" arms = [a.strip() for a in args.arms.split(",")] if args.arms else None _require_arms(args.answers, records, "checklist") @@ -193,12 +204,10 @@ def cmd_eval_checklist(args: argparse.Namespace) -> None: print("Error: no question in the answers file has expected_topics in --questions", file=sys.stderr) sys.exit(2) out = Path(args.output or Path(args.answers).with_name(Path(args.answers).stem + "_checklist.json")) - out.parent.mkdir(parents=True, exist_ok=True) # The JSON first: the paid calls are in it, and a rendering error must not lose them. - out.write_text(json.dumps({**report.to_dict(), "judge_temperature": args.judge_temperature}, indent=2), - encoding="utf-8") + write_text_or_exit(out, json.dumps({**report.to_dict(), "judge_temperature": args.judge_temperature}, indent=2), code=2) md = report.format_markdown() - out.with_suffix(".md").write_text(md, encoding="utf-8") + write_text_or_exit(out.with_suffix(".md"), md, code=2) print() print(md) print(f"✅ Coverage report saved to {out} (+ .md)") @@ -241,9 +250,8 @@ def progress(cond: str, done: int, total: int) -> None: ) if args.output: out = Path(args.output) - out.parent.mkdir(parents=True, exist_ok=True) - out.write_text(json.dumps({**report.to_dict(), "judge_temperature": args.judge_temperature}, indent=2), - encoding="utf-8") + write_text_or_exit(out, json.dumps({**report.to_dict(), "judge_temperature": args.judge_temperature}, indent=2), + code=2) print() print(report.format_summary()) if args.output: @@ -264,7 +272,6 @@ def cmd_eval_cross_judge(args: argparse.Namespace) -> None: stored_measure, unreproducible_arms, ) from dokimi.eval.arm_runner import _ground_truth_from_topics - from dokimi.eval.checklist import load_points from dokimi.eval.judge import Judge data, records = _load_answers(args.answers) @@ -274,7 +281,7 @@ def cmd_eval_cross_judge(args: argparse.Namespace) -> None: print("Error: --grounded needs --questions (their expected_topics are the ground truth)", file=sys.stderr) sys.exit(2) - truth = {q: t for q, pts in load_points(args.questions).items() + truth = {q: t for q, pts in _load_points(args.questions).items() if (t := _ground_truth_from_topics({"expected_topics": pts}))} def scorer_of(judge): @@ -407,10 +414,9 @@ def score(library, question, answer, ground_truth, context): sys.exit(2) md = report.format_markdown() out = Path(args.output or Path(args.answers).with_name(Path(args.answers).stem + "_crossjudge.json")) - out.parent.mkdir(parents=True, exist_ok=True) - out.write_text(json.dumps({**report.to_dict(), "judge_temperature": args.judge_temperature}, indent=2), - encoding="utf-8") - out.with_suffix(".md").write_text(md, encoding="utf-8") + write_text_or_exit(out, json.dumps({**report.to_dict(), "judge_temperature": args.judge_temperature}, indent=2), + code=2) + write_text_or_exit(out.with_suffix(".md"), md, code=2) print() print(md) print(f"✅ Cross-judge report saved to {out} (+ .md)") diff --git a/dokimi/dokimi/commands/orchestrate.py b/dokimi/dokimi/commands/orchestrate.py index 2f86b55f6..f31b9bbac 100644 --- a/dokimi/dokimi/commands/orchestrate.py +++ b/dokimi/dokimi/commands/orchestrate.py @@ -46,28 +46,32 @@ def cmd_evaluate(args: argparse.Namespace) -> None: context="evaluate", ) - # Create pipeline - pipeline = EvaluationPipeline( - product=args.product, - repo=getattr(args, "repo", None), - description=getattr(args, "description", None), - output_dir=Path(args.output_dir), - custom_questions_path=Path(args.custom_questions) if args.custom_questions else None, - questions_root=Path(args.questions_root) if getattr(args, "questions_root", None) else None, - model=args.model, - provider=args.provider, - judge_model=args.judge_model, - judge_provider=args.judge_provider, - judge_temperature=args.judge_temperature, - personas_count=args.personas_count, - questions_per_topic=args.questions_per_topic, - # Only pass top_k when explicitly set; otherwise let pipeline use config default - **({"top_k": args.top_k} if args.top_k is not None else {}), - rerank_threshold=args.rerank_threshold, - debug_retrieval=args.debug_retrieval, - doc_source=getattr(args, "doc_source", "context7"), - context7_id=getattr(args, "context7_id", None), - ) + # Create pipeline; it checks --custom-questions and creates the output directories + try: + pipeline = EvaluationPipeline( + product=args.product, + repo=getattr(args, "repo", None), + description=getattr(args, "description", None), + output_dir=Path(args.output_dir), + custom_questions_path=Path(args.custom_questions) if args.custom_questions else None, + questions_root=Path(args.questions_root) if getattr(args, "questions_root", None) else None, + model=args.model, + provider=args.provider, + judge_model=args.judge_model, + judge_provider=args.judge_provider, + judge_temperature=args.judge_temperature, + personas_count=args.personas_count, + questions_per_topic=args.questions_per_topic, + # Only pass top_k when explicitly set; otherwise let pipeline use config default + **({"top_k": args.top_k} if args.top_k is not None else {}), + rerank_threshold=args.rerank_threshold, + debug_retrieval=args.debug_retrieval, + doc_source=getattr(args, "doc_source", "context7"), + context7_id=getattr(args, "context7_id", None), + ) + except OSError as exc: # a missing --custom-questions file, an unusable --output-dir + print(f"Error: {exc}", file=sys.stderr) + sys.exit(1) # Run pipeline try: diff --git a/dokimi/dokimi/commands/report.py b/dokimi/dokimi/commands/report.py index f52f1c321..58d47934a 100644 --- a/dokimi/dokimi/commands/report.py +++ b/dokimi/dokimi/commands/report.py @@ -13,11 +13,30 @@ from dokimi.report.markdown_report import write_compare_report, write_run_report +def _write_combined_report(args, regular_runs, golden_runs, run_ids, warnings) -> None: + """Write the combined model-compare report, exiting cleanly if the output is unwritable.""" + from dokimi.report.model_compare import generate_combined_report + + try: + generate_combined_report( + regular_runs, + golden_runs, + run_ids, + args.out, + treatment_arm=getattr(args, "treatment_arm", None), + correction=getattr(args, "stats_correction", "holm"), + pool_replicates=getattr(args, "pool_replicates", False), + warnings=warnings, + ) + except OSError as exc: + print(f"Error: cannot write {args.out}: {exc}", file=sys.stderr) + raise SystemExit(1) from None + + def cmd_report_model_compare(args: argparse.Namespace) -> None: """Compare multiple model runs and write a Markdown report.""" from dokimi.report.model_compare import ( check_run_consistency, - generate_combined_report, load_run, ) @@ -67,16 +86,7 @@ def cmd_report_model_compare(args: argparse.Namespace) -> None: for w in all_warnings: print(w, file=sys.stderr) - generate_combined_report( - regular_runs, - golden_runs, - run_ids, - args.out, - treatment_arm=pinned_arm, - correction=getattr(args, "stats_correction", "holm"), - pool_replicates=pool_replicates, - warnings=all_warnings, - ) + _write_combined_report(args, regular_runs, golden_runs, run_ids, all_warnings) def cmd_report(args: argparse.Namespace) -> None: @@ -106,6 +116,9 @@ def cmd_report_eval(args: argparse.Namespace) -> None: except RecordError as exc: print(f"Error: {exc}", file=sys.stderr) raise SystemExit(1) from None + except OSError as exc: + print(f"Error: cannot write {out}: {exc}", file=sys.stderr) + raise SystemExit(1) from None def cmd_report_generate(args: argparse.Namespace) -> None: @@ -129,6 +142,9 @@ def cmd_report_generate(args: argparse.Namespace) -> None: try: questions_data = load_questions(args.questions).artifact + except OSError as exc: + print(f"Error: cannot read {args.questions}: {exc}", file=sys.stderr) + raise SystemExit(1) from None except ValueError as exc: # ArtifactValidationError: names the record and the fix print(f"Error: {exc}", file=sys.stderr) raise SystemExit(1) from exc @@ -150,8 +166,12 @@ def cmd_report_generate(args: argparse.Namespace) -> None: # Save output output_path = Path(args.output) if args.output else Path(f"reports/{args.product}.{args.format}") - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(report, encoding="utf-8") + try: + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(report, encoding="utf-8") + except OSError as exc: + print(f"Error: cannot write {output_path}: {exc}", file=sys.stderr) + raise SystemExit(1) from None print(f"\n✅ Generated report: {output_path}") diff --git a/dokimi/dokimi/report/eval_report.py b/dokimi/dokimi/report/eval_report.py index 5af77dfdd..ebb8c657f 100755 --- a/dokimi/dokimi/report/eval_report.py +++ b/dokimi/dokimi/report/eval_report.py @@ -173,9 +173,31 @@ def _answers_by_id(answers_path: str) -> dict[str, Record]: return out +def _load_eval(eval_path: str) -> dict: + """The eval file's JSON object, or a RecordError naming the file.""" + try: + with open(eval_path, encoding="utf-8") as f: + data = json.load(f) + except OSError as exc: + raise RecordError(f"cannot read {eval_path}: {exc}") from exc + except ValueError as exc: # not JSON, or not UTF-8 + raise RecordError(f"{eval_path} is not valid UTF-8 JSON: {exc}") from exc + if not isinstance(data, dict) or not isinstance(data.get("evaluations"), list): + raise RecordError(f"{eval_path} is not a JSON object with an `evaluations` list") + return data + + +def _write_qa_json(path: str, payload: dict) -> None: + """Write the Q&A sidecar, or raise a RecordError naming ``path``.""" + try: + with open(path, "w", encoding="utf-8") as jf: + json.dump(payload, jf, indent=2, ensure_ascii=False) + except OSError as exc: + raise RecordError(f"cannot write {path}: {exc.strerror or exc}") from exc + + def generate_report(eval_path: str, out_path: str, qa_json_out: str | None = None): - with open(eval_path, encoding="utf-8") as f: - data = json.load(f) + data = _load_eval(eval_path) evals = records(data["evaluations"], source=eval_path, kind="evaluations") meta = data.get("run_metadata", {}) @@ -449,7 +471,7 @@ def _val(key): # Sidecar JSON: questions + answers + scores if qa_json_out is None: - qa_json_out = out_path.replace(".md", ".qa.json") + qa_json_out = os.path.splitext(out_path)[0] + ".qa.json" answers_by_id = _answers_by_id(eval_path.replace("/eval/", "/answers/")) @@ -472,15 +494,14 @@ def _answer_text(a: Record | None, condition: str): "without_docs_answer": _answer_text(a, BASELINE), }) - with open(qa_json_out, "w", encoding="utf-8") as jf: - json.dump({ - "eval_path": eval_path, - "report_path": out_path, - "generated_at": datetime.now(timezone.utc).isoformat(), - "run_metadata": meta, - "count": len(qa_records), - "records": qa_records, - }, jf, indent=2, ensure_ascii=False) + _write_qa_json(qa_json_out, { + "eval_path": eval_path, + "report_path": out_path, + "generated_at": datetime.now(timezone.utc).isoformat(), + "run_metadata": meta, + "count": len(qa_records), + "records": qa_records, + }) print(f"✅ Report written to: {out_path}") print(f"✅ Q&A JSON written to: {qa_json_out}") diff --git a/dokimi/dokimi/report/model_compare.py b/dokimi/dokimi/report/model_compare.py index cec86bf7d..efa6e5267 100644 --- a/dokimi/dokimi/report/model_compare.py +++ b/dokimi/dokimi/report/model_compare.py @@ -63,13 +63,12 @@ def load_run(path: str) -> dict[str, Any]: """Load a judged arms JSON file. Raises SystemExit with a clear message on failure.""" try: - with open(path, encoding="utf-8") as f: - data = json.load(f) + data = json.loads(Path(path).read_text(encoding="utf-8")) except FileNotFoundError as exc: raise SystemExit(f"Error: File not found: {path}") from exc - except PermissionError as exc: - raise SystemExit(f"Error: Permission denied reading {path}") from exc - except json.JSONDecodeError as exc: + except OSError as exc: # permission denied, a directory, an I/O error + raise SystemExit(f"Error: cannot read {path}: {exc.strerror or exc}") from exc + except ValueError as exc: # JSONDecodeError, or bytes that are not UTF-8 raise SystemExit(f"Error: Invalid JSON in {path}: {exc}") from exc if isinstance(data, dict): for section in ("answers", "evaluations"): diff --git a/dokimi/dokimi/utils.py b/dokimi/dokimi/utils.py index 9c0ffeda9..cda2ff68c 100644 --- a/dokimi/dokimi/utils.py +++ b/dokimi/dokimi/utils.py @@ -213,3 +213,14 @@ def read_json_input(path: str | os.PathLike[str], *, flag: str) -> Any: except (OSError, ValueError, RecursionError) as exc: # JSONDecodeError and bad UTF-8 are ValueErrors reason = exc.strerror if isinstance(exc, OSError) and exc.strerror else exc raise InputError(f"{flag} {path}: {reason}") from exc + + +def write_text_or_exit(path: Path, text: str, code: int = 1) -> None: + """Write ``text`` to ``path`` (creating parents), or exit ``code`` naming ``path`` when it cannot be written.""" + import sys + try: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + except OSError as exc: + print(f"Error: cannot write {path}: {exc}", file=sys.stderr) + sys.exit(code) diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index d0d35275e..2c4d7eda2 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -4,7 +4,7 @@ "dokimi/harnesses/benchflow.py": {"lines": 869}, "dokimi/harnesses/docker_solver.py": {"lines": 900, "rationale": "pre-existing when the gate was introduced"}, "dokimi/llm.py": {"lines": 1118, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"}, - "dokimi/report/model_compare.py": {"lines": 1210, "rationale": "grew on main in #73 (judge-validity --context stored) before the ratchet landed; split it in its own PR"}, + "dokimi/report/model_compare.py": {"lines": 1209, "rationale": "grew on main in #73 (judge-validity --context stored) before the ratchet landed; split it in its own PR"}, "tests/test_arm_runner.py": {"lines": 1176, "rationale": "pre-existing when the gate was introduced"}, "tests/test_benchflow_harness.py": {"lines": 1716, "rationale": "pre-existing when the gate was introduced"}, "tests/test_container_agent.py": {"lines": 1251, "rationale": "pre-existing when the gate was introduced"}, @@ -19,7 +19,7 @@ "dokimi/commands/arms.py::_build_arms_output": {"complexity": 13}, "dokimi/commands/arms.py::_validate_expected_skills": {"complexity": 11}, "dokimi/commands/benchmark.py::cmd_benchmark_batch": {"complexity": 11}, - "dokimi/commands/benchmark.py::cmd_benchmark_preflight": {"complexity": 22}, + "dokimi/commands/benchmark.py::cmd_benchmark_preflight": {"complexity": 21}, "dokimi/commands/catalog.py::cmd_catalog_trigger_set": {"complexity": 16}, "dokimi/commands/evaluate.py::cmd_eval_judge_validity": {"complexity": 29, "rationale": "grew on main in #73 (judge-validity --context stored) from 25 to 29 before the ratchet landed; split it in its own PR"}, "dokimi/commands/judge_reliability.py::cmd_eval_cross_judge": {"complexity": 32, "rationale": "pre-existing when the gate was introduced"}, diff --git a/dokimi/tests/test_answers_command.py b/dokimi/tests/test_answers_command.py index 0fab43167..57501fdbf 100644 --- a/dokimi/tests/test_answers_command.py +++ b/dokimi/tests/test_answers_command.py @@ -63,6 +63,26 @@ def test_invalid_yaml_config_exits(tmp_path, monkeypatch, capsys): assert "invalid YAML" in capsys.readouterr().err +def test_non_utf8_config_exits_with_message(tmp_path, monkeypatch, capsys): + path = tmp_path / "products.yaml" + path.write_bytes(b"retrieval:\n top_k: \xff\xfe\n") + _point_config(monkeypatch, path) + with pytest.raises(SystemExit) as exc: + answers._load_runtime_config() + assert exc.value.code == 1 + assert f"cannot read {path}" in capsys.readouterr().err + + +def test_unreadable_config_exits_with_message(tmp_path, monkeypatch, capsys): + path = tmp_path / "products.yaml" + path.mkdir() # a directory where the file should be: read_text raises OSError + _point_config(monkeypatch, path) + with pytest.raises(SystemExit) as exc: + answers._load_runtime_config() + assert exc.value.code == 1 + assert f"cannot read {path}" in capsys.readouterr().err + + def test_non_mapping_config_exits(tmp_path, monkeypatch, capsys): path = tmp_path / "products.yaml" path.write_text("- a\n- b\n", encoding="utf-8") @@ -132,3 +152,95 @@ def save_answers(self, answers_, path): assert calls["generate"]["output_path"] == out assert calls["saved"] == (["a1"], out) assert f"Saved answers to {out}" in capsys.readouterr().out + + +def _stub_generate(monkeypatch, tmp_path, answerer_cls): + _point_config(monkeypatch, tmp_path / "absent.yaml") + monkeypatch.setattr(dokimi.questions.artifact, "load_questions", + lambda path: SimpleNamespace(questions=["q1"])) + monkeypatch.setattr(dokimi.mcp.factory, "create_doc_source_client", lambda source: "client") + monkeypatch.setattr(dokimi.mcp.factory, "resolve_library_id_for_product", + lambda product, client, explicit_id=None: "/lib/id") + monkeypatch.setattr(dokimi.eval, "Answerer", answerer_cls) + + +def test_generate_output_that_cannot_be_written_exits_2_naming_the_path(tmp_path, monkeypatch, capsys): + blocker = tmp_path / "blocker" + blocker.write_text("x", encoding="utf-8") # a regular file: nothing can be created below it + target = blocker / "a.json" + + class FakeAnswerer: + def __init__(self, **kwargs): + pass + + def generate_answers(self, output_path, **kwargs): + output_path.parent.mkdir(parents=True, exist_ok=True) + return ["a1"] + + _stub_generate(monkeypatch, tmp_path, FakeAnswerer) + with pytest.raises(SystemExit) as exc: + answers.cmd_answers_generate(_args(output=str(target))) + assert exc.value.code == 2 + assert f"Error: cannot write {target}: " in capsys.readouterr().err + + +def test_generate_final_save_that_fails_exits_2_naming_the_path(tmp_path, monkeypatch, capsys): + target = tmp_path / "a.json" + + class FakeAnswerer: + def __init__(self, **kwargs): + pass + + def generate_answers(self, **kwargs): + return ["a1"] + + def save_answers(self, answers_, path): + raise PermissionError(13, "Permission denied", f"{path}.123.0.tmp") + + _stub_generate(monkeypatch, tmp_path, FakeAnswerer) + with pytest.raises(SystemExit) as exc: + answers.cmd_answers_generate(_args(output=str(target))) + assert exc.value.code == 2 + assert f"Error: cannot write {target}: Permission denied" in capsys.readouterr().err + + +def test_generate_output_that_is_a_directory_exits_2_naming_the_path(tmp_path, monkeypatch, capsys): + from dokimi.eval import Answerer as RealAnswerer + + target = tmp_path / "outdir" + target.mkdir() + + class FakeAnswerer: + def __init__(self, **kwargs): + pass + + def generate_answers(self, **kwargs): + return [{"question_id": "q1"}] + + model = "m" + provider = "p" + + def _get_timestamp(self): + return "t" + + _build_output = RealAnswerer._build_output + + save_answers = RealAnswerer.save_answers + + _stub_generate(monkeypatch, tmp_path, FakeAnswerer) + with pytest.raises(SystemExit) as exc: + answers.cmd_answers_generate(_args(output=str(target))) + assert exc.value.code == 2 + assert f"Error: cannot write {target}: " in capsys.readouterr().err + + +@pytest.mark.parametrize("kind", ["missing", "directory"]) +def test_generate_unreadable_questions_exits_1_naming_the_path(tmp_path, monkeypatch, capsys, kind): + _point_config(monkeypatch, tmp_path / "absent.yaml") + path = tmp_path / "q.json" + if kind == "directory": + path.mkdir() + with pytest.raises(SystemExit) as exc: + answers.cmd_answers_generate(_args(questions=str(path))) + assert exc.value.code == 1 + assert f"Error: cannot read {path}" in capsys.readouterr().err diff --git a/dokimi/tests/test_benchmark_command_question_set.py b/dokimi/tests/test_benchmark_command_question_set.py index a02b4548f..70cc65579 100644 --- a/dokimi/tests/test_benchmark_command_question_set.py +++ b/dokimi/tests/test_benchmark_command_question_set.py @@ -369,6 +369,39 @@ def test_preflight_rejects_an_unwritable_output_dir(monkeypatch, tmp_path, capsy assert "output dir not writable" in capsys.readouterr().err +@pytest.mark.parametrize( + "make_source, message", + [ + (lambda p: p / "nowhere", "questions source not found"), + (lambda p: p / "bad.json", "questions source invalid"), + (lambda p: p / "adir", "questions source invalid"), + ], + ids=["missing", "invalid", "directory"], +) +@pytest.mark.parametrize("multi_run", [1, 3]) +def test_run_rejects_an_unusable_questions_from_before_any_call( + monkeypatch, tmp_path, capsys, make_source, message, multi_run +): + """benchmark run exits 1 with a message, not a traceback, and starts no pipeline.""" + calls: list[object] = [] + (tmp_path / "bad.json").write_text(json.dumps({"a": 1}), encoding="utf-8") + (tmp_path / "adir" / "questions" / "onetbb.json").mkdir(parents=True) + monkeypatch.setattr(benchmark, "_load_registry", lambda args: _Registry()) + monkeypatch.setattr(benchmark, "_run_single_library", lambda *a, **k: calls.append(k)) + + args = build_parser().parse_args([ + "benchmark", "run", "--library", "onetbb", "--multi-run", str(multi_run), + "--output-dir", str(tmp_path / "out"), + "--questions-from", str(make_source(tmp_path)), + ]) + with pytest.raises(SystemExit) as exc: + benchmark.cmd_benchmark_run(args) + + assert exc.value.code == 1 + assert message in capsys.readouterr().err + assert calls == [] + + @pytest.mark.outcome("pinned-questions-reuse") def test_multi_run_forwards_questions_root_to_every_run(monkeypatch, tmp_path): """Every run in a multi-run forwards the same --questions-root.""" diff --git a/dokimi/tests/test_eval_grounding_output_errors.py b/dokimi/tests/test_eval_grounding_output_errors.py new file mode 100644 index 000000000..2b00baf58 --- /dev/null +++ b/dokimi/tests/test_eval_grounding_output_errors.py @@ -0,0 +1,75 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""`eval grounding` and `eval judge-validity` exit 2 naming the path when --output cannot be written.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from dokimi.cli import build_parser + + +def _grounding(tmp_path: Path, output: Path) -> None: + answers = tmp_path / "answers.json" + answers.write_text(json.dumps({"answers": [ + {"question_id": "q1", "question_text": "How do I loop?", + "with_docs": {"answer": "use parallel_for", + "retrieved_docs": [{"snippet": "parallel_for runs tasks"}]}}, + ]}), encoding="utf-8") + args = build_parser().parse_args(["eval", "grounding", "--answers", str(answers), "--output", str(output)]) + args.func(args) + + +def test_grounding_report_is_written(tmp_path): + out = tmp_path / "nested" / "grounding.json" + _grounding(tmp_path, out) + assert json.loads(out.read_text(encoding="utf-8"))["schema_version"] == "grounding.v1" + + +def test_output_below_a_file_exits_2_naming_the_path(tmp_path, capsys): + blocker = tmp_path / "blocker" + blocker.write_text("x", encoding="utf-8") # a regular file: nothing can be created below it + target = blocker / "grounding.json" + with pytest.raises(SystemExit) as exc: + _grounding(tmp_path, target) + assert exc.value.code == 2 + err = capsys.readouterr().err + assert f"Error: cannot write {target}" in err + assert "Traceback" not in err + + +def test_judge_validity_output_below_a_file_exits_2_naming_the_path(tmp_path, monkeypatch, capsys): + import dokimi.eval.judge as judge_mod + import dokimi.eval.judge_validity as jv + + answer = ("Use perf stat to collect IPC, then narrow the hotspot with VTune. " + "Enable AVX-512 in the build and expect roughly 1.5x on 8 threads. ") * 12 + answers = tmp_path / "answers.json" + answers.write_text(json.dumps([ + {"question_id": f"Q{i}", "question_text": "q", "arms": {"a": {"answer": f"{i}. " + answer}}} + for i in range(20) + ]), encoding="utf-8") + monkeypatch.setattr(judge_mod, "Judge", lambda **kw: None) + monkeypatch.setattr(jv, "judge_scorer", lambda judge: lambda lib, q, a: float(len(a))) + blocker = tmp_path / "blocker" + blocker.write_text("x", encoding="utf-8") + target = blocker / "validity.json" + args = build_parser().parse_args(["eval", "judge-validity", "--answers", str(answers), "--bases", "20", + "--variants", "truncate_20", "--output", str(target)]) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 + assert f"Error: cannot write {target}" in capsys.readouterr().err + + +def test_output_that_is_a_directory_exits_2_naming_the_path(tmp_path, capsys): + target = tmp_path / "is-a-dir" + target.mkdir() + with pytest.raises(SystemExit) as exc: + _grounding(tmp_path, target) + assert exc.value.code == 2 + assert f"Error: cannot write {target}" in capsys.readouterr().err diff --git a/dokimi/tests/test_eval_output_write_errors.py b/dokimi/tests/test_eval_output_write_errors.py new file mode 100644 index 000000000..07f145cd5 --- /dev/null +++ b/dokimi/tests/test_eval_output_write_errors.py @@ -0,0 +1,256 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""eval writers and report eval: an unwritable output exits with a message naming the failing path.""" + +from __future__ import annotations + +import json + +import pytest + +from dokimi.cli import build_parser +from dokimi.commands import judge_reliability + + +def _blocked(tmp_path): + blocker = tmp_path / "blocker" + blocker.write_text("x", encoding="utf-8") # a regular file: nothing can be created below it + return blocker / "out.json" + + +def test_write_text_or_exit_names_the_path_and_exits_with_the_code(tmp_path, capsys): + from dokimi.utils import write_text_or_exit + + target = _blocked(tmp_path) + with pytest.raises(SystemExit) as exc: + write_text_or_exit(target, "{}", code=2) + assert exc.value.code == 2 + assert f"Error: cannot write {target}: " in capsys.readouterr().err + + +class _Report: + verdicts = [object()] + errors: list = [] + measures = [object()] + + def __init__(self, *a, **k): + pass + + def to_dict(self): + return {} + + def format_markdown(self): + return "md" + + def apply_validity(self, validity): + return [] + + def summary(self): + return {"arms": {}, "measured": True, "fragile_variants": []} + + def format_summary(self): + return "" + + +def _stub_judge_commands(monkeypatch): + """Replace every judge-facing dependency of the four judge_reliability writers with a stub.""" + import types + + import dokimi.eval.checklist as checklist + import dokimi.eval.cross_judge as cross_judge + import dokimi.eval.judge as judge + import dokimi.eval.judge_validity as judge_validity + import dokimi.eval.judge_wording as judge_wording + import dokimi.eval.pairwise as pairwise + + monkeypatch.setattr(judge_reliability, "_load_answers", lambda path: (None, [])) + monkeypatch.setattr(judge_reliability, "_load_points", lambda path: {}) + monkeypatch.setattr(judge_reliability, "_require_arms", lambda *a, **k: None) + monkeypatch.setattr(judge_reliability, "_check_judge_temperature", lambda *a, **k: None) + monkeypatch.setattr(judge_reliability, "pairwise_judge", lambda args: (None, "label")) + monkeypatch.setattr(pairwise, "pairwise_jobs", lambda *a, **k: None) + monkeypatch.setattr(pairwise, "run_arm_pairwise", lambda *a, **k: _Report()) + monkeypatch.setattr(checklist, "checklist_jobs", lambda *a, **k: None) + monkeypatch.setattr(checklist, "run_checklist", lambda *a, **k: _Report()) + monkeypatch.setattr(judge, "Judge", lambda **kw: types.SimpleNamespace(llm=None)) + monkeypatch.setattr(judge_validity, "_build", lambda *a, **k: types.SimpleNamespace(pairs=[])) + monkeypatch.setattr(judge_wording, "run_wording", lambda *a, **k: _Report()) + monkeypatch.setattr(cross_judge, "CrossJudgeReport", _Report) + monkeypatch.setattr(cross_judge, "scored_answers", lambda *a, **k: None) + monkeypatch.setattr(cross_judge, "score_measure", lambda *a, **k: object()) + + +_JUDGE_COMMANDS = { + "pairwise": ["eval", "pairwise"], + "checklist": ["eval", "checklist", "--questions", "q.json"], + "judge-wording": ["eval", "judge-wording"], + "cross-judge": ["eval", "cross-judge", "--judges", "openai/m", "--no-stored"], +} +_WRITES_MD = ["pairwise", "checklist", "cross-judge"] + + +def _run(command, out): + args = build_parser().parse_args( + [*_JUDGE_COMMANDS[command], "--answers", "a.json", "--output", str(out)] + ) + with pytest.raises(SystemExit) as exc: + args.func(args) + return exc.value.code + + +@pytest.mark.parametrize("command", list(_JUDGE_COMMANDS)) +def test_judge_reliability_json_write_failure_exits_2_naming_the_path( + command, tmp_path, monkeypatch, capsys +): + _stub_judge_commands(monkeypatch) + out = _blocked(tmp_path) + assert _run(command, out) == 2 + assert f"Error: cannot write {out}" in capsys.readouterr().err + + +@pytest.mark.parametrize("command", _WRITES_MD) +def test_judge_reliability_md_sidecar_write_failure_exits_2_naming_the_path( + command, tmp_path, monkeypatch, capsys +): + _stub_judge_commands(monkeypatch) + out = tmp_path / "out.json" + sidecar = out.with_suffix(".md") + sidecar.mkdir() + assert _run(command, out) == 2 + assert f"Error: cannot write {sidecar}" in capsys.readouterr().err + + +def test_ragas_results_that_cannot_be_written_exit_2_naming_the_path(tmp_path, monkeypatch, capsys): + import types + + import dokimi.eval.ragas_eval as ragas_eval + from dokimi.commands.evaluate import _run_ragas_eval + + class Evaluator: + def __init__(self, **kw): + pass + + def evaluate(self, answers, include_without_docs=True): + return types.SimpleNamespace(to_dict=dict, format_summary=lambda: "") + + monkeypatch.setattr(ragas_eval, "RagasEvaluator", Evaluator) + blocked = _blocked(tmp_path) + with pytest.raises(SystemExit) as exc: + _run_ragas_eval([], blocked, types.SimpleNamespace()) + assert exc.value.code == 2 + assert f"Error: cannot write {blocked.with_name('out_ragas.json')}" in capsys.readouterr().err + + +def test_panel_score_output_that_cannot_be_written_exits_2_naming_the_path( + tmp_path, monkeypatch, capsys +): + import types + + import dokimi.eval.panel as panel_mod + from dokimi.commands import evaluate + + target = tmp_path / "p.json" + + class Panel: + def __init__(self, **kw): + pass + + def evaluate_answers(self, *a, **k): + raise PermissionError(13, "denied", f"{target}.123.0.tmp") + + monkeypatch.setattr(panel_mod, "JudgePanel", Panel) + monkeypatch.setattr( + evaluate, "_load_answers", lambda path: types.SimpleNamespace(records=[], meta={}) + ) + args = build_parser().parse_args( + ["eval", "panel-score", "--product", "P", "--answers", "a.json", "--output", str(target)] + ) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 + assert f"Error: cannot write {target}: " in capsys.readouterr().err + + +def test_score_output_that_cannot_be_written_exits_2_naming_the_path(tmp_path, monkeypatch, capsys): + class Judge: + def evaluate_answers(self, *a, **k): + return [] + + def save_evaluations(self, evals, path): + raise PermissionError(13, "Permission denied", f"{path}.123.0.tmp") + + answers = tmp_path / "answers.json" + answers.write_text(json.dumps({"answers": []}), encoding="utf-8") + import dokimi.eval as eval_mod + + monkeypatch.setattr(eval_mod, "Judge", lambda **kw: Judge()) + target = tmp_path / "e.json" + args = build_parser().parse_args( + ["eval", "score", "--product", "P", "--answers", str(answers), "--output", str(target)] + ) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 + assert f"Error: cannot write {target}: " in capsys.readouterr().err + + +def test_report_eval_names_the_sidecar_when_it_cannot_be_written(tmp_path, monkeypatch, capsys): + from dokimi import cli + + eval_file, out = tmp_path / "eval.json", tmp_path / "r.md" + eval_file.write_text( + json.dumps( + { + "evaluations": [ + { + "question_id": "q1", + "question_text": "Q?", + "with_docs": {"aggregate": 80}, + "without_docs": {"aggregate": 60}, + "delta": 20, + } + ] + } + ), + encoding="utf-8", + ) + (tmp_path / "r.qa.json").mkdir() + monkeypatch.setattr( + "sys.argv", + ["dokimi", "report", "eval", "--product", "P", "--eval", str(eval_file), "--out", str(out)], + ) + with pytest.raises(SystemExit) as exc: + cli.main() + assert exc.value.code == 1 + err = capsys.readouterr().err + assert str(tmp_path / "r.qa.json") in err and f"cannot write {out}" not in err + + +def test_report_eval_sidecar_does_not_overwrite_an_out_without_md(tmp_path, monkeypatch): + from dokimi import cli + + eval_file, out = tmp_path / "eval.json", tmp_path / "report" + eval_file.write_text( + json.dumps( + { + "evaluations": [ + { + "question_id": "q1", + "question_text": "Q?", + "with_docs": {"aggregate": 80}, + "without_docs": {"aggregate": 60}, + "delta": 20, + } + ] + } + ), + encoding="utf-8", + ) + monkeypatch.setattr( + "sys.argv", + ["dokimi", "report", "eval", "--product", "P", "--eval", str(eval_file), "--out", str(out)], + ) + cli.main() + assert not out.read_text(encoding="utf-8").lstrip().startswith("{") + assert (tmp_path / "report.qa.json").exists() diff --git a/dokimi/tests/test_eval_questions_file_errors.py b/dokimi/tests/test_eval_questions_file_errors.py new file mode 100644 index 000000000..808295c25 --- /dev/null +++ b/dokimi/tests/test_eval_questions_file_errors.py @@ -0,0 +1,41 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""eval checklist / cross-judge --grounded: an unreadable --questions file is bad input (exit 2).""" + +from __future__ import annotations + +import json + +import pytest + + +def _arms_file(tmp_path): + path = tmp_path / "arms.json" + path.write_text(json.dumps({"answers": [{"question_id": "q1", "arms": {"skill": {"answer": "s"}}}]}), + encoding="utf-8") + return path + + +@pytest.mark.parametrize("command", [ + ["checklist"], + ["cross-judge", "--grounded", "--no-stored"], +], ids=["checklist", "cross-judge-grounded"]) +@pytest.mark.parametrize("content", [None, "{not json", "\xff\xfe"], + ids=["missing", "malformed", "not-utf8"]) +def test_an_unreadable_questions_file_exits_2_with_a_message(tmp_path, monkeypatch, capsys, command, content): + from dokimi import cli + + questions = tmp_path / "questions.json" + if content == "\xff\xfe": + questions.write_bytes(b"\xff\xfe\x00") + elif content is not None: + questions.write_text(content, encoding="utf-8") + monkeypatch.setattr("sys.argv", ["dokimi", "eval", *command, "--answers", str(_arms_file(tmp_path)), + "--questions", str(questions), "--output", str(tmp_path / "out.json")]) + with pytest.raises(SystemExit) as exc: + cli.main() + assert exc.value.code == 2 # bad input; 1 is the verdict code of the gates + err = capsys.readouterr().err + assert err.startswith("Error: ") and str(questions) in err + assert not (tmp_path / "out.json").exists() diff --git a/dokimi/tests/test_evaluate_cli_inputs.py b/dokimi/tests/test_evaluate_cli_inputs.py new file mode 100644 index 000000000..ef9902c50 --- /dev/null +++ b/dokimi/tests/test_evaluate_cli_inputs.py @@ -0,0 +1,36 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""`evaluate`: a bad --custom-questions path or --output-dir exits 1 with a message.""" + +from __future__ import annotations + +import pytest + +from dokimi.cli import build_parser + + +def _evaluate(*extra: str): + args = build_parser().parse_args(["evaluate", "--product", "p", "--description", "d", *extra]) + args.func(args) + + +def test_a_missing_custom_questions_file_exits_1_with_a_message(tmp_path, capsys): + missing = tmp_path / "missing.json" + with pytest.raises(SystemExit) as exc: + _evaluate("--output-dir", str(tmp_path / "out"), "--custom-questions", str(missing)) + assert exc.value.code == 1 + err = capsys.readouterr().err + assert f"Error: --custom-questions path does not exist: {missing}" in err + assert "Traceback" not in err + + +def test_an_unwritable_output_dir_exits_1_with_a_message(tmp_path, capsys): + blocker = tmp_path / "file" + blocker.write_text("x", encoding="utf-8") + with pytest.raises(SystemExit) as exc: + _evaluate("--output-dir", str(blocker)) + assert exc.value.code == 1 + err = capsys.readouterr().err + assert err.startswith("Error: ") and str(blocker) in err + assert "Traceback" not in err diff --git a/dokimi/tests/test_model_compare.py b/dokimi/tests/test_model_compare.py index 1f66ca9cc..9a776a424 100644 --- a/dokimi/tests/test_model_compare.py +++ b/dokimi/tests/test_model_compare.py @@ -83,6 +83,18 @@ def test_load_run_invalid_json(tmp_path): load_run(str(p)) +def test_load_run_non_utf8_file_names_the_file(tmp_path): + p = tmp_path / "latin1.json" + p.write_bytes(b'{"note": "caf\xe9"}') + with pytest.raises(SystemExit, match=r"Error: Invalid JSON in .*latin1\.json: .*utf-8"): + load_run(str(p)) + + +def test_load_run_directory_is_an_error_message_not_a_traceback(tmp_path): + with pytest.raises(SystemExit, match=r"Error: cannot read .*Is a directory"): + load_run(str(tmp_path)) + + # --------------------------------------------------------------------------- # check_run_consistency # --------------------------------------------------------------------------- diff --git a/dokimi/tests/test_report_cli_inputs.py b/dokimi/tests/test_report_cli_inputs.py index 0fad80339..170aab645 100644 --- a/dokimi/tests/test_report_cli_inputs.py +++ b/dokimi/tests/test_report_cli_inputs.py @@ -5,6 +5,8 @@ from __future__ import annotations +import json + import pytest from dokimi.cli import build_parser @@ -56,8 +58,86 @@ def test_eval_json_that_is_not_an_object_is_rejected(tmp_path, capsys, text): assert f"Error: {bad} must hold a JSON object" in _err(tmp_path, bad, capsys) +@pytest.mark.parametrize("target", ["is_a_directory", "parent_is_a_file"]) +def test_an_unwritable_output_exits_1_naming_the_path(tmp_path, capsys, monkeypatch, target): + monkeypatch.setattr("dokimi.report.ReportGenerator.generate_report", lambda self, **kw: "report") + ev = tmp_path / "ev.json" + ev.write_text('{"evaluations": []}', encoding="utf-8") + questions = tmp_path / "questions.json" + questions.write_text('{"questions": []}', encoding="utf-8") + if target == "is_a_directory": + out = tmp_path / "out.md" + out.mkdir() + else: + (tmp_path / "blocker").write_text("x", encoding="utf-8") + out = tmp_path / "blocker" / "out.md" + args = build_parser().parse_args([ + "report", "generate", "--product", "p", "--eval", str(ev), + "--questions", str(questions), "--output", str(out), + ]) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 1 + assert f"Error: cannot write {out}" in capsys.readouterr().err + + @pytest.mark.parametrize("evaluations", ["null", "5"]) def test_non_list_evaluations_exit_1_with_its_path(tmp_path, capsys, evaluations): bad = tmp_path / "evd.json" bad.write_text(f'{{"evaluations": {evaluations}}}', encoding="utf-8") assert f"{bad} evaluations: expected a list of records" in _err(tmp_path, bad, capsys) + + +@pytest.mark.parametrize("kind", ["missing", "directory"]) +def test_generate_unreadable_questions_exits_1_naming_the_path(tmp_path, capsys, kind): + ev = tmp_path / "ev.json" + ev.write_text('{"evaluations": []}', encoding="utf-8") + qs = tmp_path / "nope.json" + if kind == "directory": + qs.mkdir() + args = build_parser().parse_args([ + "report", "generate", "--product", "p", "--eval", str(ev), + "--questions", str(qs), "--output", str(tmp_path / "r.md"), + ]) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 1 + err = capsys.readouterr().err + assert err.startswith("Error: cannot read ") and str(qs) in err + assert "Traceback" not in err + + +def test_report_eval_unwritable_out_exits_1_naming_the_path(tmp_path, capsys): + ev = tmp_path / "ev.json" + ev.write_text('{"evaluations": []}', encoding="utf-8") + blocker = tmp_path / "blocker" + blocker.write_text("x", encoding="utf-8") + out = blocker / "r.md" + args = build_parser().parse_args([ + "report", "eval", "--product", "P", "--eval", str(ev), "--out", str(out), + ]) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 1 + err = capsys.readouterr().err + assert err.startswith("Error: cannot write ") and str(out) in err + + +def test_model_compare_unwritable_out_exits_1_naming_the_path(tmp_path, capsys): + run = tmp_path / "run.json" + run.write_text(json.dumps({"baseline_arm": "baseline", "evaluations": [ + {"question_id": f"q{i}", "question_text": "q", "category": "", "difficulty": "easy", + "persona": "", "scores": {"baseline": {"aggregate": 50.0}, "docs": {"aggregate": 60.0}}} + for i in range(3) + ]}), encoding="utf-8") + blocker = tmp_path / "blocker" + blocker.write_text("x", encoding="utf-8") + out = blocker / "x.md" + args = build_parser().parse_args([ + "report", "model-compare", "--regular-runs", str(run), "--run-ids", "a", "--out", str(out), + ]) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 1 + err = capsys.readouterr().err + assert err.startswith("Error: cannot write ") and str(out) in err diff --git a/dokimi/tests/test_report_eval_input_errors.py b/dokimi/tests/test_report_eval_input_errors.py new file mode 100644 index 000000000..0368742a8 --- /dev/null +++ b/dokimi/tests/test_report_eval_input_errors.py @@ -0,0 +1,50 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""report eval: an unreadable --eval file is bad input (exit 1 with a message), not a traceback.""" + +from __future__ import annotations + +import json + +import pytest + + +@pytest.mark.parametrize("content, reason", [ + (None, "cannot read"), + (b"{not json", "is not valid UTF-8 JSON"), + (b"\xff\xfe\x00", "is not valid UTF-8 JSON"), + (b"[1, 2]", "`evaluations` list"), + (b"{}", "`evaluations` list"), + (json.dumps({"evaluations": {}}).encode(), "`evaluations` list"), +], ids=["missing", "malformed", "not-utf8", "not-an-object", "no-evaluations", + "evaluations-not-a-list"]) +def test_report_eval_exits_1_with_a_message_on_an_unreadable_eval_file(tmp_path, monkeypatch, capsys, + content, reason): + from dokimi import cli + + eval_file, out = tmp_path / "eval.json", tmp_path / "r.md" + if content is not None: + eval_file.write_bytes(content) + monkeypatch.setattr("sys.argv", ["dokimi", "report", "eval", "--product", "P", "--eval", str(eval_file), + "--out", str(out)]) + with pytest.raises(SystemExit) as exc: + cli.main() + assert exc.value.code == 1 + err = capsys.readouterr().err + assert err.startswith("Error: ") and str(eval_file) in err and reason in err + assert not out.exists() + + +def test_report_eval_exits_1_with_a_message_when_the_eval_path_is_a_directory(tmp_path, monkeypatch, capsys): + from dokimi import cli + + out = tmp_path / "r.md" + monkeypatch.setattr("sys.argv", ["dokimi", "report", "eval", "--product", "P", "--eval", str(tmp_path), + "--out", str(out)]) + with pytest.raises(SystemExit) as exc: + cli.main() + assert exc.value.code == 1 + err = capsys.readouterr().err + assert err.startswith("Error: cannot read ") and str(tmp_path) in err + assert not out.exists()