diff --git a/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-tools-catalog.md b/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-tools-catalog.md new file mode 100644 index 000000000..391afc34f --- /dev/null +++ b/dokimi/changelog.d/20261025_120000_vchamarthi_cli-input-tools-catalog.md @@ -0,0 +1,3 @@ +### Fixed + +- **`tools preference` and `catalog`** exit with a message on a non-UTF-8 `--groups` file, an unwritable `--out`, `--items-out` or `--out-json`, an unreadable or wrong-shaped manifest, a malformed `SKILL.md` under `--catalog`, or a task directory without `instruction.md`, instead of a traceback. diff --git a/dokimi/dokimi/commands/catalog.py b/dokimi/dokimi/commands/catalog.py index 06ac62262..f96726609 100644 --- a/dokimi/dokimi/commands/catalog.py +++ b/dokimi/dokimi/commands/catalog.py @@ -102,7 +102,7 @@ def _scan_tasks(args: argparse.Namespace, root: Path, report: dict, as_json: boo try: tasks = read_task_manifest(root, tasks_subdir=args.tasks_subdir) except ValueError as exc: - raise SystemExit(f"suites.json: {exc}") from exc + raise SystemExit(str(exc)) from exc if not as_json: print(f"# tasks ({len(tasks)} declared under {root})") failed = False @@ -136,16 +136,47 @@ def cmd_catalog_scan(args: argparse.Namespace) -> None: raise SystemExit(1) -def cmd_catalog_trigger_set(args: argparse.Namespace) -> None: +def _write_trigger_set(args: argparse.Namespace, queries, catalog_root: Path) -> None: + """Write the query set to ``--out`` in ``--format``, or exit 1 naming ``--out`` when it cannot be written.""" import json + from dokimi.artifacts import ArtifactValidationError from dokimi.external import trigger_set as ts + + try: + if args.format == "questions": + from dokimi.questions.artifact import save_questions_artifact + + save_questions_artifact(ts.to_questions(queries), args.out) + else: + # External tools' formats: no schema of ours to hold them to. + payload = (ts.to_reach(queries, catalog_id=str(catalog_root)) if args.format == "reach" + else ts.to_skill_creator(queries, args.target)) + Path(args.out).write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + except ArtifactValidationError as exc: # the path was writable; the payload failed the schema + raise SystemExit(f"Error: questions set rejected: {exc}") from exc + except (OSError, ValueError) as exc: # unwritable --out: a directory, a file as parent, no permission, empty path (pathlib ValueError) + raise SystemExit(f"Error: cannot write --out {args.out}: {exc}") from exc + + +def _discover_catalog_skills(catalog_root: Path) -> list: from dokimi.skills import discover_skills + try: + return discover_skills(catalog_root) + except (OSError, ValueError, RecursionError) as exc: # unreadable, non-UTF-8, bad frontmatter or nesting too deep + raise SystemExit(f"--catalog {catalog_root}: {exc}") from exc + + +def cmd_catalog_trigger_set(args: argparse.Namespace) -> None: + import json + + from dokimi.external import trigger_set as ts + catalog_root = Path(args.catalog) if not catalog_root.is_dir(): raise SystemExit(f"--catalog is not a directory: {catalog_root}") - catalog = discover_skills(catalog_root) + catalog = _discover_catalog_skills(catalog_root) # Checked before any --rewrite call, so a bad invocation costs no LLM calls. if args.format == "skill-creator": if not args.target: @@ -191,15 +222,7 @@ def rewrite(text: str) -> str: # cannot pick up a set with a mistyped label. print(json.dumps(summary, indent=2)) raise SystemExit(1) - if args.format == "questions": - from dokimi.questions.artifact import save_questions_artifact - - save_questions_artifact(ts.to_questions(queries), args.out) - else: - # External tools' formats: no schema of ours to hold them to. - payload = (ts.to_reach(queries, catalog_id=str(catalog_root)) if args.format == "reach" - else ts.to_skill_creator(queries, args.target)) - Path(args.out).write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + _write_trigger_set(args, queries, catalog_root) print(json.dumps(summary, indent=2)) multi = sum(len(q.acceptable) > 1 for q in ts.labelled(queries)) diff --git a/dokimi/dokimi/commands/tools.py b/dokimi/dokimi/commands/tools.py index e0f19464f..5908fab7a 100644 --- a/dokimi/dokimi/commands/tools.py +++ b/dokimi/dokimi/commands/tools.py @@ -6,9 +6,12 @@ from __future__ import annotations import argparse +import errno import json import math +import os import sys +import threading import time from pathlib import Path @@ -91,13 +94,54 @@ def _print_skipped(skipped: dict, n_groups: int) -> None: print(f" {factor}: skipped for {len(ids)} of {n_groups} groups ({', '.join(ids)})") +def _write_items(path: str, items: list) -> None: + """Write the probe items to ``path``, or exit 1 naming it when it cannot be written.""" + try: + Path(path).write_text(json.dumps(items, indent=1), encoding="utf-8") + except OSError as exc: + print(f"Error: cannot write --items-out {path}: {exc}", file=sys.stderr) + raise SystemExit(1) from None + + +def _save_result(data: dict, path: str) -> None: + """Save the tool_preference artifact to ``path``, or exit 1 naming it when it cannot be written.""" + from dokimi.artifacts import ArtifactValidationError, save_artifact + + try: + save_artifact("tool_preference", data, path) + except ArtifactValidationError as exc: # the path was writable; the payload failed the schema + print(f"Error: tool_preference artifact rejected: {exc}", file=sys.stderr) + raise SystemExit(1) from None + except (OSError, ValueError) as exc: # unwritable path, including an empty one (pathlib ValueError) + print(f"Error: cannot write {path}: {exc}", file=sys.stderr) + raise SystemExit(1) from None + + +def _check_out_json_writable(path: str) -> None: + """Exit 2 before any probe call when a save to ``path`` would fail, probed with a temp file that is removed.""" + from dokimi.artifacts import _write_target + + target = Path(path) + try: + target.parent.mkdir(parents=True, exist_ok=True) + resolved = _write_target(target) + if resolved.is_dir(): + raise IsADirectoryError(errno.EISDIR, "Is a directory", path) + tmp = resolved.with_name(f"{resolved.name}.{os.getpid()}.{threading.get_ident()}.tmp") # the save's name + with tmp.open("xb"): + pass + tmp.unlink() + except (OSError, ValueError) as exc: + print(f"Error: cannot write {path}: {exc}", file=sys.stderr) + raise SystemExit(2) from None + + def cmd_tools_preference(args: argparse.Namespace) -> None: - from dokimi.artifacts import save_artifact from dokimi import tool_preference as tp try: doc = tp.load_groups(json.loads(Path(args.groups).read_text(encoding="utf-8"))) - except (OSError, json.JSONDecodeError, tp.GroupsError) as exc: + except (OSError, RecursionError, ValueError, tp.GroupsError) as exc: raise SystemExit(f"--groups {args.groups}: {exc}") from exc factors = list(dict.fromkeys(f for f in args.factors.split(",") if f)) if args.factors else list(tp.FACTORS) unknown = sorted(set(factors) - set(tp.FACTORS)) @@ -110,12 +154,13 @@ def cmd_tools_preference(args: argparse.Namespace) -> None: + (f", stopping at ${args.max_cost_usd:g}" if args.max_cost_usd else "")) _print_skipped(tp.skipped_factors(doc, factors), len(doc["groups"])) if args.items_out: - Path(args.items_out).write_text(json.dumps(items, indent=1), encoding="utf-8") + _write_items(args.items_out, items) print(f"items -> {args.items_out}") if args.dry_run: return if not args.model: raise SystemExit("--model is required unless --dry-run") + _check_out_json_writable(args.out_json) def progress(done: int, total: int) -> None: if done % 100 == 0 or done == total: @@ -127,7 +172,7 @@ def progress(done: int, total: int) -> None: data = tp.payload(doc, items, results, model=args.model, provider=args.provider, reps=args.reps, factors=factors, max_tokens=args.max_tokens, max_cost_usd=args.max_cost_usd) - save_artifact("tool_preference", data, args.out_json) + _save_result(data, args.out_json) _print_summary(data["summary"]) u = data["usage"] print(f"cost: {u['cost_usd']} USD over {u['n_calls']} calls ({u['cost_known_calls']} priced)") diff --git a/dokimi/dokimi/external/catalog.py b/dokimi/dokimi/external/catalog.py index b52e88120..9a916b16f 100644 --- a/dokimi/dokimi/external/catalog.py +++ b/dokimi/dokimi/external/catalog.py @@ -107,7 +107,7 @@ def _scan_skill_files(root: Path) -> list[SkillCatalogEntry]: continue try: skill = load_skill(skill_file) - except (OSError, ValueError) as exc: + except (OSError, ValueError, RecursionError) as exc: found.append( SkillCatalogEntry( name=skill_file.parent.name, @@ -141,7 +141,9 @@ def _manifest_items(manifest_path: Path) -> tuple[list, bool]: try: data = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) or {} - except yaml.YAMLError as exc: + except OSError as exc: + raise ValueError(f"cannot read skills.yaml at {manifest_path}: {exc}") from exc + except (yaml.YAMLError, UnicodeDecodeError, RecursionError) as exc: raise ValueError(f"invalid skills.yaml at {manifest_path}: {exc}") from exc if isinstance(data, dict): @@ -219,7 +221,7 @@ def failed(error: str, path: Optional[Path] = None) -> SkillCatalogEntry: return failed(f"{source} escapes root: {catalog_path!r}") try: skill = load_skill(path) - except (OSError, ValueError) as exc: + except (OSError, ValueError, RecursionError) as exc: return failed(str(exc), path) # The directory is inside the root; a SKILL.md symlinked out of it is not. if not _within(skill.path, root): @@ -228,6 +230,35 @@ def failed(error: str, path: Optional[Path] = None) -> SkillCatalogEntry: source_root=root, skill=skill) +def _manifest_tasks(data: object, manifest_path: Path) -> list[dict]: + """Return the task objects of every suite in a parsed suites.json, or raise ValueError naming the file.""" + + def invalid(why: str) -> ValueError: + return ValueError(f"invalid suites.json at {manifest_path}: {why}") + + if not isinstance(data, dict): + raise invalid(f"top level must be an object, got {type(data).__name__}") + suites = data.get("suites") + suites = [] if suites is None else suites + if not isinstance(suites, list) or not all(isinstance(s, dict) for s in suites): + raise invalid("suites must be a list of objects") + tasks: list[dict] = [] + for suite in suites: + suite_tasks = suite.get("tasks") + suite_tasks = [] if suite_tasks is None else suite_tasks + if not isinstance(suite_tasks, list) or not all(isinstance(t, dict) for t in suite_tasks): + raise invalid("each suite's tasks must be a list of objects") + tasks.extend(suite_tasks) + return tasks + + +def _discover_tasks_checked(tasks_root: Path) -> list: + try: + return discover_tasks(tasks_root) + except (OSError, ValueError) as exc: # e.g. a task directory without instruction.md + raise ValueError(f"cannot read tasks at {tasks_root}: {exc}") from exc + + def read_task_manifest( root: Path, tasks_subdir: str = DEFAULT_TASKS_SUBDIR, @@ -240,6 +271,7 @@ def read_task_manifest( manifest_path = root / DEFAULT_SUITES_MANIFEST tasks_root = root / tasks_subdir if not manifest_path.is_file(): + found = _discover_tasks_checked(tasks_root) return [ TaskCatalogEntry( name=task.name, @@ -248,88 +280,104 @@ def read_task_manifest( path=task.path, task=task, ) - for task in discover_tasks(tasks_root) + for task in found ] try: data = json.loads(manifest_path.read_text(encoding="utf-8")) - except json.JSONDecodeError as exc: + except OSError as exc: + raise ValueError(f"cannot read suites.json at {manifest_path}: {exc}") from exc + except (ValueError, RecursionError) as exc: # JSONDecodeError, non-UTF-8 bytes, or nesting too deep raise ValueError(f"invalid suites.json at {manifest_path}: {exc}") from exc entries: list[TaskCatalogEntry] = [] - for suite in data.get("suites") or []: - for item in suite.get("tasks") or []: - name = str(item.get("name") or "").strip() - status = item.get("status") - track = item.get("track") - task_dir = tasks_root / name - - if not _within(task_dir, tasks_root): - # A failure, not a skip: the manifest asked us to leave the - # checkout, which is the same thing ``read_skill_manifest`` - # reports as an error for ``catalog_path``. - entries.append( - TaskCatalogEntry( - name=name, - status=status, - track=track, - path=None, - failed=True, - reason=f"task name escapes tasks root: {name!r}", - ) + for item in _manifest_tasks(data, manifest_path): + name = str(item.get("name") or "").strip() + status = item.get("status") + track = item.get("track") + task_dir = tasks_root / name + + if not _within(task_dir, tasks_root): + # A failure, not a skip: the manifest asked us to leave the + # checkout, which is the same thing ``read_skill_manifest`` + # reports as an error for ``catalog_path``. + entries.append( + TaskCatalogEntry( + name=name, + status=status, + track=track, + path=None, + failed=True, + reason=f"task name escapes tasks root: {name!r}", ) - continue - - if not is_task_dir(task_dir): - entries.append( - TaskCatalogEntry( - name=name, - status=status, - track=track, - path=None, - skipped=True, - reason=f"no task.toml or task.md under {task_dir} (manifest status={status!r})", - ) + ) + continue + + try: + task_dir_found = is_task_dir(task_dir) + except OSError as exc: # e.g. a task name longer than the filesystem allows + entries.append( + TaskCatalogEntry( + name=name, + status=status, + track=track, + path=None, + failed=True, + reason=f"cannot inspect task directory {task_dir}: {exc}", ) - continue - - if track != "executable": - entries.append( - TaskCatalogEntry( - name=name, - status=status, - track=track, - path=task_dir, - skipped=True, - reason=f"track={track!r} is not an executable Harbor task", - ) + ) + continue + + if not task_dir_found: + entries.append( + TaskCatalogEntry( + name=name, + status=status, + track=track, + path=None, + skipped=True, + reason=f"no task.toml or task.md under {task_dir} (manifest status={status!r})", ) - continue - - try: - task = load_task(task_dir) - except (FileNotFoundError, ValueError) as exc: - # The entry is declared executable and its directory is on - # disk, so this is the case the scan exists to catch. - entries.append( - TaskCatalogEntry( - name=name, - status=status, - track=track, - path=task_dir, - failed=True, - reason=f"failed to load: {exc}", - ) + ) + continue + + if track != "executable": + entries.append( + TaskCatalogEntry( + name=name, + status=status, + track=track, + path=task_dir, + skipped=True, + reason=f"track={track!r} is not an executable Harbor task", ) - continue + ) + continue + try: + task = load_task(task_dir) + except (FileNotFoundError, ValueError) as exc: + # The entry is declared executable and its directory is on + # disk, so this is the case the scan exists to catch. entries.append( TaskCatalogEntry( name=name, status=status, track=track, path=task_dir, - task=task, + failed=True, + reason=f"failed to load: {exc}", ) ) + continue + + entries.append( + TaskCatalogEntry( + name=name, + status=status, + track=track, + path=task_dir, + task=task, + ) + ) return entries diff --git a/dokimi/dokimi/external/validate.py b/dokimi/dokimi/external/validate.py index 6c7cccf49..ff13c2c71 100644 --- a/dokimi/dokimi/external/validate.py +++ b/dokimi/dokimi/external/validate.py @@ -83,7 +83,7 @@ def _strict_meta(text: str) -> tuple[dict, Optional[Finding]]: """The frontmatter of ``text``, or a ``frontmatter-parse`` error carrying the YAML message.""" try: meta, _ = parse_frontmatter_strict(text) - except yaml.YAMLError as exc: + except (yaml.YAMLError, RecursionError) as exc: # nesting too deep for the YAML parser message = " ".join(str(exc).split()) # the YAML message is several lines; keep one return {}, Finding( "error", "frontmatter-parse", f"frontmatter is not valid YAML: {message}" diff --git a/dokimi/dokimi/tool_preference.py b/dokimi/dokimi/tool_preference.py index d58430a80..c152dfc6c 100644 --- a/dokimi/dokimi/tool_preference.py +++ b/dokimi/dokimi/tool_preference.py @@ -56,12 +56,16 @@ class GroupsError(ValueError): def load_groups(doc: dict) -> dict: """Validate a tool_preference_groups.v1 document and return it.""" + if not isinstance(doc, dict): + raise GroupsError(f"top level must be an object, got {type(doc).__name__}") problems = schema_errors(GROUPS_SCHEMA, doc) - ids = [g["id"] for g in doc.get("groups", []) if isinstance(g, dict) and isinstance(g.get("id"), str)] + raw = doc.get("groups") + groups = raw if isinstance(raw, list) else [] + ids = [g["id"] for g in groups if isinstance(g, dict) and isinstance(g.get("id"), str)] dupes = sorted({i for i in ids if ids.count(i) > 1}) if dupes: problems.append(f"groups: duplicate ids {dupes}") - for g in doc.get("groups", []) if not problems else []: + for g in groups if not problems else []: names = [g["twins"]["A"]["name"], g["twins"]["B"]["name"]] + [d["name"] for d in g["distractors"]] names += [t["vendor_name"] for t in g["twins"].values() if t.get("vendor_name")] if len(set(names)) != len(names): diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index 72aeb637b..b95b90908 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -20,7 +20,7 @@ "dokimi/commands/arms.py::_validate_expected_skills": {"complexity": 11}, "dokimi/commands/benchmark.py::cmd_benchmark_batch": {"complexity": 11}, "dokimi/commands/benchmark.py::cmd_benchmark_preflight": {"complexity": 21}, - "dokimi/commands/catalog.py::cmd_catalog_trigger_set": {"complexity": 16}, + "dokimi/commands/catalog.py::cmd_catalog_trigger_set": {"complexity": 15}, "dokimi/commands/evaluate.py::cmd_eval_judge_validity": {"complexity": 29, "rationale": "grew on main in #73 (judge-validity --context stored) from 25 to 29 before the ratchet landed; split it in its own PR"}, "dokimi/commands/judge_reliability.py::cmd_eval_cross_judge": {"complexity": 30, "rationale": "pre-existing when the gate was introduced"}, "dokimi/commands/novelty.py::cmd_novelty_routing": {"complexity": 13, "rationale": "W8 adds the --repeats guard after main's no_queries handling raised the file; the command is split with the rest of novelty.py in a separate PR"}, diff --git a/dokimi/tests/test_catalog_command.py b/dokimi/tests/test_catalog_command.py index b86fa026e..f30c258b4 100644 --- a/dokimi/tests/test_catalog_command.py +++ b/dokimi/tests/test_catalog_command.py @@ -5,12 +5,15 @@ from __future__ import annotations +import json from pathlib import Path import pytest from dokimi.cli import build_parser +_DEEP_FRONTMATTER = b"---\nname: " + b"[" * 100000 + b"\n---\nbody\n" + def _write_skill(root: Path, name: str) -> None: skill_dir = root / "skills" / name @@ -191,3 +194,171 @@ def test_catalog_scan_reports_clean_error_on_malformed_manifest(tmp_path): with pytest.raises(SystemExit) as exc_info: args.func(args) assert "invalid skills.yaml" in str(exc_info.value) + + +def test_catalog_scan_reports_clean_error_on_non_utf8_skills_manifest(tmp_path): + (tmp_path / "skills.yaml").write_bytes(b"skills: [\xff\xfe]\n") + + args = build_parser().parse_args(["catalog", "scan", "--root", str(tmp_path), "--kind", "skills"]) + with pytest.raises(SystemExit) as exc_info: + args.func(args) + assert "invalid skills.yaml" in str(exc_info.value) + + +@pytest.mark.parametrize( + ("kind", "manifest", "content"), + [ + ("skills", "skills.yaml", "skills: []\n"), + ("tasks", "evaluation/harbor/suites.json", '{"suites": []}'), + ], +) +def test_catalog_scan_reports_clean_error_on_unreadable_manifest(tmp_path, monkeypatch, kind, manifest, content): + path = tmp_path / manifest + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + real_read_text = Path.read_text + + def read_text(self, *a, **kw): + if self == path: + raise PermissionError(13, "Permission denied", str(self)) + return real_read_text(self, *a, **kw) + + monkeypatch.setattr(Path, "read_text", read_text) + + args = build_parser().parse_args(["catalog", "scan", "--root", str(tmp_path), "--kind", kind]) + with pytest.raises(SystemExit) as exc_info: + args.func(args) + assert f"cannot read {path.name} at {path}" in str(exc_info.value) + + +def test_catalog_scan_reports_clean_error_on_non_utf8_suites_manifest(tmp_path): + manifest = tmp_path / "evaluation" / "harbor" / "suites.json" + manifest.parent.mkdir(parents=True) + manifest.write_bytes(b'{"suites": ["\xff"]}') + + args = build_parser().parse_args(["catalog", "scan", "--root", str(tmp_path), "--kind", "tasks"]) + with pytest.raises(SystemExit) as exc_info: + args.func(args) + assert "invalid suites.json" in str(exc_info.value) + + +@pytest.mark.parametrize("text", ["[]", "null", '{"suites": 5}', '{"suites": {}}', '{"suites": 0}', + '{"suites": [{"tasks": {}}]}', '{"suites": [1]}', '{"suites": [{"tasks": [3]}]}', + '{"suites": [{"tasks": 3}]}', pytest.param("[" * 200000, id="deeply-nested")]) +def test_catalog_scan_reports_clean_error_on_wrong_shaped_suites_manifest(tmp_path, text): + manifest = tmp_path / "evaluation" / "harbor" / "suites.json" + manifest.parent.mkdir(parents=True) + manifest.write_text(text, encoding="utf-8") + + args = build_parser().parse_args(["catalog", "scan", "--root", str(tmp_path), "--kind", "tasks"]) + with pytest.raises(SystemExit) as exc_info: + args.func(args) + assert f"invalid suites.json at {manifest}" in str(exc_info.value) + + +@pytest.mark.parametrize("text", ["{}", '{"suites": null}', '{"suites": [{"tasks": null}]}']) +def test_catalog_scan_reads_a_null_or_missing_suites_manifest_list_as_empty(tmp_path, capsys, text): + (tmp_path / "evaluation" / "harbor" / "tasks").mkdir(parents=True) + (tmp_path / "evaluation" / "harbor" / "suites.json").write_text(text, encoding="utf-8") + + args = build_parser().parse_args(["catalog", "scan", "--root", str(tmp_path), "--kind", "tasks"]) + args.func(args) + assert f"# tasks (0 declared under {tmp_path})" in capsys.readouterr().out + + +def test_catalog_scan_reports_clean_error_on_deeply_nested_skills_manifest(tmp_path): + (tmp_path / "skills.yaml").write_text("[" * 200000, encoding="utf-8") + + args = build_parser().parse_args(["catalog", "scan", "--root", str(tmp_path), "--kind", "skills"]) + with pytest.raises(SystemExit) as exc_info: + args.func(args) + assert f"invalid skills.yaml at {tmp_path / 'skills.yaml'}" in str(exc_info.value) + + +@pytest.mark.parametrize( + "content", + [None, b"\xff\xfe", _DEEP_FRONTMATTER], + ids=["directory", "non-utf8", "deep-frontmatter"], +) +def test_catalog_scan_reports_a_malformed_skill_as_an_error_row(tmp_path, capsys, content): + skill_md = tmp_path / "broken" / "SKILL.md" + skill_md.parent.mkdir() + if content is None: + skill_md.mkdir() + else: + skill_md.write_bytes(content) + + args = build_parser().parse_args(["catalog", "scan", "--root", str(tmp_path), "--kind", "skills"]) + with pytest.raises(SystemExit) as exc_info: + args.func(args) + assert exc_info.value.code == 1 + assert f"broken\t-\t{skill_md.parent}\tERROR:" in capsys.readouterr().out + + +def test_catalog_scan_reports_a_deep_manifest_listed_skill_as_an_error_row(tmp_path, capsys): + _write_skill(tmp_path, "good") + deep = tmp_path / "skills" / "deep" + deep.mkdir() + (deep / "SKILL.md").write_bytes(_DEEP_FRONTMATTER) + (tmp_path / "skills.yaml").write_text("- name: good\n- name: deep\n", encoding="utf-8") + + args = build_parser().parse_args(["catalog", "scan", "--root", str(tmp_path), "--kind", "skills"]) + with pytest.raises(SystemExit) as exc_info: + args.func(args) + assert exc_info.value.code == 1 + out = capsys.readouterr().out + assert f"deep\t-\t{deep}\tERROR:" in out + assert f"good\t-\t{tmp_path / 'skills' / 'good'}\tok" in out + + +def test_catalog_scan_reports_a_task_name_the_filesystem_rejects(tmp_path, monkeypatch): + from dokimi.external import catalog as cat + + manifest = tmp_path / "evaluation" / "harbor" / "suites.json" + manifest.parent.mkdir(parents=True) + name = "a" * 300 + manifest.write_text(json.dumps({"suites": [{"tasks": [{"name": name, "track": "executable"}]}]}), encoding="utf-8") + + def too_long(path): + raise OSError(36, "File name too long", str(path)) + + monkeypatch.setattr(cat, "is_task_dir", too_long) + entries = cat.read_task_manifest(tmp_path) + assert entries[0].failed and "File name too long" in entries[0].reason + + +@pytest.mark.parametrize( + "content", + [b"\xff\xfe", b"---\nname: a\n---\nbody\n", _DEEP_FRONTMATTER], + ids=["non-utf8", "missing-description", "deep-frontmatter"], +) +def test_catalog_trigger_set_reports_clean_error_on_malformed_skill_md(tmp_path, content): + catalog = tmp_path / "skills" + (catalog / "a").mkdir(parents=True) + (catalog / "a" / "SKILL.md").write_bytes(content) + questions = tmp_path / "q.json" + questions.write_text(json.dumps([{"id": "q1", "question": "hi"}]), encoding="utf-8") + + args = build_parser().parse_args(["catalog", "trigger-set", "--catalog", str(catalog), + "--questions", f"{questions}=none", "--out", str(tmp_path / "o.json")]) + with pytest.raises(SystemExit) as exc_info: + args.func(args) + assert f"--catalog {catalog}:" in str(exc_info.value) + + +@pytest.mark.parametrize( + ("instruction", "task_toml"), + [(None, ""), ("Do the thing.", "[metadata\n")], + ids=["missing-instruction", "unparsable-task-toml"], +) +def test_catalog_scan_reports_clean_error_on_unloadable_task_dir(tmp_path, instruction, task_toml): + task = tmp_path / "evaluation" / "harbor" / "tasks" / "t" + task.mkdir(parents=True) + (task / "task.toml").write_text(task_toml, encoding="utf-8") + if instruction is not None: + (task / "instruction.md").write_text(instruction, encoding="utf-8") + + args = build_parser().parse_args(["catalog", "scan", "--root", str(tmp_path), "--kind", "tasks"]) + with pytest.raises(SystemExit) as exc_info: + args.func(args) + assert "cannot read tasks at" in str(exc_info.value) diff --git a/dokimi/tests/test_skill_selection.py b/dokimi/tests/test_skill_selection.py index eceed7207..f720d9d2c 100644 --- a/dokimi/tests/test_skill_selection.py +++ b/dokimi/tests/test_skill_selection.py @@ -477,6 +477,42 @@ def test_trigger_set_cli_writes_each_format(tmp_path, catalog, capsys): assert "skill-reach credits only the first" in capsys.readouterr().out +@pytest.mark.parametrize("fmt", ["reach", "skill-creator", "questions"]) +def test_trigger_set_cli_exits_with_a_message_when_out_cannot_be_written(tmp_path, catalog, fmt): + q = _qfile(tmp_path, "q.json", [{"id": "1", "text": "profile it", "skills": ["linux-perf"]}]) + out = tmp_path / "taken" + out.mkdir() # a directory where the file should go + with pytest.raises(SystemExit, match=rf"Error: cannot write --out {re.escape(str(out))}: ") as exc: + _run_cli(["catalog", "trigger-set", "--catalog", str(catalog), "--out", str(out), + "--questions", f"{q}=@skills", "--format", fmt, "--target", "linux-perf"]) + assert exc.value.code != 0 and isinstance(exc.value.code, str) + assert list(out.iterdir()) == [] + + +def test_trigger_set_cli_reports_a_schema_rejection_not_a_write_failure(tmp_path, catalog, monkeypatch): + from dokimi.artifacts import ArtifactValidationError + from dokimi.questions import artifact + + def reject(*_a, **_k): + raise ArtifactValidationError("record 1: bad shape") + + monkeypatch.setattr(artifact, "save_questions_artifact", reject) + q = _qfile(tmp_path, "q.json", [{"id": "1", "text": "profile it", "skills": ["linux-perf"]}]) + with pytest.raises(SystemExit, match=r"Error: questions set rejected: record 1: bad shape") as exc: + _run_cli(["catalog", "trigger-set", "--catalog", str(catalog), "--out", str(tmp_path / "o.json"), + "--questions", f"{q}=@skills", "--format", "questions"]) + assert "cannot write" not in exc.value.code + + +@pytest.mark.parametrize("fmt", ["reach", "skill-creator", "questions"]) +def test_trigger_set_cli_exits_with_a_message_when_out_is_empty(tmp_path, catalog, fmt): + q = _qfile(tmp_path, "q.json", [{"id": "1", "text": "profile it", "skills": ["linux-perf"]}]) + with pytest.raises(SystemExit, match=r"Error: cannot write --out : ") as exc: + _run_cli(["catalog", "trigger-set", "--catalog", str(catalog), "--out", "", + "--questions", f"{q}=@skills", "--format", fmt, "--target", "linux-perf"]) + assert isinstance(exc.value.code, str) + + # ── review round 3 ─────────────────────────────────────────────────────── def _loop_result(answer="a", skill=None): return {"answer": answer, "transcript": [], "iterations": 1, "tool_call_count": 0, diff --git a/dokimi/tests/test_tool_preference.py b/dokimi/tests/test_tool_preference.py index 2612bb6f5..034f99745 100644 --- a/dokimi/tests/test_tool_preference.py +++ b/dokimi/tests/test_tool_preference.py @@ -837,3 +837,70 @@ def test_dry_run_names_twin_a_when_only_twin_a_lacks_the_field(tmp_path, capsys) "--factors", "long_desc"]) args.func(args) assert "long_desc: skipped for 1 of 1 groups (G09 twin A)" in capsys.readouterr().out + + +@pytest.mark.parametrize("content", [bytes([0xFF, 0xFE, 0x00]), b"{not json", b"[]", b"null", None, + b'{"schema_version": "tool_preference_groups.v1", "groups": null}', + pytest.param(b"[" * 200000, id="deep-nesting"), pytest.param(b"1" * 6000, id="huge-int")]) +def test_an_unreadable_or_invalid_groups_file_exits_with_a_message(tmp_path, content): + from dokimi.cli import build_parser + + path = tmp_path / "groups.json" # content None: the path does not exist + if content is not None: + path.write_bytes(content) + args = build_parser().parse_args(["tools", "preference", "--groups", str(path), "--dry-run"]) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert str(exc.value).startswith(f"--groups {path}: ") + + +def test_an_unwritable_out_json_exits_with_a_message(monkeypatch, tmp_path, capsys): + import dokimi.llm as llm + from dokimi.cli import build_parser + + monkeypatch.setattr(llm, "chat_completion", + lambda messages, model, provider="openai", tools=None, **kw: _Resp([tools[-1]["function"]["name"]])) + blocker = tmp_path / "blocker" + blocker.write_text("x", encoding="utf-8") # a regular file where a directory is needed + out = blocker / "o.json" + args = build_parser().parse_args(["tools", "preference", "--groups", str(GROUPS), "--model", "m", + "--reps", "1", "--factors", "assertive", "--out-json", str(out)]) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 + assert f"Error: cannot write {out}: " in capsys.readouterr().err + + +def test_an_empty_out_json_exits_with_a_message(monkeypatch, capsys): + import dokimi.llm as llm + from dokimi.cli import build_parser + + monkeypatch.setattr(llm, "chat_completion", + lambda messages, model, provider="openai", tools=None, **kw: _Resp([tools[-1]["function"]["name"]])) + args = build_parser().parse_args(["tools", "preference", "--groups", str(GROUPS), "--model", "m", + "--reps", "1", "--factors", "assertive", "--out-json", ""]) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 + assert "Error: cannot write : " in capsys.readouterr().err + + +def test_a_schema_rejected_artifact_is_not_reported_as_a_write_failure(monkeypatch, capsys, tmp_path): + import dokimi.artifacts as artifacts + import dokimi.llm as llm + from dokimi.cli import build_parser + + def reject(*a, **kw): + raise artifacts.ArtifactValidationError("reps must be >= 1") + + monkeypatch.setattr(llm, "chat_completion", + lambda messages, model, provider="openai", tools=None, **kw: _Resp([tools[-1]["function"]["name"]])) + monkeypatch.setattr(artifacts, "save_artifact", reject) + args = build_parser().parse_args(["tools", "preference", "--groups", str(GROUPS), "--model", "m", + "--reps", "1", "--factors", "assertive", "--out-json", str(tmp_path / "o.json")]) + with pytest.raises(SystemExit) as exc: + args.func(args) + err = capsys.readouterr().err + assert exc.value.code == 1 + assert "tool_preference artifact rejected" in err + assert "cannot write" not in err diff --git a/dokimi/tests/test_tool_preference_out_json.py b/dokimi/tests/test_tool_preference_out_json.py new file mode 100644 index 000000000..83cd3f35e --- /dev/null +++ b/dokimi/tests/test_tool_preference_out_json.py @@ -0,0 +1,166 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The --out-json pre-flight of `tools preference`: refused before any probe call, or accepted.""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +GROUPS = Path(__file__).resolve().parent / "fixtures" / "tool_preference" / "groups.json" + + +@pytest.mark.parametrize("blocked", ["file-as-parent", "directory"]) +def test_cli_refuses_an_unwritable_out_json_before_any_call(monkeypatch, tmp_path, capsys, blocked): + import dokimi.llm as llm + from dokimi.cli import build_parser + + calls = [] + + def fake(*a, **k): + calls.append(a) + raise RuntimeError("must not be reached") + + monkeypatch.setattr(llm, "chat_completion", fake) + if blocked == "file-as-parent": + (tmp_path / "blocker").write_text("x", encoding="utf-8") + out = tmp_path / "blocker" / "pref.json" + else: + out = tmp_path / "pref.json" + out.mkdir() + args = build_parser().parse_args(["tools", "preference", "--groups", str(GROUPS), "--model", "m", + "--reps", "1", "--factors", "assertive", "--out-json", str(out)]) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 + assert calls == [] + err = capsys.readouterr().err + assert f"cannot write {out}: " in err + if blocked == "directory": + assert "Is a directory" in err + + +def test_out_json_check_creates_missing_parents_and_leaves_no_file(tmp_path): + from dokimi.commands.tools import _check_out_json_writable + + out = tmp_path / "a" / "b" / "pref.json" + _check_out_json_writable(str(out)) + assert out.parent.is_dir() + assert not out.exists() + + +@pytest.mark.skipif(os.name == "nt", reason="os.replace refuses a read-only destination on Windows, " + "which the POSIX rule pinned here does not model") +def test_out_json_check_accepts_a_read_only_file_in_a_writable_dir(tmp_path): + from dokimi.commands.tools import _check_out_json_writable + + out = tmp_path / "pref.json" + out.write_text("{}", encoding="utf-8") + out.chmod(0o444) + try: + if os.access(out, os.W_OK): + pytest.skip("chmod 0444 does not block writes here (for example, running as root)") + _check_out_json_writable(str(out)) # the save replaces the file through its directory + finally: + out.chmod(0o644) + + +def _symlink_or_skip(link: Path, target: Path) -> None: + try: + link.symlink_to(target) + except OSError as exc: # Windows needs Developer Mode or admin rights to create a symlink + pytest.skip(f"cannot create a symlink here: {exc}") + + +def test_out_json_check_leaves_no_file_at_a_dangling_symlink_target(tmp_path): + from dokimi.commands.tools import _check_out_json_writable + + real = tmp_path / "real" + real.mkdir() + link = tmp_path / "link.json" + _symlink_or_skip(link, real / "t.json") + _check_out_json_writable(str(link)) + assert not (real / "t.json").exists() + assert link.is_symlink() + + +def test_out_json_check_refuses_a_dangling_symlink_into_a_missing_dir(tmp_path, capsys): + from dokimi.commands.tools import _check_out_json_writable + + link = tmp_path / "link.json" + _symlink_or_skip(link, tmp_path / "missing" / "t.json") + with pytest.raises(SystemExit) as exc: + _check_out_json_writable(str(link)) + assert exc.value.code == 2 + assert "cannot write" in capsys.readouterr().err + + +@pytest.mark.skipif(os.name == "nt", reason="chmod 0555 does not make a directory read-only on Windows") +def test_out_json_check_refuses_a_writable_file_in_a_read_only_dir(tmp_path): + from dokimi.commands.tools import _check_out_json_writable + + ro = tmp_path / "ro" + ro.mkdir() + out = ro / "pref.json" + out.write_text("{}", encoding="utf-8") + ro.chmod(0o555) + try: + if os.access(ro, os.W_OK): + pytest.skip("chmod 0555 does not block writes here (for example, running as root)") + with pytest.raises(SystemExit) as exc: + _check_out_json_writable(str(out)) + assert exc.value.code == 2 + finally: + ro.chmod(0o755) + + +@pytest.mark.skipif(os.name == "nt", reason="chmod 0555 does not make a directory read-only on Windows") +def test_out_json_check_probes_the_symlink_target_directory(tmp_path): + from dokimi.commands.tools import _check_out_json_writable + + ro = tmp_path / "ro" + ro.mkdir() + out_dir = tmp_path / "out" + out_dir.mkdir() + link = out_dir / "link.json" + _symlink_or_skip(link, ro / "t.json") + ro.chmod(0o555) + try: + if os.access(ro, os.W_OK): + pytest.skip("chmod 0555 does not block writes here (for example, running as root)") + with pytest.raises(SystemExit) as exc: + _check_out_json_writable(str(link)) + assert exc.value.code == 2 + finally: + ro.chmod(0o755) + + +def test_out_json_check_refuses_a_name_too_long_for_the_save_temp_file(tmp_path, capsys): + from dokimi.commands.tools import _check_out_json_writable + + out = tmp_path / ("a" * 230 + ".json") + try: + out.touch() + except OSError: + pytest.skip("this file system takes no 235-character name") + out.unlink() + with pytest.raises(SystemExit) as exc: + _check_out_json_writable(str(tmp_path / ("a" * 235 + ".json"))) + assert exc.value.code == 2 + assert "cannot write" in capsys.readouterr().err + + +def test_out_json_check_refuses_a_symlink_to_a_directory(tmp_path, capsys): + from dokimi.commands.tools import _check_out_json_writable + + real = tmp_path / "dir" + real.mkdir() + link = tmp_path / "link.json" + _symlink_or_skip(link, real) + with pytest.raises(SystemExit) as exc: + _check_out_json_writable(str(link)) + assert exc.value.code == 2 + assert "Is a directory" in capsys.readouterr().err diff --git a/dokimi/tests/test_tools_preference_items_out.py b/dokimi/tests/test_tools_preference_items_out.py new file mode 100644 index 000000000..68d02e7c1 --- /dev/null +++ b/dokimi/tests/test_tools_preference_items_out.py @@ -0,0 +1,45 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""`tools preference --items-out`: a path that cannot be written exits 1 with a message.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from dokimi.cli import build_parser + +GROUPS = Path(__file__).resolve().parent / "fixtures" / "tool_preference" / "groups.json" + + +def _preference(items_out: Path) -> None: + args = build_parser().parse_args([ + "tools", "preference", "--groups", str(GROUPS), "--dry-run", "--items-out", str(items_out), + ]) + args.func(args) + + +def test_an_items_out_inside_a_regular_file_exits_1_naming_the_path(tmp_path, capsys): + blocker = tmp_path / "blocker" + blocker.write_text("x", encoding="utf-8") + target = blocker / "items.json" + with pytest.raises(SystemExit) as exc: + _preference(target) + assert exc.value.code == 1 + assert f"Error: cannot write --items-out {target}" in capsys.readouterr().err + + +def test_an_items_out_that_is_a_directory_exits_1_naming_the_path(tmp_path, capsys): + with pytest.raises(SystemExit) as exc: + _preference(tmp_path) + assert exc.value.code == 1 + assert f"Error: cannot write --items-out {tmp_path}" in capsys.readouterr().err + + +def test_a_writable_items_out_still_gets_the_items(tmp_path, capsys): + out = tmp_path / "items.json" + _preference(out) + assert out.read_text(encoding="utf-8").startswith("[") + assert f"items -> {out}" in capsys.readouterr().out