diff --git a/dokimi/changelog.d/20261028_120000_antonwolfy_import-blank-targets.md b/dokimi/changelog.d/20261028_120000_antonwolfy_import-blank-targets.md new file mode 100644 index 000000000..29a7d2c21 --- /dev/null +++ b/dokimi/changelog.d/20261028_120000_antonwolfy_import-blank-targets.md @@ -0,0 +1,12 @@ +### Fixed + +- **`questions import` leaves a blank judge assert, an empty `contains*` list + and a punctuation-only Inspect `exact` target out of the grading** — a + promptfoo `llm-rubric`/`factuality` (or other judged) assert with a blank or + missing value, or a `contains*`/`icontains*` assert with an empty list, + imported as a check that `eval reference` refuses, which aborted the run for + every question of the file; it now imports as `unsupported`, listed and not + counted, like a blank value already did. An Inspect `exact` target that + SQuAD-normalises to nothing, such as `...` or `the`, imported as a value no + answer can equal; it is now dropped, and a sample left with no target + imports as `unsupported`. diff --git a/dokimi/docs/answer-formats.md b/dokimi/docs/answer-formats.md index 80482f95c..1bae0deb3 100644 --- a/dokimi/docs/answer-formats.md +++ b/dokimi/docs/answer-formats.md @@ -65,8 +65,13 @@ A blank value is not a check. A `match` value that is blank once the check has normalised it (stripped, and with `ignore_punctuation` on, `...` too), or an empty `regex` pattern, is found in every answer, so a row whose `ideal`/`target` is blank imports as `unsupported`, and a hand-written `values` -list with such an entry is refused by `eval reference`. A promptfoo assert -whose value is blank (whitespace included) imports as `unsupported` too. +list with such an entry is refused by `eval reference`. An Inspect `exact` +target that SQuAD-normalises to nothing (`...`, `the`) equals no answer, so it +is dropped, and a sample left with no target imports as `unsupported`. A +promptfoo assert whose value is blank (whitespace included) imports as +`unsupported` too, and so do a `contains*` or `icontains*` assert whose list of +values is empty and a judged assert (`llm-rubric`, `factuality`, ...) whose +value is blank or missing. Every check accepts `mode` (`any`/`all` over its `values`), `negate` and `weight`. A question's score is the weighted mean of the checks that were diff --git a/dokimi/dokimi/task_formats/imports/inspect_ai.py b/dokimi/dokimi/task_formats/imports/inspect_ai.py index f3ae5ae56..d9cd9f6c7 100644 --- a/dokimi/dokimi/task_formats/imports/inspect_ai.py +++ b/dokimi/dokimi/task_formats/imports/inspect_ai.py @@ -19,7 +19,7 @@ from pathlib import Path from typing import Any, Mapping, Optional -from dokimi.eval.reference_checks import blank_value +from dokimi.eval.reference_checks import blank_value, squad_normalize from dokimi.task_formats.imports.common import ( Imported, as_list, choice_targets, judge, match, question, render_input, render_mcq, unsupported, ) @@ -50,10 +50,11 @@ def _choices(value: Any) -> list[str]: def _checks(scorer: str, kwargs: Mapping[str, Any], targets: list[str], imp: Imported) -> list[dict[str, Any]]: - # A target the scorer normalises to nothing ("", or "..." for `match`) is in every answer, so it - # is dropped; with none left nothing is graded. + # A target the scorer normalises to nothing ("", or "..." for `match`) is in every answer, and + # one `exact` SQuAD-normalises to nothing ("...", "the") equals none, so it is dropped; with none + # left nothing is graded. norm = {"ignore_punctuation": scorer == "match"} - targets = [t for t in targets if not blank_value("match", norm, t)] + targets = [t for t in targets if not blank_value("match", norm, t) and (scorer != "exact" or squad_normalize(t))] if not targets and scorer in ("match", "includes", "exact", "model_graded_qa", "model_graded_fact"): imp.note("sample without a usable target: not graded") return [unsupported(scorer)] diff --git a/dokimi/dokimi/task_formats/imports/promptfoo.py b/dokimi/dokimi/task_formats/imports/promptfoo.py index adbdad505..35f24f5a1 100644 --- a/dokimi/dokimi/task_formats/imports/promptfoo.py +++ b/dokimi/dokimi/task_formats/imports/promptfoo.py @@ -43,10 +43,12 @@ def _text_check(base: str, value: Any, negate: bool) -> Optional[dict[str, Any]]: """The deterministic check for a text assert, or None when ``base`` is not one.""" if base in _CONTAINS or base in ("equals", "starts-with", "regex"): - # No value to compare (`regex` would search for "None"), or a blank one, which - # eval reference refuses: left unsupported instead. + # No value to compare (`regex` would search for "None"), a blank one, or an empty + # `contains-*` list, which eval reference refuses: left unsupported instead. if value is None or any(not str(v).strip() for v in as_list(value)): return None + if base in _CONTAINS and not as_list(value): + return None if base in _CONTAINS: mode, ignore_case = _CONTAINS[base] return match(as_list(value), "any", ignore_case=ignore_case, ignore_punctuation=False, @@ -72,6 +74,8 @@ def _other_check(base: str, value: Any, negate: bool, imp: Imported) -> Optional imp.note(f"{base} schema not checked") return check if base in _JUDGED and not negate: + if not str(value or "").strip(): + return None # a judge check with no criterion or reference is refused by eval reference imp.note(f"{base}: graded by the yes/no checklist judge") return judge(native=base, **{_JUDGED[base]: str(value or "")}) return None diff --git a/dokimi/tests/test_answer_imports.py b/dokimi/tests/test_answer_imports.py index 5457343cc..37a1acadc 100644 --- a/dokimi/tests/test_answer_imports.py +++ b/dokimi/tests/test_answer_imports.py @@ -148,13 +148,18 @@ def test_inspect_blank_or_missing_target_is_unsupported_not_a_pass(tmp_path): assert _checks(imp) == _checks(imp, 1) == [{"type": "unsupported", "native": "match"}] -def test_inspect_punctuation_only_target_is_unsupported_for_match_only(tmp_path): - # `match` drops punctuation, so "..." would be found in every answer; `includes` keeps it. +def test_inspect_punctuation_only_target_is_unsupported_for_match_and_exact(tmp_path): + # `match` drops punctuation, so "..." would be found in every answer; `exact` SQuAD-normalises + # it (and "the") to "", which equals no answer; `includes` keeps it. path = _jsonl(tmp_path / "s.jsonl", [{"input": "q", "target": "..."}, - {"input": "r", "target": ["!!", "x"]}]) + {"input": "r", "target": ["!!", "x"]}, + {"input": "s", "target": ["The", "Paris"]}]) imp = import_path(path, fmt="inspect-samples") assert _checks(imp) == [{"type": "unsupported", "native": "match"}] assert _checks(imp, 1)[0]["values"] == ["x"] + exact = import_path(path, fmt="inspect-samples", scorer="exact") + assert _checks(exact) == [{"type": "unsupported", "native": "exact"}] + assert _checks(exact, 2) == [{"type": "exact", "values": ["Paris"]}] kept = import_path(path, fmt="inspect-samples", scorer="includes") assert grade(kept.questions[0]["grading"], "q", "I refuse to answer").passed is False @@ -401,6 +406,25 @@ def test_promptfoo_blank_values_are_unsupported_not_refused_later(tmp_path): assert validate_grading(imp.questions[0]["grading"]) == [] +@pytest.mark.parametrize("assertion", ["{type: llm-rubric, value: ''}", "{type: factuality, value: ' '}", + "{type: contains-any, value: []}"]) +def test_promptfoo_blank_judge_or_empty_list_is_unsupported_not_refused_later(tmp_path, assertion): + cfg = f"prompts: ['{{{{q}}}}']\ntests:\n - vars: {{q: x}}\n assert:\n - {assertion}\n" + imp = import_path(_write(tmp_path / "promptfooconfig.yaml", cfg), fmt="promptfoo") + kind = assertion.split(",")[0].removeprefix("{type: ") + assert _checks(imp) == [{"type": "unsupported", "native": kind}] + assert f"assert `{kind}` not translated" in imp.lossy + + +def test_promptfoo_blank_judge_beside_a_good_assert_keeps_the_file_gradable(tmp_path): + cfg = ("prompts: ['{{q}}']\ntests:\n - vars: {q: x}\n assert:\n" + " - {type: llm-rubric, value: ''}\n - {type: contains-any, value: []}\n" + " - {type: llm-rubric, value: Is polite}\n") + imp = import_path(_write(tmp_path / "promptfooconfig.yaml", cfg), fmt="promptfoo") + assert validate_grading(imp.questions[0]["grading"]) == [] + assert _checks(imp)[2] == {"type": "judge", "criterion": "Is polite", "native": "llm-rubric"} + + def test_promptfoo_punctuation_only_values_are_real_checks_not_a_pass(tmp_path): # promptfoo compares raw text, so "..." and "!!" are found only in answers that contain them. cfg = ("prompts: ['{{q}}']\ntests:\n - vars: {q: x}\n assert:\n"