fix(task_formats): import blank judge and empty-list asserts unsupported

dokimi/changelog.d/20261028_120000_antonwolfy_import-blank-targets.md

@@ -0,0 +1,12 @@
+### Fixed
+
+- **`questions import` leaves a blank judge assert, an empty `contains*` list
+  and a punctuation-only Inspect `exact` target out of the grading** — a
+  promptfoo `llm-rubric`/`factuality` (or other judged) assert with a blank or
+  missing value, or a `contains*`/`icontains*` assert with an empty list,
+  imported as a check that `eval reference` refuses, which aborted the run for
+  every question of the file; it now imports as `unsupported`, listed and not
+  counted, like a blank value already did. An Inspect `exact` target that
+  SQuAD-normalises to nothing, such as `...` or `the`, imported as a value no
+  answer can equal; it is now dropped, and a sample left with no target
+  imports as `unsupported`.

dokimi/docs/answer-formats.md

@@ -65,8 +65,13 @@ A blank value is not a check. A `match` value that is blank once the check has
 normalised it (stripped, and with `ignore_punctuation` on, `...` too), or an
 empty `regex` pattern, is found in every answer, so a row whose
 `ideal`/`target` is blank imports as `unsupported`, and a hand-written `values`
-list with such an entry is refused by `eval reference`. A promptfoo assert
-whose value is blank (whitespace included) imports as `unsupported` too.
+list with such an entry is refused by `eval reference`. An Inspect `exact`
+target that SQuAD-normalises to nothing (`...`, `the`) equals no answer, so it
+is dropped, and a sample left with no target imports as `unsupported`. A
+promptfoo assert whose value is blank (whitespace included) imports as
+`unsupported` too, and so do a `contains*` or `icontains*` assert whose list of
+values is empty and a judged assert (`llm-rubric`, `factuality`, ...) whose
+value is blank or missing.
 
 Every check accepts `mode` (`any`/`all` over its `values`), `negate` and
 `weight`. A question's score is the weighted mean of the checks that were

dokimi/dokimi/task_formats/imports/inspect_ai.py

@@ -19,7 +19,7 @@
 from pathlib import Path
 from typing import Any, Mapping, Optional
 
-from dokimi.eval.reference_checks import blank_value
+from dokimi.eval.reference_checks import blank_value, squad_normalize
 from dokimi.task_formats.imports.common import (
     Imported, as_list, choice_targets, judge, match, question, render_input, render_mcq, unsupported,
 )
@@ -50,10 +50,11 @@ def _choices(value: Any) -> list[str]:
 
 
 def _checks(scorer: str, kwargs: Mapping[str, Any], targets: list[str], imp: Imported) -> list[dict[str, Any]]:
-    # A target the scorer normalises to nothing ("", or "..." for `match`) is in every answer, so it
-    # is dropped; with none left nothing is graded.
+    # A target the scorer normalises to nothing ("", or "..." for `match`) is in every answer, and
+    # one `exact` SQuAD-normalises to nothing ("...", "the") equals none, so it is dropped; with none
+    # left nothing is graded.
     norm = {"ignore_punctuation": scorer == "match"}
-    targets = [t for t in targets if not blank_value("match", norm, t)]
+    targets = [t for t in targets if not blank_value("match", norm, t) and (scorer != "exact" or squad_normalize(t))]
     if not targets and scorer in ("match", "includes", "exact", "model_graded_qa", "model_graded_fact"):
         imp.note("sample without a usable target: not graded")
         return [unsupported(scorer)]

dokimi/dokimi/task_formats/imports/promptfoo.py

@@ -43,10 +43,12 @@
 def _text_check(base: str, value: Any, negate: bool) -> Optional[dict[str, Any]]:
     """The deterministic check for a text assert, or None when ``base`` is not one."""
     if base in _CONTAINS or base in ("equals", "starts-with", "regex"):
-        # No value to compare (`regex` would search for "None"), or a blank one, which
-        # eval reference refuses: left unsupported instead.
+        # No value to compare (`regex` would search for "None"), a blank one, or an empty
+        # `contains-*` list, which eval reference refuses: left unsupported instead.
         if value is None or any(not str(v).strip() for v in as_list(value)):
             return None
+        if base in _CONTAINS and not as_list(value):
+            return None
     if base in _CONTAINS:
         mode, ignore_case = _CONTAINS[base]
         return match(as_list(value), "any", ignore_case=ignore_case, ignore_punctuation=False,
@@ -72,6 +74,8 @@ def _other_check(base: str, value: Any, negate: bool, imp: Imported) -> Optional
             imp.note(f"{base} schema not checked")
         return check
     if base in _JUDGED and not negate:
+        if not str(value or "").strip():
+            return None  # a judge check with no criterion or reference is refused by eval reference
         imp.note(f"{base}: graded by the yes/no checklist judge")
         return judge(native=base, **{_JUDGED[base]: str(value or "")})
     return None

dokimi/tests/test_answer_imports.py

@@ -148,13 +148,18 @@ def test_inspect_blank_or_missing_target_is_unsupported_not_a_pass(tmp_path):
     assert _checks(imp) == _checks(imp, 1) == [{"type": "unsupported", "native": "match"}]
 
 
-def test_inspect_punctuation_only_target_is_unsupported_for_match_only(tmp_path):
-    # `match` drops punctuation, so "..." would be found in every answer; `includes` keeps it.
+def test_inspect_punctuation_only_target_is_unsupported_for_match_and_exact(tmp_path):
+    # `match` drops punctuation, so "..." would be found in every answer; `exact` SQuAD-normalises
+    # it (and "the") to "", which equals no answer; `includes` keeps it.
     path = _jsonl(tmp_path / "s.jsonl", [{"input": "q", "target": "..."},
-                                         {"input": "r", "target": ["!!", "x"]}])
+                                         {"input": "r", "target": ["!!", "x"]},
+                                         {"input": "s", "target": ["The", "Paris"]}])
     imp = import_path(path, fmt="inspect-samples")
     assert _checks(imp) == [{"type": "unsupported", "native": "match"}]
     assert _checks(imp, 1)[0]["values"] == ["x"]
+    exact = import_path(path, fmt="inspect-samples", scorer="exact")
+    assert _checks(exact) == [{"type": "unsupported", "native": "exact"}]
+    assert _checks(exact, 2) == [{"type": "exact", "values": ["Paris"]}]
     kept = import_path(path, fmt="inspect-samples", scorer="includes")
     assert grade(kept.questions[0]["grading"], "q", "I refuse to answer").passed is False
 
@@ -401,6 +406,25 @@ def test_promptfoo_blank_values_are_unsupported_not_refused_later(tmp_path):
     assert validate_grading(imp.questions[0]["grading"]) == []
 
 
+@pytest.mark.parametrize("assertion", ["{type: llm-rubric, value: ''}", "{type: factuality, value: '  '}",
+                                       "{type: contains-any, value: []}"])
+def test_promptfoo_blank_judge_or_empty_list_is_unsupported_not_refused_later(tmp_path, assertion):
+    cfg = f"prompts: ['{{{{q}}}}']\ntests:\n  - vars: {{q: x}}\n    assert:\n      - {assertion}\n"
+    imp = import_path(_write(tmp_path / "promptfooconfig.yaml", cfg), fmt="promptfoo")
+    kind = assertion.split(",")[0].removeprefix("{type: ")
+    assert _checks(imp) == [{"type": "unsupported", "native": kind}]
+    assert f"assert `{kind}` not translated" in imp.lossy
+
+
+def test_promptfoo_blank_judge_beside_a_good_assert_keeps_the_file_gradable(tmp_path):
+    cfg = ("prompts: ['{{q}}']\ntests:\n  - vars: {q: x}\n    assert:\n"
+           "      - {type: llm-rubric, value: ''}\n      - {type: contains-any, value: []}\n"
+           "      - {type: llm-rubric, value: Is polite}\n")
+    imp = import_path(_write(tmp_path / "promptfooconfig.yaml", cfg), fmt="promptfoo")
+    assert validate_grading(imp.questions[0]["grading"]) == []
+    assert _checks(imp)[2] == {"type": "judge", "criterion": "Is polite", "native": "llm-rubric"}
+
+
 def test_promptfoo_punctuation_only_values_are_real_checks_not_a_pass(tmp_path):
     # promptfoo compares raw text, so "..." and "!!" are found only in answers that contain them.
     cfg = ("prompts: ['{{q}}']\ntests:\n  - vars: {q: x}\n    assert:\n"