From 248c5f61db412a7c0b6e53e9c8865a1e9d155ce6 Mon Sep 17 00:00:00 2001 From: francisco-orkes Date: Fri, 11 Sep 2026 10:57:36 -0700 Subject: [PATCH 1/2] Re-export assert_output_satisfies from the testing package The import shown in semantic.py's own module docstring raised ImportError: testing/__init__.py never imported the module, and the name was absent from __all__, so neither the documented import nor a star import resolved it. Adds the import and the __all__ entry, matching how every other assertion in the package is exposed. semantic.py imports litellm lazily inside the function, so the package-level import does not turn that optional dependency into a hard one -- a test pins that invariant in a subprocess. --- src/conductor/ai/agents/testing/__init__.py | 5 +++ tests/unit/ai/test_testing_semantic.py | 44 +++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 tests/unit/ai/test_testing_semantic.py diff --git a/src/conductor/ai/agents/testing/__init__.py b/src/conductor/ai/agents/testing/__init__.py index cd4dffaa..30c357c6 100644 --- a/src/conductor/ai/agents/testing/__init__.py +++ b/src/conductor/ai/agents/testing/__init__.py @@ -52,6 +52,9 @@ # Record/replay from conductor.ai.agents.testing.recording import record, replay +# Semantic assertions +from conductor.ai.agents.testing.semantic import assert_output_satisfies + # Strategy validators from conductor.ai.agents.testing.strategy_validators import ( StrategyViolation, @@ -77,6 +80,8 @@ "assert_guardrail_passed", "assert_guardrail_failed", "assert_max_turns", + # Semantic assertions + "assert_output_satisfies", # Fluent API "expect", "AgentResultExpectation", diff --git a/tests/unit/ai/test_testing_semantic.py b/tests/unit/ai/test_testing_semantic.py new file mode 100644 index 00000000..d3ee91f3 --- /dev/null +++ b/tests/unit/ai/test_testing_semantic.py @@ -0,0 +1,44 @@ +"""Tests for conductor.ai.agents.testing.semantic.""" + +import os +import subprocess +import sys + +# The import shown in semantic.py's own module docstring. A failure here is a +# collection error, which is the point: the docstring must stay executable. +from conductor.ai.agents.testing import assert_output_satisfies +from conductor.ai.agents.testing.semantic import ( + assert_output_satisfies as assert_output_satisfies_direct, +) + + +def test_package_export_is_the_semantic_function(): + assert assert_output_satisfies is assert_output_satisfies_direct + + +def test_star_import_exposes_assert_output_satisfies(): + """`__all__` membership, asserted through what it actually governs.""" + namespace: dict = {} + exec("from conductor.ai.agents.testing import *", namespace) # noqa: S102 + + assert namespace["assert_output_satisfies"] is assert_output_satisfies_direct + + +def test_importing_the_package_does_not_import_litellm(): + """Re-exporting semantic.py must not drag its optional dependency in. + + ``litellm`` is imported lazily inside ``assert_output_satisfies``. If it + ever moved to module scope, this package-level re-export would turn an + optional dependency into a hard one for every importer of ``testing``. + Run in a subprocess so an unrelated test cannot pre-populate sys.modules. + """ + source = ( + "import sys\n" + "import conductor.ai.agents.testing\n" + "assert 'litellm' not in sys.modules, 'litellm imported eagerly'\n" + ) + # The subprocess inherits neither pytest's pythonpath setting nor this + # process's sys.path, so hand it the path `conductor` actually resolved + # from -- otherwise it may import a different checkout. + env = {**os.environ, "PYTHONPATH": os.pathsep.join(sys.path)} + subprocess.run([sys.executable, "-c", source], check=True, env=env) From e270970d06afef87da2cf7522de63d820a9136ce Mon Sep 17 00:00:00 2001 From: francisco-orkes Date: Tue, 15 Sep 2026 15:36:49 -0700 Subject: [PATCH 2/2] Test the judge's behaviour instead of the export Replaces the export and star-import assertions with tests for what assert_output_satisfies actually does: threshold pass and fail, the judge reason reaching the assertion message, an unparseable reply failing rather than passing, and a missing litellm raising with an install hint. The tests import through the package rather than the submodule, which is the path semantic.py's docstring documents, so a missing re-export fails collection without asserting on __all__ directly. litellm is stubbed via sys.modules, following test_guardrail.py. --- tests/unit/ai/test_testing_semantic.py | 92 +++++++++++++++++--------- 1 file changed, 60 insertions(+), 32 deletions(-) diff --git a/tests/unit/ai/test_testing_semantic.py b/tests/unit/ai/test_testing_semantic.py index d3ee91f3..af727a3b 100644 --- a/tests/unit/ai/test_testing_semantic.py +++ b/tests/unit/ai/test_testing_semantic.py @@ -1,44 +1,72 @@ -"""Tests for conductor.ai.agents.testing.semantic.""" +"""Tests for conductor.ai.agents.testing.semantic. + +Imported through the package rather than the submodule, deliberately: that is +the path ``semantic.py``'s docstring tells users to take, so exercising it here +keeps the re-export honest. ``litellm`` is an optional dependency and is +stubbed throughout, following ``test_guardrail.py``. +""" -import os -import subprocess import sys +from unittest.mock import MagicMock, patch + +import pytest -# The import shown in semantic.py's own module docstring. A failure here is a -# collection error, which is the point: the docstring must stay executable. +from conductor.ai.agents.result import AgentResult from conductor.ai.agents.testing import assert_output_satisfies -from conductor.ai.agents.testing.semantic import ( - assert_output_satisfies as assert_output_satisfies_direct, -) -def test_package_export_is_the_semantic_function(): - assert assert_output_satisfies is assert_output_satisfies_direct +def _judge_returning(payload: str) -> MagicMock: + response = MagicMock() + response.choices = [MagicMock()] + response.choices[0].message.content = payload + return response + + +def _result(output: str = "Sunny in NYC, 22C") -> AgentResult: + return AgentResult(output=output) + + +def test_passes_when_score_meets_threshold(): + with patch.dict("sys.modules", {"litellm": MagicMock()}): + sys.modules["litellm"].completion.return_value = _judge_returning( + '{"score": 0.9, "reason": "covers NYC weather"}' + ) + + assert_output_satisfies(_result(), criterion="mentions NYC weather", threshold=0.7) + + +def test_fails_below_threshold_and_reports_the_judge_reason(): + with patch.dict("sys.modules", {"litellm": MagicMock()}): + sys.modules["litellm"].completion.return_value = _judge_returning( + '{"score": 0.2, "reason": "no weather at all"}' + ) + + with pytest.raises(AssertionError, match="no weather at all"): + assert_output_satisfies(_result(), criterion="mentions NYC weather", threshold=0.7) + + +def test_criterion_and_output_reach_the_judge(): + with patch.dict("sys.modules", {"litellm": MagicMock()}): + sys.modules["litellm"].completion.return_value = _judge_returning( + '{"score": 1.0, "reason": "ok"}' + ) + + assert_output_satisfies(_result("Sunny in NYC"), criterion="mentions NYC") + prompt = sys.modules["litellm"].completion.call_args.kwargs["messages"][1] + assert "mentions NYC" in prompt["content"] + assert "Sunny in NYC" in prompt["content"] -def test_star_import_exposes_assert_output_satisfies(): - """`__all__` membership, asserted through what it actually governs.""" - namespace: dict = {} - exec("from conductor.ai.agents.testing import *", namespace) # noqa: S102 - assert namespace["assert_output_satisfies"] is assert_output_satisfies_direct +def test_unparseable_judge_reply_fails_rather_than_passing(): + with patch.dict("sys.modules", {"litellm": MagicMock()}): + sys.modules["litellm"].completion.return_value = _judge_returning("not json") + with pytest.raises(AssertionError, match="unparseable"): + assert_output_satisfies(_result(), criterion="anything") -def test_importing_the_package_does_not_import_litellm(): - """Re-exporting semantic.py must not drag its optional dependency in. - ``litellm`` is imported lazily inside ``assert_output_satisfies``. If it - ever moved to module scope, this package-level re-export would turn an - optional dependency into a hard one for every importer of ``testing``. - Run in a subprocess so an unrelated test cannot pre-populate sys.modules. - """ - source = ( - "import sys\n" - "import conductor.ai.agents.testing\n" - "assert 'litellm' not in sys.modules, 'litellm imported eagerly'\n" - ) - # The subprocess inherits neither pytest's pythonpath setting nor this - # process's sys.path, so hand it the path `conductor` actually resolved - # from -- otherwise it may import a different checkout. - env = {**os.environ, "PYTHONPATH": os.pathsep.join(sys.path)} - subprocess.run([sys.executable, "-c", source], check=True, env=env) +def test_missing_litellm_raises_with_an_install_hint(): + with patch.dict("sys.modules", {"litellm": None}): + with pytest.raises(ImportError, match="litellm"): + assert_output_satisfies(_result(), criterion="anything")