diff --git a/benchmark/core/cli.py b/benchmark/core/cli.py index 3836457..b5b67b8 100644 --- a/benchmark/core/cli.py +++ b/benchmark/core/cli.py @@ -19,7 +19,12 @@ from .gold import validate_gold_subset_table, validate_gold_table, write_gold_report from .headless_runner import HeadlessPacketRunConfig, run_headless_packet from .inventory import write_data_inventory -from .model_runner import INPUT_MODES, ModelPacketRunConfig, run_model_packet +from .model_runner import ( + INPUT_MODES, + MODEL_TRANSPORTS, + ModelPacketRunConfig, + run_model_packet, +) from .multiroute_reporting import ( build_multi_route_report_from_paths, write_multi_route_report, @@ -431,6 +436,15 @@ def _build_parser() -> argparse.ArgumentParser: run_model.add_argument("--max-tokens", type=int) run_model.add_argument("--max-retries", type=int, default=0) run_model.add_argument("--response-format", default="json_object") + run_model.add_argument( + "--transport", + choices=MODEL_TRANSPORTS, + default="chat_completions_json_object", + help=( + "model API transport; deepseek_responses_json_schema binds the " + "packet output.schema.json to DeepSeek's Responses API" + ), + ) run_model.add_argument("--endpoint") run_model.add_argument("--run-commit") run_model.add_argument("--run-id") @@ -976,6 +990,7 @@ def main(argv: Sequence[str] | None = None) -> int: max_tokens=args.max_tokens, max_retries=args.max_retries, response_format=args.response_format, + transport=args.transport, endpoint=args.endpoint, dry_run=args.dry_run, command_argv=raw_argv, diff --git a/benchmark/core/model_runner.py b/benchmark/core/model_runner.py index c8b8beb..1bf54a9 100644 --- a/benchmark/core/model_runner.py +++ b/benchmark/core/model_runner.py @@ -51,6 +51,10 @@ ".webp": "image/webp", } INPUT_MODES = ("text-only", "multimodal") +MODEL_TRANSPORTS = ( + "chat_completions_json_object", + "deepseek_responses_json_schema", +) TEXT_SUFFIXES = {".csv", ".json", ".md", ".text", ".txt"} OPENAI_COMPATIBLE_PROVIDERS = { "deepseek": { @@ -78,6 +82,7 @@ class ModelPacketRunConfig: top_p: float | None = None max_tokens: int | None = None response_format: str = "json_object" + transport: str = "chat_completions_json_object" endpoint: str | None = None max_retries: int = 0 dry_run: bool = False @@ -132,6 +137,10 @@ def run_model_packet(config: ModelPacketRunConfig) -> dict[str, Any]: raise ValueError( f"--input-mode must be one of {', '.join(INPUT_MODES)}" ) + if config.transport not in MODEL_TRANSPORTS: + raise ValueError( + f"--transport must be one of {', '.join(MODEL_TRANSPORTS)}" + ) if config.max_retries < 0: raise ValueError("--max-retries must be non-negative") model_policy_binding = ( @@ -153,6 +162,11 @@ def run_model_packet(config: ModelPacketRunConfig) -> dict[str, Any]: raise ValueError("transcription packets require --input-mode multimodal") course = CourseSpec.from_dict(_read_json(packet / "course.json")) output_contract = validate_packet_output_contract(packet, manifest, course, task) + output_schema = ( + _read_json(packet / "output.schema.json") + if config.transport == "deepseek_responses_json_schema" + else None + ) prompt_text = (packet / "prompt.txt").read_text(encoding="utf-8") rubric = _read_json(packet / "rubric.json") if task == "grade" else None student_ids = tuple(manifest.get("student_ids", ())) @@ -165,7 +179,11 @@ def run_model_packet(config: ModelPacketRunConfig) -> dict[str, Any]: image_inputs = _load_image_inputs(packet, student_ids) else: text_inputs = _load_text_inputs(packet, student_ids) - provider = _provider_from_config(config, output_contract=output_contract) + provider = _provider_from_config( + config, + output_contract=output_contract, + output_schema=output_schema, + ) output.mkdir(parents=True) (output / "outputs").mkdir() command = _write_command_records(output, config) @@ -260,10 +278,39 @@ def _provider_from_config( config: ModelPacketRunConfig, *, output_contract: str = GRADING_OUTPUT_CONTRACT_V1, + output_schema: dict[str, Any] | None = None, ) -> TextModelProvider | MultimodalModelProvider: + if config.transport not in MODEL_TRANSPORTS: + raise ValueError( + f"--transport must be one of {', '.join(MODEL_TRANSPORTS)}" + ) + if config.transport == "deepseek_responses_json_schema": + if config.provider != "deepseek": + raise ValueError( + "deepseek_responses_json_schema transport requires provider deepseek" + ) + if config.input_mode != "text-only": + raise ValueError( + "deepseek_responses_json_schema transport requires input_mode text-only" + ) + if output_schema is None: + raise ValueError( + "deepseek_responses_json_schema transport requires packet output schema" + ) if config.dry_run: return DryRunTextProvider(config.model, output_contract=output_contract) settings = _provider_settings(config.provider) + if config.transport == "deepseek_responses_json_schema": + return DeepSeekResponsesJsonSchemaProvider( + model=config.model, + endpoint=_provider_endpoint(config), + api_key_env=settings["api_key_env"], + display_name=settings["display_name"], + temperature=config.temperature, + top_p=config.top_p, + max_tokens=config.max_tokens, + output_schema=output_schema, + ) return OpenAICompatibleTextProvider( model=config.model, endpoint=_provider_endpoint(config), @@ -367,6 +414,82 @@ def _dry_run_score_row(question: Any, *, traced: bool) -> dict[str, Any]: return row +class DeepSeekResponsesJsonSchemaProvider: + """DeepSeek Responses API provider with a packet-bound JSON Schema.""" + + def __init__( + self, + *, + model: str, + endpoint: str, + api_key_env: str, + display_name: str, + temperature: float | None, + top_p: float | None, + max_tokens: int | None, + output_schema: dict[str, Any], + ): + self.model = model + self.endpoint = endpoint + self.api_key_env = api_key_env + self.display_name = display_name + self.temperature = temperature + self.top_p = top_p + self.max_tokens = max_tokens + self.output_schema = output_schema + + def complete_text( + self, + prompt: str, + *, + student_id: str, + course: CourseSpec, + task: str, + ) -> ModelProviderResult: + api_key = os.environ.get(self.api_key_env) + if not api_key: + raise ValueError( + f"{self.api_key_env} is required for non-dry {self.display_name} runs" + ) + from openai import OpenAI + + client = OpenAI(api_key=api_key, base_url=self.endpoint) + request: dict[str, Any] = { + "model": self.model, + "input": prompt, + "reasoning": {"effort": "none"}, + "text": { + "format": { + "type": "json_schema", + "name": "grading_packet_output", + "schema": self.output_schema, + } + }, + } + if self.temperature is not None: + request["temperature"] = self.temperature + if self.top_p is not None: + request["top_p"] = self.top_p + if self.max_tokens is not None: + request["max_output_tokens"] = self.max_tokens + response = client.responses.create(**request) + status = getattr(response, "status", None) + if status not in {None, "completed"}: + raise ValueError( + "DeepSeek Responses API returned non-completed status: " + f"{status}" + ) + raw_text = getattr(response, "output_text", None) + if not isinstance(raw_text, str) or not raw_text: + raise ValueError("DeepSeek Responses API returned no output_text") + return ModelProviderResult( + raw_text=raw_text, + model=getattr(response, "model", self.model), + usage=_usage_to_dict(getattr(response, "usage", None)), + system_fingerprint=getattr(response, "system_fingerprint", None), + ) + + class OpenAICompatibleTextProvider: def __init__( self, @@ -1216,7 +1339,12 @@ def _metadata( "temperature": config.temperature, "top_p": config.top_p, "max_tokens": config.max_tokens, - "response_format": config.response_format, + "response_format": ( + "json_schema" + if config.transport == "deepseek_responses_json_schema" + else config.response_format + ), + "transport": config.transport, "max_retries": config.max_retries, "retry_policy": ( "append JSON repair instruction after validation or provider failure" @@ -1280,6 +1408,8 @@ def _write_command_records(output: Path, config: ModelPacketRunConfig) -> str: config.model, "--input-mode", config.input_mode, + "--transport", + config.transport, "--packet", str(config.packet), "--output", diff --git a/tests/benchmark/core/test_model_runner.py b/tests/benchmark/core/test_model_runner.py index 0e21d36..d19bd82 100644 --- a/tests/benchmark/core/test_model_runner.py +++ b/tests/benchmark/core/test_model_runner.py @@ -12,9 +12,12 @@ from benchmark.core.cli import main from benchmark.core.model_runner import ( + DeepSeekResponsesJsonSchemaProvider, + ModelPacketRunConfig, OpenAICompatibleTextProvider, _compose_multimodal_prompt, _compose_student_prompt, + _provider_from_config, _retry_validation_prompt_suffix, _submission_page_order, _validate_grade_payload, @@ -238,6 +241,78 @@ def __init__(self, **kwargs): self.assertEqual(captured["extra_body"], {"thinking": {"type": "disabled"}}) self.assertEqual(captured["max_tokens"], 4096) + def test_deepseek_responses_provider_binds_packet_schema_and_disables_thinking(self): + captured: dict[str, object] = {} + output_schema = { + "type": "object", + "properties": {"total": {"type": "number"}}, + "required": ["total"], + "additionalProperties": False, + } + + class FakeResponses: + def create(self, **kwargs): + captured.update(kwargs) + return SimpleNamespace( + status="completed", + output_text='{"total": 5}', + model="deepseek-v4-flash", + usage=None, + ) + + class FakeOpenAI: + def __init__(self, **kwargs): + captured["client"] = kwargs + self.responses = FakeResponses() + + provider = DeepSeekResponsesJsonSchemaProvider( + model="deepseek-v4-flash", + endpoint="https://api.deepseek.com", + api_key_env="DEEPSEEK_API_KEY", + display_name="DeepSeek", + temperature=0, + top_p=None, + max_tokens=4096, + output_schema=output_schema, + ) + with patch.dict(os.environ, {"DEEPSEEK_API_KEY": "test-key"}), patch.dict( + sys.modules, {"openai": SimpleNamespace(OpenAI=FakeOpenAI)} + ): + result = provider.complete_text( + "Return JSON.", + student_id="S001", + course=None, + task="grade", + ) + + self.assertEqual(result.raw_text, '{"total": 5}') + self.assertEqual(captured["input"], "Return JSON.") + self.assertEqual(captured["reasoning"], {"effort": "none"}) + self.assertEqual(captured["max_output_tokens"], 4096) + self.assertEqual( + captured["text"], + { + "format": { + "type": "json_schema", + "name": "grading_packet_output", + "schema": output_schema, + } + }, + ) + + def test_deepseek_responses_schema_transport_rejects_other_providers(self): + config = ModelPacketRunConfig( + provider="kimi", + model="kimi-k2.6", + input_mode="text-only", + packet=Path("packet"), + output=Path("output"), + transport="deepseek_responses_json_schema", + ) + + with self.assertRaisesRegex(ValueError, "requires provider deepseek"): + _provider_from_config(config, output_schema={"type": "object"}) + def test_build_text_grading_packet_can_feed_dry_run_runner(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -454,6 +529,42 @@ def test_run_model_packet_supports_kimi_dry_run_metadata(self): self.assertNotIn("MOONSHOT_API_KEY=", command) self.assertNotIn("sk-", command) + def test_run_model_packet_records_deepseek_responses_schema_transport(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + packet = self._build_grade_packet(root, "transcript.json", b'{"answer":"ok"}') + output = root / "runs" / "deepseek-responses-schema-G1-dev-r1" + stdout = io.StringIO() + + with contextlib.redirect_stdout(stdout): + code = main( + [ + "run-model-packet", + "--provider", + "deepseek", + "--model", + "deepseek-v4-flash", + "--input-mode", + "text-only", + "--packet", + str(packet), + "--output", + str(output), + "--transport", + "deepseek_responses_json_schema", + "--dry-run", + ] + ) + + metadata = json.loads((output / "run-metadata.json").read_text(encoding="utf-8")) + command_argv = json.loads((output / "command.argv.json").read_text(encoding="utf-8")) + + self.assertEqual(code, 0) + self.assertEqual(metadata["transport"], "deepseek_responses_json_schema") + self.assertEqual(metadata["response_format"], "json_schema") + self.assertIn("--transport", command_argv) + self.assertIn("deepseek_responses_json_schema", command_argv) + def test_run_model_packet_rejects_image_inputs_for_text_only_deepseek(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp)