feat: DeepSeek Responses 严格 Schema 通道 / strict Schema transport

benchmark/core/cli.py

@@ -19,7 +19,12 @@
 from .gold import validate_gold_subset_table, validate_gold_table, write_gold_report
 from .headless_runner import HeadlessPacketRunConfig, run_headless_packet
 from .inventory import write_data_inventory
-from .model_runner import INPUT_MODES, ModelPacketRunConfig, run_model_packet
+from .model_runner import (
+    INPUT_MODES,
+    MODEL_TRANSPORTS,
+    ModelPacketRunConfig,
+    run_model_packet,
+)
 from .multiroute_reporting import (
     build_multi_route_report_from_paths,
     write_multi_route_report,
@@ -431,6 +436,15 @@ def _build_parser() -> argparse.ArgumentParser:
     run_model.add_argument("--max-tokens", type=int)
     run_model.add_argument("--max-retries", type=int, default=0)
     run_model.add_argument("--response-format", default="json_object")
+    run_model.add_argument(
+        "--transport",
+        choices=MODEL_TRANSPORTS,
+        default="chat_completions_json_object",
+        help=(
+            "model API transport; deepseek_responses_json_schema binds the "
+            "packet output.schema.json to DeepSeek's Responses API"
+        ),
+    )
     run_model.add_argument("--endpoint")
     run_model.add_argument("--run-commit")
     run_model.add_argument("--run-id")
@@ -976,6 +990,7 @@ def main(argv: Sequence[str] | None = None) -> int:
                     max_tokens=args.max_tokens,
                     max_retries=args.max_retries,
                     response_format=args.response_format,
+                    transport=args.transport,
                     endpoint=args.endpoint,
                     dry_run=args.dry_run,
                     command_argv=raw_argv,

benchmark/core/model_runner.py

@@ -51,6 +51,10 @@
     ".webp": "image/webp",
 }
 INPUT_MODES = ("text-only", "multimodal")
+MODEL_TRANSPORTS = (
+    "chat_completions_json_object",
+    "deepseek_responses_json_schema",
+)
 TEXT_SUFFIXES = {".csv", ".json", ".md", ".text", ".txt"}
 OPENAI_COMPATIBLE_PROVIDERS = {
     "deepseek": {
@@ -78,6 +82,7 @@ class ModelPacketRunConfig:
     top_p: float | None = None
     max_tokens: int | None = None
     response_format: str = "json_object"
+    transport: str = "chat_completions_json_object"
     endpoint: str | None = None
     max_retries: int = 0
     dry_run: bool = False
@@ -132,6 +137,10 @@ def run_model_packet(config: ModelPacketRunConfig) -> dict[str, Any]:
         raise ValueError(
             f"--input-mode must be one of {', '.join(INPUT_MODES)}"
         )
+    if config.transport not in MODEL_TRANSPORTS:
+        raise ValueError(
+            f"--transport must be one of {', '.join(MODEL_TRANSPORTS)}"
+        )
     if config.max_retries < 0:
         raise ValueError("--max-retries must be non-negative")
     model_policy_binding = (
@@ -153,6 +162,11 @@ def run_model_packet(config: ModelPacketRunConfig) -> dict[str, Any]:
         raise ValueError("transcription packets require --input-mode multimodal")
     course = CourseSpec.from_dict(_read_json(packet / "course.json"))
     output_contract = validate_packet_output_contract(packet, manifest, course, task)
+    output_schema = (
+        _read_json(packet / "output.schema.json")
+        if config.transport == "deepseek_responses_json_schema"
+        else None
+    )
     prompt_text = (packet / "prompt.txt").read_text(encoding="utf-8")
     rubric = _read_json(packet / "rubric.json") if task == "grade" else None
     student_ids = tuple(manifest.get("student_ids", ()))
@@ -165,7 +179,11 @@ def run_model_packet(config: ModelPacketRunConfig) -> dict[str, Any]:
         image_inputs = _load_image_inputs(packet, student_ids)
     else:
         text_inputs = _load_text_inputs(packet, student_ids)
-    provider = _provider_from_config(config, output_contract=output_contract)
+    provider = _provider_from_config(
+        config,
+        output_contract=output_contract,
+        output_schema=output_schema,
+    )
     output.mkdir(parents=True)
     (output / "outputs").mkdir()
     command = _write_command_records(output, config)
@@ -260,10 +278,39 @@ def _provider_from_config(
     config: ModelPacketRunConfig,
     *,
     output_contract: str = GRADING_OUTPUT_CONTRACT_V1,
+    output_schema: dict[str, Any] | None = None,
 ) -> TextModelProvider | MultimodalModelProvider:
+    if config.transport not in MODEL_TRANSPORTS:
+        raise ValueError(
+            f"--transport must be one of {', '.join(MODEL_TRANSPORTS)}"
+        )
+    if config.transport == "deepseek_responses_json_schema":
+        if config.provider != "deepseek":
+            raise ValueError(
+                "deepseek_responses_json_schema transport requires provider deepseek"
+            )
+        if config.input_mode != "text-only":
+            raise ValueError(
+                "deepseek_responses_json_schema transport requires input_mode text-only"
+            )
+        if output_schema is None:
+            raise ValueError(
+                "deepseek_responses_json_schema transport requires packet output schema"
+            )
     if config.dry_run:
         return DryRunTextProvider(config.model, output_contract=output_contract)
     settings = _provider_settings(config.provider)
+    if config.transport == "deepseek_responses_json_schema":
+        return DeepSeekResponsesJsonSchemaProvider(
+            model=config.model,
+            endpoint=_provider_endpoint(config),
+            api_key_env=settings["api_key_env"],
+            display_name=settings["display_name"],
+            temperature=config.temperature,
+            top_p=config.top_p,
+            max_tokens=config.max_tokens,
+            output_schema=output_schema,
+        )
     return OpenAICompatibleTextProvider(
         model=config.model,
         endpoint=_provider_endpoint(config),
@@ -367,6 +414,82 @@ def _dry_run_score_row(question: Any, *, traced: bool) -> dict[str, Any]:
     return row
 
 
+class DeepSeekResponsesJsonSchemaProvider:
+    """DeepSeek Responses API provider with a packet-bound JSON Schema."""
+
+    def __init__(
+        self,
+        *,
+        model: str,
+        endpoint: str,
+        api_key_env: str,
+        display_name: str,
+        temperature: float | None,
+        top_p: float | None,
+        max_tokens: int | None,
+        output_schema: dict[str, Any],
+    ):
+        self.model = model
+        self.endpoint = endpoint
+        self.api_key_env = api_key_env
+        self.display_name = display_name
+        self.temperature = temperature
+        self.top_p = top_p
+        self.max_tokens = max_tokens
+        self.output_schema = output_schema
+
+    def complete_text(
+        self,
+        prompt: str,
+        *,
+        student_id: str,
+        course: CourseSpec,
+        task: str,
+    ) -> ModelProviderResult:
+        api_key = os.environ.get(self.api_key_env)
+        if not api_key:
+            raise ValueError(
+                f"{self.api_key_env} is required for non-dry {self.display_name} runs"
+            )
+        from openai import OpenAI
+
+        client = OpenAI(api_key=api_key, base_url=self.endpoint)
+        request: dict[str, Any] = {
+            "model": self.model,
+            "input": prompt,
+            "reasoning": {"effort": "none"},
+            "text": {
+                "format": {
+                    "type": "json_schema",
+                    "name": "grading_packet_output",
+                    "schema": self.output_schema,
+                }
+            },
+        }
+        if self.temperature is not None:
+            request["temperature"] = self.temperature
+        if self.top_p is not None:
+            request["top_p"] = self.top_p
+        if self.max_tokens is not None:
+            request["max_output_tokens"] = self.max_tokens
+        response = client.responses.create(**request)
+        status = getattr(response, "status", None)
+        if status not in {None, "completed"}:
+            raise ValueError(
+                "DeepSeek Responses API returned non-completed status: "
+                f"{status}"
+            )
+        raw_text = getattr(response, "output_text", None)
+        if not isinstance(raw_text, str) or not raw_text:
+            raise ValueError("DeepSeek Responses API returned no output_text")
+        return ModelProviderResult(
+            raw_text=raw_text,
+            model=getattr(response, "model", self.model),
+            usage=_usage_to_dict(getattr(response, "usage", None)),
+            system_fingerprint=getattr(response, "system_fingerprint", None),
+        )
+
+
 class OpenAICompatibleTextProvider:
     def __init__(
         self,
@@ -1216,7 +1339,12 @@ def _metadata(
         "temperature": config.temperature,
         "top_p": config.top_p,
         "max_tokens": config.max_tokens,
-        "response_format": config.response_format,
+        "response_format": (
+            "json_schema"
+            if config.transport == "deepseek_responses_json_schema"
+            else config.response_format
+        ),
+        "transport": config.transport,
         "max_retries": config.max_retries,
         "retry_policy": (
             "append JSON repair instruction after validation or provider failure"
@@ -1280,6 +1408,8 @@ def _write_command_records(output: Path, config: ModelPacketRunConfig) -> str:
             config.model,
             "--input-mode",
             config.input_mode,
+            "--transport",
+            config.transport,
             "--packet",
             str(config.packet),
             "--output",

tests/benchmark/core/test_model_runner.py

@@ -12,9 +12,12 @@
 
 from benchmark.core.cli import main
 from benchmark.core.model_runner import (
+    DeepSeekResponsesJsonSchemaProvider,
+    ModelPacketRunConfig,
     OpenAICompatibleTextProvider,
     _compose_multimodal_prompt,
     _compose_student_prompt,
+    _provider_from_config,
     _retry_validation_prompt_suffix,
     _submission_page_order,
     _validate_grade_payload,
@@ -238,6 +241,78 @@ def __init__(self, **kwargs):
         self.assertEqual(captured["extra_body"], {"thinking": {"type": "disabled"}})
         self.assertEqual(captured["max_tokens"], 4096)
 
+    def test_deepseek_responses_provider_binds_packet_schema_and_disables_thinking(self):
+        captured: dict[str, object] = {}
+        output_schema = {
+            "type": "object",
+            "properties": {"total": {"type": "number"}},
+            "required": ["total"],
+            "additionalProperties": False,
+        }
+
+        class FakeResponses:
+            def create(self, **kwargs):
+                captured.update(kwargs)
+                return SimpleNamespace(
+                    status="completed",
+                    output_text='{"total": 5}',
+                    model="deepseek-v4-flash",
+                    usage=None,
+                )
+
+        class FakeOpenAI:
+            def __init__(self, **kwargs):
+                captured["client"] = kwargs
+                self.responses = FakeResponses()
+
+        provider = DeepSeekResponsesJsonSchemaProvider(
+            model="deepseek-v4-flash",
+            endpoint="https://api.deepseek.com",
+            api_key_env="DEEPSEEK_API_KEY",
+            display_name="DeepSeek",
+            temperature=0,
+            top_p=None,
+            max_tokens=4096,
+            output_schema=output_schema,
+        )
+        with patch.dict(os.environ, {"DEEPSEEK_API_KEY": "test-key"}), patch.dict(
+            sys.modules, {"openai": SimpleNamespace(OpenAI=FakeOpenAI)}
+        ):
+            result = provider.complete_text(
+                "Return JSON.",
+                student_id="S001",
+                course=None,
+                task="grade",
+            )
+
+        self.assertEqual(result.raw_text, '{"total": 5}')
+        self.assertEqual(captured["input"], "Return JSON.")
+        self.assertEqual(captured["reasoning"], {"effort": "none"})
+        self.assertEqual(captured["max_output_tokens"], 4096)
+        self.assertEqual(
+            captured["text"],
+            {
+                "format": {
+                    "type": "json_schema",
+                    "name": "grading_packet_output",
+                    "schema": output_schema,
+                }
+            },
+        )
+
+    def test_deepseek_responses_schema_transport_rejects_other_providers(self):
+        config = ModelPacketRunConfig(
+            provider="kimi",
+            model="kimi-k2.6",
+            input_mode="text-only",
+            packet=Path("packet"),
+            output=Path("output"),
+            transport="deepseek_responses_json_schema",
+        )
+
+        with self.assertRaisesRegex(ValueError, "requires provider deepseek"):
+            _provider_from_config(config, output_schema={"type": "object"})
+
     def test_build_text_grading_packet_can_feed_dry_run_runner(self):
         with tempfile.TemporaryDirectory() as tmp:
             root = Path(tmp)
@@ -454,6 +529,42 @@ def test_run_model_packet_supports_kimi_dry_run_metadata(self):
         self.assertNotIn("MOONSHOT_API_KEY=", command)
         self.assertNotIn("sk-", command)
 
+    def test_run_model_packet_records_deepseek_responses_schema_transport(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            packet = self._build_grade_packet(root, "transcript.json", b'{"answer":"ok"}')
+            output = root / "runs" / "deepseek-responses-schema-G1-dev-r1"
+            stdout = io.StringIO()
+
+            with contextlib.redirect_stdout(stdout):
+                code = main(
+                    [
+                        "run-model-packet",
+                        "--provider",
+                        "deepseek",
+                        "--model",
+                        "deepseek-v4-flash",
+                        "--input-mode",
+                        "text-only",
+                        "--packet",
+                        str(packet),
+                        "--output",
+                        str(output),
+                        "--transport",
+                        "deepseek_responses_json_schema",
+                        "--dry-run",
+                    ]
+                )
+
+            metadata = json.loads((output / "run-metadata.json").read_text(encoding="utf-8"))
+            command_argv = json.loads((output / "command.argv.json").read_text(encoding="utf-8"))
+
+        self.assertEqual(code, 0)
+        self.assertEqual(metadata["transport"], "deepseek_responses_json_schema")
+        self.assertEqual(metadata["response_format"], "json_schema")
+        self.assertIn("--transport", command_argv)
+        self.assertIn("deepseek_responses_json_schema", command_argv)
+
     def test_run_model_packet_rejects_image_inputs_for_text_only_deepseek(self):
         with tempfile.TemporaryDirectory() as tmp:
             root = Path(tmp)