From c52b9cf0dcd0901a9ec153d86346ba8b44d5b5c5 Mon Sep 17 00:00:00 2001
From: Delwin Kim <139003345+DelwinKim@users.noreply.github.com>
Date: Thu, 6 Aug 2026 23:58:11 +0000
Subject: [PATCH 1/4] Add ORT-GenAI package adapter

Preserve complete ORT-GenAI package layouts across composite conversion and cache export, including direct file and nested-directory handlers.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 olive/cache.py                                |  44 +++
 olive/olive_config.json                       |   8 +
 .../passes/onnx/composite_to_onnx_package.py  | 174 ++++++++++++
 .../onnx/test_composite_to_onnx_package.py    | 252 ++++++++++++++++++
 test/test_cache.py                            | 163 +++++++++++
 5 files changed, 641 insertions(+)
 create mode 100644 olive/passes/onnx/composite_to_onnx_package.py
 create mode 100644 test/passes/onnx/test_composite_to_onnx_package.py

diff --git a/olive/cache.py b/olive/cache.py
index 7f27a50de1..e9b8ea04c5 100644
--- a/olive/cache.py
+++ b/olive/cache.py
@@ -20,6 +20,7 @@
 from olive.common.container_client_factory import AzureContainerClientFactory
 from olive.common.utils import hash_dict, hf_repo_exists, set_nested_dict_value
 from olive.model.config.model_config import ModelConfig
+from olive.model.utils.onnx_utils import get_onnx_file_path
 from olive.resource_path import ResourcePath, create_resource_path, find_all_resources
 
 if TYPE_CHECKING:
@@ -532,6 +533,49 @@ def _save_model(
             model_path_resource = model_json["config"]["model_path"]
             source_path = Path(model_path_resource.get_path())
             onnx_file_name = model_json["config"].get("onnx_file_name")
+            model_attributes = model_json["config"].get("model_attributes") or {}
+
+            package_root = source_path if source_path.is_dir() else source_path.parent
+            for _ in range(3):
+                if (package_root / "genai_config.json").is_file():
+                    break
+                if package_root.parent == package_root:
+                    package_root = None
+                    break
+                package_root = package_root.parent
+            else:
+                package_root = None
+
+            if package_root is not None or model_attributes.get("ort_genai_package"):
+                if package_root is None:
+                    raise ValueError(f"ORT-GenAI package has no discoverable genai_config.json: {source_path}")
+                if output_dir.suffix == ".onnx":
+                    raise ValueError("ORT-GenAI packages must be saved to a directory, not an ONNX file path.")
+                package_root = package_root.resolve()
+                resolved_output_dir = output_dir.resolve()
+                if (
+                    package_root == resolved_output_dir
+                    or package_root.is_relative_to(resolved_output_dir)
+                    or resolved_output_dir.is_relative_to(package_root)
+                ):
+                    raise ValueError(
+                        "ORT-GenAI package source and output directories must not overlap: "
+                        f"source={package_root}, output={resolved_output_dir}."
+                    )
+                entry_path = Path(get_onnx_file_path(str(source_path), onnx_file_name)).resolve()
+                if not entry_path.is_relative_to(package_root) or not entry_path.is_file():
+                    raise ValueError(f"ORT-GenAI package entry point is invalid: {entry_path}")
+                if resolved_output_dir.exists():
+                    if any(resolved_output_dir.iterdir()) and not overwrite:
+                        raise FileExistsError(f"Output directory is not empty: {resolved_output_dir}")
+                    if any(resolved_output_dir.iterdir()):
+                        shutil.rmtree(resolved_output_dir)
+                    else:
+                        resolved_output_dir.rmdir()
+                shutil.copytree(package_root, resolved_output_dir)
+                model_json["config"]["model_path"] = str(resolved_output_dir)
+                model_json["config"]["onnx_file_name"] = entry_path.relative_to(package_root).as_posix()
+                return self._save_additional_files(model_json, resolved_output_dir)
 
             # Determine if source has external data or additional files
             has_additional_files = bool(onnx_file_name)
diff --git a/olive/olive_config.json b/olive/olive_config.json
index 25c4b546e5..c8cdc8f2cf 100644
--- a/olive/olive_config.json
+++ b/olive/olive_config.json
@@ -8,6 +8,14 @@
             "supported_algorithms": [  ],
             "supported_quantization_encodings": [  ]
         },
+        "CompositeToOnnxPackage": {
+            "module_path": "olive.passes.onnx.composite_to_onnx_package.CompositeToOnnxPackage",
+            "supported_providers": [ "*" ],
+            "supported_accelerators": [ "*" ],
+            "supported_precisions": [ "*" ],
+            "supported_algorithms": [  ],
+            "supported_quantization_encodings": [  ]
+        },
         "AimetQuantization": {
             "module_path": "olive.passes.onnx.aimet_quantization.AimetQuantization",
             "supported_providers": [ "*" ],
diff --git a/olive/passes/onnx/composite_to_onnx_package.py b/olive/passes/onnx/composite_to_onnx_package.py
new file mode 100644
index 0000000000..3beb9eea81
--- /dev/null
+++ b/olive/passes/onnx/composite_to_onnx_package.py
@@ -0,0 +1,174 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+# --------------------------------------------------------------------------
+"""Wrap a multi-component CompositeModel ORT-GenAI package as a single ONNXModel handler."""
+
+from __future__ import annotations
+
+import json
+import logging
+import shutil
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from olive.common.utils import hardlink_copy_dir
+from olive.model import ONNXModelHandler
+from olive.model.handler.composite import CompositeModelHandler
+from olive.passes import Pass
+from olive.passes.pass_config import BasePassConfig, PassConfigParam
+
+if TYPE_CHECKING:
+    from olive.hardware.accelerator import AcceleratorSpec
+
+logger = logging.getLogger(__name__)
+
+
+class CompositeToOnnxPackage(Pass):
+    """Repackage a CompositeModel ORT-GenAI package as a single :class:`ONNXModelHandler`.
+
+    ``MobiusBuilder`` and similar passes emit multi-component ORT-GenAI packages as a
+    :class:`CompositeModelHandler` whose components live in subdirectories::
+
+        output_dir/
+          genai_config.json
+          tokenizer.json
+          decoder/model.onnx
+          vision_encoder/model.onnx
+          audio_encoder/model.onnx
+          embedding/model.onnx
+
+    Two consumers need a concrete :class:`ONNXModelHandler` rather than a composite:
+
+    * ``LocalSystem.evaluate_model`` raises ``NotImplementedError`` for composite
+      models, so a composite package can't be evaluated directly.
+    * ORT-GenAI evaluators (``LMMSEvaluator``, ``OnnxEvaluator._inference_vision_genai``)
+      dispatch on ``ONNXModelHandler`` and locate ``genai_config.json`` relative to
+      the handler's ONNX file.
+
+    This pass performs that conversion **without flattening the layout**: the nested
+    directory structure is preserved (ONNX Runtime GenAI loads nested packages
+    directly, and ``genai_config.json`` already references the nested paths), so no
+    ONNX re-serialization or external-data rewriting is needed. The package tree is
+    hardlink-copied into the pass output directory and a single
+    :class:`ONNXModelHandler` is returned, pointing at the entry-point component
+    (defaults to ``decoder``). Evaluators discover ``genai_config.json`` by searching
+    upward from the entry ONNX file (see ``_find_genai_config`` in the evaluator).
+    """
+
+    _accepts_composite_model = True
+
+    @classmethod
+    def _default_config(cls, accelerator_spec: AcceleratorSpec) -> dict[str, PassConfigParam]:
+        return {
+            "entry_point_component": PassConfigParam(
+                type_=str,
+                default_value="decoder",
+                description=(
+                    "Name of the genai_config 'model' subsection (e.g. 'decoder', 'text') "
+                    "whose ONNX file the returned ONNXModelHandler will point at. If the "
+                    "name is not found, falls back to the first component with a 'filename' field."
+                ),
+            ),
+        }
+
+    @classmethod
+    def is_accelerator_agnostic(cls, accelerator_spec: AcceleratorSpec) -> bool:
+        # Pure file-system repackaging — no EP-specific behavior.
+        return True
+
+    def _run_for_config(
+        self,
+        model: CompositeModelHandler,
+        config: type[BasePassConfig],
+        output_model_path: str,
+    ) -> ONNXModelHandler:
+        if not isinstance(model, CompositeModelHandler):
+            raise ValueError(
+                f"CompositeToOnnxPackage expects a CompositeModelHandler input, got {type(model).__name__}."
+            )
+
+        src_dir = Path(model.model_path).resolve()
+        if not src_dir.is_dir():
+            raise ValueError(f"CompositeModel model_path is not a directory: {src_dir}")
+
+        src_genai_config = src_dir / "genai_config.json"
+        if not src_genai_config.is_file():
+            raise ValueError(
+                f"CompositeToOnnxPackage requires genai_config.json at the package root: {src_genai_config} not found."
+            )
+
+        genai_config = json.loads(src_genai_config.read_text(encoding="utf-8"))
+        model_section = genai_config.get("model")
+        if not isinstance(model_section, dict):
+            raise ValueError(f"Invalid genai_config.json at {src_genai_config}: missing 'model' section.")
+
+        entry_filename = self._select_entry_filename(model_section, config.entry_point_component)
+        if entry_filename is None:
+            raise ValueError(
+                "Failed to determine an entry-point component for CompositeToOnnxPackage. "
+                f"Requested '{config.entry_point_component}', no component matched and no fallback available."
+            )
+
+        dst_dir = self._resolve_output_dir(output_model_path).resolve()
+        # Copy the entire nested package tree (cheap hardlinks) into the pass output
+        # directory, preserving the multi-component subdirectory layout as-is.
+        hardlink_copy_dir(src_dir, dst_dir)
+        # genai_config.json is mutable deployment metadata (provider/session
+        # options are commonly adjusted after export). Give the output package an
+        # independent copy so changing it cannot mutate the input pass cache.
+        dst_genai_config = dst_dir / "genai_config.json"
+        dst_genai_config.unlink()
+        shutil.copy2(src_genai_config, dst_genai_config)
+
+        entry_path = (dst_dir / entry_filename).resolve()
+        if not entry_path.is_relative_to(dst_dir):
+            raise ValueError(
+                f"Entry-point component '{entry_filename}' escapes the ORT-GenAI package directory {dst_dir}."
+            )
+        if not entry_path.is_file():
+            raise ValueError(
+                f"Entry-point component '{entry_filename}' not found in packaged output at {entry_path}. "
+                "genai_config.json references a component file that does not exist on disk."
+            )
+
+        logger.info(
+            "CompositeToOnnxPackage: packaged %d components into '%s' (entry_point=%s, nested layout preserved)",
+            sum(1 for v in model_section.values() if isinstance(v, dict) and v.get("filename")),
+            dst_dir,
+            entry_filename,
+        )
+
+        return ONNXModelHandler(
+            model_path=str(dst_dir),
+            onnx_file_name=entry_filename,
+            model_attributes={
+                **(model.model_attributes or {}),
+                "ort_genai_package": True,
+                "entry_point_component": config.entry_point_component,
+            },
+        )
+
+    @staticmethod
+    def _resolve_output_dir(output_model_path: str) -> Path:
+        """Olive sometimes passes a `.onnx` file path; in that case use its stem as the directory."""
+        output_path = Path(output_model_path)
+        if output_path.suffix == ".onnx":
+            return output_path.parent / output_path.stem
+        return output_path
+
+    @staticmethod
+    def _select_entry_filename(model_section: dict, entry_point_component: str) -> str | None:
+        """Pick the (nested) filename for the entry-point component, falling back if missing."""
+        preferred = model_section.get(entry_point_component)
+        if isinstance(preferred, dict):
+            filename = preferred.get("filename")
+            if isinstance(filename, str):
+                return filename
+
+        for component_cfg in model_section.values():
+            if isinstance(component_cfg, dict):
+                filename = component_cfg.get("filename")
+                if isinstance(filename, str):
+                    return filename
+        return None
diff --git a/test/passes/onnx/test_composite_to_onnx_package.py b/test/passes/onnx/test_composite_to_onnx_package.py
new file mode 100644
index 0000000000..78b9d15cab
--- /dev/null
+++ b/test/passes/onnx/test_composite_to_onnx_package.py
@@ -0,0 +1,252 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+# --------------------------------------------------------------------------
+"""Tests for the CompositeToOnnxPackage pass.
+
+The pass repackages a nested multi-component CompositeModel ORT-GenAI package as a
+single ONNXModelHandler pointing at the entry-point component, preserving the nested
+directory layout (no flattening / external-data rewriting).
+"""
+
+import json
+from pathlib import Path
+
+import numpy as np
+import onnx
+import pytest
+from onnx import TensorProto, helper, numpy_helper
+
+from olive.model import ONNXModelHandler
+from olive.model.handler.composite import CompositeModelHandler
+from olive.passes.olive_pass import create_pass_from_dict
+from olive.passes.onnx.composite_to_onnx_package import CompositeToOnnxPackage
+
+
+def _write_tiny_onnx_with_external_data(onnx_path: Path, data_filename: str = "model.onnx.data") -> None:
+    """Write a minimal valid ONNX model whose single initializer lives in an external data sidecar."""
+    data = np.arange(1024, dtype=np.float32)
+    init_tensor = numpy_helper.from_array(data, name="weight")
+    output = helper.make_tensor_value_info("y", TensorProto.FLOAT, [1024])
+    node = helper.make_node("Identity", inputs=["weight"], outputs=["y"])
+    graph = helper.make_graph([node], "g", inputs=[], outputs=[output], initializer=[init_tensor])
+    model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 17)])
+
+    onnx_path.parent.mkdir(parents=True, exist_ok=True)
+    onnx.save_model(
+        model,
+        str(onnx_path),
+        save_as_external_data=True,
+        all_tensors_to_one_file=True,
+        location=data_filename,
+    )
+
+
+def _make_nested_genai_package(root: Path, components: dict[str, str]) -> Path:
+    """Build a fake nested ORT-GenAI package at ``root``.
+
+    ``components`` maps genai_config component keys (e.g. ``decoder``) to the
+    relative ONNX filename under ``root`` (e.g. ``decoder/model.onnx``).
+    """
+    root.mkdir(parents=True, exist_ok=True)
+
+    model_section: dict[str, dict[str, str]] = {}
+    for key, rel_path in components.items():
+        _write_tiny_onnx_with_external_data(root / rel_path)
+        model_section[key] = {"filename": rel_path}
+
+    # Shared root-level sidecars.
+    (root / "tokenizer.json").write_text("{}", encoding="utf-8")
+    (root / "chat_template.jinja").write_text("template", encoding="utf-8")
+
+    (root / "genai_config.json").write_text(
+        json.dumps({"model": {"type": "gemma4", **model_section}}, indent=2),
+        encoding="utf-8",
+    )
+    return root
+
+
+def _make_composite_handler(
+    root: Path, components: dict[str, str], model_attributes: dict | None = None
+) -> CompositeModelHandler:
+    component_handlers = [ONNXModelHandler(model_path=str(root / rel_path)) for rel_path in components.values()]
+    return CompositeModelHandler(
+        model_components=component_handlers,
+        model_component_names=list(components.keys()),
+        model_path=str(root),
+        model_attributes=model_attributes,
+    )
+
+
+def _find_genai_config_upward(onnx_file: Path, levels: int = 3) -> Path | None:
+    """Mirror the evaluator's upward search for genai_config.json from an ONNX file."""
+    candidate = onnx_file.parent
+    for _ in range(levels):
+        if (candidate / "genai_config.json").is_file():
+            return candidate / "genai_config.json"
+        if candidate.parent == candidate:
+            break
+        candidate = candidate.parent
+    return None
+
+
+class TestCompositeToOnnxPackage:
+    def test_preserves_nested_component_layout(self, tmp_path):
+        src_root = _make_nested_genai_package(
+            tmp_path / "src",
+            {
+                "decoder": "decoder/model.onnx",
+                "vision": "vision_encoder/model.onnx",
+                "audio": "audio_encoder/model.onnx",
+                "embedding": "embedding/model.onnx",
+            },
+        )
+        composite = _make_composite_handler(
+            src_root,
+            {
+                "decoder": "decoder/model.onnx",
+                "vision_encoder": "vision_encoder/model.onnx",
+                "audio_encoder": "audio_encoder/model.onnx",
+                "embedding": "embedding/model.onnx",
+            },
+        )
+
+        p = create_pass_from_dict(CompositeToOnnxPackage, {}, disable_search=True)
+        out = p.run(composite, str(tmp_path / "out"))
+
+        pkg_root = Path(out.model_path).parents[1]
+        # Nested layout preserved (components stay in their subdirectories).
+        assert (pkg_root / "decoder" / "model.onnx").is_file()
+        assert (pkg_root / "vision_encoder" / "model.onnx").is_file()
+        assert (pkg_root / "audio_encoder" / "model.onnx").is_file()
+        assert (pkg_root / "embedding" / "model.onnx").is_file()
+        # Shared sidecars and config copied to the package root.
+        assert (pkg_root / "genai_config.json").is_file()
+        assert (pkg_root / "tokenizer.json").is_file()
+        assert (pkg_root / "chat_template.jinja").is_file()
+
+    def test_returns_onnx_handler_pointing_at_nested_entry(self, tmp_path):
+        src_root = _make_nested_genai_package(
+            tmp_path / "src",
+            {"decoder": "decoder/model.onnx", "vision": "vision_encoder/model.onnx"},
+        )
+        composite = _make_composite_handler(
+            src_root, {"decoder": "decoder/model.onnx", "vision_encoder": "vision_encoder/model.onnx"}
+        )
+
+        p = create_pass_from_dict(CompositeToOnnxPackage, {}, disable_search=True)
+        out = p.run(composite, str(tmp_path / "out"))
+
+        assert isinstance(out, ONNXModelHandler)
+        entry = Path(out.model_path)
+        # Handler points at the nested decoder entry, which really exists.
+        assert entry.is_file()
+        assert entry.name == "model.onnx"
+        assert entry.parent.name == "decoder"
+        # genai_config.json is discoverable by searching upward from the entry ONNX file.
+        assert _find_genai_config_upward(entry) is not None
+
+    def test_enforces_package_model_attributes(self, tmp_path):
+        src_root = _make_nested_genai_package(tmp_path / "src", {"decoder": "decoder/model.onnx"})
+        composite = _make_composite_handler(
+            src_root,
+            {"decoder": "decoder/model.onnx"},
+            model_attributes={"ort_genai_package": False, "entry_point_component": "untrusted", "source": "mobius"},
+        )
+
+        p = create_pass_from_dict(CompositeToOnnxPackage, {}, disable_search=True)
+        out = p.run(composite, str(tmp_path / "out"))
+
+        assert out.model_attributes == {
+            "ort_genai_package": True,
+            "entry_point_component": "decoder",
+            "source": "mobius",
+        }
+
+    def test_does_not_rewrite_genai_config_filenames(self, tmp_path):
+        src_root = _make_nested_genai_package(
+            tmp_path / "src",
+            {"decoder": "decoder/model.onnx", "vision": "vision_encoder/model.onnx"},
+        )
+        composite = _make_composite_handler(
+            src_root, {"decoder": "decoder/model.onnx", "vision_encoder": "vision_encoder/model.onnx"}
+        )
+
+        p = create_pass_from_dict(CompositeToOnnxPackage, {}, disable_search=True)
+        out = p.run(composite, str(tmp_path / "out"))
+
+        pkg_root = Path(out.model_path).parents[1]
+        genai_config = json.loads((pkg_root / "genai_config.json").read_text(encoding="utf-8"))
+        # Nested paths are preserved verbatim (no flattening rewrite).
+        assert genai_config["model"]["decoder"]["filename"] == "decoder/model.onnx"
+        assert genai_config["model"]["vision"]["filename"] == "vision_encoder/model.onnx"
+
+    def test_genai_config_is_independent_from_input_package(self, tmp_path):
+        src_root = _make_nested_genai_package(tmp_path / "src", {"decoder": "decoder/model.onnx"})
+        composite = _make_composite_handler(src_root, {"decoder": "decoder/model.onnx"})
+
+        p = create_pass_from_dict(CompositeToOnnxPackage, {}, disable_search=True)
+        out = p.run(composite, str(tmp_path / "out"))
+
+        pkg_root = Path(out.model_path).parents[1]
+        src_config = src_root / "genai_config.json"
+        output_config = pkg_root / "genai_config.json"
+        assert src_config.stat().st_ino != output_config.stat().st_ino
+        output_config.write_text('{"model": {"type": "patched"}}', encoding="utf-8")
+        assert json.loads(src_config.read_text(encoding="utf-8"))["model"]["type"] == "gemma4"
+
+    def test_honors_explicit_entry_point_component(self, tmp_path):
+        src_root = _make_nested_genai_package(
+            tmp_path / "src",
+            {"decoder": "decoder/model.onnx", "embedding": "embedding/model.onnx"},
+        )
+        composite = _make_composite_handler(
+            src_root, {"decoder": "decoder/model.onnx", "embedding": "embedding/model.onnx"}
+        )
+
+        p = create_pass_from_dict(CompositeToOnnxPackage, {"entry_point_component": "embedding"}, disable_search=True)
+        out = p.run(composite, str(tmp_path / "out"))
+
+        assert Path(out.model_path).parent.name == "embedding"
+
+    def test_uses_fallback_entry_point_when_requested_one_missing(self, tmp_path):
+        src_root = _make_nested_genai_package(
+            tmp_path / "src",
+            {"vision": "vision_encoder/model.onnx", "embedding": "embedding/model.onnx"},
+        )
+        composite = _make_composite_handler(
+            src_root, {"vision_encoder": "vision_encoder/model.onnx", "embedding": "embedding/model.onnx"}
+        )
+
+        # 'decoder' (the default entry) is absent -> falls back to a present component.
+        p = create_pass_from_dict(CompositeToOnnxPackage, {}, disable_search=True)
+        out = p.run(composite, str(tmp_path / "out"))
+
+        assert Path(out.model_path).parent.name in {"vision_encoder", "embedding"}
+
+    def test_raises_on_non_composite_input(self, tmp_path):
+        _write_tiny_onnx_with_external_data(tmp_path / "model.onnx")
+        onnx_model = ONNXModelHandler(model_path=str(tmp_path / "model.onnx"))
+
+        p = create_pass_from_dict(CompositeToOnnxPackage, {}, disable_search=True)
+        with pytest.raises(ValueError, match="expects a CompositeModelHandler"):
+            p.run(onnx_model, str(tmp_path / "out"))
+
+    def test_raises_when_entry_component_file_missing(self, tmp_path):
+        src_root = _make_nested_genai_package(tmp_path / "src", {"decoder": "decoder/model.onnx"})
+        composite = _make_composite_handler(src_root, {"decoder": "decoder/model.onnx"})
+        # Remove the referenced component file so the config points at a missing file
+        # (after building the handler, whose components assert file existence).
+        (src_root / "decoder" / "model.onnx").unlink()
+
+        p = create_pass_from_dict(CompositeToOnnxPackage, {}, disable_search=True)
+        with pytest.raises(ValueError, match="not found"):
+            p.run(composite, str(tmp_path / "out"))
+
+    def test_rejects_entry_component_outside_package(self, tmp_path):
+        src_root = _make_nested_genai_package(tmp_path / "src", {"decoder": "../outside.onnx"})
+        composite = _make_composite_handler(src_root, {"decoder": "../outside.onnx"})
+
+        p = create_pass_from_dict(CompositeToOnnxPackage, {}, disable_search=True)
+        with pytest.raises(ValueError, match="escapes the ORT-GenAI package directory"):
+            p.run(composite, str(tmp_path / "out"))
diff --git a/test/test_cache.py b/test/test_cache.py
index 4063ff1554..1a5ae58121 100644
--- a/test/test_cache.py
+++ b/test/test_cache.py
@@ -350,6 +350,169 @@ def test_save_model_with_custom_onnx_filename(self, tmp_path):
         with open(output_json_path) as f:
             assert expected_output_path == json.load(f)["config"]["model_path"]
 
+    @pytest.mark.parametrize("model_attributes", [None, {"ort_genai_package": True}])
+    def test_save_model_preserves_ort_genai_package_layout(self, tmp_path, model_attributes):
+        model_id = "ort_genai_package"
+        cache = CacheConfig(cache_dir=tmp_path / "cache").create_cache()
+
+        source_dir = tmp_path / "source_package"
+        decoder_dir = source_dir / "decoder"
+        vision_dir = source_dir / "vision_encoder"
+        decoder_dir.mkdir(parents=True)
+        vision_dir.mkdir()
+        (source_dir / "genai_config.json").write_text("{}", encoding="utf-8")
+        (decoder_dir / "model.onnx").write_text("decoder", encoding="utf-8")
+        (vision_dir / "model.onnx").write_text("vision", encoding="utf-8")
+
+        model_config = {
+            "model_path": str(source_dir),
+            "onnx_file_name": "decoder/model.onnx",
+        }
+        if model_attributes:
+            model_config["model_attributes"] = model_attributes
+        model_json = {
+            "type": "onnxmodel",
+            "config": model_config,
+        }
+        with cache.get_model_json_path(model_id).open("w") as f:
+            json.dump(model_json, f)
+
+        output_dir = tmp_path / "output_package"
+        output_json = cache.save_model(model_id, output_dir, True)
+
+        assert output_json["config"]["model_path"] == str(output_dir.resolve())
+        assert output_json["config"]["onnx_file_name"] == "decoder/model.onnx"
+        assert (output_dir / "genai_config.json").read_text(encoding="utf-8") == "{}"
+        assert (output_dir / "decoder" / "model.onnx").read_text(encoding="utf-8") == "decoder"
+        assert (output_dir / "vision_encoder" / "model.onnx").read_text(encoding="utf-8") == "vision"
+
+    def test_save_model_preserves_file_backed_ort_genai_package_layout(self, tmp_path):
+        model_id = "ort_genai_package"
+        cache = CacheConfig(cache_dir=tmp_path / "cache").create_cache()
+
+        source_dir = tmp_path / "source_package"
+        decoder_dir = source_dir / "decoder"
+        vision_dir = source_dir / "vision_encoder"
+        decoder_dir.mkdir(parents=True)
+        vision_dir.mkdir()
+        (source_dir / "genai_config.json").write_text("{}", encoding="utf-8")
+        decoder_path = decoder_dir / "model.onnx"
+        decoder_path.write_text("decoder", encoding="utf-8")
+        (vision_dir / "model.onnx").write_text("vision", encoding="utf-8")
+        model_json = {
+            "type": "onnxmodel",
+            "config": {"model_path": str(decoder_path), "model_attributes": {"ort_genai_package": True}},
+        }
+        with cache.get_model_json_path(model_id).open("w") as f:
+            json.dump(model_json, f)
+
+        output_dir = tmp_path / "output_package"
+        output_json = cache.save_model(model_id, output_dir, True)
+
+        assert output_json["config"]["model_path"] == str(output_dir.resolve())
+        assert output_json["config"]["onnx_file_name"] == "decoder/model.onnx"
+        assert (output_dir / "genai_config.json").is_file()
+        assert (output_dir / "decoder" / "model.onnx").read_text(encoding="utf-8") == "decoder"
+        assert (output_dir / "vision_encoder" / "model.onnx").read_text(encoding="utf-8") == "vision"
+
+    @pytest.mark.parametrize("onnx_file_name", [None, "model.onnx"])
+    def test_save_model_preserves_nested_directory_backed_ort_genai_package_layout(self, tmp_path, onnx_file_name):
+        model_id = "ort_genai_package"
+        cache = CacheConfig(cache_dir=tmp_path / "cache").create_cache()
+
+        source_dir = tmp_path / "source_package"
+        decoder_dir = source_dir / "decoder"
+        vision_dir = source_dir / "vision_encoder"
+        decoder_dir.mkdir(parents=True)
+        vision_dir.mkdir()
+        (source_dir / "genai_config.json").write_text("{}", encoding="utf-8")
+        (decoder_dir / "model.onnx").write_text("decoder", encoding="utf-8")
+        (vision_dir / "model.onnx").write_text("vision", encoding="utf-8")
+        model_config = {"model_path": str(decoder_dir)}
+        if onnx_file_name is not None:
+            model_config["onnx_file_name"] = onnx_file_name
+        model_json = {"type": "onnxmodel", "config": model_config}
+        with cache.get_model_json_path(model_id).open("w") as f:
+            json.dump(model_json, f)
+
+        output_dir = tmp_path / "output_package"
+        output_json = cache.save_model(model_id, output_dir, True)
+
+        assert output_json["config"]["model_path"] == str(output_dir.resolve())
+        assert output_json["config"]["onnx_file_name"] == "decoder/model.onnx"
+        assert (output_dir / "genai_config.json").is_file()
+        assert (output_dir / "decoder" / "model.onnx").read_text(encoding="utf-8") == "decoder"
+        assert (output_dir / "vision_encoder" / "model.onnx").read_text(encoding="utf-8") == "vision"
+
+    def test_save_model_honors_overwrite_for_ort_genai_package(self, tmp_path):
+        model_id = "ort_genai_package"
+        cache = CacheConfig(cache_dir=tmp_path / "cache").create_cache()
+        source_dir = tmp_path / "source_package"
+        decoder_dir = source_dir / "decoder"
+        decoder_dir.mkdir(parents=True)
+        (source_dir / "genai_config.json").write_text("{}", encoding="utf-8")
+        (decoder_dir / "model.onnx").write_text("decoder", encoding="utf-8")
+        model_json = {
+            "type": "onnxmodel",
+            "config": {"model_path": str(source_dir), "onnx_file_name": "decoder/model.onnx"},
+        }
+        with cache.get_model_json_path(model_id).open("w") as f:
+            json.dump(model_json, f)
+
+        output_dir = tmp_path / "output_package"
+        output_dir.mkdir()
+        stale_file = output_dir / "stale.txt"
+        stale_file.write_text("stale", encoding="utf-8")
+
+        with pytest.raises(FileExistsError, match="Output directory is not empty"):
+            cache.save_model(model_id, output_dir, overwrite=False)
+
+        cache.save_model(model_id, output_dir, overwrite=True)
+        assert not stale_file.exists()
+        assert (output_dir / "genai_config.json").is_file()
+
+    def test_save_model_rejects_ort_genai_package_file_output(self, tmp_path):
+        model_id = "ort_genai_package"
+        cache = CacheConfig(cache_dir=tmp_path / "cache").create_cache()
+        source_dir = tmp_path / "source_package"
+        source_dir.mkdir()
+        (source_dir / "genai_config.json").write_text("{}", encoding="utf-8")
+        model_json = {
+            "type": "onnxmodel",
+            "config": {
+                "model_path": str(source_dir),
+                "onnx_file_name": "decoder/model.onnx",
+                "model_attributes": {"ort_genai_package": True},
+            },
+        }
+        with cache.get_model_json_path(model_id).open("w") as f:
+            json.dump(model_json, f)
+
+        with pytest.raises(ValueError, match="must be saved to a directory"):
+            cache.save_model(model_id, tmp_path / "output.onnx", True)
+
+    @pytest.mark.parametrize("output_relation", ["inside", "contains"])
+    def test_save_model_rejects_overlapping_ort_genai_package_paths(self, tmp_path, output_relation):
+        model_id = "ort_genai_package"
+        cache = CacheConfig(cache_dir=tmp_path / "cache").create_cache()
+        container_dir = tmp_path / "container"
+        source_dir = container_dir / "source_package"
+        source_dir.mkdir(parents=True)
+        (source_dir / "genai_config.json").write_text("{}", encoding="utf-8")
+        model_json = {
+            "type": "onnxmodel",
+            "config": {"model_path": str(source_dir), "onnx_file_name": "decoder/model.onnx"},
+        }
+        with cache.get_model_json_path(model_id).open("w") as f:
+            json.dump(model_json, f)
+
+        output_dir = source_dir / "output" if output_relation == "inside" else container_dir
+        with pytest.raises(ValueError, match="must not overlap"):
+            cache.save_model(model_id, output_dir, overwrite=True)
+
+        assert source_dir.is_dir()
+        assert (source_dir / "genai_config.json").is_file()
+
     def test_save_model_no_flatten_rebases_component_and_additional_file_paths(self, tmp_path):
         # setup
         model_id = "composite_model"

From 7008f22ab0e951bccfc63d6c5b16f674ba8fa804 Mon Sep 17 00:00:00 2001
From: Delwin Kim <139003345+DelwinKim@users.noreply.github.com>
Date: Thu, 6 Aug 2026 23:58:16 +0000
Subject: [PATCH 2/4] Add lmms multimodal evaluator

Integrate upstream lmms-eval generation tasks with Hugging Face wrappers and deployed ORT-GenAI image/audio packages, including custom task paths and result persistence.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 olive/evaluator/lmms_ort.py            | 1055 +++++++++++++
 olive/evaluator/olive_evaluator.py     |  536 +++++++
 test/evaluator/test_lmms_ort.py        | 1921 ++++++++++++++++++++++++
 test/evaluator/test_olive_evaluator.py |   18 +
 test/requirements-test.txt             |    2 +
 5 files changed, 3532 insertions(+)
 create mode 100644 olive/evaluator/lmms_ort.py
 create mode 100644 test/evaluator/test_lmms_ort.py

diff --git a/olive/evaluator/lmms_ort.py b/olive/evaluator/lmms_ort.py
new file mode 100644
index 0000000000..6057af99b6
--- /dev/null
+++ b/olive/evaluator/lmms_ort.py
@@ -0,0 +1,1055 @@
+# -------------------------------------------------------------------------
+# ORT-GenAI wrapper for lmms-eval multimodal evaluation
+#
+# Supports evaluating multimodal ONNX
+# models through the EvolvingLMMs-Lab/lmms-eval harness, mirroring how
+# olive/evaluator/lmeval_ort.py wraps lm-evaluation-harness for text models.
+#
+# Registers an LMMSORTGenAIEvaluator class with lmms-eval's legacy
+# @register_model registry under the name "ortgenai_mm". Consumers obtain it
+# via lmms_eval.api.registry.get_model("ortgenai_mm").
+# -------------------------------------------------------------------------
+"""ORT-GenAI wrapper for lmms-eval multimodal evaluation."""
+
+from __future__ import annotations
+
+import io
+import json
+import logging
+import math
+import re
+import tempfile
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+import PIL.Image
+from tqdm import tqdm
+
+try:
+    import onnxruntime_genai as og
+except ImportError:  # pragma: no cover - optional dep
+    og = None
+
+try:
+    from lmms_eval.api.instance import Instance
+    from lmms_eval.api.model import lmms
+    from lmms_eval.api.registry import register_model
+
+    _LMMS_EVAL_IMPORT_ERROR = None
+except ImportError as e:  # pragma: no cover - optional dep
+    Instance = Any
+    _LMMS_EVAL_IMPORT_ERROR = e
+
+    class lmms:  # noqa: N801
+        pass
+
+    def register_model(_name):
+        def decorator(cls):
+            return cls
+
+        return decorator
+
+
+logger = logging.getLogger(__name__)
+
+
+@lru_cache(maxsize=1)
+def warmup_image_decoder() -> None:
+    """Force ORT-GenAI's image decoder to initialize once, early in the process.
+
+    ORT-GenAI's ``ImageDecoder`` statically links its own zlib. Pillow, torch,
+    ``decord`` and other libraries loaded by the HuggingFace evaluation path also
+    bundle their own zlib. Because these are static copies of the same symbols
+    (an ODR violation), the *first* library to actually decode an image binds its
+    zlib process-wide. When ``LMMSEvaluator`` runs an HF input-model evaluation
+    before the ORT-GenAI (ONNX) evaluation in the same process
+    (``evaluate_input_model: true``), the HF path decodes images first, so
+    ORT-GenAI's PNG decoder later binds the wrong zlib and fails with
+    ``libpng error: bad parameters to zlib`` on every image.
+
+    Decoding a tiny throwaway PNG through ORT-GenAI here — before the HF path
+    runs — makes ORT-GenAI's zlib win the binding, so the later ONNX evaluation
+    decodes correctly. Idempotent and best-effort: a no-op if onnxruntime-genai
+    is unavailable or the warmup fails (nothing is broken for HF-only usage).
+    """
+    if og is None:
+        return
+    try:
+        with tempfile.TemporaryDirectory() as tmp:
+            path = Path(tmp) / "warmup.png"
+            PIL.Image.new("RGB", (8, 8), (0, 0, 0)).save(path, format="PNG")
+            # og.Images.open touches ORT-GenAI's libpng/zlib, binding its symbols.
+            og.Images.open(str(path))
+    except Exception as e:  # pragma: no cover - defensive, must never break a run
+        logger.debug("ORT-GenAI image decoder warmup skipped: %s", e)
+
+
+_PROVIDER_ALIASES = {
+    "cuda": "cuda",
+    "cudaexecutionprovider": "cuda",
+    "gpu": "cuda",
+    "cpu": "cpu",
+    "cpuexecutionprovider": "cpu",
+    "dml": "dml",
+    "dmlexecutionprovider": "dml",
+    "directml": "dml",
+    "webgpu": "webgpu",
+    "webgpuexecutionprovider": "webgpu",
+    "js": "web",
+    "jsexecutionprovider": "web",
+    "nvtensorrtrtx": "NvTensorRtRtx",
+    "nvtensorrtrtxexecutionprovider": "NvTensorRtRtx",
+}
+
+
+# -----------------------------------------------------------------------------
+# Helpers
+# -----------------------------------------------------------------------------
+
+
+_IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".webp", ".bmp", ".gif"}
+_AUDIO_SUFFIXES = {".wav", ".mp3", ".flac", ".ogg", ".m4a"}
+_VIDEO_SUFFIXES = {".mp4", ".avi", ".mov", ".mkv", ".webm"}
+_MEDIA_PLACEHOLDER_RE = re.compile(r"(<image>|<audio>)")
+_DEFAULT_SPEECH_CONFIG_FILENAME = "audio_processor_config.json"
+_SAMPLE_RATE_KEYS = {"sample_rate", "sampling_rate", "target_sample_rate"}
+_MULTI_SAMPLE_RATE_KEYS = {"target_sample_rates"}
+_IMAGE_SERIALIZATION_PROFILES = {"lossless", "jpeg85"}
+
+
+def _normalize_image(visual) -> PIL.Image.Image | None:
+    """Coerce an lmms-eval visual into an RGB PIL image, or None if it isn't an image.
+
+    Accepts PIL images, file paths, ``{"bytes"|"path"}`` dicts, and numpy arrays;
+    audio-shaped dicts (with ``array``/``sampling_rate``) are rejected here.
+    """
+    if isinstance(visual, PIL.Image.Image):
+        return visual.convert("RGB")
+    if isinstance(visual, (str, Path)):
+        p = Path(visual)
+        if p.suffix.lower() in _IMAGE_SUFFIXES:
+            return PIL.Image.open(p).convert("RGB")
+        return None
+    if isinstance(visual, dict):
+        # Audio dicts typically include "sampling_rate" or "array"; skip those.
+        if "sampling_rate" in visual or "array" in visual:
+            return None
+        if "bytes" in visual:
+            return PIL.Image.open(io.BytesIO(visual["bytes"])).convert("RGB")
+        if "path" in visual:
+            p = Path(visual["path"]) if visual["path"] else None
+            if p is not None and p.suffix.lower() in _IMAGE_SUFFIXES:
+                return PIL.Image.open(p).convert("RGB")
+            return None
+    if isinstance(visual, np.ndarray):
+        return PIL.Image.fromarray(np.uint8(visual)).convert("RGB")
+    return None
+
+
+def _normalize_audio(visual) -> tuple[np.ndarray, int] | None:
+    """Coerce an lmms-eval visual into ``(mono float32 waveform, sample_rate)``, or None.
+
+    Handles the legacy ``{"array", "sampling_rate"}`` dict, file paths, and
+    HF datasets 5.x ``torchcodec`` AudioDecoder objects (duck-typed, downmixed to mono).
+    """
+    if isinstance(visual, dict):
+        if "array" in visual and "sampling_rate" in visual:
+            return _normalize_audio_array(visual["array"]), int(visual["sampling_rate"])
+        if visual.get("path"):
+            return _load_audio_file(Path(visual["path"]))
+    if isinstance(visual, (str, Path)):
+        return _load_audio_file(Path(visual))
+    # torchcodec.decoders.AudioDecoder — HF datasets 5.x returns this for the
+    # "audio" feature instead of the legacy {"array", "sampling_rate"} dict.
+    # Detect by duck-typing the get_all_samples() method to avoid a hard
+    # torchcodec import (it's an optional install).
+    if hasattr(visual, "get_all_samples"):
+        try:
+            samples = visual.get_all_samples()
+            # samples.data is a torch.Tensor of shape [channels, num_samples].
+            # ORT-GenAI's processor wants mono float32; downmix if multichannel.
+            return _normalize_audio_array(samples.data.detach().cpu().numpy()), int(samples.sample_rate)
+        except RuntimeError as torchcodec_error:
+            encoded = getattr(visual, "_hf_encoded", None)
+            if not isinstance(encoded, dict):
+                raise RuntimeError(
+                    "TorchCodec failed to decode audio and no encoded source is available."
+                ) from torchcodec_error
+
+            source = io.BytesIO(encoded["bytes"]) if encoded.get("bytes") is not None else encoded.get("path")
+            if source is None:
+                raise RuntimeError(
+                    "TorchCodec failed to decode audio and its encoded source is empty."
+                ) from torchcodec_error
+
+            try:
+                import librosa
+
+                desired_sr = getattr(visual, "_desired_sample_rate", None)
+                array, sample_rate = librosa.load(source, sr=desired_sr, mono=True)
+            except (ImportError, OSError, ValueError) as fallback_error:
+                raise RuntimeError("Both TorchCodec and librosa failed to decode audio.") from fallback_error
+
+            logger.warning("TorchCodec failed to decode audio; recovered the encoded source with librosa.")
+            return _normalize_audio_array(array), int(sample_rate)
+    return None
+
+
+def _normalize_audio_array(array) -> np.ndarray:
+    """Return a contiguous mono float32 waveform from common audio layouts."""
+    arr = np.asarray(array, dtype=np.float32)
+    if arr.ndim == 2:
+        if arr.shape[0] <= 8:
+            arr = arr.mean(axis=0)
+        elif arr.shape[1] <= 8:
+            arr = arr.mean(axis=1)
+        else:
+            raise ValueError(f"Cannot infer channel axis for audio array with shape {arr.shape}.")
+    if arr.ndim != 1:
+        raise ValueError(f"Audio array must be one- or two-dimensional, got shape {arr.shape}.")
+    return np.ascontiguousarray(arr, dtype=np.float32)
+
+
+def _resample_audio(array: np.ndarray, source_rate: int, target_rate: int) -> np.ndarray:
+    """Resample a mono waveform once with a polyphase anti-aliasing filter."""
+    if source_rate <= 0 or target_rate <= 0:
+        raise ValueError(f"Audio sample rates must be positive, got source={source_rate}, target={target_rate}.")
+    if source_rate == target_rate:
+        return array
+
+    from scipy.signal import resample_poly
+
+    divisor = math.gcd(source_rate, target_rate)
+    resampled = resample_poly(array, target_rate // divisor, source_rate // divisor)
+    return np.ascontiguousarray(resampled, dtype=np.float32)
+
+
+def _collect_sample_rate_declarations(value: Any, path: str = "$") -> tuple[list[tuple[str, int]], list[str]]:
+    """Recursively collect valid and malformed sample-rate declarations."""
+    declarations: list[tuple[str, int]] = []
+    malformed: list[str] = []
+    if isinstance(value, dict):
+        for key, child in value.items():
+            child_path = f"{path}.{key}"
+            if key in _SAMPLE_RATE_KEYS:
+                if (
+                    isinstance(child, bool)
+                    or not isinstance(child, (int, float))
+                    or not float(child).is_integer()
+                    or child <= 0
+                ):
+                    malformed.append(f"{child_path}={child!r}")
+                else:
+                    declarations.append((child_path, int(child)))
+            elif key in _MULTI_SAMPLE_RATE_KEYS:
+                if not isinstance(child, list) or not child:
+                    malformed.append(f"{child_path}={child!r}")
+                else:
+                    for index, sample_rate in enumerate(child):
+                        sample_rate_path = f"{child_path}[{index}]"
+                        if (
+                            isinstance(sample_rate, bool)
+                            or not isinstance(sample_rate, (int, float))
+                            or not float(sample_rate).is_integer()
+                            or sample_rate <= 0
+                        ):
+                            malformed.append(f"{sample_rate_path}={sample_rate!r}")
+                        else:
+                            declarations.append((sample_rate_path, int(sample_rate)))
+                continue
+            child_declarations, child_malformed = _collect_sample_rate_declarations(child, child_path)
+            declarations.extend(child_declarations)
+            malformed.extend(child_malformed)
+    elif isinstance(value, list):
+        for index, child in enumerate(value):
+            child_declarations, child_malformed = _collect_sample_rate_declarations(child, f"{path}[{index}]")
+            declarations.extend(child_declarations)
+            malformed.extend(child_malformed)
+    return declarations, malformed
+
+
+def _has_audio_decoder(value: Any) -> bool:
+    """Return whether a processor config contains the standard AudioDecoder operation."""
+    if isinstance(value, dict):
+        if value.get("type") == "AudioDecoder":
+            return True
+        return any(_has_audio_decoder(child) for child in value.values())
+    if isinstance(value, list):
+        return any(_has_audio_decoder(child) for child in value)
+    return False
+
+
+def _resolve_model_audio_sample_rates(model_dir: Path, genai_config: dict[str, Any]) -> tuple[int, ...]:
+    """Resolve the target sample rates accepted by the package's speech processor."""
+    model_config = genai_config.get("model")
+    if not isinstance(model_config, dict):
+        raise ValueError("genai_config.json must contain a 'model' object.")
+    speech_config = model_config.get("speech")
+    is_whisper = str(model_config.get("type", "")).lower() == "whisper"
+    if not isinstance(speech_config, dict) and not is_whisper:
+        raise ValueError("genai_config.json does not configure model.speech for audio input.")
+
+    config_filename = (
+        speech_config.get("config_filename", _DEFAULT_SPEECH_CONFIG_FILENAME)
+        if isinstance(speech_config, dict)
+        else _DEFAULT_SPEECH_CONFIG_FILENAME
+    )
+    if not isinstance(config_filename, str) or not config_filename.strip():
+        raise ValueError("genai_config.json model.speech.config_filename must be a non-empty string.")
+    config_path = model_dir / config_filename
+    if not config_path.is_file():
+        raise ValueError(f"Speech processor config does not exist: {config_path}")
+    try:
+        speech_processor_config = json.loads(config_path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as e:
+        raise ValueError(f"Invalid speech processor config JSON: {config_path}") from e
+
+    declarations, malformed = _collect_sample_rate_declarations(speech_processor_config)
+    if malformed:
+        raise ValueError(
+            f"Speech processor config {config_path} has malformed sample-rate declarations: {', '.join(malformed)}"
+        )
+    if not declarations:
+        if is_whisper and _has_audio_decoder(speech_processor_config):
+            return (16000,)
+        raise ValueError(f"Speech processor config {config_path} does not declare a positive sample rate.")
+
+    scalar_declarations = [(path, rate) for path, rate in declarations if ".target_sample_rates[" not in path]
+    supported_declarations = [(path, rate) for path, rate in declarations if ".target_sample_rates[" in path]
+    scalar_rates = {rate for _, rate in scalar_declarations}
+    if len(scalar_rates) > 1:
+        details = ", ".join(f"{path}={rate}" for path, rate in declarations)
+        raise ValueError(f"Speech processor config {config_path} has conflicting sample rates: {details}")
+
+    supported_rates = {rate for _, rate in supported_declarations}
+    if scalar_rates:
+        scalar_rate = next(iter(scalar_rates))
+        if supported_rates and scalar_rate not in supported_rates:
+            details = ", ".join(f"{path}={rate}" for path, rate in declarations)
+            raise ValueError(f"Speech processor config {config_path} has conflicting sample rates: {details}")
+        return (scalar_rate,)
+    return tuple(sorted(supported_rates))
+
+
+def _resolve_model_audio_sample_rate(model_dir: Path, genai_config: dict[str, Any]) -> int:
+    """Resolve the preferred (highest supported) package audio sample rate."""
+    return _resolve_model_audio_sample_rates(model_dir, genai_config)[-1]
+
+
+def _load_audio_file(p: Path) -> tuple[np.ndarray, int] | None:
+    """Load an audio file to ``(mono float32 waveform, native sample_rate)`` via librosa.
+
+    Returns None if the extension is unsupported, the file is missing, or librosa isn't installed.
+    """
+    if p.suffix.lower() not in _AUDIO_SUFFIXES or not p.exists():
+        return None
+    try:
+        import librosa
+    except ImportError:
+        logger.warning("Audio file %s encountered but librosa not installed.", p)
+        return None
+    arr, sr = librosa.load(str(p), sr=None, mono=True)
+    return arr.astype(np.float32), int(sr)
+
+
+def _partition_visuals(visuals):
+    """Split a mixed list of visuals into ``(images, audios)``.
+
+    Audio is tested first (its ``array``+``sampling_rate`` signature is more distinctive
+    than the various image forms).
+    """
+    images, audios = [], []
+    for v in visuals or []:
+        if v is None:
+            continue
+        # Try audio first since its signature ("array"+"sampling_rate") is more
+        # distinctive than the image path/bytes/PIL signatures.
+        au = _normalize_audio(v)
+        if au is not None:
+            audios.append(au)
+            continue
+        img = _normalize_image(v)
+        if img is not None:
+            images.append(img)
+            continue
+        suffix = Path(v).suffix.lower() if isinstance(v, (str, Path)) else ""
+        if suffix in _VIDEO_SUFFIXES:
+            raise NotImplementedError("LMMSORTGenAIEvaluator does not support video inputs.")
+        raise ValueError(f"Unsupported lmms-eval visual input type: {type(v).__name__}.")
+    return images, audios
+
+
+def _generation_options(gen_kwargs: dict[str, Any]) -> tuple[dict[str, Any], list[str] | None]:
+    """Translate lmms-eval generation kwargs to ORT-GenAI search and stop options."""
+    if gen_kwargs.get("num_beams") not in (None, 1):
+        raise NotImplementedError("Beam search is not supported by LMMSORTGenAIEvaluator.")
+    if gen_kwargs.get("num_return_sequences") not in (None, 1):
+        raise NotImplementedError("Multiple return sequences are not supported by LMMSORTGenAIEvaluator.")
+    if gen_kwargs.get("no_repeat_ngram_size") not in (None, 0):
+        raise NotImplementedError("no_repeat_ngram_size is not supported by ORT-GenAI.")
+    if gen_kwargs.get("max_agentic_steps") is not None or gen_kwargs.get("visual_cot"):
+        raise NotImplementedError("Agentic and visual-CoT generation are not supported by LMMSORTGenAIEvaluator.")
+
+    search_options: dict[str, Any] = {"do_sample": bool(gen_kwargs.get("do_sample", False))}
+    for name in ("temperature", "top_p", "top_k", "repetition_penalty"):
+        value = gen_kwargs.get(name)
+        if value is not None:
+            search_options[name] = value
+
+    stop = gen_kwargs.get("until")
+    if isinstance(stop, str):
+        stop = [stop]
+    elif stop is not None:
+        stop = list(stop)
+    extra_stops = gen_kwargs.get("stop_sequences")
+    if isinstance(extra_stops, str):
+        extra_stops = [extra_stops]
+    if extra_stops:
+        stop = [*(stop or []), *extra_stops]
+    return search_options, stop or None
+
+
+def _build_whisper_prompt(
+    whisper_language: str = "en",
+    whisper_task: str = "transcribe",
+    whisper_timestamps: bool = False,
+) -> str:
+    """Build Whisper's decoder-start token sequence."""
+    if whisper_task not in ("transcribe", "translate"):
+        raise ValueError("whisper_task must be 'transcribe' or 'translate'.")
+    timestamp_token = "" if whisper_timestamps else "<|notimestamps|>"
+    return f"<|startoftranscript|><|{whisper_language}|><|{whisper_task}|>{timestamp_token}"
+
+
+def _normalize_execution_provider(execution_provider: Any | None) -> str:
+    """Normalize an EP spec to a canonical ORT-GenAI provider name.
+
+    Accepts a string or list/tuple (first entry used); empty input returns
+    ``"follow_config"``, meaning "use the providers already in genai_config.json".
+    """
+    if not execution_provider:
+        return "follow_config"
+    if isinstance(execution_provider, (tuple, list)):
+        execution_provider = execution_provider[0] if execution_provider else None
+    if not execution_provider:
+        return "follow_config"
+    ep = str(execution_provider).lower().replace("_", "")
+    return _PROVIDER_ALIASES.get(ep, str(execution_provider))
+
+
+# -----------------------------------------------------------------------------
+# Main adapter
+# -----------------------------------------------------------------------------
+
+
+@register_model("ortgenai_mm")
+class LMMSORTGenAIEvaluator(lmms):
+    r"""lmms-eval model wrapper for an ORT-GenAI multimodal package (model).
+
+    Implements the base lmms class interface for generate_until using ORT-GenAI's
+    multimodal processor and generator. Image and audio inputs are rendered
+    through the model's native structured-content chat template.
+    """
+
+    is_simple = True
+
+    def __init__(
+        self,
+        pretrained: str,
+        batch_size: int = 1,
+        max_new_tokens: int = 256,
+        max_length: int = 32768,
+        system_prompt: str | None = None,
+        execution_provider: str | None = None,
+        provider_options: dict | None = None,
+        fail_on_error: bool = True,
+        ignore_stop_strings: list[str] | None = None,
+        whisper_language: str = "en",
+        whisper_task: str = "transcribe",
+        whisper_timestamps: bool = False,
+        audio_target_sample_rate: int | None = None,
+        image_serialization_profile: str = "lossless",
+        **kwargs,
+    ) -> None:
+        if _LMMS_EVAL_IMPORT_ERROR is not None:
+            raise ImportError(
+                "lmms-eval is required for ortgenai_mm. Install lmms-eval before using LMMSEvaluator."
+            ) from _LMMS_EVAL_IMPORT_ERROR
+        if og is None:
+            raise ImportError(
+                "onnxruntime-genai is required for ortgenai_mm. "
+                "Install with: pip install onnxruntime-genai (or -cuda variant)."
+            )
+        super().__init__()
+        if kwargs:
+            logger.warning("Unused kwargs: %s", kwargs)
+
+        model_dir = Path(pretrained).resolve()
+        if not model_dir.is_dir():
+            raise ValueError(f"ORT-GenAI model directory does not exist: {model_dir}")
+        if not (model_dir / "genai_config.json").is_file():
+            raise ValueError(f"LMMSEvaluator requires genai_config.json in ORT-GenAI package: {model_dir}")
+        if int(batch_size) < 1:
+            raise ValueError("batch_size must be >= 1")
+        if int(max_new_tokens) < 1:
+            raise ValueError("max_new_tokens must be >= 1")
+        if int(max_length) < 1:
+            raise ValueError("max_length must be >= 1")
+
+        self.model_dir = str(model_dir)
+        self.max_new_tokens = int(max_new_tokens)
+        self.max_length = int(max_length)
+        self.batch_size_per_gpu = int(batch_size)
+        self.system_prompt = system_prompt
+        self.fail_on_error = fail_on_error
+        self.whisper_language = whisper_language
+        self.whisper_task = whisper_task
+        self.whisper_timestamps = bool(whisper_timestamps)
+        if audio_target_sample_rate is not None and (
+            isinstance(audio_target_sample_rate, bool)
+            or not isinstance(audio_target_sample_rate, int)
+            or audio_target_sample_rate <= 0
+        ):
+            raise ValueError("audio_target_sample_rate must be a positive integer.")
+        self._audio_target_sample_rate_explicit = audio_target_sample_rate is not None
+        self.audio_target_sample_rate = audio_target_sample_rate
+        self.audio_target_sample_rates = (audio_target_sample_rate,) if audio_target_sample_rate is not None else ()
+        self.image_serialization_profile = str(image_serialization_profile).lower()
+        if self.image_serialization_profile not in _IMAGE_SERIALIZATION_PROFILES:
+            raise ValueError(
+                f"image_serialization_profile must be one of {sorted(_IMAGE_SERIALIZATION_PROFILES)}, "
+                f"got {image_serialization_profile!r}."
+            )
+        # Stop strings to suppress from each request's lmms-eval ``until`` list.
+        # lmms-eval injects ``until=["\n\n"]`` (the fewshot_delimiter) for tasks
+        # that don't set their own ``until`` (lmms_eval/api/task.py). For
+        # step-by-step reasoning that double-newline truncates the model after its
+        # first paragraph, before it reaches the final answer. Listing "\n\n" here
+        # drops only that spurious stop; the model still stops normally on its
+        # EOS / end-of-turn token (handled in _run_generation).
+        self.ignore_stop_strings = set(ignore_stop_strings or [])
+
+        try:
+            cfg = json.loads((Path(self.model_dir) / "genai_config.json").read_text(encoding="utf-8"))
+        except json.JSONDecodeError as e:
+            raise ValueError(f"Invalid genai_config.json in {self.model_dir}") from e
+        model_config = cfg.get("model")
+        self._model_type = model_config.get("type", "phi4mm") if isinstance(model_config, dict) else "phi4mm"
+        self._audio_sample_rate_error: ValueError | None = None
+        if self.audio_target_sample_rate is None:
+            try:
+                self.audio_target_sample_rates = _resolve_model_audio_sample_rates(model_dir, cfg)
+                self.audio_target_sample_rate = self.audio_target_sample_rates[-1]
+            except ValueError as e:
+                self._audio_sample_rate_error = e
+        else:
+            logger.info("Using explicit audio_target_sample_rate=%d.", self.audio_target_sample_rate)
+        self._image_serialization_format = self._resolve_image_serialization_format()
+        logger.info(
+            "Using image serialization profile '%s' (%s).",
+            self.image_serialization_profile,
+            self._image_serialization_format,
+        )
+
+        logger.info("Loading ORT-GenAI model from: %s", self.model_dir)
+        ep = _normalize_execution_provider(execution_provider)
+        # CUDA GenAI packages often carry provider-specific options in genai_config.json.
+        # Clearing/re-adding CUDA can drop those options and fail to load on otherwise
+        # working packages, so follow the package config unless options are overridden.
+        if ep == "follow_config" or (ep == "cuda" and not provider_options):
+            self._model = og.Model(self.model_dir)
+        else:
+            config = og.Config(self.model_dir)
+            config.clear_providers()
+            if ep != "cpu":
+                config.append_provider(ep)
+            for key, value in (provider_options or {}).items():
+                config.set_provider_option(ep, key, value)
+            self._model = og.Model(config)
+        self._tokenizer = og.Tokenizer(self._model)
+        self._processor = self._model.create_multimodal_processor()
+
+        eos_ids = self._tokenizer.eos_token_ids
+        self._eos_token_ids = {int(t) for t in (eos_ids if eos_ids is not None else [])}
+        self._gemma_channel_start_token_id: int | None = None
+        self._gemma_channel_end_token_id: int | None = None
+        self._uses_gemma_response_channels = "gemma4" in self._model_type.lower()
+        if self._uses_gemma_response_channels:
+            tokenizer_config_path = Path(self.model_dir) / "tokenizer_config.json"
+            try:
+                tokenizer_config = json.loads(tokenizer_config_path.read_text(encoding="utf-8"))
+                self._gemma_channel_start_token_id = self._single_token_id(tokenizer_config.get("soc_token"))
+                self._gemma_channel_end_token_id = self._single_token_id(tokenizer_config.get("eoc_token"))
+            except (OSError, json.JSONDecodeError) as e:
+                raise ValueError(f"Invalid tokenizer_config.json in {self.model_dir}") from e
+            if self._gemma_channel_start_token_id is None or self._gemma_channel_end_token_id is None:
+                raise ValueError(
+                    "Gemma 4 response channel tokens must each map to exactly one tokenizer ID: "
+                    f"soc_token={tokenizer_config.get('soc_token')!r}, "
+                    f"eoc_token={tokenizer_config.get('eoc_token')!r}."
+                )
+
+        if self._model_type != "whisper" and not hasattr(self._tokenizer, "apply_chat_template"):
+            raise RuntimeError(
+                "LMMSEvaluator requires og.Tokenizer.apply_chat_template for non-Whisper models. "
+                "Upgrade onnxruntime-genai or use the preserved legacy prompt-override branch."
+            )
+
+        # Resolve the default system prompt per model family (matching lmms-eval's
+        # per-model wrappers' default system prompt) when the user did not pass one explicitly. This keeps
+        # HF-input and ORT-GenAI evaluation prompts identical
+        if self.system_prompt is None:
+            self.system_prompt = self.default_system_prompt_for_model_type(self._model_type)
+            logger.info("Using default system prompt for model type '%s': %r", self._model_type, self.system_prompt)
+
+        # Probe once whether this model's chat template injects media tokens
+        # from structured content parts (``{"type": "image"}``). Well-behaved
+        # templates (Gemma-4, Qwen2.5-VL, Qwen3-VL) do (pretty much most); Phi-4-MM's template
+        # stringifies the content list as Python repr instead. When supported,
+        # the adapter lets the template emit the correct per-model media tokens
+        # automatically.
+        self._supports_structured_content = self._model_type != "whisper" and self._probe_structured_content_support()
+
+        self._rank = 0
+        self._world_size = 1
+        logger.info("Model loaded. Model type: %s", self._model_type)
+
+    def _single_token_id(self, token: str | None) -> int | None:
+        """Return the ID for a configured special token when it maps to one token."""
+        if not token:
+            return None
+        token_ids = list(self._tokenizer.encode(token))
+        return int(token_ids[0]) if len(token_ids) == 1 else None
+
+    def _decode_generated_tokens(self, token_ids: list[int]) -> str:
+        """Decode generated IDs, excluding Gemma's optional thought channel.
+
+        ORT-GenAI's tokenizer removes channel delimiters during both streaming and
+        full-sequence decoding. Inspecting token IDs before decode preserves the
+        response-schema boundary. An unfinished thought channel has no scoreable
+        response content and therefore decodes to an empty string.
+        """
+        if not token_ids:
+            return ""
+        if (
+            not self._uses_gemma_response_channels
+            or self._gemma_channel_start_token_id is None
+            or self._gemma_channel_start_token_id not in token_ids
+        ):
+            return self._tokenizer.decode(token_ids)
+
+        if self._gemma_channel_end_token_id is None or self._gemma_channel_end_token_id not in token_ids:
+            return ""
+        content_start = len(token_ids) - 1 - token_ids[::-1].index(self._gemma_channel_end_token_id) + 1
+        response_token_ids = token_ids[content_start:]
+        if not response_token_ids:
+            return ""
+        return self._tokenizer.decode(response_token_ids)
+
+    # -------------------------------------------------------------------------
+    # lmms-eval required properties
+    # -------------------------------------------------------------------------
+    @property
+    def batch_size(self):
+        return self.batch_size_per_gpu
+
+    @property
+    def rank(self):
+        return self._rank
+
+    @property
+    def world_size(self):
+        return self._world_size
+
+    # -------------------------------------------------------------------------
+    # ORT-GenAI input plumbing
+    # -------------------------------------------------------------------------
+    def _resolve_image_serialization_format(self) -> str:
+        if self.image_serialization_profile == "jpeg85":
+            return "JPEG quality=85"
+        return "PNG"
+
+    def _build_og_images(self, images, tmp_dir: Path):
+        """Serialize images according to the configured parity profile and open them with ORT-GenAI."""
+        if not images:
+            return None
+        paths = []
+        for i, img in enumerate(images):
+            if self._image_serialization_format.startswith("JPEG"):
+                path = tmp_dir / f"image_{i}.jpg"
+                img.save(path, format="JPEG", quality=85)
+            else:
+                path = tmp_dir / f"image_{i}.png"
+                img.save(path, format="PNG")
+            paths.append(str(path))
+        return og.Images.open(*paths)
+
+    def _build_og_audios(self, audios, tmp_dir: Path):
+        """Write WAVs for ORT-GenAI, host-resampling only for an explicit user override."""
+        if not audios:
+            return None
+        if self.audio_target_sample_rate is None:
+            detail = (
+                str(self._audio_sample_rate_error) if self._audio_sample_rate_error else "target rate is unavailable"
+            )
+            raise ValueError(
+                f"Cannot process audio because the model audio sample rate could not be resolved: {detail}"
+            )
+        import soundfile as sf
+
+        paths = []
+        for i, (arr, sr) in enumerate(audios):
+            if self._audio_target_sample_rate_explicit:
+                target_sample_rate = self.audio_target_sample_rate
+                serialized_audio = _resample_audio(arr, sr, target_sample_rate)
+                serialized_sample_rate = target_sample_rate
+            else:
+                self._select_audio_target_sample_rate(sr)
+                serialized_audio = arr
+                serialized_sample_rate = sr
+            path = tmp_dir / f"audio_{i}.wav"
+            sf.write(path, serialized_audio, serialized_sample_rate, subtype="FLOAT")
+            paths.append(str(path))
+        return og.Audios.open(*paths)
+
+    def _select_audio_target_sample_rate(self, source_sample_rate: int) -> int:
+        """Mirror AudioDecoderEx by selecting the largest supported rate not above the source rate."""
+        supported_rates = self.audio_target_sample_rates
+        if not supported_rates and self.audio_target_sample_rate is not None:
+            supported_rates = (self.audio_target_sample_rate,)
+        compatible_rates = [rate for rate in supported_rates if rate <= source_sample_rate]
+        if not compatible_rates:
+            raise ValueError(
+                "Audio source sample rate is below every package-supported target rate: "
+                f"source={source_sample_rate}, supported={list(supported_rates)}."
+            )
+        return compatible_rates[-1]
+
+    def _handle_error(self, message: str, exc: Exception, default):
+        """Re-raise as RuntimeError when ``fail_on_error`` is set; otherwise log and return ``default``."""
+        if self.fail_on_error:
+            raise RuntimeError(message) from exc
+        logger.error("%s: %s", message, exc)
+        return default
+
+    # -------------------------------------------------------------------------
+    # Single-request inference primitives
+    # -------------------------------------------------------------------------
+    @staticmethod
+    def _processed_input_length(inputs) -> int:
+        """Return the processor-expanded sequence length from ORT-GenAI inputs."""
+        try:
+            shape = inputs["input_ids"].shape()
+        except (KeyError, RuntimeError, TypeError, AttributeError) as e:
+            raise ValueError("ORT-GenAI multimodal processor output is missing a valid input_ids tensor.") from e
+        if not shape or int(shape[-1]) <= 0:
+            raise ValueError(f"ORT-GenAI multimodal processor returned an invalid input_ids shape: {shape}.")
+        return int(shape[-1])
+
+    def _run_generation(
+        self,
+        prompt: str,
+        images,
+        audios,
+        max_new_tokens: int,
+        stop_strings: list[str] | None = None,
+        search_options: dict[str, Any] | None = None,
+    ) -> str:
+        """Greedily decode a single multimodal request and return the generated text.
+
+        Feeds ``prompt`` + media through the ORT-GenAI processor, then decodes
+        token-by-token until EOS, the ``max_new_tokens`` budget, or a ``stop_strings``
+        match (early-truncated). Returns "" on processor/input failure unless
+        ``fail_on_error`` is set.
+        """
+        if max_new_tokens < 1:
+            error = ValueError(f"max_new_tokens must be >= 1, got {max_new_tokens}.")
+            return self._handle_error("Invalid ORT-GenAI completion budget.", error, "")
+
+        with tempfile.TemporaryDirectory() as tmp:
+            tmp_dir = Path(tmp)
+            try:
+                og_images = self._build_og_images(images, tmp_dir)
+                og_audios = self._build_og_audios(audios, tmp_dir)
+                # ORT-GenAI multimodal processors disagree on argument shape:
+                #   - Phi-4-MM expects a bare string. Passing [prompt] raises
+                #     "Number of image tokens does not match the number of images"
+                #     because the processor interprets the list as one prompt per
+                #     image (verified against pre-built phi4mm INT4 package).
+                #   - Whisper's processor (per ORT-GenAI's reference
+                #     benchmark_multimodal.py) is exercised with a list of
+                #     prompts.
+                # Branch on model type rather than guess.
+                processor_input = [prompt] if self._model_type == "whisper" else prompt
+                inputs = self._processor(processor_input, images=og_images, audios=og_audios)
+            except Exception as e:  # pragma: no cover
+                return self._handle_error("ORT-GenAI multimodal preprocessing failed.", e, "")
+
+            try:
+                input_length = self._processed_input_length(inputs)
+            except ValueError as e:
+                return self._handle_error("Failed to determine the processed prompt length.", e, "")
+            available_new_tokens = self.max_length - input_length
+            if available_new_tokens <= 0:
+                error = ValueError(
+                    f"Current processed prompt length {input_length} leaves no completion capacity within "
+                    f"max_length={self.max_length}."
+                )
+                return self._handle_error("ORT-GenAI processed prompt exceeds max_length.", error, "")
+            effective_new_tokens = min(max_new_tokens, available_new_tokens)
+            if effective_new_tokens < max_new_tokens:
+                logger.warning(
+                    "Reducing max_new_tokens from %d to %d because the processed prompt uses %d of max_length=%d.",
+                    max_new_tokens,
+                    effective_new_tokens,
+                    input_length,
+                    self.max_length,
+                )
+
+            request_max_length = input_length + effective_new_tokens
+            params = og.GeneratorParams(self._model)
+            params.set_search_options(
+                max_length=request_max_length,
+                **(search_options or {"do_sample": False}),
+            )
+            generator = og.Generator(self._model, params)
+            try:
+                try:
+                    generator.set_inputs(inputs)
+                except RuntimeError as e:
+                    return self._handle_error("ORT-GenAI generator input setup failed.", e, "")
+
+                # Whisper's BOS == EOS (token 50257 = <|startoftranscript|> = <|endoftext|>),
+                # so the very first generated token can collide with EOS. Skip the
+                # EOS check until we've emitted at least one non-EOS token.
+                decoded = ""
+                generated_token_ids: list[int] = []
+                stream = self._tokenizer.create_stream()
+                steps = 0
+                generated_any = False
+                while not generator.is_done() and steps < effective_new_tokens:
+                    generator.generate_next_token()
+                    tok = int(generator.get_next_tokens()[0])
+                    if tok in self._eos_token_ids:
+                        if self._model_type != "whisper" or generated_any:
+                            break
+                        # Whisper's first-step EOS can collide with its BOS token.
+                        steps += 1
+                        continue
+                    generated_any = True
+                    generated_token_ids.append(tok)
+                    decoded += stream.decode(tok)
+                    if stop_strings:
+                        scoreable_text = (
+                            self._decode_generated_tokens(generated_token_ids)
+                            if self._uses_gemma_response_channels
+                            else decoded
+                        )
+                        for s in stop_strings:
+                            if s in scoreable_text:
+                                return scoreable_text.split(s, 1)[0]
+                    steps += 1
+            finally:
+                del generator
+
+        return self._decode_generated_tokens(generated_token_ids) if self._uses_gemma_response_channels else decoded
+
+    # The generic system prompt used by the majority of lmms-eval wrappers
+    # (qwen2_5_vl.py, qwen3_vl.py, gemma3.py, llava_*, minicpm_o, ...). Also the
+    # fallback for model types without a more specific mapping.
+    _DEFAULT_SYSTEM_PROMPT = "You are a helpful assistant."
+    # Qwen Omni models (qwen2_5_omni.py / qwen3_omni.py) are trained with a fixed
+    # identity system prompt; reuse it verbatim for prompt parity.
+    _QWEN_OMNI_SYSTEM_PROMPT = (
+        "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, "
+        "capable of perceiving auditory and visual inputs, as well as generating text and speech."
+    )
+    # Per-model-family default system prompts. Each entry is a tuple of substrings
+    # that must ALL appear in the (lowercased) genai_config ``model.type``, paired
+    # with the prompt to use. First match wins, so more specific rules come first.
+    # These mirror lmms-eval's per-model wrappers so HF-input and ORT-GenAI
+    # evaluation use identical prompts out of the box. Only applied when the user
+    # does not pass an explicit ``system_prompt``.
+    #   - Qwen Omni (qwen2_5_omni / qwen3_omni_moe): fixed identity prompt.
+    #   - Qwen-VL / Gemma: generic "You are a helpful assistant.".
+    #   - Phi-4-MM: lmms-eval's phi4_multimodal wrapper uses NO system turn, so
+    #     the parity-correct default is an empty string.
+    _MODEL_TYPE_SYSTEM_PROMPTS: tuple[tuple[tuple[str, ...], str], ...] = (
+        (("qwen", "omni"), _QWEN_OMNI_SYSTEM_PROMPT),
+        (("qwen",), _DEFAULT_SYSTEM_PROMPT),
+        (("gemma",), _DEFAULT_SYSTEM_PROMPT),
+        (("phi",), ""),
+    )
+
+    @classmethod
+    def default_system_prompt_for_model_type(cls, model_type: str | None) -> str:
+        """Return the parity-correct default system prompt for a model type (see ``_MODEL_TYPE_SYSTEM_PROMPTS``)."""
+        model_type_lower = (model_type or "").lower()
+        for required_substrings, prompt in cls._MODEL_TYPE_SYSTEM_PROMPTS:
+            if all(substring in model_type_lower for substring in required_substrings):
+                return prompt
+        return cls._DEFAULT_SYSTEM_PROMPT
+
+    def _probe_structured_content_support(self) -> bool:
+        """Detect whether the model's chat template correctly injects media tokens from structured content.
+
+        Renders a probe message whose content is a list of typed parts
+        (``[{"type": "image"}, {"type": "text", ...}]``) and checks the result:
+
+        - Well-behaved templates (Gemma-4, Qwen2.5-VL, Qwen3-VL) replace the
+          image part with the model's own media token (e.g. ``<|image|>`` or
+          ``<|vision_start|>...``), so the rendered string contains no Python
+          dict repr.
+        - Broken templates (Phi-4-MM) stringify the list as Python repr, so the
+          rendered string contains ``{'type'`` / ``"type"``.
+
+        Probed once at load. Returns False only when the template renders but
+        does not support structured media; execution failures remain distinct.
+        """
+        try:
+            probe = [{"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "x"}]}]
+            rendered = self._tokenizer.apply_chat_template(json.dumps(probe), add_generation_prompt=True)
+            text_only = [{"role": "user", "content": [{"type": "text", "text": "x"}]}]
+            rendered_text_only = self._tokenizer.apply_chat_template(json.dumps(text_only), add_generation_prompt=True)
+        except Exception as e:
+            raise RuntimeError("Structured-content capability probe failed while rendering the chat template.") from e
+        # A useful template must neither leak the content dictionaries nor ignore
+        # the image part entirely.
+        return "{'type'" not in rendered and '"type"' not in rendered and rendered != rendered_text_only
+
+    def _build_structured_chat_prompt(self, user_text: str, num_images: int, num_audios: int) -> str:
+        """Build the prompt via structured content parts so the chat template injects media tokens.
+
+        Only used when :meth:`_probe_structured_content_support` returned True.
+        """
+        content: list[dict[str, Any]] = []
+        if _MEDIA_PLACEHOLDER_RE.search(user_text):
+            image_count = 0
+            audio_count = 0
+            for part in _MEDIA_PLACEHOLDER_RE.split(user_text):
+                if part == "<image>":
+                    image_count += 1
+                    content.append({"type": "image"})
+                elif part == "<audio>":
+                    audio_count += 1
+                    content.append({"type": "audio"})
+                elif part:
+                    content.append({"type": "text", "text": part})
+            if image_count != num_images or audio_count != num_audios:
+                raise ValueError(
+                    "Media placeholder counts do not match the extracted media: "
+                    f"{image_count} image placeholders for {num_images} images, "
+                    f"{audio_count} audio placeholders for {num_audios} audios."
+                )
+        else:
+            content.extend({"type": "image"} for _ in range(num_images))
+            content.extend({"type": "audio"} for _ in range(num_audios))
+            content.append({"type": "text", "text": user_text})
+
+        messages: list[dict[str, Any]] = []
+        if self.system_prompt:
+            system_content: str | list[dict[str, str]] = self.system_prompt
+            if "gemma" in self._model_type.lower():
+                system_content = [{"type": "text", "text": self.system_prompt}]
+            messages.append({"role": "system", "content": system_content})
+        messages.append({"role": "user", "content": content})
+        return self._tokenizer.apply_chat_template(json.dumps(messages), add_generation_prompt=True)
+
+    def _build_prompt_for_request(self, user_text: str, num_images: int, num_audios: int) -> str:
+        """Build the final prompt string fed to ``og.MultiModalProcessor``.
+
+        Whisper uses its decoder-start sequence. Other models use their native
+        chat template. Multimodal requests require the template to support
+        structured content parts so that it emits its own media tokens.
+        """
+        if self._model_type == "whisper":
+            return _build_whisper_prompt(
+                whisper_language=self.whisper_language,
+                whisper_task=self.whisper_task,
+                whisper_timestamps=self.whisper_timestamps,
+            )
+
+        has_media = bool(num_images or num_audios or _MEDIA_PLACEHOLDER_RE.search(user_text))
+        if has_media:
+            if not self._supports_structured_content:
+                raise ValueError(
+                    f"Model type {self._model_type!r} does not provide a chat template that supports "
+                    "structured image/audio content. Manual media-token prompt overrides are not supported."
+                )
+            return self._build_structured_chat_prompt(user_text, num_images, num_audios)
+
+        messages: list[dict[str, Any]] = []
+        if self.system_prompt:
+            system_content: str | list[dict[str, str]] = self.system_prompt
+            if "gemma" in self._model_type.lower():
+                system_content = [{"type": "text", "text": self.system_prompt}]
+            messages.append({"role": "system", "content": system_content})
+        messages.append({"role": "user", "content": user_text})
+
+        return self._tokenizer.apply_chat_template(json.dumps(messages), add_generation_prompt=True)
+
+    def _get_doc_and_visuals(self, doc_to_visual, doc_id, task, split):
+        """Look up an lmms-eval document and its visual inputs.
+
+        ``doc_to_visual`` is the task-specific function that extracts images or
+        audio from the document.
+        """
+        try:
+            doc = self.task_dict[task][split][doc_id]
+        except (KeyError, IndexError, TypeError) as e:
+            raise KeyError(f"Failed to find lmms-eval document={task!r}, split={split!r}, doc_id={doc_id!r}") from e
+
+        visuals = doc_to_visual(doc) if doc_to_visual else []
+        if visuals is None:
+            visuals = []
+        if not isinstance(visuals, list):
+            visuals = [visuals]
+        return doc, visuals
+
+    # -------------------------------------------------------------------------
+    # lmms-eval Model interface
+    # -------------------------------------------------------------------------
+    def generate_until(self, requests: list[Instance], disable_tqdm: bool = False) -> list[str]:
+        """lmms-eval entry point: generate a text completion for each request.
+
+        Per request, resolves media, the ``max_new_tokens`` budget (task YAML wins),
+        and ``until`` stop strings (minus ``ignore_stop_strings``), builds the prompt,
+        and greedily decodes. Returns one string per request, in order.
+        """
+        results = []
+        pbar = tqdm(total=len(requests), desc="ortgenai_mm generate_until", disable=disable_tqdm)
+        for req in requests:
+            contexts, gen_kwargs, doc_to_visual, doc_id, task, split = req.args
+            # NOTE, visuals could contain both images and audio
+            _, visuals = self._get_doc_and_visuals(doc_to_visual, doc_id, task, split)
+            images, audios = _partition_visuals(visuals)
+
+            gen_kwargs = gen_kwargs or {}
+            max_new_tokens = int(gen_kwargs.get("max_new_tokens", self.max_new_tokens))
+            search_options, stop = _generation_options(gen_kwargs)
+            if stop and self.ignore_stop_strings:
+                stop = [s for s in stop if s not in self.ignore_stop_strings] or None
+
+            prompt = self._build_prompt_for_request(contexts, len(images), len(audios))
+            text = self._run_generation(
+                prompt,
+                images,
+                audios,
+                max_new_tokens,
+                stop,
+                search_options=search_options,
+            )
+            results.append(text)
+            self.cache_hook.add_partial("generate_until", (contexts, gen_kwargs), text)
+            pbar.update(1)
+        pbar.close()
+        return results
+
+    def loglikelihood(self, requests: list[Instance], disable_tqdm: bool = False) -> list[tuple[float, bool]]:
+        """Reject loglikelihood tasks until upstream lmms-eval preserves tuple responses."""
+        raise NotImplementedError("ortgenai_mm does not support lmms-eval loglikelihood tasks yet.")
+
+    def generate_until_multi_round(self, requests) -> list[str]:
+        raise NotImplementedError("ortgenai_mm does not support lmms-eval multi-round generation yet.")
diff --git a/olive/evaluator/olive_evaluator.py b/olive/evaluator/olive_evaluator.py
index 23054e1973..e633f8fe26 100644
--- a/olive/evaluator/olive_evaluator.py
+++ b/olive/evaluator/olive_evaluator.py
@@ -575,6 +575,149 @@ def _get_genai_model_dir(model: ONNXModelHandler) -> str:
     return str(Path(model.model_path).parent)
 
 
+def _lmms_model_size_on_disk(model: "OliveModelHandler") -> int:
+    """Return the full package size for ORT-GenAI models, otherwise the handler size."""
+    if isinstance(model, ONNXModelHandler):
+        genai_config_path = _find_genai_config(model)
+        if genai_config_path is not None:
+            return sum(path.stat().st_size for path in genai_config_path.parent.rglob("*") if path.is_file())
+    return model.size_on_disk
+
+
+def _lmms_output_path(output_path: str, model: "OliveModelHandler") -> Path:
+    """Avoid clobbering an input-model result with a different backend's result."""
+    out = Path(output_path)
+    backend = "hf" if isinstance(model, HfModelHandler) else "onnx"
+    if "{model_type}" in str(out):
+        return Path(str(out).format(model_type=backend))
+    if not out.is_file():
+        return out
+    try:
+        existing_backend = json.loads(out.read_text(encoding="utf-8")).get("model_type")
+    except (json.JSONDecodeError, OSError):
+        existing_backend = None
+    if existing_backend in (None, backend):
+        return out
+    return out.with_name(f"{out.stem}_{backend}{out.suffix}")
+
+
+def _wrap_generate_until_drop_stops(lm, ignore_stop_strings) -> None:
+    r"""Wrap ``generate_until`` to drop unwanted request stop strings.
+
+    lmms-eval injects ``until=[fewshot_delimiter]`` (``"\\n\\n"``) for generate_until
+    tasks that don't set their own ``until`` (lmms_eval/api/task.py), which truncates
+    step-by-step reasoning at the first blank line. The ORT-GenAI adapter honors
+    ``ignore_stop_strings`` directly; for HF wrappers (whose ``generate_until`` we
+    don't own) we filter each request's ``gen_kwargs["until"]`` before delegating.
+    Mirrors the diagnostic probe's monkeypatch, but scoped to a single instance.
+    """
+    ignore = set(ignore_stop_strings or [])
+    if not ignore or not hasattr(lm, "generate_until"):
+        return
+    orig = lm.generate_until
+
+    def _patched(requests, *args, **kwargs):
+        for req in requests:
+            gen_kwargs = next((arg for arg in req.args if isinstance(arg, dict)), None)
+            if gen_kwargs is None:
+                continue
+            until = gen_kwargs.get("until")
+            if isinstance(until, str):
+                until = [until]
+            if until:
+                filtered = [u for u in until if u not in ignore]
+                gen_kwargs["until"] = filtered
+        return orig(requests, *args, **kwargs)
+
+    lm.generate_until = _patched
+
+
+def _linify_lmms_sample(obj):
+    r"""Recursively split multi-line strings into lists of lines.
+
+    This lets ``json.dumps`` render long generations readably instead of one
+    escaped ``\\n`` blob.
+    """
+    if isinstance(obj, str):
+        return obj.split("\n") if "\n" in obj else obj
+    if isinstance(obj, list):
+        return [_linify_lmms_sample(x) for x in obj]
+    if isinstance(obj, tuple):
+        return [_linify_lmms_sample(x) for x in obj]
+    if isinstance(obj, dict):
+        return {k: _linify_lmms_sample(v) for k, v in obj.items()}
+    return obj
+
+
+def _has_correct_lmms_score(value: Any) -> bool:
+    """Return True when a nested lmms-eval score contains a successful value."""
+    if isinstance(value, dict):
+        return any(_has_correct_lmms_score(item) for item in value.values())
+    if isinstance(value, (list, tuple)):
+        return any(_has_correct_lmms_score(item) for item in value)
+    return value in (1, 1.0, True)
+
+
+def _summarize_lmms_samples(task: str, samples: list[dict]) -> dict:
+    """Reduce lmms-eval sample records to fields useful for inspection.
+
+    Generated text remains full and untruncated, with multi-line text represented
+    as a list of lines. The bulky ``doc`` field is dropped.
+    """
+    rows = []
+    n_empty = 0
+    n_scored_correct = 0
+    is_mathvision = "mathvision" in task
+    for s in samples:
+        target = s.get("target")
+        resps = s.get("resps")
+        filtered = s.get("filtered_resps")
+        score_keys = [k for k in s if k in ("mmmu_acc", "exact_match", "mathvision_standard_eval", "acc", "score")]
+        score = {k: s.get(k) for k in score_keys}
+
+        gen_text = ""
+        try:
+            if isinstance(resps, list) and resps:
+                gen_text = resps[0][0] if isinstance(resps[0], list) else str(resps[0])
+        except (IndexError, TypeError):
+            gen_text = str(resps)
+
+        if gen_text is None or (isinstance(gen_text, str) and not gen_text.strip()):
+            n_empty += 1
+        if _has_correct_lmms_score(score):
+            n_scored_correct += 1
+
+        if is_mathvision:
+            score_compact = {}
+            for k, v in score.items():
+                if isinstance(v, dict):
+                    score_compact[k] = {kk: vv for kk, vv in v.items() if kk != "response"}
+                else:
+                    score_compact[k] = v
+            row = {
+                "doc_id": s.get("doc_id"),
+                "target": target,
+                "generated": _linify_lmms_sample(gen_text or ""),
+                "score": _linify_lmms_sample(score_compact),
+            }
+        else:
+            row = {
+                "doc_id": s.get("doc_id"),
+                "target": target,
+                "generated": _linify_lmms_sample(gen_text or ""),
+                "filtered": _linify_lmms_sample(filtered) if filtered is not None else None,
+                "score": _linify_lmms_sample(score),
+            }
+        rows.append(row)
+    return {
+        "task": task,
+        "n": len(samples),
+        "n_empty_generations": n_empty,
+        "n_scored_correct": n_scored_correct,
+        "rows": rows,
+    }
+
+
 def _normalize_audio_batch(input_data) -> tuple[list, list]:
     """Return (audio_arrays, file_names) from the various speech input shapes.
 
@@ -2327,6 +2470,399 @@ def evaluate(
         return flatten_metric_result(metrics)
 
 
+@Registry.register("LMMSEvaluator")
+class LMMSEvaluator(OliveEvaluator):
+    """Evaluator for multimodal models using lmms-eval (EvolvingLMMs-Lab/lmms-eval).
+
+    Supports two model handler types:
+
+    1. :class:`ONNXModelHandler` whose path is an ORT-GenAI multimodal package
+       (directory containing ``genai_config.json`` plus respective ONNX subgraph files
+       Dispatches to the ``ortgenai_mm`` adapter in :mod:`olive.evaluator.lmms_ort`.
+
+    2. :class:`HfModelHandler` for HuggingFace PyTorch multimodal models.
+       Dispatches to lmms-eval's native wrapper for the model architecture
+       (e.g. ``phi4_multimodal``, ``qwen2_5_vl``). The wrapper is auto-detected
+       from the HF ``model_type`` field; pass ``model_class`` in the recipe to
+       override.
+
+    Raw single-file ONNX models are intentionally not supported: multimodal
+    inference requires the vision/audio preprocessing pipeline that ORT-GenAI's
+    multimodal processor provides; a bare ``onnxruntime.InferenceSession``
+    cannot do image or audio tokenization on its own.
+
+    ``include_path`` accepts one custom-task directory or a list of directories.
+    ONNX audio uses the package-declared speech sample rate unless
+    ``audio_target_sample_rate`` is explicitly set. ``image_serialization_profile``
+    defaults to historical lossless PNG; set ``jpeg85`` explicitly for parity
+    with native wrappers that JPEG-encode inputs.
+    """
+
+    # HuggingFace model_type -> lmms-eval model_class (canonical id).
+    # Covers the multimodal architectures most relevant to Olive sweeps; other
+    # architectures still work if ``model_class`` is set explicitly in the
+    # recipe. Verified against lmms-eval's AVAILABLE_SIMPLE_MODELS /
+    # AVAILABLE_CHAT_TEMPLATE_MODELS registries.
+    _HF_MODEL_TYPE_TO_LMMS_CLASS: ClassVar[dict[str, str]] = {
+        "phi4mm": "phi4_multimodal",
+        "phi3_v": "phi3v",
+        "qwen2_vl": "qwen2_vl",
+        "qwen2_5_vl": "qwen2_5_vl",
+        "qwen3_vl": "qwen3_vl",
+        "qwen2_audio": "qwen2_audio",
+        "qwen2_5_omni": "qwen2_5_omni",
+        "qwen3_omni": "qwen3_omni",
+        "whisper": "whisper",
+        "gemma3": "gemma3",
+        "minicpm_o": "minicpm_o",
+        "llava": "llava",
+        "llava_onevision": "llava_onevision",
+        "internvl_chat": "internvl",
+    }
+
+    def __init__(self, tasks: list[str], **kwargs):
+        super().__init__(**kwargs)
+        self.tasks = tasks
+        include_path = kwargs.get("include_path")
+        if include_path is None:
+            self.include_path = None
+        else:
+            include_paths = [include_path] if isinstance(include_path, str) else include_path
+            if (
+                not isinstance(include_paths, list)
+                or not include_paths
+                or not all(isinstance(path, str) and path.strip() for path in include_paths)
+            ):
+                raise ValueError(
+                    "include_path must be a non-empty directory path or list of non-empty directory paths."
+                )
+            resolved_include_paths = [str(Path(path).expanduser().resolve()) for path in include_paths]
+            invalid_paths = [path for path in resolved_include_paths if not Path(path).is_dir()]
+            if invalid_paths:
+                raise ValueError(f"include_path entries must be existing directories: {invalid_paths}")
+            self.include_path = resolved_include_paths[0] if isinstance(include_path, str) else resolved_include_paths
+        self.limit = kwargs.get("limit")
+        self.model_class = kwargs.get("model_class")
+        self.batch_size = kwargs.get("batch_size", 1)
+        # Task generation kwargs take precedence; this value is the fallback for
+        # tasks that do not provide max_new_tokens.
+        self.max_new_tokens = kwargs.get("max_new_tokens", 256)
+        self.max_length = kwargs.get("max_length", 32768)
+        # Default to None (not a hardcoded string) so the adapter's per-model
+        # system-prompt auto-resolution takes over when the recipe doesn't set
+        # one. This keeps olive-run and standalone-adapter usage consistent, and
+        # lets model-family defaults apply (e.g. Gemma/Qwen-VL "You are a helpful
+        # assistant.", Qwen-Omni identity prompt, Phi-4-MM empty system turn),
+        # matching lmms-eval's per-model wrappers. An explicit recipe value still
+        # overrides.
+        self.system_prompt = kwargs.get("system_prompt")
+        self.ep = kwargs.get("execution_provider")
+        self.ep_options = kwargs.get("provider_options")
+        self.log_samples = bool(kwargs.get("log_samples", False))
+        self.output_path = kwargs.get("output_path")
+        self.fail_on_error = bool(kwargs.get("fail_on_error", True))
+        self.whisper_language = kwargs.get("whisper_language", "en")
+        self.whisper_task = kwargs.get("whisper_task", "transcribe")
+        self.whisper_timestamps = bool(kwargs.get("whisper_timestamps", False))
+        self.audio_target_sample_rate = kwargs.get("audio_target_sample_rate")
+        # "lossless" preserves historical Olive runs; "jpeg85" forces JPEG.
+        self.image_serialization_profile = kwargs.get("image_serialization_profile", "lossless")
+        # Stop strings to suppress from each task's lmms-eval ``until`` list (e.g.
+        # ["\n\n"]). lmms-eval defaults ``until`` to the fewshot_delimiter ("\n\n")
+        # for tasks that don't set their own, which truncates step-by-step
+        # reasoning at the first blank line. Set this (plus a larger
+        # ``max_new_tokens``) for reasoning tasks so generation runs to the
+        # model's EOS instead. Only used by the ortgenai_mm adapter.
+        self.ignore_stop_strings = kwargs.get("ignore_stop_strings")
+        # HF-only knobs (forwarded to lmms-eval's native wrapper if present).
+        # ``trust_remote_code`` defaults to False to match the rest of Olive
+        # (e.g. olive/common/hf/utils.py, olive/data/component/load_dataset.py)
+        # and avoid silently executing arbitrary Hub code at load time. Users
+        # who need Phi-4-MM, MiniCPM-o, etc. opt in explicitly via the recipe.
+        self.dtype = kwargs.get("dtype", "auto")
+        self.trust_remote_code = bool(kwargs.get("trust_remote_code", False))
+        self.hf_model_kwargs = kwargs.get("hf_model_kwargs") or {}
+
+    @staticmethod
+    def _device_for_hf(device: Device) -> str:
+        # lmms-eval's HF wrappers accept "cuda", "cpu", or a torch.device.
+        return "cuda" if device == Device.GPU else "cpu"
+
+    def _build_ortgenai_mm_lm(
+        self,
+        model: ONNXModelHandler,
+        execution_providers: Optional[Union[str, list[str]]],
+    ):
+        """Build the lmms-eval wrapper for an ORT-GenAI multimodal package."""
+        from olive.evaluator.lmms_ort import LMMSORTGenAIEvaluator
+
+        if _find_genai_config(model) is None:
+            raise ValueError(
+                "LMMSEvaluator requires an ORT-GenAI package "
+                "(directory containing genai_config.json) for ONNXModelHandler input. "
+                f"Got ONNXModelHandler without genai_config at {model.model_path}. "
+                "Raw single-file ONNX is not supported for multimodal evaluation because "
+                "the vision/audio preprocessing pipeline lives in ORT-GenAI's multimodal "
+                "processor; use HfModelHandler or an ORT-GenAI package instead."
+            )
+
+        model_dir = _get_genai_model_dir(model)
+        logger.info("Running lmms-eval (model_class=ortgenai_mm, model_dir=%s)", model_dir)
+        return LMMSORTGenAIEvaluator(
+            pretrained=model_dir,
+            batch_size=self.batch_size,
+            max_new_tokens=self.max_new_tokens,
+            max_length=self.max_length,
+            system_prompt=self.system_prompt,
+            execution_provider=self.ep or execution_providers,
+            provider_options=self.ep_options,
+            fail_on_error=self.fail_on_error,
+            ignore_stop_strings=self.ignore_stop_strings,
+            whisper_language=self.whisper_language,
+            whisper_task=self.whisper_task,
+            whisper_timestamps=self.whisper_timestamps,
+            audio_target_sample_rate=self.audio_target_sample_rate,
+            image_serialization_profile=self.image_serialization_profile,
+        )
+
+    def _resolve_hf_model_class(self, model: HfModelHandler, hf_model_type: str | None = None) -> str:
+        if self.model_class:
+            return self.model_class
+        hf_model_type = hf_model_type or model.get_hf_model_type()
+        lmms_model_class = self._HF_MODEL_TYPE_TO_LMMS_CLASS.get(hf_model_type)
+        if not lmms_model_class:
+            raise ValueError(
+                f"Could not auto-detect lmms-eval model_class for HF model_type={hf_model_type!r}. "
+                f"Pass 'model_class' in the evaluator config (e.g. one of "
+                f"{sorted(self._HF_MODEL_TYPE_TO_LMMS_CLASS.values())}, or any other "
+                f"name registered with lmms-eval)."
+            )
+        return lmms_model_class
+
+    def _build_hf_lm(self, model: HfModelHandler, device: Device):
+        """Build lmms-eval's native HF wrapper for a HuggingFace PyTorch multimodal model."""
+        import inspect
+
+        from lmms_eval.models import get_model
+
+        if model.adapter_path:
+            raise ValueError(
+                "LMMSEvaluator does not support HfModelHandler.adapter_path because lmms-eval wrappers load the "
+                "base checkpoint directly. Merge the adapter into a checkpoint before evaluation."
+            )
+        handler_load_kwargs = model.get_load_kwargs()
+        if handler_load_kwargs:
+            raise ValueError(
+                "LMMSEvaluator does not forward HfModelHandler.load_kwargs because lmms-eval model wrappers have "
+                "incompatible constructor options. Move wrapper-supported options to the evaluator's "
+                f"hf_model_kwargs or remove the handler load_kwargs. Got keys: {sorted(handler_load_kwargs)}."
+            )
+
+        hf_model_type = model.get_hf_model_type()
+        model_class = self._resolve_hf_model_class(model, hf_model_type)
+        # The ORT-GenAI adapter implements lmms-eval's simple request protocol.
+        # Prefer the matching simple HF wrapper when a model registers both a
+        # simple and chat implementation, so input/output evaluation uses the
+        # same task construction and media layout.
+        lm_cls = get_model(model_class, force_simple=True)
+        from olive.evaluator.lmms_ort import LMMSORTGenAIEvaluator
+
+        system_prompt = self.system_prompt
+        if system_prompt is None:
+            system_prompt = LMMSORTGenAIEvaluator.default_system_prompt_for_model_type(hf_model_type)
+
+        # lmms-eval wrappers have inconsistent constructor signatures: phi4_multimodal
+        # accepts dtype/trust_remote_code as named params, qwen2_5_vl asserts
+        # ``kwargs == {}`` at runtime even though it has ``**kwargs`` in its signature.
+        # Because the signature alone cannot tell "absorbs unknown kwargs" from
+        # "rejects unknown kwargs at runtime", we conservatively only forward
+        # ``device/dtype/trust_remote_code`` to wrappers that name them explicitly
+        # as parameters. For wrappers that take a different set of kwargs (e.g.
+        # backend-specific knobs), users pass them through ``hf_model_kwargs``,
+        # which is always forwarded unfiltered as an explicit user opt-in.
+        try:
+            accepted = set(inspect.signature(lm_cls.__init__).parameters)
+        except (TypeError, ValueError):
+            accepted = set()
+
+        optional_kwargs = {
+            "device": self._device_for_hf(device),
+            "dtype": self.dtype,
+            "trust_remote_code": self.trust_remote_code,
+            "system_prompt": system_prompt,
+            "language": self.whisper_language,
+            "task": self.whisper_task,
+        }
+        if hf_model_type == "whisper" and self.whisper_timestamps:
+            raise NotImplementedError("Whisper timestamp parity is not supported for HF input-model evaluation.")
+        forwarded = {k: v for k, v in optional_kwargs.items() if k in accepted}
+
+        init_kwargs = {
+            "pretrained": str(model.model_name_or_path),
+            "batch_size": self.batch_size,
+            **forwarded,
+            **self.hf_model_kwargs,
+        }
+        logger.info(
+            "Running lmms-eval (model_class=%s, pretrained=%s, forwarded_kwargs=%s)",
+            model_class,
+            init_kwargs["pretrained"],
+            sorted(set(init_kwargs) - {"pretrained", "batch_size"}),
+        )
+        lm = lm_cls(**init_kwargs)
+        if self.ignore_stop_strings:
+            _wrap_generate_until_drop_stops(lm, self.ignore_stop_strings)
+        return lm
+
+    def evaluate(
+        self,
+        model: "OliveModelHandler",
+        metrics: list[Metric],
+        device: Device = Device.CPU,
+        execution_providers: Optional[Union[str, list[str]]] = None,
+    ) -> MetricResult:
+        metrics_dict: dict[str, MetricResult] = {}
+        tasks = list(self.tasks)
+
+        if MetricType.SIZE_ON_DISK.value in tasks:
+            tasks.remove(MetricType.SIZE_ON_DISK.value)
+            metrics_dict[MetricType.SIZE_ON_DISK.value] = MetricResult.model_validate(
+                {
+                    SizeOnDiskSubType.BYTES.value: {
+                        "value": _lmms_model_size_on_disk(model),
+                        "priority": -1,
+                        "higher_is_better": False,
+                    }
+                }
+            )
+
+        if tasks:
+            from lmms_eval.evaluator import simple_evaluate
+
+            from olive.evaluator.lmms_ort import warmup_image_decoder
+
+            # Bind ORT-GenAI's image-decoder zlib before any HF input-model eval in
+            # this process decodes images; otherwise the ONNX eval hits
+            # "libpng error: bad parameters to zlib". See warmup_image_decoder().
+            warmup_image_decoder()
+
+            # Checks runtime type of model and routes to appropriate lmms-eval
+            # model wrapper (exposes generate_until over the resolved backend).
+            if isinstance(model, ONNXModelHandler):
+                lm = self._build_ortgenai_mm_lm(model, execution_providers)
+            elif isinstance(model, HfModelHandler):
+                lm = self._build_hf_lm(model, device)
+            else:
+                raise ValueError(
+                    "LMMSEvaluator supports ONNXModelHandler (ORT-GenAI multimodal package) "
+                    f"and HfModelHandler. Got {type(model).__name__}."
+                )
+
+            task_manager_kwargs = {}
+            if self.include_path is not None:
+                from lmms_eval.tasks import TaskManager
+
+                task_manager_kwargs["task_manager"] = TaskManager(include_path=self.include_path, model_name=lm)
+            results = simple_evaluate(
+                model=lm,
+                tasks=tasks,
+                batch_size=self.batch_size,
+                limit=self.limit,
+                log_samples=self.log_samples,
+                **task_manager_kwargs,
+            )
+
+            # Explicitly write lmms-eval results to json.
+            if self.output_path:
+                out = _lmms_output_path(self.output_path, model)
+                out.parent.mkdir(parents=True, exist_ok=True)
+                is_hf_model = isinstance(model, HfModelHandler)
+                compact = {
+                    "model_type": "hf" if is_hf_model else "onnx",
+                    "results": results.get("results", {}),
+                    "higher_is_better": results.get("higher_is_better", {}),
+                    "configs": {k: str(v) for k, v in results.get("configs", {}).items()},
+                    "evaluator_config": {
+                        "include_path": self.include_path,
+                        "image_serialization_profile": (
+                            "native HF wrapper" if is_hf_model else self.image_serialization_profile
+                        ),
+                        "resolved_image_serialization_format": (
+                            "native HF wrapper"
+                            if is_hf_model
+                            else getattr(lm, "_image_serialization_format", "unknown")
+                        ),
+                        "audio_target_sample_rate": (
+                            "native HF wrapper"
+                            if is_hf_model
+                            else getattr(lm, "audio_target_sample_rate", self.audio_target_sample_rate)
+                        ),
+                        "audio_target_sample_rates": (
+                            "native HF wrapper" if is_hf_model else list(getattr(lm, "audio_target_sample_rates", ()))
+                        ),
+                        "audio_resampling_mode": (
+                            "native HF wrapper"
+                            if is_hf_model
+                            else (
+                                "host explicit override"
+                                if getattr(lm, "_audio_target_sample_rate_explicit", False)
+                                else "package processor"
+                            )
+                        ),
+                    },
+                }
+                out.write_text(json.dumps(compact, indent=2, default=str), encoding="utf-8")
+                logger.info("Wrote lmms-eval results to %s", out)
+
+                # When log_samples is on, persist per-doc generations to a sibling
+                # *_samples.json (probe-style: doc_id, target, full untruncated
+                # generation, score). The main results file stays compact.
+                samples = results.get("samples") or {}
+                if samples:
+                    summarized = {
+                        task_name: _summarize_lmms_samples(task_name, task_samples)
+                        for task_name, task_samples in samples.items()
+                    }
+                    samples_out = out.with_name(f"{out.stem}_samples{out.suffix}")
+                    samples_out.write_text(json.dumps(summarized, indent=2, default=str), encoding="utf-8")
+                    logger.info("Wrote lmms-eval per-sample generations to %s", samples_out)
+            elif self.log_samples:
+                logger.warning(
+                    "log_samples is set but output_path is not; per-sample generations "
+                    "were computed but not persisted. Set 'output_path' to save them."
+                )
+
+            # Convert lmms-eval results into Olive's MetricResult shape (mirrors LMEvaluator).
+            for task_name in sorted(results.get("results", {}).keys()):
+                task_results = results["results"][task_name]
+                task_directions = (results.get("higher_is_better") or {}).get(task_name) or {}
+                task_metrics = {}
+                for mf, v in sorted(task_results.items()):
+                    if mf == "alias" or not isinstance(v, (int, float)):
+                        continue
+                    metric_name, _, filter_name = mf.partition(",")
+                    # Skip all stderr variants. lmms-eval emits "<metric>_stderr" as
+                    # well as "<metric>_stderr_clt" / "<metric>_stderr_clustered"
+                    # (the latter two are numeric and would otherwise leak in as
+                    # bogus metrics), so match the "_stderr" substring, not just the
+                    # "_stderr" suffix.
+                    if "_stderr" in metric_name:
+                        continue
+                    metric_key = metric_name if filter_name in ("", "none") else f"{metric_name},{filter_name}"
+                    direction = task_directions.get(metric_name)
+                    task_metrics[metric_key] = SubMetricResult(
+                        value=float(v),
+                        priority=-1,
+                        higher_is_better=True if direction is None else bool(direction),
+                    )
+                if task_metrics:
+                    metrics_dict[task_name] = MetricResult.model_validate(task_metrics)
+        elif self.log_samples:
+            logger.warning("log_samples is set but no lmms-eval tasks were provided; no samples were generated.")
+
+        return flatten_metric_result(metrics_dict)
+
+
 @Registry.register("MTEBEvaluator")
 class MTEBEvaluator(OliveEvaluator):
     """Evaluator for embedding models using the MTEB (Massive Text Embedding Benchmark) library.
diff --git a/test/evaluator/test_lmms_ort.py b/test/evaluator/test_lmms_ort.py
new file mode 100644
index 0000000000..e5e1551336
--- /dev/null
+++ b/test/evaluator/test_lmms_ort.py
@@ -0,0 +1,1921 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+# --------------------------------------------------------------------------
+# Tests intentionally exercise the protected _run_generation runtime method and
+# configure fake collaborators by setting attributes
+# directly on the fake. Both are normal in unit tests, so suppress pylint's
+# protected-access / attribute-defined-outside-init warnings for this file.
+# pylint: disable=protected-access,attribute-defined-outside-init,no-value-for-parameter,unexpected-keyword-arg
+import json
+import sys
+from pathlib import Path
+from types import ModuleType, SimpleNamespace
+from typing import ClassVar
+from unittest.mock import MagicMock, patch
+
+import numpy as np
+import PIL.Image
+import pytest
+
+import olive.evaluator.olive_evaluator as olive_evaluator_module
+from olive.evaluator.lmms_ort import (
+    LMMSORTGenAIEvaluator,
+    _build_whisper_prompt,
+    _normalize_execution_provider,
+    _resample_audio,
+    _resolve_model_audio_sample_rate,
+    _resolve_model_audio_sample_rates,
+)
+from olive.model import ONNXModelHandler
+
+LMMSEvaluator = olive_evaluator_module.LMMSEvaluator
+
+
+class _FakeTaskManager:
+    instances: ClassVar[list] = []
+
+    def __init__(self, include_path=None, model_name=None):
+        self.include_path = include_path
+        self.model_name = model_name
+        type(self).instances.append(self)
+
+
+_LMMS_EVAL_TASKS_MODULE = ModuleType("lmms_eval.tasks")
+_LMMS_EVAL_TASKS_MODULE.TaskManager = _FakeTaskManager
+
+
+def test_build_whisper_prompt_configures_language_and_task():
+    prompt = _build_whisper_prompt(
+        whisper_language="fr",
+        whisper_task="translate",
+        whisper_timestamps=False,
+    )
+
+    assert prompt == "<|startoftranscript|><|fr|><|translate|><|notimestamps|>"
+
+
+@pytest.mark.parametrize(
+    ("model_type", "expected"),
+    [
+        ("qwen2_5_vl", "You are a helpful assistant."),
+        ("qwen2_5_vl_text", "You are a helpful assistant."),
+        ("qwen3_vl", "You are a helpful assistant."),
+        ("gemma4", "You are a helpful assistant."),
+        ("gemma3", "You are a helpful assistant."),
+        (
+            "qwen2_5_omni",
+            "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, "
+            "capable of perceiving auditory and visual inputs, as well as generating text and speech.",
+        ),
+        (
+            "qwen3_omni_moe",
+            "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, "
+            "capable of perceiving auditory and visual inputs, as well as generating text and speech.",
+        ),
+        ("phi4mm", ""),
+        ("phi3v", ""),
+        ("whisper", "You are a helpful assistant."),
+        (None, "You are a helpful assistant."),
+    ],
+)
+def test_default_system_prompt_for_model_type_matches_lmms_eval_wrappers(model_type, expected):
+    assert LMMSORTGenAIEvaluator.default_system_prompt_for_model_type(model_type) == expected
+
+
+@pytest.mark.parametrize(
+    ("execution_provider", "expected"),
+    [
+        ("CUDAExecutionProvider", "cuda"),
+        ("CPUExecutionProvider", "cpu"),
+        ("DmlExecutionProvider", "dml"),
+        ("gpu", "cuda"),
+        (None, "follow_config"),
+        (("CUDAExecutionProvider", {"device_id": "0"}), "cuda"),
+    ],
+)
+def test_normalize_execution_provider(execution_provider, expected):
+    assert _normalize_execution_provider(execution_provider) == expected
+
+
+def _make_evaluator_for_prompt_tests():
+    """Construct an LMMSORTGenAIEvaluator with __init__ skipped.
+
+    Lets us unit-test the prompt-building path without needing a real ORT-GenAI
+    model on disk.
+    """
+    inst = LMMSORTGenAIEvaluator.__new__(LMMSORTGenAIEvaluator)
+    inst._tokenizer = MagicMock(name="og.Tokenizer")
+    inst._model_type = "test_model"
+    inst.system_prompt = "You are helpful."
+    inst._supports_structured_content = True
+    inst.whisper_language = "en"
+    inst.whisper_task = "transcribe"
+    inst.whisper_timestamps = False
+    return inst
+
+
+def test_build_prompt_for_request_uses_apply_chat_template_by_default():
+    inst = _make_evaluator_for_prompt_tests()
+    inst._tokenizer.apply_chat_template.return_value = "<rendered chat prompt>"
+
+    out = inst._build_prompt_for_request("What is in the image?", num_images=1, num_audios=0)
+
+    assert out == "<rendered chat prompt>"
+    inst._tokenizer.apply_chat_template.assert_called_once()
+    messages_json_arg = inst._tokenizer.apply_chat_template.call_args.args[0]
+    assert inst._tokenizer.apply_chat_template.call_args.kwargs.get("add_generation_prompt") is True
+
+    messages = json.loads(messages_json_arg)
+    assert messages[0] == {"role": "system", "content": "You are helpful."}
+    assert messages[1] == {
+        "role": "user",
+        "content": [{"type": "image"}, {"type": "text", "text": "What is in the image?"}],
+    }
+
+
+def test_build_prompt_for_request_uses_typed_system_text_for_gemma_only():
+    gemma = _make_evaluator_for_prompt_tests()
+    gemma._model_type = "gemma4"
+    gemma._tokenizer.apply_chat_template.return_value = "gemma"
+    qwen = _make_evaluator_for_prompt_tests()
+    qwen._model_type = "qwen2_5_vl"
+    qwen._tokenizer.apply_chat_template.return_value = "qwen"
+
+    gemma._build_prompt_for_request("Q", num_images=1, num_audios=0)
+    qwen._build_prompt_for_request("Q", num_images=1, num_audios=0)
+
+    gemma_messages = json.loads(gemma._tokenizer.apply_chat_template.call_args.args[0])
+    qwen_messages = json.loads(qwen._tokenizer.apply_chat_template.call_args.args[0])
+    assert gemma_messages[0] == {
+        "role": "system",
+        "content": [{"type": "text", "text": "You are helpful."}],
+    }
+    assert qwen_messages[0] == {"role": "system", "content": "You are helpful."}
+
+
+def test_build_prompt_for_request_skips_system_when_empty():
+    inst = _make_evaluator_for_prompt_tests()
+    inst.system_prompt = ""
+    inst._tokenizer.apply_chat_template.return_value = "out"
+
+    inst._build_prompt_for_request("Q", num_images=0, num_audios=0)
+
+    messages = json.loads(inst._tokenizer.apply_chat_template.call_args.args[0])
+    assert all(m["role"] != "system" for m in messages)
+    assert messages[-1] == {"role": "user", "content": "Q"}
+
+
+def test_build_prompt_for_request_uses_structured_media_parts():
+    inst = _make_evaluator_for_prompt_tests()
+    inst.system_prompt = ""
+    inst._tokenizer.apply_chat_template.return_value = "out"
+
+    inst._build_prompt_for_request("Q", num_images=2, num_audios=1)
+
+    messages = json.loads(inst._tokenizer.apply_chat_template.call_args.args[0])
+    assert messages[0]["content"] == [
+        {"type": "image"},
+        {"type": "image"},
+        {"type": "audio"},
+        {"type": "text", "text": "Q"},
+    ]
+
+
+def test_build_prompt_for_request_uses_structured_content_when_supported():
+    """Use structured content parts when supported."""
+    inst = _make_evaluator_for_prompt_tests()
+    inst._tokenizer.apply_chat_template.return_value = "<rendered>"
+
+    inst._build_prompt_for_request("What is this?", num_images=1, num_audios=1)
+
+    messages = json.loads(inst._tokenizer.apply_chat_template.call_args.args[0])
+    user_msg = messages[-1]
+    assert user_msg["role"] == "user"
+    # Content remains a list of typed parts rather than a flat string.
+    assert user_msg["content"] == [
+        {"type": "image"},
+        {"type": "audio"},
+        {"type": "text", "text": "What is this?"},
+    ]
+
+
+def test_build_prompt_for_request_rejects_media_when_structured_content_unsupported():
+    inst = _make_evaluator_for_prompt_tests()
+    inst._supports_structured_content = False
+
+    with pytest.raises(ValueError, match="structured image/audio content"):
+        inst._build_prompt_for_request("What is this?", num_images=1, num_audios=0)
+
+
+def test_probe_structured_content_support_detects_injection():
+    """A template that injects the media token (no dict repr) -> supported."""
+    inst = LMMSORTGenAIEvaluator.__new__(LMMSORTGenAIEvaluator)
+    inst._tokenizer = MagicMock()
+    inst._tokenizer.apply_chat_template.side_effect = [
+        "<|im_start|>user\n<|image|>x<|im_end|>",
+        "<|im_start|>user\nx<|im_end|>",
+    ]
+
+    assert inst._probe_structured_content_support() is True
+
+
+def test_probe_structured_content_support_detects_stringified_repr():
+    """A template that leaks the Python dict repr -> not supported."""
+    inst = LMMSORTGenAIEvaluator.__new__(LMMSORTGenAIEvaluator)
+    inst._tokenizer = MagicMock()
+    inst._tokenizer.apply_chat_template.return_value = "<|user|>[{'type': 'image'}, ...]<|end|>"
+
+    assert inst._probe_structured_content_support() is False
+
+
+def test_probe_structured_content_support_detects_ignored_image_part():
+    inst = LMMSORTGenAIEvaluator.__new__(LMMSORTGenAIEvaluator)
+    inst._tokenizer = MagicMock()
+    inst._tokenizer.apply_chat_template.side_effect = ["same-rendering", "same-rendering"]
+
+    assert inst._probe_structured_content_support() is False
+
+
+def test_probe_structured_content_support_surfaces_probe_exception():
+    inst = LMMSORTGenAIEvaluator.__new__(LMMSORTGenAIEvaluator)
+    inst._tokenizer = MagicMock()
+    inst._tokenizer.apply_chat_template.side_effect = ValueError("invalid template expression")
+
+    with pytest.raises(RuntimeError, match="capability probe failed") as exc_info:
+        inst._probe_structured_content_support()
+
+    assert isinstance(exc_info.value.__cause__, ValueError)
+    assert "invalid template expression" in str(exc_info.value.__cause__)
+
+
+def _make_evaluator_for_generate_until(ignore_stop_strings):
+    """Construct an LMMSORTGenAIEvaluator with __init__ skipped.
+
+    Wired so generate_until runs without a real model: _run_generation is a stub
+    that records the stop list it was given.
+    """
+    inst = LMMSORTGenAIEvaluator.__new__(LMMSORTGenAIEvaluator)
+    inst.max_new_tokens = 256
+    inst.ignore_stop_strings = set(ignore_stop_strings or [])
+    inst.cache_hook = SimpleNamespace(add_partial=lambda *a, **k: None)
+    inst._get_doc_and_visuals = lambda *a, **k: (None, [])
+    inst._build_prompt_for_request = lambda *a, **k: "<prompt>"
+    inst.recorded_stops = []
+
+    def _fake_run_generation(prompt, images, audios, max_new, stop, search_options=None):
+        inst.recorded_stops.append(stop)
+        inst.recorded_search_options = search_options
+        return "ok"
+
+    inst._run_generation = _fake_run_generation
+    return inst
+
+
+def _make_generate_until_request(until):
+    gen_kwargs = {"until": until}
+    return SimpleNamespace(args=("ctx", gen_kwargs, None, 0, "task", "split"))
+
+
+def test_generate_until_drops_ignored_stop_strings():
+    inst = _make_evaluator_for_generate_until(ignore_stop_strings=["\n\n"])
+    req = _make_generate_until_request(until=["\n\n", "Q:"])
+
+    inst.generate_until([req], disable_tqdm=True)
+
+    # "\n\n" is suppressed, the legitimate "Q:" stop is preserved.
+    assert inst.recorded_stops == [["Q:"]]
+
+
+def test_generate_until_sets_stop_none_when_all_ignored():
+    inst = _make_evaluator_for_generate_until(ignore_stop_strings=["\n\n"])
+    req = _make_generate_until_request(until=["\n\n"])
+
+    inst.generate_until([req], disable_tqdm=True)
+
+    # The only stop string was suppressed -> None (generation runs to EOS).
+    assert inst.recorded_stops == [None]
+
+
+def test_generate_until_keeps_stops_when_nothing_to_ignore():
+    inst = _make_evaluator_for_generate_until(ignore_stop_strings=[])
+    req = _make_generate_until_request(until=["\n\n", "Q:"])
+
+    inst.generate_until([req], disable_tqdm=True)
+
+    # No ignore list configured -> stops pass through unchanged.
+    assert inst.recorded_stops == [["\n\n", "Q:"]]
+
+
+def test_generate_until_forwards_supported_search_options():
+    inst = _make_evaluator_for_generate_until(ignore_stop_strings=[])
+    req = SimpleNamespace(
+        args=(
+            "ctx",
+            {
+                "do_sample": True,
+                "temperature": 0.7,
+                "top_p": 0.9,
+                "top_k": 10,
+                "repetition_penalty": 1.1,
+            },
+            None,
+            0,
+            "task",
+            "split",
+        )
+    )
+
+    inst.generate_until([req], disable_tqdm=True)
+
+    assert inst.recorded_search_options == {
+        "do_sample": True,
+        "temperature": 0.7,
+        "top_p": 0.9,
+        "top_k": 10,
+        "repetition_penalty": 1.1,
+    }
+
+
+def test_generate_until_rejects_unsupported_beam_search():
+    inst = _make_evaluator_for_generate_until(ignore_stop_strings=[])
+    req = SimpleNamespace(args=("ctx", {"num_beams": 2}, None, 0, "task", "split"))
+
+    with pytest.raises(NotImplementedError, match="Beam search"):
+        inst.generate_until([req], disable_tqdm=True)
+
+
+def test_wrap_generate_until_drop_stops_filters_hf_wrapper_until():
+    """Strip ignored stops before delegating to the HF wrapper.
+
+    The patch updates each request's generation kwargs in place.
+    """
+    seen_until = []
+
+    class FakeHfWrapper:
+        def generate_until(self, requests, *args, **kwargs):
+            seen_until.extend(r.args[1].get("until") for r in requests)
+            return ["ok"] * len(requests)
+
+    lm = FakeHfWrapper()
+    olive_evaluator_module._wrap_generate_until_drop_stops(lm, ["\n\n"])
+
+    reqs = [
+        _make_generate_until_request(until=["\n\n", "Q:"]),  # partial -> keep "Q:"
+        _make_generate_until_request(until=["\n\n"]),  # all ignored -> None
+        _make_generate_until_request(until="\n\n"),  # str form -> None
+    ]
+    out = lm.generate_until(reqs)
+
+    assert out == ["ok", "ok", "ok"]
+    assert seen_until == [["Q:"], [], []]
+
+
+def test_wrap_generate_until_drop_stops_finds_chat_wrapper_kwargs():
+    seen_until = []
+
+    class FakeChatWrapper:
+        def generate_until(self, requests, *args, **kwargs):
+            seen_until.extend(request.args[2]["until"] for request in requests)
+            return ["ok"] * len(requests)
+
+    lm = FakeChatWrapper()
+    olive_evaluator_module._wrap_generate_until_drop_stops(lm, ["\n\n"])
+    request = SimpleNamespace(args=("ctx", lambda doc: doc, {"until": ["\n\n", "END"]}, 0, "task", "split"))
+
+    lm.generate_until([request])
+
+    assert seen_until == [["END"]]
+
+
+def test_wrap_generate_until_drop_stops_noop_without_ignore_list():
+    class FakeHfWrapper:
+        def generate_until(self, requests, *args, **kwargs):
+            return "orig"
+
+    lm = FakeHfWrapper()
+    olive_evaluator_module._wrap_generate_until_drop_stops(lm, [])
+    # Empty ignore list -> generate_until not wrapped; original behavior intact.
+    assert "generate_until" not in vars(lm)
+    assert lm.generate_until([]) == "orig"
+
+
+def test_lmms_evaluator_converts_lmms_results(tmp_path):
+    model_dir = tmp_path / "model"
+    model_dir.mkdir()
+    model_path = model_dir / "text.onnx"
+    model_path.touch()
+    (model_dir / "genai_config.json").write_text('{"model": {"type": "phi4mm"}}', encoding="utf-8")
+
+    output_path = tmp_path / "results.json"
+    evaluator = LMMSEvaluator(
+        tasks=["ai2d_lite"],
+        batch_size=1,
+        limit=2,
+        output_path=str(output_path),
+        fail_on_error=False,
+    )
+    model = ONNXModelHandler(model_path=str(model_path))
+
+    simple_evaluate_result = {
+        "results": {
+            "ai2d_lite": {
+                "alias": "AI2D Lite",
+                "exact_match,none": 0.5,
+                "exact_match_stderr,none": 0.1,
+                "samples": [{"ignored": True}],
+            }
+        },
+        "configs": {"ai2d_lite": {"task": "ai2d_lite"}},
+    }
+
+    simple_evaluate_mock = MagicMock(return_value=simple_evaluate_result)
+    lmms_eval_module = ModuleType("lmms_eval")
+    lmms_eval_evaluator_module = ModuleType("lmms_eval.evaluator")
+    lmms_eval_evaluator_module.simple_evaluate = simple_evaluate_mock
+
+    with (
+        patch.dict(
+            sys.modules,
+            {
+                "lmms_eval": lmms_eval_module,
+                "lmms_eval.evaluator": lmms_eval_evaluator_module,
+                "lmms_eval.tasks": _LMMS_EVAL_TASKS_MODULE,
+            },
+        ),
+        patch(
+            "olive.evaluator.lmms_ort.LMMSORTGenAIEvaluator",
+            return_value=SimpleNamespace(_image_serialization_format="PNG", audio_target_sample_rate=None),
+        ) as lm_mock,
+    ):
+        result = evaluator.evaluate(model, [], execution_providers=["CUDAExecutionProvider"])
+
+    lm_mock.assert_called_once_with(
+        pretrained=str(model_dir),
+        batch_size=1,
+        max_new_tokens=256,
+        max_length=32768,
+        system_prompt=None,
+        execution_provider=["CUDAExecutionProvider"],
+        provider_options=None,
+        fail_on_error=False,
+        ignore_stop_strings=None,
+        whisper_language="en",
+        whisper_task="transcribe",
+        whisper_timestamps=False,
+        audio_target_sample_rate=None,
+        image_serialization_profile="lossless",
+    )
+    simple_evaluate_mock.assert_called_once()
+    assert "task_manager" not in simple_evaluate_mock.call_args.kwargs
+    assert not _FakeTaskManager.instances
+    assert result.get_value("ai2d_lite", "exact_match") == 0.5
+    assert output_path.exists()
+    persisted = json.loads(output_path.read_text(encoding="utf-8"))
+    assert persisted["evaluator_config"]["image_serialization_profile"] == "lossless"
+    assert persisted["evaluator_config"]["resolved_image_serialization_format"] == "PNG"
+
+
+def test_lmms_evaluator_passes_external_include_path_to_onnx_task_manager(tmp_path):
+    from lmms_eval.tasks import TaskManager as RealTaskManager
+
+    task_dir = tmp_path / "external_tasks"
+    task_dir.mkdir()
+    (task_dir / "phase5_custom.yaml").write_text("task: phase5_custom\n", encoding="utf-8")
+    assert "phase5_custom" in RealTaskManager(include_path=str(task_dir), include_defaults=False).all_tasks
+    model_dir = tmp_path / "model"
+    model_dir.mkdir()
+    model_path = model_dir / "text.onnx"
+    model_path.touch()
+    (model_dir / "genai_config.json").write_text('{"model": {"type": "phi4mm"}}', encoding="utf-8")
+    evaluator = LMMSEvaluator(tasks=["phase5_custom"], include_path=str(task_dir))
+    model = ONNXModelHandler(model_path=str(model_path))
+    _FakeTaskManager.instances = []
+
+    def simple_evaluate(**kwargs):
+        manager = kwargs["task_manager"]
+        assert Path(manager.include_path) == task_dir.resolve()
+        assert (Path(manager.include_path) / "phase5_custom.yaml").is_file()
+        return {"results": {}, "configs": {}}
+
+    lmms_eval_module = ModuleType("lmms_eval")
+    lmms_eval_evaluator_module = ModuleType("lmms_eval.evaluator")
+    lmms_eval_evaluator_module.simple_evaluate = simple_evaluate
+    with (
+        patch.dict(
+            sys.modules,
+            {
+                "lmms_eval": lmms_eval_module,
+                "lmms_eval.evaluator": lmms_eval_evaluator_module,
+                "lmms_eval.tasks": _LMMS_EVAL_TASKS_MODULE,
+            },
+        ),
+        patch("olive.evaluator.lmms_ort.LMMSORTGenAIEvaluator", return_value=SimpleNamespace()),
+    ):
+        evaluator.evaluate(model, [])
+
+    assert len(_FakeTaskManager.instances) == 1
+    assert _FakeTaskManager.instances[0].model_name is not None
+
+
+def test_lmms_evaluator_passes_multiple_include_paths_to_hf_task_manager(tmp_path, monkeypatch):
+    task_dirs = [tmp_path / "tasks_a", tmp_path / "tasks_b"]
+    for index, task_dir in enumerate(task_dirs):
+        task_dir.mkdir()
+        (task_dir / f"custom_{index}.yaml").write_text(f"task: custom_{index}\n", encoding="utf-8")
+    handler_stub = _make_hf_model_handler_stub("/p/Qwen2.5-VL-3B-Instruct", "qwen2_5_vl")
+    _patch_isinstance_for_hf(handler_stub, monkeypatch)
+    output_path = tmp_path / "hf_results.json"
+    evaluator = LMMSEvaluator(
+        tasks=["custom_0", "custom_1"],
+        include_path=[str(path) for path in task_dirs],
+        output_path=str(output_path),
+        image_serialization_profile="jpeg85",
+        audio_target_sample_rate=16000,
+    )
+    _FakeTaskManager.instances = []
+
+    simple_evaluate_mock = MagicMock(return_value={"results": {}, "configs": {}})
+    lmms_eval_module = ModuleType("lmms_eval")
+    lmms_eval_evaluator_module = ModuleType("lmms_eval.evaluator")
+    lmms_eval_evaluator_module.simple_evaluate = simple_evaluate_mock
+    lmms_eval_models_module = ModuleType("lmms_eval.models")
+    lmms_eval_models_module.get_model = MagicMock(return_value=_FakeQwenWrapper)
+    with patch.dict(
+        sys.modules,
+        {
+            "lmms_eval": lmms_eval_module,
+            "lmms_eval.evaluator": lmms_eval_evaluator_module,
+            "lmms_eval.models": lmms_eval_models_module,
+            "lmms_eval.tasks": _LMMS_EVAL_TASKS_MODULE,
+        },
+    ):
+        evaluator.evaluate(handler_stub, [])
+
+    assert len(_FakeTaskManager.instances) == 1
+    assert _FakeTaskManager.instances[0].include_path == [str(path.resolve()) for path in task_dirs]
+    assert simple_evaluate_mock.call_args.kwargs["task_manager"] is _FakeTaskManager.instances[0]
+    assert _FakeTaskManager.instances[0].model_name is simple_evaluate_mock.call_args.kwargs["model"]
+    persisted = json.loads(output_path.read_text(encoding="utf-8"))
+    assert persisted["evaluator_config"]["image_serialization_profile"] == "native HF wrapper"
+    assert persisted["evaluator_config"]["resolved_image_serialization_format"] == "native HF wrapper"
+    assert persisted["evaluator_config"]["audio_target_sample_rate"] == "native HF wrapper"
+
+
+@pytest.mark.parametrize("include_path", ["missing", ["missing"], [], [""], [1]])
+def test_lmms_evaluator_rejects_invalid_include_paths(tmp_path, include_path):
+    if include_path == "missing":
+        include_path = str(tmp_path / "missing")
+    elif include_path == ["missing"]:
+        include_path = [str(tmp_path / "missing")]
+
+    with pytest.raises(ValueError, match="include_path"):
+        LMMSEvaluator(tasks=["custom"], include_path=include_path)
+
+
+def test_lmms_evaluator_preserves_filter_names_and_metric_direction(tmp_path):
+    model_dir = tmp_path / "model"
+    model_dir.mkdir()
+    model_path = model_dir / "text.onnx"
+    model_path.touch()
+    (model_dir / "genai_config.json").write_text('{"model": {"type": "phi4mm"}}', encoding="utf-8")
+    evaluator = LMMSEvaluator(tasks=["speech_task"])
+    model = ONNXModelHandler(model_path=str(model_path))
+    simple_evaluate_result = {
+        "results": {
+            "speech_task": {
+                "alias": "Speech",
+                "wer,strict": 0.2,
+                "wer,flexible": 0.1,
+                "wer_stderr,strict": 0.01,
+            }
+        },
+        "higher_is_better": {"speech_task": {"wer": False}},
+        "configs": {},
+    }
+    lmms_eval_module = ModuleType("lmms_eval")
+    lmms_eval_evaluator_module = ModuleType("lmms_eval.evaluator")
+    lmms_eval_evaluator_module.simple_evaluate = MagicMock(return_value=simple_evaluate_result)
+
+    with (
+        patch.dict(
+            sys.modules,
+            {
+                "lmms_eval": lmms_eval_module,
+                "lmms_eval.evaluator": lmms_eval_evaluator_module,
+                "lmms_eval.tasks": _LMMS_EVAL_TASKS_MODULE,
+            },
+        ),
+        patch("olive.evaluator.lmms_ort.LMMSORTGenAIEvaluator", return_value=SimpleNamespace()),
+    ):
+        result = evaluator.evaluate(model, [])
+
+    assert result.root["speech_task-wer,strict"].value == 0.2
+    assert result.root["speech_task-wer,flexible"].value == 0.1
+    assert result.root["speech_task-wer,strict"].higher_is_better is False
+    assert all("_stderr" not in key for key in result.root)
+
+
+def test_lmms_output_path_avoids_cross_backend_overwrite(tmp_path):
+    output_path = tmp_path / "results.json"
+    output_path.write_text('{"model_type": "hf"}', encoding="utf-8")
+    model_path = tmp_path / "model.onnx"
+    model_path.touch()
+    model = ONNXModelHandler(model_path=str(model_path))
+
+    assert olive_evaluator_module._lmms_output_path(str(output_path), model) == tmp_path / "results_onnx.json"
+
+
+def test_lmms_model_size_counts_whole_genai_package(tmp_path):
+    package = tmp_path / "package"
+    decoder = package / "decoder"
+    vision = package / "vision"
+    decoder.mkdir(parents=True)
+    vision.mkdir()
+    (package / "genai_config.json").write_text("{}", encoding="utf-8")
+    (decoder / "model.onnx").write_bytes(b"decoder")
+    (vision / "model.onnx").write_bytes(b"vision")
+    model = ONNXModelHandler(model_path=str(package), onnx_file_name="decoder/model.onnx")
+
+    assert olive_evaluator_module._lmms_model_size_on_disk(model) == sum(
+        path.stat().st_size for path in package.rglob("*") if path.is_file()
+    )
+
+
+def test_summarize_lmms_samples_counts_nested_mathvision_score():
+    summary = olive_evaluator_module._summarize_lmms_samples(
+        "mathvision_test",
+        [
+            {
+                "doc_id": 0,
+                "target": "A",
+                "resps": [["answer"]],
+                "mathvision_standard_eval": {"scores": [True], "response": ["answer"]},
+            }
+        ],
+    )
+
+    assert summary["n_scored_correct"] == 1
+
+
+def test_lmms_evaluator_requires_genai_config(tmp_path):
+    model_path = tmp_path / "text.onnx"
+    model_path.touch()
+    evaluator = LMMSEvaluator(tasks=["ai2d_lite"])
+    model = ONNXModelHandler(model_path=str(model_path))
+
+    with pytest.raises(ValueError, match="requires an ORT-GenAI package"):
+        evaluator.evaluate(model, [])
+
+
+# -----------------------------------------------------------------------------
+# HuggingFace dispatch
+# -----------------------------------------------------------------------------
+
+
+def _make_hf_model_handler_stub(model_name_or_path: str, hf_model_type: str):
+    handler = MagicMock(name="HfModelHandler")
+    handler.model_name_or_path = model_name_or_path
+    handler.adapter_path = None
+    handler.get_load_kwargs.return_value = {}
+    handler.get_hf_model_type.return_value = hf_model_type
+    return handler
+
+
+def _patch_isinstance_for_hf(handler_stub, monkeypatch):
+    """Force isinstance(handler_stub, HfModelHandler) to True for the test path.
+
+    Avoids constructing a real HfModelHandler (which would require a real HF
+    model on disk) while still exercising the dispatch logic.
+    """
+    real_isinstance = isinstance
+
+    def _isinstance(obj, cls):
+        if obj is handler_stub and cls is olive_evaluator_module.HfModelHandler:
+            return True
+        if obj is handler_stub and cls is olive_evaluator_module.ONNXModelHandler:
+            return False
+        return real_isinstance(obj, cls)
+
+    monkeypatch.setattr(olive_evaluator_module, "isinstance", _isinstance, raising=False)
+    olive_evaluator_module.isinstance = _isinstance
+
+
+class _FakePhi4Wrapper:
+    """Fake lmms-eval wrapper with a phi4_multimodal-style signature.
+
+    Accepts dtype + trust_remote_code (mirrors lmms-eval's phi4_multimodal class).
+    """
+
+    last_kwargs: ClassVar[dict] = {}
+
+    def __init__(
+        self,
+        pretrained,
+        device="cuda",
+        dtype="auto",
+        batch_size=1,
+        trust_remote_code=True,
+        system_prompt=None,
+        **kwargs,
+    ):
+        type(self).last_kwargs = {
+            "pretrained": pretrained,
+            "device": device,
+            "dtype": dtype,
+            "batch_size": batch_size,
+            "trust_remote_code": trust_remote_code,
+            "system_prompt": system_prompt,
+            **kwargs,
+        }
+
+
+class _FakeQwenWrapper:
+    """Fake lmms-eval wrapper with a qwen2_5_vl-style signature.
+
+    Does NOT accept dtype or trust_remote_code (mirrors lmms-eval's qwen2_5_vl
+    class which asserts kwargs == {}).
+    """
+
+    last_kwargs: ClassVar[dict] = {}
+
+    def __init__(
+        self,
+        pretrained,
+        device="cuda",
+        device_map="auto",
+        batch_size=1,
+        system_prompt=None,
+        **kwargs,
+    ):
+        if kwargs:
+            raise AssertionError(f"Unexpected kwargs: {kwargs}")
+        type(self).last_kwargs = {
+            "pretrained": pretrained,
+            "device": device,
+            "device_map": device_map,
+            "batch_size": batch_size,
+            "system_prompt": system_prompt,
+        }
+
+
+class _FakeKwargsWrapper:
+    """Fake lmms-eval wrapper that absorbs ALL options via ``**kwargs``.
+
+    Mirrors lmms-eval wrappers (and HF model wrappers more generally) that take
+    only the required ``pretrained`` argument by name and pass everything else
+    through ``**kwargs`` to the underlying HF transformers model. Used to verify
+    LMMSEvaluator forwards optional kwargs (dtype, trust_remote_code, device) to
+    such wrappers instead of silently dropping them because they aren't in
+    ``inspect.signature(...).parameters`` as named params.
+    """
+
+    last_kwargs: ClassVar[dict] = {}
+
+    def __init__(self, pretrained, **kwargs):
+        type(self).last_kwargs = {"pretrained": pretrained, **kwargs}
+
+
+class _FakeWhisperWrapper:
+    last_kwargs: ClassVar[dict] = {}
+
+    def __init__(self, pretrained, device="cuda", batch_size=1, language="en", task="transcribe"):
+        type(self).last_kwargs = {
+            "pretrained": pretrained,
+            "device": device,
+            "batch_size": batch_size,
+            "language": language,
+            "task": task,
+        }
+
+
+def test_lmms_evaluator_rejects_hf_adapter_path():
+    handler_stub = _make_hf_model_handler_stub("/p/model", "qwen2_5_vl")
+    handler_stub.adapter_path = "/p/adapter"
+    evaluator = LMMSEvaluator(tasks=["ai2d"])
+
+    with pytest.raises(ValueError, match=r"does not support HfModelHandler\.adapter_path"):
+        evaluator._build_hf_lm(handler_stub, olive_evaluator_module.Device.CPU)
+
+
+def test_lmms_evaluator_rejects_hf_handler_load_kwargs():
+    handler_stub = _make_hf_model_handler_stub("/p/model", "qwen2_5_vl")
+    handler_stub.get_load_kwargs.return_value = {"revision": "main"}
+    evaluator = LMMSEvaluator(tasks=["ai2d"])
+
+    with pytest.raises(ValueError, match=r"does not forward HfModelHandler\.load_kwargs"):
+        evaluator._build_hf_lm(handler_stub, olive_evaluator_module.Device.CPU)
+
+
+def test_lmms_evaluator_auto_detects_hf_model_class_from_model_type(tmp_path, monkeypatch):
+    """When model_class is unset, auto-detect from HfModelHandler.get_hf_model_type()."""
+    handler_stub = _make_hf_model_handler_stub("/local/path/Phi-4-multimodal-instruct", "phi4mm")
+    _patch_isinstance_for_hf(handler_stub, monkeypatch)
+
+    output_path = tmp_path / "results.json"
+    evaluator = LMMSEvaluator(tasks=["ai2d_lite"], batch_size=2, limit=4, output_path=str(output_path))
+
+    simple_evaluate_mock = MagicMock(
+        return_value={"results": {"ai2d_lite": {"alias": "AI2D", "exact_match,none": 0.75}}, "configs": {}}
+    )
+    lmms_eval_module = ModuleType("lmms_eval")
+    lmms_eval_evaluator_module = ModuleType("lmms_eval.evaluator")
+    lmms_eval_evaluator_module.simple_evaluate = simple_evaluate_mock
+    lmms_eval_models_module = ModuleType("lmms_eval.models")
+    lmms_eval_models_module.get_model = MagicMock(return_value=_FakePhi4Wrapper)
+
+    with patch.dict(
+        sys.modules,
+        {
+            "lmms_eval": lmms_eval_module,
+            "lmms_eval.evaluator": lmms_eval_evaluator_module,
+            "lmms_eval.models": lmms_eval_models_module,
+            "lmms_eval.tasks": _LMMS_EVAL_TASKS_MODULE,
+        },
+    ):
+        result = evaluator.evaluate(handler_stub, [])
+
+    lmms_eval_models_module.get_model.assert_called_once_with("phi4_multimodal", force_simple=True)
+    assert _FakePhi4Wrapper.last_kwargs["pretrained"] == "/local/path/Phi-4-multimodal-instruct"
+    assert _FakePhi4Wrapper.last_kwargs["batch_size"] == 2
+    # trust_remote_code defaults to False (see olive/evaluator/olive_evaluator.py
+    # LMMSEvaluator.__init__); users opt in explicitly in the recipe.
+    assert _FakePhi4Wrapper.last_kwargs["trust_remote_code"] is False
+    assert _FakePhi4Wrapper.last_kwargs["dtype"] == "auto"
+    assert _FakePhi4Wrapper.last_kwargs["system_prompt"] == ""
+    simple_evaluate_mock.assert_called_once()
+    assert result.get_value("ai2d_lite", "exact_match") == 0.75
+
+
+def test_lmms_evaluator_filters_kwargs_for_qwen_style_wrapper(monkeypatch):
+    """Wrappers like qwen2_5_vl reject unknown kwargs.
+
+    LMMSEvaluator must inspect the wrapper signature and only forward kwargs
+    the wrapper actually declares as named parameters.
+    """
+    handler_stub = _make_hf_model_handler_stub("/p/Qwen2.5-VL-3B-Instruct", "qwen2_5_vl")
+    _patch_isinstance_for_hf(handler_stub, monkeypatch)
+
+    evaluator = LMMSEvaluator(tasks=["mmstar"], batch_size=1, dtype="bfloat16", trust_remote_code=True)
+
+    simple_evaluate_mock = MagicMock(return_value={"results": {}, "configs": {}})
+    lmms_eval_module = ModuleType("lmms_eval")
+    lmms_eval_evaluator_module = ModuleType("lmms_eval.evaluator")
+    lmms_eval_evaluator_module.simple_evaluate = simple_evaluate_mock
+    lmms_eval_models_module = ModuleType("lmms_eval.models")
+    lmms_eval_models_module.get_model = MagicMock(return_value=_FakeQwenWrapper)
+
+    with patch.dict(
+        sys.modules,
+        {
+            "lmms_eval": lmms_eval_module,
+            "lmms_eval.evaluator": lmms_eval_evaluator_module,
+            "lmms_eval.models": lmms_eval_models_module,
+            "lmms_eval.tasks": _LMMS_EVAL_TASKS_MODULE,
+        },
+    ):
+        evaluator.evaluate(handler_stub, [])
+
+    # dtype + trust_remote_code must NOT have been forwarded (Qwen wrapper would error)
+    assert "dtype" not in _FakeQwenWrapper.last_kwargs
+    assert "trust_remote_code" not in _FakeQwenWrapper.last_kwargs
+    # but pretrained, device, batch_size MUST have been forwarded
+    assert _FakeQwenWrapper.last_kwargs["pretrained"] == "/p/Qwen2.5-VL-3B-Instruct"
+    assert _FakeQwenWrapper.last_kwargs["device"] == "cpu"  # Device.CPU default
+    assert _FakeQwenWrapper.last_kwargs["batch_size"] == 1
+    assert _FakeQwenWrapper.last_kwargs["system_prompt"] == "You are a helpful assistant."
+
+
+def test_lmms_evaluator_does_not_forward_to_pure_var_keyword_wrappers(monkeypatch):
+    """Verify ``device``/``dtype``/``trust_remote_code`` are NOT auto-forwarded to ``**kwargs`` wrappers.
+
+    Rationale: the signature alone cannot tell "absorbs unknowns" from "rejects
+    unknowns at runtime" (qwen2_5_vl has ``**kwargs`` and asserts ``kwargs ==
+    {}``). To stay safe, only kwargs named explicitly as parameters are
+    auto-forwarded. Users who want to forward additional kwargs to a pure
+    ``**kwargs`` wrapper must use ``hf_model_kwargs`` (explicit user opt-in).
+    """
+    handler_stub = _make_hf_model_handler_stub("/p/some-vlm", "qwen2_5_vl")
+    _patch_isinstance_for_hf(handler_stub, monkeypatch)
+
+    evaluator = LMMSEvaluator(
+        tasks=["mmstar"],
+        batch_size=1,
+        dtype="bfloat16",
+        trust_remote_code=True,
+        # The escape hatch for forwarding arbitrary kwargs to a wrapper that
+        # absorbs them via **kwargs:
+        hf_model_kwargs={"custom_backend_opt": "value"},
+    )
+
+    simple_evaluate_mock = MagicMock(return_value={"results": {}, "configs": {}})
+    lmms_eval_module = ModuleType("lmms_eval")
+    lmms_eval_evaluator_module = ModuleType("lmms_eval.evaluator")
+    lmms_eval_evaluator_module.simple_evaluate = simple_evaluate_mock
+    lmms_eval_models_module = ModuleType("lmms_eval.models")
+    lmms_eval_models_module.get_model = MagicMock(return_value=_FakeKwargsWrapper)
+
+    with patch.dict(
+        sys.modules,
+        {
+            "lmms_eval": lmms_eval_module,
+            "lmms_eval.evaluator": lmms_eval_evaluator_module,
+            "lmms_eval.models": lmms_eval_models_module,
+            "lmms_eval.tasks": _LMMS_EVAL_TASKS_MODULE,
+        },
+    ):
+        evaluator.evaluate(handler_stub, [])
+
+    # Required kwargs are always forwarded.
+    assert _FakeKwargsWrapper.last_kwargs["pretrained"] == "/p/some-vlm"
+    assert _FakeKwargsWrapper.last_kwargs["batch_size"] == 1
+    # Optional kwargs are NOT forwarded to pure **kwargs wrappers.
+    assert "dtype" not in _FakeKwargsWrapper.last_kwargs
+    assert "trust_remote_code" not in _FakeKwargsWrapper.last_kwargs
+    assert "device" not in _FakeKwargsWrapper.last_kwargs
+    # The explicit hf_model_kwargs escape hatch IS forwarded.
+    assert _FakeKwargsWrapper.last_kwargs["custom_backend_opt"] == "value"
+
+
+def test_lmms_evaluator_uses_explicit_model_class_when_set(monkeypatch):
+    """An explicit ``model_class`` in the recipe overrides auto-detection."""
+    handler_stub = _make_hf_model_handler_stub("/p/some-vlm", "some-unknown-vlm-type")
+    _patch_isinstance_for_hf(handler_stub, monkeypatch)
+
+    evaluator = LMMSEvaluator(tasks=["mmstar"], model_class="qwen2_5_vl", batch_size=1)
+
+    simple_evaluate_mock = MagicMock(return_value={"results": {}, "configs": {}})
+    lmms_eval_module = ModuleType("lmms_eval")
+    lmms_eval_evaluator_module = ModuleType("lmms_eval.evaluator")
+    lmms_eval_evaluator_module.simple_evaluate = simple_evaluate_mock
+    lmms_eval_models_module = ModuleType("lmms_eval.models")
+    lmms_eval_models_module.get_model = MagicMock(return_value=_FakeQwenWrapper)
+
+    with patch.dict(
+        sys.modules,
+        {
+            "lmms_eval": lmms_eval_module,
+            "lmms_eval.evaluator": lmms_eval_evaluator_module,
+            "lmms_eval.models": lmms_eval_models_module,
+            "lmms_eval.tasks": _LMMS_EVAL_TASKS_MODULE,
+        },
+    ):
+        evaluator.evaluate(handler_stub, [])
+
+    lmms_eval_models_module.get_model.assert_called_once_with("qwen2_5_vl", force_simple=True)
+    # Explicit model_class bypasses class auto-detection, but model_type is still
+    # consulted to resolve the parity-correct default system prompt.
+    handler_stub.get_hf_model_type.assert_called_once()
+
+
+def test_lmms_evaluator_forwards_whisper_language_and_task(monkeypatch):
+    handler_stub = _make_hf_model_handler_stub("/p/whisper", "whisper")
+    _patch_isinstance_for_hf(handler_stub, monkeypatch)
+    evaluator = LMMSEvaluator(
+        tasks=["fleurs_fr"],
+        whisper_language="fr",
+        whisper_task="translate",
+    )
+    lmms_eval_module = ModuleType("lmms_eval")
+    lmms_eval_evaluator_module = ModuleType("lmms_eval.evaluator")
+    lmms_eval_evaluator_module.simple_evaluate = MagicMock(return_value={"results": {}, "configs": {}})
+    lmms_eval_models_module = ModuleType("lmms_eval.models")
+    lmms_eval_models_module.get_model = MagicMock(return_value=_FakeWhisperWrapper)
+
+    with patch.dict(
+        sys.modules,
+        {
+            "lmms_eval": lmms_eval_module,
+            "lmms_eval.evaluator": lmms_eval_evaluator_module,
+            "lmms_eval.models": lmms_eval_models_module,
+            "lmms_eval.tasks": _LMMS_EVAL_TASKS_MODULE,
+        },
+    ):
+        evaluator.evaluate(handler_stub, [])
+
+    assert _FakeWhisperWrapper.last_kwargs["language"] == "fr"
+    assert _FakeWhisperWrapper.last_kwargs["task"] == "translate"
+
+
+@pytest.mark.parametrize("model_type", ["some-exotic-vlm", "gemma4", "gemma4_unified"])
+def test_lmms_evaluator_raises_when_hf_model_type_is_unmapped(monkeypatch, model_type):
+    """If we can't auto-detect and the user didn't set model_class, fail loudly."""
+    handler_stub = _make_hf_model_handler_stub("/p/exotic-model", model_type)
+    _patch_isinstance_for_hf(handler_stub, monkeypatch)
+
+    evaluator = LMMSEvaluator(tasks=["mmstar"])
+
+    # Even with lmms_eval modules mocked, the error fires before reaching them.
+    lmms_eval_module = ModuleType("lmms_eval")
+    lmms_eval_evaluator_module = ModuleType("lmms_eval.evaluator")
+    lmms_eval_evaluator_module.simple_evaluate = MagicMock()
+    lmms_eval_models_module = ModuleType("lmms_eval.models")
+    lmms_eval_models_module.get_model = MagicMock()
+    with (
+        patch.dict(
+            sys.modules,
+            {
+                "lmms_eval": lmms_eval_module,
+                "lmms_eval.evaluator": lmms_eval_evaluator_module,
+                "lmms_eval.models": lmms_eval_models_module,
+                "lmms_eval.tasks": _LMMS_EVAL_TASKS_MODULE,
+            },
+        ),
+        pytest.raises(ValueError, match=r"Could not auto-detect lmms-eval model_class"),
+    ):
+        evaluator.evaluate(handler_stub, [])
+
+
+def test_lmms_evaluator_rejects_unsupported_handler_type():
+    """LMMSEvaluator only supports HfModelHandler and ONNXModelHandler-as-ortgenai."""
+    evaluator = LMMSEvaluator(tasks=["mmstar"])
+    bogus = SimpleNamespace()  # neither HfModelHandler nor ONNXModelHandler
+
+    lmms_eval_module = ModuleType("lmms_eval")
+    lmms_eval_evaluator_module = ModuleType("lmms_eval.evaluator")
+    lmms_eval_evaluator_module.simple_evaluate = MagicMock()
+    with (
+        patch.dict(
+            sys.modules,
+            {
+                "lmms_eval": lmms_eval_module,
+                "lmms_eval.evaluator": lmms_eval_evaluator_module,
+                "lmms_eval.tasks": _LMMS_EVAL_TASKS_MODULE,
+            },
+        ),
+        pytest.raises(ValueError, match=r"ONNXModelHandler.*HfModelHandler"),
+    ):
+        evaluator.evaluate(bogus, [])
+
+
+# -----------------------------------------------------------------------------
+# lmms-eval MODEL_REGISTRY_V2 entry-point integration
+# -----------------------------------------------------------------------------
+
+
+def test_lmms_ort_genai_evaluator_is_simple_flag_matches_registration():
+    """Verify is_simple matches the lmms-eval registration type.
+
+    MODEL_REGISTRY_V2._validate_model_class requires the class' ``is_simple``
+    flag to match the registered model_type (``simple`` vs ``chat``). Our
+    adapter is registered with ``simple_class_path``, so ``is_simple`` must
+    be ``True``.
+    """
+    assert LMMSORTGenAIEvaluator.is_simple is True
+
+
+# -----------------------------------------------------------------------------
+# Visual partitioning
+# -----------------------------------------------------------------------------
+
+
+def test_partition_visuals_separates_images_and_audios_and_skips_nones():
+    from olive.evaluator.lmms_ort import _partition_visuals
+
+    img1 = PIL.Image.new("RGB", (4, 4))
+    img2_dict = {"bytes": _png_bytes(PIL.Image.new("RGB", (2, 2)))}
+    audio_dict = {"array": np.zeros(16, dtype=np.float32), "sampling_rate": 16000}
+
+    images, audios = _partition_visuals([img1, None, img2_dict, audio_dict, None])
+
+    assert len(images) == 2
+    assert all(isinstance(img, PIL.Image.Image) for img in images)
+    assert len(audios) == 1
+    arr, sr = audios[0]
+    assert arr.shape == (16,)
+    assert sr == 16000
+
+
+def test_partition_visuals_handles_none_input():
+    from olive.evaluator.lmms_ort import _partition_visuals
+
+    assert _partition_visuals(None) == ([], [])
+    assert _partition_visuals([]) == ([], [])
+
+
+def test_partition_visuals_downmixes_channels_first_and_last_audio():
+    from olive.evaluator.lmms_ort import _partition_visuals
+
+    channels_first = {"array": np.ones((2, 16), dtype=np.float32), "sampling_rate": 16000}
+    channels_last = {"array": np.ones((16, 2), dtype=np.float32), "sampling_rate": 16000}
+
+    _, audios = _partition_visuals([channels_first, channels_last])
+
+    assert [arr.shape for arr, _ in audios] == [(16,), (16,)]
+
+
+def test_partition_visuals_falls_back_to_librosa_when_torchcodec_fails():
+    import io
+
+    import soundfile as sf
+
+    from olive.evaluator.lmms_ort import _partition_visuals
+
+    buffer = io.BytesIO()
+    sf.write(buffer, np.linspace(-0.5, 0.5, 800, dtype=np.float32), 8000, format="WAV")
+
+    class BrokenAudioDecoder:
+        def __init__(self):
+            self._desired_sample_rate = 16000
+            self._hf_encoded = {"bytes": buffer.getvalue(), "path": None}
+
+        @staticmethod
+        def get_all_samples():
+            raise RuntimeError("invalid packet")
+
+    _, audios = _partition_visuals([BrokenAudioDecoder()])
+
+    assert len(audios) == 1
+    array, sample_rate = audios[0]
+    assert array.dtype == np.float32
+    assert array.ndim == 1
+    assert sample_rate == 16000
+    assert len(array) == 1600
+
+
+def test_partition_visuals_rejects_video():
+    from olive.evaluator.lmms_ort import _partition_visuals
+
+    with pytest.raises(NotImplementedError, match="video"):
+        _partition_visuals(["sample.mp4"])
+
+
+@pytest.mark.parametrize(("source_rate", "sample_count"), [(192000, 192000), (44100, 44100)])
+def test_resample_audio_preserves_one_second_duration_at_16khz(source_rate, sample_count):
+    waveform = np.sin(2 * np.pi * 440 * np.arange(sample_count, dtype=np.float32) / source_rate)
+
+    resampled = _resample_audio(waveform, source_rate, 16000)
+
+    assert resampled.dtype == np.float32
+    assert len(resampled) == 16000
+
+
+def test_resample_audio_bypasses_matching_rate_sample_identically():
+    waveform = np.linspace(-1, 1, 16000, dtype=np.float32)
+
+    resampled = _resample_audio(waveform, 16000, 16000)
+
+    assert resampled is waveform
+    assert np.array_equal(resampled, waveform)
+
+
+def test_build_og_audios_downmixes_resamples_once_and_writes_target_rate(tmp_path):
+    import soundfile as sf
+    from scipy.signal import resample_poly
+
+    from olive.evaluator import lmms_ort as lmms_ort_module
+    from olive.evaluator.lmms_ort import _partition_visuals
+
+    stereo = np.stack(
+        [
+            np.sin(2 * np.pi * 220 * np.arange(44100, dtype=np.float32) / 44100),
+            np.sin(2 * np.pi * 440 * np.arange(44100, dtype=np.float32) / 44100),
+        ]
+    )
+    _, audios = _partition_visuals([{"array": stereo, "sampling_rate": 44100}])
+    evaluator = LMMSORTGenAIEvaluator.__new__(LMMSORTGenAIEvaluator)
+    evaluator._audio_target_sample_rate_explicit = True
+    evaluator.audio_target_sample_rate = 16000
+    evaluator.audio_target_sample_rates = (16000,)
+    evaluator._audio_sample_rate_error = None
+    captured = {}
+
+    def open_audio(path):
+        captured["array"], captured["sample_rate"] = sf.read(path, dtype="float32")
+        return "audio-handle"
+
+    fake_og = SimpleNamespace(Audios=SimpleNamespace(open=open_audio))
+    with (
+        patch.object(lmms_ort_module, "og", fake_og),
+        patch("scipy.signal.resample_poly", wraps=resample_poly) as resample_mock,
+    ):
+        handle = evaluator._build_og_audios(audios, tmp_path)
+
+    assert handle == "audio-handle"
+    resample_mock.assert_called_once()
+    assert captured["sample_rate"] == 16000
+    assert captured["array"].ndim == 1
+    assert len(captured["array"]) == 16000
+
+
+def test_build_og_audios_explicit_override_allows_upsampling(tmp_path):
+    import soundfile as sf
+
+    from olive.evaluator import lmms_ort as lmms_ort_module
+
+    evaluator = LMMSORTGenAIEvaluator.__new__(LMMSORTGenAIEvaluator)
+    evaluator._audio_target_sample_rate_explicit = True
+    evaluator.audio_target_sample_rate = 16000
+    evaluator.audio_target_sample_rates = (16000,)
+    evaluator._audio_sample_rate_error = None
+    waveform = np.zeros(8000, dtype=np.float32)
+    captured = {}
+
+    def open_audio(path):
+        captured["array"], captured["sample_rate"] = sf.read(path, dtype="float32")
+        return "audio-handle"
+
+    fake_og = SimpleNamespace(Audios=SimpleNamespace(open=open_audio))
+    with patch.object(lmms_ort_module, "og", fake_og):
+        handle = evaluator._build_og_audios([(waveform, 8000)], tmp_path)
+
+    assert handle == "audio-handle"
+    assert captured["sample_rate"] == 16000
+    assert len(captured["array"]) == 16000
+
+
+def test_build_og_audios_preserves_source_rate_for_package_processor(tmp_path):
+    import soundfile as sf
+    from scipy.signal import resample_poly
+
+    from olive.evaluator import lmms_ort as lmms_ort_module
+
+    evaluator = LMMSORTGenAIEvaluator.__new__(LMMSORTGenAIEvaluator)
+    evaluator._audio_target_sample_rate_explicit = False
+    evaluator.audio_target_sample_rate = 16000
+    evaluator.audio_target_sample_rates = (8000, 16000)
+    evaluator._audio_sample_rate_error = None
+    waveform = np.zeros(44100, dtype=np.float32)
+    captured = {}
+
+    def open_audio(path):
+        captured["array"], captured["sample_rate"] = sf.read(path, dtype="float32")
+        return "audio-handle"
+
+    fake_og = SimpleNamespace(Audios=SimpleNamespace(open=open_audio))
+    with (
+        patch.object(lmms_ort_module, "og", fake_og),
+        patch("scipy.signal.resample_poly", wraps=resample_poly) as resample_mock,
+    ):
+        handle = evaluator._build_og_audios([(waveform, 44100)], tmp_path)
+
+    assert handle == "audio-handle"
+    resample_mock.assert_not_called()
+    assert captured["sample_rate"] == 44100
+    assert len(captured["array"]) == 44100
+
+
+def _write_speech_package(tmp_path, speech_processor_config, *, config_filename="speech_config.json"):
+    model_dir = tmp_path / "model"
+    model_dir.mkdir()
+    genai_config = {
+        "model": {
+            "type": "gemma4",
+            "speech": {"filename": "audio/model.onnx", "config_filename": config_filename},
+        }
+    }
+    (model_dir / "genai_config.json").write_text(json.dumps(genai_config), encoding="utf-8")
+    (model_dir / config_filename).write_text(json.dumps(speech_processor_config), encoding="utf-8")
+    return model_dir, genai_config
+
+
+def test_resolve_model_audio_sample_rate_finds_nested_unambiguous_declaration(tmp_path):
+    model_dir, config = _write_speech_package(
+        tmp_path,
+        {"feature_extraction": {"sequence": [{"operation": {"attrs": {"sampling_rate": 16000}}}]}},
+    )
+
+    assert _resolve_model_audio_sample_rate(model_dir, config) == 16000
+
+
+def test_resolve_model_audio_sample_rate_supports_runtime_default_config_filename(tmp_path):
+    model_dir = tmp_path / "model"
+    model_dir.mkdir()
+    config = {"model": {"type": "whisper", "speech": {"filename": "encoder.onnx"}}}
+    (model_dir / "audio_processor_config.json").write_text('{"sample_rate": 16000}', encoding="utf-8")
+
+    assert _resolve_model_audio_sample_rate(model_dir, config) == 16000
+
+
+def test_resolve_model_audio_sample_rate_supports_audio_decoder_target_sample_rate(tmp_path):
+    model_dir, config = _write_speech_package(
+        tmp_path,
+        {"feature_extraction": {"sequence": [{"operation": {"attrs": {"target_sample_rate": 16000}}}]}},
+    )
+
+    assert _resolve_model_audio_sample_rates(model_dir, config) == (16000,)
+    assert _resolve_model_audio_sample_rate(model_dir, config) == 16000
+
+
+@pytest.mark.parametrize("attrs", [None, {"stereo_to_mono": 0}, {"max_samples": 240000}])
+def test_resolve_model_audio_sample_rate_supports_whisper_audio_decoder_default(tmp_path, attrs):
+    model_dir = tmp_path / "model"
+    model_dir.mkdir()
+    config = {"model": {"type": "whisper"}}
+    decoder = {"name": "audio_decoder", "type": "AudioDecoder"}
+    if attrs is not None:
+        decoder["attrs"] = attrs
+    (model_dir / "audio_processor_config.json").write_text(
+        json.dumps(
+            {
+                "feature_extraction": {
+                    "sequence": [
+                        {"operation": decoder},
+                    ]
+                }
+            }
+        ),
+        encoding="utf-8",
+    )
+
+    assert _resolve_model_audio_sample_rates(model_dir, config) == (16000,)
+    assert _resolve_model_audio_sample_rate(model_dir, config) == 16000
+
+
+def test_resolve_model_audio_sample_rate_rejects_whisper_without_audio_decoder_or_rate(tmp_path):
+    model_dir = tmp_path / "model"
+    model_dir.mkdir()
+    config = {"model": {"type": "whisper"}}
+    (model_dir / "audio_processor_config.json").write_text(
+        '{"feature_extraction": {"sequence": [{"operation": {"type": "STFTNorm"}}]}}',
+        encoding="utf-8",
+    )
+
+    with pytest.raises(ValueError, match="does not declare"):
+        _resolve_model_audio_sample_rates(model_dir, config)
+
+
+def test_resolve_model_audio_sample_rate_supports_runtime_target_sample_rates(tmp_path):
+    model_dir, config = _write_speech_package(
+        tmp_path,
+        {"feature_extraction": {"sequence": [{"operation": {"attrs": {"target_sample_rates": [8000, 16000]}}}]}},
+    )
+
+    assert _resolve_model_audio_sample_rates(model_dir, config) == (8000, 16000)
+    assert _resolve_model_audio_sample_rate(model_dir, config) == 16000
+
+
+@pytest.mark.parametrize(
+    ("source_sample_rate", "expected_target_sample_rate"),
+    [(8000, 8000), (12000, 8000), (16000, 16000), (44100, 16000), (192000, 16000)],
+)
+def test_select_audio_target_sample_rate_matches_audio_decoder_ex(source_sample_rate, expected_target_sample_rate):
+    evaluator = LMMSORTGenAIEvaluator.__new__(LMMSORTGenAIEvaluator)
+    evaluator.audio_target_sample_rate = 16000
+    evaluator.audio_target_sample_rates = (8000, 16000)
+
+    assert evaluator._select_audio_target_sample_rate(source_sample_rate) == expected_target_sample_rate
+
+
+def test_select_audio_target_sample_rate_rejects_source_below_supported_rates():
+    evaluator = LMMSORTGenAIEvaluator.__new__(LMMSORTGenAIEvaluator)
+    evaluator.audio_target_sample_rate = 16000
+    evaluator.audio_target_sample_rates = (8000, 16000)
+
+    with pytest.raises(ValueError, match="below every package-supported target rate"):
+        evaluator._select_audio_target_sample_rate(4000)
+
+
+@pytest.mark.parametrize(
+    ("processor_config", "message"),
+    [
+        ({"sampling_rate": "16000"}, "malformed"),
+        ({"sampling_rate": 0}, "malformed"),
+        ({"target_sample_rates": []}, "malformed"),
+        ({"target_sample_rates": [8000, "16000"]}, "malformed"),
+        ({"a": {"sampling_rate": 16000}, "b": {"sample_rate": 44100}}, "conflicting"),
+        ({"sampling_rate": 16000, "target_sample_rates": [8000]}, "conflicting"),
+        ({"feature_size": 128}, "does not declare"),
+    ],
+)
+def test_resolve_model_audio_sample_rate_rejects_bad_declarations(tmp_path, processor_config, message):
+    model_dir, config = _write_speech_package(tmp_path, processor_config)
+
+    with pytest.raises(ValueError, match=message):
+        _resolve_model_audio_sample_rate(model_dir, config)
+
+
+def test_build_og_audios_fails_when_model_has_no_speech_config(tmp_path):
+    fake_model = _FakeOgModel(tokenizer=_FakeTokenizer({}))
+    evaluator, og_patcher = _build_lmms_ortgenai_evaluator(tmp_path, fake_model)
+
+    with og_patcher, pytest.raises(ValueError, match=r"model audio sample rate.*model\.speech"):
+        evaluator._build_og_audios([(np.zeros(16000, dtype=np.float32), 16000)], tmp_path)
+
+
+def test_explicit_audio_target_sample_rate_allows_package_without_speech_config(tmp_path):
+    fake_model = _FakeOgModel(tokenizer=_FakeTokenizer({}))
+    evaluator, og_patcher = _build_lmms_ortgenai_evaluator(
+        tmp_path,
+        fake_model,
+        audio_target_sample_rate=16000,
+    )
+
+    with og_patcher:
+        handle = evaluator._build_og_audios([(np.zeros(16000, dtype=np.float32), 16000)], tmp_path)
+
+    assert evaluator.audio_target_sample_rate == 16000
+    assert handle[0] == "AUDIO"
+
+
+def _capture_serialized_image(evaluator, image, tmp_path):
+    from olive.evaluator import lmms_ort as lmms_ort_module
+
+    captured = {}
+
+    def open_image(path):
+        captured["bytes"] = Path(path).read_bytes()
+        with PIL.Image.open(path) as serialized:
+            captured["pixels"] = np.asarray(serialized.convert("RGB"))
+        return "image-handle"
+
+    with patch.object(lmms_ort_module, "og", SimpleNamespace(Images=SimpleNamespace(open=open_image))):
+        handle = evaluator._build_og_images([image], tmp_path)
+    return handle, captured
+
+
+def test_jpeg85_image_profile_serializes_jpeg_bytes_and_changes_pixels(tmp_path):
+    pixels = np.indices((32, 32)).sum(axis=0) % 2 * 255
+    image = PIL.Image.fromarray(np.uint8(np.stack([pixels, 255 - pixels, pixels], axis=-1)), mode="RGB")
+    evaluator = LMMSORTGenAIEvaluator.__new__(LMMSORTGenAIEvaluator)
+    evaluator.image_serialization_profile = "jpeg85"
+    evaluator._image_serialization_format = evaluator._resolve_image_serialization_format()
+
+    handle, captured = _capture_serialized_image(evaluator, image, tmp_path)
+
+    assert handle == "image-handle"
+    assert captured["bytes"].startswith(b"\xff\xd8")
+    assert not np.array_equal(captured["pixels"], np.asarray(image))
+
+
+def test_lossless_image_profile_preserves_png_pixels(tmp_path):
+    pixels = np.arange(16 * 16 * 3, dtype=np.uint8).reshape(16, 16, 3)
+    image = PIL.Image.fromarray(pixels, mode="RGB")
+    evaluator = LMMSORTGenAIEvaluator.__new__(LMMSORTGenAIEvaluator)
+    evaluator.image_serialization_profile = "lossless"
+    evaluator._image_serialization_format = evaluator._resolve_image_serialization_format()
+
+    _, captured = _capture_serialized_image(evaluator, image, tmp_path)
+
+    assert captured["bytes"].startswith(b"\x89PNG\r\n\x1a\n")
+    assert np.array_equal(captured["pixels"], pixels)
+
+
+def test_handle_error_logs_synthetic_exception_detail():
+    evaluator = LMMSORTGenAIEvaluator.__new__(LMMSORTGenAIEvaluator)
+    evaluator.fail_on_error = False
+
+    with patch("olive.evaluator.lmms_ort.logger.error") as error_mock:
+        result = evaluator._handle_error("Invalid completion budget.", ValueError("max_new_tokens=-1"), "")
+
+    assert result == ""
+    error_mock.assert_called_once()
+    assert error_mock.call_args.args[0] == "%s: %s"
+    assert error_mock.call_args.args[1] == "Invalid completion budget."
+    assert str(error_mock.call_args.args[2]) == "max_new_tokens=-1"
+
+
+def test_structured_prompt_preserves_placeholder_order():
+    inst = _make_evaluator_for_prompt_tests()
+    inst._tokenizer.apply_chat_template.return_value = "<rendered>"
+
+    inst._build_prompt_for_request("before <image> middle <audio> after", num_images=1, num_audios=1)
+
+    messages = json.loads(inst._tokenizer.apply_chat_template.call_args.args[0])
+    assert messages[-1]["content"] == [
+        {"type": "text", "text": "before "},
+        {"type": "image"},
+        {"type": "text", "text": " middle "},
+        {"type": "audio"},
+        {"type": "text", "text": " after"},
+    ]
+
+
+def _png_bytes(image):
+    import io as _io
+
+    buf = _io.BytesIO()
+    image.save(buf, format="PNG")
+    return buf.getvalue()
+
+
+# -----------------------------------------------------------------------------
+# Runtime: _run_generation
+#
+# These tests construct a LMMSORTGenAIEvaluator with the onnxruntime_genai
+# module wholesale-mocked, so the generation flow is exercised
+# end-to-end without needing an actual ORT-GenAI model on disk.
+# -----------------------------------------------------------------------------
+
+
+class _FakeTokenStream:
+    def decode(self, tok):
+        return f"<t{tok}>"
+
+
+class _FakeTokenizer:
+    """Minimal stub for og.Tokenizer used by _run_generation.
+
+    ``tokens_for`` is a dict mapping the exact input string to the list of
+    token ids encode() should return; this lets each test inject specific
+    tokenizations to drive generation behavior.
+    """
+
+    def __init__(self, tokens_for, eos_token_ids=(99,)):
+        self._tokens_for = tokens_for
+        self.eos_token_ids = list(eos_token_ids)
+
+    def encode(self, text):
+        if text not in self._tokens_for:
+            raise KeyError(f"_FakeTokenizer: no canned tokenization for text={text!r}")
+        return list(self._tokens_for[text])
+
+    def create_stream(self):
+        return _FakeTokenStream()
+
+    def decode(self, token_ids):
+        return "".join(f"<t{tok}>" for tok in token_ids)
+
+    def apply_chat_template(self, messages, add_generation_prompt=True):
+        media_token = "<|image|>" if '"type": "image"' in messages else ""
+        return f"{media_token}x"
+
+
+class _FakeGenerator:
+    """Records call order so tests can assert the score/generation protocol.
+
+    ``logits_queue`` is a list of numpy arrays consumed in order, one per forward
+    pass: ``generate_next_token`` and ``append_tokens`` each consume one entry
+    into ``_current_logits``, which ``get_logits()`` returns.
+    """
+
+    instances: ClassVar[list] = []
+
+    def __init__(self, model, params):
+        type(self).instances.append(self)
+        self._model = model
+        self._params = params
+        self._logits_queue = list(model._next_logits_queue)
+        self._sampled_queue = list(model._next_sampled_queue)
+        self._current_logits = None
+        self._token_count = 0
+        self._done = False
+        self._last_sampled = -1
+        self.calls = []
+
+    def _consume_forward_pass(self):
+        if not self._logits_queue:
+            raise RuntimeError("_FakeGenerator: forward pass exhausted (logits_queue empty)")
+        self._current_logits = self._logits_queue.pop(0)
+
+    def set_inputs(self, inputs):
+        self.calls.append(("set_inputs", inputs))
+        self._token_count = int(inputs["input_ids"].shape()[-1])
+        # ORT-GenAI SetInputs appends prompt tokens and computes prompt logits.
+        self._consume_forward_pass()
+
+    def generate_next_token(self):
+        if self._current_logits is None:
+            self._consume_forward_pass()
+        sampled = self._sampled_queue.pop(0) if self._sampled_queue else -1
+        self._last_sampled = sampled
+        self._token_count += 1
+        self.calls.append(("generate_next_token", sampled))
+        self._current_logits = None
+
+    def get_logits(self):
+        self.calls.append(("get_logits",))
+        if self._current_logits is None:
+            raise RuntimeError("_FakeGenerator: get_logits called before any forward pass")
+        return np.asarray(self._current_logits, dtype=np.float32)
+
+    def get_next_tokens(self):
+        return np.array([self._last_sampled], dtype=np.int32)
+
+    def append_tokens(self, tok_array):
+        toks = [int(t) for t in np.asarray(tok_array).reshape(-1)]
+        self.calls.append(("append_tokens", toks))
+        self._token_count += len(toks)
+        # Each appended token batch is one forward pass (computes new last-position logits).
+        self._consume_forward_pass()
+
+    def token_count(self):
+        return self._token_count
+
+    def is_done(self):
+        max_length = self._params.search_options.get("max_length")
+        return self._done or (max_length is not None and self._token_count >= max_length)
+
+    def rewind_to(self, n):
+        self.calls.append(("rewind_to", n))
+        self._token_count = n
+
+
+class _FakeGeneratorParams:
+    def __init__(self, model):
+        self._model = model
+        self.search_options = {}
+
+    def set_search_options(self, **kwargs):
+        self.search_options.update(kwargs)
+
+
+class _FakeTensor:
+    def __init__(self, shape):
+        self._shape = shape
+
+    def shape(self):
+        return self._shape
+
+
+class _FakeProcessor:
+    def __init__(self, input_length):
+        self._input_length = input_length
+
+    def __call__(self, prompt, images=None, audios=None):
+        return {
+            "input_ids": _FakeTensor([1, self._input_length]),
+            "_prompt": prompt,
+            "_images": images,
+            "_audios": audios,
+        }
+
+
+class _FakeOgModel:
+    def __init__(self, tokenizer=None, next_logits_queue=None, next_sampled_queue=None, processed_input_length=4):
+        self.tokenizer = tokenizer
+        self._next_logits_queue = list(next_logits_queue or [])
+        self._next_sampled_queue = list(next_sampled_queue or [])
+        self.processed_input_length = processed_input_length
+
+    def create_multimodal_processor(self):
+        return _FakeProcessor(self.processed_input_length)
+
+
+def _make_fake_og(model):
+    """Build a SimpleNamespace mimicking the onnxruntime_genai module surface.
+
+    Covers the API surface used by LMMSORTGenAIEvaluator's __init__ + runtime paths.
+    """
+    return SimpleNamespace(
+        Model=lambda *a, **kw: model,
+        Tokenizer=lambda m: model.tokenizer,
+        Generator=_FakeGenerator,
+        GeneratorParams=_FakeGeneratorParams,
+        Images=SimpleNamespace(open=lambda *paths: ("IMG", list(paths))),
+        Audios=SimpleNamespace(open=lambda *paths: ("AUDIO", list(paths))),
+        Config=lambda *a, **kw: SimpleNamespace(
+            clear_providers=lambda: None,
+            append_provider=lambda *a, **kw: None,
+            set_provider_option=lambda *a, **kw: None,
+        ),
+    )
+
+
+def _build_lmms_ortgenai_evaluator(
+    tmp_path,
+    fake_model,
+    *,
+    model_type="phi4mm",
+    tokenizer_config=None,
+    **evaluator_kwargs,
+):
+    """Construct an LMMSORTGenAIEvaluator wired to a fake onnxruntime_genai.
+
+    Returns ``(evaluator, og_patcher)``. The patcher is a context manager that
+    swaps in the fake ``og`` module; tests should call evaluator methods inside
+    ``with og_patcher: ...`` so ``_run_generation`` uses the fake instead of
+    the real ORT-GenAI.
+    """
+    from olive.evaluator import lmms_ort as lmms_ort_mod
+
+    model_dir = tmp_path / "model"
+    model_dir.mkdir()
+    (model_dir / "genai_config.json").write_text(
+        json.dumps({"model": {"type": model_type}}),
+        encoding="utf-8",
+    )
+    if tokenizer_config is not None:
+        (model_dir / "tokenizer_config.json").write_text(
+            json.dumps(tokenizer_config),
+            encoding="utf-8",
+        )
+
+    fake_og = _make_fake_og(fake_model)
+    _FakeGenerator.instances = []
+    og_patcher = patch.object(lmms_ort_mod, "og", fake_og)
+    with og_patcher:
+        evaluator = LMMSORTGenAIEvaluator(
+            pretrained=str(model_dir),
+            batch_size=1,
+            max_new_tokens=8,
+            max_length=64,
+            execution_provider="cpu",
+            fail_on_error=True,
+            **evaluator_kwargs,
+        )
+    # Return a fresh patcher for the test to use during runtime calls.
+    return evaluator, patch.object(lmms_ort_mod, "og", fake_og)
+
+
+def test_loglikelihood_is_not_supported():
+    evaluator = LMMSORTGenAIEvaluator.__new__(LMMSORTGenAIEvaluator)
+
+    with pytest.raises(NotImplementedError, match="does not support lmms-eval loglikelihood"):
+        evaluator.loglikelihood([], disable_tqdm=True)
+
+
+def test_run_generation_stops_on_eos_token(tmp_path):
+    """Verify ``_run_generation`` stops at EOS tokens.
+
+    ``_run_generation`` must respect ``_eos_token_ids`` and stop emitting text
+    once the model samples an EOS token, even before max_new_tokens is reached.
+    """
+    fake_tokenizer = _FakeTokenizer({}, eos_token_ids=[99])
+    logits = np.zeros(100, dtype=np.float32)
+    fake_model = _FakeOgModel(
+        tokenizer=fake_tokenizer,
+        next_logits_queue=[logits] * 5,
+        next_sampled_queue=[7, 8, 99, 10, 11],  # third sample is EOS
+    )
+
+    evaluator, og_patcher = _build_lmms_ortgenai_evaluator(tmp_path, fake_model)
+    with og_patcher:
+        out = evaluator._run_generation("p", images=[], audios=[], max_new_tokens=5)
+
+    # Stream produces "<t{tok}>" per non-EOS token; EOS stops generation.
+    assert out == "<t7><t8>"
+    gen = _FakeGenerator.instances[-1]
+    op_names = [c[0] for c in gen.calls]
+    # Exactly 3 generate_next_token calls happened (7, 8, then EOS stops loop).
+    assert op_names.count("generate_next_token") == 3
+    assert gen._params.search_options["max_length"] == 9
+
+
+def test_run_generation_media_preprocessing_honors_fail_on_error_false():
+    evaluator = LMMSORTGenAIEvaluator.__new__(LMMSORTGenAIEvaluator)
+    evaluator.fail_on_error = False
+    evaluator._build_og_images = MagicMock(return_value=None)
+    evaluator._build_og_audios = MagicMock(side_effect=ValueError("unsupported audio rate"))
+
+    output = evaluator._run_generation("prompt", images=[], audios=[object()], max_new_tokens=1)
+
+    assert output == ""
+
+
+def test_init_resolves_gemma_response_channel_token_ids(tmp_path):
+    fake_tokenizer = _FakeTokenizer(
+        {
+            "<|channel>": [100],
+            "<channel|>": [101],
+        }
+    )
+    fake_model = _FakeOgModel(tokenizer=fake_tokenizer)
+
+    evaluator, _ = _build_lmms_ortgenai_evaluator(
+        tmp_path,
+        fake_model,
+        model_type="gemma4",
+        tokenizer_config={
+            "soc_token": "<|channel>",
+            "eoc_token": "<channel|>",
+        },
+    )
+
+    assert evaluator._uses_gemma_response_channels is True
+    assert evaluator._gemma_channel_start_token_id == 100
+    assert evaluator._gemma_channel_end_token_id == 101
+
+
+def test_init_rejects_gemma_response_channel_that_is_not_one_token(tmp_path):
+    fake_tokenizer = _FakeTokenizer(
+        {
+            "<|channel>": [100, 102],
+            "<channel|>": [101],
+        }
+    )
+    fake_model = _FakeOgModel(tokenizer=fake_tokenizer)
+
+    with pytest.raises(ValueError, match="must each map to exactly one tokenizer ID"):
+        _build_lmms_ortgenai_evaluator(
+            tmp_path,
+            fake_model,
+            model_type="gemma4",
+            tokenizer_config={
+                "soc_token": "<|channel>",
+                "eoc_token": "<channel|>",
+            },
+        )
+
+
+def test_run_generation_returns_only_gemma_content_after_thought_channel(tmp_path):
+    fake_tokenizer = _FakeTokenizer({}, eos_token_ids=[99])
+    logits = np.zeros(110, dtype=np.float32)
+    fake_model = _FakeOgModel(
+        tokenizer=fake_tokenizer,
+        next_logits_queue=[logits] * 6,
+        next_sampled_queue=[100, 7, 8, 101, 9, 99],
+    )
+    evaluator, og_patcher = _build_lmms_ortgenai_evaluator(tmp_path, fake_model)
+    evaluator._model_type = "gemma4"
+    evaluator._uses_gemma_response_channels = True
+    evaluator._gemma_channel_start_token_id = 100
+    evaluator._gemma_channel_end_token_id = 101
+
+    with og_patcher:
+        output = evaluator._run_generation("p", images=[], audios=[], max_new_tokens=6)
+
+    assert output == "<t9>"
+
+
+def test_run_generation_returns_empty_for_truncated_gemma_thought_channel(tmp_path):
+    fake_tokenizer = _FakeTokenizer({}, eos_token_ids=[])
+    logits = np.zeros(110, dtype=np.float32)
+    fake_model = _FakeOgModel(
+        tokenizer=fake_tokenizer,
+        next_logits_queue=[logits] * 3,
+        next_sampled_queue=[100, 7, 8],
+    )
+    evaluator, og_patcher = _build_lmms_ortgenai_evaluator(tmp_path, fake_model)
+    evaluator._model_type = "gemma4"
+    evaluator._uses_gemma_response_channels = True
+    evaluator._gemma_channel_start_token_id = 100
+    evaluator._gemma_channel_end_token_id = 101
+
+    with og_patcher:
+        output = evaluator._run_generation("p", images=[], audios=[], max_new_tokens=3)
+
+    assert output == ""
+
+
+def test_decode_generated_tokens_returns_empty_without_response_tokens():
+    evaluator = LMMSORTGenAIEvaluator.__new__(LMMSORTGenAIEvaluator)
+    evaluator._tokenizer = SimpleNamespace(decode=MagicMock(side_effect=AssertionError("decode must not be called")))
+    evaluator._uses_gemma_response_channels = True
+    evaluator._gemma_channel_start_token_id = 10
+    evaluator._gemma_channel_end_token_id = 11
+
+    assert evaluator._decode_generated_tokens([]) == ""
+    assert evaluator._decode_generated_tokens([10, 11]) == ""
+    evaluator._tokenizer.decode.assert_not_called()
+
+
+def test_run_generation_clamps_budget_to_remaining_context(tmp_path):
+    fake_tokenizer = _FakeTokenizer({}, eos_token_ids=[])
+    logits = np.zeros(10, dtype=np.float32)
+    fake_model = _FakeOgModel(
+        tokenizer=fake_tokenizer,
+        next_logits_queue=[logits] * 2,
+        next_sampled_queue=[7, 8, 9],
+        processed_input_length=62,
+    )
+    evaluator, og_patcher = _build_lmms_ortgenai_evaluator(tmp_path, fake_model)
+
+    with og_patcher, patch("olive.evaluator.lmms_ort.logger.warning") as warning_mock:
+        output = evaluator._run_generation("p", images=[], audios=[], max_new_tokens=5)
+
+    assert output == "<t7><t8>"
+    generator = _FakeGenerator.instances[-1]
+    assert generator._params.search_options["max_length"] == 64
+    warning_mock.assert_called_once_with(
+        "Reducing max_new_tokens from %d to %d because the processed prompt uses %d of max_length=%d.",
+        5,
+        2,
+        62,
+        64,
+    )
+
+
+def test_run_generation_rejects_prompt_at_context_ceiling(tmp_path):
+    fake_model = _FakeOgModel(
+        tokenizer=_FakeTokenizer({}, eos_token_ids=[]),
+        processed_input_length=64,
+    )
+    evaluator, og_patcher = _build_lmms_ortgenai_evaluator(tmp_path, fake_model)
+
+    with og_patcher, pytest.raises(RuntimeError, match="processed prompt exceeds max_length"):
+        evaluator._run_generation("p", images=[], audios=[], max_new_tokens=1)
+
+    assert not _FakeGenerator.instances
+
+
+def test_run_generation_reports_set_inputs_failure_without_prompt_length_guess(tmp_path):
+    fake_model = _FakeOgModel(
+        tokenizer=_FakeTokenizer({}, eos_token_ids=[]),
+        next_logits_queue=[],
+    )
+    evaluator, og_patcher = _build_lmms_ortgenai_evaluator(tmp_path, fake_model)
+
+    with og_patcher, pytest.raises(RuntimeError, match="generator input setup failed") as exc_info:
+        evaluator._run_generation("p", images=[], audios=[], max_new_tokens=1)
+
+    assert "prompt may exceed" not in str(exc_info.value).lower()
+
+
+def test_run_generation_stops_on_explicit_stop_string(tmp_path):
+    """``stop_strings`` should truncate output as soon as a stop sequence appears."""
+    fake_tokenizer = _FakeTokenizer({}, eos_token_ids=[])
+    logits = np.zeros(10, dtype=np.float32)
+    fake_model = _FakeOgModel(
+        tokenizer=fake_tokenizer,
+        next_logits_queue=[logits] * 4,
+        next_sampled_queue=[1, 2, 3, 4],
+    )
+
+    evaluator, og_patcher = _build_lmms_ortgenai_evaluator(tmp_path, fake_model)
+    # _FakeTokenStream emits "<t1><t2><t3>..."; "<t2>" appears after the 2nd token.
+    with og_patcher:
+        out = evaluator._run_generation("p", images=[], audios=[], max_new_tokens=10, stop_strings=["<t2>"])
+
+    # Output is truncated at (but not including) the stop string.
+    assert out == "<t1>"
diff --git a/test/evaluator/test_olive_evaluator.py b/test/evaluator/test_olive_evaluator.py
index 0c3bc2157b..169074b2f0 100644
--- a/test/evaluator/test_olive_evaluator.py
+++ b/test/evaluator/test_olive_evaluator.py
@@ -20,6 +20,7 @@
     OpenVINOEvaluator,
     PyTorchEvaluator,
     _is_vision_metric,
+    _summarize_lmms_samples,
     _validate_vision_task_metric,
 )
 from olive.exception import OliveEvaluationError
@@ -110,6 +111,23 @@ class TestOliveEvaluator:
         ),
     ]
 
+    def test_summarize_lmms_samples_preserves_loglikelihood_response(self):
+        summary = _summarize_lmms_samples(
+            "mmlu_abstract_algebra",
+            [
+                {
+                    "doc_id": 0,
+                    "target": 1,
+                    "resps": [[(-1.25, False), (-0.5, True)]],
+                    "filtered_resps": [1],
+                    "acc": 1.0,
+                }
+            ],
+        )
+
+        assert summary["n_empty_generations"] == 0
+        assert summary["rows"][0]["generated"] == [-1.25, False]
+
     @pytest.mark.parametrize(
         ("evaluator", "model_loader", "metric_func", "acc_subtype", "expected_res"),
         ACCURACY_TEST_CASE,
diff --git a/test/requirements-test.txt b/test/requirements-test.txt
index bebf4bb260..95b5392869 100644
--- a/test/requirements-test.txt
+++ b/test/requirements-test.txt
@@ -8,6 +8,8 @@ datasets
 diffusers
 docker>=7.1.0
 evaluate
+# Includes extensionless task templates omitted from the 0.7.2 PyPI wheel (lmms-eval#1390).
+lmms-eval[audio,metrics] @ git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git@3e675904f8cba6793de12b91979b04d91754bdf3
 marshmallow
 ml_dtypes
 mlflow

From 08a6a2b2684d69d91061b5135eb0acd97bfacd73 Mon Sep 17 00:00:00 2001
From: Delwin Kim <139003345+DelwinKim@users.noreply.github.com>
Date: Thu, 6 Aug 2026 23:58:22 +0000
Subject: [PATCH 3/4] Document lmms evaluator usage

Describe the evaluator boundary, upstream dependency pin, supported handlers, custom task loading, and current runtime limitations.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 .../metrics-configuration.md                  | 90 +++++++++++++++++++
 1 file changed, 90 insertions(+)

diff --git a/docs/source/how-to/configure-workflows/metrics-configuration.md b/docs/source/how-to/configure-workflows/metrics-configuration.md
index 710d03ccfe..06824ceb16 100644
--- a/docs/source/how-to/configure-workflows/metrics-configuration.md
+++ b/docs/source/how-to/configure-workflows/metrics-configuration.md
@@ -281,3 +281,93 @@ Olive provides three built-in accuracy sub-types for evaluating vision/multimoda
 - Multiple valid answers (lists) are joined with `|` and the metric matches against any valid answer.
 - For ONNX models, provide a custom pre-process that applies the processor/tokenizer to produce numeric tensors.
 ```
+
+## Standard Multimodal Benchmarks with lmms-eval
+
+Use `LMMSEvaluator` when a public image or audio generation benchmark is available
+in [lmms-eval](https://github.com/EvolvingLMMs-Lab/lmms-eval). It delegates task
+loading, prompting, and scoring to lmms-eval so results use the benchmark's
+declared protocol and metric, such as DocVQA ANLS, TextVQA accuracy, WER, or
+BLEU.
+
+Olive's existing `OnnxEvaluator` and custom evaluator support remain the
+appropriate choices for proprietary tasks, lightweight smoke tests, and
+workflows that should not depend on lmms-eval. `LMMSEvaluator` does not
+deprecate those paths.
+
+Install Olive and the pinned upstream lmms-eval dependency before using this
+evaluator:
+
+```bash
+pip install olive-ai
+pip install \
+  "lmms-eval[audio,metrics] @ git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git@3e675904f8cba6793de12b91979b04d91754bdf3"
+```
+
+Olive cannot publish a package extra containing a direct Git dependency. The
+installation command therefore pins the upstream lmms-eval commit containing
+[the wheel package-data fix](https://github.com/EvolvingLMMs-Lab/lmms-eval/pull/1390).
+The published `0.7.2` wheel omits extensionless task templates and cannot load
+its default task registry. A normal `olive-ai[lmms-eval]` extra can replace this
+command after upstream publishes a release containing that fix.
+
+For an ONNX input, also install the ORT-GenAI package for the target provider,
+such as `onnxruntime-genai` or `onnxruntime-genai-cuda`.
+
+`LMMSEvaluator` accepts:
+
+- an `HfModelHandler`, dispatched to an upstream lmms-eval model wrapper; or
+- an `ONNXModelHandler` that represents a complete ORT-GenAI package containing
+  `genai_config.json` and all referenced model components.
+
+`MobiusBuilder` returns a `CompositeModelHandler`. Add
+`CompositeToOnnxPackage` after it to preserve the package layout while exposing
+an `ONNXModelHandler` to the evaluator:
+
+```json
+{
+    "passes": {
+        "build": {
+            "type": "MobiusBuilder"
+        },
+        "as_onnx_package": {
+            "type": "CompositeToOnnxPackage"
+        }
+    },
+    "evaluators": {
+        "multimodal_benchmarks": {
+            "type": "LMMSEvaluator",
+            "tasks": ["ai2d"],
+            "batch_size": 1,
+            "log_samples": true,
+            "output_path": "results/lmms_eval.json"
+        }
+    },
+    "evaluator": "multimodal_benchmarks"
+}
+```
+
+Use `include_path` with one directory or a list of directories to load custom
+lmms-eval tasks. Custom task names must not collide with built-in task names.
+Public benchmark fixes should be contributed to lmms-eval rather than hidden
+behind a colliding local task.
+
+The following limitations apply:
+
+- Raw single-file ONNX models are not supported because they do not contain the
+  ORT-GenAI multimodal preprocessing pipeline.
+- The ORT-GenAI adapter processes requests individually. It does not implement
+  batched generation, video, multi-round or interleaved generation, beam
+  search, multiple return sequences, or loglikelihood tasks.
+- The adapter is registered through lmms-eval's legacy model registry for
+  in-process use by Olive. It is not an lmms-eval command-line entry point.
+- Automatic Hugging Face dispatch is limited to model wrappers in the pinned
+  upstream lmms-eval release. Set `model_class` explicitly for another wrapper
+  that is registered in the installed lmms-eval environment.
+- `HfModelHandler.adapter_path` and handler `load_kwargs` are rejected rather
+  than silently ignored. Merge an adapter into the checkpoint first, and pass
+  wrapper-supported constructor options through `hf_model_kwargs`.
+
+`image_serialization_profile` accepts `lossless` (PNG, the default) or
+`jpeg85`. ONNX audio uses the sample rates declared by the ORT-GenAI package;
+`audio_target_sample_rate` is an explicit host-resampling override.

From 765fe9a689a8a6793a03e9c54ceb4c999144c8b8 Mon Sep 17 00:00:00 2001
From: Delwin Kim <139003345+DelwinKim@users.noreply.github.com>
Date: Fri, 7 Aug 2026 23:50:03 +0000
Subject: [PATCH 4/4] Preserve sibling artifacts on package overwrite

Replace only ORT-GenAI package-owned output entries so CLI test workflows retain their saved reference model.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 olive/cache.py     | 21 +++++++++++++--------
 test/test_cache.py | 11 ++++++++++-
 2 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/olive/cache.py b/olive/cache.py
index e9b8ea04c5..8d25e6f402 100644
--- a/olive/cache.py
+++ b/olive/cache.py
@@ -565,14 +565,19 @@ def _save_model(
                 entry_path = Path(get_onnx_file_path(str(source_path), onnx_file_name)).resolve()
                 if not entry_path.is_relative_to(package_root) or not entry_path.is_file():
                     raise ValueError(f"ORT-GenAI package entry point is invalid: {entry_path}")
-                if resolved_output_dir.exists():
-                    if any(resolved_output_dir.iterdir()) and not overwrite:
-                        raise FileExistsError(f"Output directory is not empty: {resolved_output_dir}")
-                    if any(resolved_output_dir.iterdir()):
-                        shutil.rmtree(resolved_output_dir)
-                    else:
-                        resolved_output_dir.rmdir()
-                shutil.copytree(package_root, resolved_output_dir)
+                if resolved_output_dir.exists() and any(resolved_output_dir.iterdir()) and not overwrite:
+                    raise FileExistsError(f"Output directory is not empty: {resolved_output_dir}")
+                resolved_output_dir.mkdir(parents=True, exist_ok=True)
+                if overwrite:
+                    # Replace package-owned entries without deleting sibling run artifacts such as
+                    # the reference model saved by the CLI's --test workflow.
+                    for source_item in package_root.iterdir():
+                        output_item = resolved_output_dir / source_item.name
+                        if output_item.is_dir() and not output_item.is_symlink():
+                            shutil.rmtree(output_item)
+                        elif output_item.exists() or output_item.is_symlink():
+                            output_item.unlink()
+                shutil.copytree(package_root, resolved_output_dir, dirs_exist_ok=True)
                 model_json["config"]["model_path"] = str(resolved_output_dir)
                 model_json["config"]["onnx_file_name"] = entry_path.relative_to(package_root).as_posix()
                 return self._save_additional_files(model_json, resolved_output_dir)
diff --git a/test/test_cache.py b/test/test_cache.py
index 1a5ae58121..316aa61184 100644
--- a/test/test_cache.py
+++ b/test/test_cache.py
@@ -463,12 +463,21 @@ def test_save_model_honors_overwrite_for_ort_genai_package(self, tmp_path):
         output_dir.mkdir()
         stale_file = output_dir / "stale.txt"
         stale_file.write_text("stale", encoding="utf-8")
+        reference_model = output_dir / "reference_hf_model"
+        reference_model.mkdir()
+        (reference_model / "model.safetensors").write_text("reference", encoding="utf-8")
+        stale_decoder_file = output_dir / "decoder" / "stale.bin"
+        stale_decoder_file.parent.mkdir()
+        stale_decoder_file.write_text("stale decoder", encoding="utf-8")
 
         with pytest.raises(FileExistsError, match="Output directory is not empty"):
             cache.save_model(model_id, output_dir, overwrite=False)
 
         cache.save_model(model_id, output_dir, overwrite=True)
-        assert not stale_file.exists()
+        assert stale_file.read_text(encoding="utf-8") == "stale"
+        assert (reference_model / "model.safetensors").read_text(encoding="utf-8") == "reference"
+        assert not stale_decoder_file.exists()
+        assert (output_dir / "decoder" / "model.onnx").read_text(encoding="utf-8") == "decoder"
         assert (output_dir / "genai_config.json").is_file()
 
     def test_save_model_rejects_ort_genai_package_file_output(self, tmp_path):