diff --git a/olive/evaluator/lmms_ort.py b/olive/evaluator/lmms_ort.py
new file mode 100644
index 000000000..f1f8948c2
--- /dev/null
+++ b/olive/evaluator/lmms_ort.py
@@ -0,0 +1,876 @@
+# -------------------------------------------------------------------------
+# ORT-GenAI wrapper for lmms-eval multimodal evaluation
+#
+# Supports evaluating multimodal ONNX
+# models through the EvolvingLMMs-Lab/lmms-eval harness, mirroring how
+# olive/evaluator/lmeval_ort.py wraps lm-evaluation-harness for text models.
+#
+# Registers an LMMSORTGenAIEvaluator class with lmms-eval's legacy
+# @register_model registry under the name "ortgenai_mm". Consumers obtain it
+# via lmms_eval.api.registry.get_model("ortgenai_mm").
+# -------------------------------------------------------------------------
+"""ORT-GenAI wrapper for lmms-eval multimodal evaluation."""
+
+from __future__ import annotations
+
+import io
+import json
+import logging
+import re
+import tempfile
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+import PIL.Image
+from tqdm import tqdm
+
+try:
+    import onnxruntime_genai as og
+except ImportError:  # pragma: no cover - optional dep
+    og = None
+
+try:
+    from lmms_eval.api.instance import Instance
+    from lmms_eval.api.model import lmms
+    from lmms_eval.api.registry import register_model
+
+    _LMMS_EVAL_IMPORT_ERROR = None
+except ImportError as e:  # pragma: no cover - optional dep
+    Instance = Any
+    _LMMS_EVAL_IMPORT_ERROR = e
+
+    class lmms:  # noqa: N801
+        pass
+
+    def register_model(_name):
+        def decorator(cls):
+            return cls
+
+        return decorator
+
+
+logger = logging.getLogger(__name__)
+
+
+@lru_cache(maxsize=1)
+def warmup_image_decoder() -> None:
+    """Force ORT-GenAI's image decoder to initialize once, early in the process.
+
+    ORT-GenAI's ``ImageDecoder`` statically links its own zlib. Pillow, torch,
+    ``decord`` and other libraries loaded by the HuggingFace evaluation path also
+    bundle their own zlib. Because these are static copies of the same symbols
+    (an ODR violation), the *first* library to actually decode an image binds its
+    zlib process-wide. When ``LMMSEvaluator`` runs an HF input-model evaluation
+    before the ORT-GenAI (ONNX) evaluation in the same process
+    (``evaluate_input_model: true``), the HF path decodes images first, so
+    ORT-GenAI's PNG decoder later binds the wrong zlib and fails with
+    ``libpng error: bad parameters to zlib`` on every image.
+
+    Decoding a tiny throwaway PNG through ORT-GenAI here — before the HF path
+    runs — makes ORT-GenAI's zlib win the binding, so the later ONNX evaluation
+    decodes correctly. Idempotent and best-effort: a no-op if onnxruntime-genai
+    is unavailable or the warmup fails (nothing is broken for HF-only usage).
+    """
+    if og is None:
+        return
+    try:
+        with tempfile.TemporaryDirectory() as tmp:
+            path = Path(tmp) / "warmup.png"
+            PIL.Image.new("RGB", (8, 8), (0, 0, 0)).save(path, format="PNG")
+            # og.Images.open touches ORT-GenAI's libpng/zlib, binding its symbols.
+            og.Images.open(str(path))
+    except Exception as e:  # pragma: no cover - defensive, must never break a run
+        logger.debug("ORT-GenAI image decoder warmup skipped: %s", e)
+
+
+_PROVIDER_ALIASES = {
+    "cuda": "cuda",
+    "cudaexecutionprovider": "cuda",
+    "gpu": "cuda",
+    "cpu": "cpu",
+    "cpuexecutionprovider": "cpu",
+    "dml": "dml",
+    "dmlexecutionprovider": "dml",
+    "directml": "dml",
+    "webgpu": "webgpu",
+    "webgpuexecutionprovider": "webgpu",
+    "js": "web",
+    "jsexecutionprovider": "web",
+    "nvtensorrtrtx": "NvTensorRtRtx",
+    "nvtensorrtrtxexecutionprovider": "NvTensorRtRtx",
+}
+
+
+# -----------------------------------------------------------------------------
+# Helpers
+# -----------------------------------------------------------------------------
+
+
+_IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".webp", ".bmp", ".gif"}
+_AUDIO_SUFFIXES = {".wav", ".mp3", ".flac", ".ogg", ".m4a"}
+_VIDEO_SUFFIXES = {".mp4", ".avi", ".mov", ".mkv", ".webm"}
+_MEDIA_PLACEHOLDER_RE = re.compile(r"(<image>|<audio>)")
+
+
+def _normalize_image(visual) -> PIL.Image.Image | None:
+    """Coerce an lmms-eval visual into an RGB PIL image, or None if it isn't an image.
+
+    Accepts PIL images, file paths, ``{"bytes"|"path"}`` dicts, and numpy arrays;
+    audio-shaped dicts (with ``array``/``sampling_rate``) are rejected here.
+    """
+    if isinstance(visual, PIL.Image.Image):
+        return visual.convert("RGB")
+    if isinstance(visual, (str, Path)):
+        p = Path(visual)
+        if p.suffix.lower() in _IMAGE_SUFFIXES:
+            return PIL.Image.open(p).convert("RGB")
+        return None
+    if isinstance(visual, dict):
+        # Audio dicts typically include "sampling_rate" or "array"; skip those.
+        if "sampling_rate" in visual or "array" in visual:
+            return None
+        if "bytes" in visual:
+            return PIL.Image.open(io.BytesIO(visual["bytes"])).convert("RGB")
+        if "path" in visual:
+            p = Path(visual["path"]) if visual["path"] else None
+            if p is not None and p.suffix.lower() in _IMAGE_SUFFIXES:
+                return PIL.Image.open(p).convert("RGB")
+            return None
+    if isinstance(visual, np.ndarray):
+        return PIL.Image.fromarray(np.uint8(visual)).convert("RGB")
+    return None
+
+
+def _normalize_audio(visual) -> tuple[np.ndarray, int] | None:
+    """Coerce an lmms-eval visual into ``(mono float32 waveform, sample_rate)``, or None.
+
+    Handles the legacy ``{"array", "sampling_rate"}`` dict, file paths, and
+    HF datasets 5.x ``torchcodec`` AudioDecoder objects (duck-typed, downmixed to mono).
+    """
+    if isinstance(visual, dict):
+        if "array" in visual and "sampling_rate" in visual:
+            return _normalize_audio_array(visual["array"]), int(visual["sampling_rate"])
+        if visual.get("path"):
+            return _load_audio_file(Path(visual["path"]))
+    if isinstance(visual, (str, Path)):
+        return _load_audio_file(Path(visual))
+    # torchcodec.decoders.AudioDecoder — HF datasets 5.x returns this for the
+    # "audio" feature instead of the legacy {"array", "sampling_rate"} dict.
+    # Detect by duck-typing the get_all_samples() method to avoid a hard
+    # torchcodec import (it's an optional install).
+    if hasattr(visual, "get_all_samples"):
+        try:
+            samples = visual.get_all_samples()
+            # samples.data is a torch.Tensor of shape [channels, num_samples].
+            # ORT-GenAI's processor wants mono float32; downmix if multichannel.
+            return _normalize_audio_array(samples.data.detach().cpu().numpy()), int(samples.sample_rate)
+        except Exception as e:  # pragma: no cover
+            logger.warning("Failed to decode AudioDecoder visual: %s", e)
+            return None
+    return None
+
+
+def _normalize_audio_array(array) -> np.ndarray:
+    """Return a contiguous mono float32 waveform from common audio layouts."""
+    arr = np.asarray(array, dtype=np.float32)
+    if arr.ndim == 2:
+        if arr.shape[0] <= 8:
+            arr = arr.mean(axis=0)
+        elif arr.shape[1] <= 8:
+            arr = arr.mean(axis=1)
+        else:
+            raise ValueError(f"Cannot infer channel axis for audio array with shape {arr.shape}.")
+    if arr.ndim != 1:
+        raise ValueError(f"Audio array must be one- or two-dimensional, got shape {arr.shape}.")
+    return np.ascontiguousarray(arr, dtype=np.float32)
+
+
+def _load_audio_file(p: Path) -> tuple[np.ndarray, int] | None:
+    """Load an audio file to ``(mono float32 waveform, native sample_rate)`` via librosa.
+
+    Returns None if the extension is unsupported, the file is missing, or librosa isn't installed.
+    """
+    if p.suffix.lower() not in _AUDIO_SUFFIXES or not p.exists():
+        return None
+    try:
+        import librosa
+    except ImportError:
+        logger.warning("Audio file %s encountered but librosa not installed.", p)
+        return None
+    arr, sr = librosa.load(str(p), sr=None, mono=True)
+    return arr.astype(np.float32), int(sr)
+
+
+def _partition_visuals(visuals):
+    """Split a mixed list of visuals into ``(images, audios)``.
+
+    Audio is tested first (its ``array``+``sampling_rate`` signature is more distinctive
+    than the various image forms).
+    """
+    images, audios = [], []
+    for v in visuals or []:
+        if v is None:
+            continue
+        # Try audio first since its signature ("array"+"sampling_rate") is more
+        # distinctive than the image path/bytes/PIL signatures.
+        au = _normalize_audio(v)
+        if au is not None:
+            audios.append(au)
+            continue
+        img = _normalize_image(v)
+        if img is not None:
+            images.append(img)
+            continue
+        suffix = Path(v).suffix.lower() if isinstance(v, (str, Path)) else ""
+        if suffix in _VIDEO_SUFFIXES:
+            raise NotImplementedError("LMMSORTGenAIEvaluator does not support video inputs.")
+        raise ValueError(f"Unsupported lmms-eval visual input type: {type(v).__name__}.")
+    return images, audios
+
+
+def _generation_options(gen_kwargs: dict[str, Any]) -> tuple[dict[str, Any], list[str] | None]:
+    """Translate lmms-eval generation kwargs to ORT-GenAI search and stop options."""
+    if gen_kwargs.get("num_beams") not in (None, 1):
+        raise NotImplementedError("Beam search is not supported by LMMSORTGenAIEvaluator.")
+    if gen_kwargs.get("num_return_sequences") not in (None, 1):
+        raise NotImplementedError("Multiple return sequences are not supported by LMMSORTGenAIEvaluator.")
+    if gen_kwargs.get("no_repeat_ngram_size") not in (None, 0):
+        raise NotImplementedError("no_repeat_ngram_size is not supported by ORT-GenAI.")
+    if gen_kwargs.get("max_agentic_steps") is not None or gen_kwargs.get("visual_cot"):
+        raise NotImplementedError("Agentic and visual-CoT generation are not supported by LMMSORTGenAIEvaluator.")
+
+    search_options: dict[str, Any] = {"do_sample": bool(gen_kwargs.get("do_sample", False))}
+    for name in ("temperature", "top_p", "top_k", "repetition_penalty"):
+        value = gen_kwargs.get(name)
+        if value is not None:
+            search_options[name] = value
+
+    stop = gen_kwargs.get("until")
+    if isinstance(stop, str):
+        stop = [stop]
+    elif stop is not None:
+        stop = list(stop)
+    extra_stops = gen_kwargs.get("stop_sequences")
+    if isinstance(extra_stops, str):
+        extra_stops = [extra_stops]
+    if extra_stops:
+        stop = [*(stop or []), *extra_stops]
+    return search_options, stop or None
+
+
+def _build_whisper_prompt(
+    whisper_language: str = "en",
+    whisper_task: str = "transcribe",
+    whisper_timestamps: bool = False,
+) -> str:
+    """Build Whisper's decoder-start token sequence."""
+    if whisper_task not in ("transcribe", "translate"):
+        raise ValueError("whisper_task must be 'transcribe' or 'translate'.")
+    timestamp_token = "" if whisper_timestamps else "<|notimestamps|>"
+    return f"<|startoftranscript|><|{whisper_language}|><|{whisper_task}|>{timestamp_token}"
+
+
+def _normalize_execution_provider(execution_provider: Any | None) -> str:
+    """Normalize an EP spec to a canonical ORT-GenAI provider name.
+
+    Accepts a string or list/tuple (first entry used); empty input returns
+    ``"follow_config"``, meaning "use the providers already in genai_config.json".
+    """
+    if not execution_provider:
+        return "follow_config"
+    if isinstance(execution_provider, (tuple, list)):
+        execution_provider = execution_provider[0] if execution_provider else None
+    if not execution_provider:
+        return "follow_config"
+    ep = str(execution_provider).lower().replace("_", "")
+    return _PROVIDER_ALIASES.get(ep, str(execution_provider))
+
+
+# -----------------------------------------------------------------------------
+# Main adapter
+# -----------------------------------------------------------------------------
+
+
+@register_model("ortgenai_mm")
+class LMMSORTGenAIEvaluator(lmms):
+    r"""lmms-eval model wrapper for an ORT-GenAI multimodal package (model).
+
+    Implements the base lmms class interface for generate_until and loglikelihood
+    using ORT-GenAI's multimodal processor and generator. Image and audio inputs
+    are rendered through the model's native structured-content chat template.
+    """
+
+    is_simple = True
+
+    def __init__(
+        self,
+        pretrained: str,
+        batch_size: int = 1,
+        max_new_tokens: int = 256,
+        max_length: int = 32768,
+        system_prompt: str | None = None,
+        execution_provider: str | None = None,
+        provider_options: dict | None = None,
+        fail_on_error: bool = True,
+        ignore_stop_strings: list[str] | None = None,
+        whisper_language: str = "en",
+        whisper_task: str = "transcribe",
+        whisper_timestamps: bool = False,
+        **kwargs,
+    ) -> None:
+        if _LMMS_EVAL_IMPORT_ERROR is not None:
+            raise ImportError(
+                "lmms-eval is required for ortgenai_mm. Install lmms-eval before using LMMSEvaluator."
+            ) from _LMMS_EVAL_IMPORT_ERROR
+        if og is None:
+            raise ImportError(
+                "onnxruntime-genai is required for ortgenai_mm. "
+                "Install with: pip install onnxruntime-genai (or -cuda variant)."
+            )
+        super().__init__()
+        if kwargs:
+            logger.warning("Unused kwargs: %s", kwargs)
+
+        model_dir = Path(pretrained).resolve()
+        if not model_dir.is_dir():
+            raise ValueError(f"ORT-GenAI model directory does not exist: {model_dir}")
+        if not (model_dir / "genai_config.json").is_file():
+            raise ValueError(f"LMMSEvaluator requires genai_config.json in ORT-GenAI package: {model_dir}")
+        if int(batch_size) < 1:
+            raise ValueError("batch_size must be >= 1")
+        if int(max_new_tokens) < 1:
+            raise ValueError("max_new_tokens must be >= 1")
+        if int(max_length) < 1:
+            raise ValueError("max_length must be >= 1")
+
+        self.model_dir = str(model_dir)
+        self.max_new_tokens = int(max_new_tokens)
+        self.max_length = int(max_length)
+        self.batch_size_per_gpu = int(batch_size)
+        self.system_prompt = system_prompt
+        self.fail_on_error = fail_on_error
+        self.whisper_language = whisper_language
+        self.whisper_task = whisper_task
+        self.whisper_timestamps = bool(whisper_timestamps)
+        # Stop strings to suppress from each request's lmms-eval ``until`` list.
+        # lmms-eval injects ``until=["\n\n"]`` (the fewshot_delimiter) for tasks
+        # that don't set their own ``until`` (lmms_eval/api/task.py). For
+        # step-by-step reasoning that double-newline truncates the model after its
+        # first paragraph, before it reaches the final answer. Listing "\n\n" here
+        # drops only that spurious stop; the model still stops normally on its
+        # EOS / end-of-turn token (handled in _run_generation).
+        self.ignore_stop_strings = set(ignore_stop_strings or [])
+
+        logger.info("Loading ORT-GenAI model from: %s", self.model_dir)
+        ep = _normalize_execution_provider(execution_provider)
+        # CUDA GenAI packages often carry provider-specific options in genai_config.json.
+        # Clearing/re-adding CUDA can drop those options and fail to load on otherwise
+        # working packages, so follow the package config unless options are overridden.
+        if ep == "follow_config" or (ep == "cuda" and not provider_options):
+            self._model = og.Model(self.model_dir)
+        else:
+            config = og.Config(self.model_dir)
+            config.clear_providers()
+            if ep != "cpu":
+                config.append_provider(ep)
+            for key, value in (provider_options or {}).items():
+                config.set_provider_option(ep, key, value)
+            self._model = og.Model(config)
+        self._tokenizer = og.Tokenizer(self._model)
+        self._processor = self._model.create_multimodal_processor()
+
+        eos_ids = self._tokenizer.eos_token_ids
+        self._eos_token_ids = {int(t) for t in (eos_ids if eos_ids is not None else [])}
+
+        try:
+            cfg = json.loads((Path(self.model_dir) / "genai_config.json").read_text(encoding="utf-8"))
+            self._model_type = cfg.get("model", {}).get("type", "phi4mm")
+        except json.JSONDecodeError as e:
+            raise ValueError(f"Invalid genai_config.json in {self.model_dir}") from e
+        if self._model_type != "whisper" and not hasattr(self._tokenizer, "apply_chat_template"):
+            raise RuntimeError(
+                "LMMSEvaluator requires og.Tokenizer.apply_chat_template for non-Whisper models. "
+                "Upgrade onnxruntime-genai or use the preserved legacy prompt-override branch."
+            )
+
+        # Resolve the default system prompt per model family (matching lmms-eval's
+        # per-model wrappers' default system prompt) when the user did not pass one explicitly. This keeps
+        # HF-input and ORT-GenAI evaluation prompts identical
+        if self.system_prompt is None:
+            self.system_prompt = self.default_system_prompt_for_model_type(self._model_type)
+            logger.info("Using default system prompt for model type '%s': %r", self._model_type, self.system_prompt)
+
+        # Probe once whether this model's chat template injects media tokens
+        # from structured content parts (``{"type": "image"}``). Well-behaved
+        # templates (Gemma-4, Qwen2.5-VL, Qwen3-VL) do (pretty much most); Phi-4-MM's template
+        # stringifies the content list as Python repr instead. When supported,
+        # the adapter lets the template emit the correct per-model media tokens
+        # automatically.
+        self._supports_structured_content = self._model_type != "whisper" and self._probe_structured_content_support()
+
+        self._rank = 0
+        self._world_size = 1
+        logger.info("Model loaded. Model type: %s", self._model_type)
+
+    # -------------------------------------------------------------------------
+    # lmms-eval required properties
+    # -------------------------------------------------------------------------
+    @property
+    def batch_size(self):
+        return self.batch_size_per_gpu
+
+    @property
+    def rank(self):
+        return self._rank
+
+    @property
+    def world_size(self):
+        return self._world_size
+
+    # -------------------------------------------------------------------------
+    # ORT-GenAI input plumbing
+    # -------------------------------------------------------------------------
+    def _build_og_images(self, images, tmp_dir: Path):
+        """Write PIL images to temp PNGs and open them as an ``og.Images`` handle (None if no images)."""
+        if not images:
+            return None
+        paths = []
+        for i, img in enumerate(images):
+            path = tmp_dir / f"image_{i}.png"
+            img.save(path, format="PNG")
+            paths.append(str(path))
+        return og.Images.open(*paths)
+
+    def _build_og_audios(self, audios, tmp_dir: Path):
+        """Write ``(waveform, sr)`` pairs to temp WAVs and open them as an ``og.Audios`` handle (None if none)."""
+        if not audios:
+            return None
+        import soundfile as sf
+
+        paths = []
+        for i, (arr, sr) in enumerate(audios):
+            path = tmp_dir / f"audio_{i}.wav"
+            sf.write(path, arr, sr, subtype="FLOAT")
+            paths.append(str(path))
+        return og.Audios.open(*paths)
+
+    def _handle_error(self, message: str, exc: Exception, default):
+        """Re-raise as RuntimeError when ``fail_on_error`` is set; otherwise log and return ``default``."""
+        if self.fail_on_error:
+            raise RuntimeError(message) from exc
+        logger.exception("%s", message)
+        return default
+
+    # -------------------------------------------------------------------------
+    # Single-request inference primitives
+    # -------------------------------------------------------------------------
+    @staticmethod
+    def _processed_input_length(inputs) -> int:
+        """Return the processor-expanded sequence length from ORT-GenAI inputs."""
+        try:
+            shape = inputs["input_ids"].shape()
+        except (KeyError, RuntimeError, TypeError, AttributeError) as e:
+            raise ValueError("ORT-GenAI multimodal processor output is missing a valid input_ids tensor.") from e
+        if not shape or int(shape[-1]) <= 0:
+            raise ValueError(f"ORT-GenAI multimodal processor returned an invalid input_ids shape: {shape}.")
+        return int(shape[-1])
+
+    def _run_generation(
+        self,
+        prompt: str,
+        images,
+        audios,
+        max_new_tokens: int,
+        stop_strings: list[str] | None = None,
+        search_options: dict[str, Any] | None = None,
+    ) -> str:
+        """Greedily decode a single multimodal request and return the generated text.
+
+        Feeds ``prompt`` + media through the ORT-GenAI processor, then decodes
+        token-by-token until EOS, the ``max_new_tokens`` budget, or a ``stop_strings``
+        match (early-truncated). Returns "" on processor/input failure unless
+        ``fail_on_error`` is set.
+        """
+        if max_new_tokens < 1:
+            error = ValueError(f"max_new_tokens must be >= 1, got {max_new_tokens}.")
+            return self._handle_error("Invalid ORT-GenAI completion budget.", error, "")
+
+        with tempfile.TemporaryDirectory() as tmp:
+            tmp_dir = Path(tmp)
+            og_images = self._build_og_images(images, tmp_dir)
+            og_audios = self._build_og_audios(audios, tmp_dir)
+
+            try:
+                # ORT-GenAI multimodal processors disagree on argument shape:
+                #   - Phi-4-MM expects a bare string. Passing [prompt] raises
+                #     "Number of image tokens does not match the number of images"
+                #     because the processor interprets the list as one prompt per
+                #     image (verified against pre-built phi4mm INT4 package).
+                #   - Whisper's processor (per ORT-GenAI's reference
+                #     benchmark_multimodal.py) is exercised with a list of
+                #     prompts.
+                # Branch on model type rather than guess.
+                processor_input = [prompt] if self._model_type == "whisper" else prompt
+                inputs = self._processor(processor_input, images=og_images, audios=og_audios)
+            except Exception as e:  # pragma: no cover
+                return self._handle_error("ORT-GenAI multimodal processor failed.", e, "")
+
+            try:
+                input_length = self._processed_input_length(inputs)
+            except ValueError as e:
+                return self._handle_error("Failed to determine the processed prompt length.", e, "")
+            available_new_tokens = self.max_length - input_length
+            if available_new_tokens <= 0:
+                error = ValueError(
+                    f"Current processed prompt length {input_length} leaves no completion capacity within "
+                    f"max_length={self.max_length}."
+                )
+                return self._handle_error("ORT-GenAI processed prompt exceeds max_length.", error, "")
+            effective_new_tokens = min(max_new_tokens, available_new_tokens)
+            if effective_new_tokens < max_new_tokens:
+                logger.warning(
+                    "Reducing max_new_tokens from %d to %d because the processed prompt uses %d of max_length=%d.",
+                    max_new_tokens,
+                    effective_new_tokens,
+                    input_length,
+                    self.max_length,
+                )
+
+            request_max_length = input_length + effective_new_tokens
+            params = og.GeneratorParams(self._model)
+            params.set_search_options(
+                max_length=request_max_length,
+                **(search_options or {"do_sample": False}),
+            )
+            generator = og.Generator(self._model, params)
+            try:
+                generator.set_inputs(inputs)
+            except RuntimeError as e:
+                del generator
+                return self._handle_error(
+                    "ORT-GenAI generator input setup failed. The prompt may exceed max_length.", e, ""
+                )
+
+            # Whisper's BOS == EOS (token 50257 = <|startoftranscript|> = <|endoftext|>),
+            # so the very first generated token can collide with EOS. Skip the
+            # EOS check until we've emitted at least one non-EOS token.
+            decoded = ""
+            stream = self._tokenizer.create_stream()
+            steps = 0
+            generated_any = False
+            while not generator.is_done() and steps < effective_new_tokens:
+                generator.generate_next_token()
+                tok = int(generator.get_next_tokens()[0])
+                if tok in self._eos_token_ids:
+                    if self._model_type != "whisper" or generated_any:
+                        break
+                    # Whisper's first-step EOS can collide with its BOS token.
+                    steps += 1
+                    continue
+                generated_any = True
+                decoded += stream.decode(tok)
+                if stop_strings:
+                    for s in stop_strings:
+                        if s in decoded:
+                            decoded = decoded.split(s, 1)[0]
+                            del generator
+                            return decoded
+                steps += 1
+
+        del generator
+        return decoded
+
+    def _score_continuation(self, prompt: str, continuation: str, images, audios) -> tuple[float, bool]:
+        """Return ``(continuation loss, is_greedy)`` for a loglikelihood request.
+
+        Scores the continuation tokens under the model given ``prompt`` + media.
+        lmms-eval's multiple-choice tasks select the minimum returned value, so
+        this method returns positive negative-log-likelihood rather than log-probability.
+        ``is_greedy`` is True iff every continuation token was the argmax at its step.
+        """
+        # Tokenize prompt and prompt+continuation jointly, then slice to obtain
+        # the continuation token IDs as they would actually appear extending the
+        # prompt. Critical for sentencepiece/BPE tokenizers where ``encode("A")``
+        # differs from the suffix of ``encode("prompt A")`` (leading-space
+        # handling, BOS injection, etc.).
+        prompt_tokens = list(self._tokenizer.encode(prompt))
+        full_tokens = list(self._tokenizer.encode(prompt + continuation))
+        cont_tokens = full_tokens[len(prompt_tokens) :]
+        if len(cont_tokens) == 0:
+            return 0.0, True
+
+        with tempfile.TemporaryDirectory() as tmp:
+            tmp_dir = Path(tmp)
+            og_images = self._build_og_images(images, tmp_dir)
+            og_audios = self._build_og_audios(audios, tmp_dir)
+            try:
+                processor_input = [prompt] if self._model_type == "whisper" else prompt
+                inputs = self._processor(processor_input, images=og_images, audios=og_audios)
+            except Exception as e:  # pragma: no cover
+                return self._handle_error("ORT-GenAI multimodal processor failed in loglikelihood.", e, (1e9, False))
+
+            try:
+                input_length = self._processed_input_length(inputs)
+            except ValueError as e:
+                return self._handle_error(
+                    "Failed to determine the processed prompt length in loglikelihood.", e, (1e9, False)
+                )
+            request_max_length = input_length + len(cont_tokens)
+            if request_max_length > self.max_length:
+                error = ValueError(
+                    f"Processed prompt length {input_length} plus continuation length {len(cont_tokens)} "
+                    f"exceeds max_length={self.max_length}."
+                )
+                return self._handle_error(
+                    "ORT-GenAI continuation cannot be scored within max_length.", error, (1e9, False)
+                )
+
+            params = og.GeneratorParams(self._model)
+            params.set_search_options(
+                max_length=request_max_length,
+                do_sample=False,
+            )
+            generator = og.Generator(self._model, params)
+            try:
+                generator.set_inputs(inputs)
+            except RuntimeError as e:
+                del generator
+                return self._handle_error("ORT-GenAI generator input setup failed in loglikelihood.", e, (1e9, False))
+
+            # SetInputs appends the prompt tokens and runs the decoder prefill, so
+            # get_logits() immediately returns the distribution for the first
+            # continuation token. Appending a chosen continuation token advances
+            # the generator and computes logits for the following token.
+            total_loss = 0.0
+            all_greedy = True
+            for i, tok_id in enumerate(cont_tokens):
+                if generator.is_done():
+                    total_loss += 50.0
+                    all_greedy = False
+                    continue
+                logits = np.asarray(generator.get_logits(), dtype=np.float64).reshape(-1)
+                if tok_id >= logits.shape[0]:
+                    del generator
+                    raise ValueError(f"Token id {tok_id} is outside logits vocabulary size {logits.shape[0]}.")
+                log_denom = np.logaddexp.reduce(logits)
+                total_loss += float(log_denom - logits[tok_id])
+                if int(np.argmax(logits)) != tok_id:
+                    all_greedy = False
+                if i + 1 < len(cont_tokens):
+                    generator.append_tokens(np.array([tok_id], dtype=np.int32))
+
+        del generator
+        return total_loss, all_greedy
+
+    # The generic system prompt used by the majority of lmms-eval wrappers
+    # (qwen2_5_vl.py, qwen3_vl.py, gemma3.py, llava_*, minicpm_o, ...). Also the
+    # fallback for model types without a more specific mapping.
+    _DEFAULT_SYSTEM_PROMPT = "You are a helpful assistant."
+    # Qwen Omni models (qwen2_5_omni.py / qwen3_omni.py) are trained with a fixed
+    # identity system prompt; reuse it verbatim for prompt parity.
+    _QWEN_OMNI_SYSTEM_PROMPT = (
+        "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, "
+        "capable of perceiving auditory and visual inputs, as well as generating text and speech."
+    )
+    # Per-model-family default system prompts. Each entry is a tuple of substrings
+    # that must ALL appear in the (lowercased) genai_config ``model.type``, paired
+    # with the prompt to use. First match wins, so more specific rules come first.
+    # These mirror lmms-eval's per-model wrappers so HF-input and ORT-GenAI
+    # evaluation use identical prompts out of the box. Only applied when the user
+    # does not pass an explicit ``system_prompt``.
+    #   - Qwen Omni (qwen2_5_omni / qwen3_omni_moe): fixed identity prompt.
+    #   - Qwen-VL / Gemma: generic "You are a helpful assistant.".
+    #   - Phi-4-MM: lmms-eval's phi4_multimodal wrapper uses NO system turn, so
+    #     the parity-correct default is an empty string.
+    _MODEL_TYPE_SYSTEM_PROMPTS: tuple[tuple[tuple[str, ...], str], ...] = (
+        (("qwen", "omni"), _QWEN_OMNI_SYSTEM_PROMPT),
+        (("qwen",), _DEFAULT_SYSTEM_PROMPT),
+        (("gemma",), _DEFAULT_SYSTEM_PROMPT),
+        (("phi",), ""),
+    )
+
+    @classmethod
+    def default_system_prompt_for_model_type(cls, model_type: str | None) -> str:
+        """Return the parity-correct default system prompt for a model type (see ``_MODEL_TYPE_SYSTEM_PROMPTS``)."""
+        model_type_lower = (model_type or "").lower()
+        for required_substrings, prompt in cls._MODEL_TYPE_SYSTEM_PROMPTS:
+            if all(substring in model_type_lower for substring in required_substrings):
+                return prompt
+        return cls._DEFAULT_SYSTEM_PROMPT
+
+    def _probe_structured_content_support(self) -> bool:
+        """Detect whether the model's chat template correctly injects media tokens from structured content.
+
+        Renders a probe message whose content is a list of typed parts
+        (``[{"type": "image"}, {"type": "text", ...}]``) and checks the result:
+
+        - Well-behaved templates (Gemma-4, Qwen2.5-VL, Qwen3-VL) replace the
+          image part with the model's own media token (e.g. ``<|image|>`` or
+          ``<|vision_start|>...``), so the rendered string contains no Python
+          dict repr.
+        - Broken templates (Phi-4-MM) stringify the list as Python repr, so the
+          rendered string contains ``{'type'`` / ``"type"``.
+
+        Probed once at load. Returns False if the template cannot render
+        structured content or the probe raises.
+        """
+        try:
+            probe = [{"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "x"}]}]
+            rendered = self._tokenizer.apply_chat_template(json.dumps(probe), add_generation_prompt=True)
+            text_only = [{"role": "user", "content": [{"type": "text", "text": "x"}]}]
+            rendered_text_only = self._tokenizer.apply_chat_template(json.dumps(text_only), add_generation_prompt=True)
+        except Exception as e:  # pragma: no cover - defensive
+            logger.debug("Structured-content probe failed: %s", e)
+            return False
+        # A useful template must neither leak the content dictionaries nor ignore
+        # the image part entirely.
+        return "{'type'" not in rendered and '"type"' not in rendered and rendered != rendered_text_only
+
+    def _build_structured_chat_prompt(self, user_text: str, num_images: int, num_audios: int) -> str:
+        """Build the prompt via structured content parts so the chat template injects media tokens.
+
+        Only used when :meth:`_probe_structured_content_support` returned True.
+        """
+        content: list[dict[str, Any]] = []
+        if _MEDIA_PLACEHOLDER_RE.search(user_text):
+            image_count = 0
+            audio_count = 0
+            for part in _MEDIA_PLACEHOLDER_RE.split(user_text):
+                if part == "<image>":
+                    image_count += 1
+                    content.append({"type": "image"})
+                elif part == "<audio>":
+                    audio_count += 1
+                    content.append({"type": "audio"})
+                elif part:
+                    content.append({"type": "text", "text": part})
+            if image_count != num_images or audio_count != num_audios:
+                raise ValueError(
+                    "Media placeholder counts do not match the extracted media: "
+                    f"{image_count} image placeholders for {num_images} images, "
+                    f"{audio_count} audio placeholders for {num_audios} audios."
+                )
+        else:
+            content.extend({"type": "image"} for _ in range(num_images))
+            content.extend({"type": "audio"} for _ in range(num_audios))
+            content.append({"type": "text", "text": user_text})
+
+        messages: list[dict[str, Any]] = []
+        if self.system_prompt:
+            messages.append({"role": "system", "content": self.system_prompt})
+        messages.append({"role": "user", "content": content})
+        return self._tokenizer.apply_chat_template(json.dumps(messages), add_generation_prompt=True)
+
+    def _build_prompt_for_request(self, user_text: str, num_images: int, num_audios: int) -> str:
+        """Build the final prompt string fed to ``og.MultiModalProcessor``.
+
+        Whisper uses its decoder-start sequence. Other models use their native
+        chat template. Multimodal requests require the template to support
+        structured content parts so that it emits its own media tokens.
+        """
+        if self._model_type == "whisper":
+            return _build_whisper_prompt(
+                whisper_language=self.whisper_language,
+                whisper_task=self.whisper_task,
+                whisper_timestamps=self.whisper_timestamps,
+            )
+
+        has_media = bool(num_images or num_audios or _MEDIA_PLACEHOLDER_RE.search(user_text))
+        if has_media:
+            if not self._supports_structured_content:
+                raise ValueError(
+                    f"Model type {self._model_type!r} does not provide a chat template that supports "
+                    "structured image/audio content. Manual media-token prompt overrides are not supported."
+                )
+            return self._build_structured_chat_prompt(user_text, num_images, num_audios)
+
+        messages: list[dict[str, Any]] = []
+        if self.system_prompt:
+            messages.append({"role": "system", "content": self.system_prompt})
+        messages.append({"role": "user", "content": user_text})
+
+        return self._tokenizer.apply_chat_template(json.dumps(messages), add_generation_prompt=True)
+
+    def _get_doc_and_visuals(self, doc_to_visual, doc_id, task, split):
+        """Look up an lmms-eval document and its visual inputs.
+
+        ``doc_to_visual`` is the task-specific function that extracts images or
+        audio from the document.
+        """
+        try:
+            doc = self.task_dict[task][split][doc_id]
+        except (KeyError, IndexError, TypeError) as e:
+            raise KeyError(f"Failed to find lmms-eval document={task!r}, split={split!r}, doc_id={doc_id!r}") from e
+
+        visuals = doc_to_visual(doc) if doc_to_visual else []
+        if visuals is None:
+            visuals = []
+        if not isinstance(visuals, list):
+            visuals = [visuals]
+        return doc, visuals
+
+    # -------------------------------------------------------------------------
+    # lmms-eval Model interface
+    # -------------------------------------------------------------------------
+    def generate_until(self, requests: list[Instance], disable_tqdm: bool = False) -> list[str]:
+        """lmms-eval entry point: generate a text completion for each request.
+
+        Per request, resolves media, the ``max_new_tokens`` budget (task YAML wins),
+        and ``until`` stop strings (minus ``ignore_stop_strings``), builds the prompt,
+        and greedily decodes. Returns one string per request, in order.
+        """
+        results = []
+        pbar = tqdm(total=len(requests), desc="ortgenai_mm generate_until", disable=disable_tqdm)
+        for req in requests:
+            contexts, gen_kwargs, doc_to_visual, doc_id, task, split = req.args
+            # NOTE, visuals could contain both images and audio
+            _, visuals = self._get_doc_and_visuals(doc_to_visual, doc_id, task, split)
+            images, audios = _partition_visuals(visuals)
+
+            gen_kwargs = gen_kwargs or {}
+            max_new_tokens = int(gen_kwargs.get("max_new_tokens", self.max_new_tokens))
+            search_options, stop = _generation_options(gen_kwargs)
+            if stop and self.ignore_stop_strings:
+                stop = [s for s in stop if s not in self.ignore_stop_strings] or None
+
+            prompt = self._build_prompt_for_request(contexts, len(images), len(audios))
+            text = self._run_generation(
+                prompt,
+                images,
+                audios,
+                max_new_tokens,
+                stop,
+                search_options=search_options,
+            )
+            results.append(text)
+            self.cache_hook.add_partial("generate_until", (contexts, gen_kwargs), text)
+            pbar.update(1)
+        pbar.close()
+        return results
+
+    def loglikelihood(self, requests: list[Instance], disable_tqdm: bool = False) -> list[tuple[float, bool]]:
+        """lmms-eval entry point: score each request's target continuation.
+
+        Returns one ``(loss, is_greedy)`` per request. lmms-eval multiple-choice
+        tasks select the candidate with the minimum loss.
+        """
+        results = []
+        pbar = tqdm(total=len(requests), desc="ortgenai_mm loglikelihood", disable=disable_tqdm)
+        for req in requests:
+            contexts, doc_to_target, doc_to_visual, doc_id, task, split = req.args
+            doc, visuals = self._get_doc_and_visuals(doc_to_visual, doc_id, task, split)
+            images, audios = _partition_visuals(visuals)
+            continuation = doc_to_target if isinstance(doc_to_target, str) else str(doc_to_target(doc))
+
+            prompt = self._build_prompt_for_request(contexts, len(images), len(audios))
+            loss, is_greedy = self._score_continuation(prompt, continuation, images, audios)
+            results.append((loss, is_greedy))
+            self.cache_hook.add_partial("loglikelihood", (contexts, continuation), (loss, is_greedy))
+            pbar.update(1)
+        pbar.close()
+        return results
+
+    def generate_until_multi_round(self, requests) -> list[str]:
+        raise NotImplementedError("ortgenai_mm does not support lmms-eval multi-round generation yet.")
diff --git a/olive/evaluator/olive_evaluator.py b/olive/evaluator/olive_evaluator.py
index 75d743602..a441d8671 100644
--- a/olive/evaluator/olive_evaluator.py
+++ b/olive/evaluator/olive_evaluator.py
@@ -575,6 +575,147 @@ def _get_genai_model_dir(model: ONNXModelHandler) -> str:
     return str(Path(model.model_path).parent)
 
 
+def _lmms_model_size_on_disk(model: "OliveModelHandler") -> int:
+    """Return the full package size for ORT-GenAI models, otherwise the handler size."""
+    if isinstance(model, ONNXModelHandler):
+        genai_config_path = _find_genai_config(model)
+        if genai_config_path is not None:
+            return sum(path.stat().st_size for path in genai_config_path.parent.rglob("*") if path.is_file())
+    return model.size_on_disk
+
+
+def _lmms_output_path(output_path: str, model: "OliveModelHandler") -> Path:
+    """Avoid clobbering an input-model result with a different backend's result."""
+    out = Path(output_path)
+    backend = "hf" if isinstance(model, HfModelHandler) else "onnx"
+    if "{model_type}" in str(out):
+        return Path(str(out).format(model_type=backend))
+    if not out.is_file():
+        return out
+    try:
+        existing_backend = json.loads(out.read_text(encoding="utf-8")).get("model_type")
+    except (json.JSONDecodeError, OSError):
+        existing_backend = None
+    if existing_backend in (None, backend):
+        return out
+    return out.with_name(f"{out.stem}_{backend}{out.suffix}")
+
+
+def _wrap_generate_until_drop_stops(lm, ignore_stop_strings) -> None:
+    r"""Wrap ``generate_until`` to drop unwanted request stop strings.
+
+    lmms-eval injects ``until=[fewshot_delimiter]`` (``"\\n\\n"``) for generate_until
+    tasks that don't set their own ``until`` (lmms_eval/api/task.py), which truncates
+    step-by-step reasoning at the first blank line. The ORT-GenAI adapter honors
+    ``ignore_stop_strings`` directly; for HF wrappers (whose ``generate_until`` we
+    don't own) we filter each request's ``gen_kwargs["until"]`` before delegating.
+    Mirrors the diagnostic probe's monkeypatch, but scoped to a single instance.
+    """
+    ignore = set(ignore_stop_strings or [])
+    if not ignore or not hasattr(lm, "generate_until"):
+        return
+    orig = lm.generate_until
+
+    def _patched(requests, *args, **kwargs):
+        for req in requests:
+            gen_kwargs = next((arg for arg in req.args if isinstance(arg, dict)), None)
+            if gen_kwargs is None:
+                continue
+            until = gen_kwargs.get("until")
+            if isinstance(until, str):
+                until = [until]
+            if until:
+                filtered = [u for u in until if u not in ignore]
+                gen_kwargs["until"] = filtered
+        return orig(requests, *args, **kwargs)
+
+    lm.generate_until = _patched
+
+
+def _linify_lmms_sample(obj):
+    r"""Recursively split multi-line strings into lists of lines.
+
+    This lets ``json.dumps`` render long generations readably instead of one
+    escaped ``\\n`` blob.
+    """
+    if isinstance(obj, str):
+        return obj.split("\n") if "\n" in obj else obj
+    if isinstance(obj, list):
+        return [_linify_lmms_sample(x) for x in obj]
+    if isinstance(obj, dict):
+        return {k: _linify_lmms_sample(v) for k, v in obj.items()}
+    return obj
+
+
+def _has_correct_lmms_score(value: Any) -> bool:
+    """Return True when a nested lmms-eval score contains a successful value."""
+    if isinstance(value, dict):
+        return any(_has_correct_lmms_score(item) for item in value.values())
+    if isinstance(value, (list, tuple)):
+        return any(_has_correct_lmms_score(item) for item in value)
+    return value in (1, 1.0, True)
+
+
+def _summarize_lmms_samples(task: str, samples: list[dict]) -> dict:
+    """Reduce lmms-eval sample records to fields useful for inspection.
+
+    Generated text remains full and untruncated, with multi-line text represented
+    as a list of lines. The bulky ``doc`` field is dropped.
+    """
+    rows = []
+    n_empty = 0
+    n_scored_correct = 0
+    is_mathvision = "mathvision" in task
+    for s in samples:
+        target = s.get("target")
+        resps = s.get("resps")
+        filtered = s.get("filtered_resps")
+        score_keys = [k for k in s if k in ("mmmu_acc", "exact_match", "mathvision_standard_eval", "acc", "score")]
+        score = {k: s.get(k) for k in score_keys}
+
+        gen_text = ""
+        try:
+            if isinstance(resps, list) and resps:
+                gen_text = resps[0][0] if isinstance(resps[0], list) else str(resps[0])
+        except (IndexError, TypeError):
+            gen_text = str(resps)
+
+        if not (gen_text or "").strip():
+            n_empty += 1
+        if _has_correct_lmms_score(score):
+            n_scored_correct += 1
+
+        if is_mathvision:
+            score_compact = {}
+            for k, v in score.items():
+                if isinstance(v, dict):
+                    score_compact[k] = {kk: vv for kk, vv in v.items() if kk != "response"}
+                else:
+                    score_compact[k] = v
+            row = {
+                "doc_id": s.get("doc_id"),
+                "target": target,
+                "generated": _linify_lmms_sample(gen_text or ""),
+                "score": _linify_lmms_sample(score_compact),
+            }
+        else:
+            row = {
+                "doc_id": s.get("doc_id"),
+                "target": target,
+                "generated": _linify_lmms_sample(gen_text or ""),
+                "filtered": _linify_lmms_sample(filtered) if filtered is not None else None,
+                "score": _linify_lmms_sample(score),
+            }
+        rows.append(row)
+    return {
+        "task": task,
+        "n": len(samples),
+        "n_empty_generations": n_empty,
+        "n_scored_correct": n_scored_correct,
+        "rows": rows,
+    }
+
+
 def _normalize_audio_batch(input_data) -> tuple[list, list]:
     """Return (audio_arrays, file_names) from the various speech input shapes.
 
@@ -2267,6 +2408,325 @@ def evaluate(
         return flatten_metric_result(metrics)
 
 
+@Registry.register("LMMSEvaluator")
+class LMMSEvaluator(OliveEvaluator):
+    """Evaluator for multimodal models using lmms-eval (EvolvingLMMs-Lab/lmms-eval).
+
+    Supports two model handler types:
+
+    1. :class:`ONNXModelHandler` whose path is an ORT-GenAI multimodal package
+       (directory containing ``genai_config.json`` plus respective ONNX subgraph files
+       Dispatches to the ``ortgenai_mm`` adapter in :mod:`olive.evaluator.lmms_ort`.
+
+    2. :class:`HfModelHandler` for HuggingFace PyTorch multimodal models.
+       Dispatches to lmms-eval's native wrapper for the model architecture
+       (e.g. ``phi4_multimodal``, ``qwen2_5_vl``). The wrapper is auto-detected
+       from the HF ``model_type`` field; pass ``model_class`` in the recipe to
+       override.
+
+    Raw single-file ONNX models are intentionally not supported: multimodal
+    inference requires the vision/audio preprocessing pipeline that ORT-GenAI's
+    multimodal processor provides; a bare ``onnxruntime.InferenceSession``
+    cannot do image or audio tokenization on its own.
+    """
+
+    # HuggingFace model_type -> lmms-eval model_class (canonical id).
+    # Covers the multimodal architectures most relevant to Olive sweeps; other
+    # architectures still work if ``model_class`` is set explicitly in the
+    # recipe. Verified against lmms-eval's AVAILABLE_SIMPLE_MODELS /
+    # AVAILABLE_CHAT_TEMPLATE_MODELS registries.
+    _HF_MODEL_TYPE_TO_LMMS_CLASS: ClassVar[dict[str, str]] = {
+        "phi4mm": "phi4_multimodal",
+        "phi3_v": "phi3v",
+        "qwen2_vl": "qwen2_vl",
+        "qwen2_5_vl": "qwen2_5_vl",
+        "qwen3_vl": "qwen3_vl",
+        "qwen2_audio": "qwen2_audio",
+        "qwen2_5_omni": "qwen2_5_omni",
+        "qwen3_omni": "qwen3_omni",
+        "whisper": "whisper",
+        "gemma3": "gemma3",
+        # requires gemma4.py wrapper in lmms-eval (not yet merged upstream), check my fork
+        "gemma4": "gemma4",
+        "gemma4_unified": "gemma4",
+        "minicpm_o": "minicpm_o",
+        "llava": "llava",
+        "llava_onevision": "llava_onevision",
+        "internvl_chat": "internvl",
+    }
+
+    def __init__(self, tasks: list[str], **kwargs):
+        super().__init__(**kwargs)
+        self.tasks = tasks
+        self.limit = kwargs.get("limit")
+        self.model_class = kwargs.get("model_class")
+        self.batch_size = kwargs.get("batch_size", 1)
+        # Task generation kwargs take precedence; this value is the fallback for
+        # tasks that do not provide max_new_tokens.
+        self.max_new_tokens = kwargs.get("max_new_tokens", 256)
+        self.max_length = kwargs.get("max_length", 32768)
+        # Default to None (not a hardcoded string) so the adapter's per-model
+        # system-prompt auto-resolution takes over when the recipe doesn't set
+        # one. This keeps olive-run and standalone-adapter usage consistent, and
+        # lets model-family defaults apply (e.g. Gemma/Qwen-VL "You are a helpful
+        # assistant.", Qwen-Omni identity prompt, Phi-4-MM empty system turn),
+        # matching lmms-eval's per-model wrappers. An explicit recipe value still
+        # overrides.
+        self.system_prompt = kwargs.get("system_prompt")
+        self.ep = kwargs.get("execution_provider")
+        self.ep_options = kwargs.get("provider_options")
+        self.log_samples = bool(kwargs.get("log_samples", False))
+        self.output_path = kwargs.get("output_path")
+        self.fail_on_error = bool(kwargs.get("fail_on_error", True))
+        self.whisper_language = kwargs.get("whisper_language", "en")
+        self.whisper_task = kwargs.get("whisper_task", "transcribe")
+        self.whisper_timestamps = bool(kwargs.get("whisper_timestamps", False))
+        # Stop strings to suppress from each task's lmms-eval ``until`` list (e.g.
+        # ["\n\n"]). lmms-eval defaults ``until`` to the fewshot_delimiter ("\n\n")
+        # for tasks that don't set their own, which truncates step-by-step
+        # reasoning at the first blank line. Set this (plus a larger
+        # ``max_new_tokens``) for reasoning tasks so generation runs to the
+        # model's EOS instead. Only used by the ortgenai_mm adapter.
+        self.ignore_stop_strings = kwargs.get("ignore_stop_strings")
+        # HF-only knobs (forwarded to lmms-eval's native wrapper if present).
+        # ``trust_remote_code`` defaults to False to match the rest of Olive
+        # (e.g. olive/common/hf/utils.py, olive/data/component/load_dataset.py)
+        # and avoid silently executing arbitrary Hub code at load time. Users
+        # who need Phi-4-MM, MiniCPM-o, etc. opt in explicitly via the recipe.
+        self.dtype = kwargs.get("dtype", "auto")
+        self.trust_remote_code = bool(kwargs.get("trust_remote_code", False))
+        self.hf_model_kwargs = kwargs.get("hf_model_kwargs") or {}
+
+    @staticmethod
+    def _device_for_hf(device: Device) -> str:
+        # lmms-eval's HF wrappers accept "cuda", "cpu", or a torch.device.
+        return "cuda" if device == Device.GPU else "cpu"
+
+    def _build_ortgenai_mm_lm(
+        self,
+        model: ONNXModelHandler,
+        execution_providers: Optional[Union[str, list[str]]],
+    ):
+        """Build the lmms-eval wrapper for an ORT-GenAI multimodal package."""
+        from olive.evaluator.lmms_ort import LMMSORTGenAIEvaluator
+
+        if _find_genai_config(model) is None:
+            raise ValueError(
+                "LMMSEvaluator requires an ORT-GenAI package "
+                "(directory containing genai_config.json) for ONNXModelHandler input. "
+                f"Got ONNXModelHandler without genai_config at {model.model_path}. "
+                "Raw single-file ONNX is not supported for multimodal evaluation because "
+                "the vision/audio preprocessing pipeline lives in ORT-GenAI's multimodal "
+                "processor; use HfModelHandler or an ORT-GenAI package instead."
+            )
+
+        model_dir = _get_genai_model_dir(model)
+        logger.info("Running lmms-eval (model_class=ortgenai_mm, model_dir=%s)", model_dir)
+        return LMMSORTGenAIEvaluator(
+            pretrained=model_dir,
+            batch_size=self.batch_size,
+            max_new_tokens=self.max_new_tokens,
+            max_length=self.max_length,
+            system_prompt=self.system_prompt,
+            execution_provider=self.ep or execution_providers,
+            provider_options=self.ep_options,
+            fail_on_error=self.fail_on_error,
+            ignore_stop_strings=self.ignore_stop_strings,
+            whisper_language=self.whisper_language,
+            whisper_task=self.whisper_task,
+            whisper_timestamps=self.whisper_timestamps,
+        )
+
+    def _resolve_hf_model_class(self, model: HfModelHandler, hf_model_type: str | None = None) -> str:
+        if self.model_class:
+            return self.model_class
+        hf_model_type = hf_model_type or model.get_hf_model_type()
+        lmms_model_class = self._HF_MODEL_TYPE_TO_LMMS_CLASS.get(hf_model_type)
+        if not lmms_model_class:
+            raise ValueError(
+                f"Could not auto-detect lmms-eval model_class for HF model_type={hf_model_type!r}. "
+                f"Pass 'model_class' in the evaluator config (e.g. one of "
+                f"{sorted(self._HF_MODEL_TYPE_TO_LMMS_CLASS.values())}, or any other "
+                f"name registered with lmms-eval)."
+            )
+        return lmms_model_class
+
+    def _build_hf_lm(self, model: HfModelHandler, device: Device):
+        """Build lmms-eval's native HF wrapper for a HuggingFace PyTorch multimodal model."""
+        import inspect
+
+        from lmms_eval.models import get_model
+
+        hf_model_type = model.get_hf_model_type()
+        model_class = self._resolve_hf_model_class(model, hf_model_type)
+        # The ORT-GenAI adapter implements lmms-eval's simple request protocol.
+        # Prefer the matching simple HF wrapper when a model registers both a
+        # simple and chat implementation, so input/output evaluation uses the
+        # same task construction and media layout.
+        lm_cls = get_model(model_class, force_simple=True)
+        from olive.evaluator.lmms_ort import LMMSORTGenAIEvaluator
+
+        system_prompt = self.system_prompt
+        if system_prompt is None:
+            system_prompt = LMMSORTGenAIEvaluator.default_system_prompt_for_model_type(hf_model_type)
+
+        # lmms-eval wrappers have inconsistent constructor signatures: phi4_multimodal
+        # accepts dtype/trust_remote_code as named params, qwen2_5_vl asserts
+        # ``kwargs == {}`` at runtime even though it has ``**kwargs`` in its signature.
+        # Because the signature alone cannot tell "absorbs unknown kwargs" from
+        # "rejects unknown kwargs at runtime", we conservatively only forward
+        # ``device/dtype/trust_remote_code`` to wrappers that name them explicitly
+        # as parameters. For wrappers that take a different set of kwargs (e.g.
+        # backend-specific knobs), users pass them through ``hf_model_kwargs``,
+        # which is always forwarded unfiltered as an explicit user opt-in.
+        try:
+            accepted = set(inspect.signature(lm_cls.__init__).parameters)
+        except (TypeError, ValueError):
+            accepted = set()
+
+        optional_kwargs = {
+            "device": self._device_for_hf(device),
+            "dtype": self.dtype,
+            "trust_remote_code": self.trust_remote_code,
+            "system_prompt": system_prompt,
+            "language": self.whisper_language,
+            "task": self.whisper_task,
+        }
+        if hf_model_type == "whisper" and self.whisper_timestamps:
+            raise NotImplementedError("Whisper timestamp parity is not supported for HF input-model evaluation.")
+        forwarded = {k: v for k, v in optional_kwargs.items() if k in accepted}
+
+        init_kwargs = {
+            "pretrained": str(model.model_name_or_path),
+            "batch_size": self.batch_size,
+            **forwarded,
+            **self.hf_model_kwargs,
+        }
+        logger.info(
+            "Running lmms-eval (model_class=%s, pretrained=%s, forwarded_kwargs=%s)",
+            model_class,
+            init_kwargs["pretrained"],
+            sorted(set(init_kwargs) - {"pretrained", "batch_size"}),
+        )
+        lm = lm_cls(**init_kwargs)
+        if self.ignore_stop_strings:
+            _wrap_generate_until_drop_stops(lm, self.ignore_stop_strings)
+        return lm
+
+    def evaluate(
+        self,
+        model: "OliveModelHandler",
+        metrics: list[Metric],
+        device: Device = Device.CPU,
+        execution_providers: Optional[Union[str, list[str]]] = None,
+    ) -> MetricResult:
+        metrics_dict: dict[str, MetricResult] = {}
+        tasks = list(self.tasks)
+
+        if MetricType.SIZE_ON_DISK.value in tasks:
+            tasks.remove(MetricType.SIZE_ON_DISK.value)
+            metrics_dict[MetricType.SIZE_ON_DISK.value] = MetricResult.model_validate(
+                {
+                    SizeOnDiskSubType.BYTES.value: {
+                        "value": _lmms_model_size_on_disk(model),
+                        "priority": -1,
+                        "higher_is_better": False,
+                    }
+                }
+            )
+
+        if tasks:
+            from lmms_eval.evaluator import simple_evaluate
+
+            from olive.evaluator.lmms_ort import warmup_image_decoder
+
+            # Bind ORT-GenAI's image-decoder zlib before any HF input-model eval in
+            # this process decodes images; otherwise the ONNX eval hits
+            # "libpng error: bad parameters to zlib". See warmup_image_decoder().
+            warmup_image_decoder()
+
+            # Checks runtime type of model and routes to appropriate lmms-eval
+            # model wrapper (exposes generate_until/loglikelihood over the resolved backend).
+            if isinstance(model, ONNXModelHandler):
+                lm = self._build_ortgenai_mm_lm(model, execution_providers)
+            elif isinstance(model, HfModelHandler):
+                lm = self._build_hf_lm(model, device)
+            else:
+                raise ValueError(
+                    "LMMSEvaluator supports ONNXModelHandler (ORT-GenAI multimodal package) "
+                    f"and HfModelHandler. Got {type(model).__name__}."
+                )
+
+            results = simple_evaluate(
+                model=lm,
+                tasks=tasks,
+                batch_size=self.batch_size,
+                limit=self.limit,
+                log_samples=self.log_samples,
+            )
+
+            # Explicitly write lmms-eval results to json.
+            if self.output_path:
+                out = _lmms_output_path(self.output_path, model)
+                out.parent.mkdir(parents=True, exist_ok=True)
+                compact = {
+                    "model_type": "hf" if isinstance(model, HfModelHandler) else "onnx",
+                    "results": results.get("results", {}),
+                    "higher_is_better": results.get("higher_is_better", {}),
+                    "configs": {k: str(v) for k, v in results.get("configs", {}).items()},
+                }
+                out.write_text(json.dumps(compact, indent=2, default=str), encoding="utf-8")
+                logger.info("Wrote lmms-eval results to %s", out)
+
+                # When log_samples is on, persist per-doc generations to a sibling
+                # *_samples.json (probe-style: doc_id, target, full untruncated
+                # generation, score). The main results file stays compact.
+                samples = results.get("samples") or {}
+                if samples:
+                    summarized = {
+                        task_name: _summarize_lmms_samples(task_name, task_samples)
+                        for task_name, task_samples in samples.items()
+                    }
+                    samples_out = out.with_name(f"{out.stem}_samples{out.suffix}")
+                    samples_out.write_text(json.dumps(summarized, indent=2, default=str), encoding="utf-8")
+                    logger.info("Wrote lmms-eval per-sample generations to %s", samples_out)
+            elif self.log_samples:
+                logger.warning(
+                    "log_samples is set but output_path is not; per-sample generations "
+                    "were computed but not persisted. Set 'output_path' to save them."
+                )
+
+            # Convert lmms-eval results into Olive's MetricResult shape (mirrors LMEvaluator).
+            for task_name in sorted(results.get("results", {}).keys()):
+                task_results = results["results"][task_name]
+                task_directions = (results.get("higher_is_better") or {}).get(task_name) or {}
+                task_metrics = {}
+                for mf, v in sorted(task_results.items()):
+                    if mf == "alias" or not isinstance(v, (int, float)):
+                        continue
+                    metric_name, _, filter_name = mf.partition(",")
+                    # Skip all stderr variants. lmms-eval emits "<metric>_stderr" as
+                    # well as "<metric>_stderr_clt" / "<metric>_stderr_clustered"
+                    # (the latter two are numeric and would otherwise leak in as
+                    # bogus metrics), so match the "_stderr" substring, not just the
+                    # "_stderr" suffix.
+                    if "_stderr" in metric_name:
+                        continue
+                    metric_key = metric_name if filter_name in ("", "none") else f"{metric_name},{filter_name}"
+                    direction = task_directions.get(metric_name)
+                    task_metrics[metric_key] = SubMetricResult(
+                        value=float(v),
+                        priority=-1,
+                        higher_is_better=True if direction is None else bool(direction),
+                    )
+                if task_metrics:
+                    metrics_dict[task_name] = MetricResult.model_validate(task_metrics)
+        elif self.log_samples:
+            logger.warning("log_samples is set but no lmms-eval tasks were provided; no samples were generated.")
+
+        return flatten_metric_result(metrics_dict)
+
+
 @Registry.register("MTEBEvaluator")
 class MTEBEvaluator(OliveEvaluator):
     """Evaluator for embedding models using the MTEB (Massive Text Embedding Benchmark) library.
diff --git a/olive/olive_config.json b/olive/olive_config.json
index 25c4b546e..c8cdc8f2c 100644
--- a/olive/olive_config.json
+++ b/olive/olive_config.json
@@ -8,6 +8,14 @@
             "supported_algorithms": [  ],
             "supported_quantization_encodings": [  ]
         },
+        "CompositeToOnnxPackage": {
+            "module_path": "olive.passes.onnx.composite_to_onnx_package.CompositeToOnnxPackage",
+            "supported_providers": [ "*" ],
+            "supported_accelerators": [ "*" ],
+            "supported_precisions": [ "*" ],
+            "supported_algorithms": [  ],
+            "supported_quantization_encodings": [  ]
+        },
         "AimetQuantization": {
             "module_path": "olive.passes.onnx.aimet_quantization.AimetQuantization",
             "supported_providers": [ "*" ],
diff --git a/olive/passes/onnx/composite_to_onnx_package.py b/olive/passes/onnx/composite_to_onnx_package.py
new file mode 100644
index 000000000..55f03e65b
--- /dev/null
+++ b/olive/passes/onnx/composite_to_onnx_package.py
@@ -0,0 +1,170 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+# --------------------------------------------------------------------------
+"""Wrap a multi-component CompositeModel ORT-GenAI package as a single ONNXModel handler."""
+
+from __future__ import annotations
+
+import json
+import logging
+import shutil
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from olive.common.utils import hardlink_copy_dir
+from olive.model import ONNXModelHandler
+from olive.model.handler.composite import CompositeModelHandler
+from olive.passes import Pass
+from olive.passes.pass_config import BasePassConfig, PassConfigParam
+
+if TYPE_CHECKING:
+    from olive.hardware.accelerator import AcceleratorSpec
+
+logger = logging.getLogger(__name__)
+
+
+class CompositeToOnnxPackage(Pass):
+    """Repackage a CompositeModel ORT-GenAI package as a single :class:`ONNXModelHandler`.
+
+    ``MobiusBuilder`` and similar passes emit multi-component ORT-GenAI packages as a
+    :class:`CompositeModelHandler` whose components live in subdirectories::
+
+        output_dir/
+          genai_config.json
+          tokenizer.json
+          decoder/model.onnx
+          vision_encoder/model.onnx
+          audio_encoder/model.onnx
+          embedding/model.onnx
+
+    Two consumers need a concrete :class:`ONNXModelHandler` rather than a composite:
+
+    * ``LocalSystem.evaluate_model`` raises ``NotImplementedError`` for composite
+      models, so a composite package can't be evaluated directly.
+    * ORT-GenAI evaluators (``LMMSEvaluator``, ``OnnxEvaluator._inference_vision_genai``)
+      dispatch on ``ONNXModelHandler`` and locate ``genai_config.json`` relative to
+      the handler's ONNX file.
+
+    This pass performs that conversion **without flattening the layout**: the nested
+    directory structure is preserved (ONNX Runtime GenAI loads nested packages
+    directly, and ``genai_config.json`` already references the nested paths), so no
+    ONNX re-serialization or external-data rewriting is needed. The package tree is
+    hardlink-copied into the pass output directory and a single
+    :class:`ONNXModelHandler` is returned, pointing at the entry-point component
+    (defaults to ``decoder``). Evaluators discover ``genai_config.json`` by searching
+    upward from the entry ONNX file (see ``_find_genai_config`` in the evaluator).
+    """
+
+    _accepts_composite_model = True
+
+    @classmethod
+    def _default_config(cls, accelerator_spec: AcceleratorSpec) -> dict[str, PassConfigParam]:
+        return {
+            "entry_point_component": PassConfigParam(
+                type_=str,
+                default_value="decoder",
+                description=(
+                    "Name of the genai_config 'model' subsection (e.g. 'decoder', 'text') "
+                    "whose ONNX file the returned ONNXModelHandler will point at. If the "
+                    "name is not found, falls back to the first component with a 'filename' field."
+                ),
+            ),
+        }
+
+    @classmethod
+    def is_accelerator_agnostic(cls, accelerator_spec: AcceleratorSpec) -> bool:
+        # Pure file-system repackaging — no EP-specific behavior.
+        return True
+
+    def _run_for_config(
+        self,
+        model: CompositeModelHandler,
+        config: type[BasePassConfig],
+        output_model_path: str,
+    ) -> ONNXModelHandler:
+        if not isinstance(model, CompositeModelHandler):
+            raise ValueError(
+                f"CompositeToOnnxPackage expects a CompositeModelHandler input, got {type(model).__name__}."
+            )
+
+        src_dir = Path(model.model_path).resolve()
+        if not src_dir.is_dir():
+            raise ValueError(f"CompositeModel model_path is not a directory: {src_dir}")
+
+        src_genai_config = src_dir / "genai_config.json"
+        if not src_genai_config.is_file():
+            raise ValueError(
+                f"CompositeToOnnxPackage requires genai_config.json at the package root: {src_genai_config} not found."
+            )
+
+        genai_config = json.loads(src_genai_config.read_text(encoding="utf-8"))
+        model_section = genai_config.get("model")
+        if not isinstance(model_section, dict):
+            raise ValueError(f"Invalid genai_config.json at {src_genai_config}: missing 'model' section.")
+
+        entry_filename = self._select_entry_filename(model_section, config.entry_point_component)
+        if entry_filename is None:
+            raise ValueError(
+                "Failed to determine an entry-point component for CompositeToOnnxPackage. "
+                f"Requested '{config.entry_point_component}', no component matched and no fallback available."
+            )
+
+        dst_dir = self._resolve_output_dir(output_model_path)
+        # Copy the entire nested package tree (cheap hardlinks) into the pass output
+        # directory, preserving the multi-component subdirectory layout as-is.
+        hardlink_copy_dir(src_dir, dst_dir)
+        # genai_config.json is mutable deployment metadata (provider/session
+        # options are commonly adjusted after export). Give the output package an
+        # independent copy so changing it cannot mutate the input pass cache.
+        dst_genai_config = dst_dir / "genai_config.json"
+        dst_genai_config.unlink()
+        shutil.copy2(src_genai_config, dst_genai_config)
+
+        entry_path = dst_dir / entry_filename
+        if not entry_path.is_file():
+            raise ValueError(
+                f"Entry-point component '{entry_filename}' not found in packaged output at {entry_path}. "
+                "genai_config.json references a component file that does not exist on disk."
+            )
+
+        logger.info(
+            "CompositeToOnnxPackage: packaged %d components into '%s' (entry_point=%s, nested layout preserved)",
+            sum(1 for v in model_section.values() if isinstance(v, dict) and v.get("filename")),
+            dst_dir,
+            entry_filename,
+        )
+
+        return ONNXModelHandler(
+            model_path=str(dst_dir),
+            onnx_file_name=entry_filename,
+            model_attributes={
+                "ort_genai_package": True,
+                "entry_point_component": config.entry_point_component,
+                **(model.model_attributes or {}),
+            },
+        )
+
+    @staticmethod
+    def _resolve_output_dir(output_model_path: str) -> Path:
+        """Olive sometimes passes a `.onnx` file path; in that case use its stem as the directory."""
+        output_path = Path(output_model_path)
+        if output_path.suffix == ".onnx":
+            return output_path.parent / output_path.stem
+        return output_path
+
+    @staticmethod
+    def _select_entry_filename(model_section: dict, entry_point_component: str) -> str | None:
+        """Pick the (nested) filename for the entry-point component, falling back if missing."""
+        preferred = model_section.get(entry_point_component)
+        if isinstance(preferred, dict):
+            filename = preferred.get("filename")
+            if isinstance(filename, str):
+                return filename
+
+        for component_cfg in model_section.values():
+            if isinstance(component_cfg, dict):
+                filename = component_cfg.get("filename")
+                if isinstance(filename, str):
+                    return filename
+        return None
diff --git a/test/evaluator/test_lmms_ort.py b/test/evaluator/test_lmms_ort.py
new file mode 100644
index 000000000..343ac4304
--- /dev/null
+++ b/test/evaluator/test_lmms_ort.py
@@ -0,0 +1,1372 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+# --------------------------------------------------------------------------
+# Tests intentionally exercise "protected" runtime methods (_score_continuation,
+# _run_generation) and configure fake collaborators by setting attributes
+# directly on the fake. Both are normal in unit tests, so suppress pylint's
+# protected-access / attribute-defined-outside-init warnings for this file.
+# pylint: disable=protected-access,attribute-defined-outside-init,no-value-for-parameter,unexpected-keyword-arg
+import sys
+from types import ModuleType, SimpleNamespace
+from typing import ClassVar
+from unittest.mock import MagicMock, patch
+
+import numpy as np
+import PIL.Image
+import pytest
+
+from olive.evaluator.lmms_ort import (
+    LMMSORTGenAIEvaluator,
+    _build_whisper_prompt,
+    _normalize_execution_provider,
+)
+from olive.evaluator.olive_evaluator import LMMSEvaluator
+from olive.model import ONNXModelHandler
+
+
+def test_build_whisper_prompt_configures_language_and_task():
+    prompt = _build_whisper_prompt(
+        whisper_language="fr",
+        whisper_task="translate",
+        whisper_timestamps=False,
+    )
+
+    assert prompt == "<|startoftranscript|><|fr|><|translate|><|notimestamps|>"
+
+
+@pytest.mark.parametrize(
+    ("model_type", "expected"),
+    [
+        ("qwen2_5_vl", "You are a helpful assistant."),
+        ("qwen2_5_vl_text", "You are a helpful assistant."),
+        ("qwen3_vl", "You are a helpful assistant."),
+        ("gemma4", "You are a helpful assistant."),
+        ("gemma3", "You are a helpful assistant."),
+        (
+            "qwen2_5_omni",
+            "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, "
+            "capable of perceiving auditory and visual inputs, as well as generating text and speech.",
+        ),
+        (
+            "qwen3_omni_moe",
+            "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, "
+            "capable of perceiving auditory and visual inputs, as well as generating text and speech.",
+        ),
+        ("phi4mm", ""),
+        ("phi3v", ""),
+        ("whisper", "You are a helpful assistant."),
+        (None, "You are a helpful assistant."),
+    ],
+)
+def test_default_system_prompt_for_model_type_matches_lmms_eval_wrappers(model_type, expected):
+    assert LMMSORTGenAIEvaluator.default_system_prompt_for_model_type(model_type) == expected
+
+
+@pytest.mark.parametrize(
+    ("execution_provider", "expected"),
+    [
+        ("CUDAExecutionProvider", "cuda"),
+        ("CPUExecutionProvider", "cpu"),
+        ("DmlExecutionProvider", "dml"),
+        ("gpu", "cuda"),
+        (None, "follow_config"),
+        (("CUDAExecutionProvider", {"device_id": "0"}), "cuda"),
+    ],
+)
+def test_normalize_execution_provider(execution_provider, expected):
+    assert _normalize_execution_provider(execution_provider) == expected
+
+
+def _make_evaluator_for_prompt_tests():
+    """Construct an LMMSORTGenAIEvaluator with __init__ skipped.
+
+    Lets us unit-test the prompt-building path without needing a real ORT-GenAI
+    model on disk.
+    """
+    inst = LMMSORTGenAIEvaluator.__new__(LMMSORTGenAIEvaluator)
+    inst._tokenizer = MagicMock(name="og.Tokenizer")
+    inst._model_type = "test_model"
+    inst.system_prompt = "You are helpful."
+    inst._supports_structured_content = True
+    inst.whisper_language = "en"
+    inst.whisper_task = "transcribe"
+    inst.whisper_timestamps = False
+    return inst
+
+
+def test_build_prompt_for_request_uses_apply_chat_template_by_default():
+    inst = _make_evaluator_for_prompt_tests()
+    inst._tokenizer.apply_chat_template.return_value = "<rendered chat prompt>"
+
+    out = inst._build_prompt_for_request("What is in the image?", num_images=1, num_audios=0)
+
+    assert out == "<rendered chat prompt>"
+    inst._tokenizer.apply_chat_template.assert_called_once()
+    messages_json_arg = inst._tokenizer.apply_chat_template.call_args.args[0]
+    assert inst._tokenizer.apply_chat_template.call_args.kwargs.get("add_generation_prompt") is True
+
+    import json as _json
+
+    messages = _json.loads(messages_json_arg)
+    assert messages[0] == {"role": "system", "content": "You are helpful."}
+    assert messages[1] == {
+        "role": "user",
+        "content": [{"type": "image"}, {"type": "text", "text": "What is in the image?"}],
+    }
+
+
+def test_build_prompt_for_request_skips_system_when_empty():
+    inst = _make_evaluator_for_prompt_tests()
+    inst.system_prompt = ""
+    inst._tokenizer.apply_chat_template.return_value = "out"
+
+    inst._build_prompt_for_request("Q", num_images=0, num_audios=0)
+
+    import json as _json
+
+    messages = _json.loads(inst._tokenizer.apply_chat_template.call_args.args[0])
+    assert all(m["role"] != "system" for m in messages)
+    assert messages[-1] == {"role": "user", "content": "Q"}
+
+
+def test_build_prompt_for_request_uses_structured_media_parts():
+    inst = _make_evaluator_for_prompt_tests()
+    inst.system_prompt = ""
+    inst._tokenizer.apply_chat_template.return_value = "out"
+
+    inst._build_prompt_for_request("Q", num_images=2, num_audios=1)
+
+    import json as _json
+
+    messages = _json.loads(inst._tokenizer.apply_chat_template.call_args.args[0])
+    assert messages[0]["content"] == [
+        {"type": "image"},
+        {"type": "image"},
+        {"type": "audio"},
+        {"type": "text", "text": "Q"},
+    ]
+
+
+def test_build_prompt_for_request_uses_structured_content_when_supported():
+    """Use structured content parts when supported."""
+    inst = _make_evaluator_for_prompt_tests()
+    inst._tokenizer.apply_chat_template.return_value = "<rendered>"
+
+    inst._build_prompt_for_request("What is this?", num_images=1, num_audios=1)
+
+    import json as _json
+
+    messages = _json.loads(inst._tokenizer.apply_chat_template.call_args.args[0])
+    user_msg = messages[-1]
+    assert user_msg["role"] == "user"
+    # Content remains a list of typed parts rather than a flat string.
+    assert user_msg["content"] == [
+        {"type": "image"},
+        {"type": "audio"},
+        {"type": "text", "text": "What is this?"},
+    ]
+
+
+def test_build_prompt_for_request_rejects_media_when_structured_content_unsupported():
+    inst = _make_evaluator_for_prompt_tests()
+    inst._supports_structured_content = False
+
+    with pytest.raises(ValueError, match="structured image/audio content"):
+        inst._build_prompt_for_request("What is this?", num_images=1, num_audios=0)
+
+
+def test_probe_structured_content_support_detects_injection():
+    """A template that injects the media token (no dict repr) -> supported."""
+    inst = LMMSORTGenAIEvaluator.__new__(LMMSORTGenAIEvaluator)
+    inst._tokenizer = MagicMock()
+    inst._tokenizer.apply_chat_template.side_effect = [
+        "<|im_start|>user\n<|image|>x<|im_end|>",
+        "<|im_start|>user\nx<|im_end|>",
+    ]
+
+    assert inst._probe_structured_content_support() is True
+
+
+def test_probe_structured_content_support_detects_stringified_repr():
+    """A template that leaks the Python dict repr -> not supported."""
+    inst = LMMSORTGenAIEvaluator.__new__(LMMSORTGenAIEvaluator)
+    inst._tokenizer = MagicMock()
+    inst._tokenizer.apply_chat_template.return_value = "<|user|>[{'type': 'image'}, ...]<|end|>"
+
+    assert inst._probe_structured_content_support() is False
+
+
+def test_probe_structured_content_support_detects_ignored_image_part():
+    inst = LMMSORTGenAIEvaluator.__new__(LMMSORTGenAIEvaluator)
+    inst._tokenizer = MagicMock()
+    inst._tokenizer.apply_chat_template.side_effect = ["same-rendering", "same-rendering"]
+
+    assert inst._probe_structured_content_support() is False
+
+
+def _make_evaluator_for_generate_until(ignore_stop_strings):
+    """Construct an LMMSORTGenAIEvaluator with __init__ skipped.
+
+    Wired so generate_until runs without a real model: _run_generation is a stub
+    that records the stop list it was given.
+    """
+    inst = LMMSORTGenAIEvaluator.__new__(LMMSORTGenAIEvaluator)
+    inst.max_new_tokens = 256
+    inst.ignore_stop_strings = set(ignore_stop_strings or [])
+    inst.cache_hook = SimpleNamespace(add_partial=lambda *a, **k: None)
+    inst._get_doc_and_visuals = lambda *a, **k: (None, [])
+    inst._build_prompt_for_request = lambda *a, **k: "<prompt>"
+    inst.recorded_stops = []
+
+    def _fake_run_generation(prompt, images, audios, max_new, stop, search_options=None):
+        inst.recorded_stops.append(stop)
+        inst.recorded_search_options = search_options
+        return "ok"
+
+    inst._run_generation = _fake_run_generation
+    return inst
+
+
+def _make_generate_until_request(until):
+    gen_kwargs = {"until": until}
+    return SimpleNamespace(args=("ctx", gen_kwargs, None, 0, "task", "split"))
+
+
+def test_generate_until_drops_ignored_stop_strings():
+    inst = _make_evaluator_for_generate_until(ignore_stop_strings=["\n\n"])
+    req = _make_generate_until_request(until=["\n\n", "Q:"])
+
+    inst.generate_until([req], disable_tqdm=True)
+
+    # "\n\n" is suppressed, the legitimate "Q:" stop is preserved.
+    assert inst.recorded_stops == [["Q:"]]
+
+
+def test_generate_until_sets_stop_none_when_all_ignored():
+    inst = _make_evaluator_for_generate_until(ignore_stop_strings=["\n\n"])
+    req = _make_generate_until_request(until=["\n\n"])
+
+    inst.generate_until([req], disable_tqdm=True)
+
+    # The only stop string was suppressed -> None (generation runs to EOS).
+    assert inst.recorded_stops == [None]
+
+
+def test_generate_until_keeps_stops_when_nothing_to_ignore():
+    inst = _make_evaluator_for_generate_until(ignore_stop_strings=[])
+    req = _make_generate_until_request(until=["\n\n", "Q:"])
+
+    inst.generate_until([req], disable_tqdm=True)
+
+    # No ignore list configured -> stops pass through unchanged.
+    assert inst.recorded_stops == [["\n\n", "Q:"]]
+
+
+def test_generate_until_forwards_supported_search_options():
+    inst = _make_evaluator_for_generate_until(ignore_stop_strings=[])
+    req = SimpleNamespace(
+        args=(
+            "ctx",
+            {
+                "do_sample": True,
+                "temperature": 0.7,
+                "top_p": 0.9,
+                "top_k": 10,
+                "repetition_penalty": 1.1,
+            },
+            None,
+            0,
+            "task",
+            "split",
+        )
+    )
+
+    inst.generate_until([req], disable_tqdm=True)
+
+    assert inst.recorded_search_options == {
+        "do_sample": True,
+        "temperature": 0.7,
+        "top_p": 0.9,
+        "top_k": 10,
+        "repetition_penalty": 1.1,
+    }
+
+
+def test_generate_until_rejects_unsupported_beam_search():
+    inst = _make_evaluator_for_generate_until(ignore_stop_strings=[])
+    req = SimpleNamespace(args=("ctx", {"num_beams": 2}, None, 0, "task", "split"))
+
+    with pytest.raises(NotImplementedError, match="Beam search"):
+        inst.generate_until([req], disable_tqdm=True)
+
+
+def test_wrap_generate_until_drop_stops_filters_hf_wrapper_until():
+    """Strip ignored stops before delegating to the HF wrapper.
+
+    The patch updates each request's generation kwargs in place.
+    """
+    from olive.evaluator.olive_evaluator import _wrap_generate_until_drop_stops
+
+    seen_until = []
+
+    class FakeHfWrapper:
+        def generate_until(self, requests, *args, **kwargs):
+            seen_until.extend(r.args[1].get("until") for r in requests)
+            return ["ok"] * len(requests)
+
+    lm = FakeHfWrapper()
+    _wrap_generate_until_drop_stops(lm, ["\n\n"])
+
+    reqs = [
+        _make_generate_until_request(until=["\n\n", "Q:"]),  # partial -> keep "Q:"
+        _make_generate_until_request(until=["\n\n"]),  # all ignored -> None
+        _make_generate_until_request(until="\n\n"),  # str form -> None
+    ]
+    out = lm.generate_until(reqs)
+
+    assert out == ["ok", "ok", "ok"]
+    assert seen_until == [["Q:"], [], []]
+
+
+def test_wrap_generate_until_drop_stops_finds_chat_wrapper_kwargs():
+    from olive.evaluator.olive_evaluator import _wrap_generate_until_drop_stops
+
+    seen_until = []
+
+    class FakeChatWrapper:
+        def generate_until(self, requests, *args, **kwargs):
+            seen_until.extend(request.args[2]["until"] for request in requests)
+            return ["ok"] * len(requests)
+
+    lm = FakeChatWrapper()
+    _wrap_generate_until_drop_stops(lm, ["\n\n"])
+    request = SimpleNamespace(args=("ctx", lambda doc: doc, {"until": ["\n\n", "END"]}, 0, "task", "split"))
+
+    lm.generate_until([request])
+
+    assert seen_until == [["END"]]
+
+
+def test_wrap_generate_until_drop_stops_noop_without_ignore_list():
+    from olive.evaluator.olive_evaluator import _wrap_generate_until_drop_stops
+
+    class FakeHfWrapper:
+        def generate_until(self, requests, *args, **kwargs):
+            return "orig"
+
+    lm = FakeHfWrapper()
+    _wrap_generate_until_drop_stops(lm, [])
+    # Empty ignore list -> generate_until not wrapped; original behavior intact.
+    assert "generate_until" not in vars(lm)
+    assert lm.generate_until([]) == "orig"
+
+
+def test_lmms_evaluator_converts_lmms_results(tmp_path):
+    model_dir = tmp_path / "model"
+    model_dir.mkdir()
+    model_path = model_dir / "text.onnx"
+    model_path.touch()
+    (model_dir / "genai_config.json").write_text('{"model": {"type": "phi4mm"}}', encoding="utf-8")
+
+    output_path = tmp_path / "results.json"
+    evaluator = LMMSEvaluator(
+        tasks=["ai2d_lite"],
+        batch_size=1,
+        limit=2,
+        output_path=str(output_path),
+        fail_on_error=False,
+    )
+    model = ONNXModelHandler(model_path=str(model_path))
+
+    simple_evaluate_result = {
+        "results": {
+            "ai2d_lite": {
+                "alias": "AI2D Lite",
+                "exact_match,none": 0.5,
+                "exact_match_stderr,none": 0.1,
+                "samples": [{"ignored": True}],
+            }
+        },
+        "configs": {"ai2d_lite": {"task": "ai2d_lite"}},
+    }
+
+    simple_evaluate_mock = MagicMock(return_value=simple_evaluate_result)
+    lmms_eval_module = ModuleType("lmms_eval")
+    lmms_eval_evaluator_module = ModuleType("lmms_eval.evaluator")
+    lmms_eval_evaluator_module.simple_evaluate = simple_evaluate_mock
+
+    with (
+        patch.dict(
+            sys.modules,
+            {"lmms_eval": lmms_eval_module, "lmms_eval.evaluator": lmms_eval_evaluator_module},
+        ),
+        patch("olive.evaluator.lmms_ort.LMMSORTGenAIEvaluator", return_value=SimpleNamespace()) as lm_mock,
+    ):
+        result = evaluator.evaluate(model, [], execution_providers=["CUDAExecutionProvider"])
+
+    lm_mock.assert_called_once_with(
+        pretrained=str(model_dir),
+        batch_size=1,
+        max_new_tokens=256,
+        max_length=32768,
+        system_prompt=None,
+        execution_provider=["CUDAExecutionProvider"],
+        provider_options=None,
+        fail_on_error=False,
+        ignore_stop_strings=None,
+        whisper_language="en",
+        whisper_task="transcribe",
+        whisper_timestamps=False,
+    )
+    simple_evaluate_mock.assert_called_once()
+    assert result.get_value("ai2d_lite", "exact_match") == 0.5
+    assert output_path.exists()
+
+
+def test_lmms_evaluator_preserves_filter_names_and_metric_direction(tmp_path):
+    model_dir = tmp_path / "model"
+    model_dir.mkdir()
+    model_path = model_dir / "text.onnx"
+    model_path.touch()
+    (model_dir / "genai_config.json").write_text('{"model": {"type": "phi4mm"}}', encoding="utf-8")
+    evaluator = LMMSEvaluator(tasks=["speech_task"])
+    model = ONNXModelHandler(model_path=str(model_path))
+    simple_evaluate_result = {
+        "results": {
+            "speech_task": {
+                "alias": "Speech",
+                "wer,strict": 0.2,
+                "wer,flexible": 0.1,
+                "wer_stderr,strict": 0.01,
+            }
+        },
+        "higher_is_better": {"speech_task": {"wer": False}},
+        "configs": {},
+    }
+    lmms_eval_module = ModuleType("lmms_eval")
+    lmms_eval_evaluator_module = ModuleType("lmms_eval.evaluator")
+    lmms_eval_evaluator_module.simple_evaluate = MagicMock(return_value=simple_evaluate_result)
+
+    with (
+        patch.dict(
+            sys.modules,
+            {"lmms_eval": lmms_eval_module, "lmms_eval.evaluator": lmms_eval_evaluator_module},
+        ),
+        patch("olive.evaluator.lmms_ort.LMMSORTGenAIEvaluator", return_value=SimpleNamespace()),
+    ):
+        result = evaluator.evaluate(model, [])
+
+    assert result.root["speech_task-wer,strict"].value == 0.2
+    assert result.root["speech_task-wer,flexible"].value == 0.1
+    assert result.root["speech_task-wer,strict"].higher_is_better is False
+    assert all("_stderr" not in key for key in result.root)
+
+
+def test_lmms_output_path_avoids_cross_backend_overwrite(tmp_path):
+    from olive.evaluator.olive_evaluator import _lmms_output_path
+
+    output_path = tmp_path / "results.json"
+    output_path.write_text('{"model_type": "hf"}', encoding="utf-8")
+    model_path = tmp_path / "model.onnx"
+    model_path.touch()
+    model = ONNXModelHandler(model_path=str(model_path))
+
+    assert _lmms_output_path(str(output_path), model) == tmp_path / "results_onnx.json"
+
+
+def test_lmms_model_size_counts_whole_genai_package(tmp_path):
+    from olive.evaluator.olive_evaluator import _lmms_model_size_on_disk
+
+    package = tmp_path / "package"
+    decoder = package / "decoder"
+    vision = package / "vision"
+    decoder.mkdir(parents=True)
+    vision.mkdir()
+    (package / "genai_config.json").write_text("{}", encoding="utf-8")
+    (decoder / "model.onnx").write_bytes(b"decoder")
+    (vision / "model.onnx").write_bytes(b"vision")
+    model = ONNXModelHandler(model_path=str(package), onnx_file_name="decoder/model.onnx")
+
+    assert _lmms_model_size_on_disk(model) == sum(path.stat().st_size for path in package.rglob("*") if path.is_file())
+
+
+def test_summarize_lmms_samples_counts_nested_mathvision_score():
+    from olive.evaluator.olive_evaluator import _summarize_lmms_samples
+
+    summary = _summarize_lmms_samples(
+        "mathvision_test",
+        [
+            {
+                "doc_id": 0,
+                "target": "A",
+                "resps": [["answer"]],
+                "mathvision_standard_eval": {"scores": [True], "response": ["answer"]},
+            }
+        ],
+    )
+
+    assert summary["n_scored_correct"] == 1
+
+
+def test_lmms_evaluator_requires_genai_config(tmp_path):
+    model_path = tmp_path / "text.onnx"
+    model_path.touch()
+    evaluator = LMMSEvaluator(tasks=["ai2d_lite"])
+    model = ONNXModelHandler(model_path=str(model_path))
+
+    with pytest.raises(ValueError, match="requires an ORT-GenAI package"):
+        evaluator.evaluate(model, [])
+
+
+# -----------------------------------------------------------------------------
+# HuggingFace dispatch
+# -----------------------------------------------------------------------------
+
+
+def _make_hf_model_handler_stub(model_name_or_path: str, hf_model_type: str):
+    handler = MagicMock(name="HfModelHandler")
+    handler.model_name_or_path = model_name_or_path
+    handler.get_hf_model_type.return_value = hf_model_type
+    return handler
+
+
+def _patch_isinstance_for_hf(handler_stub, monkeypatch):
+    """Force isinstance(handler_stub, HfModelHandler) to True for the test path.
+
+    Avoids constructing a real HfModelHandler (which would require a real HF
+    model on disk) while still exercising the dispatch logic.
+    """
+    import olive.evaluator.olive_evaluator as oe
+
+    real_isinstance = isinstance
+
+    def _isinstance(obj, cls):
+        if obj is handler_stub and cls is oe.HfModelHandler:
+            return True
+        if obj is handler_stub and cls is oe.ONNXModelHandler:
+            return False
+        return real_isinstance(obj, cls)
+
+    monkeypatch.setattr(oe, "isinstance", _isinstance, raising=False)
+    oe.isinstance = _isinstance
+
+
+class _FakePhi4Wrapper:
+    """Fake lmms-eval wrapper with a phi4_multimodal-style signature.
+
+    Accepts dtype + trust_remote_code (mirrors lmms-eval's phi4_multimodal class).
+    """
+
+    last_kwargs: ClassVar[dict] = {}
+
+    def __init__(
+        self,
+        pretrained,
+        device="cuda",
+        dtype="auto",
+        batch_size=1,
+        trust_remote_code=True,
+        system_prompt=None,
+        **kwargs,
+    ):
+        type(self).last_kwargs = {
+            "pretrained": pretrained,
+            "device": device,
+            "dtype": dtype,
+            "batch_size": batch_size,
+            "trust_remote_code": trust_remote_code,
+            "system_prompt": system_prompt,
+            **kwargs,
+        }
+
+
+class _FakeQwenWrapper:
+    """Fake lmms-eval wrapper with a qwen2_5_vl-style signature.
+
+    Does NOT accept dtype or trust_remote_code (mirrors lmms-eval's qwen2_5_vl
+    class which asserts kwargs == {}).
+    """
+
+    last_kwargs: ClassVar[dict] = {}
+
+    def __init__(
+        self,
+        pretrained,
+        device="cuda",
+        device_map="auto",
+        batch_size=1,
+        system_prompt=None,
+        **kwargs,
+    ):
+        if kwargs:
+            raise AssertionError(f"Unexpected kwargs: {kwargs}")
+        type(self).last_kwargs = {
+            "pretrained": pretrained,
+            "device": device,
+            "device_map": device_map,
+            "batch_size": batch_size,
+            "system_prompt": system_prompt,
+        }
+
+
+class _FakeKwargsWrapper:
+    """Fake lmms-eval wrapper that absorbs ALL options via ``**kwargs``.
+
+    Mirrors lmms-eval wrappers (and HF model wrappers more generally) that take
+    only the required ``pretrained`` argument by name and pass everything else
+    through ``**kwargs`` to the underlying HF transformers model. Used to verify
+    LMMSEvaluator forwards optional kwargs (dtype, trust_remote_code, device) to
+    such wrappers instead of silently dropping them because they aren't in
+    ``inspect.signature(...).parameters`` as named params.
+    """
+
+    last_kwargs: ClassVar[dict] = {}
+
+    def __init__(self, pretrained, **kwargs):
+        type(self).last_kwargs = {"pretrained": pretrained, **kwargs}
+
+
+class _FakeWhisperWrapper:
+    last_kwargs: ClassVar[dict] = {}
+
+    def __init__(self, pretrained, device="cuda", batch_size=1, language="en", task="transcribe"):
+        type(self).last_kwargs = {
+            "pretrained": pretrained,
+            "device": device,
+            "batch_size": batch_size,
+            "language": language,
+            "task": task,
+        }
+
+
+def test_lmms_evaluator_auto_detects_hf_model_class_from_model_type(tmp_path, monkeypatch):
+    """When model_class is unset, auto-detect from HfModelHandler.get_hf_model_type()."""
+    handler_stub = _make_hf_model_handler_stub("/local/path/Phi-4-multimodal-instruct", "phi4mm")
+    _patch_isinstance_for_hf(handler_stub, monkeypatch)
+
+    output_path = tmp_path / "results.json"
+    evaluator = LMMSEvaluator(tasks=["ai2d_lite"], batch_size=2, limit=4, output_path=str(output_path))
+
+    simple_evaluate_mock = MagicMock(
+        return_value={"results": {"ai2d_lite": {"alias": "AI2D", "exact_match,none": 0.75}}, "configs": {}}
+    )
+    lmms_eval_module = ModuleType("lmms_eval")
+    lmms_eval_evaluator_module = ModuleType("lmms_eval.evaluator")
+    lmms_eval_evaluator_module.simple_evaluate = simple_evaluate_mock
+    lmms_eval_models_module = ModuleType("lmms_eval.models")
+    lmms_eval_models_module.get_model = MagicMock(return_value=_FakePhi4Wrapper)
+
+    with patch.dict(
+        sys.modules,
+        {
+            "lmms_eval": lmms_eval_module,
+            "lmms_eval.evaluator": lmms_eval_evaluator_module,
+            "lmms_eval.models": lmms_eval_models_module,
+        },
+    ):
+        result = evaluator.evaluate(handler_stub, [])
+
+    lmms_eval_models_module.get_model.assert_called_once_with("phi4_multimodal", force_simple=True)
+    assert _FakePhi4Wrapper.last_kwargs["pretrained"] == "/local/path/Phi-4-multimodal-instruct"
+    assert _FakePhi4Wrapper.last_kwargs["batch_size"] == 2
+    # trust_remote_code defaults to False (see olive/evaluator/olive_evaluator.py
+    # LMMSEvaluator.__init__); users opt in explicitly in the recipe.
+    assert _FakePhi4Wrapper.last_kwargs["trust_remote_code"] is False
+    assert _FakePhi4Wrapper.last_kwargs["dtype"] == "auto"
+    assert _FakePhi4Wrapper.last_kwargs["system_prompt"] == ""
+    simple_evaluate_mock.assert_called_once()
+    assert result.get_value("ai2d_lite", "exact_match") == 0.75
+
+
+def test_lmms_evaluator_filters_kwargs_for_qwen_style_wrapper(monkeypatch):
+    """Wrappers like qwen2_5_vl reject unknown kwargs.
+
+    LMMSEvaluator must inspect the wrapper signature and only forward kwargs
+    the wrapper actually declares as named parameters.
+    """
+    handler_stub = _make_hf_model_handler_stub("/p/Qwen2.5-VL-3B-Instruct", "qwen2_5_vl")
+    _patch_isinstance_for_hf(handler_stub, monkeypatch)
+
+    evaluator = LMMSEvaluator(tasks=["mmstar"], batch_size=1, dtype="bfloat16", trust_remote_code=True)
+
+    simple_evaluate_mock = MagicMock(return_value={"results": {}, "configs": {}})
+    lmms_eval_module = ModuleType("lmms_eval")
+    lmms_eval_evaluator_module = ModuleType("lmms_eval.evaluator")
+    lmms_eval_evaluator_module.simple_evaluate = simple_evaluate_mock
+    lmms_eval_models_module = ModuleType("lmms_eval.models")
+    lmms_eval_models_module.get_model = MagicMock(return_value=_FakeQwenWrapper)
+
+    with patch.dict(
+        sys.modules,
+        {
+            "lmms_eval": lmms_eval_module,
+            "lmms_eval.evaluator": lmms_eval_evaluator_module,
+            "lmms_eval.models": lmms_eval_models_module,
+        },
+    ):
+        evaluator.evaluate(handler_stub, [])
+
+    # dtype + trust_remote_code must NOT have been forwarded (Qwen wrapper would error)
+    assert "dtype" not in _FakeQwenWrapper.last_kwargs
+    assert "trust_remote_code" not in _FakeQwenWrapper.last_kwargs
+    # but pretrained, device, batch_size MUST have been forwarded
+    assert _FakeQwenWrapper.last_kwargs["pretrained"] == "/p/Qwen2.5-VL-3B-Instruct"
+    assert _FakeQwenWrapper.last_kwargs["device"] == "cpu"  # Device.CPU default
+    assert _FakeQwenWrapper.last_kwargs["batch_size"] == 1
+    assert _FakeQwenWrapper.last_kwargs["system_prompt"] == "You are a helpful assistant."
+
+
+def test_lmms_evaluator_does_not_forward_to_pure_var_keyword_wrappers(monkeypatch):
+    """Verify ``device``/``dtype``/``trust_remote_code`` are NOT auto-forwarded to ``**kwargs`` wrappers.
+
+    Rationale: the signature alone cannot tell "absorbs unknowns" from "rejects
+    unknowns at runtime" (qwen2_5_vl has ``**kwargs`` and asserts ``kwargs ==
+    {}``). To stay safe, only kwargs named explicitly as parameters are
+    auto-forwarded. Users who want to forward additional kwargs to a pure
+    ``**kwargs`` wrapper must use ``hf_model_kwargs`` (explicit user opt-in).
+    """
+    handler_stub = _make_hf_model_handler_stub("/p/some-vlm", "qwen2_5_vl")
+    _patch_isinstance_for_hf(handler_stub, monkeypatch)
+
+    evaluator = LMMSEvaluator(
+        tasks=["mmstar"],
+        batch_size=1,
+        dtype="bfloat16",
+        trust_remote_code=True,
+        # The escape hatch for forwarding arbitrary kwargs to a wrapper that
+        # absorbs them via **kwargs:
+        hf_model_kwargs={"custom_backend_opt": "value"},
+    )
+
+    simple_evaluate_mock = MagicMock(return_value={"results": {}, "configs": {}})
+    lmms_eval_module = ModuleType("lmms_eval")
+    lmms_eval_evaluator_module = ModuleType("lmms_eval.evaluator")
+    lmms_eval_evaluator_module.simple_evaluate = simple_evaluate_mock
+    lmms_eval_models_module = ModuleType("lmms_eval.models")
+    lmms_eval_models_module.get_model = MagicMock(return_value=_FakeKwargsWrapper)
+
+    with patch.dict(
+        sys.modules,
+        {
+            "lmms_eval": lmms_eval_module,
+            "lmms_eval.evaluator": lmms_eval_evaluator_module,
+            "lmms_eval.models": lmms_eval_models_module,
+        },
+    ):
+        evaluator.evaluate(handler_stub, [])
+
+    # Required kwargs are always forwarded.
+    assert _FakeKwargsWrapper.last_kwargs["pretrained"] == "/p/some-vlm"
+    assert _FakeKwargsWrapper.last_kwargs["batch_size"] == 1
+    # Optional kwargs are NOT forwarded to pure **kwargs wrappers.
+    assert "dtype" not in _FakeKwargsWrapper.last_kwargs
+    assert "trust_remote_code" not in _FakeKwargsWrapper.last_kwargs
+    assert "device" not in _FakeKwargsWrapper.last_kwargs
+    # The explicit hf_model_kwargs escape hatch IS forwarded.
+    assert _FakeKwargsWrapper.last_kwargs["custom_backend_opt"] == "value"
+
+
+def test_lmms_evaluator_uses_explicit_model_class_when_set(monkeypatch):
+    """An explicit ``model_class`` in the recipe overrides auto-detection."""
+    handler_stub = _make_hf_model_handler_stub("/p/some-vlm", "some-unknown-vlm-type")
+    _patch_isinstance_for_hf(handler_stub, monkeypatch)
+
+    evaluator = LMMSEvaluator(tasks=["mmstar"], model_class="qwen2_5_vl", batch_size=1)
+
+    simple_evaluate_mock = MagicMock(return_value={"results": {}, "configs": {}})
+    lmms_eval_module = ModuleType("lmms_eval")
+    lmms_eval_evaluator_module = ModuleType("lmms_eval.evaluator")
+    lmms_eval_evaluator_module.simple_evaluate = simple_evaluate_mock
+    lmms_eval_models_module = ModuleType("lmms_eval.models")
+    lmms_eval_models_module.get_model = MagicMock(return_value=_FakeQwenWrapper)
+
+    with patch.dict(
+        sys.modules,
+        {
+            "lmms_eval": lmms_eval_module,
+            "lmms_eval.evaluator": lmms_eval_evaluator_module,
+            "lmms_eval.models": lmms_eval_models_module,
+        },
+    ):
+        evaluator.evaluate(handler_stub, [])
+
+    lmms_eval_models_module.get_model.assert_called_once_with("qwen2_5_vl", force_simple=True)
+    # Explicit model_class bypasses class auto-detection, but model_type is still
+    # consulted to resolve the parity-correct default system prompt.
+    handler_stub.get_hf_model_type.assert_called_once()
+
+
+def test_lmms_evaluator_forwards_whisper_language_and_task(monkeypatch):
+    handler_stub = _make_hf_model_handler_stub("/p/whisper", "whisper")
+    _patch_isinstance_for_hf(handler_stub, monkeypatch)
+    evaluator = LMMSEvaluator(
+        tasks=["fleurs_fr"],
+        whisper_language="fr",
+        whisper_task="translate",
+    )
+    lmms_eval_module = ModuleType("lmms_eval")
+    lmms_eval_evaluator_module = ModuleType("lmms_eval.evaluator")
+    lmms_eval_evaluator_module.simple_evaluate = MagicMock(return_value={"results": {}, "configs": {}})
+    lmms_eval_models_module = ModuleType("lmms_eval.models")
+    lmms_eval_models_module.get_model = MagicMock(return_value=_FakeWhisperWrapper)
+
+    with patch.dict(
+        sys.modules,
+        {
+            "lmms_eval": lmms_eval_module,
+            "lmms_eval.evaluator": lmms_eval_evaluator_module,
+            "lmms_eval.models": lmms_eval_models_module,
+        },
+    ):
+        evaluator.evaluate(handler_stub, [])
+
+    assert _FakeWhisperWrapper.last_kwargs["language"] == "fr"
+    assert _FakeWhisperWrapper.last_kwargs["task"] == "translate"
+
+
+def test_lmms_evaluator_raises_when_hf_model_type_is_unmapped(monkeypatch):
+    """If we can't auto-detect and the user didn't set model_class, fail loudly."""
+    handler_stub = _make_hf_model_handler_stub("/p/exotic-model", "some-exotic-vlm")
+    _patch_isinstance_for_hf(handler_stub, monkeypatch)
+
+    evaluator = LMMSEvaluator(tasks=["mmstar"])
+
+    # Even with lmms_eval modules mocked, the error fires before reaching them.
+    lmms_eval_module = ModuleType("lmms_eval")
+    lmms_eval_evaluator_module = ModuleType("lmms_eval.evaluator")
+    lmms_eval_evaluator_module.simple_evaluate = MagicMock()
+    lmms_eval_models_module = ModuleType("lmms_eval.models")
+    lmms_eval_models_module.get_model = MagicMock()
+    with (
+        patch.dict(
+            sys.modules,
+            {
+                "lmms_eval": lmms_eval_module,
+                "lmms_eval.evaluator": lmms_eval_evaluator_module,
+                "lmms_eval.models": lmms_eval_models_module,
+            },
+        ),
+        pytest.raises(ValueError, match=r"Could not auto-detect lmms-eval model_class"),
+    ):
+        evaluator.evaluate(handler_stub, [])
+
+
+def test_lmms_evaluator_rejects_unsupported_handler_type():
+    """LMMSEvaluator only supports HfModelHandler and ONNXModelHandler-as-ortgenai."""
+    evaluator = LMMSEvaluator(tasks=["mmstar"])
+    bogus = SimpleNamespace()  # neither HfModelHandler nor ONNXModelHandler
+
+    lmms_eval_module = ModuleType("lmms_eval")
+    lmms_eval_evaluator_module = ModuleType("lmms_eval.evaluator")
+    lmms_eval_evaluator_module.simple_evaluate = MagicMock()
+    with (
+        patch.dict(sys.modules, {"lmms_eval": lmms_eval_module, "lmms_eval.evaluator": lmms_eval_evaluator_module}),
+        pytest.raises(ValueError, match=r"ONNXModelHandler.*HfModelHandler"),
+    ):
+        evaluator.evaluate(bogus, [])
+
+
+# -----------------------------------------------------------------------------
+# lmms-eval MODEL_REGISTRY_V2 entry-point integration
+# -----------------------------------------------------------------------------
+
+
+def test_lmms_ort_genai_evaluator_is_simple_flag_matches_registration():
+    """Verify is_simple matches the lmms-eval registration type.
+
+    MODEL_REGISTRY_V2._validate_model_class requires the class' ``is_simple``
+    flag to match the registered model_type (``simple`` vs ``chat``). Our
+    adapter is registered with ``simple_class_path``, so ``is_simple`` must
+    be ``True``.
+    """
+    assert LMMSORTGenAIEvaluator.is_simple is True
+
+
+# -----------------------------------------------------------------------------
+# Visual partitioning
+# -----------------------------------------------------------------------------
+
+
+def test_partition_visuals_separates_images_and_audios_and_skips_nones():
+    from olive.evaluator.lmms_ort import _partition_visuals
+
+    img1 = PIL.Image.new("RGB", (4, 4))
+    img2_dict = {"bytes": _png_bytes(PIL.Image.new("RGB", (2, 2)))}
+    audio_dict = {"array": np.zeros(16, dtype=np.float32), "sampling_rate": 16000}
+
+    images, audios = _partition_visuals([img1, None, img2_dict, audio_dict, None])
+
+    assert len(images) == 2
+    assert all(isinstance(img, PIL.Image.Image) for img in images)
+    assert len(audios) == 1
+    arr, sr = audios[0]
+    assert arr.shape == (16,)
+    assert sr == 16000
+
+
+def test_partition_visuals_handles_none_input():
+    from olive.evaluator.lmms_ort import _partition_visuals
+
+    assert _partition_visuals(None) == ([], [])
+    assert _partition_visuals([]) == ([], [])
+
+
+def test_partition_visuals_downmixes_channels_first_and_last_audio():
+    from olive.evaluator.lmms_ort import _partition_visuals
+
+    channels_first = {"array": np.ones((2, 16), dtype=np.float32), "sampling_rate": 16000}
+    channels_last = {"array": np.ones((16, 2), dtype=np.float32), "sampling_rate": 16000}
+
+    _, audios = _partition_visuals([channels_first, channels_last])
+
+    assert [arr.shape for arr, _ in audios] == [(16,), (16,)]
+
+
+def test_partition_visuals_rejects_video():
+    from olive.evaluator.lmms_ort import _partition_visuals
+
+    with pytest.raises(NotImplementedError, match="video"):
+        _partition_visuals(["sample.mp4"])
+
+
+def test_structured_prompt_preserves_placeholder_order():
+    inst = _make_evaluator_for_prompt_tests()
+    inst._tokenizer.apply_chat_template.return_value = "<rendered>"
+
+    inst._build_prompt_for_request("before <image> middle <audio> after", num_images=1, num_audios=1)
+
+    import json as _json
+
+    messages = _json.loads(inst._tokenizer.apply_chat_template.call_args.args[0])
+    assert messages[-1]["content"] == [
+        {"type": "text", "text": "before "},
+        {"type": "image"},
+        {"type": "text", "text": " middle "},
+        {"type": "audio"},
+        {"type": "text", "text": " after"},
+    ]
+
+
+def _png_bytes(image):
+    import io as _io
+
+    buf = _io.BytesIO()
+    image.save(buf, format="PNG")
+    return buf.getvalue()
+
+
+# -----------------------------------------------------------------------------
+# Runtime: _score_continuation and _run_generation
+#
+# These tests construct a LMMSORTGenAIEvaluator with the onnxruntime_genai
+# module wholesale-mocked, so the generation/scoring flows are exercised
+# end-to-end without needing an actual ORT-GenAI model on disk.
+# -----------------------------------------------------------------------------
+
+
+class _FakeTokenStream:
+    def decode(self, tok):
+        return f"<t{tok}>"
+
+
+class _FakeTokenizer:
+    """Minimal stub for og.Tokenizer used by _score_continuation/_run_generation.
+
+    ``tokens_for`` is a dict mapping the exact input string to the list of
+    token ids encode() should return; this lets each test inject specific
+    prompt + (prompt+continuation) tokenizations to drive the slicing logic.
+    """
+
+    def __init__(self, tokens_for, eos_token_ids=(99,)):
+        self._tokens_for = tokens_for
+        self.eos_token_ids = list(eos_token_ids)
+
+    def encode(self, text):
+        if text not in self._tokens_for:
+            raise KeyError(f"_FakeTokenizer: no canned tokenization for text={text!r}")
+        return list(self._tokens_for[text])
+
+    def create_stream(self):
+        return _FakeTokenStream()
+
+    def apply_chat_template(self, messages, add_generation_prompt=True):
+        media_token = "<|image|>" if '"type": "image"' in messages else ""
+        return f"{media_token}x"
+
+
+class _FakeGenerator:
+    """Records call order so tests can assert the score/generation protocol.
+
+    ``logits_queue`` is a list of numpy arrays consumed in order, one per forward
+    pass: ``generate_next_token`` and ``append_tokens`` each consume one entry
+    into ``_current_logits``, which ``get_logits()`` returns.
+    """
+
+    instances: ClassVar[list] = []
+
+    def __init__(self, model, params):
+        type(self).instances.append(self)
+        self._model = model
+        self._params = params
+        self._logits_queue = list(model._next_logits_queue)
+        self._sampled_queue = list(model._next_sampled_queue)
+        self._current_logits = None
+        self._token_count = 0
+        self._done = False
+        self._last_sampled = -1
+        self.calls = []
+
+    def _consume_forward_pass(self):
+        if not self._logits_queue:
+            raise RuntimeError("_FakeGenerator: forward pass exhausted (logits_queue empty)")
+        self._current_logits = self._logits_queue.pop(0)
+
+    def set_inputs(self, inputs):
+        self.calls.append(("set_inputs", inputs))
+        self._token_count = int(inputs["input_ids"].shape()[-1])
+        # ORT-GenAI SetInputs appends prompt tokens and computes prompt logits.
+        self._consume_forward_pass()
+
+    def generate_next_token(self):
+        if self._current_logits is None:
+            self._consume_forward_pass()
+        sampled = self._sampled_queue.pop(0) if self._sampled_queue else -1
+        self._last_sampled = sampled
+        self._token_count += 1
+        self.calls.append(("generate_next_token", sampled))
+        self._current_logits = None
+
+    def get_logits(self):
+        self.calls.append(("get_logits",))
+        if self._current_logits is None:
+            raise RuntimeError("_FakeGenerator: get_logits called before any forward pass")
+        return np.asarray(self._current_logits, dtype=np.float32)
+
+    def get_next_tokens(self):
+        return np.array([self._last_sampled], dtype=np.int32)
+
+    def append_tokens(self, tok_array):
+        toks = [int(t) for t in np.asarray(tok_array).reshape(-1)]
+        self.calls.append(("append_tokens", toks))
+        self._token_count += len(toks)
+        # Each appended token batch is one forward pass (computes new last-position logits).
+        self._consume_forward_pass()
+
+    def token_count(self):
+        return self._token_count
+
+    def is_done(self):
+        max_length = self._params.search_options.get("max_length")
+        return self._done or (max_length is not None and self._token_count >= max_length)
+
+    def rewind_to(self, n):
+        self.calls.append(("rewind_to", n))
+        self._token_count = n
+
+
+class _FakeGeneratorParams:
+    def __init__(self, model):
+        self._model = model
+        self.search_options = {}
+
+    def set_search_options(self, **kwargs):
+        self.search_options.update(kwargs)
+
+
+class _FakeTensor:
+    def __init__(self, shape):
+        self._shape = shape
+
+    def shape(self):
+        return self._shape
+
+
+class _FakeProcessor:
+    def __init__(self, input_length):
+        self._input_length = input_length
+
+    def __call__(self, prompt, images=None, audios=None):
+        return {
+            "input_ids": _FakeTensor([1, self._input_length]),
+            "_prompt": prompt,
+            "_images": images,
+            "_audios": audios,
+        }
+
+
+class _FakeOgModel:
+    def __init__(self, tokenizer=None, next_logits_queue=None, next_sampled_queue=None, processed_input_length=4):
+        self.tokenizer = tokenizer
+        self._next_logits_queue = list(next_logits_queue or [])
+        self._next_sampled_queue = list(next_sampled_queue or [])
+        self.processed_input_length = processed_input_length
+
+    def create_multimodal_processor(self):
+        return _FakeProcessor(self.processed_input_length)
+
+
+def _make_fake_og(model):
+    """Build a SimpleNamespace mimicking the onnxruntime_genai module surface.
+
+    Covers the API surface used by LMMSORTGenAIEvaluator's __init__ + runtime paths.
+    """
+    return SimpleNamespace(
+        Model=lambda *a, **kw: model,
+        Tokenizer=lambda m: model.tokenizer,
+        Generator=_FakeGenerator,
+        GeneratorParams=_FakeGeneratorParams,
+        Images=SimpleNamespace(open=lambda *paths: ("IMG", list(paths))),
+        Audios=SimpleNamespace(open=lambda *paths: ("AUDIO", list(paths))),
+        Config=lambda *a, **kw: SimpleNamespace(
+            clear_providers=lambda: None,
+            append_provider=lambda *a, **kw: None,
+            set_provider_option=lambda *a, **kw: None,
+        ),
+    )
+
+
+def _build_lmms_ortgenai_evaluator(tmp_path, fake_model):
+    """Construct an LMMSORTGenAIEvaluator wired to a fake onnxruntime_genai.
+
+    Returns ``(evaluator, og_patcher)``. The patcher is a context manager that
+    swaps in the fake ``og`` module; tests should call evaluator methods inside
+    ``with og_patcher: ...`` so runtime paths (``_score_continuation``,
+    ``_run_generation``) also use the fake instead of the real ORT-GenAI.
+    """
+    from olive.evaluator import lmms_ort as lmms_ort_mod
+
+    model_dir = tmp_path / "model"
+    model_dir.mkdir()
+    (model_dir / "genai_config.json").write_text('{"model": {"type": "phi4mm"}}', encoding="utf-8")
+
+    fake_og = _make_fake_og(fake_model)
+    _FakeGenerator.instances = []
+    og_patcher = patch.object(lmms_ort_mod, "og", fake_og)
+    with og_patcher:
+        evaluator = LMMSORTGenAIEvaluator(
+            pretrained=str(model_dir),
+            batch_size=1,
+            max_new_tokens=8,
+            max_length=64,
+            execution_provider="cpu",
+            fail_on_error=True,
+        )
+    # Return a fresh patcher for the test to use during runtime calls.
+    return evaluator, patch.object(lmms_ort_mod, "og", fake_og)
+
+
+def test_score_continuation_uses_joint_tokenization_to_slice_continuation(tmp_path):
+    """Encoding ``continuation`` standalone is wrong for sentencepiece/BPE tokenizers.
+
+    The adapter must encode ``prompt + continuation`` jointly and slice off the
+    prompt-aligned prefix so the scored tokens are the ones the model would
+    actually produce extending the prompt. Verify by giving the tokenizer
+    DIFFERENT tokens for ``continuation`` vs the prompt-suffix of
+    ``prompt + continuation``: only the latter should land in append_tokens.
+    """
+    prompt = "<|user|>What is in the image?<|end|><|assistant|>"
+    continuation = "A"
+
+    fake_tokenizer = _FakeTokenizer(
+        {
+            prompt: [1, 2, 3, 4],
+            prompt + continuation: [1, 2, 3, 4, 17, 18],  # cont tokens = [17, 18]
+            continuation: [99999],  # standalone-encoded - MUST NOT be used
+        }
+    )
+
+    # Two forward passes: prompt prefill in set_inputs + cont[0] append.
+    vocab_size = 50
+    logits_prompt_end = np.full(vocab_size, -10.0, dtype=np.float32)
+    logits_prompt_end[17] = 5.0  # greedy = 17
+    logits_after_17 = np.full(vocab_size, -10.0, dtype=np.float32)
+    logits_after_17[18] = 5.0  # greedy = 18
+    logits_after_18 = np.full(vocab_size, -10.0, dtype=np.float32)
+
+    fake_model = _FakeOgModel(
+        tokenizer=fake_tokenizer,
+        next_logits_queue=[logits_prompt_end, logits_after_17, logits_after_18],
+    )
+
+    evaluator, og_patcher = _build_lmms_ortgenai_evaluator(tmp_path, fake_model)
+
+    with og_patcher:
+        loss, is_greedy = evaluator._score_continuation(prompt, continuation, images=[], audios=[])
+
+    gen = _FakeGenerator.instances[-1]
+    set_inputs_idx = next(i for i, c in enumerate(gen.calls) if c[0] == "set_inputs")
+    append_calls = [c for c in gen.calls if c[0] == "append_tokens"]
+
+    assert set_inputs_idx < next(i for i, c in enumerate(gen.calls) if c[0] == "get_logits")
+    assert not [c for c in gen.calls if c[0] in ("generate_next_token", "rewind_to")]
+    # cont_tokens were correctly sliced from prompt+continuation, NOT taken from
+    # encode(continuation) standalone (which would have been [99999]).
+    assert [tok for _, tok in append_calls] == [[17]]
+    # Both predicted tokens were greedy (== argmax of their position's logits).
+    assert is_greedy is True
+    assert loss > 0.0
+    assert gen._params.search_options["max_length"] == 6
+
+
+def test_score_continuation_uses_prefill_logits_from_set_inputs(tmp_path):
+    """Read prompt logits immediately after SetInputs without sampling a token."""
+    prompt = "<|user|>x<|end|><|assistant|>"
+    continuation = "y"
+    fake_tokenizer = _FakeTokenizer({prompt: [1, 2], prompt + continuation: [1, 2, 5], continuation: [777]})
+    logits = np.zeros(10, dtype=np.float32)
+    logits[5] = 1.0
+    fake_model = _FakeOgModel(
+        tokenizer=fake_tokenizer,
+        next_logits_queue=[logits],
+    )
+
+    evaluator, og_patcher = _build_lmms_ortgenai_evaluator(tmp_path, fake_model)
+    with og_patcher:
+        evaluator._score_continuation(prompt, continuation, images=[], audios=[])
+
+    gen = _FakeGenerator.instances[-1]
+    op_names = [c[0] for c in gen.calls]
+    set_inputs_idx = op_names.index("set_inputs")
+    get_logits_idx = op_names.index("get_logits")
+    assert set_inputs_idx < get_logits_idx
+    assert "generate_next_token" not in op_names
+    assert "rewind_to" not in op_names
+
+
+def test_score_continuation_returns_zero_when_continuation_tokenizes_to_empty_suffix(tmp_path):
+    """Short-circuit cleanly when continuation contributes no tokens.
+
+    Happens when prompt+cont tokenizes to the same length as prompt (e.g. cont
+    is just whitespace absorbed by tokenizer normalization). The adapter must
+    short-circuit instead of feeding an empty cont_tokens list to the loop.
+    """
+    prompt = "<|user|>x<|end|><|assistant|>"
+    continuation = ""
+    fake_tokenizer = _FakeTokenizer({prompt: [1, 2, 3], prompt + continuation: [1, 2, 3]})
+    fake_model = _FakeOgModel(tokenizer=fake_tokenizer)
+
+    evaluator, og_patcher = _build_lmms_ortgenai_evaluator(tmp_path, fake_model)
+    with og_patcher:
+        logprob, is_greedy = evaluator._score_continuation(prompt, continuation, images=[], audios=[])
+
+    assert (logprob, is_greedy) == (0.0, True)
+    # No generator should have been constructed for an empty continuation.
+    assert not _FakeGenerator.instances
+
+
+def test_score_continuation_rejects_target_beyond_context_ceiling(tmp_path):
+    prompt = "<|user|>x<|end|><|assistant|>"
+    continuation = "yz"
+    fake_tokenizer = _FakeTokenizer(
+        {
+            prompt: [1, 2],
+            prompt + continuation: [1, 2, 5, 6],
+        }
+    )
+    fake_model = _FakeOgModel(tokenizer=fake_tokenizer, processed_input_length=63)
+    evaluator, og_patcher = _build_lmms_ortgenai_evaluator(tmp_path, fake_model)
+
+    with og_patcher, pytest.raises(RuntimeError, match="cannot be scored within max_length"):
+        evaluator._score_continuation(prompt, continuation, images=[], audios=[])
+
+    assert not _FakeGenerator.instances
+
+
+def test_loglikelihood_accepts_string_continuation_from_multiple_choice():
+    evaluator = LMMSORTGenAIEvaluator.__new__(LMMSORTGenAIEvaluator)
+    evaluator.cache_hook = SimpleNamespace(add_partial=lambda *args, **kwargs: None)
+    evaluator._get_doc_and_visuals = lambda *args, **kwargs: ({"answer": "A"}, [])
+    evaluator._build_prompt_for_request = lambda *args, **kwargs: "<prompt>"
+    evaluator._score_continuation = MagicMock(return_value=(0.25, True))
+    request = SimpleNamespace(args=("context", " A", None, 0, "arc_easy", "test"))
+
+    result = evaluator.loglikelihood([request], disable_tqdm=True)
+
+    assert result == [(0.25, True)]
+    evaluator._score_continuation.assert_called_once_with("<prompt>", " A", [], [])
+
+
+def test_run_generation_stops_on_eos_token(tmp_path):
+    """Verify ``_run_generation`` stops at EOS tokens.
+
+    ``_run_generation`` must respect ``_eos_token_ids`` and stop emitting text
+    once the model samples an EOS token, even before max_new_tokens is reached.
+    """
+    fake_tokenizer = _FakeTokenizer({}, eos_token_ids=[99])
+    logits = np.zeros(100, dtype=np.float32)
+    fake_model = _FakeOgModel(
+        tokenizer=fake_tokenizer,
+        next_logits_queue=[logits] * 5,
+        next_sampled_queue=[7, 8, 99, 10, 11],  # third sample is EOS
+    )
+
+    evaluator, og_patcher = _build_lmms_ortgenai_evaluator(tmp_path, fake_model)
+    with og_patcher:
+        out = evaluator._run_generation("p", images=[], audios=[], max_new_tokens=5)
+
+    # Stream produces "<t{tok}>" per non-EOS token; EOS stops generation.
+    assert out == "<t7><t8>"
+    gen = _FakeGenerator.instances[-1]
+    op_names = [c[0] for c in gen.calls]
+    # Exactly 3 generate_next_token calls happened (7, 8, then EOS stops loop).
+    assert op_names.count("generate_next_token") == 3
+    assert gen._params.search_options["max_length"] == 9
+
+
+def test_run_generation_clamps_budget_to_remaining_context(tmp_path):
+    fake_tokenizer = _FakeTokenizer({}, eos_token_ids=[])
+    logits = np.zeros(10, dtype=np.float32)
+    fake_model = _FakeOgModel(
+        tokenizer=fake_tokenizer,
+        next_logits_queue=[logits] * 2,
+        next_sampled_queue=[7, 8, 9],
+        processed_input_length=62,
+    )
+    evaluator, og_patcher = _build_lmms_ortgenai_evaluator(tmp_path, fake_model)
+
+    with og_patcher, patch("olive.evaluator.lmms_ort.logger.warning") as warning_mock:
+        output = evaluator._run_generation("p", images=[], audios=[], max_new_tokens=5)
+
+    assert output == "<t7><t8>"
+    generator = _FakeGenerator.instances[-1]
+    assert generator._params.search_options["max_length"] == 64
+    warning_mock.assert_called_once_with(
+        "Reducing max_new_tokens from %d to %d because the processed prompt uses %d of max_length=%d.",
+        5,
+        2,
+        62,
+        64,
+    )
+
+
+def test_run_generation_rejects_prompt_at_context_ceiling(tmp_path):
+    fake_model = _FakeOgModel(
+        tokenizer=_FakeTokenizer({}, eos_token_ids=[]),
+        processed_input_length=64,
+    )
+    evaluator, og_patcher = _build_lmms_ortgenai_evaluator(tmp_path, fake_model)
+
+    with og_patcher, pytest.raises(RuntimeError, match="processed prompt exceeds max_length"):
+        evaluator._run_generation("p", images=[], audios=[], max_new_tokens=1)
+
+    assert not _FakeGenerator.instances
+
+
+def test_run_generation_stops_on_explicit_stop_string(tmp_path):
+    """``stop_strings`` should truncate output as soon as a stop sequence appears."""
+    fake_tokenizer = _FakeTokenizer({}, eos_token_ids=[])
+    logits = np.zeros(10, dtype=np.float32)
+    fake_model = _FakeOgModel(
+        tokenizer=fake_tokenizer,
+        next_logits_queue=[logits] * 4,
+        next_sampled_queue=[1, 2, 3, 4],
+    )
+
+    evaluator, og_patcher = _build_lmms_ortgenai_evaluator(tmp_path, fake_model)
+    # _FakeTokenStream emits "<t1><t2><t3>..."; "<t2>" appears after the 2nd token.
+    with og_patcher:
+        out = evaluator._run_generation("p", images=[], audios=[], max_new_tokens=10, stop_strings=["<t2>"])
+
+    # Output is truncated at (but not including) the stop string.
+    assert out == "<t1>"
diff --git a/test/passes/onnx/test_composite_to_onnx_package.py b/test/passes/onnx/test_composite_to_onnx_package.py
new file mode 100644
index 000000000..cde6073dd
--- /dev/null
+++ b/test/passes/onnx/test_composite_to_onnx_package.py
@@ -0,0 +1,224 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+# --------------------------------------------------------------------------
+"""Tests for the CompositeToOnnxPackage pass.
+
+The pass repackages a nested multi-component CompositeModel ORT-GenAI package as a
+single ONNXModelHandler pointing at the entry-point component, preserving the nested
+directory layout (no flattening / external-data rewriting).
+"""
+
+import json
+from pathlib import Path
+
+import numpy as np
+import onnx
+import pytest
+from onnx import TensorProto, helper, numpy_helper
+
+from olive.model import ONNXModelHandler
+from olive.model.handler.composite import CompositeModelHandler
+from olive.passes.olive_pass import create_pass_from_dict
+from olive.passes.onnx.composite_to_onnx_package import CompositeToOnnxPackage
+
+
+def _write_tiny_onnx_with_external_data(onnx_path: Path, data_filename: str = "model.onnx.data") -> None:
+    """Write a minimal valid ONNX model whose single initializer lives in an external data sidecar."""
+    data = np.arange(1024, dtype=np.float32)
+    init_tensor = numpy_helper.from_array(data, name="weight")
+    output = helper.make_tensor_value_info("y", TensorProto.FLOAT, [1024])
+    node = helper.make_node("Identity", inputs=["weight"], outputs=["y"])
+    graph = helper.make_graph([node], "g", inputs=[], outputs=[output], initializer=[init_tensor])
+    model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 17)])
+
+    onnx_path.parent.mkdir(parents=True, exist_ok=True)
+    onnx.save_model(
+        model,
+        str(onnx_path),
+        save_as_external_data=True,
+        all_tensors_to_one_file=True,
+        location=data_filename,
+    )
+
+
+def _make_nested_genai_package(root: Path, components: dict[str, str]) -> Path:
+    """Build a fake nested ORT-GenAI package at ``root``.
+
+    ``components`` maps genai_config component keys (e.g. ``decoder``) to the
+    relative ONNX filename under ``root`` (e.g. ``decoder/model.onnx``).
+    """
+    root.mkdir(parents=True, exist_ok=True)
+
+    model_section: dict[str, dict[str, str]] = {}
+    for key, rel_path in components.items():
+        _write_tiny_onnx_with_external_data(root / rel_path)
+        model_section[key] = {"filename": rel_path}
+
+    # Shared root-level sidecars.
+    (root / "tokenizer.json").write_text("{}", encoding="utf-8")
+    (root / "chat_template.jinja").write_text("template", encoding="utf-8")
+
+    (root / "genai_config.json").write_text(
+        json.dumps({"model": {"type": "gemma4", **model_section}}, indent=2),
+        encoding="utf-8",
+    )
+    return root
+
+
+def _make_composite_handler(root: Path, components: dict[str, str]) -> CompositeModelHandler:
+    component_handlers = [ONNXModelHandler(model_path=str(root / rel_path)) for rel_path in components.values()]
+    return CompositeModelHandler(
+        model_components=component_handlers,
+        model_component_names=list(components.keys()),
+        model_path=str(root),
+    )
+
+
+def _find_genai_config_upward(onnx_file: Path, levels: int = 3) -> Path | None:
+    """Mirror the evaluator's upward search for genai_config.json from an ONNX file."""
+    candidate = onnx_file.parent
+    for _ in range(levels):
+        if (candidate / "genai_config.json").is_file():
+            return candidate / "genai_config.json"
+        if candidate.parent == candidate:
+            break
+        candidate = candidate.parent
+    return None
+
+
+class TestCompositeToOnnxPackage:
+    def test_preserves_nested_component_layout(self, tmp_path):
+        src_root = _make_nested_genai_package(
+            tmp_path / "src",
+            {
+                "decoder": "decoder/model.onnx",
+                "vision": "vision_encoder/model.onnx",
+                "audio": "audio_encoder/model.onnx",
+                "embedding": "embedding/model.onnx",
+            },
+        )
+        composite = _make_composite_handler(
+            src_root,
+            {
+                "decoder": "decoder/model.onnx",
+                "vision_encoder": "vision_encoder/model.onnx",
+                "audio_encoder": "audio_encoder/model.onnx",
+                "embedding": "embedding/model.onnx",
+            },
+        )
+
+        p = create_pass_from_dict(CompositeToOnnxPackage, {}, disable_search=True)
+        out = p.run(composite, str(tmp_path / "out"))
+
+        pkg_root = Path(out.model_path).parents[1]
+        # Nested layout preserved (components stay in their subdirectories).
+        assert (pkg_root / "decoder" / "model.onnx").is_file()
+        assert (pkg_root / "vision_encoder" / "model.onnx").is_file()
+        assert (pkg_root / "audio_encoder" / "model.onnx").is_file()
+        assert (pkg_root / "embedding" / "model.onnx").is_file()
+        # Shared sidecars and config copied to the package root.
+        assert (pkg_root / "genai_config.json").is_file()
+        assert (pkg_root / "tokenizer.json").is_file()
+        assert (pkg_root / "chat_template.jinja").is_file()
+
+    def test_returns_onnx_handler_pointing_at_nested_entry(self, tmp_path):
+        src_root = _make_nested_genai_package(
+            tmp_path / "src",
+            {"decoder": "decoder/model.onnx", "vision": "vision_encoder/model.onnx"},
+        )
+        composite = _make_composite_handler(
+            src_root, {"decoder": "decoder/model.onnx", "vision_encoder": "vision_encoder/model.onnx"}
+        )
+
+        p = create_pass_from_dict(CompositeToOnnxPackage, {}, disable_search=True)
+        out = p.run(composite, str(tmp_path / "out"))
+
+        assert isinstance(out, ONNXModelHandler)
+        entry = Path(out.model_path)
+        # Handler points at the nested decoder entry, which really exists.
+        assert entry.is_file()
+        assert entry.name == "model.onnx"
+        assert entry.parent.name == "decoder"
+        # genai_config.json is discoverable by searching upward from the entry ONNX file.
+        assert _find_genai_config_upward(entry) is not None
+
+    def test_does_not_rewrite_genai_config_filenames(self, tmp_path):
+        src_root = _make_nested_genai_package(
+            tmp_path / "src",
+            {"decoder": "decoder/model.onnx", "vision": "vision_encoder/model.onnx"},
+        )
+        composite = _make_composite_handler(
+            src_root, {"decoder": "decoder/model.onnx", "vision_encoder": "vision_encoder/model.onnx"}
+        )
+
+        p = create_pass_from_dict(CompositeToOnnxPackage, {}, disable_search=True)
+        out = p.run(composite, str(tmp_path / "out"))
+
+        pkg_root = Path(out.model_path).parents[1]
+        genai_config = json.loads((pkg_root / "genai_config.json").read_text(encoding="utf-8"))
+        # Nested paths are preserved verbatim (no flattening rewrite).
+        assert genai_config["model"]["decoder"]["filename"] == "decoder/model.onnx"
+        assert genai_config["model"]["vision"]["filename"] == "vision_encoder/model.onnx"
+
+    def test_genai_config_is_independent_from_input_package(self, tmp_path):
+        src_root = _make_nested_genai_package(tmp_path / "src", {"decoder": "decoder/model.onnx"})
+        composite = _make_composite_handler(src_root, {"decoder": "decoder/model.onnx"})
+
+        p = create_pass_from_dict(CompositeToOnnxPackage, {}, disable_search=True)
+        out = p.run(composite, str(tmp_path / "out"))
+
+        pkg_root = Path(out.model_path).parents[1]
+        src_config = src_root / "genai_config.json"
+        output_config = pkg_root / "genai_config.json"
+        assert src_config.stat().st_ino != output_config.stat().st_ino
+        output_config.write_text('{"model": {"type": "patched"}}', encoding="utf-8")
+        assert json.loads(src_config.read_text(encoding="utf-8"))["model"]["type"] == "gemma4"
+
+    def test_honors_explicit_entry_point_component(self, tmp_path):
+        src_root = _make_nested_genai_package(
+            tmp_path / "src",
+            {"decoder": "decoder/model.onnx", "embedding": "embedding/model.onnx"},
+        )
+        composite = _make_composite_handler(
+            src_root, {"decoder": "decoder/model.onnx", "embedding": "embedding/model.onnx"}
+        )
+
+        p = create_pass_from_dict(CompositeToOnnxPackage, {"entry_point_component": "embedding"}, disable_search=True)
+        out = p.run(composite, str(tmp_path / "out"))
+
+        assert Path(out.model_path).parent.name == "embedding"
+
+    def test_uses_fallback_entry_point_when_requested_one_missing(self, tmp_path):
+        src_root = _make_nested_genai_package(
+            tmp_path / "src",
+            {"vision": "vision_encoder/model.onnx", "embedding": "embedding/model.onnx"},
+        )
+        composite = _make_composite_handler(
+            src_root, {"vision_encoder": "vision_encoder/model.onnx", "embedding": "embedding/model.onnx"}
+        )
+
+        # 'decoder' (the default entry) is absent -> falls back to a present component.
+        p = create_pass_from_dict(CompositeToOnnxPackage, {}, disable_search=True)
+        out = p.run(composite, str(tmp_path / "out"))
+
+        assert Path(out.model_path).parent.name in {"vision_encoder", "embedding"}
+
+    def test_raises_on_non_composite_input(self, tmp_path):
+        _write_tiny_onnx_with_external_data(tmp_path / "model.onnx")
+        onnx_model = ONNXModelHandler(model_path=str(tmp_path / "model.onnx"))
+
+        p = create_pass_from_dict(CompositeToOnnxPackage, {}, disable_search=True)
+        with pytest.raises(ValueError, match="expects a CompositeModelHandler"):
+            p.run(onnx_model, str(tmp_path / "out"))
+
+    def test_raises_when_entry_component_file_missing(self, tmp_path):
+        src_root = _make_nested_genai_package(tmp_path / "src", {"decoder": "decoder/model.onnx"})
+        composite = _make_composite_handler(src_root, {"decoder": "decoder/model.onnx"})
+        # Remove the referenced component file so the config points at a missing file
+        # (after building the handler, whose components assert file existence).
+        (src_root / "decoder" / "model.onnx").unlink()
+
+        p = create_pass_from_dict(CompositeToOnnxPackage, {}, disable_search=True)
+        with pytest.raises(ValueError, match="not found"):
+            p.run(composite, str(tmp_path / "out"))