From eb0eb9f7ee86db4691b2272bdaa25225aa5a2eb3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= <xadupre@microsoft.com>
Date: Thu, 9 Jul 2026 17:49:36 +0200
Subject: [PATCH 01/21] fix wrong architecture in --test

---
 olive/common/hf/utils.py               | 41 +++++++++++++++++++--
 olive/passes/onnx/discrepancy_check.py |  8 ++++-
 test/common/test_hf.py                 | 49 ++++++++++++++++++++++++++
 3 files changed, 95 insertions(+), 3 deletions(-)

diff --git a/olive/common/hf/utils.py b/olive/common/hf/utils.py
index 69c8b3b7af..ca44c9f5cc 100644
--- a/olive/common/hf/utils.py
+++ b/olive/common/hf/utils.py
@@ -100,6 +100,30 @@ def _apply_test_model_config(
     return model_config
 
 
+def get_model_class_from_config(model_config: "PretrainedConfig") -> Optional[type]:
+    """Resolve the concrete transformers model class declared in ``config.architectures``.
+
+    This is the same signal ONNX Runtime GenAI's model builder relies on in the non-test path, so
+    reusing it keeps the test/reference model architecture consistent with the optimized model
+    (e.g. an encoder-decoder model such as Whisper resolves to ``WhisperForConditionalGeneration``
+    rather than the decoder-only ``WhisperForCausalLM`` implied by the default text-generation task).
+
+    Returns the first architecture class that exists in the top-level ``transformers`` namespace and
+    can be instantiated from a config, or ``None`` when the config declares no architecture or the
+    class is not available (e.g. custom remote-code architectures). Callers should fall back to
+    task-based class selection when ``None`` is returned.
+    """
+    import transformers
+
+    for arch in getattr(model_config, "architectures", None) or []:
+        model_class = getattr(transformers, arch, None)
+        # Concrete architecture classes expose the private ``_from_config`` while the ``AutoModel*``
+        # helpers expose the public ``from_config``; accept either so the class can be built from config.
+        if model_class is not None and (hasattr(model_class, "from_config") or hasattr(model_class, "_from_config")):
+            return model_class
+    return None
+
+
 def _load_test_model(
     model_class: type,
     model_config: "PretrainedConfig",
@@ -112,8 +136,12 @@ def _load_test_model(
     through to ``from_config`` so the random test model uses the requested attention implementation
     rather than relying on the transformers default (which can be ``"eager"`` on some versions).
     This keeps the generated test model consistent with the base/reference model.
+
+    ``model_class`` may be an ``AutoModel*`` helper (public ``from_config``) or a concrete architecture
+    class such as ``WhisperForConditionalGeneration`` (private ``_from_config``); both are supported.
     """
-    from_config_signature = inspect.signature(model_class.from_config)
+    from_config = getattr(model_class, "from_config", None) or model_class._from_config
+    from_config_signature = inspect.signature(from_config)
     accepts_var_keyword = any(
         parameter.kind == inspect.Parameter.VAR_KEYWORD for parameter in from_config_signature.parameters.values()
     )
@@ -126,7 +154,7 @@ def _load_test_model(
         accepts_var_keyword or "attn_implementation" in from_config_signature.parameters
     ) and attn_implementation is not None:
         from_config_kwargs["attn_implementation"] = attn_implementation
-    model = model_class.from_config(model_config, **from_config_kwargs)
+    model = from_config(model_config, **from_config_kwargs)
     logger.info("Generating test model class %s", type(model))
     return model
 
@@ -230,6 +258,15 @@ def load_model_from_task(
             AUTO_QUANTIZER_MAPPING["olive"] = OliveHfQuantizer
 
     class_tuple = targeted_task["pt"] or (AutoModel,)
+    if test_model_config:
+        # The reference test model must match the *real* model's architecture. Deriving the class
+        # from the task (e.g. the default "text-generation" -> AutoModelForCausalLM) would coerce
+        # encoder-decoder / seq2seq models such as Whisper into a decoder-only *ForCausalLM head.
+        # The model config already declares the concrete architecture (as ONNX Runtime GenAI's model
+        # builder relies on in the non-test path), so prefer that when it is resolvable.
+        arch_class = get_model_class_from_config(model_config)
+        if arch_class is not None:
+            class_tuple = (arch_class,)
     model = None
     for i, model_class in enumerate(class_tuple):
         try:
diff --git a/olive/passes/onnx/discrepancy_check.py b/olive/passes/onnx/discrepancy_check.py
index ab282d6178..17c53cef3a 100644
--- a/olive/passes/onnx/discrepancy_check.py
+++ b/olive/passes/onnx/discrepancy_check.py
@@ -434,6 +434,8 @@ def _load_reference_model(self, model: ONNXModelHandler, config: type[BasePassCo
         # Load reference PyTorch model
         from transformers import AutoConfig, AutoModelForCausalLM
 
+        from olive.common.hf.utils import get_model_class_from_config
+
         # Resolve the reference model path.  Use the configured path if it exists as a local
         # directory; otherwise fall back to a ``reference_hf_model`` directory saved alongside the
         # ONNX output.  The reference model is normally kept at ``<output_path>/reference_hf_model``
@@ -465,10 +467,14 @@ def _load_reference_model(self, model: ONNXModelHandler, config: type[BasePassCo
                 f"Got architectures={architectures}"
             )
 
+        # Load the reference model using the concrete class declared in its config.architectures
+        # (shared with the test-model save path) rather than assuming AutoModelForCausalLM, falling
+        # back to AutoModelForCausalLM only when the architecture cannot be resolved.
         # The attention implementation is baked into the reference model's config.json
         # (as ``_attn_implementation``) by the SaveTestModelConfig pass, so it is picked up
         # automatically here without needing to pass ``attn_implementation`` explicitly.
-        ref_model = AutoModelForCausalLM.from_pretrained(ref_path, config=ref_cfg)
+        model_class = get_model_class_from_config(ref_cfg) or AutoModelForCausalLM
+        ref_model = model_class.from_pretrained(ref_path, config=ref_cfg)
         ref_model.eval()
         logger.info(
             "Loaded reference model from %s with attn_implementation=%s",
diff --git a/test/common/test_hf.py b/test/common/test_hf.py
index 4204606e0b..dad6f56d54 100644
--- a/test/common/test_hf.py
+++ b/test/common/test_hf.py
@@ -54,6 +54,55 @@ def test_load_model_from_task_test_model_config(model_config, hidden_layers_attr
     assert getattr(mock_model_class.from_config.call_args.args[0], hidden_layers_attr) == 2
 
 
+def test_get_model_class_from_config_resolves_declared_architecture():
+    from transformers import WhisperConfig, WhisperForConditionalGeneration
+
+    from olive.common.hf.utils import get_model_class_from_config
+
+    config = WhisperConfig(architectures=["WhisperForConditionalGeneration"])  # pylint: disable=unexpected-keyword-arg
+    assert get_model_class_from_config(config) is WhisperForConditionalGeneration
+
+
+def test_get_model_class_from_config_returns_none_when_unresolvable():
+    from transformers import BertConfig
+
+    from olive.common.hf.utils import get_model_class_from_config
+
+    # No architectures declared.
+    assert get_model_class_from_config(BertConfig()) is None
+    # Architecture name not present in the transformers namespace (e.g. custom remote-code model).
+    assert (
+        get_model_class_from_config(BertConfig(architectures=["TotallyNotARealArchitecture"]))  # pylint: disable=unexpected-keyword-arg
+        is None
+    )
+
+
+def test_load_model_from_task_test_model_config_prefers_config_architecture():
+    """In --test mode the reference model class comes from config.architectures, not the task.
+
+    A seq2seq model such as Whisper must not be coerced into a decoder-only *ForCausalLM head just
+    because the default task is text-generation.
+    """
+    from transformers import WhisperConfig, WhisperForConditionalGeneration
+
+    model_config = WhisperConfig(architectures=["WhisperForConditionalGeneration"])  # pylint: disable=unexpected-keyword-arg
+    created_model = MagicMock(spec=torch.nn.Module)
+
+    with (
+        patch("transformers.pipelines.check_task") as mock_check_task,
+        patch("olive.common.hf.utils.from_pretrained", return_value=model_config),
+        patch("olive.common.hf.utils._load_test_model", return_value=created_model) as mock_load_test_model,
+    ):
+        # The task resolves to a causal-LM class, which must be ignored in favor of the config arch.
+        mock_check_task.return_value = ("text-generation", {"pt": (MagicMock(),)}, None)
+
+        model = load_model_from_task("text-generation-with-past", "dummy-model", test_model_config={"hidden_layers": 2})
+
+    assert model is created_model
+    mock_load_test_model.assert_called_once()
+    assert mock_load_test_model.call_args.args[0] is WhisperForConditionalGeneration
+
+
 def test_load_model_from_task_test_model_config_saves_tokenizer(tmp_path):
     """The reference tokenizer should be saved into the test model directory."""
     model_config = BertConfig(num_hidden_layers=12)  # pylint: disable=unexpected-keyword-arg

From 3d43c6009cf1dd24e1e0d9491c7b8e7174203f34 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= <xadupre@microsoft.com>
Date: Thu, 9 Jul 2026 18:26:06 +0200
Subject: [PATCH 02/21] add support for Whisper

---
 olive/passes/onnx/discrepancy_check.py     | 395 ++++++++++++++++++---
 test/passes/onnx/test_discrepancy_check.py | 162 +++++++++
 2 files changed, 516 insertions(+), 41 deletions(-)

diff --git a/olive/passes/onnx/discrepancy_check.py b/olive/passes/onnx/discrepancy_check.py
index 17c53cef3a..d4da88f3f4 100644
--- a/olive/passes/onnx/discrepancy_check.py
+++ b/olive/passes/onnx/discrepancy_check.py
@@ -228,8 +228,19 @@ class OnnxDiscrepancyCheck(Pass):
       and ONNX Runtime GenAI generation (when enabled)
 
     The pass status is marked as failed if any configured threshold is exceeded.
+
+    For encoder-decoder speech models (e.g. Whisper) the ONNX model is a composite
+    (separate encoder/decoder graphs) rather than a single ``input_ids -> logits`` graph,
+    so the single-graph logits/MAE/speedup comparison does not apply. In that case the
+    pass runs only the generation comparison (transformers ``generate(input_features)``
+    vs ONNX Runtime GenAI audio transcription).
     """
 
+    # Speech encoder-decoder models (e.g. Whisper) are exported as composite models
+    # (encoder.onnx + decoder.onnx). Accept the composite as a whole so the pass can run
+    # the audio-based generation comparison instead of being applied per component.
+    _accepts_composite_model: bool = True
+
     @classmethod
     def _default_config(cls, accelerator_spec: AcceleratorSpec) -> dict[str, PassConfigParam]:
         return {
@@ -311,6 +322,16 @@ def _default_config(cls, accelerator_spec: AcceleratorSpec) -> dict[str, PassCon
                 default_value="The capital of France is",
                 description="Text prompt used for generation comparison between transformers and GenAI.",
             ),
+            "speech_audio_path": PassConfigParam(
+                type_=Optional[str],
+                default_value=None,
+                description=(
+                    "Path to an audio file (e.g. a 16 kHz mono ``.wav``) used as the input for the "
+                    "generation comparison of encoder-decoder speech models such as Whisper. When "
+                    "not set, a short synthetic audio signal is generated in-code. Ignored for "
+                    "text/causal-LM models, which use ``generate_prompt`` instead."
+                ),
+            ),
             "generate_max_new_tokens": PassConfigParam(
                 type_=int,
                 default_value=32,
@@ -369,14 +390,29 @@ def _default_config(cls, accelerator_spec: AcceleratorSpec) -> dict[str, PassCon
     def _run_for_config(
         self, model: ONNXModelHandler, config: type[BasePassConfig], output_model_path: str
     ) -> ONNXModelHandler:
-        dataloader, io_config = self._prepare_dataloader(model)
         ref_model, ref_path = self._load_reference_model(model, config)
+        report_dir = self._resolve_report_dir(config, output_model_path)
+
+        # Encoder-decoder speech models (e.g. Whisper) are exported as composite encoder/decoder
+        # graphs, so the single-graph logits/MAE/speedup comparison does not apply. Run only the
+        # audio-based generation comparison for them.
+        if self._is_speech_seq2seq(ref_model):
+            logger.info(
+                "OnnxDiscrepancyCheck detected an encoder-decoder speech model (%s); "
+                "running the audio-based generation comparison only.",
+                type(ref_model).__name__,
+            )
+            _, torch_device = self._resolve_devices()
+            ref_model = self._cast_reference_model(ref_model, None, torch_device)
+            results = self._run_speech_generation_comparison(model, config, ref_model, ref_path)
+            self._save_results(model, results, report_dir)
+            return model
+
+        dataloader, io_config = self._prepare_dataloader(model)
 
         device, execution_provider, torch_device, weight_dtype = self._resolve_execution_device(model)
         ref_model = self._cast_reference_model(ref_model, weight_dtype, torch_device)
 
-        report_dir = self._resolve_report_dir(config, output_model_path)
-
         session = model.prepare_session(
             device=device,
             execution_providers=[execution_provider] if execution_provider else None,
@@ -399,6 +435,15 @@ def _run_for_config(
         self._save_results(model, results, report_dir)
         return model
 
+    @staticmethod
+    def _is_speech_seq2seq(ref_model) -> bool:
+        # Speech encoder-decoder models (e.g. Whisper) take audio ``input_features`` rather than
+        # text ``input_ids`` as their main input; this is the robust signal to route them to the
+        # audio-based generation comparison.
+        config = getattr(ref_model, "config", None)
+        is_encoder_decoder = bool(getattr(config, "is_encoder_decoder", False))
+        return is_encoder_decoder and getattr(ref_model, "main_input_name", "input_ids") == "input_features"
+
     def _prepare_dataloader(self, model: ONNXModelHandler):
         from olive.common.config_utils import validate_config
         from olive.data.template import dummy_data_config_template
@@ -461,9 +506,14 @@ def _load_reference_model(self, model: ONNXModelHandler, config: type[BasePassCo
 
         ref_cfg = AutoConfig.from_pretrained(ref_path)
         architectures = getattr(ref_cfg, "architectures", None) or []
-        if not any("ForCausalLM" in arch for arch in architectures):
+        is_causal_lm = any("ForCausalLM" in arch for arch in architectures)
+        is_conditional_generation = getattr(ref_cfg, "is_encoder_decoder", False) or any(
+            "ForConditionalGeneration" in arch for arch in architectures
+        )
+        if not (is_causal_lm or is_conditional_generation):
             raise ValueError(
-                "OnnxDiscrepancyCheck currently supports only HuggingFace causal language models (ForCausalLM). "
+                "OnnxDiscrepancyCheck supports HuggingFace causal language models (ForCausalLM) and "
+                "encoder-decoder conditional-generation models (ForConditionalGeneration, e.g. Whisper). "
                 f"Got architectures={architectures}"
             )
 
@@ -483,19 +533,11 @@ def _load_reference_model(self, model: ONNXModelHandler, config: type[BasePassCo
         )
         return ref_model, ref_path
 
-    def _resolve_execution_device(self, model: ONNXModelHandler):
+    def _resolve_devices(self):
+        """Resolve the accelerator Device and matching torch device (independent of the ONNX model)."""
         import torch
 
-        # Determine the floating-point dtype used by the ONNX model weights and
-        # cast the reference PyTorch model to match, so the comparison uses the
-        # same numeric precision for the weights on both sides.
-        weight_dtype = None
-        onnx_weight_dtype = _infer_onnx_weight_dtype(model.load_model())
-        if onnx_weight_dtype is not None:
-            weight_dtype = _onnx_dtype_to_torch(onnx_weight_dtype)
-        # Prepare ONNX session on the target device (fallback to CPU)
         device = self.accelerator_spec.accelerator_type if self.accelerator_spec else None
-        execution_provider = self.accelerator_spec.execution_provider if self.accelerator_spec else None
         if device is None:
             device = Device.CPU
         elif not isinstance(device, Device):
@@ -505,10 +547,22 @@ def _resolve_execution_device(self, model: ONNXModelHandler):
                 logger.warning("Unknown accelerator_type=%s; falling back to CPU.", device)
                 device = Device.CPU
 
-        # Determine the torch device matching the accelerator spec
         torch_device = torch.device("cpu")
         if device == Device.GPU and torch.cuda.is_available():
             torch_device = torch.device("cuda")
+        return device, torch_device
+
+    def _resolve_execution_device(self, model: ONNXModelHandler):
+        # Determine the floating-point dtype used by the ONNX model weights and
+        # cast the reference PyTorch model to match, so the comparison uses the
+        # same numeric precision for the weights on both sides.
+        weight_dtype = None
+        onnx_weight_dtype = _infer_onnx_weight_dtype(model.load_model())
+        if onnx_weight_dtype is not None:
+            weight_dtype = _onnx_dtype_to_torch(onnx_weight_dtype)
+        # Prepare ONNX session on the target device (fallback to CPU)
+        device, torch_device = self._resolve_devices()
+        execution_provider = self.accelerator_spec.execution_provider if self.accelerator_spec else None
         return device, execution_provider, torch_device, weight_dtype
 
     def _cast_reference_model(self, ref_model, weight_dtype, torch_device):
@@ -668,16 +722,15 @@ def _check_error_thresholds(self, config: type[BasePassConfig], results, effecti
         else:
             results["status"] = "passed"
 
-    def _run_generation_comparison(
-        self, model: ONNXModelHandler, config, ref_model, ref_path, generation_metrics, results
-    ):
-        # Generation token sequence comparison (transformers vs ONNX Runtime GenAI).
-        # Runs when an explicit genai_model_path is configured or when any generation-based
-        # test metric (first_token_20 / tft / tf5t) is requested.  In the latter case the
-        # optimized ONNX model directory is used as the GenAI model when it exposes a
-        # genai_config.json (as produced by the ModelBuilder pass).
+    def _resolve_genai_model_path(self, model, config, generation_metrics):
+        """Resolve the ONNX Runtime GenAI model directory.
+
+        Uses an explicitly configured ``genai_model_path`` when set; otherwise falls back to the
+        optimized model directory when it exposes a ``genai_config.json`` (as produced by the
+        ModelBuilder pass). Returns ``None`` when no GenAI model can be located.
+        """
         genai_model_path = config.genai_model_path
-        if genai_model_path is None and generation_metrics:
+        if genai_model_path is None:
             model_dir = Path(model.model_path)
             model_dir = model_dir if model_dir.is_dir() else model_dir.parent
             if (model_dir / "genai_config.json").is_file():
@@ -686,30 +739,18 @@ def _run_generation_comparison(
                     "Using optimized ONNX model directory %s as the GenAI model for generation metrics.",
                     genai_model_path,
                 )
-            else:
+            elif generation_metrics:
                 logger.warning(
                     "Generation metrics %s requested but no genai_config.json was found in %s; skipping them.",
                     sorted(generation_metrics),
                     model_dir,
                 )
+        return genai_model_path
 
-        if not genai_model_path:
-            return
-
-        # first_token_20 generates 20 tokens; tf5t measures the time to the first 5 tokens.
-        gen_max_new_tokens = 20 if "first_token_20" in generation_metrics else config.generate_max_new_tokens
-        gen_first_n = 5 if "tf5t" in generation_metrics else config.first_n_tokens_timed
-        gen_results = self.compare_generation(
-            config,
-            ref_model,
-            ref_model_path=ref_path,
-            genai_model_path=genai_model_path,
-            max_new_tokens=gen_max_new_tokens,
-            first_n=gen_first_n,
-        )
+    def _surface_generation_metrics(self, config, generation_metrics, gen_results, results):
+        """Merge generation-comparison results into ``results`` and record threshold failures."""
         longest_common = gen_results["longest_common_token_sequence"]
         results.update(gen_results)
-        results["genai_model_path"] = genai_model_path
 
         # Surface the explicitly requested named metrics for easy inspection.
         if "first_token_20" in generation_metrics:
@@ -763,6 +804,70 @@ def _run_generation_comparison(
             results.setdefault("failures", []).append(gen_failure)
             logger.error("ONNX model discrepancy check FAILED: %s", gen_failure)
 
+    def _run_generation_comparison(
+        self, model: ONNXModelHandler, config, ref_model, ref_path, generation_metrics, results
+    ):
+        # Generation token sequence comparison (transformers vs ONNX Runtime GenAI).
+        # Runs when an explicit genai_model_path is configured or when any generation-based
+        # test metric (first_token_20 / tft / tf5t) is requested.  In the latter case the
+        # optimized ONNX model directory is used as the GenAI model when it exposes a
+        # genai_config.json (as produced by the ModelBuilder pass).
+        if config.genai_model_path is None and not generation_metrics:
+            return
+        genai_model_path = self._resolve_genai_model_path(model, config, generation_metrics)
+        if not genai_model_path:
+            return
+
+        # first_token_20 generates 20 tokens; tf5t measures the time to the first 5 tokens.
+        gen_max_new_tokens = 20 if "first_token_20" in generation_metrics else config.generate_max_new_tokens
+        gen_first_n = 5 if "tf5t" in generation_metrics else config.first_n_tokens_timed
+        gen_results = self.compare_generation(
+            config,
+            ref_model,
+            ref_model_path=ref_path,
+            genai_model_path=genai_model_path,
+            max_new_tokens=gen_max_new_tokens,
+            first_n=gen_first_n,
+        )
+        results["genai_model_path"] = genai_model_path
+        self._surface_generation_metrics(config, generation_metrics, gen_results, results)
+
+    def _run_speech_generation_comparison(self, model, config, ref_model, ref_path):
+        """Run the audio-based generation comparison for encoder-decoder speech models (Whisper).
+
+        These models are exported as composite encoder/decoder graphs, so only the generation
+        comparison (transformers ``generate(input_features)`` vs ONNX Runtime GenAI audio
+        transcription) is applicable. Returns the results dict (also saved to disk by the caller).
+        """
+        _, _, generation_metrics = self._resolve_metric_settings(config)
+        results = {"model_kind": "speech-seq2seq", "status": "passed"}
+
+        genai_model_path = self._resolve_genai_model_path(model, config, generation_metrics or {"first_token_20"})
+        if not genai_model_path:
+            logger.warning(
+                "OnnxDiscrepancyCheck: no GenAI model (genai_config.json) found for the speech model; "
+                "skipping the generation comparison."
+            )
+            results["status"] = "skipped"
+            results["skip_reason"] = "no genai_config.json found for speech model"
+            return results
+
+        gen_max_new_tokens = 20 if "first_token_20" in generation_metrics else config.generate_max_new_tokens
+        gen_first_n = 5 if "tf5t" in generation_metrics else config.first_n_tokens_timed
+        gen_results = self._compare_generation_speech(
+            config,
+            ref_model,
+            ref_model_path=ref_path,
+            genai_model_path=genai_model_path,
+            max_new_tokens=gen_max_new_tokens,
+            first_n=gen_first_n,
+        )
+        results["genai_model_path"] = genai_model_path
+        # For speech models every generation metric is derived from the audio comparison, so always
+        # surface first_token_20 alongside any explicitly requested timing metrics.
+        self._surface_generation_metrics(config, generation_metrics | {"first_token_20"}, gen_results, results)
+        return results
+
     def _run_llama_cpp_comparison(
         self, model: ONNXModelHandler, config, ref_model, ref_path, report_dir, generation_metrics, results
     ):
@@ -911,6 +1016,214 @@ def _measure_speedup(
 
         return pytorch_time, onnx_time, speedup
 
+    def _load_or_make_audio(self, config):
+        """Return ``(audio_array, sample_rate)`` for the speech comparison.
+
+        Uses ``config.speech_audio_path`` when set (read as mono); otherwise generates a short
+        synthetic signal so the comparison always has an input available.
+        """
+        import numpy as np
+
+        audio_path = getattr(config, "speech_audio_path", None)
+        if audio_path:
+            import soundfile as sf
+
+            audio, sample_rate = sf.read(audio_path, dtype="float32")
+            if getattr(audio, "ndim", 1) > 1:
+                # Downmix to mono.
+                audio = audio.mean(axis=1)
+            logger.info("OnnxDiscrepancyCheck using speech audio from %s (sample_rate=%d).", audio_path, sample_rate)
+            return np.asarray(audio, dtype=np.float32), int(sample_rate)
+
+        # Synthetic fallback: 2 seconds of a low-amplitude 440 Hz tone at 16 kHz.
+        sample_rate = 16000
+        duration_s = 2.0
+        t = np.linspace(0, duration_s, int(duration_s * sample_rate), endpoint=False, dtype=np.float32)
+        audio = (0.1 * np.sin(2 * np.pi * 440.0 * t)).astype(np.float32)
+        logger.info(
+            "OnnxDiscrepancyCheck using synthetic %.1fs audio at %d Hz for the speech comparison.",
+            duration_s,
+            sample_rate,
+        )
+        return audio, sample_rate
+
+    @staticmethod
+    def _audio_to_wav_bytes(audio, sample_rate: int) -> bytes:
+        import io
+
+        import soundfile as sf
+
+        buffer = io.BytesIO()
+        sf.write(buffer, audio, samplerate=sample_rate, format="WAV")
+        return buffer.getvalue()
+
+    @staticmethod
+    def _whisper_decoder_prompt(genai_model_path: str) -> str:
+        """Build the Whisper decoder prompt string, mirroring olive_evaluator's genai speech path."""
+        genai_config = {}
+        genai_config_path = Path(genai_model_path) / "genai_config.json"
+        if genai_config_path.is_file():
+            with genai_config_path.open() as f:
+                genai_config = json.load(f)
+        # English-only Whisper checkpoints (vocab_size=51864) use a shorter prompt without a
+        # language/task selector.
+        vocab_size = genai_config.get("model", {}).get("vocab_size", 51865)
+        if vocab_size == 51864:
+            prompt_tokens = ["<|startoftranscript|>", "<|notimestamps|>"]
+        else:
+            prompt_tokens = ["<|startoftranscript|>", "<|en|>", "<|transcribe|>", "<|notimestamps|>"]
+        return "".join(prompt_tokens)
+
+    def _compare_generation_speech(
+        self,
+        config: type[BasePassConfig],
+        ref_model,
+        *,
+        ref_model_path: str,
+        genai_model_path: str,
+        max_new_tokens: Optional[int] = None,
+        first_n: Optional[int] = None,
+    ) -> dict:
+        """Compare transformers vs ONNX Runtime GenAI generation for a Whisper-style speech model.
+
+        The transformers side runs ``generate(input_features=...)`` on mel features extracted from
+        the audio; the GenAI side transcribes the same audio through the multimodal processor
+        (reusing the proven olive_evaluator genai-whisper flow). Returns the same-shaped dict as
+        :meth:`compare_generation` (longest common leading token sequence, first/second token
+        matches and latency metrics), computed over the full decoder token sequences (both of which
+        start with the shared ``<|startoftranscript|>`` preamble).
+        """
+        try:
+            import onnxruntime_genai as og
+        except ImportError as exc:
+            raise ImportError("Please install `onnxruntime-genai` to enable generation comparison.") from exc
+
+        import torch
+        from transformers import AutoProcessor, StoppingCriteria, StoppingCriteriaList
+
+        max_new_tokens = config.generate_max_new_tokens if max_new_tokens is None else max_new_tokens
+        first_n_config = config.first_n_tokens_timed if first_n is None else first_n
+        first_n = max(1, min(first_n_config, max_new_tokens)) if max_new_tokens > 0 else 0
+
+        audio, sample_rate = self._load_or_make_audio(config)
+
+        # ---- transformers generation (audio mel features -> decoder tokens) ----
+        processor = AutoProcessor.from_pretrained(ref_model_path)
+        features = processor(audio, sampling_rate=sample_rate, return_tensors="pt")
+        input_features = features.input_features.to(device=ref_model.device, dtype=ref_model.dtype)
+        use_cuda_sync = ref_model.device.type == "cuda"
+
+        transformers_latency = {"start": None, "ttft": None, "ttfn": None}
+        prompt_token_count = {"value": None}
+
+        class _TransformersLatencyStopCriteria(StoppingCriteria):
+            def __call__(self, generated_ids, scores, **kwargs) -> bool:
+                if prompt_token_count["value"] is None:
+                    # First invocation carries the decoder prompt (start-of-transcript preamble).
+                    prompt_token_count["value"] = generated_ids.shape[-1] - 1
+                generated_token_count = generated_ids.shape[-1] - prompt_token_count["value"]
+                if generated_token_count >= 1 and transformers_latency["ttft"] is None:
+                    transformers_latency["ttft"] = time.perf_counter() - transformers_latency["start"]
+                if generated_token_count >= first_n and transformers_latency["ttfn"] is None:
+                    transformers_latency["ttfn"] = time.perf_counter() - transformers_latency["start"]
+                return False
+
+        with torch.no_grad():
+            if use_cuda_sync:
+                torch.cuda.synchronize()
+            start = time.perf_counter()
+            ref_model.generate(input_features=input_features, max_new_tokens=3, do_sample=False)
+            warmup_time = time.perf_counter() - start
+            if use_cuda_sync:
+                torch.cuda.synchronize()
+            start = time.perf_counter()
+            transformers_latency["start"] = start
+            transformers_output = ref_model.generate(
+                input_features=input_features,
+                max_new_tokens=max_new_tokens,
+                do_sample=False,
+                stopping_criteria=StoppingCriteriaList([_TransformersLatencyStopCriteria()]),
+            )
+            if use_cuda_sync:
+                torch.cuda.synchronize()
+            transformers_elapsed = time.perf_counter() - start
+        transformers_tokens = transformers_output[0].cpu().tolist()
+        if max_new_tokens > 0:
+            transformers_ttft = (
+                transformers_latency["ttft"] if transformers_latency["ttft"] is not None else transformers_elapsed
+            )
+            transformers_ttfn = (
+                transformers_latency["ttfn"] if transformers_latency["ttfn"] is not None else transformers_elapsed
+            )
+        else:
+            transformers_ttft = None
+            transformers_ttfn = None
+
+        # ---- ONNX Runtime GenAI generation (audio -> decoder tokens) ----
+        genai_model = og.Model(genai_model_path)
+        genai_processor = genai_model.create_multimodal_processor()
+        prompt = self._whisper_decoder_prompt(genai_model_path)
+        audios = og.Audios.open_bytes(self._audio_to_wav_bytes(audio, sample_rate))
+        inputs = genai_processor([prompt], audios=audios)
+
+        params = og.GeneratorParams(genai_model)
+        params.set_search_options(do_sample=False, max_length=max_new_tokens + 64, min_length=0, batch_size=1)
+        generator = og.Generator(genai_model, params)
+        generator.set_inputs(inputs)
+
+        genai_ttft = None
+        genai_ttfn = None
+        num_generated = 0
+        start = time.perf_counter()
+        while not generator.is_done():
+            generator.generate_next_token()
+            num_generated += 1
+            if num_generated == 1:
+                genai_ttft = time.perf_counter() - start
+            if num_generated == first_n:
+                genai_ttfn = time.perf_counter() - start
+            if num_generated >= max_new_tokens:
+                break
+        genai_tokens = list(generator.get_sequence(0))
+        del generator
+
+        # Both token streams begin with the shared start-of-transcript decoder preamble, so the
+        # comparison is over the full decoder sequences (unlike the causal-LM path, which strips a
+        # known text prompt first).
+        longest_common = _longest_common_token_sequence(transformers_tokens, genai_tokens)
+
+        transformers_first_token = transformers_tokens[0] if transformers_tokens else None
+        genai_first_token = genai_tokens[0] if genai_tokens else None
+        first_token_matches = transformers_first_token is not None and transformers_first_token == genai_first_token
+        transformers_second_token = transformers_tokens[1] if len(transformers_tokens) > 1 else None
+        genai_second_token = genai_tokens[1] if len(genai_tokens) > 1 else None
+        second_token_matches = transformers_second_token is not None and transformers_second_token == genai_second_token
+
+        gen_results = {
+            "longest_common_token_sequence": longest_common,
+            "first_n_tokens_timed": first_n,
+            "transformers_first_token": transformers_first_token,
+            "genai_first_token": genai_first_token,
+            "first_token_matches": first_token_matches,
+            "transformers_second_token": transformers_second_token,
+            "genai_second_token": genai_second_token,
+            "second_token_matches": second_token_matches,
+            "transformers_ttft_s": transformers_ttft,
+            "transformers_ttfn_s": transformers_ttfn,
+            "genai_ttft_s": genai_ttft,
+            "genai_ttfn_s": genai_ttfn,
+            "transformers_warmup_s": warmup_time,
+        }
+        logger.info(
+            "OnnxDiscrepancyCheck speech generation comparison: transformers_len=%d, genai_len=%d, "
+            "longest_common_token_sequence=%d, first_token_matches=%s",
+            len(transformers_tokens),
+            len(genai_tokens),
+            longest_common,
+            first_token_matches,
+        )
+        return gen_results
+
     def compare_generation(
         self,
         config: type[BasePassConfig],
diff --git a/test/passes/onnx/test_discrepancy_check.py b/test/passes/onnx/test_discrepancy_check.py
index 2f003c236c..dcc5d42b81 100644
--- a/test/passes/onnx/test_discrepancy_check.py
+++ b/test/passes/onnx/test_discrepancy_check.py
@@ -825,3 +825,165 @@ def test_compare_llama_cpp_returns_stderr_and_stdout_on_helper_error(self, tmp_p
             )
 
         assert result == {"llama_cpp_out": "stderr text", "llama_cpp_err": "stdout text"}
+
+
+class TestSpeechSeq2Seq:
+    """Unit tests for the encoder-decoder speech (Whisper) generation comparison path."""
+
+    @staticmethod
+    def _speech_ref_model():
+        import torch
+
+        ref_model = MagicMock()
+        ref_model.device = torch.device("cpu")
+        ref_model.dtype = torch.float32
+        ref_model.main_input_name = "input_features"
+        ref_model.config.is_encoder_decoder = True
+        return ref_model
+
+    def test_is_speech_seq2seq_true_for_whisper_like_model(self):
+        from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck
+
+        assert OnnxDiscrepancyCheck._is_speech_seq2seq(self._speech_ref_model()) is True
+
+    def test_is_speech_seq2seq_false_for_causal_lm(self):
+        import torch
+
+        from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck
+
+        ref_model = MagicMock()
+        ref_model.device = torch.device("cpu")
+        ref_model.main_input_name = "input_ids"
+        ref_model.config.is_encoder_decoder = False
+        assert OnnxDiscrepancyCheck._is_speech_seq2seq(ref_model) is False
+
+    def test_load_or_make_audio_returns_synthetic_when_unset(self):
+        import numpy as np
+
+        from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck
+
+        config = MagicMock()
+        config.speech_audio_path = None
+        pass_instance = OnnxDiscrepancyCheck.__new__(OnnxDiscrepancyCheck)
+        audio, sample_rate = pass_instance._load_or_make_audio(config)
+        assert sample_rate == 16000
+        assert isinstance(audio, np.ndarray)
+        assert audio.dtype == np.float32
+        assert audio.shape[0] == int(2.0 * 16000)
+
+    def test_load_or_make_audio_reads_configured_path(self):
+        import numpy as np
+
+        from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck
+
+        config = MagicMock()
+        config.speech_audio_path = "audio.wav"
+        # Stereo audio should be downmixed to mono.
+        stereo = np.ones((100, 2), dtype=np.float32)
+        mock_sf = MagicMock()
+        mock_sf.read.return_value = (stereo, 22050)
+
+        pass_instance = OnnxDiscrepancyCheck.__new__(OnnxDiscrepancyCheck)
+        with patch.dict(sys.modules, {"soundfile": mock_sf}):
+            audio, sample_rate = pass_instance._load_or_make_audio(config)
+        assert sample_rate == 22050
+        assert audio.ndim == 1
+        assert audio.shape[0] == 100
+
+    def test_whisper_decoder_prompt_multilingual_default(self, tmp_path):
+        from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck
+
+        # No genai_config.json -> multilingual prompt.
+        prompt = OnnxDiscrepancyCheck._whisper_decoder_prompt(str(tmp_path))
+        assert prompt == "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>"
+
+    def test_whisper_decoder_prompt_english_only(self, tmp_path):
+        import json
+
+        from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck
+
+        (tmp_path / "genai_config.json").write_text(json.dumps({"model": {"vocab_size": 51864}}))
+        prompt = OnnxDiscrepancyCheck._whisper_decoder_prompt(str(tmp_path))
+        assert prompt == "<|startoftranscript|><|notimestamps|>"
+
+    def test_compare_generation_speech_computes_common_prefix(self, tmp_path):
+        import torch
+
+        from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck
+
+        config = MagicMock()
+        config.speech_audio_path = None
+        config.generate_max_new_tokens = 10
+        config.first_n_tokens_timed = 5
+
+        ref_model = self._speech_ref_model()
+        # transformers decoder tokens (start-of-transcript preamble + content).
+        ref_model.generate.return_value = torch.tensor([[50258, 50259, 50359, 50363, 100, 200, 300]])
+
+        # Processor(audio) -> object exposing .input_features.
+        mock_processor = MagicMock()
+        mock_features = MagicMock()
+        mock_features.input_features = torch.zeros((1, 80, 3000))
+        mock_processor.return_value = mock_features
+
+        # GenAI sequence diverges at the last token (999 vs 300) -> longest common = 6.
+        genai_sequence = [50258, 50259, 50359, 50363, 100, 200, 999]
+        num_new = len(genai_sequence) - 4  # 4 prompt tokens
+
+        mock_generator = MagicMock()
+        counter = {"n": 0}
+
+        def is_done():
+            return counter["n"] >= num_new
+
+        def generate_next_token():
+            counter["n"] += 1
+
+        mock_generator.is_done = is_done
+        mock_generator.generate_next_token = generate_next_token
+        mock_generator.get_sequence.return_value = genai_sequence
+
+        mock_og = MagicMock()
+        mock_og.Generator.return_value = mock_generator
+
+        mock_sf = MagicMock()
+
+        with (
+            patch.dict(sys.modules, {"onnxruntime_genai": mock_og, "soundfile": mock_sf}),
+            patch("transformers.AutoProcessor.from_pretrained", return_value=mock_processor),
+        ):
+            pass_instance = OnnxDiscrepancyCheck.__new__(OnnxDiscrepancyCheck)
+            result = pass_instance._compare_generation_speech(
+                config,
+                ref_model,
+                ref_model_path=str(tmp_path),
+                genai_model_path=str(tmp_path),
+            )
+
+        # transformers was driven with audio input_features, not input_ids.
+        _, kwargs = ref_model.generate.call_args
+        assert "input_features" in kwargs
+        mock_generator.set_inputs.assert_called_once()
+        assert result["longest_common_token_sequence"] == 6
+        assert result["first_token_matches"] is True
+        assert result["second_token_matches"] is True
+        assert result["genai_first_token"] == 50258
+
+    def test_run_speech_generation_comparison_marks_skipped_without_genai(self):
+        from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck
+
+        config = MagicMock()
+        config.test_metrics = None
+        config.genai_model_path = None
+        config.timing_iterations = 0
+        config.max_mae = None
+
+        model = MagicMock()
+        # model_path without a genai_config.json -> generation comparison is skipped.
+        model.model_path = "/nonexistent/model/dir"
+
+        pass_instance = OnnxDiscrepancyCheck.__new__(OnnxDiscrepancyCheck)
+        with patch.object(OnnxDiscrepancyCheck, "_resolve_genai_model_path", return_value=None):
+            results = pass_instance._run_speech_generation_comparison(model, config, MagicMock(), "ref_path")
+        assert results["model_kind"] == "speech-seq2seq"
+        assert results["status"] == "skipped"

From 61c99d2e90355feaf43c3e76cabd927ed7d388ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= <xadupre@microsoft.com>
Date: Thu, 9 Jul 2026 18:52:35 +0200
Subject: [PATCH 03/21] modif

---
 olive/common/hf/utils.py               | 15 +++++++++
 olive/passes/onnx/discrepancy_check.py | 44 ++++++++++++++++++++++++--
 2 files changed, 57 insertions(+), 2 deletions(-)

diff --git a/olive/common/hf/utils.py b/olive/common/hf/utils.py
index ca44c9f5cc..06399d8184 100644
--- a/olive/common/hf/utils.py
+++ b/olive/common/hf/utils.py
@@ -177,6 +177,21 @@ def _save_test_model(
             save_tokenizer(tokenizer, str(output_path))
         except Exception as e:  # pylint: disable=broad-except
             logger.debug("Could not save tokenizer for test model from %r: %s", model_name_or_path, e)
+        # Also save the processor / feature extractor when the model has one (e.g. Whisper and other
+        # speech/multimodal models). This writes ``preprocessor_config.json`` so downstream passes
+        # such as OnnxDiscrepancyCheck can build audio ``input_features`` from the reference model
+        # directory without needing the original model. Best-effort: text-only models have no
+        # processor and are already covered by the tokenizer save above.
+        try:
+            from transformers import AutoProcessor
+            from transformers.tokenization_utils_base import PreTrainedTokenizerBase
+
+            processor = AutoProcessor.from_pretrained(model_name_or_path)
+            if not isinstance(processor, PreTrainedTokenizerBase):
+                processor.save_pretrained(str(output_path))
+                logger.debug("Saved processor/feature extractor for test model to %s", output_path)
+        except Exception as e:  # pylint: disable=broad-except
+            logger.debug("No processor/feature extractor saved for test model from %r: %s", model_name_or_path, e)
     _write_test_model_marker(output_path, test_model_config)
 
 
diff --git a/olive/passes/onnx/discrepancy_check.py b/olive/passes/onnx/discrepancy_check.py
index d4da88f3f4..21ddbc5933 100644
--- a/olive/passes/onnx/discrepancy_check.py
+++ b/olive/passes/onnx/discrepancy_check.py
@@ -1074,6 +1074,46 @@ def _whisper_decoder_prompt(genai_model_path: str) -> str:
             prompt_tokens = ["<|startoftranscript|>", "<|en|>", "<|transcribe|>", "<|notimestamps|>"]
         return "".join(prompt_tokens)
 
+    @staticmethod
+    def _load_speech_processor(ref_model_path: str, genai_model_path: str, ref_model):
+        """Load the audio processor / feature extractor for the reference speech model.
+
+        Prefers the reference model directory (written self-contained by SaveTestModelConfig), then
+        falls back to the GenAI model directory and finally to the original model id recorded in the
+        config, so an older reference directory saved without a ``preprocessor_config.json`` still
+        works.
+        """
+        from transformers import AutoProcessor
+
+        candidates = [ref_model_path, genai_model_path]
+        original_name = getattr(getattr(ref_model, "config", None), "_name_or_path", None)
+        if original_name:
+            candidates.append(original_name)
+
+        last_error = None
+        seen = set()
+        for candidate in candidates:
+            if not candidate or candidate in seen:
+                continue
+            seen.add(candidate)
+            try:
+                processor = AutoProcessor.from_pretrained(candidate)
+                if candidate != ref_model_path:
+                    logger.info(
+                        "OnnxDiscrepancyCheck loaded the speech processor from %r (reference model "
+                        "directory %r did not contain one).",
+                        candidate,
+                        ref_model_path,
+                    )
+                return processor
+            except Exception as exc:  # pylint: disable=broad-except
+                last_error = exc
+                logger.debug("Could not load speech processor from %r: %s", candidate, exc)
+        raise RuntimeError(
+            f"Could not load an audio processor/feature extractor for the speech model from any of "
+            f"{candidates}. Ensure the reference model directory contains a preprocessor_config.json."
+        ) from last_error
+
     def _compare_generation_speech(
         self,
         config: type[BasePassConfig],
@@ -1099,7 +1139,7 @@ def _compare_generation_speech(
             raise ImportError("Please install `onnxruntime-genai` to enable generation comparison.") from exc
 
         import torch
-        from transformers import AutoProcessor, StoppingCriteria, StoppingCriteriaList
+        from transformers import StoppingCriteria, StoppingCriteriaList
 
         max_new_tokens = config.generate_max_new_tokens if max_new_tokens is None else max_new_tokens
         first_n_config = config.first_n_tokens_timed if first_n is None else first_n
@@ -1108,7 +1148,7 @@ def _compare_generation_speech(
         audio, sample_rate = self._load_or_make_audio(config)
 
         # ---- transformers generation (audio mel features -> decoder tokens) ----
-        processor = AutoProcessor.from_pretrained(ref_model_path)
+        processor = self._load_speech_processor(ref_model_path, genai_model_path, ref_model)
         features = processor(audio, sampling_rate=sample_rate, return_tensors="pt")
         input_features = features.input_features.to(device=ref_model.device, dtype=ref_model.dtype)
         use_cuda_sync = ref_model.device.type == "cuda"

From 27a02455431951abf107761a1b77e36d8c6be3dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= <xadupre@microsoft.com>
Date: Thu, 9 Jul 2026 19:00:01 +0200
Subject: [PATCH 04/21] test

---
 test/common/test_hf.py | 62 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 62 insertions(+)

diff --git a/test/common/test_hf.py b/test/common/test_hf.py
index dad6f56d54..ae931740de 100644
--- a/test/common/test_hf.py
+++ b/test/common/test_hf.py
@@ -131,6 +131,68 @@ def test_load_model_from_task_test_model_config_saves_tokenizer(tmp_path):
     mock_save_tokenizer.assert_called_once_with(mock_tokenizer, str(test_model_path))
 
 
+def test_load_model_from_task_test_model_config_saves_processor(tmp_path):
+    """A speech/multimodal model's processor should be saved into the test model directory."""
+    model_config = BertConfig(num_hidden_layers=12)  # pylint: disable=unexpected-keyword-arg
+    created_model = MagicMock()
+    test_model_path = tmp_path / "saved_test_model"
+    # A feature-extractor-style processor (not a tokenizer) should be persisted so that
+    # preprocessor_config.json ends up in the reference model directory (e.g. for Whisper).
+    mock_processor = MagicMock()
+
+    with (
+        patch("transformers.pipelines.check_task") as mock_check_task,
+        patch("olive.common.hf.utils.from_pretrained", return_value=model_config),
+        patch("olive.common.hf.utils.get_tokenizer", return_value=MagicMock()),
+        patch("olive.common.hf.utils.save_tokenizer"),
+        patch("transformers.AutoProcessor.from_pretrained", return_value=mock_processor) as mock_get_processor,
+    ):
+        mock_model_class = MagicMock()
+        mock_model_class.from_config.return_value = created_model
+        mock_check_task.return_value = ("text-classification", {"pt": (mock_model_class,)}, None)
+
+        load_model_from_task(
+            "text-classification",
+            "dummy-model",
+            test_model_config={"num_hidden_layers": 2},
+            test_model_path=str(test_model_path),
+        )
+
+    mock_get_processor.assert_called_once_with("dummy-model")
+    mock_processor.save_pretrained.assert_called_once_with(str(test_model_path))
+
+
+def test_load_model_from_task_test_model_config_skips_tokenizer_only_processor(tmp_path):
+    """When AutoProcessor returns a plain tokenizer, it should not be re-saved as a processor."""
+    from transformers.tokenization_utils_base import PreTrainedTokenizerBase
+
+    model_config = BertConfig(num_hidden_layers=12)  # pylint: disable=unexpected-keyword-arg
+    created_model = MagicMock()
+    test_model_path = tmp_path / "saved_test_model"
+    # Text-only models return a tokenizer from AutoProcessor; it must not be saved via the processor path.
+    mock_processor = MagicMock(spec=PreTrainedTokenizerBase)
+
+    with (
+        patch("transformers.pipelines.check_task") as mock_check_task,
+        patch("olive.common.hf.utils.from_pretrained", return_value=model_config),
+        patch("olive.common.hf.utils.get_tokenizer", return_value=MagicMock()),
+        patch("olive.common.hf.utils.save_tokenizer"),
+        patch("transformers.AutoProcessor.from_pretrained", return_value=mock_processor),
+    ):
+        mock_model_class = MagicMock()
+        mock_model_class.from_config.return_value = created_model
+        mock_check_task.return_value = ("text-classification", {"pt": (mock_model_class,)}, None)
+
+        load_model_from_task(
+            "text-classification",
+            "dummy-model",
+            test_model_config={"num_hidden_layers": 2},
+            test_model_path=str(test_model_path),
+        )
+
+    mock_processor.save_pretrained.assert_not_called()
+
+
 def test_load_model_from_task_test_model_config_fails_without_fallback():
     model_config = BertConfig(num_hidden_layers=12)  # pylint: disable=unexpected-keyword-arg
 

From aeb5e373b33044bf57457e91a86039e727764a08 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= <xadupre@microsoft.com>
Date: Thu, 9 Jul 2026 19:11:56 +0200
Subject: [PATCH 05/21] update

---
 olive/passes/onnx/discrepancy_check.py     | 32 ++++++++++++++++------
 test/passes/onnx/test_discrepancy_check.py | 26 ++++++++++++++++++
 2 files changed, 50 insertions(+), 8 deletions(-)

diff --git a/olive/passes/onnx/discrepancy_check.py b/olive/passes/onnx/discrepancy_check.py
index 21ddbc5933..285269b88f 100644
--- a/olive/passes/onnx/discrepancy_check.py
+++ b/olive/passes/onnx/discrepancy_check.py
@@ -854,15 +854,31 @@ def _run_speech_generation_comparison(self, model, config, ref_model, ref_path):
 
         gen_max_new_tokens = 20 if "first_token_20" in generation_metrics else config.generate_max_new_tokens
         gen_first_n = 5 if "tf5t" in generation_metrics else config.first_n_tokens_timed
-        gen_results = self._compare_generation_speech(
-            config,
-            ref_model,
-            ref_model_path=ref_path,
-            genai_model_path=genai_model_path,
-            max_new_tokens=gen_max_new_tokens,
-            first_n=gen_first_n,
-        )
         results["genai_model_path"] = genai_model_path
+        try:
+            gen_results = self._compare_generation_speech(
+                config,
+                ref_model,
+                ref_model_path=ref_path,
+                genai_model_path=genai_model_path,
+                max_new_tokens=gen_max_new_tokens,
+                first_n=gen_first_n,
+            )
+        except Exception as exc:  # pylint: disable=broad-except
+            # The audio-based generation comparison is an optional diagnostic. onnxruntime-genai can
+            # raise runtime errors for some Whisper builds (e.g. "Invalid output name:
+            # present_key_cross_*" on a genai/model-builder version mismatch). Degrade gracefully so
+            # the optimize workflow still completes and the failure is recorded in the report.
+            logger.warning(
+                "OnnxDiscrepancyCheck speech generation comparison could not be completed (%s). This is "
+                "typically an onnxruntime-genai / model-builder version incompatibility for the Whisper "
+                "GenAI model; skipping the comparison.",
+                exc,
+            )
+            results["status"] = "skipped"
+            results["skip_reason"] = f"speech generation comparison failed: {exc}"
+            return results
+
         # For speech models every generation metric is derived from the audio comparison, so always
         # surface first_token_20 alongside any explicitly requested timing metrics.
         self._surface_generation_metrics(config, generation_metrics | {"first_token_20"}, gen_results, results)
diff --git a/test/passes/onnx/test_discrepancy_check.py b/test/passes/onnx/test_discrepancy_check.py
index dcc5d42b81..51b43d96b1 100644
--- a/test/passes/onnx/test_discrepancy_check.py
+++ b/test/passes/onnx/test_discrepancy_check.py
@@ -987,3 +987,29 @@ def test_run_speech_generation_comparison_marks_skipped_without_genai(self):
             results = pass_instance._run_speech_generation_comparison(model, config, MagicMock(), "ref_path")
         assert results["model_kind"] == "speech-seq2seq"
         assert results["status"] == "skipped"
+
+    def test_run_speech_generation_comparison_degrades_on_genai_error(self):
+        from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck
+
+        config = MagicMock()
+        config.test_metrics = None
+        config.genai_model_path = "/some/genai/dir"
+        config.generate_max_new_tokens = 10
+        config.first_n_tokens_timed = 5
+
+        model = MagicMock()
+        pass_instance = OnnxDiscrepancyCheck.__new__(OnnxDiscrepancyCheck)
+        # An onnxruntime-genai runtime failure (e.g. "Invalid output name: present_key_cross_*")
+        # must not abort the workflow; it is recorded as a skipped comparison.
+        with (
+            patch.object(OnnxDiscrepancyCheck, "_resolve_genai_model_path", return_value="/some/genai/dir"),
+            patch.object(
+                OnnxDiscrepancyCheck,
+                "_compare_generation_speech",
+                side_effect=RuntimeError("Invalid output name: present_key_cross_2"),
+            ),
+        ):
+            results = pass_instance._run_speech_generation_comparison(model, config, MagicMock(), "ref_path")
+        assert results["status"] == "skipped"
+        assert "present_key_cross_2" in results["skip_reason"]
+        assert results["genai_model_path"] == "/some/genai/dir"

From 3d302504608d9e9bbd0facc4db5d85f1bac25556 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= <xadupre@microsoft.com>
Date: Fri, 10 Jul 2026 10:25:33 +0200
Subject: [PATCH 06/21] fix test

---
 olive/common/hf/utils.py                       | 17 +++++++++++++++--
 test/common/test_hf.py                         | 18 ++++++++++++++++++
 test/passes/onnx/test_session_params_tuning.py |  4 +++-
 3 files changed, 36 insertions(+), 3 deletions(-)

diff --git a/olive/common/hf/utils.py b/olive/common/hf/utils.py
index 06399d8184..58cd439af0 100644
--- a/olive/common/hf/utils.py
+++ b/olive/common/hf/utils.py
@@ -79,8 +79,21 @@ def _apply_test_model_config(
     updated = False
     # Common Hugging Face configs do not use a single canonical field:
     # BERT-style models use num_hidden_layers while GPT-style models often use n_layer/n_layers/num_layers.
-    for attr_name in ("num_hidden_layers", "num_layers", "n_layer", "n_layers"):
-        if hasattr(model_config, attr_name):
+    # Encoder-decoder models (e.g. Whisper, BART, T5) keep separate encoder/decoder layer counts that
+    # must ALL be reduced consistently: reducing only num_hidden_layers while leaving encoder_layers/
+    # decoder_layers at their original value produces an inconsistent model where, for example, the
+    # ONNX decoder graph exports more cross-attention KV outputs (present_key_cross_*) than the GenAI
+    # config expects, which makes onnxruntime-genai fail with "Invalid output name: present_key_cross_*".
+    for attr_name in (
+        "num_hidden_layers",
+        "num_layers",
+        "n_layer",
+        "n_layers",
+        "encoder_layers",
+        "decoder_layers",
+        "num_decoder_layers",
+    ):
+        if getattr(model_config, attr_name, None) is not None:
             setattr(model_config, attr_name, hidden_layers)
             updated = True
 
diff --git a/test/common/test_hf.py b/test/common/test_hf.py
index ae931740de..394daa11c9 100644
--- a/test/common/test_hf.py
+++ b/test/common/test_hf.py
@@ -18,6 +18,24 @@
 )
 
 
+def test_apply_test_model_config_reduces_encoder_decoder_layers_consistently():
+    """Encoder-decoder models (e.g. Whisper) must reduce encoder AND decoder layer counts together.
+
+    Reducing only num_hidden_layers while leaving encoder_layers/decoder_layers unchanged produces an
+    inconsistent model whose exported ONNX decoder emits more cross-attention KV outputs than the
+    GenAI config expects, causing onnxruntime-genai to fail with "Invalid output name:
+    present_key_cross_*".
+    """
+    from transformers import WhisperConfig
+
+    from olive.common.hf.utils import _apply_test_model_config
+
+    reduced = _apply_test_model_config(WhisperConfig(), {"hidden_layers": 2})
+    assert reduced.encoder_layers == 2
+    assert reduced.decoder_layers == 2
+    assert reduced.num_hidden_layers == 2
+
+
 def test_load_model_from_task():
     # The model name and task type is gotten from
     # https://huggingface.co/docs/transformers/v4.28.1/en/main_classes/pipelines#transformers.pipeline
diff --git a/test/passes/onnx/test_session_params_tuning.py b/test/passes/onnx/test_session_params_tuning.py
index 4eed0653a9..3c7d8dfefb 100644
--- a/test/passes/onnx/test_session_params_tuning.py
+++ b/test/passes/onnx/test_session_params_tuning.py
@@ -171,4 +171,6 @@ def test_ort_session_params_tuning_pass_with_dynamic_shapes(mock_get_io_config,
     with pytest.raises(TypeError) as e:
         # execute
         p.run(input_model, output_folder)
-    assert "ones() received an invalid combination of arguments" in str(e.value)
+    assert "ones() received an invalid combination of arguments" in str(e.value) or (
+        "ones()" in str(e.value) and "must be tuple of ints" in str(e.value)
+    )

From 8ca18c0db249988763b091ba117bc10eaf670796 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= <xadupre@microsoft.com>
Date: Fri, 10 Jul 2026 11:29:03 +0200
Subject: [PATCH 07/21] fix whisper

---
 olive/passes/onnx/discrepancy_check.py     | 218 ++++++++++++++++++---
 test/passes/onnx/test_discrepancy_check.py | 138 ++++++++++++-
 2 files changed, 329 insertions(+), 27 deletions(-)

diff --git a/olive/passes/onnx/discrepancy_check.py b/olive/passes/onnx/discrepancy_check.py
index 285269b88f..85cdf337e9 100644
--- a/olive/passes/onnx/discrepancy_check.py
+++ b/olive/passes/onnx/discrepancy_check.py
@@ -4,6 +4,8 @@
 # --------------------------------------------------------------------------
 import json
 import logging
+import os
+import shutil
 import subprocess
 import time
 from pathlib import Path
@@ -34,6 +36,69 @@ def _json_sanitize(obj):
     return obj
 
 
+def _expand_genai_output_names(template: str, num_layers: int) -> list:
+    """Expand a genai_config output-name value into the concrete ONNX output names.
+
+    ONNX Runtime GenAI encodes per-layer cache outputs as ``%d`` templates (e.g.
+    ``present_key_cross_%d``) that it expands over the number of layers, while plain outputs
+    such as ``logits`` are used verbatim.
+    """
+    if "%d" in template:
+        return [template % layer for layer in range(num_layers)]
+    return [template]
+
+
+def _reconcile_genai_speech_output_names(genai_config: dict, actual_outputs: dict):
+    """Prune Whisper ``genai_config.json`` output names that are absent from the ONNX graphs.
+
+    A genai / model-builder version mismatch can leave the ``model.encoder`` / ``model.decoder``
+    ``outputs`` maps referencing tensors (typically the ``present_key_cross_*`` /
+    ``present_value_cross_*`` cross-attention cache) that the actual ONNX graph does not produce,
+    which makes ``onnxruntime_genai.Model`` fail with ``Invalid output name: ...``. Dropping the
+    unmatched entries lets the model load so the discrepancy comparison can still run.
+
+    Args:
+        genai_config: Parsed ``genai_config.json`` contents. Not mutated; a reconciled copy is
+            returned instead.
+        actual_outputs: Mapping of section name (``"encoder"`` / ``"decoder"``) to the set of
+            output names actually present in that section's ONNX graph. Sections missing from the
+            mapping are left untouched.
+
+    Returns:
+        Tuple ``(reconciled_config, pruned)`` where ``pruned`` is a list of
+        ``(section, key, template)`` tuples describing each removed entry.
+
+    """
+    import copy
+
+    reconciled = copy.deepcopy(genai_config)
+    pruned = []
+    model_section = reconciled.get("model")
+    if not isinstance(model_section, dict):
+        return reconciled, pruned
+
+    for section_name, present in actual_outputs.items():
+        section = model_section.get(section_name)
+        if not isinstance(section, dict):
+            continue
+        outputs = section.get("outputs")
+        if not isinstance(outputs, dict):
+            continue
+        num_layers = section.get("num_hidden_layers", 0) or 0
+        for key in list(outputs):
+            value = outputs[key]
+            if not isinstance(value, str):
+                continue
+            expanded = _expand_genai_output_names(value, num_layers)
+            # Keep the entry only when every concrete output name it references actually exists;
+            # a plain output with no expansion (e.g. logits) is kept when its single name exists.
+            if expanded and not all(name in present for name in expanded):
+                del outputs[key]
+                pruned.append((section_name, key, value))
+
+    return reconciled, pruned
+
+
 def _infer_shape(dynamic_shape, known_values=None):
     # Use an empty past-KV cache (past_sequence_length=0) so the discrepancy check is a clean
     # prefill comparison.  The dummy dataloader passes ``past_key_values.<i>.key/value`` tensors,
@@ -1130,6 +1195,102 @@ def _load_speech_processor(ref_model_path: str, genai_model_path: str, ref_model
             f"{candidates}. Ensure the reference model directory contains a preprocessor_config.json."
         ) from last_error
 
+    @staticmethod
+    def _load_genai_speech_model(genai_model_path: str):
+        """Load an ``onnxruntime_genai.Model`` for a Whisper model, tolerating output-name mismatches.
+
+        A genai / model-builder version incompatibility can leave ``genai_config.json`` declaring
+        ONNX outputs (typically ``present_key_cross_*``) that the graph does not produce, which makes
+        ``og.Model`` raise ``Invalid output name: ...``. When that happens, the ``genai_config.json``
+        is reconciled against the real graph outputs in a temporary directory (the large ONNX files
+        are hard-linked, falling back to copies) and the load is retried once.
+
+        Returns a tuple ``(genai_model, temp_dir)`` where ``temp_dir`` is a
+        ``tempfile.TemporaryDirectory`` that the caller must keep alive while the model is in use and
+        clean up afterwards (``None`` when no reconciliation was needed).
+        """
+        import onnxruntime_genai as og
+
+        try:
+            return og.Model(genai_model_path), None
+        except Exception as exc:  # pylint: disable=broad-except
+            if "invalid output name" not in str(exc).lower():
+                raise
+
+            model_dir = Path(genai_model_path)
+            genai_config_path = model_dir / "genai_config.json"
+            if not genai_config_path.is_file():
+                raise
+            with genai_config_path.open() as f:
+                genai_config = json.load(f)
+
+            actual_outputs = OnnxDiscrepancyCheck._collect_genai_section_outputs(model_dir, genai_config)
+            reconciled, pruned = _reconcile_genai_speech_output_names(genai_config, actual_outputs)
+            if not pruned:
+                # Nothing to reconcile; the failure is unrelated to stale output names.
+                raise
+
+            logger.warning(
+                "OnnxDiscrepancyCheck reconciling Whisper genai_config.json: %d output name(s) declared "
+                "but absent from the ONNX graph were removed so the model can load (%s). This indicates a "
+                "genai / model-builder version mismatch.",
+                len(pruned),
+                ", ".join(f"{section}.{key}={template}" for section, key, template in pruned),
+            )
+
+            import tempfile
+
+            temp_dir = tempfile.TemporaryDirectory(prefix="olive_genai_reconciled_")
+            temp_path = Path(temp_dir.name)
+            for item in model_dir.iterdir():
+                if item.name == "genai_config.json" or not item.is_file():
+                    continue
+                target = temp_path / item.name
+                try:
+                    os.link(item, target)
+                except OSError:
+                    shutil.copy2(item, target)
+            with (temp_path / "genai_config.json").open("w") as f:
+                json.dump(reconciled, f, indent=4)
+
+            try:
+                return og.Model(str(temp_path)), temp_dir
+            except Exception:
+                temp_dir.cleanup()
+                raise
+
+    @staticmethod
+    def _collect_genai_section_outputs(model_dir: Path, genai_config: dict) -> dict:
+        """Read the actual ONNX graph output names for each genai_config model section.
+
+        Returns a mapping of section name (``"encoder"`` / ``"decoder"``) to the set of output names
+        found in that section's ONNX file. Sections whose ONNX file cannot be read are omitted so the
+        reconciliation leaves them untouched.
+        """
+        import onnx
+
+        actual_outputs = {}
+        model_section = genai_config.get("model", {})
+        if not isinstance(model_section, dict):
+            return actual_outputs
+        for section_name in ("encoder", "decoder"):
+            section = model_section.get(section_name)
+            if not isinstance(section, dict):
+                continue
+            filename = section.get("filename")
+            if not filename:
+                continue
+            onnx_path = model_dir / filename
+            if not onnx_path.is_file():
+                continue
+            try:
+                onnx_model = onnx.load(str(onnx_path), load_external_data=False)
+            except Exception as exc:  # pylint: disable=broad-except
+                logger.debug("Could not read ONNX outputs from %s: %s", onnx_path, exc)
+                continue
+            actual_outputs[section_name] = {output.name for output in onnx_model.graph.output}
+        return actual_outputs
+
     def _compare_generation_speech(
         self,
         config: type[BasePassConfig],
@@ -1216,32 +1377,37 @@ def __call__(self, generated_ids, scores, **kwargs) -> bool:
             transformers_ttfn = None
 
         # ---- ONNX Runtime GenAI generation (audio -> decoder tokens) ----
-        genai_model = og.Model(genai_model_path)
-        genai_processor = genai_model.create_multimodal_processor()
-        prompt = self._whisper_decoder_prompt(genai_model_path)
-        audios = og.Audios.open_bytes(self._audio_to_wav_bytes(audio, sample_rate))
-        inputs = genai_processor([prompt], audios=audios)
-
-        params = og.GeneratorParams(genai_model)
-        params.set_search_options(do_sample=False, max_length=max_new_tokens + 64, min_length=0, batch_size=1)
-        generator = og.Generator(genai_model, params)
-        generator.set_inputs(inputs)
-
-        genai_ttft = None
-        genai_ttfn = None
-        num_generated = 0
-        start = time.perf_counter()
-        while not generator.is_done():
-            generator.generate_next_token()
-            num_generated += 1
-            if num_generated == 1:
-                genai_ttft = time.perf_counter() - start
-            if num_generated == first_n:
-                genai_ttfn = time.perf_counter() - start
-            if num_generated >= max_new_tokens:
-                break
-        genai_tokens = list(generator.get_sequence(0))
-        del generator
+        genai_model, genai_temp_dir = self._load_genai_speech_model(genai_model_path)
+        try:
+            genai_processor = genai_model.create_multimodal_processor()
+            prompt = self._whisper_decoder_prompt(genai_model_path)
+            audios = og.Audios.open_bytes(self._audio_to_wav_bytes(audio, sample_rate))
+            inputs = genai_processor([prompt], audios=audios)
+
+            params = og.GeneratorParams(genai_model)
+            params.set_search_options(do_sample=False, max_length=max_new_tokens + 64, min_length=0, batch_size=1)
+            generator = og.Generator(genai_model, params)
+            generator.set_inputs(inputs)
+
+            genai_ttft = None
+            genai_ttfn = None
+            num_generated = 0
+            start = time.perf_counter()
+            while not generator.is_done():
+                generator.generate_next_token()
+                num_generated += 1
+                if num_generated == 1:
+                    genai_ttft = time.perf_counter() - start
+                if num_generated == first_n:
+                    genai_ttfn = time.perf_counter() - start
+                if num_generated >= max_new_tokens:
+                    break
+            genai_tokens = list(generator.get_sequence(0))
+            del generator
+        finally:
+            del genai_model
+            if genai_temp_dir is not None:
+                genai_temp_dir.cleanup()
 
         # Both token streams begin with the shared start-of-transcript decoder preamble, so the
         # comparison is over the full decoder sequences (unlike the causal-LM path, which strips a
diff --git a/test/passes/onnx/test_discrepancy_check.py b/test/passes/onnx/test_discrepancy_check.py
index 51b43d96b1..1798427587 100644
--- a/test/passes/onnx/test_discrepancy_check.py
+++ b/test/passes/onnx/test_discrepancy_check.py
@@ -9,7 +9,11 @@
 
 import pytest
 
-from olive.passes.onnx.discrepancy_check import _longest_common_token_sequence
+from olive.passes.onnx.discrepancy_check import (
+    _expand_genai_output_names,
+    _longest_common_token_sequence,
+    _reconcile_genai_speech_output_names,
+)
 
 
 class TestLongestCommonTokenSequence:
@@ -47,6 +51,138 @@ def test_common_tokens_later_not_counted(self):
         assert _longest_common_token_sequence([10, 1, 2, 3], [20, 1, 2, 3]) == 0
 
 
+def _whisper_genai_config(num_layers=2):
+    """Build a minimal Whisper-style genai_config with cross-attention cache outputs."""
+    return {
+        "model": {
+            "decoder": {
+                "filename": "decoder_model_merged.onnx",
+                "num_hidden_layers": num_layers,
+                "outputs": {
+                    "logits": "logits",
+                    "present_key_names": "present_key_self_%d",
+                    "present_value_names": "present_value_self_%d",
+                },
+            },
+            "encoder": {
+                "filename": "encoder_model.onnx",
+                "num_hidden_layers": num_layers,
+                "outputs": {
+                    "encoder_hidden_states": "hidden_states",
+                    "cross_present_key_names": "present_key_cross_%d",
+                    "cross_present_value_names": "present_value_cross_%d",
+                },
+            },
+        }
+    }
+
+
+class TestExpandGenaiOutputNames:
+    """Unit tests for _expand_genai_output_names helper."""
+
+    def test_expands_templated_name_over_layers(self):
+        assert _expand_genai_output_names("present_key_cross_%d", 3) == [
+            "present_key_cross_0",
+            "present_key_cross_1",
+            "present_key_cross_2",
+        ]
+
+    def test_plain_name_is_returned_verbatim(self):
+        assert _expand_genai_output_names("logits", 4) == ["logits"]
+
+    def test_templated_name_with_zero_layers_is_empty(self):
+        assert _expand_genai_output_names("present_key_cross_%d", 0) == []
+
+
+class TestReconcileGenaiSpeechOutputNames:
+    """Unit tests for _reconcile_genai_speech_output_names helper."""
+
+    def test_prunes_output_names_absent_from_graph(self):
+        config = _whisper_genai_config(num_layers=2)
+        actual_outputs = {
+            "decoder": {
+                "logits",
+                "present_key_self_0",
+                "present_value_self_0",
+                "present_key_self_1",
+                "present_value_self_1",
+            },
+            # The encoder graph does not produce the cross-attention cache outputs.
+            "encoder": {"hidden_states"},
+        }
+
+        reconciled, pruned = _reconcile_genai_speech_output_names(config, actual_outputs)
+
+        encoder_outputs = reconciled["model"]["encoder"]["outputs"]
+        assert "cross_present_key_names" not in encoder_outputs
+        assert "cross_present_value_names" not in encoder_outputs
+        assert encoder_outputs["encoder_hidden_states"] == "hidden_states"
+        assert {(section, key) for section, key, _ in pruned} == {
+            ("encoder", "cross_present_key_names"),
+            ("encoder", "cross_present_value_names"),
+        }
+
+    def test_keeps_outputs_when_all_names_present(self):
+        config = _whisper_genai_config(num_layers=2)
+        actual_outputs = {
+            "encoder": {
+                "hidden_states",
+                "present_key_cross_0",
+                "present_key_cross_1",
+                "present_value_cross_0",
+                "present_value_cross_1",
+            },
+            "decoder": {
+                "logits",
+                "present_key_self_0",
+                "present_key_self_1",
+                "present_value_self_0",
+                "present_value_self_1",
+            },
+        }
+
+        reconciled, pruned = _reconcile_genai_speech_output_names(config, actual_outputs)
+
+        assert pruned == []
+        assert reconciled == config
+
+    def test_prunes_when_only_some_layer_outputs_missing(self):
+        config = _whisper_genai_config(num_layers=2)
+        # Only layer 0 cross outputs exist; layer 1 is missing, so the whole entry is pruned.
+        actual_outputs = {
+            "encoder": {"hidden_states", "present_key_cross_0", "present_value_cross_0"},
+        }
+
+        reconciled, pruned = _reconcile_genai_speech_output_names(config, actual_outputs)
+
+        encoder_outputs = reconciled["model"]["encoder"]["outputs"]
+        assert "cross_present_key_names" not in encoder_outputs
+        assert "cross_present_value_names" not in encoder_outputs
+        assert len(pruned) == 2
+
+    def test_does_not_mutate_input_config(self):
+        config = _whisper_genai_config(num_layers=2)
+        actual_outputs = {"encoder": {"hidden_states"}}
+
+        _reconcile_genai_speech_output_names(config, actual_outputs)
+
+        assert "cross_present_key_names" in config["model"]["encoder"]["outputs"]
+
+    def test_leaves_sections_without_actual_outputs_untouched(self):
+        config = _whisper_genai_config(num_layers=2)
+
+        reconciled, pruned = _reconcile_genai_speech_output_names(config, {})
+
+        assert pruned == []
+        assert reconciled == config
+
+    def test_handles_config_without_model_section(self):
+        reconciled, pruned = _reconcile_genai_speech_output_names({}, {"encoder": {"hidden_states"}})
+
+        assert pruned == []
+        assert reconciled == {}
+
+
 class TestCompareGeneration:
     """Unit tests for OnnxDiscrepancyCheck.compare_generation."""
 

From e5665729db38869f64425e704e45af4e066a3c17 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= <xadupre@microsoft.com>
Date: Fri, 10 Jul 2026 14:07:45 +0200
Subject: [PATCH 08/21] whister again

---
 olive/passes/onnx/discrepancy_check.py     | 48 ++++++++++++---
 test/passes/onnx/test_discrepancy_check.py | 68 ++++++++++++++--------
 2 files changed, 82 insertions(+), 34 deletions(-)

diff --git a/olive/passes/onnx/discrepancy_check.py b/olive/passes/onnx/discrepancy_check.py
index 85cdf337e9..e8b6953342 100644
--- a/olive/passes/onnx/discrepancy_check.py
+++ b/olive/passes/onnx/discrepancy_check.py
@@ -52,10 +52,17 @@ def _reconcile_genai_speech_output_names(genai_config: dict, actual_outputs: dic
     """Prune Whisper ``genai_config.json`` output names that are absent from the ONNX graphs.
 
     A genai / model-builder version mismatch can leave the ``model.encoder`` / ``model.decoder``
-    ``outputs`` maps referencing tensors (typically the ``present_key_cross_*`` /
-    ``present_value_cross_*`` cross-attention cache) that the actual ONNX graph does not produce,
-    which makes ``onnxruntime_genai.Model`` fail with ``Invalid output name: ...``. Dropping the
-    unmatched entries lets the model load so the discrepancy comparison can still run.
+    ``outputs`` maps referencing tensors that the actual ONNX graph does not produce, which makes
+    ``onnxruntime_genai.Model`` fail with ``Invalid output name: ...``. Dropping such unmatched
+    entries lets the model load so the discrepancy comparison can still run.
+
+    An output is only pruned when it is *not consumed* elsewhere in the pipeline. Whisper wires each
+    KV-cache ``present`` output into the paired ``past`` input of the next step (e.g. the encoder's
+    ``present_key_cross_%d`` output feeds the decoder's ``past_key_cross_%d`` input, and the
+    decoder's ``present_key_self_%d`` output feeds its own ``past_key_self_%d`` input). Removing a
+    consumed output would leave that cache input unwired, so ``onnxruntime_genai`` loads the model
+    but then crashes (segfault) during generation. Such mismatches are left in place so the load
+    fails cleanly with ``Invalid output name`` and the caller can skip the comparison in-process.
 
     Args:
         genai_config: Parsed ``genai_config.json`` contents. Not mutated; a reconciled copy is
@@ -77,6 +84,8 @@ def _reconcile_genai_speech_output_names(genai_config: dict, actual_outputs: dic
     if not isinstance(model_section, dict):
         return reconciled, pruned
 
+    consumed_input_templates = _collect_genai_input_templates(model_section)
+
     for section_name, present in actual_outputs.items():
         section = model_section.get(section_name)
         if not isinstance(section, dict):
@@ -90,15 +99,36 @@ def _reconcile_genai_speech_output_names(genai_config: dict, actual_outputs: dic
             if not isinstance(value, str):
                 continue
             expanded = _expand_genai_output_names(value, num_layers)
-            # Keep the entry only when every concrete output name it references actually exists;
-            # a plain output with no expansion (e.g. logits) is kept when its single name exists.
-            if expanded and not all(name in present for name in expanded):
-                del outputs[key]
-                pruned.append((section_name, key, value))
+            # Keep the entry when every concrete output name it references actually exists; a plain
+            # output with no expansion (e.g. logits) is kept when its single name exists.
+            if not expanded or all(name in present for name in expanded):
+                continue
+            # A ``present`` KV output whose matching ``past`` input is declared is consumed by the
+            # decode loop; pruning it would leave that cache input unwired and crash onnxruntime-genai
+            # at generation time, so leave it and let the load fail cleanly instead.
+            if value.replace("present", "past") in consumed_input_templates:
+                continue
+            del outputs[key]
+            pruned.append((section_name, key, value))
 
     return reconciled, pruned
 
 
+def _collect_genai_input_templates(model_section: dict) -> set:
+    """Collect every declared input name template across all genai_config model sections."""
+    templates = set()
+    for section in model_section.values():
+        if not isinstance(section, dict):
+            continue
+        inputs = section.get("inputs")
+        if not isinstance(inputs, dict):
+            continue
+        for value in inputs.values():
+            if isinstance(value, str):
+                templates.add(value)
+    return templates
+
+
 def _infer_shape(dynamic_shape, known_values=None):
     # Use an empty past-KV cache (past_sequence_length=0) so the discrepancy check is a clean
     # prefill comparison.  The dummy dataloader passes ``past_key_values.<i>.key/value`` tensors,
diff --git a/test/passes/onnx/test_discrepancy_check.py b/test/passes/onnx/test_discrepancy_check.py
index 1798427587..f8e1b3a141 100644
--- a/test/passes/onnx/test_discrepancy_check.py
+++ b/test/passes/onnx/test_discrepancy_check.py
@@ -58,6 +58,13 @@ def _whisper_genai_config(num_layers=2):
             "decoder": {
                 "filename": "decoder_model_merged.onnx",
                 "num_hidden_layers": num_layers,
+                "inputs": {
+                    "input_ids": "input_ids",
+                    "past_key_names": "past_key_self_%d",
+                    "past_value_names": "past_value_self_%d",
+                    "cross_past_key_names": "past_key_cross_%d",
+                    "cross_past_value_names": "past_value_cross_%d",
+                },
                 "outputs": {
                     "logits": "logits",
                     "present_key_names": "present_key_self_%d",
@@ -67,6 +74,7 @@ def _whisper_genai_config(num_layers=2):
             "encoder": {
                 "filename": "encoder_model.onnx",
                 "num_hidden_layers": num_layers,
+                "inputs": {"audio_features": "audio_features"},
                 "outputs": {
                     "encoder_hidden_states": "hidden_states",
                     "cross_present_key_names": "present_key_cross_%d",
@@ -97,9 +105,26 @@ def test_templated_name_with_zero_layers_is_empty(self):
 class TestReconcileGenaiSpeechOutputNames:
     """Unit tests for _reconcile_genai_speech_output_names helper."""
 
-    def test_prunes_output_names_absent_from_graph(self):
+    def test_prunes_absent_output_not_consumed_downstream(self):
+        config = _whisper_genai_config(num_layers=2)
+        # An optional diagnostic output that no input consumes and that the graph does not produce.
+        config["model"]["encoder"]["outputs"]["extra_debug"] = "debug_tensor"
+        actual_outputs = {"encoder": {"hidden_states"}}
+
+        reconciled, pruned = _reconcile_genai_speech_output_names(config, actual_outputs)
+
+        encoder_outputs = reconciled["model"]["encoder"]["outputs"]
+        assert "extra_debug" not in encoder_outputs
+        assert encoder_outputs["encoder_hidden_states"] == "hidden_states"
+        assert {(section, key) for section, key, _ in pruned} == {("encoder", "extra_debug")}
+
+    def test_keeps_absent_but_consumed_cross_kv_outputs(self):
+        # The encoder cross-attention present outputs are missing from the graph but are consumed as
+        # the decoder's cross past inputs, so pruning them would produce a model that segfaults;
+        # they must be left in place so the load fails cleanly instead.
         config = _whisper_genai_config(num_layers=2)
         actual_outputs = {
+            "encoder": {"hidden_states"},
             "decoder": {
                 "logits",
                 "present_key_self_0",
@@ -107,20 +132,26 @@ def test_prunes_output_names_absent_from_graph(self):
                 "present_key_self_1",
                 "present_value_self_1",
             },
-            # The encoder graph does not produce the cross-attention cache outputs.
-            "encoder": {"hidden_states"},
         }
 
         reconciled, pruned = _reconcile_genai_speech_output_names(config, actual_outputs)
 
         encoder_outputs = reconciled["model"]["encoder"]["outputs"]
-        assert "cross_present_key_names" not in encoder_outputs
-        assert "cross_present_value_names" not in encoder_outputs
-        assert encoder_outputs["encoder_hidden_states"] == "hidden_states"
-        assert {(section, key) for section, key, _ in pruned} == {
-            ("encoder", "cross_present_key_names"),
-            ("encoder", "cross_present_value_names"),
-        }
+        assert encoder_outputs["cross_present_key_names"] == "present_key_cross_%d"
+        assert encoder_outputs["cross_present_value_names"] == "present_value_cross_%d"
+        assert pruned == []
+
+    def test_keeps_absent_but_consumed_self_kv_outputs(self):
+        # Self-attention present outputs feed the decoder's own past inputs and must not be pruned.
+        config = _whisper_genai_config(num_layers=2)
+        actual_outputs = {"decoder": {"logits"}}
+
+        reconciled, pruned = _reconcile_genai_speech_output_names(config, actual_outputs)
+
+        decoder_outputs = reconciled["model"]["decoder"]["outputs"]
+        assert decoder_outputs["present_key_names"] == "present_key_self_%d"
+        assert decoder_outputs["present_value_names"] == "present_value_self_%d"
+        assert pruned == []
 
     def test_keeps_outputs_when_all_names_present(self):
         config = _whisper_genai_config(num_layers=2)
@@ -146,27 +177,14 @@ def test_keeps_outputs_when_all_names_present(self):
         assert pruned == []
         assert reconciled == config
 
-    def test_prunes_when_only_some_layer_outputs_missing(self):
-        config = _whisper_genai_config(num_layers=2)
-        # Only layer 0 cross outputs exist; layer 1 is missing, so the whole entry is pruned.
-        actual_outputs = {
-            "encoder": {"hidden_states", "present_key_cross_0", "present_value_cross_0"},
-        }
-
-        reconciled, pruned = _reconcile_genai_speech_output_names(config, actual_outputs)
-
-        encoder_outputs = reconciled["model"]["encoder"]["outputs"]
-        assert "cross_present_key_names" not in encoder_outputs
-        assert "cross_present_value_names" not in encoder_outputs
-        assert len(pruned) == 2
-
     def test_does_not_mutate_input_config(self):
         config = _whisper_genai_config(num_layers=2)
+        config["model"]["encoder"]["outputs"]["extra_debug"] = "debug_tensor"
         actual_outputs = {"encoder": {"hidden_states"}}
 
         _reconcile_genai_speech_output_names(config, actual_outputs)
 
-        assert "cross_present_key_names" in config["model"]["encoder"]["outputs"]
+        assert "extra_debug" in config["model"]["encoder"]["outputs"]
 
     def test_leaves_sections_without_actual_outputs_untouched(self):
         config = _whisper_genai_config(num_layers=2)

From 6d0cc2a0019ab2d714398f0ff33720d6ee36420f Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Fri, 10 Jul 2026 12:27:14 +0000
Subject: [PATCH 09/21] fix: resolve lint issues from code scanning alerts

---
 olive/common/hf/utils.py                   |  2 +-
 olive/passes/onnx/discrepancy_check.py     |  2 +-
 test/common/test_hf.py                     |  4 ----
 test/passes/onnx/test_discrepancy_check.py | 10 +++++-----
 4 files changed, 7 insertions(+), 11 deletions(-)

diff --git a/olive/common/hf/utils.py b/olive/common/hf/utils.py
index 58cd439af0..fec2db0bd0 100644
--- a/olive/common/hf/utils.py
+++ b/olive/common/hf/utils.py
@@ -153,7 +153,7 @@ def _load_test_model(
     ``model_class`` may be an ``AutoModel*`` helper (public ``from_config``) or a concrete architecture
     class such as ``WhisperForConditionalGeneration`` (private ``_from_config``); both are supported.
     """
-    from_config = getattr(model_class, "from_config", None) or model_class._from_config
+    from_config = getattr(model_class, "from_config", None) or getattr(model_class, "_from_config")
     from_config_signature = inspect.signature(from_config)
     accepts_var_keyword = any(
         parameter.kind == inspect.Parameter.VAR_KEYWORD for parameter in from_config_signature.parameters.values()
diff --git a/olive/passes/onnx/discrepancy_check.py b/olive/passes/onnx/discrepancy_check.py
index e8b6953342..ed0bf153b4 100644
--- a/olive/passes/onnx/discrepancy_check.py
+++ b/olive/passes/onnx/discrepancy_check.py
@@ -1270,7 +1270,7 @@ def _load_genai_speech_model(genai_model_path: str):
 
             import tempfile
 
-            temp_dir = tempfile.TemporaryDirectory(prefix="olive_genai_reconciled_")
+            temp_dir = tempfile.TemporaryDirectory(prefix="olive_genai_reconciled_")  # pylint: disable=consider-using-with
             temp_path = Path(temp_dir.name)
             for item in model_dir.iterdir():
                 if item.name == "genai_config.json" or not item.is_file():
diff --git a/test/common/test_hf.py b/test/common/test_hf.py
index 394daa11c9..daf947a9b6 100644
--- a/test/common/test_hf.py
+++ b/test/common/test_hf.py
@@ -28,8 +28,6 @@ def test_apply_test_model_config_reduces_encoder_decoder_layers_consistently():
     """
     from transformers import WhisperConfig
 
-    from olive.common.hf.utils import _apply_test_model_config
-
     reduced = _apply_test_model_config(WhisperConfig(), {"hidden_layers": 2})
     assert reduced.encoder_layers == 2
     assert reduced.decoder_layers == 2
@@ -82,8 +80,6 @@ def test_get_model_class_from_config_resolves_declared_architecture():
 
 
 def test_get_model_class_from_config_returns_none_when_unresolvable():
-    from transformers import BertConfig
-
     from olive.common.hf.utils import get_model_class_from_config
 
     # No architectures declared.
diff --git a/test/passes/onnx/test_discrepancy_check.py b/test/passes/onnx/test_discrepancy_check.py
index f8e1b3a141..67c627ccae 100644
--- a/test/passes/onnx/test_discrepancy_check.py
+++ b/test/passes/onnx/test_discrepancy_check.py
@@ -139,7 +139,7 @@ def test_keeps_absent_but_consumed_cross_kv_outputs(self):
         encoder_outputs = reconciled["model"]["encoder"]["outputs"]
         assert encoder_outputs["cross_present_key_names"] == "present_key_cross_%d"
         assert encoder_outputs["cross_present_value_names"] == "present_value_cross_%d"
-        assert pruned == []
+        assert not pruned
 
     def test_keeps_absent_but_consumed_self_kv_outputs(self):
         # Self-attention present outputs feed the decoder's own past inputs and must not be pruned.
@@ -151,7 +151,7 @@ def test_keeps_absent_but_consumed_self_kv_outputs(self):
         decoder_outputs = reconciled["model"]["decoder"]["outputs"]
         assert decoder_outputs["present_key_names"] == "present_key_self_%d"
         assert decoder_outputs["present_value_names"] == "present_value_self_%d"
-        assert pruned == []
+        assert not pruned
 
     def test_keeps_outputs_when_all_names_present(self):
         config = _whisper_genai_config(num_layers=2)
@@ -174,7 +174,7 @@ def test_keeps_outputs_when_all_names_present(self):
 
         reconciled, pruned = _reconcile_genai_speech_output_names(config, actual_outputs)
 
-        assert pruned == []
+        assert not pruned
         assert reconciled == config
 
     def test_does_not_mutate_input_config(self):
@@ -191,13 +191,13 @@ def test_leaves_sections_without_actual_outputs_untouched(self):
 
         reconciled, pruned = _reconcile_genai_speech_output_names(config, {})
 
-        assert pruned == []
+        assert not pruned
         assert reconciled == config
 
     def test_handles_config_without_model_section(self):
         reconciled, pruned = _reconcile_genai_speech_output_names({}, {"encoder": {"hidden_states"}})
 
-        assert pruned == []
+        assert not pruned
         assert reconciled == {}
 
 

From 44ef5bd8199baaba63f5a55adf8fae6589321204 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= <xadupre@microsoft.com>
Date: Fri, 10 Jul 2026 14:27:33 +0200
Subject: [PATCH 10/21] move imports

---
 olive/passes/onnx/discrepancy_check.py | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/olive/passes/onnx/discrepancy_check.py b/olive/passes/onnx/discrepancy_check.py
index e8b6953342..0529c62ba1 100644
--- a/olive/passes/onnx/discrepancy_check.py
+++ b/olive/passes/onnx/discrepancy_check.py
@@ -2,15 +2,20 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
 # --------------------------------------------------------------------------
+import copy
 import json
 import logging
 import os
 import shutil
 import subprocess
 import time
+from collections import Counter
 from pathlib import Path
 from typing import Optional
 
+import numpy as np
+import onnx
+
 from olive.data.config import DataConfig
 from olive.hardware import AcceleratorSpec
 from olive.hardware.accelerator import Device
@@ -23,8 +28,6 @@
 
 def _json_sanitize(obj):
     """Recursively convert numpy scalars/arrays to native Python types for JSON serialization."""
-    import numpy as np
-
     if isinstance(obj, dict):
         return {key: _json_sanitize(value) for key, value in obj.items()}
     if isinstance(obj, (list, tuple)):
@@ -76,8 +79,6 @@ def _reconcile_genai_speech_output_names(genai_config: dict, actual_outputs: dic
         ``(section, key, template)`` tuples describing each removed entry.
 
     """
-    import copy
-
     reconciled = copy.deepcopy(genai_config)
     pruned = []
     model_section = reconciled.get("model")
@@ -166,10 +167,6 @@ def _infer_onnx_weight_dtype(onnx_model):
     floating-point ONNX TensorProto data type. Returns ``None`` when no
     floating-point initializer is found.
     """
-    from collections import Counter
-
-    import onnx
-
     float_types = {
         onnx.TensorProto.FLOAT,
         onnx.TensorProto.FLOAT16,
@@ -190,7 +187,6 @@ def _infer_onnx_weight_dtype(onnx_model):
 
 def _onnx_dtype_to_torch(onnx_dtype):
     """Map an ONNX TensorProto floating-point data type to a torch dtype."""
-    import onnx
     import torch
 
     mapping = {

From 6c67638163cc1c76062b0f789fa565563544f084 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= <xadupre@microsoft.com>
Date: Fri, 10 Jul 2026 14:48:45 +0200
Subject: [PATCH 11/21] refators

---
 olive/passes/onnx/_genai_speech_worker.py  | 310 +++++++++++++++++++++
 olive/passes/onnx/discrepancy_check.py     | 295 ++++----------------
 test/passes/onnx/test_discrepancy_check.py | 101 +++++--
 3 files changed, 446 insertions(+), 260 deletions(-)
 create mode 100644 olive/passes/onnx/_genai_speech_worker.py

diff --git a/olive/passes/onnx/_genai_speech_worker.py b/olive/passes/onnx/_genai_speech_worker.py
new file mode 100644
index 0000000000..1ba9213453
--- /dev/null
+++ b/olive/passes/onnx/_genai_speech_worker.py
@@ -0,0 +1,310 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+# --------------------------------------------------------------------------
+"""Isolated worker for the Whisper GenAI speech-generation discrepancy comparison.
+
+``onnxruntime_genai`` runs native code that can hard-crash (segfault) for some Whisper builds,
+e.g. on a genai / model-builder version incompatibility. A native crash cannot be caught by a
+Python ``try/except`` in the calling process, so the GenAI generation is executed here in a
+separate process: if it crashes, the parent simply observes a non-zero exit code and degrades
+gracefully instead of taking down the whole optimize workflow.
+
+The module is intentionally self-contained (standard library plus ``onnx`` / ``onnxruntime_genai``
+only, no ``olive`` / ``torch`` / ``transformers`` imports) so it can be launched by file path with a
+fast, side-effect-free startup.
+"""
+
+import json
+import logging
+import os
+import shutil
+import sys
+import tempfile
+import time
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+def expand_output_names(template: str, num_layers: int) -> list:
+    """Expand a genai_config output-name value into the concrete ONNX output names.
+
+    ONNX Runtime GenAI encodes per-layer cache outputs as ``%d`` templates (e.g.
+    ``present_key_cross_%d``) that it expands over the number of layers, while plain outputs
+    such as ``logits`` are used verbatim.
+    """
+    if "%d" in template:
+        return [template % layer for layer in range(num_layers)]
+    return [template]
+
+
+def collect_input_templates(model_section: dict) -> set:
+    """Collect every declared input name template across all genai_config model sections."""
+    templates = set()
+    for section in model_section.values():
+        if not isinstance(section, dict):
+            continue
+        inputs = section.get("inputs")
+        if not isinstance(inputs, dict):
+            continue
+        for value in inputs.values():
+            if isinstance(value, str):
+                templates.add(value)
+    return templates
+
+
+def reconcile_output_names(genai_config: dict, actual_outputs: dict):
+    """Prune Whisper ``genai_config.json`` output names that are absent from the ONNX graphs.
+
+    A genai / model-builder version mismatch can leave the ``model.encoder`` / ``model.decoder``
+    ``outputs`` maps referencing tensors that the actual ONNX graph does not produce, which makes
+    ``onnxruntime_genai.Model`` fail with ``Invalid output name: ...``. Dropping such unmatched
+    entries lets the model load so the discrepancy comparison can still run.
+
+    An output is only pruned when it is *not consumed* elsewhere in the pipeline. Whisper wires each
+    KV-cache ``present`` output into the paired ``past`` input of the next step (e.g. the encoder's
+    ``present_key_cross_%d`` output feeds the decoder's ``past_key_cross_%d`` input, and the
+    decoder's ``present_key_self_%d`` output feeds its own ``past_key_self_%d`` input). Removing a
+    consumed output would leave that cache input unwired, so ``onnxruntime_genai`` loads the model
+    but then crashes during generation. Such mismatches are left in place so the load fails cleanly
+    with ``Invalid output name`` and the caller can skip the comparison instead.
+
+    Args:
+        genai_config: Parsed ``genai_config.json`` contents. Not mutated; a reconciled copy is
+            returned instead.
+        actual_outputs: Mapping of section name (``"encoder"`` / ``"decoder"``) to the set of
+            output names actually present in that section's ONNX graph. Sections missing from the
+            mapping are left untouched.
+
+    Returns:
+        Tuple ``(reconciled_config, pruned)`` where ``pruned`` is a list of
+        ``(section, key, template)`` tuples describing each removed entry.
+
+    """
+    import copy
+
+    reconciled = copy.deepcopy(genai_config)
+    pruned = []
+    model_section = reconciled.get("model")
+    if not isinstance(model_section, dict):
+        return reconciled, pruned
+
+    consumed_input_templates = collect_input_templates(model_section)
+
+    for section_name, present in actual_outputs.items():
+        section = model_section.get(section_name)
+        if not isinstance(section, dict):
+            continue
+        outputs = section.get("outputs")
+        if not isinstance(outputs, dict):
+            continue
+        num_layers = section.get("num_hidden_layers", 0) or 0
+        for key in list(outputs):
+            value = outputs[key]
+            if not isinstance(value, str):
+                continue
+            expanded = expand_output_names(value, num_layers)
+            # Keep the entry when every concrete output name it references actually exists; a plain
+            # output with no expansion (e.g. logits) is kept when its single name exists.
+            if not expanded or all(name in present for name in expanded):
+                continue
+            # A ``present`` KV output whose matching ``past`` input is declared is consumed by the
+            # decode loop; pruning it would leave that cache input unwired and crash onnxruntime-genai
+            # at generation time, so leave it and let the load fail cleanly instead.
+            if value.replace("present", "past") in consumed_input_templates:
+                continue
+            del outputs[key]
+            pruned.append((section_name, key, value))
+
+    return reconciled, pruned
+
+
+def collect_section_outputs(model_dir: Path, genai_config: dict) -> dict:
+    """Read the actual ONNX graph output names for each genai_config model section.
+
+    Returns a mapping of section name (``"encoder"`` / ``"decoder"``) to the set of output names
+    found in that section's ONNX file. Sections whose ONNX file cannot be read are omitted so the
+    reconciliation leaves them untouched.
+    """
+    import onnx
+
+    actual_outputs = {}
+    model_section = genai_config.get("model", {})
+    if not isinstance(model_section, dict):
+        return actual_outputs
+    for section_name in ("encoder", "decoder"):
+        section = model_section.get(section_name)
+        if not isinstance(section, dict):
+            continue
+        filename = section.get("filename")
+        if not filename:
+            continue
+        onnx_path = model_dir / filename
+        if not onnx_path.is_file():
+            continue
+        try:
+            onnx_model = onnx.load(str(onnx_path), load_external_data=False)
+        except Exception as exc:  # pylint: disable=broad-except
+            logger.debug("Could not read ONNX outputs from %s: %s", onnx_path, exc)
+            continue
+        actual_outputs[section_name] = {output.name for output in onnx_model.graph.output}
+    return actual_outputs
+
+
+def load_genai_speech_model(genai_model_path: str):
+    """Load an ``onnxruntime_genai.Model`` for a Whisper model, tolerating output-name mismatches.
+
+    A genai / model-builder version incompatibility can leave ``genai_config.json`` declaring ONNX
+    outputs that the graph does not produce, which makes ``og.Model`` raise ``Invalid output
+    name: ...``. When that happens, the ``genai_config.json`` is reconciled against the real graph
+    outputs in a temporary directory (the large ONNX files are hard-linked, falling back to copies)
+    and the load is retried once.
+
+    Returns a tuple ``(genai_model, temp_dir)`` where ``temp_dir`` is a
+    ``tempfile.TemporaryDirectory`` that the caller must keep alive while the model is in use and
+    clean up afterwards (``None`` when no reconciliation was needed).
+    """
+    import onnxruntime_genai as og
+
+    try:
+        return og.Model(genai_model_path), None
+    except Exception as exc:  # pylint: disable=broad-except
+        if "invalid output name" not in str(exc).lower():
+            raise
+
+        model_dir = Path(genai_model_path)
+        genai_config_path = model_dir / "genai_config.json"
+        if not genai_config_path.is_file():
+            raise
+        with genai_config_path.open() as f:
+            genai_config = json.load(f)
+
+        actual_outputs = collect_section_outputs(model_dir, genai_config)
+        reconciled, pruned = reconcile_output_names(genai_config, actual_outputs)
+        if not pruned:
+            # Nothing safe to reconcile; the failure is unrelated to prunable output names.
+            raise
+
+        logger.warning(
+            "Reconciling Whisper genai_config.json: %d output name(s) declared but absent from the "
+            "ONNX graph were removed so the model can load (%s). This indicates a genai / "
+            "model-builder version mismatch.",
+            len(pruned),
+            ", ".join(f"{section}.{key}={template}" for section, key, template in pruned),
+        )
+
+        temp_dir = tempfile.TemporaryDirectory(prefix="olive_genai_reconciled_")
+        temp_path = Path(temp_dir.name)
+        for item in model_dir.iterdir():
+            if item.name == "genai_config.json" or not item.is_file():
+                continue
+            target = temp_path / item.name
+            try:
+                os.link(item, target)
+            except OSError:
+                shutil.copy2(item, target)
+        with (temp_path / "genai_config.json").open("w") as f:
+            json.dump(reconciled, f, indent=4)
+
+        try:
+            return og.Model(str(temp_path)), temp_dir
+        except Exception:
+            temp_dir.cleanup()
+            raise
+
+
+def whisper_decoder_prompt(genai_model_path: str) -> str:
+    """Build the Whisper decoder prompt string, mirroring olive_evaluator's genai speech path."""
+    genai_config = {}
+    genai_config_path = Path(genai_model_path) / "genai_config.json"
+    if genai_config_path.is_file():
+        with genai_config_path.open() as f:
+            genai_config = json.load(f)
+    # English-only Whisper checkpoints (vocab_size=51864) use a shorter prompt without a
+    # language/task selector.
+    vocab_size = genai_config.get("model", {}).get("vocab_size", 51865)
+    if vocab_size == 51864:
+        prompt_tokens = ["<|startoftranscript|>", "<|notimestamps|>"]
+    else:
+        prompt_tokens = ["<|startoftranscript|>", "<|en|>", "<|transcribe|>", "<|notimestamps|>"]
+    return "".join(prompt_tokens)
+
+
+def generate(genai_model_path: str, wav_path: str, max_new_tokens: int, first_n: int) -> dict:
+    """Run ONNX Runtime GenAI audio transcription and return the decoded tokens and timings.
+
+    Returns a dict with ``genai_tokens`` (list of ints), ``genai_ttft_s`` and ``genai_ttfn_s``.
+    """
+    import onnxruntime_genai as og
+
+    genai_model, genai_temp_dir = load_genai_speech_model(genai_model_path)
+    try:
+        genai_processor = genai_model.create_multimodal_processor()
+        prompt = whisper_decoder_prompt(genai_model_path)
+        with open(wav_path, "rb") as f:
+            wav_bytes = f.read()
+        audios = og.Audios.open_bytes(wav_bytes)
+        inputs = genai_processor([prompt], audios=audios)
+
+        params = og.GeneratorParams(genai_model)
+        params.set_search_options(do_sample=False, max_length=max_new_tokens + 64, min_length=0, batch_size=1)
+        og_generator = og.Generator(genai_model, params)
+        og_generator.set_inputs(inputs)
+
+        genai_ttft = None
+        genai_ttfn = None
+        num_generated = 0
+        start = time.perf_counter()
+        while not og_generator.is_done():
+            og_generator.generate_next_token()
+            num_generated += 1
+            if num_generated == 1:
+                genai_ttft = time.perf_counter() - start
+            if num_generated == first_n:
+                genai_ttfn = time.perf_counter() - start
+            if num_generated >= max_new_tokens:
+                break
+        genai_tokens = list(og_generator.get_sequence(0))
+        del og_generator
+    finally:
+        del genai_model
+        if genai_temp_dir is not None:
+            genai_temp_dir.cleanup()
+
+    return {
+        "genai_tokens": [int(token) for token in genai_tokens],
+        "genai_ttft_s": genai_ttft,
+        "genai_ttfn_s": genai_ttfn,
+    }
+
+
+def main(argv=None) -> int:
+    """Entry point: read a request JSON and write the generation result JSON.
+
+    Usage: ``python _genai_speech_worker.py <request_json_path> <result_json_path>``. The request
+    JSON provides ``genai_model_path``, ``wav_path``, ``max_new_tokens`` and ``first_n``.
+    """
+    logging.basicConfig(level=logging.WARNING)
+    argv = list(sys.argv[1:] if argv is None else argv)
+    if len(argv) != 2:
+        sys.stderr.write("usage: _genai_speech_worker.py <request_json_path> <result_json_path>\n")
+        return 2
+
+    request_path, result_path = argv
+    with open(request_path) as f:
+        request = json.load(f)
+
+    result = generate(
+        genai_model_path=request["genai_model_path"],
+        wav_path=request["wav_path"],
+        max_new_tokens=int(request["max_new_tokens"]),
+        first_n=int(request["first_n"]),
+    )
+    with open(result_path, "w") as f:
+        json.dump(result, f)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/olive/passes/onnx/discrepancy_check.py b/olive/passes/onnx/discrepancy_check.py
index e8b6953342..ffdc9e1ea1 100644
--- a/olive/passes/onnx/discrepancy_check.py
+++ b/olive/passes/onnx/discrepancy_check.py
@@ -4,9 +4,9 @@
 # --------------------------------------------------------------------------
 import json
 import logging
-import os
-import shutil
 import subprocess
+import sys
+import tempfile
 import time
 from pathlib import Path
 from typing import Optional
@@ -16,6 +16,7 @@
 from olive.hardware.accelerator import Device
 from olive.model import ONNXModelHandler
 from olive.passes import Pass
+from olive.passes.onnx import _genai_speech_worker
 from olive.passes.pass_config import BasePassConfig, PassConfigParam
 
 logger = logging.getLogger(__name__)
@@ -37,96 +38,13 @@ def _json_sanitize(obj):
 
 
 def _expand_genai_output_names(template: str, num_layers: int) -> list:
-    """Expand a genai_config output-name value into the concrete ONNX output names.
-
-    ONNX Runtime GenAI encodes per-layer cache outputs as ``%d`` templates (e.g.
-    ``present_key_cross_%d``) that it expands over the number of layers, while plain outputs
-    such as ``logits`` are used verbatim.
-    """
-    if "%d" in template:
-        return [template % layer for layer in range(num_layers)]
-    return [template]
+    """Backwards-compatible alias for the worker helper (see ``_genai_speech_worker``)."""
+    return _genai_speech_worker.expand_output_names(template, num_layers)
 
 
 def _reconcile_genai_speech_output_names(genai_config: dict, actual_outputs: dict):
-    """Prune Whisper ``genai_config.json`` output names that are absent from the ONNX graphs.
-
-    A genai / model-builder version mismatch can leave the ``model.encoder`` / ``model.decoder``
-    ``outputs`` maps referencing tensors that the actual ONNX graph does not produce, which makes
-    ``onnxruntime_genai.Model`` fail with ``Invalid output name: ...``. Dropping such unmatched
-    entries lets the model load so the discrepancy comparison can still run.
-
-    An output is only pruned when it is *not consumed* elsewhere in the pipeline. Whisper wires each
-    KV-cache ``present`` output into the paired ``past`` input of the next step (e.g. the encoder's
-    ``present_key_cross_%d`` output feeds the decoder's ``past_key_cross_%d`` input, and the
-    decoder's ``present_key_self_%d`` output feeds its own ``past_key_self_%d`` input). Removing a
-    consumed output would leave that cache input unwired, so ``onnxruntime_genai`` loads the model
-    but then crashes (segfault) during generation. Such mismatches are left in place so the load
-    fails cleanly with ``Invalid output name`` and the caller can skip the comparison in-process.
-
-    Args:
-        genai_config: Parsed ``genai_config.json`` contents. Not mutated; a reconciled copy is
-            returned instead.
-        actual_outputs: Mapping of section name (``"encoder"`` / ``"decoder"``) to the set of
-            output names actually present in that section's ONNX graph. Sections missing from the
-            mapping are left untouched.
-
-    Returns:
-        Tuple ``(reconciled_config, pruned)`` where ``pruned`` is a list of
-        ``(section, key, template)`` tuples describing each removed entry.
-
-    """
-    import copy
-
-    reconciled = copy.deepcopy(genai_config)
-    pruned = []
-    model_section = reconciled.get("model")
-    if not isinstance(model_section, dict):
-        return reconciled, pruned
-
-    consumed_input_templates = _collect_genai_input_templates(model_section)
-
-    for section_name, present in actual_outputs.items():
-        section = model_section.get(section_name)
-        if not isinstance(section, dict):
-            continue
-        outputs = section.get("outputs")
-        if not isinstance(outputs, dict):
-            continue
-        num_layers = section.get("num_hidden_layers", 0) or 0
-        for key in list(outputs):
-            value = outputs[key]
-            if not isinstance(value, str):
-                continue
-            expanded = _expand_genai_output_names(value, num_layers)
-            # Keep the entry when every concrete output name it references actually exists; a plain
-            # output with no expansion (e.g. logits) is kept when its single name exists.
-            if not expanded or all(name in present for name in expanded):
-                continue
-            # A ``present`` KV output whose matching ``past`` input is declared is consumed by the
-            # decode loop; pruning it would leave that cache input unwired and crash onnxruntime-genai
-            # at generation time, so leave it and let the load fail cleanly instead.
-            if value.replace("present", "past") in consumed_input_templates:
-                continue
-            del outputs[key]
-            pruned.append((section_name, key, value))
-
-    return reconciled, pruned
-
-
-def _collect_genai_input_templates(model_section: dict) -> set:
-    """Collect every declared input name template across all genai_config model sections."""
-    templates = set()
-    for section in model_section.values():
-        if not isinstance(section, dict):
-            continue
-        inputs = section.get("inputs")
-        if not isinstance(inputs, dict):
-            continue
-        for value in inputs.values():
-            if isinstance(value, str):
-                templates.add(value)
-    return templates
+    """Backwards-compatible alias for the worker helper (see ``_genai_speech_worker``)."""
+    return _genai_speech_worker.reconcile_output_names(genai_config, actual_outputs)
 
 
 def _infer_shape(dynamic_shape, known_values=None):
@@ -1168,23 +1086,6 @@ def _audio_to_wav_bytes(audio, sample_rate: int) -> bytes:
         sf.write(buffer, audio, samplerate=sample_rate, format="WAV")
         return buffer.getvalue()
 
-    @staticmethod
-    def _whisper_decoder_prompt(genai_model_path: str) -> str:
-        """Build the Whisper decoder prompt string, mirroring olive_evaluator's genai speech path."""
-        genai_config = {}
-        genai_config_path = Path(genai_model_path) / "genai_config.json"
-        if genai_config_path.is_file():
-            with genai_config_path.open() as f:
-                genai_config = json.load(f)
-        # English-only Whisper checkpoints (vocab_size=51864) use a shorter prompt without a
-        # language/task selector.
-        vocab_size = genai_config.get("model", {}).get("vocab_size", 51865)
-        if vocab_size == 51864:
-            prompt_tokens = ["<|startoftranscript|>", "<|notimestamps|>"]
-        else:
-            prompt_tokens = ["<|startoftranscript|>", "<|en|>", "<|transcribe|>", "<|notimestamps|>"]
-        return "".join(prompt_tokens)
-
     @staticmethod
     def _load_speech_processor(ref_model_path: str, genai_model_path: str, ref_model):
         """Load the audio processor / feature extractor for the reference speech model.
@@ -1225,101 +1126,47 @@ def _load_speech_processor(ref_model_path: str, genai_model_path: str, ref_model
             f"{candidates}. Ensure the reference model directory contains a preprocessor_config.json."
         ) from last_error
 
-    @staticmethod
-    def _load_genai_speech_model(genai_model_path: str):
-        """Load an ``onnxruntime_genai.Model`` for a Whisper model, tolerating output-name mismatches.
-
-        A genai / model-builder version incompatibility can leave ``genai_config.json`` declaring
-        ONNX outputs (typically ``present_key_cross_*``) that the graph does not produce, which makes
-        ``og.Model`` raise ``Invalid output name: ...``. When that happens, the ``genai_config.json``
-        is reconciled against the real graph outputs in a temporary directory (the large ONNX files
-        are hard-linked, falling back to copies) and the load is retried once.
-
-        Returns a tuple ``(genai_model, temp_dir)`` where ``temp_dir`` is a
-        ``tempfile.TemporaryDirectory`` that the caller must keep alive while the model is in use and
-        clean up afterwards (``None`` when no reconciliation was needed).
-        """
-        import onnxruntime_genai as og
-
-        try:
-            return og.Model(genai_model_path), None
-        except Exception as exc:  # pylint: disable=broad-except
-            if "invalid output name" not in str(exc).lower():
-                raise
-
-            model_dir = Path(genai_model_path)
-            genai_config_path = model_dir / "genai_config.json"
-            if not genai_config_path.is_file():
-                raise
-            with genai_config_path.open() as f:
-                genai_config = json.load(f)
-
-            actual_outputs = OnnxDiscrepancyCheck._collect_genai_section_outputs(model_dir, genai_config)
-            reconciled, pruned = _reconcile_genai_speech_output_names(genai_config, actual_outputs)
-            if not pruned:
-                # Nothing to reconcile; the failure is unrelated to stale output names.
-                raise
-
-            logger.warning(
-                "OnnxDiscrepancyCheck reconciling Whisper genai_config.json: %d output name(s) declared "
-                "but absent from the ONNX graph were removed so the model can load (%s). This indicates a "
-                "genai / model-builder version mismatch.",
-                len(pruned),
-                ", ".join(f"{section}.{key}={template}" for section, key, template in pruned),
-            )
-
-            import tempfile
-
-            temp_dir = tempfile.TemporaryDirectory(prefix="olive_genai_reconciled_")
-            temp_path = Path(temp_dir.name)
-            for item in model_dir.iterdir():
-                if item.name == "genai_config.json" or not item.is_file():
-                    continue
-                target = temp_path / item.name
-                try:
-                    os.link(item, target)
-                except OSError:
-                    shutil.copy2(item, target)
-            with (temp_path / "genai_config.json").open("w") as f:
-                json.dump(reconciled, f, indent=4)
-
-            try:
-                return og.Model(str(temp_path)), temp_dir
-            except Exception:
-                temp_dir.cleanup()
-                raise
+    def _run_genai_speech_subprocess(self, genai_model_path, audio, sample_rate, *, max_new_tokens, first_n):
+        """Run the native onnxruntime-genai audio transcription in an isolated subprocess.
 
-    @staticmethod
-    def _collect_genai_section_outputs(model_dir: Path, genai_config: dict) -> dict:
-        """Read the actual ONNX graph output names for each genai_config model section.
-
-        Returns a mapping of section name (``"encoder"`` / ``"decoder"``) to the set of output names
-        found in that section's ONNX file. Sections whose ONNX file cannot be read are omitted so the
-        reconciliation leaves them untouched.
+        Running the GenAI generation out-of-process means a native crash (segfault) surfaces as a
+        non-zero subprocess exit code that raises here, so the caller can degrade gracefully instead
+        of the crash killing the whole optimize process. Returns a dict with ``genai_tokens``,
+        ``genai_ttft_s`` and ``genai_ttfn_s``.
         """
-        import onnx
-
-        actual_outputs = {}
-        model_section = genai_config.get("model", {})
-        if not isinstance(model_section, dict):
-            return actual_outputs
-        for section_name in ("encoder", "decoder"):
-            section = model_section.get(section_name)
-            if not isinstance(section, dict):
-                continue
-            filename = section.get("filename")
-            if not filename:
-                continue
-            onnx_path = model_dir / filename
-            if not onnx_path.is_file():
-                continue
-            try:
-                onnx_model = onnx.load(str(onnx_path), load_external_data=False)
-            except Exception as exc:  # pylint: disable=broad-except
-                logger.debug("Could not read ONNX outputs from %s: %s", onnx_path, exc)
-                continue
-            actual_outputs[section_name] = {output.name for output in onnx_model.graph.output}
-        return actual_outputs
+        worker = Path(_genai_speech_worker.__file__)
+        with tempfile.TemporaryDirectory(prefix="olive_genai_speech_") as work_dir:
+            work_path = Path(work_dir)
+            wav_path = work_path / "audio.wav"
+            wav_path.write_bytes(self._audio_to_wav_bytes(audio, sample_rate))
+            request_path = work_path / "request.json"
+            result_path = work_path / "result.json"
+            with request_path.open("w") as f:
+                json.dump(
+                    {
+                        "genai_model_path": str(genai_model_path),
+                        "wav_path": str(wav_path),
+                        "max_new_tokens": int(max_new_tokens),
+                        "first_n": int(first_n),
+                    },
+                    f,
+                )
+            proc = subprocess.run(
+                [sys.executable, str(worker), str(request_path), str(result_path)],
+                capture_output=True,
+                text=True,
+                check=False,
+            )
+            if proc.returncode != 0 or not result_path.is_file():
+                stderr_tail = (proc.stderr or "").strip()[-2000:]
+                raise RuntimeError(
+                    f"onnxruntime-genai speech generation subprocess failed (exit code "
+                    f"{proc.returncode}). This typically indicates a native crash in onnxruntime-genai "
+                    f"for this Whisper build (e.g. a genai / model-builder version incompatibility). "
+                    f"stderr tail: {stderr_tail}"
+                )
+            with result_path.open() as f:
+                return json.load(f)
 
     def _compare_generation_speech(
         self,
@@ -1340,10 +1187,10 @@ def _compare_generation_speech(
         matches and latency metrics), computed over the full decoder token sequences (both of which
         start with the shared ``<|startoftranscript|>`` preamble).
         """
-        try:
-            import onnxruntime_genai as og
-        except ImportError as exc:
-            raise ImportError("Please install `onnxruntime-genai` to enable generation comparison.") from exc
+        import importlib.util
+
+        if importlib.util.find_spec("onnxruntime_genai") is None:
+            raise ImportError("Please install `onnxruntime-genai` to enable generation comparison.")
 
         import torch
         from transformers import StoppingCriteria, StoppingCriteriaList
@@ -1407,37 +1254,17 @@ def __call__(self, generated_ids, scores, **kwargs) -> bool:
             transformers_ttfn = None
 
         # ---- ONNX Runtime GenAI generation (audio -> decoder tokens) ----
-        genai_model, genai_temp_dir = self._load_genai_speech_model(genai_model_path)
-        try:
-            genai_processor = genai_model.create_multimodal_processor()
-            prompt = self._whisper_decoder_prompt(genai_model_path)
-            audios = og.Audios.open_bytes(self._audio_to_wav_bytes(audio, sample_rate))
-            inputs = genai_processor([prompt], audios=audios)
-
-            params = og.GeneratorParams(genai_model)
-            params.set_search_options(do_sample=False, max_length=max_new_tokens + 64, min_length=0, batch_size=1)
-            generator = og.Generator(genai_model, params)
-            generator.set_inputs(inputs)
-
-            genai_ttft = None
-            genai_ttfn = None
-            num_generated = 0
-            start = time.perf_counter()
-            while not generator.is_done():
-                generator.generate_next_token()
-                num_generated += 1
-                if num_generated == 1:
-                    genai_ttft = time.perf_counter() - start
-                if num_generated == first_n:
-                    genai_ttfn = time.perf_counter() - start
-                if num_generated >= max_new_tokens:
-                    break
-            genai_tokens = list(generator.get_sequence(0))
-            del generator
-        finally:
-            del genai_model
-            if genai_temp_dir is not None:
-                genai_temp_dir.cleanup()
+        # onnxruntime-genai runs native code that can hard-crash (segfault) for some Whisper builds
+        # (e.g. a genai / model-builder version incompatibility). A native crash cannot be caught by
+        # a Python try/except, so the GenAI generation runs in an isolated subprocess: a crash then
+        # surfaces as a non-zero exit code that raises here and is degraded gracefully by the caller,
+        # instead of taking down the whole optimize workflow.
+        gen_result = self._run_genai_speech_subprocess(
+            genai_model_path, audio, sample_rate, max_new_tokens=max_new_tokens, first_n=first_n
+        )
+        genai_tokens = gen_result["genai_tokens"]
+        genai_ttft = gen_result["genai_ttft_s"]
+        genai_ttfn = gen_result["genai_ttfn_s"]
 
         # Both token streams begin with the shared start-of-transcript decoder preamble, so the
         # comparison is over the full decoder sequences (unlike the causal-LM path, which strips a
diff --git a/test/passes/onnx/test_discrepancy_check.py b/test/passes/onnx/test_discrepancy_check.py
index f8e1b3a141..7247eac577 100644
--- a/test/passes/onnx/test_discrepancy_check.py
+++ b/test/passes/onnx/test_discrepancy_check.py
@@ -201,6 +201,73 @@ def test_handles_config_without_model_section(self):
         assert reconciled == {}
 
 
+class TestRunGenaiSpeechSubprocess:
+    """Unit tests for OnnxDiscrepancyCheck._run_genai_speech_subprocess (crash isolation)."""
+
+    def _make_self(self):
+        mock_self = MagicMock()
+        mock_self._audio_to_wav_bytes.return_value = b"RIFF-fake-wav"
+        # Bind the real method to the mock instance.
+        from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck
+
+        mock_self._run_genai_speech_subprocess = OnnxDiscrepancyCheck._run_genai_speech_subprocess.__get__(mock_self)
+        return mock_self
+
+    def test_raises_when_subprocess_crashes(self):
+        import numpy as np
+
+        mock_self = self._make_self()
+        crashed = MagicMock(returncode=-11, stderr="Segmentation fault (core dumped)\n")
+
+        with (
+            patch("olive.passes.onnx.discrepancy_check.subprocess.run", return_value=crashed) as run_mock,
+            pytest.raises(RuntimeError) as exc_info,
+        ):
+            mock_self._run_genai_speech_subprocess(
+                "genai_dir", np.zeros(16000, dtype=np.float32), 16000, max_new_tokens=20, first_n=5
+            )
+
+        run_mock.assert_called_once()
+        assert "exit code -11" in str(exc_info.value)
+
+    def test_returns_result_on_success(self):
+        import json as _json
+
+        import numpy as np
+
+        mock_self = self._make_self()
+        expected = {"genai_tokens": [1, 2, 3], "genai_ttft_s": 0.1, "genai_ttfn_s": 0.2}
+
+        def fake_run(cmd, **kwargs):
+            # cmd = [python, worker, request_path, result_path]; write the result the caller expects.
+            with open(cmd[3], "w") as f:
+                _json.dump(expected, f)
+            return MagicMock(returncode=0, stderr="")
+
+        with patch("olive.passes.onnx.discrepancy_check.subprocess.run", side_effect=fake_run):
+            result = mock_self._run_genai_speech_subprocess(
+                "genai_dir", np.zeros(16000, dtype=np.float32), 16000, max_new_tokens=20, first_n=5
+            )
+
+        assert result == expected
+
+    def test_raises_when_result_missing_despite_zero_exit(self):
+        import numpy as np
+
+        mock_self = self._make_self()
+
+        with (
+            patch(
+                "olive.passes.onnx.discrepancy_check.subprocess.run",
+                return_value=MagicMock(returncode=0, stderr=""),
+            ),
+            pytest.raises(RuntimeError),
+        ):
+            mock_self._run_genai_speech_subprocess(
+                "genai_dir", np.zeros(16000, dtype=np.float32), 16000, max_new_tokens=20, first_n=5
+            )
+
+
 class TestCompareGeneration:
     """Unit tests for OnnxDiscrepancyCheck.compare_generation."""
 
@@ -1045,19 +1112,19 @@ def test_load_or_make_audio_reads_configured_path(self):
         assert audio.shape[0] == 100
 
     def test_whisper_decoder_prompt_multilingual_default(self, tmp_path):
-        from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck
+        from olive.passes.onnx._genai_speech_worker import whisper_decoder_prompt
 
         # No genai_config.json -> multilingual prompt.
-        prompt = OnnxDiscrepancyCheck._whisper_decoder_prompt(str(tmp_path))
+        prompt = whisper_decoder_prompt(str(tmp_path))
         assert prompt == "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>"
 
     def test_whisper_decoder_prompt_english_only(self, tmp_path):
         import json
 
-        from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck
+        from olive.passes.onnx._genai_speech_worker import whisper_decoder_prompt
 
         (tmp_path / "genai_config.json").write_text(json.dumps({"model": {"vocab_size": 51864}}))
-        prompt = OnnxDiscrepancyCheck._whisper_decoder_prompt(str(tmp_path))
+        prompt = whisper_decoder_prompt(str(tmp_path))
         assert prompt == "<|startoftranscript|><|notimestamps|>"
 
     def test_compare_generation_speech_computes_common_prefix(self, tmp_path):
@@ -1080,31 +1147,14 @@ def test_compare_generation_speech_computes_common_prefix(self, tmp_path):
         mock_features.input_features = torch.zeros((1, 80, 3000))
         mock_processor.return_value = mock_features
 
-        # GenAI sequence diverges at the last token (999 vs 300) -> longest common = 6.
+        # GenAI runs in an isolated subprocess; its result is mocked here. The GenAI sequence diverges
+        # at the last token (999 vs 300) -> longest common = 6.
         genai_sequence = [50258, 50259, 50359, 50363, 100, 200, 999]
-        num_new = len(genai_sequence) - 4  # 4 prompt tokens
-
-        mock_generator = MagicMock()
-        counter = {"n": 0}
-
-        def is_done():
-            return counter["n"] >= num_new
-
-        def generate_next_token():
-            counter["n"] += 1
-
-        mock_generator.is_done = is_done
-        mock_generator.generate_next_token = generate_next_token
-        mock_generator.get_sequence.return_value = genai_sequence
-
-        mock_og = MagicMock()
-        mock_og.Generator.return_value = mock_generator
-
-        mock_sf = MagicMock()
+        gen_result = {"genai_tokens": genai_sequence, "genai_ttft_s": 0.01, "genai_ttfn_s": 0.02}
 
         with (
-            patch.dict(sys.modules, {"onnxruntime_genai": mock_og, "soundfile": mock_sf}),
             patch("transformers.AutoProcessor.from_pretrained", return_value=mock_processor),
+            patch.object(OnnxDiscrepancyCheck, "_run_genai_speech_subprocess", return_value=gen_result),
         ):
             pass_instance = OnnxDiscrepancyCheck.__new__(OnnxDiscrepancyCheck)
             result = pass_instance._compare_generation_speech(
@@ -1117,7 +1167,6 @@ def generate_next_token():
         # transformers was driven with audio input_features, not input_ids.
         _, kwargs = ref_model.generate.call_args
         assert "input_features" in kwargs
-        mock_generator.set_inputs.assert_called_once()
         assert result["longest_common_token_sequence"] == 6
         assert result["first_token_matches"] is True
         assert result["second_token_matches"] is True

From 1b649a35a875dd888ba0c61dcbf671602f90856f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= <xadupre@users.noreply.github.com>
Date: Fri, 10 Jul 2026 14:54:11 +0200
Subject: [PATCH 12/21] Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
---
 olive/passes/onnx/discrepancy_check.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/olive/passes/onnx/discrepancy_check.py b/olive/passes/onnx/discrepancy_check.py
index b296bdd6d1..d96406f826 100644
--- a/olive/passes/onnx/discrepancy_check.py
+++ b/olive/passes/onnx/discrepancy_check.py
@@ -614,8 +614,11 @@ def _load_reference_model(self, model: ONNXModelHandler, config: type[BasePassCo
         # The attention implementation is baked into the reference model's config.json
         # (as ``_attn_implementation``) by the SaveTestModelConfig pass, so it is picked up
         # automatically here without needing to pass ``attn_implementation`` explicitly.
-        model_class = get_model_class_from_config(ref_cfg) or AutoModelForCausalLM
-        ref_model = model_class.from_pretrained(ref_path, config=ref_cfg)
+from transformers import AutoModelForSeq2SeqLM
+
+fallback_class = AutoModelForSeq2SeqLM if is_conditional_generation and not is_causal_lm else AutoModelForCausalLM
+model_class = get_model_class_from_config(ref_cfg) or fallback_class
+ref_model = model_class.from_pretrained(ref_path, config=ref_cfg)
         ref_model.eval()
         logger.info(
             "Loaded reference model from %s with attn_implementation=%s",

From 8cc671d3b9a33f438eb7887e500d5b5c93af3e67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= <xadupre@users.noreply.github.com>
Date: Fri, 10 Jul 2026 14:54:24 +0200
Subject: [PATCH 13/21] Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
---
 olive/passes/onnx/discrepancy_check.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/olive/passes/onnx/discrepancy_check.py b/olive/passes/onnx/discrepancy_check.py
index d96406f826..5c69565711 100644
--- a/olive/passes/onnx/discrepancy_check.py
+++ b/olive/passes/onnx/discrepancy_check.py
@@ -946,8 +946,8 @@ def _run_speech_generation_comparison(self, model, config, ref_model, ref_path):
             results["skip_reason"] = "no genai_config.json found for speech model"
             return results
 
-        gen_max_new_tokens = 20 if "first_token_20" in generation_metrics else config.generate_max_new_tokens
-        gen_first_n = 5 if "tf5t" in generation_metrics else config.first_n_tokens_timed
+gen_max_new_tokens = 20
+gen_first_n = 5 if "tf5t" in generation_metrics else config.first_n_tokens_timed
         results["genai_model_path"] = genai_model_path
         try:
             gen_results = self._compare_generation_speech(

From bea0dabef5d0f5913979a184dd03853047c5f9c9 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Fri, 10 Jul 2026 12:54:34 +0000
Subject: [PATCH 14/21] fix: resolve remaining lint warnings

---
 olive/common/hf/utils.py               | 5 ++++-
 olive/passes/onnx/discrepancy_check.py | 4 ----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/olive/common/hf/utils.py b/olive/common/hf/utils.py
index fec2db0bd0..d873a29f10 100644
--- a/olive/common/hf/utils.py
+++ b/olive/common/hf/utils.py
@@ -153,7 +153,10 @@ def _load_test_model(
     ``model_class`` may be an ``AutoModel*`` helper (public ``from_config``) or a concrete architecture
     class such as ``WhisperForConditionalGeneration`` (private ``_from_config``); both are supported.
     """
-    from_config = getattr(model_class, "from_config", None) or getattr(model_class, "_from_config")
+    try:
+        from_config = model_class.from_config
+    except AttributeError:
+        from_config = model_class._from_config  # pylint: disable=protected-access
     from_config_signature = inspect.signature(from_config)
     accepts_var_keyword = any(
         parameter.kind == inspect.Parameter.VAR_KEYWORD for parameter in from_config_signature.parameters.values()
diff --git a/olive/passes/onnx/discrepancy_check.py b/olive/passes/onnx/discrepancy_check.py
index 5c69565711..b4bb28250f 100644
--- a/olive/passes/onnx/discrepancy_check.py
+++ b/olive/passes/onnx/discrepancy_check.py
@@ -1132,8 +1132,6 @@ def _load_or_make_audio(self, config):
         Uses ``config.speech_audio_path`` when set (read as mono); otherwise generates a short
         synthetic signal so the comparison always has an input available.
         """
-        import numpy as np
-
         audio_path = getattr(config, "speech_audio_path", None)
         if audio_path:
             import soundfile as sf
@@ -1296,8 +1294,6 @@ def _collect_genai_section_outputs(model_dir: Path, genai_config: dict) -> dict:
         found in that section's ONNX file. Sections whose ONNX file cannot be read are omitted so the
         reconciliation leaves them untouched.
         """
-        import onnx
-
         actual_outputs = {}
         model_section = genai_config.get("model", {})
         if not isinstance(model_section, dict):

From 6690f4eba36782b81b32e25abe8692dac173fc6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= <xadupre@users.noreply.github.com>
Date: Fri, 10 Jul 2026 14:54:42 +0200
Subject: [PATCH 15/21] Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
---
 olive/passes/onnx/discrepancy_check.py | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/olive/passes/onnx/discrepancy_check.py b/olive/passes/onnx/discrepancy_check.py
index b4bb28250f..593c67b636 100644
--- a/olive/passes/onnx/discrepancy_check.py
+++ b/olive/passes/onnx/discrepancy_check.py
@@ -1418,15 +1418,13 @@ def __call__(self, generated_ids, scores, **kwargs) -> bool:
             genai_ttfn = None
             num_generated = 0
             start = time.perf_counter()
-            while not generator.is_done():
-                generator.generate_next_token()
-                num_generated += 1
-                if num_generated == 1:
-                    genai_ttft = time.perf_counter() - start
-                if num_generated == first_n:
-                    genai_ttfn = time.perf_counter() - start
-                if num_generated >= max_new_tokens:
-                    break
+while num_generated < max_new_tokens and not generator.is_done():
+    generator.generate_next_token()
+    num_generated += 1
+    if num_generated == 1:
+        genai_ttft = time.perf_counter() - start
+    if num_generated == first_n:
+        genai_ttfn = time.perf_counter() - start
             genai_tokens = list(generator.get_sequence(0))
             del generator
         finally:

From 09a1418dec23cb212e0c342997ee9b412b8453ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= <xadupre@microsoft.com>
Date: Fri, 10 Jul 2026 15:41:14 +0200
Subject: [PATCH 16/21] import

---
 olive/passes/onnx/_genai_speech_worker.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/olive/passes/onnx/_genai_speech_worker.py b/olive/passes/onnx/_genai_speech_worker.py
index 1ba9213453..d33b5e3459 100644
--- a/olive/passes/onnx/_genai_speech_worker.py
+++ b/olive/passes/onnx/_genai_speech_worker.py
@@ -15,6 +15,7 @@
 fast, side-effect-free startup.
 """
 
+import copy
 import json
 import logging
 import os
@@ -24,6 +25,9 @@
 import time
 from pathlib import Path
 
+import onnx
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -82,8 +86,6 @@ def reconcile_output_names(genai_config: dict, actual_outputs: dict):
         ``(section, key, template)`` tuples describing each removed entry.
 
     """
-    import copy
-
     reconciled = copy.deepcopy(genai_config)
     pruned = []
     model_section = reconciled.get("model")
@@ -127,8 +129,6 @@ def collect_section_outputs(model_dir: Path, genai_config: dict) -> dict:
     found in that section's ONNX file. Sections whose ONNX file cannot be read are omitted so the
     reconciliation leaves them untouched.
     """
-    import onnx
-
     actual_outputs = {}
     model_section = genai_config.get("model", {})
     if not isinstance(model_section, dict):

From 8baa43a1baf3fdf5959def61b2c44b7c84b44caa Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Fri, 10 Jul 2026 14:10:04 +0000
Subject: [PATCH 17/21] fix: resolve remaining lint issues

---
 olive/passes/onnx/_genai_speech_worker.py  |  3 +--
 olive/passes/onnx/discrepancy_check.py     |  2 --
 test/passes/onnx/test_discrepancy_check.py | 22 ++++++++++++----------
 3 files changed, 13 insertions(+), 14 deletions(-)

diff --git a/olive/passes/onnx/_genai_speech_worker.py b/olive/passes/onnx/_genai_speech_worker.py
index d33b5e3459..44bae1800c 100644
--- a/olive/passes/onnx/_genai_speech_worker.py
+++ b/olive/passes/onnx/_genai_speech_worker.py
@@ -27,7 +27,6 @@
 
 import onnx
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -194,7 +193,7 @@ def load_genai_speech_model(genai_model_path: str):
             ", ".join(f"{section}.{key}={template}" for section, key, template in pruned),
         )
 
-        temp_dir = tempfile.TemporaryDirectory(prefix="olive_genai_reconciled_")
+        temp_dir = tempfile.TemporaryDirectory(prefix="olive_genai_reconciled_")  # pylint: disable=consider-using-with
         temp_path = Path(temp_dir.name)
         for item in model_dir.iterdir():
             if item.name == "genai_config.json" or not item.is_file():
diff --git a/olive/passes/onnx/discrepancy_check.py b/olive/passes/onnx/discrepancy_check.py
index e874e48dbe..055e549370 100644
--- a/olive/passes/onnx/discrepancy_check.py
+++ b/olive/passes/onnx/discrepancy_check.py
@@ -1048,8 +1048,6 @@ def _load_or_make_audio(self, config):
         Uses ``config.speech_audio_path`` when set (read as mono); otherwise generates a short
         synthetic signal so the comparison always has an input available.
         """
-        import numpy as np
-
         audio_path = getattr(config, "speech_audio_path", None)
         if audio_path:
             import soundfile as sf
diff --git a/test/passes/onnx/test_discrepancy_check.py b/test/passes/onnx/test_discrepancy_check.py
index aaca538f36..6e9c206fc7 100644
--- a/test/passes/onnx/test_discrepancy_check.py
+++ b/test/passes/onnx/test_discrepancy_check.py
@@ -207,15 +207,13 @@ class TestRunGenaiSpeechSubprocess:
     def _make_self(self):
         mock_self = MagicMock()
         mock_self._audio_to_wav_bytes.return_value = b"RIFF-fake-wav"
-        # Bind the real method to the mock instance.
-        from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck
-
-        mock_self._run_genai_speech_subprocess = OnnxDiscrepancyCheck._run_genai_speech_subprocess.__get__(mock_self)
         return mock_self
 
     def test_raises_when_subprocess_crashes(self):
         import numpy as np
 
+        from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck
+
         mock_self = self._make_self()
         crashed = MagicMock(returncode=-11, stderr="Segmentation fault (core dumped)\n")
 
@@ -223,8 +221,8 @@ def test_raises_when_subprocess_crashes(self):
             patch("olive.passes.onnx.discrepancy_check.subprocess.run", return_value=crashed) as run_mock,
             pytest.raises(RuntimeError) as exc_info,
         ):
-            mock_self._run_genai_speech_subprocess(
-                "genai_dir", np.zeros(16000, dtype=np.float32), 16000, max_new_tokens=20, first_n=5
+            OnnxDiscrepancyCheck._run_genai_speech_subprocess(
+                mock_self, "genai_dir", np.zeros(16000, dtype=np.float32), 16000, max_new_tokens=20, first_n=5
             )
 
         run_mock.assert_called_once()
@@ -235,6 +233,8 @@ def test_returns_result_on_success(self):
 
         import numpy as np
 
+        from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck
+
         mock_self = self._make_self()
         expected = {"genai_tokens": [1, 2, 3], "genai_ttft_s": 0.1, "genai_ttfn_s": 0.2}
 
@@ -245,8 +245,8 @@ def fake_run(cmd, **kwargs):
             return MagicMock(returncode=0, stderr="")
 
         with patch("olive.passes.onnx.discrepancy_check.subprocess.run", side_effect=fake_run):
-            result = mock_self._run_genai_speech_subprocess(
-                "genai_dir", np.zeros(16000, dtype=np.float32), 16000, max_new_tokens=20, first_n=5
+            result = OnnxDiscrepancyCheck._run_genai_speech_subprocess(
+                mock_self, "genai_dir", np.zeros(16000, dtype=np.float32), 16000, max_new_tokens=20, first_n=5
             )
 
         assert result == expected
@@ -254,6 +254,8 @@ def fake_run(cmd, **kwargs):
     def test_raises_when_result_missing_despite_zero_exit(self):
         import numpy as np
 
+        from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck
+
         mock_self = self._make_self()
 
         with (
@@ -263,8 +265,8 @@ def test_raises_when_result_missing_despite_zero_exit(self):
             ),
             pytest.raises(RuntimeError),
         ):
-            mock_self._run_genai_speech_subprocess(
-                "genai_dir", np.zeros(16000, dtype=np.float32), 16000, max_new_tokens=20, first_n=5
+            OnnxDiscrepancyCheck._run_genai_speech_subprocess(
+                mock_self, "genai_dir", np.zeros(16000, dtype=np.float32), 16000, max_new_tokens=20, first_n=5
             )
 
 

From c8180f2cad7cc016b147c8c37a153be3df0556d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= <xadupre@microsoft.com>
Date: Fri, 10 Jul 2026 17:04:36 +0200
Subject: [PATCH 18/21] improves figures

---
 olive/passes/onnx/discrepancy_check.py     | 79 ++++++++++++++-----
 test/passes/onnx/test_discrepancy_check.py | 90 +++++++++++++++++++++-
 2 files changed, 149 insertions(+), 20 deletions(-)

diff --git a/olive/passes/onnx/discrepancy_check.py b/olive/passes/onnx/discrepancy_check.py
index 055e549370..2bbb8194c7 100644
--- a/olive/passes/onnx/discrepancy_check.py
+++ b/olive/passes/onnx/discrepancy_check.py
@@ -805,7 +805,11 @@ def _surface_generation_metrics(self, config, generation_metrics, gen_results, r
                 _format_seconds(gen_results.get("genai_ttfn_s")),
             )
 
-        if config.min_longest_common_tokens is not None and longest_common < config.min_longest_common_tokens:
+        if (
+            config.min_longest_common_tokens is not None
+            and longest_common is not None
+            and longest_common < config.min_longest_common_tokens
+        ):
             results["status"] = "failed"
             gen_failure = (
                 f"Longest common token sequence length {longest_common} is below "
@@ -892,6 +896,16 @@ def _run_speech_generation_comparison(self, model, config, ref_model, ref_path):
         # For speech models every generation metric is derived from the audio comparison, so always
         # surface first_token_20 alongside any explicitly requested timing metrics.
         self._surface_generation_metrics(config, generation_metrics | {"first_token_20"}, gen_results, results)
+        # A GenAI failure does not discard the transformers figures: surface them and record why the
+        # GenAI-vs-transformers comparison could not be completed.
+        if gen_results.get("genai_error"):
+            results["genai_generation_error"] = gen_results["genai_error"]
+            logger.warning(
+                "OnnxDiscrepancyCheck speech generation: reported transformers-only metrics; the "
+                "GenAI comparison was skipped (%s). This is typically an onnxruntime-genai / "
+                "model-builder version incompatibility for the Whisper GenAI model.",
+                gen_results["genai_error"],
+            )
         return results
 
     def _run_llama_cpp_comparison(
@@ -1252,26 +1266,52 @@ def __call__(self, generated_ids, scores, **kwargs) -> bool:
         # onnxruntime-genai runs native code that can hard-crash (segfault) for some Whisper builds
         # (e.g. a genai / model-builder version incompatibility). A native crash cannot be caught by
         # a Python try/except, so the GenAI generation runs in an isolated subprocess: a crash then
-        # surfaces as a non-zero exit code that raises here and is degraded gracefully by the caller,
-        # instead of taking down the whole optimize workflow.
-        gen_result = self._run_genai_speech_subprocess(
-            genai_model_path, audio, sample_rate, max_new_tokens=max_new_tokens, first_n=first_n
-        )
-        genai_tokens = gen_result["genai_tokens"]
-        genai_ttft = gen_result["genai_ttft_s"]
-        genai_ttfn = gen_result["genai_ttfn_s"]
-
-        # Both token streams begin with the shared start-of-transcript decoder preamble, so the
-        # comparison is over the full decoder sequences (unlike the causal-LM path, which strips a
-        # known text prompt first).
-        longest_common = _longest_common_token_sequence(transformers_tokens, genai_tokens)
+        # surfaces as a non-zero exit code that raises here.
+        #
+        # The transformers metrics above are already computed, so a GenAI failure must NOT discard
+        # them: degrade gracefully to a transformers-only result (GenAI/comparison fields left None
+        # and a ``genai_error`` recorded) so the report still surfaces the transformers figures.
+        genai_error = None
+        try:
+            gen_result = self._run_genai_speech_subprocess(
+                genai_model_path, audio, sample_rate, max_new_tokens=max_new_tokens, first_n=first_n
+            )
+            genai_tokens = gen_result["genai_tokens"]
+            genai_ttft = gen_result["genai_ttft_s"]
+            genai_ttfn = gen_result["genai_ttfn_s"]
+        except Exception as exc:  # pylint: disable=broad-except
+            genai_error = str(exc)
+            genai_tokens = []
+            genai_ttft = None
+            genai_ttfn = None
+            logger.warning(
+                "OnnxDiscrepancyCheck speech generation: onnxruntime-genai generation failed (%s); "
+                "reporting transformers-only generation metrics.",
+                exc,
+            )
 
         transformers_first_token = transformers_tokens[0] if transformers_tokens else None
-        genai_first_token = genai_tokens[0] if genai_tokens else None
-        first_token_matches = transformers_first_token is not None and transformers_first_token == genai_first_token
         transformers_second_token = transformers_tokens[1] if len(transformers_tokens) > 1 else None
-        genai_second_token = genai_tokens[1] if len(genai_tokens) > 1 else None
-        second_token_matches = transformers_second_token is not None and transformers_second_token == genai_second_token
+
+        if genai_error is None:
+            # Both token streams begin with the shared start-of-transcript decoder preamble, so the
+            # comparison is over the full decoder sequences (unlike the causal-LM path, which strips a
+            # known text prompt first).
+            longest_common = _longest_common_token_sequence(transformers_tokens, genai_tokens)
+            genai_first_token = genai_tokens[0] if genai_tokens else None
+            first_token_matches = transformers_first_token is not None and transformers_first_token == genai_first_token
+            genai_second_token = genai_tokens[1] if len(genai_tokens) > 1 else None
+            second_token_matches = (
+                transformers_second_token is not None and transformers_second_token == genai_second_token
+            )
+        else:
+            # GenAI unavailable: no comparison is possible, so leave the comparison fields None so the
+            # threshold check is skipped and only the transformers figures are surfaced.
+            longest_common = None
+            genai_first_token = None
+            first_token_matches = None
+            genai_second_token = None
+            second_token_matches = None
 
         gen_results = {
             "longest_common_token_sequence": longest_common,
@@ -1287,10 +1327,11 @@ def __call__(self, generated_ids, scores, **kwargs) -> bool:
             "genai_ttft_s": genai_ttft,
             "genai_ttfn_s": genai_ttfn,
             "transformers_warmup_s": warmup_time,
+            "genai_error": genai_error,
         }
         logger.info(
             "OnnxDiscrepancyCheck speech generation comparison: transformers_len=%d, genai_len=%d, "
-            "longest_common_token_sequence=%d, first_token_matches=%s",
+            "longest_common_token_sequence=%s, first_token_matches=%s",
             len(transformers_tokens),
             len(genai_tokens),
             longest_common,
diff --git a/test/passes/onnx/test_discrepancy_check.py b/test/passes/onnx/test_discrepancy_check.py
index 6e9c206fc7..b4f2621e5d 100644
--- a/test/passes/onnx/test_discrepancy_check.py
+++ b/test/passes/onnx/test_discrepancy_check.py
@@ -1174,7 +1174,95 @@ def test_compare_generation_speech_computes_common_prefix(self, tmp_path):
         assert result["second_token_matches"] is True
         assert result["genai_first_token"] == 50258
 
-    def test_run_speech_generation_comparison_marks_skipped_without_genai(self):
+    def test_compare_generation_speech_reports_transformers_only_on_genai_failure(self, tmp_path):
+        import torch
+
+        from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck
+
+        config = MagicMock()
+        config.speech_audio_path = None
+        config.generate_max_new_tokens = 10
+        config.first_n_tokens_timed = 5
+
+        ref_model = self._speech_ref_model()
+        ref_model.generate.return_value = torch.tensor([[50258, 50259, 50359, 50363, 100, 200, 300]])
+
+        mock_processor = MagicMock()
+        mock_features = MagicMock()
+        mock_features.input_features = torch.zeros((1, 80, 3000))
+        mock_processor.return_value = mock_features
+
+        # A GenAI subprocess failure (e.g. native crash / version mismatch) must NOT discard the
+        # transformers metrics that were already computed.
+        with (
+            patch("transformers.AutoProcessor.from_pretrained", return_value=mock_processor),
+            patch.object(
+                OnnxDiscrepancyCheck,
+                "_run_genai_speech_subprocess",
+                side_effect=RuntimeError("Invalid output name: present_key_cross_2"),
+            ),
+        ):
+            pass_instance = OnnxDiscrepancyCheck.__new__(OnnxDiscrepancyCheck)
+            result = pass_instance._compare_generation_speech(
+                config,
+                ref_model,
+                ref_model_path=str(tmp_path),
+                genai_model_path=str(tmp_path),
+            )
+
+        # transformers figures are present.
+        assert result["transformers_first_token"] == 50258
+        assert result["transformers_second_token"] == 50259
+        assert result["transformers_ttft_s"] is not None
+        # GenAI/comparison fields are left None and the failure is recorded.
+        assert "present_key_cross_2" in result["genai_error"]
+        assert result["longest_common_token_sequence"] is None
+        assert result["genai_first_token"] is None
+        assert result["first_token_matches"] is None
+
+    def test_run_speech_generation_comparison_surfaces_transformers_on_genai_failure(self):
+        from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck
+
+        config = MagicMock()
+        config.test_metrics = None
+        config.genai_model_path = "/some/genai/dir"
+        config.generate_max_new_tokens = 10
+        config.first_n_tokens_timed = 5
+        config.min_longest_common_tokens = 4
+
+        # Partial (transformers-only) generation result produced when GenAI fails.
+        gen_results = {
+            "longest_common_token_sequence": None,
+            "first_n_tokens_timed": 5,
+            "transformers_first_token": 50258,
+            "genai_first_token": None,
+            "first_token_matches": None,
+            "transformers_second_token": 50259,
+            "genai_second_token": None,
+            "second_token_matches": None,
+            "transformers_ttft_s": 0.01,
+            "transformers_ttfn_s": 0.02,
+            "genai_ttft_s": None,
+            "genai_ttfn_s": None,
+            "transformers_warmup_s": 0.05,
+            "genai_error": "Invalid output name: present_key_cross_2",
+        }
+
+        model = MagicMock()
+        pass_instance = OnnxDiscrepancyCheck.__new__(OnnxDiscrepancyCheck)
+        with (
+            patch.object(OnnxDiscrepancyCheck, "_resolve_genai_model_path", return_value="/some/genai/dir"),
+            patch.object(OnnxDiscrepancyCheck, "_compare_generation_speech", return_value=gen_results),
+        ):
+            results = pass_instance._run_speech_generation_comparison(model, config, MagicMock(), "ref_path")
+
+        # Status is not skipped: transformers figures are surfaced despite the GenAI failure.
+        assert results["status"] == "passed"
+        assert results["transformers_first_token"] == 50258
+        assert results["transformers_ttft_s"] == 0.01
+        assert "present_key_cross_2" in results["genai_generation_error"]
+        # A missing longest-common comparison must not trip the min-longest-common threshold check.
+        assert "failures" not in results
         from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck
 
         config = MagicMock()

From d72c8d2233def4d31da78c2bdd74daee4d8134d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= <xadupre@users.noreply.github.com>
Date: Fri, 10 Jul 2026 17:18:43 +0200
Subject: [PATCH 19/21] Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
---
 olive/passes/onnx/discrepancy_check.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/olive/passes/onnx/discrepancy_check.py b/olive/passes/onnx/discrepancy_check.py
index 2bbb8194c7..d5b59d50e2 100644
--- a/olive/passes/onnx/discrepancy_check.py
+++ b/olive/passes/onnx/discrepancy_check.py
@@ -406,6 +406,14 @@ def _run_for_config(
         # Encoder-decoder speech models (e.g. Whisper) are exported as composite encoder/decoder
         # graphs, so the single-graph logits/MAE/speedup comparison does not apply. Run only the
         # audio-based generation comparison for them.
+        from olive.model import CompositeModelHandler
+
+        if isinstance(model, CompositeModelHandler) and not self._is_speech_seq2seq(ref_model):
+            raise ValueError(
+                "OnnxDiscrepancyCheck only supports composite ONNX models for encoder-decoder speech models "
+                "(e.g. Whisper)."
+            )
+
         if self._is_speech_seq2seq(ref_model):
             logger.info(
                 "OnnxDiscrepancyCheck detected an encoder-decoder speech model (%s); "

From dd58836af91f8624c1f9565cba82697660d11add Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= <xadupre@users.noreply.github.com>
Date: Fri, 10 Jul 2026 17:19:00 +0200
Subject: [PATCH 20/21] Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
---
 test/passes/onnx/test_discrepancy_check.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/passes/onnx/test_discrepancy_check.py b/test/passes/onnx/test_discrepancy_check.py
index b4f2621e5d..5e65fa7d86 100644
--- a/test/passes/onnx/test_discrepancy_check.py
+++ b/test/passes/onnx/test_discrepancy_check.py
@@ -1263,6 +1263,8 @@ def test_run_speech_generation_comparison_surfaces_transformers_on_genai_failure
         assert "present_key_cross_2" in results["genai_generation_error"]
         # A missing longest-common comparison must not trip the min-longest-common threshold check.
         assert "failures" not in results
+
+    def test_run_speech_generation_comparison_skips_when_no_genai_model(self):
         from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck
 
         config = MagicMock()
@@ -1280,7 +1282,6 @@ def test_run_speech_generation_comparison_surfaces_transformers_on_genai_failure
             results = pass_instance._run_speech_generation_comparison(model, config, MagicMock(), "ref_path")
         assert results["model_kind"] == "speech-seq2seq"
         assert results["status"] == "skipped"
-
     def test_run_speech_generation_comparison_degrades_on_genai_error(self):
         from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck
 

From 59c3c6d7870ec3cffec7512c18fe2b6bd9870799 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Fri, 10 Jul 2026 15:58:08 +0000
Subject: [PATCH 21/21] fix: address remaining ruff-format lint in discrepancy
 check tests

---
 test/passes/onnx/test_discrepancy_check.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/passes/onnx/test_discrepancy_check.py b/test/passes/onnx/test_discrepancy_check.py
index 5e65fa7d86..4ff24aad9e 100644
--- a/test/passes/onnx/test_discrepancy_check.py
+++ b/test/passes/onnx/test_discrepancy_check.py
@@ -1282,6 +1282,7 @@ def test_run_speech_generation_comparison_skips_when_no_genai_model(self):
             results = pass_instance._run_speech_generation_comparison(model, config, MagicMock(), "ref_path")
         assert results["model_kind"] == "speech-seq2seq"
         assert results["status"] == "skipped"
+
     def test_run_speech_generation_comparison_degrades_on_genai_error(self):
         from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck