OpenVINO HF Exporter

docs/source/en/exporters.md

@@ -43,6 +43,7 @@ removals as we follow upstream.
 | ---------------------- | -------------------------- | --------------------------------------------- |
 | [`DynamoExporter`]     | `ExportedProgram`          | Any PyTorch runtime, AOT compilation          |
 | [`OnnxExporter`]       | `ONNXProgram`              | Any ONNX runtime (ORT, TensorRT, OpenVINO, …) |
+| [`OpenVINOExporter`]   | `openvino.Model`           | OpenVINO runtime (Intel CPU/GPU/NPU)          |
 | [`ExecutorchExporter`] | `ExecutorchProgramManager` | Mobile and edge devices (ExecuTorch)          |
 
 [`AutoHfExporter`] picks the right exporter from a config and [`AutoExportConfig`] picks the right
@@ -72,6 +73,13 @@ pip install transformers "torch==2.12.0" "onnx==1.21.0" "onnxscript==0.7.0" onnx
 pip install transformers "torch==2.12.0" "executorch==1.3.1"
 ```
 
+</hfoption>
+<hfoption id="OpenVINO">
+
+```bash
+pip install transformers "torch==2.12.0" "openvino==2025.0.0"
+```
+
 </hfoption>
 </hfoptions>
 
@@ -156,6 +164,30 @@ method = program.load_method("forward")
 outputs = method.execute(list(inputs.values()))
 ```
 
+</hfoption>
+<hfoption id="OpenVINO">
+
+```python
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers.exporters import OpenVINOExporter, OpenVINOConfig
+
+model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-0.6B")
+tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B")
+inputs = tokenizer("Hello, world!", return_tensors="pt")
+
+exporter = OpenVINOExporter()
+config = OpenVINOConfig(dynamic=True)
+ov_model = exporter.export(model, inputs, config=config)
+
+ov_model.save("model.xml")
+
+# compile and run on CPU (or "GPU" / "NPU" if available)
+import openvino as ov
+compiled = ov.Core().compile_model(ov_model, "CPU")
+ov_inputs = {k: v.numpy() for k, v in inputs.items()}
+outputs = compiled(ov_inputs)
+```
+
 </hfoption>
 </hfoptions>
 
@@ -251,6 +283,33 @@ config = ExecutorchConfig(
 et_program = exporter.export(model, inputs, config=config)
 ```
 
+</hfoption>
+<hfoption id="OpenVINO">
+
+```python
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers.exporters import OpenVINOExporter, OpenVINOConfig
+
+model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-0.6B")
+tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B")
+inputs = tokenizer(["Hello, world!", "Hi"], padding=True, return_tensors="pt")
+
+batch = torch.export.Dim("batch", min=1, max=32)
+seq = torch.export.Dim("seq", min=1, max=2048)
+
+exporter = OpenVINOExporter()
+config = OpenVINOConfig(
+    dynamic_shapes={"input_ids": {0: batch, 1: seq}, "attention_mask": {0: batch, 1: seq}},
+    # Emit data-dependent shape guards as runtime asserts instead of failing the export when a
+    # guard wouldn't hold across the explicit symbolic range — most LLMs need this under fine-grained
+    # ``Dim(min=, max=)`` bounds. Not needed with ``dynamic=True`` / ``Dim.AUTO``, where torch.export
+    # infers shape relations instead of verifying them against user-stated bounds.
+    prefer_deferred_runtime_asserts_over_guards=True,
+)
+ov_model = exporter.export(model, inputs, config=config)
+```
+
 </hfoption>
 </hfoptions>
 
@@ -325,6 +384,25 @@ components = exporter.export_for_generation(model, inputs, config=config)
 # components = {"image_encoder": ExecutorchProgramManager, "language_model": ..., "lm_head": ..., "decode": ...}
 ```
 
+</hfoption>
+<hfoption id="OpenVINO">
+
+```python
+from transformers import AutoModelForImageTextToText, AutoProcessor
+from transformers.exporters import OpenVINOExporter, OpenVINOConfig
+
+model = AutoModelForImageTextToText.from_pretrained("Qwen/Qwen2-VL-2B-Instruct")
+processor = AutoProcessor.from_pretrained("Qwen/Qwen2-VL-2B-Instruct")
+messages = [{"role": "user", "content": [{"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"}, {"type": "text", "text": "Describe this image."}]}]
+text = processor.apply_chat_template(messages, add_generation_prompt=True, tokenize=False)
+inputs = processor(text=text, images=messages[0]["content"][0]["url"], return_tensors="pt").to(model.device)
+
+exporter = OpenVINOExporter()
+config = OpenVINOConfig(dynamic=True)
+components = exporter.export_for_generation(model, inputs, config=config)
+# components = {"image_encoder": openvino.Model, "language_model": openvino.Model, "lm_head": openvino.Model, "decode": openvino.Model}
+```
+
 </hfoption>
 </hfoptions>
 
@@ -380,9 +458,7 @@ visible from the public `export()` API, but the most common things to know:
 - `grouped_mm` traces fine through `DynamoExporter` and is auto-translated for `OnnxExporter`;
   for `ExecutorchExporter` with the XNNPACK backend, the exporter swaps MoE experts to
   `batched_mm` because XNNPACK has no `_grouped_mm.out` kernel.
-- A short list of models (`EXPORT_SKIP_MODEL_CLASSES`) is skipped from the export sweep when
-  the model itself is fundamentally non-exportable; each entry carries a TODO with the
-  model-side change needed.
+- Not every architecture exports cleanly yet — a few hit data-dependent control flow that can't be vectorised, or exceed practical export time under dynamic shapes. When that happens the failure surfaces at `export()` time with a concrete error, not silently.
 
 <details>
 <summary>Export pipeline — internals (per-backend stages and how to extend)</summary>
@@ -466,22 +542,6 @@ The split is intentional:
   translation, an ORT validation quirk, an FX decomposition that emits a dead op. Keep the
   workaround in the exporter and the modeling code stays clean.
 
-### Known upstream workarounds
-
-A small number of model classes hit confirmed bugs in `onnxscript`'s graph optimizer
-(constant folding crashing on `SplitToSequence`, FPN initialisers being dropped). For those,
-ONNX optimisation is selectively disabled via
-[`ONNX_DISABLE_OPTIMIZE_MODEL_CLASSES`](https://github.com/huggingface/transformers/blob/main/tests/exporters/test_utils.py)
-in the test suite — each entry is annotated with the upstream issue it works around. This
-list is **expected to shrink** as upstream bugs land; it is not an extension point for
-arbitrary skipping, and new entries should reference a specific upstream bug.
-
-A second list, [`EXPORT_SKIP_MODEL_CLASSES`](https://github.com/huggingface/transformers/blob/main/tests/exporters/test_utils.py),
-opts a handful of model classes out of the entire export sweep when the model itself is
-fundamentally non-exportable as-is (data-dependent control flow that can't be vectorised,
-modules treated as forward arguments, …). Same expectations: every entry carries a TODO
-naming the underlying model change needed; the list should shrink, not grow.
-
 </details>
 
 ## API reference

docs/source/en/main_classes/exporters.md

@@ -47,3 +47,7 @@ Learn how to use the built-in exporters in the [Exporters](../exporters) guide.
 ## ExecutorchConfig
 
 [[autodoc]] exporters.configs.ExecutorchConfig
+
+## OpenVINOConfig
+
+[[autodoc]] exporters.configs.OpenVINOConfig

src/transformers/cache_utils.py

@@ -916,8 +916,10 @@ def lazy_initialization(
             self.is_conv_states_initialized = True
 
         if recurrent_states is not None:
-            # The shape is always static, so we init as such
-            self.recurrent_states = torch.zeros_like(recurrent_states, dtype=self.dtype, device=self.device)
+            # The shape is always static, so we init as such. Preserve the recurrent tensor's OWN
+            # dtype (not `self.dtype`, which follows the conv state): models like recurrent_gemma
+            # accumulate the recurrence in float32 while the conv state stays in the compute dtype.
+            self.recurrent_states = torch.zeros_like(recurrent_states)
             # Mark as static address to be able to use cudagraphs
             if not is_torchdynamo_compiling():
                 torch._dynamo.mark_static_address(self.recurrent_states)

src/transformers/exporters/__init__.py

@@ -14,7 +14,8 @@
 # limitations under the License.
 from .auto import AutoExportConfig, AutoHfExporter, get_hf_exporter, register_export_config, register_exporter
 from .base import HfExporter
-from .configs import DynamoConfig, ExecutorchConfig, ExportConfigMixin, ExportFormat, OnnxConfig
+from .configs import DynamoConfig, ExecutorchConfig, ExportConfigMixin, ExportFormat, OnnxConfig, OpenVINOConfig
 from .exporter_dynamo import DynamoExporter
 from .exporter_executorch import ExecutorchExporter
 from .exporter_onnx import OnnxExporter
+from .exporter_openvino import OpenVINOExporter

src/transformers/exporters/auto.py

@@ -22,16 +22,19 @@
 from .exporter_dynamo import DynamoConfig, DynamoExporter
 from .exporter_executorch import ExecutorchConfig, ExecutorchExporter
 from .exporter_onnx import OnnxConfig, OnnxExporter
+from .exporter_openvino import OpenVINOConfig, OpenVINOExporter
 
 
 AUTO_EXPORTER_MAPPING = {
     "executorch": ExecutorchExporter,
+    "openvino": OpenVINOExporter,
     "dynamo": DynamoExporter,
     "onnx": OnnxExporter,
 }
 
 AUTO_EXPORT_CONFIG_MAPPING = {
     "executorch": ExecutorchConfig,
+    "openvino": OpenVINOConfig,
     "dynamo": DynamoConfig,
     "onnx": OnnxConfig,
 }
@@ -69,7 +72,7 @@ def from_dict(cls, export_config_dict: dict):
 
 class AutoHfExporter:
     """
-    The Auto-HF expoerter class that takes care of automatically instantiating to the correct
+    The Auto-HF exporter class that takes care of automatically instantiating to the correct
     `HfExporter` given the `ExportConfig`.
     """
 

src/transformers/exporters/configs.py

@@ -28,6 +28,7 @@ class ExportFormat(Enum):
     """Identifies the export backend. Stored in [`ExportConfigMixin`] for serialisation round-trips."""
 
     EXECUTORCH = "executorch"
+    OPENVINO = "openvino"
     DYNAMO = "dynamo"
     ONNX = "onnx"
 
@@ -99,8 +100,8 @@ class DynamoConfig(ExportConfigMixin):
     """
 
     export_format: ExportFormat = ExportFormat.DYNAMO
-    dynamic: bool = False
 
+    dynamic: bool = False
     strict: bool = False
     dynamic_shapes: dict[str, Any] | None = None
     prefer_deferred_runtime_asserts_over_guards: bool = False
@@ -141,7 +142,6 @@ class OnnxConfig(DynamoConfig):
     export_format: ExportFormat = ExportFormat.ONNX
 
     output_path: str | PathLike | None = None
-    dynamic_shapes: dict[str, Any] | None = None
     opset_version: int | None = None
     external_data: bool = True
     optimize: bool = True
@@ -168,3 +168,33 @@ class ExecutorchConfig(DynamoConfig):
     export_format: ExportFormat = ExportFormat.EXECUTORCH
 
     backend: str = "xnnpack"
+
+
+@dataclass
+class OpenVINOConfig(DynamoConfig):
+    """
+    Configuration class for exporting models to OpenVINO IR via ``openvino.convert_model``.
+
+    Inherits all fields from [`DynamoConfig`] (`dynamic`, `strict`, `dynamic_shapes`,
+    `prefer_deferred_runtime_asserts_over_guards`).
+
+    Args:
+        output_path (`str` or `PathLike`, *optional*):
+            Output path for the `.xml` file (the matching `.bin` is written alongside). When
+            `None` (default) the converted model is kept in memory as an ``openvino.Model``.
+        compress_to_fp16 (`bool`, *optional*, defaults to `True`):
+            Compress floating-point weights to FP16 when saving — halves on-disk size with
+            negligible accuracy impact on most models. Only applied when ``output_path`` is set.
+        stateful (`bool`, *optional*, defaults to `True`):
+            Fold round-tripped state tensors (KV cache, SSM states, …) into internal OV
+            variables (``ReadValue``/``Assign``). The runtime then carries state across
+            ``infer()`` calls instead of marshalling cache tensors through inputs/outputs on
+            every step, and a fused ``beam_idx`` input reorders state in-graph for beam search.
+            No-op for models without round-tripped state (encoders, prefill-only exports).
+    """
+
+    export_format: ExportFormat = ExportFormat.OPENVINO
+
+    output_path: str | PathLike | None = None
+    compress_to_fp16: bool = True
+    stateful: bool = True

src/transformers/exporters/exporter_dynamo.py

@@ -43,6 +43,7 @@
 import importlib
 import inspect
 import sys
+import types
 from collections.abc import MutableMapping
 from contextlib import contextmanager
 from typing import Any
@@ -351,8 +352,8 @@ def _to_batched(t):
         enable_gqa=getattr(self, "num_key_value_heads", self.num_heads) != self.num_heads,
     )
 
-    # (n_seg, heads, seg_len, dim) → (n_seg, seg_len, heads, dim) → (seq, heads*dim)
-    attn_output = attn_output.transpose(1, 2).reshape(seq_length, -1).contiguous()
+    # (n_seg, heads, seg_len, dim) → (n_seg, seg_len, heads, dim) → (seq, heads*dim).
+    attn_output = attn_output.transpose(1, 2).reshape(seq_length, -1)
     out_proj = self.proj if hasattr(self, "proj") else self.out_proj
     attn_output = out_proj(attn_output)
 
@@ -382,6 +383,8 @@ def _to_batched(t):
     "transformers.models.glm_image.modeling_glm_image.GlmImageVisionAttention.forward",
     # Separate `.q` / `.k` / `.v` + single rotary tensor + `.proj`
     "transformers.models.qwen2_5_omni.modeling_qwen2_5_omni.Qwen2_5OmniVisionAttention.forward",
+    # Separate `q_proj`/`k_proj`/`v_proj` + `(cos, sin)` rotary + `.proj` (single return)
+    "transformers.models.kimi_k25.modeling_kimi_k25.Kimi_K25VisionAttention.forward",
     # Separate `_proj` + `(cos, sin)` rotary + `.out_proj` (tuple return)
     "transformers.models.video_llama_3.modeling_video_llama_3.VideoLlama3VisionAttention.forward",
     "transformers.models.paddleocr_vl.modeling_paddleocr_vl.PaddleOCRVisionAttention.forward",
@@ -449,6 +452,22 @@ def _flatten_to_context(obj: Any, tensors: list) -> Any:
     if isinstance(obj, torch.layout):
         return {"_t": "layout", "n": str(obj).removeprefix("torch.")}
     if isinstance(obj, (torch.SymInt, torch.SymFloat, torch.SymBool)):
+        # A Sym* that has already specialized to a concrete constant is not a genuine dynamic
+        # graph output — bake it as a plain scalar instead of a leaf. Leaving it as a leaf makes
+        # flatten non-deterministic: the same field can be a constant-valued SymInt at one trace
+        # point (the dynamo out_spec capture) and an already-materialized python scalar at another
+        # (the aot-decomposition retrace), so the leaf count flips and `treespec.unflatten` fails
+        # with an off-by-one. deepseek_v4 hits this via two sibling cache layers
+        # (DeepseekV4HCACache / DeepseekV4CSACache) sharing a `cumulative_length` counter that one
+        # path leaves as a constant SymInt and the other as a python int.
+        if isinstance(obj, torch.SymBool):
+            const = obj.node.maybe_as_bool()
+        elif isinstance(obj, torch.SymFloat):
+            const = obj.node.maybe_as_float()
+        else:
+            const = obj.node.maybe_as_int()
+        if const is not None:
+            return const
         idx = len(tensors)
         tensors.append(obj)
         return {"_t": "sym", "i": idx}
@@ -471,12 +490,21 @@ def _flatten_to_context(obj: Any, tensors: list) -> Any:
             "p": _class_to_path(cls),
             "v": [_flatten_to_context(i, tensors) for i in obj],
         }
+    if isinstance(obj, types.MethodType):
+        # Self-bound methods are handled by the `"obj"` branch below (rebound to the
+        # reconstructed instance); a bare bound method has no instance to rebind to.
+        raise TypeError("Cannot flatten a bound method not stored on its own instance for pytree context")
     if hasattr(obj, "__dict__"):
-        return {
-            "_t": "obj",
-            "p": _class_to_path(cls),
-            "s": {k: _flatten_to_context(v, tensors) for k, v in vars(obj).items()},
-        }
+        state = {}
+        for k, v in vars(obj).items():
+            if isinstance(v, types.MethodType) and v.__self__ is obj:
+                # Methods bound onto the instance itself (e.g. recurrent_gemma binds
+                # `get_seq_length` onto its `DynamicCache`) — store the underlying
+                # function's import path; unflatten rebinds it to the new instance.
+                state[k] = {"_t": "method", "f": f"{v.__func__.__module__}:{v.__func__.__qualname__}"}
+            else:
+                state[k] = _flatten_to_context(v, tensors)
+        return {"_t": "obj", "p": _class_to_path(cls), "s": state}
 
     raise TypeError(f"Cannot flatten {type(obj).__name__} for pytree context")
 
@@ -523,9 +551,12 @@ def _unflatten_from_context(ctx: Any, tensors: list) -> Any:
             return cls(*items)  # NamedTuple (requires positional args)
     if t == "obj":
         cls = _path_to_class(ctx["p"])
-        state = {k: _unflatten_from_context(v, tensors) for k, v in ctx["s"].items()}
         instance = cls.__new__(cls)
-        instance.__dict__.update(state)
+        for k, v in ctx["s"].items():
+            if type(v) is dict and v.get("_t") == "method":
+                instance.__dict__[k] = _path_to_class(v["f"]).__get__(instance)
+            else:
+                instance.__dict__[k] = _unflatten_from_context(v, tensors)
         return instance
 
     raise TypeError(f"Unknown tag {t!r} in pytree context")
@@ -566,6 +597,13 @@ def _iter_subclasses(cls: type):
         yield from _iter_subclasses(subclass)
 
 
+def is_cache_object(value: Any) -> bool:
+    """Whether ``value`` is a cache — a [`Cache`] instance or a model-specific class following
+    the ``*Cache`` naming convention (e.g. ``xLSTMCache``, ``MimiConv1dPaddingCache``), matching
+    what [`register_cache_pytrees_for_model`] registers as pytree nodes."""
+    return isinstance(value, Cache) or type(value).__name__.endswith("Cache")
+
+
 def register_cache_pytrees_for_model(model: PreTrainedModel):
     """Register all relevant cache types as pytree nodes for torch.export."""
     # All transformers Cache subclasses