Route byte-level llama tokenizers to TokenizersBackend

src/transformers/models/auto/tokenization_auto.py

@@ -429,6 +429,48 @@
     "salesforce/instructblip-flan-t5-*",
 ]
 
+# Model types whose single Hub `tokenizer_class` spans BOTH SentencePiece (older) and byte-level
+# (newer) tokenizers. For these, the declared class alone is ambiguous, so we must look at the
+# serialized tokenizer.json to decide. Kept intentionally small — most model types are consistent.
+_DUAL_SCHEME_MODEL_TYPES = {"llama"}
+
+
+def _tokenizer_json_is_byte_level(pretrained_model_name_or_path, **kwargs) -> bool:
+    """True if the repo's tokenizer.json declares a ByteLevel pre_tokenizer/decoder (GPT-2 / tiktoken
+    style, e.g. Llama-3), as opposed to SentencePiece/Metaspace (Llama-1/2). Best-effort; returns
+    False on any error so callers fall back to the normal resolution."""
+    try:
+        _pass = {
+            k: kwargs[k]
+            for k in ("cache_dir", "force_download", "proxies", "token", "revision", "local_files_only", "subfolder")
+            if k in kwargs
+        }
+        resolved = cached_file(
+            pretrained_model_name_or_path,
+            "tokenizer.json",
+            _raise_exceptions_for_gated_repo=False,
+            _raise_exceptions_for_missing_entries=False,
+            _raise_exceptions_for_connection_errors=False,
+            **_pass,
+        )
+        if resolved is None:
+            return False
+        with open(resolved, encoding="utf-8") as f:
+            tok_json = json.load(f)
+
+        def _has_byte_level(node):
+            if isinstance(node, dict):
+                if node.get("type") == "ByteLevel":
+                    return True
+                return any(_has_byte_level(v) for v in node.values())
+            if isinstance(node, list):
+                return any(_has_byte_level(v) for v in node)
+            return False
+
+        return _has_byte_level(tok_json.get("pre_tokenizer")) or _has_byte_level(tok_json.get("decoder"))
+    except Exception:
+        return False
+
 
 def load_vocab(vocab_file):
     """Loads a vocabulary file into a dictionary."""
@@ -766,6 +808,19 @@ def from_pretrained(
             else:
                 tokenizer_auto_map = tokenizer_config["auto_map"].get("AutoTokenizer", None)
 
+        # Dual-scheme model types (e.g. `llama`): the Hub tokenizer_class (LlamaTokenizerFast) is
+        # ambiguous because Llama-1/2 are SentencePiece while Llama-3 is byte-level. Forcing the
+        # SentencePiece LlamaTokenizer (Metaspace) onto a byte-level tokenizer.json silently drops
+        # spaces (see #45488). If tokenizer.json is byte-level, respect it via TokenizersBackend;
+        # SentencePiece Llama-1/2 (no ByteLevel in tokenizer.json) stays on LlamaTokenizer unchanged.
+        if (
+            tokenizer_auto_map is None
+            and TokenizersBackend is not None
+            and config_model_type in _DUAL_SCHEME_MODEL_TYPES
+            and _tokenizer_json_is_byte_level(pretrained_model_name_or_path, **kwargs)
+        ):
+            return TokenizersBackend.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
+
         # Some specific checkpoints need TokenizersBackend because their config on the Hub really needs to be updated.
         _config_name_or_path = (
             name.lower() if isinstance((name := getattr(config, "_name_or_path", None)), str) else ""

tests/models/auto/test_tokenization_auto.py

@@ -81,6 +81,39 @@ class AutoTokenizerTest(unittest.TestCase):
     def setUp(self):
         transformers.dynamic_module_utils.TIME_OUT_REMOTE_CODE = 0
 
+    def test_tokenizer_json_is_byte_level(self):
+        from transformers.models.auto.tokenization_auto import _tokenizer_json_is_byte_level
+
+        cases = {
+            # byte-level (GPT-2 / tiktoken style, e.g. Llama-3): detected -> True
+            "byte_level_flat": ({"pre_tokenizer": {"type": "ByteLevel"}, "decoder": {"type": "ByteLevel"}}, True),
+            # byte-level nested inside a Sequence pre_tokenizer (real Llama-3 layout)
+            "byte_level_seq": (
+                {
+                    "pre_tokenizer": {"type": "Sequence", "pretokenizers": [{"type": "Split"}, {"type": "ByteLevel"}]},
+                    "decoder": {"type": "ByteLevel"},
+                },
+                True,
+            ),
+            # SentencePiece / Metaspace (Llama-1/2): not byte-level -> False
+            "metaspace": (
+                {
+                    "pre_tokenizer": {"type": "Metaspace", "replacement": "▁"},
+                    "decoder": {"type": "Metaspace", "replacement": "▁"},
+                },
+                False,
+            ),
+        }
+        for name, (tok_json, expected) in cases.items():
+            with tempfile.TemporaryDirectory() as tmp_dir:
+                with open(os.path.join(tmp_dir, "tokenizer.json"), "w", encoding="utf-8") as f:
+                    json.dump(tok_json, f)
+                self.assertEqual(_tokenizer_json_is_byte_level(tmp_dir), expected, msg=name)
+
+        # No tokenizer.json available -> best-effort False (falls back to normal resolution).
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            self.assertFalse(_tokenizer_json_is_byte_level(tmp_dir))
+
     @slow
     def test_tokenizer_from_pretrained(self):
         for model_name in ("google-bert/bert-base-uncased", "google-bert/bert-base-cased"):