From b0f5a9f4a2acb5a8bd11942a63a81eccdd7aa25e Mon Sep 17 00:00:00 2001 From: subin9 Date: Thu, 2 Jul 2026 05:03:26 +0000 Subject: [PATCH] Route byte-level llama tokenizers to TokenizersBackend Model type "llama" spans both SentencePiece (Llama-1/2) and byte-level (Llama-3 / tiktoken) tokenizers under one Hub tokenizer_class (LlamaTokenizerFast). In v5, LlamaTokenizer.__init__ unconditionally installs a Metaspace pre-tokenizer/decoder, which silently drops spaces for byte-level repos (see #45488), e.g. deepseek-ai/DeepSeek-R1-Distill-Llama-*. The existing MODEL_IDS_TO_TOKENIZERS_BACKEND allowlist only covers specific checkpoints (e.g. the 8B) and misses others (the 70B is still broken). Instead, for the small set of dual-scheme model types, inspect the serialized tokenizer.json: if it declares a ByteLevel pre_tokenizer/decoder, route to TokenizersBackend (which respects tokenizer.json). SentencePiece Llama-1/2 stays on LlamaTokenizer unchanged. --- .../models/auto/tokenization_auto.py | 55 +++++++++++++++++++ tests/models/auto/test_tokenization_auto.py | 33 +++++++++++ 2 files changed, 88 insertions(+) diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index 678144ab8669..adbf3fbe5e5c 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -429,6 +429,48 @@ "salesforce/instructblip-flan-t5-*", ] +# Model types whose single Hub `tokenizer_class` spans BOTH SentencePiece (older) and byte-level +# (newer) tokenizers. For these, the declared class alone is ambiguous, so we must look at the +# serialized tokenizer.json to decide. Kept intentionally small — most model types are consistent. +_DUAL_SCHEME_MODEL_TYPES = {"llama"} + + +def _tokenizer_json_is_byte_level(pretrained_model_name_or_path, **kwargs) -> bool: + """True if the repo's tokenizer.json declares a ByteLevel pre_tokenizer/decoder (GPT-2 / tiktoken + style, e.g. Llama-3), as opposed to SentencePiece/Metaspace (Llama-1/2). Best-effort; returns + False on any error so callers fall back to the normal resolution.""" + try: + _pass = { + k: kwargs[k] + for k in ("cache_dir", "force_download", "proxies", "token", "revision", "local_files_only", "subfolder") + if k in kwargs + } + resolved = cached_file( + pretrained_model_name_or_path, + "tokenizer.json", + _raise_exceptions_for_gated_repo=False, + _raise_exceptions_for_missing_entries=False, + _raise_exceptions_for_connection_errors=False, + **_pass, + ) + if resolved is None: + return False + with open(resolved, encoding="utf-8") as f: + tok_json = json.load(f) + + def _has_byte_level(node): + if isinstance(node, dict): + if node.get("type") == "ByteLevel": + return True + return any(_has_byte_level(v) for v in node.values()) + if isinstance(node, list): + return any(_has_byte_level(v) for v in node) + return False + + return _has_byte_level(tok_json.get("pre_tokenizer")) or _has_byte_level(tok_json.get("decoder")) + except Exception: + return False + def load_vocab(vocab_file): """Loads a vocabulary file into a dictionary.""" @@ -766,6 +808,19 @@ def from_pretrained( else: tokenizer_auto_map = tokenizer_config["auto_map"].get("AutoTokenizer", None) + # Dual-scheme model types (e.g. `llama`): the Hub tokenizer_class (LlamaTokenizerFast) is + # ambiguous because Llama-1/2 are SentencePiece while Llama-3 is byte-level. Forcing the + # SentencePiece LlamaTokenizer (Metaspace) onto a byte-level tokenizer.json silently drops + # spaces (see #45488). If tokenizer.json is byte-level, respect it via TokenizersBackend; + # SentencePiece Llama-1/2 (no ByteLevel in tokenizer.json) stays on LlamaTokenizer unchanged. + if ( + tokenizer_auto_map is None + and TokenizersBackend is not None + and config_model_type in _DUAL_SCHEME_MODEL_TYPES + and _tokenizer_json_is_byte_level(pretrained_model_name_or_path, **kwargs) + ): + return TokenizersBackend.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) + # Some specific checkpoints need TokenizersBackend because their config on the Hub really needs to be updated. _config_name_or_path = ( name.lower() if isinstance((name := getattr(config, "_name_or_path", None)), str) else "" diff --git a/tests/models/auto/test_tokenization_auto.py b/tests/models/auto/test_tokenization_auto.py index 7c1c703a906a..0e4c1b54e4e9 100644 --- a/tests/models/auto/test_tokenization_auto.py +++ b/tests/models/auto/test_tokenization_auto.py @@ -81,6 +81,39 @@ class AutoTokenizerTest(unittest.TestCase): def setUp(self): transformers.dynamic_module_utils.TIME_OUT_REMOTE_CODE = 0 + def test_tokenizer_json_is_byte_level(self): + from transformers.models.auto.tokenization_auto import _tokenizer_json_is_byte_level + + cases = { + # byte-level (GPT-2 / tiktoken style, e.g. Llama-3): detected -> True + "byte_level_flat": ({"pre_tokenizer": {"type": "ByteLevel"}, "decoder": {"type": "ByteLevel"}}, True), + # byte-level nested inside a Sequence pre_tokenizer (real Llama-3 layout) + "byte_level_seq": ( + { + "pre_tokenizer": {"type": "Sequence", "pretokenizers": [{"type": "Split"}, {"type": "ByteLevel"}]}, + "decoder": {"type": "ByteLevel"}, + }, + True, + ), + # SentencePiece / Metaspace (Llama-1/2): not byte-level -> False + "metaspace": ( + { + "pre_tokenizer": {"type": "Metaspace", "replacement": "▁"}, + "decoder": {"type": "Metaspace", "replacement": "▁"}, + }, + False, + ), + } + for name, (tok_json, expected) in cases.items(): + with tempfile.TemporaryDirectory() as tmp_dir: + with open(os.path.join(tmp_dir, "tokenizer.json"), "w", encoding="utf-8") as f: + json.dump(tok_json, f) + self.assertEqual(_tokenizer_json_is_byte_level(tmp_dir), expected, msg=name) + + # No tokenizer.json available -> best-effort False (falls back to normal resolution). + with tempfile.TemporaryDirectory() as tmp_dir: + self.assertFalse(_tokenizer_json_is_byte_level(tmp_dir)) + @slow def test_tokenizer_from_pretrained(self): for model_name in ("google-bert/bert-base-uncased", "google-bert/bert-base-cased"):