diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 2c5e62a16fbe..ef3a70228460 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -184,7 +184,10 @@ def main() -> None: if args.remote: hf_repo_id = args.model from huggingface_hub import snapshot_download - allowed_patterns = ["LICENSE", "*.json", "*.md", "*.txt", "tokenizer.model"] + # "*.model" covers tokenizer.model and tiktoken.model; "*.py" is needed + # because several converters call AutoTokenizer with trust_remote_code=True, + # which resolves tokenizer code from the model directory. + allowed_patterns = ["LICENSE", "*.json", "*.md", "*.txt", "*.model", "*.py"] if args.sentence_transformers_dense_modules: # include sentence-transformers dense modules safetensors files allowed_patterns.append("*.safetensors")