safer access to tokenizer attrs

Signed-off-by: alessiodevoto <devoto.alessio@gmail.com>

Author: alessiodevoto

logits_processor_zoo/utils.py

@@ -25,8 +25,8 @@ def text_to_token(tokenizer: PreTrainedTokenizer, text: str, last: bool):
 
     # We allow 2 tokens to account for the BOS or prefix token
     max_token_count = 1
-    bos_token_added = tokenizer.bos_token and tokenizer.bos_token_id in tokens
-    prefix_token_added = tokenizer.add_prefix_space is not False
+    bos_token_added = getattr(tokenizer, 'bos_token', None) and getattr(tokenizer, 'bos_token_id', None) in tokens
+    prefix_token_added = getattr(tokenizer, 'add_prefix_space', None) is not False
     if bos_token_added or prefix_token_added:
         max_token_count = 2