fixed via add_prefix_token

Signed-off-by: alessiodevoto <devoto.alessio@gmail.com>

Author: alessiodevoto

logits_processor_zoo/utils.py

@@ -23,8 +23,12 @@
 def text_to_token(tokenizer: PreTrainedTokenizer, text: str, last: bool):
     tokens = tokenizer.encode(text, add_special_tokens=False)
 
-    # We allow 2 tokens to account for the BOS token
-    max_token_count = 2 if tokenizer.bos_token and tokenizer.bos_token_id in tokens else 1
+    # We allow 2 tokens to account for the BOS or prefix token
+    max_token_count = 1
+    bos_token_added = tokenizer.bos_token and tokenizer.bos_token_id in tokens
+    prefix_token_added = tokenizer.add_prefix_space is not False
+    if bos_token_added or prefix_token_added:
+        max_token_count = 2
 
     if not last and len(tokens) > max_token_count:
         raise Exception(f"Can't convert {text} to token. It has {len(tokens)} tokens.")