Add the vocab params fix patch

Author: JamePeng

llama_cpp/llama.py

@@ -1430,7 +1430,7 @@ def logit_bias_processor(
                             )
                         )
                         top_logprob = {
-                            self.detokenize([i]).decode(
+                            self.detokenize(self._vocab, [i]).decode(
                                 "utf-8", errors="ignore"
                             ): logprob
                             for logprob, i in sorted_logprobs[:logprobs]
@@ -1559,6 +1559,7 @@ def logit_bias_processor(
                     )
                     text_offset = len(prompt) + len(
                         self.detokenize(
+                            self._vocab,
                             completion_tokens[:returned_tokens],
                             prev_tokens=prompt_tokens
                             + completion_tokens[:returned_tokens],
@@ -1574,7 +1575,7 @@ def logit_bias_processor(
                         )
                     )
                     top_logprob = {
-                        self.detokenize([i]).decode("utf-8", errors="ignore"): logprob
+                        self.detokenize(self._vocab, [i]).decode("utf-8", errors="ignore"): logprob
                         for logprob, i in sorted_logprobs[:logprobs]
                     }
                     top_logprob.update({token_str: current_logprobs[int(token)]})
@@ -1617,7 +1618,7 @@ def logit_bias_processor(
                     "model": model_name,
                     "choices": [
                         {
-                            "text": self.detokenize([token]).decode(
+                            "text": self.detokenize(self._vocab, [token]).decode(
                                 "utf-8", errors="ignore"
                             ),
                             "index": 0,
@@ -1680,7 +1681,7 @@ def logit_bias_processor(
                 all_tokens = completion_tokens
 
             all_token_strs = [
-                self.detokenize([token], prev_tokens=all_tokens[:i]).decode(
+                self.detokenize(self._vocab, [token], prev_tokens=all_tokens[:i]).decode(
                     "utf-8", errors="ignore"
                 )
                 for i, token in enumerate(all_tokens)
@@ -1695,7 +1696,7 @@ def logit_bias_processor(
                 text_offsets.append(
                     text_offset
                     + len(
-                        self.detokenize(all_tokens[:idx]).decode(
+                        self.detokenize(self._vocab, all_tokens[:idx]).decode(
                             "utf-8", errors="ignore"
                         )
                     )
@@ -1708,7 +1709,7 @@ def logit_bias_processor(
                 )
                 token_logprobs.append(logprobs_token[int(token)])
                 top_logprob: Optional[Dict[str, float]] = {
-                    self.detokenize([i], prev_tokens=all_tokens[:idx]).decode(
+                    self.detokenize(self._vocab, [i], prev_tokens=all_tokens[:idx]).decode(
                         "utf-8", errors="ignore"
                     ): logprob
                     for logprob, i in sorted_logprobs[:logprobs]

llama_cpp/llama_chat_format.py

@@ -594,7 +594,7 @@ def chat_completion_handler(
             tool_choice=tool_choice,
         )
         prompt = llama.tokenize(
-            vocab=llama.llama_model_get_vocab(model),
+            vocab=llama.llama_model_get_vocab(llama.model),
             text=result.prompt.encode("utf-8"),
             add_bos=not result.added_special,
             special=True,
@@ -2813,8 +2813,8 @@ def __call__(
         text = template.render(
             messages=messages,
             add_generation_prompt=True,
-            eos_token=llama.detokenize([llama.token_eos()]),
-            bos_token=llama.detokenize([llama.token_bos()]),
+            eos_token=llama.detokenize(vocab=llama.llama_model_get_vocab(llama.model), tokens=[llama.token_eos()]),
+            bos_token=llama.detokenize(vocab=llama.llama_model_get_vocab(llama.model), tokens=[llama.token_bos()]),
         )
         split_text = self.split_text_on_image_urls(text, image_urls)
 
@@ -2828,7 +2828,8 @@ def __call__(
         for type_, value in split_text:
             if type_ == "text":
                 tokens = llama.tokenize(
-                    value.encode("utf8"), add_bos=False, special=True
+                    vocab=llama.llama_model_get_vocab(llama.model),
+                    text=value.encode("utf8"), add_bos=False, special=True
                 )
                 if llama.n_tokens + len(tokens) > llama.n_ctx():
                     raise ValueError(

llama_cpp/llama_tokenizer.py

@@ -62,14 +62,14 @@ def detokenize(
         return self._model.detokenize(vocab, tokens, special=special)
 
     def encode(
-        self, text: str, add_bos: bool = True, special: bool = True
+        self, vocab:llama_cpp.llama_vocab_p, text: str, add_bos: bool = True, special: bool = True
     ) -> List[int]:
         return self.tokenize(
-            text.encode("utf-8", errors="ignore"), add_bos=add_bos, special=special
+            vocab, text.encode("utf-8", errors="ignore"), add_bos=add_bos, special=special
         )
 
-    def decode(self, tokens: List[int]) -> str:
-        return self.detokenize(tokens).decode("utf-8", errors="ignore")
+    def decode(self, vocab:llama_cpp.llama_vocab_p, tokens: List[int]) -> str:
+        return self.detokenize(vocab, tokens).decode("utf-8", errors="ignore")
 
     @classmethod
     def from_ggml_file(cls, path: str) -> "LlamaTokenizer":

llama_cpp/server/app.py

@@ -227,9 +227,10 @@ def _logit_bias_tokens_to_input_ids(
     logit_bias: Dict[str, float],
 ) -> Dict[str, float]:
     to_bias: Dict[str, float] = {}
+    vocab=llama.llama_model_get_vocab(llama.model)
     for token, score in logit_bias.items():
         token = token.encode("utf-8")
-        for input_id in llama.tokenize(token, add_bos=False, special=True):
+        for input_id in llama.tokenize(vocab, token, add_bos=False, special=True):
             to_bias[str(input_id)] = score
     return to_bias