Skip to content

Commit d890289

Browse files
o7sio7si
committed
vocab : refactor normalizer flags into options struct, add strip_accents
1 parent 49f3542 commit d890289

7 files changed

Lines changed: 83 additions & 51 deletions

File tree

‎gguf-py/gguf/constants.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -272,7 +272,8 @@ class Tokenizer:
272272
CHAT_TEMPLATE_N = "tokenizer.chat_template.{name}"
273273
CHAT_TEMPLATES = "tokenizer.chat_templates"
274274
# Normalizer constants
275-
NORMALIZER_LOWERCASE = "tokenizer.ggml.normalizer.lowercase"
275+
NORMALIZER_LOWERCASE = "tokenizer.ggml.normalizer.lowercase"
276+
NORMALIZER_STRIP_ACCENTS = "tokenizer.ggml.normalizer.strip_accents"
276277
# FIM/Infill special tokens constants
277278
FIM_PRE_ID = "tokenizer.ggml.fim_pre_token_id"
278279
FIM_SUF_ID = "tokenizer.ggml.fim_suf_token_id"

‎gguf-py/gguf/gguf_writer.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1124,6 +1124,9 @@ def add_suppress_tokens(self, tokens: Sequence[int]) -> None:
11241124
def add_normalizer_lowercase(self, value: bool) -> None:
11251125
self.add_bool(Keys.Tokenizer.NORMALIZER_LOWERCASE, value)
11261126

1127+
def add_normalizer_strip_accents(self, value: bool) -> None:
1128+
self.add_bool(Keys.Tokenizer.NORMALIZER_STRIP_ACCENTS, value)
1129+
11271130
def add_eot_token_id(self, id: int) -> None:
11281131
self.add_uint32(Keys.Tokenizer.EOT_ID, id)
11291132

‎gguf-py/gguf/vocab.py‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ class SpecialVocab:
5353
special_token_ids: dict[str, int]
5454
chat_template: str | Sequence[Mapping[str, str]] | None
5555
normalizer_lowercase: bool | None
56+
normalizer_strip_accents: bool | None
5657

5758
def __init__(
5859
self, path: str | os.PathLike[str], load_merges: bool = False,
@@ -66,6 +67,7 @@ def __init__(
6667
self.merges = []
6768
self.chat_template = None
6869
self.normalizer_lowercase = None
70+
self.normalizer_strip_accents = None
6971
if special_token_types is not None:
7072
self.special_token_types = special_token_types
7173
else:
@@ -108,6 +110,10 @@ def add_to_gguf(self, gw: GGUFWriter, quiet: bool = False) -> None:
108110
if not quiet:
109111
logger.info(f'Setting normalizer_lowercase to {self.normalizer_lowercase}')
110112
gw.add_normalizer_lowercase(self.normalizer_lowercase)
113+
if self.normalizer_strip_accents is not None:
114+
if not quiet:
115+
logger.info(f'Setting normalizer_strip_accents to {self.normalizer_strip_accents}')
116+
gw.add_normalizer_strip_accents(self.normalizer_strip_accents)
111117

112118
def _load(self, path: Path) -> None:
113119
self._try_load_from_tokenizer_json(path)
@@ -155,17 +161,21 @@ def _set_special_token(self, typ: str, tid: Any) -> None:
155161
def _parse_normalizer(self, normalizer: dict) -> None:
156162
# ref: https://huggingface.co/docs/tokenizers/api/normalizers
157163
#
158-
# Detects lowercase normalization in three possible formats:
159-
# 1. Standalone: {"type": "Lowercase"}
160-
# 2. BertNormalizer attribute: {"type": "BertNormalizer", "lowercase": true, ...}
161-
# 3. Nested in Sequence: {"type": "Sequence", "normalizers": [...]}
164+
# Extracts normalizer flags from three possible formats:
165+
# 1. Standalone: {"type": "Lowercase"}
166+
# 2. BertNormalizer attrs: {"type": "BertNormalizer", ...}
167+
# 3. Nested in Sequence: {"type": "Sequence", "normalizers": [...]}
162168

163169
normalizer_type = normalizer.get('type')
164170
if normalizer_type == 'Lowercase':
165171
self.normalizer_lowercase = True
172+
elif normalizer_type == 'StripAccents':
173+
self.normalizer_strip_accents = True
166174
elif normalizer_type == 'BertNormalizer':
167175
if 'lowercase' in normalizer:
168176
self.normalizer_lowercase = normalizer['lowercase']
177+
if 'strip_accents' in normalizer:
178+
self.normalizer_strip_accents = normalizer['strip_accents']
169179
elif normalizer_type == 'Sequence':
170180
for norm in normalizer.get('normalizers', []):
171181
self._parse_normalizer(norm)

‎src/llama-arch.cpp‎

Lines changed: 34 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -299,39 +299,40 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
299299
{ LLM_KV_DENSE_3_FEAT_IN, "%s.dense_3_feat_in" },
300300
{ LLM_KV_DENSE_3_FEAT_OUT, "%s.dense_3_feat_out" },
301301

302-
{ LLM_KV_TOKENIZER_MODEL, "tokenizer.ggml.model" },
303-
{ LLM_KV_TOKENIZER_PRE, "tokenizer.ggml.pre" },
304-
{ LLM_KV_TOKENIZER_LIST, "tokenizer.ggml.tokens" },
305-
{ LLM_KV_TOKENIZER_TOKEN_TYPE, "tokenizer.ggml.token_type" },
306-
{ LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, "tokenizer.ggml.token_type_count" },
307-
{ LLM_KV_TOKENIZER_SCORES, "tokenizer.ggml.scores" },
308-
{ LLM_KV_TOKENIZER_MERGES, "tokenizer.ggml.merges" },
309-
{ LLM_KV_TOKENIZER_BOS_ID, "tokenizer.ggml.bos_token_id" },
310-
{ LLM_KV_TOKENIZER_EOS_ID, "tokenizer.ggml.eos_token_id" },
311-
{ LLM_KV_TOKENIZER_EOT_ID, "tokenizer.ggml.eot_token_id" },
312-
{ LLM_KV_TOKENIZER_EOM_ID, "tokenizer.ggml.eom_token_id" },
313-
{ LLM_KV_TOKENIZER_UNK_ID, "tokenizer.ggml.unknown_token_id" },
314-
{ LLM_KV_TOKENIZER_SEP_ID, "tokenizer.ggml.seperator_token_id" },
315-
{ LLM_KV_TOKENIZER_PAD_ID, "tokenizer.ggml.padding_token_id" },
316-
{ LLM_KV_TOKENIZER_CLS_ID, "tokenizer.ggml.cls_token_id" },
317-
{ LLM_KV_TOKENIZER_MASK_ID, "tokenizer.ggml.mask_token_id" },
318-
{ LLM_KV_TOKENIZER_ADD_BOS, "tokenizer.ggml.add_bos_token" },
319-
{ LLM_KV_TOKENIZER_ADD_EOS, "tokenizer.ggml.add_eos_token" },
320-
{ LLM_KV_TOKENIZER_ADD_SEP, "tokenizer.ggml.add_sep_token" },
321-
{ LLM_KV_TOKENIZER_ADD_PREFIX, "tokenizer.ggml.add_space_prefix" },
322-
{ LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, "tokenizer.ggml.remove_extra_whitespaces" },
323-
{ LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, "tokenizer.ggml.precompiled_charsmap" },
324-
{ LLM_KV_TOKENIZER_HF_JSON, "tokenizer.huggingface.json" },
325-
{ LLM_KV_TOKENIZER_RWKV, "tokenizer.rwkv.world" },
326-
{ LLM_KV_TOKENIZER_CHAT_TEMPLATE, "tokenizer.chat_template" },
327-
{ LLM_KV_TOKENIZER_NORMALIZER_LOWERCASE, "tokenizer.ggml.normalizer.lowercase" },
328-
{ LLM_KV_TOKENIZER_FIM_PRE_ID, "tokenizer.ggml.fim_pre_token_id" },
329-
{ LLM_KV_TOKENIZER_FIM_SUF_ID, "tokenizer.ggml.fim_suf_token_id" },
330-
{ LLM_KV_TOKENIZER_FIM_MID_ID, "tokenizer.ggml.fim_mid_token_id" },
331-
{ LLM_KV_TOKENIZER_FIM_PAD_ID, "tokenizer.ggml.fim_pad_token_id" },
332-
{ LLM_KV_TOKENIZER_FIM_REP_ID, "tokenizer.ggml.fim_rep_token_id" },
333-
{ LLM_KV_TOKENIZER_FIM_SEP_ID, "tokenizer.ggml.fim_sep_token_id" },
334-
{ LLM_KV_TOKENIZER_SUPPRESS_TOKENS, "tokenizer.ggml.suppress_tokens" },
302+
{ LLM_KV_TOKENIZER_MODEL, "tokenizer.ggml.model" },
303+
{ LLM_KV_TOKENIZER_PRE, "tokenizer.ggml.pre" },
304+
{ LLM_KV_TOKENIZER_LIST, "tokenizer.ggml.tokens" },
305+
{ LLM_KV_TOKENIZER_TOKEN_TYPE, "tokenizer.ggml.token_type" },
306+
{ LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, "tokenizer.ggml.token_type_count" },
307+
{ LLM_KV_TOKENIZER_SCORES, "tokenizer.ggml.scores" },
308+
{ LLM_KV_TOKENIZER_MERGES, "tokenizer.ggml.merges" },
309+
{ LLM_KV_TOKENIZER_BOS_ID, "tokenizer.ggml.bos_token_id" },
310+
{ LLM_KV_TOKENIZER_EOS_ID, "tokenizer.ggml.eos_token_id" },
311+
{ LLM_KV_TOKENIZER_EOT_ID, "tokenizer.ggml.eot_token_id" },
312+
{ LLM_KV_TOKENIZER_EOM_ID, "tokenizer.ggml.eom_token_id" },
313+
{ LLM_KV_TOKENIZER_UNK_ID, "tokenizer.ggml.unknown_token_id" },
314+
{ LLM_KV_TOKENIZER_SEP_ID, "tokenizer.ggml.seperator_token_id" },
315+
{ LLM_KV_TOKENIZER_PAD_ID, "tokenizer.ggml.padding_token_id" },
316+
{ LLM_KV_TOKENIZER_CLS_ID, "tokenizer.ggml.cls_token_id" },
317+
{ LLM_KV_TOKENIZER_MASK_ID, "tokenizer.ggml.mask_token_id" },
318+
{ LLM_KV_TOKENIZER_ADD_BOS, "tokenizer.ggml.add_bos_token" },
319+
{ LLM_KV_TOKENIZER_ADD_EOS, "tokenizer.ggml.add_eos_token" },
320+
{ LLM_KV_TOKENIZER_ADD_SEP, "tokenizer.ggml.add_sep_token" },
321+
{ LLM_KV_TOKENIZER_ADD_PREFIX, "tokenizer.ggml.add_space_prefix" },
322+
{ LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, "tokenizer.ggml.remove_extra_whitespaces" },
323+
{ LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, "tokenizer.ggml.precompiled_charsmap" },
324+
{ LLM_KV_TOKENIZER_HF_JSON, "tokenizer.huggingface.json" },
325+
{ LLM_KV_TOKENIZER_RWKV, "tokenizer.rwkv.world" },
326+
{ LLM_KV_TOKENIZER_CHAT_TEMPLATE, "tokenizer.chat_template" },
327+
{ LLM_KV_TOKENIZER_NORMALIZER_LOWERCASE, "tokenizer.ggml.normalizer.lowercase" },
328+
{ LLM_KV_TOKENIZER_NORMALIZER_STRIP_ACCENTS, "tokenizer.ggml.normalizer.strip_accents" },
329+
{ LLM_KV_TOKENIZER_FIM_PRE_ID, "tokenizer.ggml.fim_pre_token_id" },
330+
{ LLM_KV_TOKENIZER_FIM_SUF_ID, "tokenizer.ggml.fim_suf_token_id" },
331+
{ LLM_KV_TOKENIZER_FIM_MID_ID, "tokenizer.ggml.fim_mid_token_id" },
332+
{ LLM_KV_TOKENIZER_FIM_PAD_ID, "tokenizer.ggml.fim_pad_token_id" },
333+
{ LLM_KV_TOKENIZER_FIM_REP_ID, "tokenizer.ggml.fim_rep_token_id" },
334+
{ LLM_KV_TOKENIZER_FIM_SEP_ID, "tokenizer.ggml.fim_sep_token_id" },
335+
{ LLM_KV_TOKENIZER_SUPPRESS_TOKENS, "tokenizer.ggml.suppress_tokens" },
335336

336337
{ LLM_KV_ADAPTER_TYPE, "adapter.type" },
337338
{ LLM_KV_ADAPTER_LORA_ALPHA, "adapter.lora.alpha" },

‎src/llama-arch.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -314,6 +314,7 @@ enum llm_kv {
314314
LLM_KV_TOKENIZER_RWKV,
315315
LLM_KV_TOKENIZER_CHAT_TEMPLATE,
316316
LLM_KV_TOKENIZER_NORMALIZER_LOWERCASE,
317+
LLM_KV_TOKENIZER_NORMALIZER_STRIP_ACCENTS,
317318
LLM_KV_TOKENIZER_FIM_PRE_ID,
318319
LLM_KV_TOKENIZER_FIM_SUF_ID,
319320
LLM_KV_TOKENIZER_FIM_MID_ID,

‎src/llama-vocab.cpp‎

Lines changed: 22 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -764,7 +764,7 @@ struct llm_tokenizer_wpm_session {
764764

765765
void tokenize(const std::string & text, std::vector<llama_token> & output) {
766766
// normalize and split by whitespace
767-
std::vector<std::string> words = preprocess(text, vocab.get_normalizer_lowercase());
767+
std::vector<std::string> words = preprocess(text, vocab.get_normalizer_opts());
768768
// bos token prepended already
769769

770770
// find the longest tokens that form the words
@@ -809,11 +809,14 @@ struct llm_tokenizer_wpm_session {
809809
}
810810

811811
// TODO: reduce string copies by using cpts_offs array
812-
static std::vector<std::string> preprocess(const std::string & text, bool lowercase) {
813-
const std::vector<uint32_t> cpts_nfd = unicode_cpts_normalize_nfd(unicode_cpts_from_utf8(text));
812+
static std::vector<std::string> preprocess(const std::string & text, const llama_vocab::normalizer_options & normalizer_opts) {
813+
std::vector<uint32_t> cpts = unicode_cpts_from_utf8(text);
814+
if (normalizer_opts.strip_accents) {
815+
cpts = unicode_cpts_normalize_nfd(cpts);
816+
}
814817
std::vector<std::string> words(1, "");
815818

816-
for (const uint32_t cpt : cpts_nfd) {
819+
for (const uint32_t cpt : cpts) {
817820
const auto flags = unicode_cpt_flags_from_cpt(cpt);
818821

819822
if (flags.is_whitespace) {
@@ -828,7 +831,11 @@ struct llm_tokenizer_wpm_session {
828831
continue;
829832
}
830833

831-
const std::string s = unicode_cpt_to_utf8(lowercase ? unicode_tolower(cpt) : cpt);
834+
if (normalizer_opts.strip_accents && flags.is_accent_mark) {
835+
continue;
836+
}
837+
838+
const std::string s = unicode_cpt_to_utf8(normalizer_opts.lowercase ? unicode_tolower(cpt) : cpt);
832839
if (flags.is_punctuation || ( cpt < 0x7F && flags.is_symbol ) || is_chinese_char(cpt)) {
833840
if (words.back().size()) { // finish previous word if any
834841
words.emplace_back();
@@ -1692,7 +1699,7 @@ struct llm_tokenizer_whitespace_session : llm_tokenizer_bpe_session {
16921699
llm_tokenizer_whitespace_session(const llama_vocab & vocab, const llm_tokenizer_bpe & tokenizer) : llm_tokenizer_bpe_session{vocab, tokenizer}, vocab{vocab} {}
16931700

16941701
void tokenize(const std::string & text, std::vector<llama_token> & output) override {
1695-
const bool lowercase = vocab.get_normalizer_lowercase();
1702+
const bool lowercase = vocab.get_normalizer_opts().lowercase;
16961703

16971704
std::string segment;
16981705
auto flush = [&]() {
@@ -1797,7 +1804,9 @@ struct llama_vocab::impl {
17971804
bool remove_extra_whitespaces = false;
17981805
bool escape_whitespaces = true;
17991806
bool treat_whitespace_as_suffix = false;
1800-
bool normalizer_lowercase = true; // Lowercase normalizer (tokenizer.json)
1807+
1808+
// BertNormalizer options
1809+
llama_vocab::normalizer_options normalizer_opts;
18011810

18021811
std::unordered_map<std::string, llama_token> token_to_id;
18031812
std::vector<token_data> id_to_token;
@@ -2172,7 +2181,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
21722181
} else if (
21732182
tokenizer_pre == "whitespace") {
21742183
pre_type = LLAMA_VOCAB_PRE_TYPE_WHITESPACE;
2175-
normalizer_lowercase = false;
2184+
normalizer_opts.lowercase = false;
21762185
} else if (
21772186
tokenizer_pre == "refact") {
21782187
pre_type = LLAMA_VOCAB_PRE_TYPE_REFACT;
@@ -2532,8 +2541,9 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
25322541
}
25332542
}
25342543

2535-
// Lowercase normalizer flag (consulted by WPM / whitespace BPE)
2536-
ml.get_key(LLM_KV_TOKENIZER_NORMALIZER_LOWERCASE, normalizer_lowercase, false);
2544+
// BertNormalizer options
2545+
ml.get_key(LLM_KV_TOKENIZER_NORMALIZER_LOWERCASE, normalizer_opts.lowercase, false);
2546+
ml.get_key(LLM_KV_TOKENIZER_NORMALIZER_STRIP_ACCENTS, normalizer_opts.strip_accents, false);
25372547

25382548
// suppress tokens
25392549
{
@@ -3969,8 +3979,8 @@ bool llama_vocab::get_treat_whitespace_as_suffix() const {
39693979
return pimpl->treat_whitespace_as_suffix;
39703980
}
39713981

3972-
bool llama_vocab::get_normalizer_lowercase() const {
3973-
return pimpl->normalizer_lowercase;
3982+
const llama_vocab::normalizer_options & llama_vocab::get_normalizer_opts() const {
3983+
return pimpl->normalizer_opts;
39743984
}
39753985

39763986
const std::vector<llama_token> & llama_vocab::get_suppress_tokens() const {

‎src/llama-vocab.h‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,12 @@ struct llama_vocab {
7676
llama_token_attr attr;
7777
};
7878

79+
struct normalizer_options {
80+
bool lowercase = true;
81+
bool strip_accents = false;
82+
// TODO: clean_text, handle_chinese_chars
83+
};
84+
7985
llama_vocab();
8086
~llama_vocab();
8187

@@ -141,7 +147,7 @@ struct llama_vocab {
141147
bool get_remove_extra_whitespaces () const;
142148
bool get_escape_whitespaces () const;
143149
bool get_treat_whitespace_as_suffix() const;
144-
bool get_normalizer_lowercase () const;
150+
const normalizer_options & get_normalizer_opts() const;
145151

146152
const std::vector<llama_token> & get_suppress_tokens() const;
147153

0 commit comments

Comments
 (0)