Skip to content

Motif3 #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
timkhronos wants to merge 8 commits into from
Closed

Motif3 #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
51b715b
Initial Motif 3 (beta) support
timkhronos Jul 28, 2026
743daa4
Fix pretokenizer
timkhronos Jul 28, 2026
42ce6b5
Update motif3.py
timkhronos Jul 28, 2026
de8ebef
Fix pretokenizer
timkhronos Jul 28, 2026
77605d6
Update motif3.py
timkhronos Jul 28, 2026
c0786e9
Update motif3.py
timkhronos Jul 28, 2026
7f015d6
Update llama-vocab.cpp
timkhronos Jul 29, 2026
c72519a
Precision Improvements
timkhronos Jul 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions conversion/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,7 @@
"ModernBertForSequenceClassification": "bert",
"ModernBertModel": "bert",
"NanbeigeForCausalLM": "nanbeige",
"MotifForCausalLM": "motif3",
"NemotronForCausalLM": "nemotron",
"NemotronHForCausalLM": "nemotron",
"NeoBERT": "bert",
Expand Down
353 changes: 353 additions & 0 deletions conversion/motif3.py
View file
Open in desktop

Large diffs are not rendered by default.

70 changes: 70 additions & 0 deletions gguf-py/gguf/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -555,6 +555,7 @@ class MODEL_ARCH(IntEnum):
TALKIE = auto()
MELLUM = auto()
NANBEIGE = auto()
MOTIF3 = auto()


class VISION_PROJECTOR_TYPE(IntEnum):
Expand Down Expand Up @@ -722,6 +723,20 @@ class MODEL_TENSOR(IntEnum):
HC_FFN_FN = auto()
HC_FFN_BASE = auto()
HC_FFN_SCALE = auto()
ATTN_LAMBDA = auto() # motif3 differential-attention lambda proj
FFN_POLY = auto() # motif3 PolyNorm coefficients (dense mlp)
FFN_POLY_EXPS = auto() # motif3 per-expert PolyNorm coefficients
FFN_POLY_SHEXP = auto() # motif3 shared-expert PolyNorm coefficients
MHC_ATTN_NORM = auto() # motif3 mHC rms-norm (attn sublayer)
MHC_ATTN_PRE = auto()
MHC_ATTN_POST = auto()
MHC_ATTN_RES = auto()
MHC_ATTN_ALPHA = auto()
MHC_FFN_NORM = auto()
MHC_FFN_PRE = auto()
MHC_FFN_POST = auto()
MHC_FFN_RES = auto()
MHC_FFN_ALPHA = auto()
ATTN_COMPRESSOR_WKV = auto()
ATTN_COMPRESSOR_WGATE = auto()
ATTN_COMPRESSOR_APE = auto()
Expand Down Expand Up @@ -1163,6 +1178,7 @@ class MODEL_TENSOR(IntEnum):
MODEL_ARCH.TALKIE: "talkie",
MODEL_ARCH.MELLUM: "mellum",
MODEL_ARCH.NANBEIGE: "nanbeige",
MODEL_ARCH.MOTIF3: "motif3",
}

VISION_PROJECTOR_TYPE_NAMES: dict[VISION_PROJECTOR_TYPE, str] = {
Expand Down Expand Up @@ -1328,6 +1344,20 @@ class MODEL_TENSOR(IntEnum):
MODEL_TENSOR.HC_FFN_FN: "blk.{bid}.hc_ffn_fn",
MODEL_TENSOR.HC_FFN_BASE: "blk.{bid}.hc_ffn_base",
MODEL_TENSOR.HC_FFN_SCALE: "blk.{bid}.hc_ffn_scale",
MODEL_TENSOR.ATTN_LAMBDA: "blk.{bid}.attn_lambda",
MODEL_TENSOR.FFN_POLY: "blk.{bid}.ffn_poly",
MODEL_TENSOR.FFN_POLY_EXPS: "blk.{bid}.ffn_poly_exps",
MODEL_TENSOR.FFN_POLY_SHEXP: "blk.{bid}.ffn_poly_shexp",
MODEL_TENSOR.MHC_ATTN_NORM: "blk.{bid}.mhc_attn_norm",
MODEL_TENSOR.MHC_ATTN_PRE: "blk.{bid}.mhc_attn_pre",
MODEL_TENSOR.MHC_ATTN_POST: "blk.{bid}.mhc_attn_post",
MODEL_TENSOR.MHC_ATTN_RES: "blk.{bid}.mhc_attn_res",
MODEL_TENSOR.MHC_ATTN_ALPHA: "blk.{bid}.mhc_attn_alpha",
MODEL_TENSOR.MHC_FFN_NORM: "blk.{bid}.mhc_ffn_norm",
MODEL_TENSOR.MHC_FFN_PRE: "blk.{bid}.mhc_ffn_pre",
MODEL_TENSOR.MHC_FFN_POST: "blk.{bid}.mhc_ffn_post",
MODEL_TENSOR.MHC_FFN_RES: "blk.{bid}.mhc_ffn_res",
MODEL_TENSOR.MHC_FFN_ALPHA: "blk.{bid}.mhc_ffn_alpha",
MODEL_TENSOR.ATTN_COMPRESSOR_WKV: "blk.{bid}.attn_compressor_kv",
MODEL_TENSOR.ATTN_COMPRESSOR_WGATE: "blk.{bid}.attn_compressor_gate",
MODEL_TENSOR.ATTN_COMPRESSOR_APE: "blk.{bid}.attn_compressor_ape",
Expand Down Expand Up @@ -3170,6 +3200,46 @@ class MODEL_TENSOR(IntEnum):
MODEL_TENSOR.FFN_DOWN_SHEXP,
MODEL_TENSOR.FFN_UP_SHEXP,
],
MODEL_ARCH.MOTIF3: [
MODEL_TENSOR.TOKEN_EMBD,
MODEL_TENSOR.OUTPUT_NORM,
MODEL_TENSOR.OUTPUT,
MODEL_TENSOR.ATTN_NORM,
MODEL_TENSOR.ATTN_Q_A,
MODEL_TENSOR.ATTN_Q_A_NORM,
MODEL_TENSOR.ATTN_Q_B,
MODEL_TENSOR.ATTN_GATE,
MODEL_TENSOR.ATTN_KV_A_MQA,
MODEL_TENSOR.ATTN_KV_A_NORM,
MODEL_TENSOR.ATTN_KV_B,
MODEL_TENSOR.ATTN_LAMBDA,
MODEL_TENSOR.ATTN_OUT,
MODEL_TENSOR.FFN_NORM,
MODEL_TENSOR.FFN_GATE,
MODEL_TENSOR.FFN_UP,
MODEL_TENSOR.FFN_DOWN,
MODEL_TENSOR.FFN_POLY,
MODEL_TENSOR.FFN_GATE_INP,
MODEL_TENSOR.FFN_EXP_PROBS_B,
MODEL_TENSOR.FFN_GATE_EXP,
MODEL_TENSOR.FFN_UP_EXP,
MODEL_TENSOR.FFN_DOWN_EXP,
MODEL_TENSOR.FFN_POLY_EXPS,
MODEL_TENSOR.FFN_GATE_SHEXP,
MODEL_TENSOR.FFN_UP_SHEXP,
MODEL_TENSOR.FFN_DOWN_SHEXP,
MODEL_TENSOR.FFN_POLY_SHEXP,
MODEL_TENSOR.MHC_ATTN_NORM,
MODEL_TENSOR.MHC_ATTN_PRE,
MODEL_TENSOR.MHC_ATTN_POST,
MODEL_TENSOR.MHC_ATTN_RES,
MODEL_TENSOR.MHC_ATTN_ALPHA,
MODEL_TENSOR.MHC_FFN_NORM,
MODEL_TENSOR.MHC_FFN_PRE,
MODEL_TENSOR.MHC_FFN_POST,
MODEL_TENSOR.MHC_FFN_RES,
MODEL_TENSOR.MHC_FFN_ALPHA,
],
MODEL_ARCH.DEEPSEEK2: [
MODEL_TENSOR.TOKEN_EMBD,
MODEL_TENSOR.OUTPUT_NORM,
Expand Down
37 changes: 37 additions & 0 deletions src/llama-arch.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
{ LLM_ARCH_TALKIE, "talkie" },
{ LLM_ARCH_MELLUM, "mellum" },
{ LLM_ARCH_NANBEIGE, "nanbeige" },
{ LLM_ARCH_MOTIF3, "motif3" },
{ LLM_ARCH_UNKNOWN, "(unknown)" },
};

Expand Down Expand Up @@ -270,6 +271,14 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
{ LLM_KV_HYPER_CONNECTION_COUNT, "%s.hyper_connection.count" },
{ LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, "%s.hyper_connection.sinkhorn_iterations" },
{ LLM_KV_HYPER_CONNECTION_EPSILON, "%s.hyper_connection.epsilon" },
{ LLM_KV_MHC_H_POST_COEFF, "%s.hyper_connection.h_post_coeff" },
{ LLM_KV_ATTENTION_NOISE_HEAD_COUNT, "%s.attention.noise_head_count" },
{ LLM_KV_ATTENTION_YARN_MSCALE, "%s.attention.yarn_mscale" },
{ LLM_KV_POLYNORM_EPS, "%s.polynorm.epsilon" },
{ LLM_KV_POLYNORM_OUTPUT_SCALE, "%s.polynorm.output_scale" },
{ LLM_KV_POLYNORM_BIAS_CLAMP, "%s.polynorm.bias_clamp" },
{ LLM_KV_POLYNORM_HIDDEN_CLAMP, "%s.polynorm.hidden_clamp" },
{ LLM_KV_POLYNORM_SIGMOID_WEIGHT, "%s.polynorm.sigmoid_weight" },

{ LLM_KV_HASH_LAYER_COUNT, "%s.hash_layer_count" },

Expand Down Expand Up @@ -474,6 +483,20 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {
{ LLM_TENSOR_HC_FFN_FN, "blk.%d.hc_ffn_fn" },
{ LLM_TENSOR_HC_FFN_BASE, "blk.%d.hc_ffn_base" },
{ LLM_TENSOR_HC_FFN_SCALE, "blk.%d.hc_ffn_scale" },
{ LLM_TENSOR_ATTN_LAMBDA, "blk.%d.attn_lambda" },
{ LLM_TENSOR_FFN_POLY, "blk.%d.ffn_poly" },
{ LLM_TENSOR_FFN_POLY_EXPS, "blk.%d.ffn_poly_exps" },
{ LLM_TENSOR_FFN_POLY_SHEXP, "blk.%d.ffn_poly_shexp" },
{ LLM_TENSOR_MHC_ATTN_NORM, "blk.%d.mhc_attn_norm" },
{ LLM_TENSOR_MHC_ATTN_PRE, "blk.%d.mhc_attn_pre" },
{ LLM_TENSOR_MHC_ATTN_POST, "blk.%d.mhc_attn_post" },
{ LLM_TENSOR_MHC_ATTN_RES, "blk.%d.mhc_attn_res" },
{ LLM_TENSOR_MHC_ATTN_ALPHA, "blk.%d.mhc_attn_alpha" },
{ LLM_TENSOR_MHC_FFN_NORM, "blk.%d.mhc_ffn_norm" },
{ LLM_TENSOR_MHC_FFN_PRE, "blk.%d.mhc_ffn_pre" },
{ LLM_TENSOR_MHC_FFN_POST, "blk.%d.mhc_ffn_post" },
{ LLM_TENSOR_MHC_FFN_RES, "blk.%d.mhc_ffn_res" },
{ LLM_TENSOR_MHC_FFN_ALPHA, "blk.%d.mhc_ffn_alpha" },
{ LLM_TENSOR_ATTN_COMPRESSOR_WKV, "blk.%d.attn_compressor_kv" },
{ LLM_TENSOR_ATTN_COMPRESSOR_WGATE, "blk.%d.attn_compressor_gate" },
{ LLM_TENSOR_ATTN_COMPRESSOR_APE, "blk.%d.attn_compressor_ape" },
Expand Down Expand Up @@ -678,6 +701,20 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {
{LLM_TENSOR_HC_FFN_FN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_HC_FFN_BASE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}},
{LLM_TENSOR_HC_FFN_SCALE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_ATTN_LAMBDA, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_FFN_POLY, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_FFN_POLY_EXPS, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_GET_ROWS}},
{LLM_TENSOR_FFN_POLY_SHEXP, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_MHC_ATTN_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_MHC_ATTN_PRE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_MHC_ATTN_POST, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_MHC_ATTN_RES, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_MHC_ATTN_ALPHA, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_MHC_FFN_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_MHC_FFN_PRE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_MHC_FFN_POST, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_MHC_FFN_RES, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_MHC_FFN_ALPHA, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_ATTN_COMPRESSOR_WKV, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_ATTN_COMPRESSOR_WGATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_ATTN_COMPRESSOR_APE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_GET_ROWS}},
Expand Down
24 changes: 24 additions & 0 deletions src/llama-arch.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,7 @@ enum llm_arch {
LLM_ARCH_MINIMAX_M3,
LLM_ARCH_DFLASH,
LLM_ARCH_NANBEIGE,
LLM_ARCH_MOTIF3,
LLM_ARCH_UNKNOWN,
};

Expand Down Expand Up @@ -276,6 +277,15 @@ enum llm_kv {
LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS,
LLM_KV_HYPER_CONNECTION_EPSILON,

LLM_KV_ATTENTION_NOISE_HEAD_COUNT,
LLM_KV_ATTENTION_YARN_MSCALE,
LLM_KV_POLYNORM_EPS,
LLM_KV_POLYNORM_OUTPUT_SCALE,
LLM_KV_POLYNORM_BIAS_CLAMP,
LLM_KV_POLYNORM_HIDDEN_CLAMP,
LLM_KV_POLYNORM_SIGMOID_WEIGHT,
LLM_KV_MHC_H_POST_COEFF,

LLM_KV_HASH_LAYER_COUNT,

LLM_KV_ROPE_DIMENSION_COUNT,
Expand Down Expand Up @@ -539,6 +549,20 @@ enum llm_tensor {
LLM_TENSOR_HC_FFN_FN,
LLM_TENSOR_HC_FFN_BASE,
LLM_TENSOR_HC_FFN_SCALE,
LLM_TENSOR_ATTN_LAMBDA,
LLM_TENSOR_FFN_POLY,
LLM_TENSOR_FFN_POLY_EXPS,
LLM_TENSOR_FFN_POLY_SHEXP,
LLM_TENSOR_MHC_ATTN_NORM,
LLM_TENSOR_MHC_ATTN_PRE,
LLM_TENSOR_MHC_ATTN_POST,
LLM_TENSOR_MHC_ATTN_RES,
LLM_TENSOR_MHC_ATTN_ALPHA,
LLM_TENSOR_MHC_FFN_NORM,
LLM_TENSOR_MHC_FFN_PRE,
LLM_TENSOR_MHC_FFN_POST,
LLM_TENSOR_MHC_FFN_RES,
LLM_TENSOR_MHC_FFN_ALPHA,
LLM_TENSOR_ATTN_COMPRESSOR_WKV,
LLM_TENSOR_ATTN_COMPRESSOR_WGATE,
LLM_TENSOR_ATTN_COMPRESSOR_APE,
Expand Down
1 change: 1 addition & 0 deletions src/llama-context.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2339,6 +2339,7 @@ uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const {
model.arch == LLM_ARCH_QWEN35 ||
model.arch == LLM_ARCH_QWEN35MOE ||
model.arch == LLM_ARCH_DEEPSEEK4 ||
model.arch == LLM_ARCH_MOTIF3 ||
model.arch == LLM_ARCH_NANBEIGE ||
model.arch == LLM_ARCH_MINIMAX_M3) {
return std::max<uint32_t>(n_tokens * 40, 32u * model.n_tensors());
Expand Down
12 changes: 12 additions & 0 deletions src/llama-hparams.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,18 @@ struct llama_hparams {
// DeepSeek-V4
uint32_t dsv4_o_group_count = 0;
uint32_t dsv4_o_lora_rank = 0;
// Motif-3
uint32_t motif_n_noise_heads = 0;
float motif_mscale = 1.0f;
float motif_poly_eps = 1e-6f;
float motif_poly_out_scale = 1.0f;
float motif_poly_bias_clamp = 0.0f;
float motif_poly_hidden_clamp = 0.0f;
bool motif_poly_sigmoid_w = true;
uint32_t motif_mhc_mult = 0;
uint32_t motif_mhc_iters = 20;
float motif_mhc_post_coeff = 1.0f;

uint32_t dsv4_hc_mult = 0;
uint32_t dsv4_hc_sinkhorn_iters = 0;
uint32_t dsv4_hash_layer_count = 0;
Expand Down
4 changes: 4 additions & 0 deletions src/llama-model.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
return new llama_model_mellum(params);
case LLM_ARCH_NANBEIGE:
return new llama_model_nanbeige(params);
case LLM_ARCH_MOTIF3:
return new llama_model_motif3(params);
case LLM_ARCH_QWEN:
return new llama_model_qwen(params);
case LLM_ARCH_QWEN2:
Expand Down Expand Up @@ -826,6 +828,7 @@ const char * llm_type_name(llm_type type) {
case LLM_TYPE_235B_A22B: return "235B.A22B";
case LLM_TYPE_300B_A47B: return "300B.A47B";
case LLM_TYPE_310B_A15B: return "310B.A15B";
case LLM_TYPE_314B_A13B: return "314B.A13B";
case LLM_TYPE_355B_A32B: return "355B.A32B";
case LLM_TYPE_397B_A17B: return "397B.A17B";
case LLM_TYPE_685B_A37B: return "685B.A37B";
Expand Down Expand Up @@ -2528,6 +2531,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
case LLM_ARCH_GEMMA:
case LLM_ARCH_GEMMA2:
case LLM_ARCH_GEMMA3:
case LLM_ARCH_MOTIF3:
case LLM_ARCH_GEMMA3N:
case LLM_ARCH_GEMMA4:
case LLM_ARCH_GEMMA4_ASSISTANT:
Expand Down
27 changes: 27 additions & 0 deletions src/llama-model.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,7 @@ enum llm_type {
LLM_TYPE_235B_A22B,
LLM_TYPE_300B_A47B, // Ernie MoE big
LLM_TYPE_310B_A15B, // /MiMo-V2-Flash
LLM_TYPE_314B_A13B, // Motif-3
LLM_TYPE_355B_A32B, // GLM-4.5
LLM_TYPE_397B_A17B, // Qwen3.5
LLM_TYPE_685B_A37B, // DeepSeek V3.2
Expand Down Expand Up @@ -336,6 +337,32 @@ struct llama_layer {
struct ggml_tensor * ffn_up_b = nullptr; // b3
struct ggml_tensor * ffn_act = nullptr;
struct ggml_tensor * ffn_exp_probs_b = nullptr;

// Motif-3
struct ggml_tensor * wq_b_gate = nullptr;
struct ggml_tensor * attn_lambda = nullptr;
struct ggml_tensor * ffn_poly_w = nullptr;
struct ggml_tensor * ffn_poly_b = nullptr;
struct ggml_tensor * ffn_poly_exps_w = nullptr;
struct ggml_tensor * ffn_poly_exps_b = nullptr;
struct ggml_tensor * ffn_poly_shexp_w = nullptr;
struct ggml_tensor * ffn_poly_shexp_b = nullptr;
struct ggml_tensor * mhc_attn_norm = nullptr;
struct ggml_tensor * mhc_attn_pre = nullptr;
struct ggml_tensor * mhc_attn_pre_b = nullptr;
struct ggml_tensor * mhc_attn_post = nullptr;
struct ggml_tensor * mhc_attn_post_b = nullptr;
struct ggml_tensor * mhc_attn_res = nullptr;
struct ggml_tensor * mhc_attn_res_b = nullptr;
struct ggml_tensor * mhc_attn_alpha = nullptr;
struct ggml_tensor * mhc_ffn_norm = nullptr;
struct ggml_tensor * mhc_ffn_pre = nullptr;
struct ggml_tensor * mhc_ffn_pre_b = nullptr;
struct ggml_tensor * mhc_ffn_post = nullptr;
struct ggml_tensor * mhc_ffn_post_b = nullptr;
struct ggml_tensor * mhc_ffn_res = nullptr;
struct ggml_tensor * mhc_ffn_res_b = nullptr;
struct ggml_tensor * mhc_ffn_alpha = nullptr;
struct ggml_tensor * ffn_gate_tid2eid = nullptr;

// mamba proj
Expand Down
2 changes: 2 additions & 0 deletions src/llama-vocab.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2596,6 +2596,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
|| t.first == "<|im_end|>"
|| t.first == "<|end|>"
|| t.first == "<end_of_turn>"
|| t.first == "<|endofturn|>" // Motif-3
|| t.first == "<|endoftext|>"
|| t.first == "<|end_of_text|>" // granite
|| t.first == "<EOT>"
Expand Down Expand Up @@ -2796,6 +2797,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
|| t.first == "<|flush|>" // solar-open
|| t.first == "<|calls|>" // solar-open
|| t.first == "<end_of_turn>"
|| t.first == "<|endofturn|>" // Motif-3
|| t.first == "<|endoftext|>"
|| t.first == "</s>" // paddleocr
|| t.first == "<|eom_id|>"
Expand Down
Loading
Loading