mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-07-10 05:00:29 +00:00
llama : fix command-r detokenization
This commit is contained in:
@@ -5407,6 +5407,7 @@ static void llm_load_vocab(
|
||||
} else if (
|
||||
tokenizer_pre == "command-r") {
|
||||
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_COMMAND_R;
|
||||
vocab.tokenizer_clean_spaces = false;
|
||||
} else if (
|
||||
tokenizer_pre == "qwen2") {
|
||||
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_QWEN2;
|
||||
|
||||
Reference in New Issue
Block a user