[Bugfix] Update mistral tokenizer test for continue_final_message fix (#44622)

Signed-off-by: Xu Zhou <xuzhou9417@163.com>
Co-authored-by: Xu Zhou <xuzhou9417@163.com>
This commit is contained in:
XuZhou
2026-06-05 16:13:26 +08:00
committed by GitHub
parent d2f70da116
commit d61d8566ec
+2 -2
View File
@@ -797,11 +797,11 @@ class TestMistralTokenizer:
True, True,
( (
[1, 3, 23325, 2294, 1686, 4, 23325], [1, 3, 23325, 2294, 1686, 4, 23325],
[1, 3, 22177, 4304, 2662, 4, 22177, 2], [1, 3, 22177, 4304, 2662, 4, 22177],
), ),
( (
"<s>[INST]▁Hello▁world▁![/INST]▁Hello", "<s>[INST]▁Hello▁world▁![/INST]▁Hello",
("<s>[INST]Hello world ![/INST]Hello</s>"), "<s>[INST]Hello world ![/INST]Hello",
), ),
), ),
], ],