mirror of
https://github.com/NVIDIA/TensorRT-LLM.git
synced 2026-01-13 22:18:36 +08:00
31bc14b350
Signed-off-by: Chang Liu (Enterprise Products) <9713593+chang-l@users.noreply.github.com>
22 lines
943 B
Python
22 lines
943 B
Python
# Backward compatibility shim - the tokenizer module has moved to tensorrt_llm.tokenizer
|
|
# All imports from tensorrt_llm.llmapi.tokenizer will continue to work.
|
|
from tensorrt_llm.tokenizer import (TLLM_INCREMENTAL_DETOKENIZATION_BACKEND,
|
|
TLLM_STREAM_INTERVAL_THRESHOLD,
|
|
TokenizerBase, TransformersTokenizer,
|
|
_llguidance_tokenizer_info,
|
|
_xgrammar_tokenizer_info, load_hf_tokenizer,
|
|
tokenizer_factory)
|
|
from tensorrt_llm.tokenizer.deepseek_v32 import DeepseekV32Tokenizer
|
|
|
|
__all__ = [
|
|
"TLLM_INCREMENTAL_DETOKENIZATION_BACKEND",
|
|
"TLLM_STREAM_INTERVAL_THRESHOLD",
|
|
"TokenizerBase",
|
|
"TransformersTokenizer",
|
|
"DeepseekV32Tokenizer",
|
|
"tokenizer_factory",
|
|
"_xgrammar_tokenizer_info",
|
|
"_llguidance_tokenizer_info",
|
|
"load_hf_tokenizer",
|
|
]
|