[Tokenizer] Use HF config for HF tokenizers (#49907)

Signed-off-by: Andreas Karatzas <[email protected]>

vllm/tokenizers/registry.py

@@ -205,17 +205,27 @@ def get_tokenizer(
         **kwargs,
     )
 
+    if tokenizer_cls == TokenizerLike:
+        tokenizer_cls_ = TokenizerRegistry.load_tokenizer_cls(tokenizer_mode)
+    else:
+        tokenizer_cls_ = tokenizer_cls
+
     # Ensure that, if the config were to come from vllm.transformers_utils.config, it is
     # registered with AutoConfig before the tokenizer is loaded. This is necessary since
     # tokenizer_cls_.from_pretrained will call AutoConfig.from_pretrained internally.
     # This may fail for paths that don't have a model config (e.g. LoRA adapters),
     # which is fine — those don't need custom config registration.
+    # HF-backed tokenizers must receive the HF config. In a dual-format Mistral
+    # repository, auto detection intentionally prefers params.json, but passing
+    # that generic config to AutoTokenizer can select the wrong tokenizer class.
+    config_format = "hf" if tokenizer_cls_ is CachedHfTokenizer else "auto"
     config = None
     with contextlib.suppress(ValueError, OSError):
         config = get_config(
             tokenizer_name,
             trust_remote_code=trust_remote_code,
             revision=revision,
+            config_format=config_format,
         )
 
     # Some models have an incorrect tokenizer_class on the hub.
@@ -229,10 +239,6 @@ def get_tokenizer(
             model_type,
         )
         tokenizer_cls_ = TokenizersBackend
-    elif tokenizer_cls == TokenizerLike:
-        tokenizer_cls_ = TokenizerRegistry.load_tokenizer_cls(tokenizer_mode)
-    else:
-        tokenizer_cls_ = tokenizer_cls
 
     if config is not None and tokenizer_cls_ is CachedHfTokenizer:
         # AutoTokenizer otherwise reloads config.json internally. Reuse the