[Model] Fused mm preprocess normalisation on the Device (#50411)

Signed-off-by: wang.yuqi <[email protected]>
Signed-off-by: wang.yuqi <[email protected]>
Co-authored-by: Cyrus Leung <[email protected]>

docs/design/mm_processing.md

@@ -61,3 +61,45 @@ Some HF processors, such as the one for Qwen2-VL, are [very slow](https://github
 When new data is passed in, we first check which items are in the cache, and which ones are missing. The missing items are passed into the HF processor in a single batch and cached, before being merged with the existing items in the cache.
 
 Since we only process the missing multi-modal data items, the number of input placeholder tokens no longer corresponds to the number of the multi-modal inputs, so they can't be passed alongside the text prompt to HF processor. Therefore, we process the text and multi-modal inputs separately, using [dummy text](#dummy-text) to avoid HF errors. Since this skips HF's prompt updating code, we apply [automatic prompt updating](#automatic-prompt-updating) afterwards to keep the output tokens and multi-modal data consistent with each other.
+
+## Speeding Up Multi‑Modal Data Processing
+
+### Fused Normalisation on the Device
+
+To accelerate the multi‑modal data pipeline (decoding, resizing, normalisation, and rescaling), we offload the heavy numerical preprocessing from the CPU to the GPU and optimise data movement.
+
+#### Fusing Normalisation and Rescaling on the GPU
+
+Traditionally, the CPU would divide pixel values by 255, then subtract the mean and divide by the standard deviation. We fuse these steps into one operation and run it entirely on the GPU.
+
+- **How it works**: We use a dedicated `FusedInputNorm` module — essentially a frozen `BatchNorm1d` with the rescale factor baked into its statistics — and bake the rescale factor (typically 1/255) directly into the effective mean and standard deviation:
+    - Effective mean = `image_mean * (1/rescale_factor)`
+    - Effective std  = `image_std  * (1/rescale_factor)`
+- **At runtime**: The `FusedInputNorm` layer takes raw `uint8` pixel values (0–255) and performs the full normalised mapping in a single GPU kernel—no CPU involvement.
+
+#### Optimized Data Path for Fused Normalisation
+
+Performing fused normalisation directly on the device allows us to keep the entire transfer path—from **Entrypoint** through **Engine Core** to **GPU memory**—in **`uint8`**. This halves PCIe bandwidth and reduces CPU memory footprint.
+
+Only after data reaches GPU memory do we cast to `fp32` for the `FusedInputNorm` layer (to ensure numerical accuracy), then cast to `bf16` for subsequent layers—all within the GPU, avoiding any host‑side conversions.
+
+Overall path: **`Entrypoint (uint8) → Engine Core (uint8) → GPU Memory (uint8)`** → GPU‑local `fp32` `FusedInputNorm` → `bf16` output.
+
+#### Toggle: `mm_device_do_normalize`
+
+This GPU‑side fusion is controlled by a config flag called **`mm_device_do_normalize`**.
+
+- When `True`, normalisation and rescaling are done on the GPU using the `FusedInputNorm` layer; when `False`, we fall back to the old CPU‑side path.
+- The flag is **enabled by default** for all models that support it.
+- Currently, it’s on by default for these architectures:
+
+| name         | Architecture                         | Example HF Models                   |
+|--------------|--------------------------------------|-------------------------------------|
+| `qwen2-vl`   | `Qwen2VLForConditionalGeneration`    | `Qwen/Qwen2-VL-2B-Instruct`, etc.   |
+| `qwen2.5-vl` | `Qwen2_5_VLForConditionalGeneration` | `Qwen/Qwen2.5-VL-3B-Instruct`, etc. |
+
+#### What We Gain Overall
+
+- **CPU offload**: The arithmetic for normalisation and rescaling is completely gone from the CPU.
+- **PCIe savings**: Sending `uint8` (1 byte) instead of `bf16` (2 bytes) slashes data transfer volume by **50%** .
+- **GPU overhead**: The fused kernel is very lightweight and can often be merged with subsequent CUDA operations, so it hardly adds any extra cost.

tests/models/multimodal/generation_ppl_test/ppl_utils.py

@@ -72,6 +72,7 @@ def vqa_ppl_test(
         # and avoid batch different requests together.
 
         model_config = vllm_model.llm.llm_engine.model_config
+        mm_device_do_normalize = model_config.multimodal_config.mm_device_do_normalize
 
         # Confirm whether vllm is using the correct architecture
         if model_info.architecture:
@@ -147,6 +148,7 @@ def vqa_ppl_test(
 
     differ = (vllm_ppl - hf_ppl) / hf_ppl
     print("Model:", model_info.name)
+    print("mm_device_do_normalize:", mm_device_do_normalize)
     print("VLLM:", f"dtype:{vllm_dtype}", f"head_dtype:{head_dtype}", vllm_ppl)
     print("Transformers:", hf_dtype, hf_ppl)
     print("Difference (%):", differ * 100)

tests/models/multimodal/generation_ppl_test/test_qwen.py

@@ -8,8 +8,8 @@ from tests.models.utils import GenerateModelInfo
 from .ppl_utils import vqa_ppl_test
 
 MODELS = [
-    GenerateModelInfo("Qwen/Qwen2-VL-2B-Instruct"),
-    GenerateModelInfo("Qwen/Qwen2.5-VL-3B-Instruct"),
+    GenerateModelInfo("Qwen/Qwen2-VL-2B-Instruct", hf_ppl=41081356.0),
+    GenerateModelInfo("Qwen/Qwen2.5-VL-3B-Instruct", hf_ppl=18330016.0),
 ]
 
 
@@ -20,7 +20,14 @@ mm_processor_kwargs = {
 
 
 @pytest.mark.parametrize("model_info", MODELS)
-def test_ppl(hf_runner, vllm_runner, model_info: GenerateModelInfo):
+@pytest.mark.parametrize("mm_device_do_normalize", [True, False])
+def test_ppl(
+    hf_runner, vllm_runner, model_info: GenerateModelInfo, mm_device_do_normalize: bool
+):
     vqa_ppl_test(
-        hf_runner, vllm_runner, model_info, mm_processor_kwargs=mm_processor_kwargs
+        hf_runner,
+        vllm_runner,
+        model_info,
+        vllm_extra_kwargs={"mm_device_do_normalize": mm_device_do_normalize},
+        mm_processor_kwargs=mm_processor_kwargs,
     )

tests/models/multimodal/processing/test_qwen2_vl.py

@@ -2,9 +2,11 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
 import pytest
+import torch
 from packaging.version import Version
 from transformers import __version__ as TRANSFORMERS_VERSION
 
+from vllm.model_executor.models.vision import FusedInputNorm
 from vllm.multimodal import MULTIMODAL_REGISTRY
 
 from ....conftest import ImageTestAssets
@@ -128,3 +130,52 @@ def test_get_image_size_with_most_features(
         t, h, w = grid_thw[0]
         tokens = (t * h * w) // (merge_size**2)
         assert tokens < max_tokens
+
+
+@pytest.mark.parametrize(
+    "model_id", ["Qwen/Qwen2-VL-2B-Instruct", "Qwen/Qwen2.5-VL-3B-Instruct"]
+)
+@pytest.mark.parametrize("num_imgs", [1, 2])
+def test_mm_device_do_normalize(
+    image_assets: ImageTestAssets,
+    model_id: str,
+    num_imgs: int,
+):
+    """Ensure that enable mm_device_do_normalize yields the correct result."""
+
+    ctx = build_model_context(
+        model_id,
+        limit_mm_per_prompt={"image": num_imgs},
+    )
+    ctx.model_config.multimodal_config.mm_device_do_normalize = False
+    processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
+
+    # Build the image str / prompt based on the number of images we pass
+    prompt = "<|vision_start|><|image_pad|><|vision_end|>" * num_imgs
+    mm_data = {"image": [image_assets[0].pil_image] * num_imgs}
+
+    processed_inputs_with_normalize = processor(
+        prompt,
+        mm_items=processor.info.parse_mm_data(mm_data),
+    )
+    pixel_values_with_normalize = processed_inputs_with_normalize[
+        "mm_kwargs"
+    ].get_data()["pixel_values"]
+    dtype = pixel_values_with_normalize.dtype
+
+    processed_inputs_without_normalize = processor(
+        prompt,
+        mm_items=processor.info.parse_mm_data(mm_data),
+        hf_processor_mm_kwargs={"do_normalize": False, "do_rescale": False},
+    )
+    pixel_values_without_normalize = processed_inputs_without_normalize[
+        "mm_kwargs"
+    ].get_data()["pixel_values"]
+
+    ctx.model_config.multimodal_config.mm_device_do_normalize = True
+    input_norm = FusedInputNorm.from_model_config(ctx.model_config)
+    pixel_values_do_input_norm = input_norm(
+        pixel_values_without_normalize.to(dtype), dtype
+    )
+
+    torch.testing.assert_close(pixel_values_with_normalize, pixel_values_do_input_norm)

vllm/config/model.py

@@ -396,6 +396,7 @@ class ModelConfig:
     video_pruning_method: InitVar[str | None] = None
     mm_tensor_ipc: InitVar[MMTensorIPC] = None
     mm_ipc_gpu_memory_gb: InitVar[float | None] = None
+    mm_device_do_normalize: InitVar[bool | None] = None
     mm_processor_device: InitVar[MMProcessorDevice | None] = None
 
     def compute_hash(self) -> str:
@@ -527,6 +528,7 @@ class ModelConfig:
         video_pruning_method: str | None,
         mm_tensor_ipc: MMTensorIPC,
         mm_ipc_gpu_memory_gb: float | None,
+        mm_device_do_normalize: bool | None,
         mm_processor_device: MMProcessorDevice | None,
     ) -> None:
         # Keep set served_model_name before maybe_model_redirect(self.model)
@@ -793,6 +795,9 @@ class ModelConfig:
                 video_pruning_method=video_pruning_method,
                 mm_tensor_ipc=mm_tensor_ipc,
                 mm_ipc_gpu_memory_gb=mm_ipc_gpu_memory_gb,
+                mm_device_do_normalize=self._resolve_mm_device_do_normalize(
+                    mm_device_do_normalize
+                ),
             )
 
             mm_config_kwargs = {
@@ -927,6 +932,51 @@ class ModelConfig:
             )
         return self
 
+    def _resolve_mm_device_do_normalize(
+        self, mm_device_do_normalize: bool | None
+    ) -> bool:
+        if mm_device_do_normalize is None:
+            if envs.VLLM_USE_RUST_FRONTEND:
+                logger.debug(
+                    "VLLM_USE_RUST_FRONTEND is set. "
+                    "Rust frontend does not currently support mm_device_do_normalize, "
+                    "forcing mm_device_do_normalize = False."
+                )
+                mm_device_do_normalize = False
+            else:
+                mm_device_do_normalize = (
+                    self._model_info.supports_mm_device_do_normalize
+                )
+                logger.debug(
+                    "mm_device_do_normalize is %s by default.",
+                    "enabled" if mm_device_do_normalize else "disabled",
+                )
+        else:
+            if mm_device_do_normalize and envs.VLLM_USE_RUST_FRONTEND:
+                logger.warning(
+                    "VLLM_USE_RUST_FRONTEND is set. "
+                    "Rust frontend does not currently support mm_device_do_normalize, "
+                    "forcing mm_device_do_normalize = False."
+                )
+                mm_device_do_normalize = False
+
+            if (
+                mm_device_do_normalize
+                and not self._model_info.supports_mm_device_do_normalize
+            ):
+                logger.warning(
+                    "Model does not support mm_device_do_normalize, "
+                    "forcing mm_device_do_normalize = False."
+                )
+                mm_device_do_normalize = False
+
+            logger.debug(
+                "mm_device_do_normalize is %s.",
+                "enabled" if mm_device_do_normalize else "disabled",
+            )
+
+        return mm_device_do_normalize
+
     def _get_transformers_backend_cls(self) -> str:
         """Determine which Transformers modeling backend class will be used if
         `model_impl` is set to `transformers` or `auto`."""

vllm/config/multimodal.py

@@ -144,6 +144,11 @@ class MultiModalConfig:
 
     For example, for Phi-3-Vision:
     `{"num_crops": 4}`."""
+    mm_device_do_normalize: bool | None = True
+    """
+    Move the do_normalize computation in the mm preprocessing to before the ViT, 
+    and let the device do it, so that CPU computation can be saved.
+    """
     mm_processor_cache_gb: float = Field(default=4, ge=0)
     """The size (in GiB) of the multi-modal processor cache, which is used to
     avoid re-processing past multi-modal inputs.
@@ -474,6 +479,7 @@ class MultiModalConfig:
             self.mm_encoder_tp_mode,
             self.mm_encoder_attn_dtype,
             self.mm_encoder_fp8_scale_path,
+            self.mm_device_do_normalize,
         ]
         hash_str = safe_hash(str(factors).encode(), usedforsecurity=False).hexdigest()
         return hash_str
@@ -503,6 +509,9 @@ class MultiModalConfig:
         according to the extra arguments passed during inference.
         """
         kwargs = self.mm_processor_kwargs or {}
+        if self.mm_device_do_normalize:
+            kwargs["do_normalize"] = False
+            kwargs["do_rescale"] = False
         return kwargs | dict(inference_kwargs)
 
     def use_gpu_video_backend(self) -> bool:

vllm/engine/arg_utils.py

@@ -599,6 +599,7 @@ class EngineArgs:
     mm_tensor_ipc: MMTensorIPC = MultiModalConfig.mm_tensor_ipc
     mm_processor_device: MMProcessorDevice = "auto"
     mm_ipc_gpu_memory_gb: float = MultiModalConfig.mm_ipc_gpu_memory_gb
+    mm_device_do_normalize: bool | None = MultiModalConfig.mm_device_do_normalize
     # LoRA fields
     enable_lora: bool = False
     max_loras: int = LoRAConfig.max_loras
@@ -1383,6 +1384,13 @@ class EngineArgs:
             "--mm-ipc-gpu-memory-gb",
             **multimodal_kwargs["mm_ipc_gpu_memory_gb"],
         )
+        multimodal_group.add_argument(
+            "--mm-device-do-normalize",
+            **{
+                **multimodal_kwargs["mm_device_do_normalize"],
+                "default": None,
+            },
+        )
 
         # LoRA related configs
         lora_kwargs = get_kwargs(LoRAConfig)
@@ -1769,6 +1777,7 @@ class EngineArgs:
             video_pruning_method=self.video_pruning_method,
             mm_tensor_ipc=self.mm_tensor_ipc,
             mm_ipc_gpu_memory_gb=self.mm_ipc_gpu_memory_gb,
+            mm_device_do_normalize=self.mm_device_do_normalize,
             mm_processor_device=self.mm_processor_device,
             io_processor_plugin=self.io_processor_plugin,
             renderer_num_workers=self.renderer_num_workers,

vllm/model_executor/models/interfaces.py

@@ -149,6 +149,12 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
     `multimodal_config.mm_encoder_tp_mode="data"`.
     """
 
+    supports_mm_device_do_normalize: ClassVar[bool] = False
+    """
+    A flag that indicates whether this model supports
+    `multimodal_config.mm_device_do_normalize`.
+    """
+
     requires_raw_input_tokens: ClassVar[bool] = False
     """
     A flag that indicates this model processes input id tokens

vllm/model_executor/models/qwen2_5_vl.py

@@ -112,6 +112,7 @@ from .utils import (
     maybe_prefix,
 )
 from .vision import (
+    FusedInputNorm,
     get_fp8_padded_hidden_size,
     get_vit_attn_backend,
     is_vit_use_data_parallel,
@@ -1254,6 +1255,7 @@ class Qwen2_5_VLForConditionalGeneration(
     )
 
     supports_encoder_tp_data = True
+    supports_mm_device_do_normalize = True
 
     def iter_mm_grid_thw(
         self, mm_features: list[MultiModalFeatureSpec]
@@ -1363,6 +1365,7 @@ class Qwen2_5_VLForConditionalGeneration(
                 quant_config=self.quant_config,
                 prefix=maybe_prefix(prefix, "visual"),
             )
+            self.input_norm = FusedInputNorm.from_model_config(self.model_config)
 
         with self._mark_language_model(vllm_config):
             self.language_model = init_vllm_registered_model(
@@ -1437,6 +1440,8 @@ class Qwen2_5_VLForConditionalGeneration(
             image_embeds = image_input["image_embeds"].type(self.visual.dtype)
         else:
             pixel_values = image_input["pixel_values"]
+            pixel_values = self.input_norm(pixel_values, self.visual.dtype)
+
             if self.use_data_parallel:
                 return run_dp_sharded_mrope_vision_model(
                     self.visual, pixel_values, grid_thw_list, rope_type="rope_3d"
@@ -1493,6 +1498,10 @@ class Qwen2_5_VLForConditionalGeneration(
             video_embeds = video_input["video_embeds"].type(self.visual.dtype)
         else:
             pixel_values_videos = video_input["pixel_values_videos"]
+            pixel_values_videos = self.input_norm(
+                pixel_values_videos, self.visual.dtype
+            )
+
             if self.use_data_parallel:
                 return run_dp_sharded_mrope_vision_model(
                     self.visual,

vllm/model_executor/models/qwen2_vl.py

@@ -103,6 +103,7 @@ from .utils import (
     maybe_prefix,
 )
 from .vision import (
+    FusedInputNorm,
     get_vit_attn_backend,
     is_vit_use_data_parallel,
     run_dp_sharded_mrope_vision_model,
@@ -1192,6 +1193,7 @@ class Qwen2VLForConditionalGeneration(
     )
 
     supports_encoder_tp_data = True
+    supports_mm_device_do_normalize = True
 
     def iter_mm_grid_thw(
         self, mm_features: list[MultiModalFeatureSpec]
@@ -1296,6 +1298,7 @@ class Qwen2VLForConditionalGeneration(
                 quant_config=quant_config,
                 prefix=maybe_prefix(prefix, "visual"),
             )
+            self.input_norm = FusedInputNorm.from_model_config(self.model_config)
 
         with self._mark_language_model(vllm_config):
             self.language_model = init_vllm_registered_model(
@@ -1363,9 +1366,10 @@ class Qwen2VLForConditionalGeneration(
         assert grid_thw.ndim == 2
 
         if image_input["type"] == "image_embeds":
-            image_embeds = image_input["image_embeds"]
+            image_embeds = image_input["image_embeds"].type(self.visual.dtype)
         else:
             pixel_values = image_input["pixel_values"]
+            pixel_values = self.input_norm(pixel_values, self.visual.dtype)
 
             if self.use_data_parallel:
                 return run_dp_sharded_mrope_vision_model(
@@ -1386,9 +1390,12 @@ class Qwen2VLForConditionalGeneration(
         assert grid_thw.ndim == 2
 
         if video_input["type"] == "video_embeds":
-            video_embeds = video_input["video_embeds"]
+            video_embeds = video_input["video_embeds"].type(self.visual.dtype)
         else:
             pixel_values_videos = video_input["pixel_values_videos"]
+            pixel_values_videos = self.input_norm(
+                pixel_values_videos, self.visual.dtype
+            )
             if self.use_data_parallel:
                 return run_dp_sharded_mrope_vision_model(
                     self.visual,

vllm/model_executor/models/registry.py

@@ -814,6 +814,7 @@ class _ModelInfo:
     supports_transcription: bool
     supports_transcription_only: bool
     supported_video_pruning_methods: tuple[str, ...]
+    supports_mm_device_do_normalize: bool
 
     @staticmethod
     def from_model_cls(model: type[nn.Module]) -> "_ModelInfo":
@@ -847,6 +848,9 @@ class _ModelInfo:
             supported_video_pruning_methods=getattr(
                 model, "supported_video_pruning_methods", ()
             ),
+            supports_mm_device_do_normalize=getattr(
+                model, "supports_mm_device_do_normalize", False
+            ),
         )
 
 

vllm/model_executor/models/vision.py

@@ -8,9 +8,11 @@ from collections.abc import Callable
 from typing import Final, Generic, Literal, Protocol, TypeAlias, TypeVar
 
 import torch
+import torch.nn as nn
+import torch.nn.functional as F
 from transformers import PretrainedConfig
 
-from vllm.config import MultiModalConfig, get_current_vllm_config_or_none
+from vllm.config import ModelConfig, MultiModalConfig, get_current_vllm_config_or_none
 from vllm.distributed import (
     get_tensor_model_parallel_rank,
     get_tensor_model_parallel_world_size,
@@ -18,6 +20,7 @@ from vllm.distributed import (
 )
 from vllm.logger import init_logger
 from vllm.platforms import current_platform
+from vllm.transformers_utils.processor import get_processor, get_processor_config
 from vllm.utils.math_utils import round_up
 from vllm.v1.attention.backends.registry import AttentionBackendEnum
 
@@ -580,3 +583,140 @@ def run_dp_sharded_mrope_vision_model(
         "Found unassigned embeddings"
     )
     return out_embeddings
+
+
+class FusedInputNorm(nn.Module):
+    """
+    Module that applies rescaling and normalization to input images.
+    Equivalent to: output = (input * rescale_factor - mean) / std
+    """
+
+    def __init__(
+        self,
+        image_mean: list[float],
+        image_std: list[float],
+        rescale_factor: float,
+        channel: int = 3,
+        dtype: torch.dtype = torch.float32,
+    ):
+        super().__init__()
+
+        self.channel = channel
+
+        image_mean_tensor = torch.tensor(image_mean, dtype=dtype) * (
+            1.0 / rescale_factor
+        )
+        image_std_tensor = torch.tensor(image_std, dtype=dtype) * (1.0 / rescale_factor)
+        weight = 1.0 / image_std_tensor
+        bias = -image_mean_tensor / image_std_tensor
+
+        self.is_identity = bool(
+            torch.allclose(weight, torch.ones_like(weight))
+            and torch.allclose(bias, torch.zeros_like(bias))
+        )
+
+        if not self.is_identity:
+            self.register_buffer("weight", weight)
+            self.register_buffer("bias", bias)
+            self.register_buffer("running_mean", torch.zeros_like(image_mean_tensor))
+            self.register_buffer("running_var", torch.ones_like(image_mean_tensor))
+        else:
+            self.register_buffer("weight", None)
+            self.register_buffer("bias", None)
+            self.register_buffer("running_mean", None)
+            self.register_buffer("running_var", None)
+
+    @property
+    def dtype(self) -> torch.dtype:
+        return self.weight.dtype
+
+    @classmethod
+    def identity(
+        cls, channel: int = 3, dtype: torch.dtype = torch.float32
+    ) -> "FusedInputNorm":
+        return cls(
+            image_mean=[0.0, 0.0, 0.0],
+            image_std=[1.0, 1.0, 1.0],
+            rescale_factor=1.0,
+            channel=channel,
+            dtype=dtype,
+        )
+
+    @classmethod
+    def from_model_config(cls, model_config: "ModelConfig") -> nn.Module:
+        if not model_config.multimodal_config.mm_device_do_normalize:
+            return cls.identity()
+
+        model = model_config.model
+        revision = model_config.revision
+
+        # Try to read parameters from the processor config
+        config = get_processor_config(model, revision=revision)
+        do_rescale = config.get("do_rescale", None)
+        do_normalize = config.get("do_normalize", None)
+        image_mean = config.get("image_mean", None)
+        image_std = config.get("image_std", None)
+        rescale_factor = config.get("rescale_factor", None)
+
+        # Fallback to the image_processor object if any parameter is missing
+        if None in [do_rescale, do_normalize, image_mean, image_std, rescale_factor]:
+            image_processor = get_processor(model, revision=revision).image_processor
+
+            if do_rescale is None:
+                do_rescale = getattr(image_processor, "do_rescale", None)
+            if do_normalize is None:
+                do_normalize = getattr(image_processor, "do_normalize", None)
+            if image_mean is None:
+                image_mean = getattr(image_processor, "image_mean", None)
+            if image_std is None:
+                image_std = getattr(image_processor, "image_std", None)
+            if rescale_factor is None:
+                rescale_factor = getattr(image_processor, "rescale_factor", None)
+
+        # Apply defaults based on flags
+        if not do_rescale:
+            rescale_factor = 1.0
+        if not do_normalize:
+            image_mean = [0.0, 0.0, 0.0]
+            image_std = [1.0, 1.0, 1.0]
+
+        # Ensure all required parameters are resolved
+        assert None not in [
+            do_rescale,
+            do_normalize,
+            image_mean,
+            image_std,
+            rescale_factor,
+        ], "Some normalization parameters are still None after resolution."
+
+        # If no processing is needed, return an identity module
+        if not do_rescale and not do_normalize:
+            return cls.identity()
+
+        return cls(
+            image_mean=image_mean, image_std=image_std, rescale_factor=rescale_factor
+        )
+
+    def forward(
+        self,
+        grid_thw: torch.Tensor,
+        visual_dtype: torch.dtype,
+    ) -> torch.Tensor:
+        if self.is_identity:
+            return grid_thw.to(visual_dtype)
+
+        assert grid_thw.ndim == 2
+        patches, size = grid_thw.shape
+        patch_size = size // self.channel
+
+        grid_thw = grid_thw.view(patches, self.channel, patch_size)
+        grid_thw = F.batch_norm(
+            grid_thw.to(self.dtype),
+            running_mean=self.running_mean,
+            running_var=self.running_var,
+            weight=self.weight,
+            bias=self.bias,
+            training=False,
+            eps=0.0,
+        )
+        return grid_thw.view(patches, size).to(visual_dtype)

vllm/transformers_utils/processor.py

@@ -142,10 +142,10 @@ def _merge_mm_kwargs(
     return allowed_kwargs
 
 
-def get_processor_cls_name_from_config(
+def get_processor_config(
     processor_name: str,
     revision: str | None = "main",
-) -> str | None:
+) -> dict:
     config_file = [
         "processor_config.json",
         "preprocessor_config.json",
@@ -154,8 +154,20 @@ def get_processor_cls_name_from_config(
     for file in config_file:
         config = get_hf_file_to_dict(file, processor_name, revision=revision)
         if config and "processor_class" in config:
-            return config["processor_class"]
-    return None
+            return config
+    return {}
+
+
+def get_processor_cls_name_from_config(
+    processor_name: str,
+    revision: str | None = "main",
+) -> str | None:
+    config = get_processor_config(processor_name, revision=revision)
+
+    if not config:
+        return None
+
+    return config["processor_class"]
 
 
 def get_video_processor_cls_name_from_config(