[Bugfix][MM] Fix MiniCPM-V placeholder replacement and image processor loading on Transformers v5 (#48413)

Signed-off-by: YunzhuLu <[email protected]>
Signed-off-by: Yunzhu Lu <[email protected]>
Co-authored-by: Cyrus Leung <[email protected]>

tests/models/multimodal/processing/test_minicpmv.py

@@ -0,0 +1,74 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+"""Tests for MiniCPMV's multimodal preprocessing."""
+
+import numpy as np
+import pytest
+
+from vllm.multimodal import MULTIMODAL_REGISTRY
+
+from ...utils import build_model_context
+
+
+@pytest.mark.parametrize("model_id", ["openbmb/MiniCPM-V-4"])
+def test_get_hf_processor_for_same_model_different_kwargs(model_id: str):
+    """Calls with different kwargs must not reuse stale processor instances."""
+    ctx = build_model_context(
+        model_id,
+        limit_mm_per_prompt={"image": 1},
+    )
+    processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
+    info = processor.info
+    processor_1 = info.get_hf_processor(max_slice_nums=1)
+    processor_2 = info.get_hf_processor(max_slice_nums=2)
+    assert processor_1.image_processor.max_slice_nums == 1
+    assert processor_2.image_processor.max_slice_nums == 2
+
+
+@pytest.mark.parametrize(
+    "model_ids", [("openbmb/MiniCPM-Llama3-V-2_5", "openbmb/MiniCPM-V-4")]
+)
+def test_image_processor_for_dif_model(model_ids):
+    model_id_25, model_id_4 = model_ids
+
+    ctx_25 = build_model_context(model_id_25, limit_mm_per_prompt={"image": 1})
+    processor_25 = MULTIMODAL_REGISTRY.create_processor(ctx_25.model_config)
+    image_processor_25 = processor_25.info.get_image_processor()
+
+    ctx_4 = build_model_context(model_id_4, limit_mm_per_prompt={"image": 1})
+    processor_4 = MULTIMODAL_REGISTRY.create_processor(ctx_4.model_config)
+    image_processor_4 = processor_4.info.get_image_processor()
+
+    assert type(image_processor_25) is not type(image_processor_4)
+    assert type(image_processor_25).__module__ != type(image_processor_4).__module__
+
+
+@pytest.mark.parametrize("model_id", ["openbmb/MiniCPM-V-4"])
+def test_prompt_has_dif_BPE_boundaries_in_context(model_id: str):
+    ctx = build_model_context(
+        model_id,
+        limit_mm_per_prompt={"image": 1},
+    )
+    processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
+    tokenizer = ctx.get_tokenizer()
+
+    messages = [
+        {"role": "user", "content": "(<image>./</image>)\nWhat is in this image?"}
+    ]
+    prompt = tokenizer.apply_chat_template(
+        messages,
+        tokenize=False,
+        add_generation_prompt=True,
+    )
+    image = np.zeros((768, 1024, 3), dtype=np.uint8)
+
+    mm_items = processor.info.parse_mm_data({"image": [image]})
+    processed = processor(
+        prompt,
+        mm_items=mm_items,
+        hf_processor_mm_kwargs={},
+    )
+    image_placeholders = processed["mm_placeholders"].get("image", [])
+    assert len(image_placeholders) == 1
+    assert image_placeholders[0].length > 0

tests/models/registry.py

@@ -1107,13 +1107,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
             "4.0": "openbmb/MiniCPM-V-4",
             "4.5": "openbmb/MiniCPM-V-4_5",
         },
-        max_transformers_version="4.57",
-        transformers_version_reason={
-            "vllm": (
-                "MiniCPMVBatchFeature is incompatible with its base class in "
-                "Transformers v5. See https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5/discussions/78"
-            )
-        },
         trust_remote_code=True,
     ),
     "MiniCPMV4_6ForConditionalGeneration": _HfExamplesInfo(

vllm/model_executor/models/minicpmv.py

@@ -27,7 +27,7 @@
 import math
 from collections import defaultdict
 from collections.abc import Callable, Iterable, Mapping, Sequence, Set
-from functools import partial
+from functools import cached_property, partial
 from itertools import chain
 from typing import Annotated, Any, Literal, TypeAlias
 
@@ -37,6 +37,10 @@ import torch.types
 from torch import nn
 from torch.nn.init import trunc_normal_
 from transformers import BatchFeature, PretrainedConfig
+from transformers.dynamic_module_utils import (
+    get_class_from_dynamic_module,
+    resolve_trust_remote_code,
+)
 from typing_extensions import TypeVar
 
 from vllm.config import VllmConfig
@@ -74,6 +78,8 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder
 from vllm.multimodal.processing.processor import (
     BaseMultiModalProcessor,
     BaseProcessingInfo,
+    MultiModalPromptUpdates,
+    PlaceholderFeaturesInfo,
     PromptReplacement,
     PromptUpdate,
     PromptUpdateDetails,
@@ -82,6 +88,11 @@ from vllm.multimodal.processing.processor import (
 )
 from vllm.platforms import current_platform
 from vllm.sequence import IntermediateTensors
+from vllm.transformers_utils.processor import (
+    _merge_mm_kwargs,
+    cached_get_image_processor,
+)
+from vllm.transformers_utils.utils import convert_model_repo_to_path
 from vllm.utils.collection_utils import flatten_2d_lists
 from vllm.utils.tensor_schema import TensorSchema, TensorShape
 from vllm.utils.torch_utils import set_default_torch_dtype
@@ -559,16 +570,63 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
     image_pattern = "(<image>./</image>)"
     video_pattern = "(<video>./</video>)"
 
+    @cached_property
+    def _image_processor_cls(self):
+        model_config = self.ctx.model_config
+        model_path = convert_model_repo_to_path(model_config.model)
+
+        from transformers import ImageProcessingMixin
+
+        image_processor_config, _ = ImageProcessingMixin.get_image_processor_dict(
+            model_path,
+            revision=model_config.revision,
+            token=model_config.hf_token,
+        )
+
+        auto_map = image_processor_config.get("auto_map") or {}
+        class_ref = auto_map.get("AutoImageProcessor")
+        if not class_ref:
+            raise ValueError(
+                "Missing auto_map['AutoImageProcessor'] in image processor config "
+                f"for {model_config.model!r}"
+            )
+
+        resolve_trust_remote_code(
+            model_config.trust_remote_code,
+            model_config.model,
+            has_local_code=False,
+            has_remote_code=True,
+        )
+        return get_class_from_dynamic_module(
+            class_ref,
+            model_config.model,
+            revision=model_config.revision,
+        )
+
     def get_hf_config(self):
         return self.ctx.get_hf_config()
 
     def get_hf_processor(self, **kwargs: object):
+        model_config = self.ctx.model_config
+        processor_cls = self._image_processor_cls
+        merged_kwargs = _merge_mm_kwargs(model_config, processor_cls, **kwargs)
+
+        # AutoProcessor only for tokenizer; its image_processor is resolved by
+        # class name and can pick the wrong checkpoint across MiniCPM-V versions.
         hf_processor = self.ctx.get_hf_processor(**kwargs)
 
+        image_processor = cached_get_image_processor(
+            model_config.model,
+            revision=model_config.revision,
+            trust_remote_code=model_config.trust_remote_code,
+            processor_cls_overrides=processor_cls,
+            **merged_kwargs,
+        )
+
         from vllm.transformers_utils.processors.minicpmv import MiniCPMVProcessor
 
         vendored_processor = MiniCPMVProcessor(
-            image_processor=hf_processor.image_processor,
+            image_processor=image_processor,
             tokenizer=hf_processor.tokenizer,
             version=self.get_model_version(),
         )
@@ -576,7 +634,6 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
 
         # NumPy arrays are considered as Iterable but not Sequence in
         # https://github.com/huggingface/transformers/blob/main/src/transformers/image_transforms.py#L428
-        image_processor = hf_processor.image_processor  # type: ignore
         # transformers v5+ renamed `mean`/`std` -> `image_mean`/`image_std`
         for attr in ("mean", "std", "image_mean", "image_std"):
             val = getattr(image_processor, attr, None)
@@ -863,6 +920,57 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
             **self.process_videos(mm_data, mm_kwargs, tok_kwargs),
         }
 
+    def _apply_prompt_updates(
+        self,
+        token_ids: list[int],
+        mm_prompt_updates: MultiModalPromptUpdates,
+    ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
+        """Apply multi-modal prompt updates to token IDs."""
+        tokenizer = self.info.get_tokenizer()
+
+        new_token_ids, match_result = self._apply_token_matches(
+            token_ids,
+            mm_prompt_updates,
+        )
+
+        # If the search text does not represent a special token,
+        # it may have different token IDs in the prompt, because
+        # the tokens may go across the boundaries of the search text.
+        # ----
+        # e.g. when searching for "foo" in "food", if "food" itself makes
+        # up a token, then the token ID of "foo" will not appear at all
+        # ----
+        # Since it is inefficient to search for all possible tokenizations
+        # of the search text in the prompt, we instead perform string-based
+        # updates on the decoded token IDs, then encode them back.
+        if not all(
+            all(update_idx is not None for update_idx in update_idxs)
+            for update_idxs in match_result.values()
+        ):
+            new_token_ids, match_result = self._apply_text_matches_as_segmented_tokens(
+                _seq2text(tokenizer, token_ids, use_cache=False),
+                mm_prompt_updates,
+            )
+
+        matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list)
+        for modality, update_idxs in match_result.items():
+            for item_idx, update_idx in enumerate(update_idxs):
+                assert update_idx is not None, (
+                    "Failed to apply prompt replacement for "
+                    f"mm_items[{modality!r}][{item_idx}]"
+                )
+
+                matched_updates[modality].append(
+                    [mm_prompt_updates[modality][item_idx][update_idx]]
+                )
+
+        placeholders = self._find_mm_placeholders(
+            new_token_ids,
+            dict(matched_updates),
+        )
+
+        return new_token_ids, placeholders
+
     def _base_call_hf_processor(
         self,
         prompts: list[str],

vllm/multimodal/processing/processor.py

@@ -903,6 +903,29 @@ def apply_text_matches(
     return "".join(texts), result
 
 
+def apply_text_matches_as_segmented_tokens(
+    prompt: str,
+    mm_prompt_updates: "MultiModalPromptUpdates",
+    tokenizer: TokenizerLike | None,
+) -> tuple[list[int], "MultiModalPromptUpdatesApplyResult"]:
+    """
+    Apply the updates in `mm_prompt_updates` to `prompt`.
+
+    Matches are exclusive even when multiple modalities share
+    the same placeholder tokens. In that case, the modality that
+    appears earlier in `mm_prompt_updates` takes priority.
+
+    Each segment is encoded separately instead of being joined into one
+    string and encoded in a single pass. Joining first would let BPE merge
+    tokens across a segment boundary, silently change how a text
+    (non-special-token) placeholder is tokenized.
+    """
+    texts, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
+    token_id_seqs = [_seq2tokens(tokenizer, text, use_cache=False) for text in texts]
+
+    return flatten_2d_lists(token_id_seqs), result
+
+
 def _iter_placeholders(
     prompt: list[int],
     mm_prompt_updates: "MultiModalPromptUpdates",
@@ -1584,6 +1607,16 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
         tokenizer = self.info.get_tokenizer()
         return apply_text_matches(prompt, mm_prompt_updates, tokenizer)
 
+    def _apply_text_matches_as_segmented_tokens(
+        self,
+        prompt: str,
+        mm_prompt_updates: MultiModalPromptUpdates,
+    ) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
+        tokenizer = self.info.get_tokenizer()
+        return apply_text_matches_as_segmented_tokens(
+            prompt, mm_prompt_updates, tokenizer
+        )
+
     def _apply_prompt_updates(
         self,
         token_ids: list[int],