From f84df12c6ac30fb96a24491bb9846b28f04cf24c Mon Sep 17 00:00:00 2001 From: Yunzhu Lu Date: Thu, 6 Aug 2026 12:12:00 +0800 Subject: [PATCH] [Bugfix][MM] Fix MiniCPM-V placeholder replacement and image processor loading on Transformers v5 (#48413) Signed-off-by: YunzhuLu Signed-off-by: Yunzhu Lu Co-authored-by: Cyrus Leung --- .../multimodal/processing/test_minicpmv.py | 74 ++++++++++++ tests/models/registry.py | 7 -- vllm/model_executor/models/minicpmv.py | 114 +++++++++++++++++- vllm/multimodal/processing/processor.py | 33 +++++ 4 files changed, 218 insertions(+), 10 deletions(-) create mode 100644 tests/models/multimodal/processing/test_minicpmv.py diff --git a/tests/models/multimodal/processing/test_minicpmv.py b/tests/models/multimodal/processing/test_minicpmv.py new file mode 100644 index 00000000000..9424ea39595 --- /dev/null +++ b/tests/models/multimodal/processing/test_minicpmv.py @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Tests for MiniCPMV's multimodal preprocessing.""" + +import numpy as np +import pytest + +from vllm.multimodal import MULTIMODAL_REGISTRY + +from ...utils import build_model_context + + +@pytest.mark.parametrize("model_id", ["openbmb/MiniCPM-V-4"]) +def test_get_hf_processor_for_same_model_different_kwargs(model_id: str): + """Calls with different kwargs must not reuse stale processor instances.""" + ctx = build_model_context( + model_id, + limit_mm_per_prompt={"image": 1}, + ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + info = processor.info + processor_1 = info.get_hf_processor(max_slice_nums=1) + processor_2 = info.get_hf_processor(max_slice_nums=2) + assert processor_1.image_processor.max_slice_nums == 1 + assert processor_2.image_processor.max_slice_nums == 2 + + +@pytest.mark.parametrize( + "model_ids", [("openbmb/MiniCPM-Llama3-V-2_5", "openbmb/MiniCPM-V-4")] +) +def test_image_processor_for_dif_model(model_ids): + model_id_25, model_id_4 = model_ids + + ctx_25 = build_model_context(model_id_25, limit_mm_per_prompt={"image": 1}) + processor_25 = MULTIMODAL_REGISTRY.create_processor(ctx_25.model_config) + image_processor_25 = processor_25.info.get_image_processor() + + ctx_4 = build_model_context(model_id_4, limit_mm_per_prompt={"image": 1}) + processor_4 = MULTIMODAL_REGISTRY.create_processor(ctx_4.model_config) + image_processor_4 = processor_4.info.get_image_processor() + + assert type(image_processor_25) is not type(image_processor_4) + assert type(image_processor_25).__module__ != type(image_processor_4).__module__ + + +@pytest.mark.parametrize("model_id", ["openbmb/MiniCPM-V-4"]) +def test_prompt_has_dif_BPE_boundaries_in_context(model_id: str): + ctx = build_model_context( + model_id, + limit_mm_per_prompt={"image": 1}, + ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + tokenizer = ctx.get_tokenizer() + + messages = [ + {"role": "user", "content": "(./)\nWhat is in this image?"} + ] + prompt = tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True, + ) + image = np.zeros((768, 1024, 3), dtype=np.uint8) + + mm_items = processor.info.parse_mm_data({"image": [image]}) + processed = processor( + prompt, + mm_items=mm_items, + hf_processor_mm_kwargs={}, + ) + image_placeholders = processed["mm_placeholders"].get("image", []) + assert len(image_placeholders) == 1 + assert image_placeholders[0].length > 0 diff --git a/tests/models/registry.py b/tests/models/registry.py index a37d02299ef..e3b615dd7da 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -1107,13 +1107,6 @@ _MULTIMODAL_EXAMPLE_MODELS = { "4.0": "openbmb/MiniCPM-V-4", "4.5": "openbmb/MiniCPM-V-4_5", }, - max_transformers_version="4.57", - transformers_version_reason={ - "vllm": ( - "MiniCPMVBatchFeature is incompatible with its base class in " - "Transformers v5. See https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5/discussions/78" - ) - }, trust_remote_code=True, ), "MiniCPMV4_6ForConditionalGeneration": _HfExamplesInfo( diff --git a/vllm/model_executor/models/minicpmv.py b/vllm/model_executor/models/minicpmv.py index 97485d17ed0..4a3ad03ed78 100644 --- a/vllm/model_executor/models/minicpmv.py +++ b/vllm/model_executor/models/minicpmv.py @@ -27,7 +27,7 @@ import math from collections import defaultdict from collections.abc import Callable, Iterable, Mapping, Sequence, Set -from functools import partial +from functools import cached_property, partial from itertools import chain from typing import Annotated, Any, Literal, TypeAlias @@ -37,6 +37,10 @@ import torch.types from torch import nn from torch.nn.init import trunc_normal_ from transformers import BatchFeature, PretrainedConfig +from transformers.dynamic_module_utils import ( + get_class_from_dynamic_module, + resolve_trust_remote_code, +) from typing_extensions import TypeVar from vllm.config import VllmConfig @@ -74,6 +78,8 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder from vllm.multimodal.processing.processor import ( BaseMultiModalProcessor, BaseProcessingInfo, + MultiModalPromptUpdates, + PlaceholderFeaturesInfo, PromptReplacement, PromptUpdate, PromptUpdateDetails, @@ -82,6 +88,11 @@ from vllm.multimodal.processing.processor import ( ) from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors +from vllm.transformers_utils.processor import ( + _merge_mm_kwargs, + cached_get_image_processor, +) +from vllm.transformers_utils.utils import convert_model_repo_to_path from vllm.utils.collection_utils import flatten_2d_lists from vllm.utils.tensor_schema import TensorSchema, TensorShape from vllm.utils.torch_utils import set_default_torch_dtype @@ -559,16 +570,63 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo): image_pattern = "(./)" video_pattern = "()" + @cached_property + def _image_processor_cls(self): + model_config = self.ctx.model_config + model_path = convert_model_repo_to_path(model_config.model) + + from transformers import ImageProcessingMixin + + image_processor_config, _ = ImageProcessingMixin.get_image_processor_dict( + model_path, + revision=model_config.revision, + token=model_config.hf_token, + ) + + auto_map = image_processor_config.get("auto_map") or {} + class_ref = auto_map.get("AutoImageProcessor") + if not class_ref: + raise ValueError( + "Missing auto_map['AutoImageProcessor'] in image processor config " + f"for {model_config.model!r}" + ) + + resolve_trust_remote_code( + model_config.trust_remote_code, + model_config.model, + has_local_code=False, + has_remote_code=True, + ) + return get_class_from_dynamic_module( + class_ref, + model_config.model, + revision=model_config.revision, + ) + def get_hf_config(self): return self.ctx.get_hf_config() def get_hf_processor(self, **kwargs: object): + model_config = self.ctx.model_config + processor_cls = self._image_processor_cls + merged_kwargs = _merge_mm_kwargs(model_config, processor_cls, **kwargs) + + # AutoProcessor only for tokenizer; its image_processor is resolved by + # class name and can pick the wrong checkpoint across MiniCPM-V versions. hf_processor = self.ctx.get_hf_processor(**kwargs) + image_processor = cached_get_image_processor( + model_config.model, + revision=model_config.revision, + trust_remote_code=model_config.trust_remote_code, + processor_cls_overrides=processor_cls, + **merged_kwargs, + ) + from vllm.transformers_utils.processors.minicpmv import MiniCPMVProcessor vendored_processor = MiniCPMVProcessor( - image_processor=hf_processor.image_processor, + image_processor=image_processor, tokenizer=hf_processor.tokenizer, version=self.get_model_version(), ) @@ -576,7 +634,6 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo): # NumPy arrays are considered as Iterable but not Sequence in # https://github.com/huggingface/transformers/blob/main/src/transformers/image_transforms.py#L428 - image_processor = hf_processor.image_processor # type: ignore # transformers v5+ renamed `mean`/`std` -> `image_mean`/`image_std` for attr in ("mean", "std", "image_mean", "image_std"): val = getattr(image_processor, attr, None) @@ -863,6 +920,57 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]): **self.process_videos(mm_data, mm_kwargs, tok_kwargs), } + def _apply_prompt_updates( + self, + token_ids: list[int], + mm_prompt_updates: MultiModalPromptUpdates, + ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]: + """Apply multi-modal prompt updates to token IDs.""" + tokenizer = self.info.get_tokenizer() + + new_token_ids, match_result = self._apply_token_matches( + token_ids, + mm_prompt_updates, + ) + + # If the search text does not represent a special token, + # it may have different token IDs in the prompt, because + # the tokens may go across the boundaries of the search text. + # ---- + # e.g. when searching for "foo" in "food", if "food" itself makes + # up a token, then the token ID of "foo" will not appear at all + # ---- + # Since it is inefficient to search for all possible tokenizations + # of the search text in the prompt, we instead perform string-based + # updates on the decoded token IDs, then encode them back. + if not all( + all(update_idx is not None for update_idx in update_idxs) + for update_idxs in match_result.values() + ): + new_token_ids, match_result = self._apply_text_matches_as_segmented_tokens( + _seq2text(tokenizer, token_ids, use_cache=False), + mm_prompt_updates, + ) + + matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list) + for modality, update_idxs in match_result.items(): + for item_idx, update_idx in enumerate(update_idxs): + assert update_idx is not None, ( + "Failed to apply prompt replacement for " + f"mm_items[{modality!r}][{item_idx}]" + ) + + matched_updates[modality].append( + [mm_prompt_updates[modality][item_idx][update_idx]] + ) + + placeholders = self._find_mm_placeholders( + new_token_ids, + dict(matched_updates), + ) + + return new_token_ids, placeholders + def _base_call_hf_processor( self, prompts: list[str], diff --git a/vllm/multimodal/processing/processor.py b/vllm/multimodal/processing/processor.py index 7addd653601..8fafccc3e72 100644 --- a/vllm/multimodal/processing/processor.py +++ b/vllm/multimodal/processing/processor.py @@ -903,6 +903,29 @@ def apply_text_matches( return "".join(texts), result +def apply_text_matches_as_segmented_tokens( + prompt: str, + mm_prompt_updates: "MultiModalPromptUpdates", + tokenizer: TokenizerLike | None, +) -> tuple[list[int], "MultiModalPromptUpdatesApplyResult"]: + """ + Apply the updates in `mm_prompt_updates` to `prompt`. + + Matches are exclusive even when multiple modalities share + the same placeholder tokens. In that case, the modality that + appears earlier in `mm_prompt_updates` takes priority. + + Each segment is encoded separately instead of being joined into one + string and encoded in a single pass. Joining first would let BPE merge + tokens across a segment boundary, silently change how a text + (non-special-token) placeholder is tokenized. + """ + texts, result = _apply_matches(prompt, mm_prompt_updates, tokenizer) + token_id_seqs = [_seq2tokens(tokenizer, text, use_cache=False) for text in texts] + + return flatten_2d_lists(token_id_seqs), result + + def _iter_placeholders( prompt: list[int], mm_prompt_updates: "MultiModalPromptUpdates", @@ -1584,6 +1607,16 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): tokenizer = self.info.get_tokenizer() return apply_text_matches(prompt, mm_prompt_updates, tokenizer) + def _apply_text_matches_as_segmented_tokens( + self, + prompt: str, + mm_prompt_updates: MultiModalPromptUpdates, + ) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]: + tokenizer = self.info.get_tokenizer() + return apply_text_matches_as_segmented_tokens( + prompt, mm_prompt_updates, tokenizer + ) + def _apply_prompt_updates( self, token_ids: list[int],