From 45273b8dcbfb2d2c300c2f4a55c4dc283adca06a Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Fri, 7 Aug 2026 18:30:43 +0100 Subject: [PATCH] [1/N] Harden Transformers modelling backend multi-modal path (#51408) Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> --- .../multimodal/processing/test_common.py | 32 ++++ .../processing/test_transformers_audio.py | 47 +++++- .../processing/test_transformers_image.py | 38 ++++- .../models/transformers/multimodal.py | 139 ++++++++++++------ 4 files changed, 204 insertions(+), 52 deletions(-) diff --git a/tests/models/multimodal/processing/test_common.py b/tests/models/multimodal/processing/test_common.py index 6d31e87003d..0754892e4e0 100644 --- a/tests/models/multimodal/processing/test_common.py +++ b/tests/models/multimodal/processing/test_common.py @@ -134,6 +134,17 @@ def get_model_ids_to_test(): return _get_model_ids_to_test(vllm_only_archs) +def get_transformers_backend_model_ids_to_test(): + return sorted( + { + model_id + for arch, info in _TRANSFORMERS_BACKEND_MODELS.items() + if "MultiModal" in arch + for model_id in (info.default, *info.extras.values()) + } + ) + + def get_text_token_prompts( processor: BaseMultiModalProcessor, mm_data: MultiModalDataDict, @@ -211,6 +222,7 @@ def _test_processing_correctness( hit_rate: float, num_batches: int, simplify_rate: float, + model_impl: str = "auto", ): if model_id_or_arch in HF_EXAMPLE_MODELS.get_supported_archs(): # Use model architecture to get the default model id @@ -238,6 +250,7 @@ def _test_processing_correctness( enable_mm_embeds=model_info.require_embed_inputs, enforce_eager=model_info.enforce_eager, dtype=model_info.dtype, + model_impl=model_impl, ) # Ensure that the cache can fit all of the data # (set after because ModelConfig would set it to 0 for encoder-decoder models) @@ -480,6 +493,25 @@ def test_processing_correctness( ) +@pytest.mark.parametrize("model_id", get_transformers_backend_model_ids_to_test()) +@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0]) +@pytest.mark.parametrize("num_batches", [32]) +@pytest.mark.parametrize("simplify_rate", [1.0]) +def test_processing_correctness_transformers( + model_id: str, + hit_rate: float, + num_batches: int, + simplify_rate: float, +): + _test_processing_correctness( + model_id, + hit_rate=hit_rate, + num_batches=num_batches, + simplify_rate=simplify_rate, + model_impl="transformers", + ) + + def _assert_inputs_equal( a: MultiModalInput, b: MultiModalInput, diff --git a/tests/models/multimodal/processing/test_transformers_audio.py b/tests/models/multimodal/processing/test_transformers_audio.py index 3ff3e04379f..bc0dedcd1d2 100644 --- a/tests/models/multimodal/processing/test_transformers_audio.py +++ b/tests/models/multimodal/processing/test_transformers_audio.py @@ -113,9 +113,7 @@ def test_audio_multimodal_processor(model_id): ) -def test_audio_multiple_inputs(): - """Multiple audios per prompt are each detected as a separate placeholder - and multi-modal item by the Transformers backend.""" +def _process_granite_speech(separator: str): model_id = "ibm-granite/granite-speech-3.3-2b" model_config = ModelConfig(model=model_id, model_impl="transformers") mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config) @@ -124,15 +122,54 @@ def test_audio_multiple_inputs(): # One token per audio; the processor expands each to its placeholder run. prompt = ( "<|start_of_role|>user<|end_of_role|>" - f"{audio_token} and {audio_token} transcribe<|end_of_text|>\n" + f"{audio_token}{separator}{audio_token} transcribe<|end_of_text|>\n" ) audios = [np.zeros(16000, dtype=np.float32), np.zeros(24000, dtype=np.float32)] - result = mm_processor( + return mm_processor( prompt=prompt, mm_items=mm_processor.info.parse_mm_data({"audio": audios}), hf_processor_mm_kwargs={}, ) + +def test_audio_multiple_inputs(): + """Multiple audios per prompt are each detected as a separate placeholder + and multi-modal item by the Transformers modelling backend.""" + result = _process_granite_speech(separator=" and ") + assert len(result["mm_placeholders"]["audio"]) == 2 assert len(result["mm_kwargs"]["audio"]) == 2 + + +def test_audio_fields_not_claimed_by_image(): + """Audio fields survive when the image branch is also active.""" + model_id = "ibm-granite/granite-speech-3.3-2b" + model_config = ModelConfig(model=model_id, model_impl="transformers") + mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config) + + audio_keys = ["input_features", "input_features_mask"] + owned = mm_processor._partition_keys_by_modality(audio_keys, ["audio", "image"]) + + assert owned["audio"] == audio_keys + assert owned["image"] == [] + + +def test_unclaimed_fields_warn_rather_than_raise(): + """Keys no sub-processor declares are dropped with a warning, not an error.""" + model_id = "ibm-granite/granite-speech-3.3-2b" + model_config = ModelConfig(model=model_id, model_impl="transformers") + mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config) + + owned = mm_processor._partition_keys_by_modality( + ["input_features", "surprise_field"], ["audio", "image"] + ) + + assert owned["audio"] == ["input_features"] + assert owned["image"] == [] + + +def test_audio_adjacent_inputs(): + """Adjacent audios are rejected rather than silently merged into one placeholder.""" + with pytest.raises(ValueError, match="told apart"): + _process_granite_speech(separator="") diff --git a/tests/models/multimodal/processing/test_transformers_image.py b/tests/models/multimodal/processing/test_transformers_image.py index 2c31bcc6347..6aa42fc71c9 100644 --- a/tests/models/multimodal/processing/test_transformers_image.py +++ b/tests/models/multimodal/processing/test_transformers_image.py @@ -56,24 +56,52 @@ def test_multimodal_processor(model_id): ) -def test_image_multiple_inputs(): - """Multiple images per prompt are each detected as a separate placeholder - and multi-modal item by the Transformers backend.""" +def _process_two_images(separator: str): model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf" model_config = ModelConfig(model=model_id, model_impl="transformers") mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config) image = ImageAsset("cherry_blossom").pil_image prompt = ( - "<|im_start|>user \n and \n" + f"<|im_start|>user {separator}\n" "What do these images show?<|im_end|><|im_start|>assistant\n" ) - result = mm_processor( + return mm_processor( prompt=prompt, mm_items=mm_processor.info.parse_mm_data({"image": [image, image]}), hf_processor_mm_kwargs={}, ) + +def test_image_multiple_inputs(): + """Multiple images per prompt are each detected as a separate placeholder + and multi-modal item by the Transformers modelling backend.""" + result = _process_two_images(separator="\n and ") + assert len(result["mm_placeholders"]["image"]) == 2 assert len(result["mm_kwargs"]["image"]) == 2 + + +def test_image_adjacent_inputs(): + """Adjacent images stay separate placeholders rather than merging into one.""" + result = _process_two_images(separator="") + + assert len(result["mm_placeholders"]["image"]) == 2 + assert len(result["mm_kwargs"]["image"]) == 2 + + +def test_text_only_prompt(): + """An image model still accepts a prompt with no images.""" + model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf" + model_config = ModelConfig(model=model_id, model_impl="transformers") + mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config) + + result = mm_processor( + prompt="<|im_start|>user Hello!<|im_end|><|im_start|>assistant\n", + mm_items=mm_processor.info.parse_mm_data({}), + hf_processor_mm_kwargs={}, + ) + + assert len(result["prompt_token_ids"]) > 0 + assert not result["mm_placeholders"] diff --git a/vllm/model_executor/models/transformers/multimodal.py b/vllm/model_executor/models/transformers/multimodal.py index 3d077ac021f..7b84c3e07ad 100644 --- a/vllm/model_executor/models/transformers/multimodal.py +++ b/vllm/model_executor/models/transformers/multimodal.py @@ -168,7 +168,8 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn if self.info._is_audio_model() and (num_audios := mm_counts.get("audio", 0)): processor = self.info.get_hf_processor() audio_token = getattr(processor, "audio_token", "") - text += audio_token * num_audios + # Separated so that `_apply_audio` can tell the placeholders apart + text += " ".join([audio_token] * num_audios) if self.info._is_image_model() and (num_images := mm_counts.get("image", 0)): processor = self.info.get_hf_processor() if "gemma3" in processor.__class__.__name__.lower(): @@ -230,6 +231,54 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]): """ return None + def _get_modality_field_names(self, modality: str) -> set[str]: + """Field names the sub-processor for `modality` produces.""" + # TODO: use else branch only once huggingface/transformers#44394 lands. + if modality == "audio": + sub_processor = self.info._get_audio_processor() + else: + processor = self.info.get_hf_processor() + sub_processor = getattr(processor, f"{modality}_processor", None) + + # Pre-computed embeddings bypass the sub-processor entirely + names = {f"{modality}_embeds"} + for name in getattr(sub_processor, "model_input_names", None) or (): + # Companion masks are emitted but not always declared + names.update((name, f"{name}_mask")) + return names + + def _partition_keys_by_modality( + self, + keys: list[str], + modalities: list[str], + ) -> dict[str, list[str]]: + """Attribute each HF processor output key to the modality that produced it.""" + if len(modalities) == 1: + return {modalities[0]: keys} + + claimed = {m: self._get_modality_field_names(m) for m in modalities} + + owned: dict[str, list[str]] = {modality: [] for modality in modalities} + unclaimed = [] + for key in keys: + for modality in modalities: + if key in claimed[modality]: + owned[modality].append(key) + break + else: + unclaimed.append(key) + + if unclaimed: + logger.warning_once( + "Unable to attribute %s to any of the modalities %s, so they " + "will not be passed to the model. Add them to the relevant " + "sub-processor's `model_input_names` to fix this.", + tuple(unclaimed), + tuple(modalities), + ) + + return owned + def _get_mm_fields_config( self, hf_inputs: "BatchFeature", @@ -238,34 +287,29 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]): # HF Processors always return a mask but vLLM doesn't need it hf_inputs.pop("attention_mask", None) - mm_fields: dict[str, MultiModalFieldConfig] = {} - if self.info._is_audio_model(): - num_audio_tokens = hf_inputs.get("num_audio_tokens") - mm_fields.update( - { - key: MultiModalFieldConfig.flat_from_sizes( - "audio", num_audio_tokens - ) - for key in hf_inputs - } - ) - mm_fields["num_audio_tokens"] = MultiModalFieldConfig.batched("audio") - if self.info._is_image_model(): - num_image_patches = hf_inputs.get("num_image_patches") - mm_fields.update( - { - key: MultiModalFieldConfig.flat_from_sizes( - "image", num_image_patches - ) - for key in hf_inputs - } - ) - mm_fields["image_embeds"] = MultiModalFieldConfig.flat_from_sizes( - "image", num_image_patches - ) + # Written by `_apply_audio`/`_apply_vision`; absent if the modality had no items + sizes = { + "audio": hf_inputs.get("num_audio_tokens"), + "image": hf_inputs.get("num_image_patches"), + } + modalities = [m for m, size in sizes.items() if size is not None] - # Keep these as batched, as they always have batch size as first dim + size_keys = {"num_audio_tokens", "num_image_patches"} + keys = [key for key in hf_inputs if key not in size_keys] + owned = self._partition_keys_by_modality(keys, modalities) + + mm_fields: dict[str, MultiModalFieldConfig] = { + key: MultiModalFieldConfig.flat_from_sizes(modality, sizes[modality]) + for modality in modalities + for key in owned[modality] + } + + # Keep these as batched, as they always have batch size as first dim + if "audio" in modalities: + mm_fields["num_audio_tokens"] = MultiModalFieldConfig.batched("audio") + if "image" in modalities: mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched("image") + # TODO: route to "video" once the video modality is supported mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched("image") mm_fields["num_image_patches"] = MultiModalFieldConfig.batched( "image", keep_on_cpu=True @@ -291,6 +335,7 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]): self, prompt_ids: list[int], processed_data: "BatchFeature", + num_audios: int, ) -> dict[str, list[PlaceholderRange]]: audio_token_id = self.info._get_audio_token_id() prompt_tensor = torch.tensor(prompt_ids) @@ -301,17 +346,21 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]): padded = torch.cat([torch.tensor([False]), is_audio, torch.tensor([False])]) transitions = padded.int().diff() - starts = torch.where(transitions == 1)[0] - ends = torch.where(transitions == -1)[0] - lengths = ends - starts + offsets = torch.where(transitions == 1)[0] + lengths = torch.where(transitions == -1)[0] - offsets + + if len(offsets) != num_audios: + raise ValueError( + f"Found {len(offsets)} run(s) of the audio token in the prompt but " + f"{num_audios} audio item(s) were passed. The Transformers backend " + "locates audio placeholders by finding contiguous runs of the audio " + "token, so placeholders with no text between them cannot yet be told " + "apart. Separate them in the prompt to work around this." + ) ranges = [ - PlaceholderRange( - offset=s.item(), - length=ln.item(), - is_embed=torch.ones(ln.item(), dtype=torch.bool), - ) - for s, ln in zip(starts, lengths) + PlaceholderRange(offset=offset.item(), length=length.item()) + for offset, length in zip(offsets, lengths) ] processed_data["num_audio_tokens"] = lengths return {"audio": ranges} @@ -324,6 +373,7 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]): hf_processor_mm_kwargs: Mapping[str, object], mm_token_type_ids: torch.Tensor | None, ) -> dict[str, list[PlaceholderRange]]: + # Placeholders can't be located without them, so give up rather than guess if mm_token_type_ids is None: return {} @@ -383,10 +433,13 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]): with timing_ctx.record("apply_hf_processor"): hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) if not isinstance(prompt, str): - # the prompt is the tokenized ids which is not supported - # by the hf_processor, which is why we would need to decode the ids - # into string + # HF processors only accept text, and the decoded string already + # contains any special tokens, so don't let them be added again prompt = hf_processor.decode(prompt) + tokenization_kwargs = { + **tokenization_kwargs, + "add_special_tokens": False, + } # Bypass cached processor and always apply to the full set of mm inputs # NOTE: we can't just set caching=False because base class method @@ -412,9 +465,11 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]): mm_token_type_ids = processed_data.pop("mm_token_type_ids", mm_token_type_ids) mm_placeholders: dict[str, list[PlaceholderRange]] = {} - if self.info._is_audio_model(): - mm_placeholders.update(self._apply_audio(prompt_ids, processed_data)) - if self.info._is_image_model(): + if num_audios := mm_items.get_count("audio", strict=False): + mm_placeholders.update( + self._apply_audio(prompt_ids, processed_data, num_audios) + ) + if mm_items.get_count("image", strict=False): mm_placeholders.update( self._apply_vision( prompt_ids,