obscura/vllm
2
0
Fork 0
mirror of https://github.com/vllm-project/vllm.git synced 2026-08-21 05:00:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

[1/N] Harden Transformers modelling backend multi-modal path (#51408)

Browse Source 
Signed-off-by: Harry Mellor <[email protected]>
This commit is contained in:
Harry Mellor
2026-08-07 17:30:43 +00:00
committed by GitHub
parent fcde8e1460
commit 45273b8dcb
4 changed files with 204 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/models/multimodal/processing/test_common.py
+32
View File
@@ -134,6 +134,17 @@ def get_model_ids_to_test():
return _get_model_ids_to_test(vllm_only_archs)
def get_transformers_backend_model_ids_to_test():
return sorted(
{
model_id
for arch, info in _TRANSFORMERS_BACKEND_MODELS.items()
if "MultiModal" in arch
for model_id in (info.default, *info.extras.values())
}
)
def get_text_token_prompts(
processor: BaseMultiModalProcessor,
mm_data: MultiModalDataDict,
@@ -211,6 +222,7 @@ def _test_processing_correctness(
hit_rate: float,
num_batches: int,
simplify_rate: float,
model_impl: str = "auto",
):
if model_id_or_arch in HF_EXAMPLE_MODELS.get_supported_archs():
# Use model architecture to get the default model id
@@ -238,6 +250,7 @@ def _test_processing_correctness(
enable_mm_embeds=model_info.require_embed_inputs,
enforce_eager=model_info.enforce_eager,
dtype=model_info.dtype,
model_impl=model_impl,
)
# Ensure that the cache can fit all of the data
# (set after because ModelConfig would set it to 0 for encoder-decoder models)
@@ -480,6 +493,25 @@ def test_processing_correctness(
)
@pytest.mark.parametrize("model_id", get_transformers_backend_model_ids_to_test())
@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
@pytest.mark.parametrize("num_batches", [32])
@pytest.mark.parametrize("simplify_rate", [1.0])
def test_processing_correctness_transformers(
model_id: str,
hit_rate: float,
num_batches: int,
simplify_rate: float,
):
_test_processing_correctness(
model_id,
hit_rate=hit_rate,
num_batches=num_batches,
simplify_rate=simplify_rate,
model_impl="transformers",
)
def _assert_inputs_equal(
a: MultiModalInput,
b: MultiModalInput,
tests/models/multimodal/processing/test_transformers_audio.py
+42 -5
View File
@@ -113,9 +113,7 @@ def test_audio_multimodal_processor(model_id):
)
def test_audio_multiple_inputs():
"""Multiple audios per prompt are each detected as a separate placeholder
and multi-modal item by the Transformers backend."""
def _process_granite_speech(separator: str):
model_id = "ibm-granite/granite-speech-3.3-2b"
model_config = ModelConfig(model=model_id, model_impl="transformers")
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
@@ -124,15 +122,54 @@ def test_audio_multiple_inputs():
# One token per audio; the processor expands each to its placeholder run.
prompt = (
"<|start_of_role|>user<|end_of_role|>"
f"{audio_token} and {audio_token} transcribe<|end_of_text|>\n"
f"{audio_token}{separator}{audio_token} transcribe<|end_of_text|>\n"
)
audios = [np.zeros(16000, dtype=np.float32), np.zeros(24000, dtype=np.float32)]
result = mm_processor(
return mm_processor(
prompt=prompt,
mm_items=mm_processor.info.parse_mm_data({"audio": audios}),
hf_processor_mm_kwargs={},
)
def test_audio_multiple_inputs():
"""Multiple audios per prompt are each detected as a separate placeholder
and multi-modal item by the Transformers modelling backend."""
result = _process_granite_speech(separator=" and ")
assert len(result["mm_placeholders"]["audio"]) == 2
assert len(result["mm_kwargs"]["audio"]) == 2
def test_audio_fields_not_claimed_by_image():
"""Audio fields survive when the image branch is also active."""
model_id = "ibm-granite/granite-speech-3.3-2b"
model_config = ModelConfig(model=model_id, model_impl="transformers")
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
audio_keys = ["input_features", "input_features_mask"]
owned = mm_processor._partition_keys_by_modality(audio_keys, ["audio", "image"])
assert owned["audio"] == audio_keys
assert owned["image"] == []
def test_unclaimed_fields_warn_rather_than_raise():
"""Keys no sub-processor declares are dropped with a warning, not an error."""
model_id = "ibm-granite/granite-speech-3.3-2b"
model_config = ModelConfig(model=model_id, model_impl="transformers")
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
owned = mm_processor._partition_keys_by_modality(
["input_features", "surprise_field"], ["audio", "image"]
)
assert owned["audio"] == ["input_features"]
assert owned["image"] == []
def test_audio_adjacent_inputs():
"""Adjacent audios are rejected rather than silently merged into one placeholder."""
with pytest.raises(ValueError, match="told apart"):
_process_granite_speech(separator="")
tests/models/multimodal/processing/test_transformers_image.py
+33 -5
View File
@@ -56,24 +56,52 @@ def test_multimodal_processor(model_id):
)
def test_image_multiple_inputs():
"""Multiple images per prompt are each detected as a separate placeholder
and multi-modal item by the Transformers backend."""
def _process_two_images(separator: str):
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
model_config = ModelConfig(model=model_id, model_impl="transformers")
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
image = ImageAsset("cherry_blossom").pil_image
prompt = (
"<|im_start|>user <image>\n and <image>\n"
f"<|im_start|>user <image>{separator}<image>\n"
"What do these images show?<|im_end|><|im_start|>assistant\n"
)
result = mm_processor(
return mm_processor(
prompt=prompt,
mm_items=mm_processor.info.parse_mm_data({"image": [image, image]}),
hf_processor_mm_kwargs={},
)
def test_image_multiple_inputs():
"""Multiple images per prompt are each detected as a separate placeholder
and multi-modal item by the Transformers modelling backend."""
result = _process_two_images(separator="\n and ")
assert len(result["mm_placeholders"]["image"]) == 2
assert len(result["mm_kwargs"]["image"]) == 2
def test_image_adjacent_inputs():
"""Adjacent images stay separate placeholders rather than merging into one."""
result = _process_two_images(separator="")
assert len(result["mm_placeholders"]["image"]) == 2
assert len(result["mm_kwargs"]["image"]) == 2
def test_text_only_prompt():
"""An image model still accepts a prompt with no images."""
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
model_config = ModelConfig(model=model_id, model_impl="transformers")
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
result = mm_processor(
prompt="<|im_start|>user Hello!<|im_end|><|im_start|>assistant\n",
mm_items=mm_processor.info.parse_mm_data({}),
hf_processor_mm_kwargs={},
)
assert len(result["prompt_token_ids"]) > 0
assert not result["mm_placeholders"]
vllm/model_executor/models/transformers/multimodal.py
+97 -42
View File
@@ -168,7 +168,8 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn
if self.info._is_audio_model() and (num_audios := mm_counts.get("audio", 0)):
processor = self.info.get_hf_processor()
audio_token = getattr(processor, "audio_token", "")
text += audio_token * num_audios
# Separated so that `_apply_audio` can tell the placeholders apart
text += " ".join([audio_token] * num_audios)
if self.info._is_image_model() and (num_images := mm_counts.get("image", 0)):
processor = self.info.get_hf_processor()
if "gemma3" in processor.__class__.__name__.lower():
@@ -230,6 +231,54 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
"""
return None
def _get_modality_field_names(self, modality: str) -> set[str]:
"""Field names the sub-processor for `modality` produces."""
# TODO: use else branch only once huggingface/transformers#44394 lands.
if modality == "audio":
sub_processor = self.info._get_audio_processor()
else:
processor = self.info.get_hf_processor()
sub_processor = getattr(processor, f"{modality}_processor", None)
# Pre-computed embeddings bypass the sub-processor entirely
names = {f"{modality}_embeds"}
for name in getattr(sub_processor, "model_input_names", None) or ():
# Companion masks are emitted but not always declared
names.update((name, f"{name}_mask"))
return names
def _partition_keys_by_modality(
self,
keys: list[str],
modalities: list[str],
) -> dict[str, list[str]]:
"""Attribute each HF processor output key to the modality that produced it."""
if len(modalities) == 1:
return {modalities[0]: keys}
claimed = {m: self._get_modality_field_names(m) for m in modalities}
owned: dict[str, list[str]] = {modality: [] for modality in modalities}
unclaimed = []
for key in keys:
for modality in modalities:
if key in claimed[modality]:
owned[modality].append(key)
break
else:
unclaimed.append(key)
if unclaimed:
logger.warning_once(
"Unable to attribute %s to any of the modalities %s, so they "
"will not be passed to the model. Add them to the relevant "
"sub-processor's `model_input_names` to fix this.",
tuple(unclaimed),
tuple(modalities),
)
return owned
def _get_mm_fields_config(
self,
hf_inputs: "BatchFeature",
@@ -238,34 +287,29 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
# HF Processors always return a mask but vLLM doesn't need it
hf_inputs.pop("attention_mask", None)
mm_fields: dict[str, MultiModalFieldConfig] = {}
if self.info._is_audio_model():
num_audio_tokens = hf_inputs.get("num_audio_tokens")
mm_fields.update(
{
key: MultiModalFieldConfig.flat_from_sizes(
"audio", num_audio_tokens
)
for key in hf_inputs
}
)
mm_fields["num_audio_tokens"] = MultiModalFieldConfig.batched("audio")
if self.info._is_image_model():
num_image_patches = hf_inputs.get("num_image_patches")
mm_fields.update(
{
key: MultiModalFieldConfig.flat_from_sizes(
"image", num_image_patches
)
for key in hf_inputs
}
)
mm_fields["image_embeds"] = MultiModalFieldConfig.flat_from_sizes(
"image", num_image_patches
)
# Written by `_apply_audio`/`_apply_vision`; absent if the modality had no items
sizes = {
"audio": hf_inputs.get("num_audio_tokens"),
"image": hf_inputs.get("num_image_patches"),
}
modalities = [m for m, size in sizes.items() if size is not None]
# Keep these as batched, as they always have batch size as first dim
size_keys = {"num_audio_tokens", "num_image_patches"}
keys = [key for key in hf_inputs if key not in size_keys]
owned = self._partition_keys_by_modality(keys, modalities)
mm_fields: dict[str, MultiModalFieldConfig] = {
key: MultiModalFieldConfig.flat_from_sizes(modality, sizes[modality])
for modality in modalities
for key in owned[modality]
}
# Keep these as batched, as they always have batch size as first dim
if "audio" in modalities:
mm_fields["num_audio_tokens"] = MultiModalFieldConfig.batched("audio")
if "image" in modalities:
mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched("image")
# TODO: route to "video" once the video modality is supported
mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched("image")
mm_fields["num_image_patches"] = MultiModalFieldConfig.batched(
"image", keep_on_cpu=True
@@ -291,6 +335,7 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
self,
prompt_ids: list[int],
processed_data: "BatchFeature",
num_audios: int,
) -> dict[str, list[PlaceholderRange]]:
audio_token_id = self.info._get_audio_token_id()
prompt_tensor = torch.tensor(prompt_ids)
@@ -301,17 +346,21 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
padded = torch.cat([torch.tensor([False]), is_audio, torch.tensor([False])])
transitions = padded.int().diff()
starts = torch.where(transitions == 1)[0]
ends = torch.where(transitions == -1)[0]
lengths = ends - starts
offsets = torch.where(transitions == 1)[0]
lengths = torch.where(transitions == -1)[0] - offsets
if len(offsets) != num_audios:
raise ValueError(
f"Found {len(offsets)} run(s) of the audio token in the prompt but "
f"{num_audios} audio item(s) were passed. The Transformers backend "
"locates audio placeholders by finding contiguous runs of the audio "
"token, so placeholders with no text between them cannot yet be told "
"apart. Separate them in the prompt to work around this."
)
ranges = [
PlaceholderRange(
offset=s.item(),
length=ln.item(),
is_embed=torch.ones(ln.item(), dtype=torch.bool),
)
for s, ln in zip(starts, lengths)
PlaceholderRange(offset=offset.item(), length=length.item())
for offset, length in zip(offsets, lengths)
]
processed_data["num_audio_tokens"] = lengths
return {"audio": ranges}
@@ -324,6 +373,7 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
hf_processor_mm_kwargs: Mapping[str, object],
mm_token_type_ids: torch.Tensor | None,
) -> dict[str, list[PlaceholderRange]]:
# Placeholders can't be located without them, so give up rather than guess
if mm_token_type_ids is None:
return {}
@@ -383,10 +433,13 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
with timing_ctx.record("apply_hf_processor"):
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
if not isinstance(prompt, str):
# the prompt is the tokenized ids which is not supported
# by the hf_processor, which is why we would need to decode the ids
# into string
# HF processors only accept text, and the decoded string already
# contains any special tokens, so don't let them be added again
prompt = hf_processor.decode(prompt)
tokenization_kwargs = {
**tokenization_kwargs,
"add_special_tokens": False,
}
# Bypass cached processor and always apply to the full set of mm inputs
# NOTE: we can't just set caching=False because base class method
@@ -412,9 +465,11 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
mm_token_type_ids = processed_data.pop("mm_token_type_ids", mm_token_type_ids)
mm_placeholders: dict[str, list[PlaceholderRange]] = {}
if self.info._is_audio_model():
mm_placeholders.update(self._apply_audio(prompt_ids, processed_data))
if self.info._is_image_model():
if num_audios := mm_items.get_count("audio", strict=False):
mm_placeholders.update(
self._apply_audio(prompt_ids, processed_data, num_audios)
)
if mm_items.get_count("image", strict=False):
mm_placeholders.update(
self._apply_vision(
prompt_ids,