From 53fa09d0857fb07da061137ea03c932da5e05f5d Mon Sep 17 00:00:00 2001 From: XiaoZ <57973980+xiaozcy@users.noreply.github.com> Date: Tue, 2 Jun 2026 23:15:06 +0800 Subject: [PATCH] [Misc] Support local image encoding in benchmarks (#43843) Signed-off-by: xiaoz --- docs/benchmarking/cli.md | 10 +- tests/benchmarks/test_custom_image_dataset.py | 140 ++++++++++++++++++ vllm/benchmarks/datasets/datasets.py | 103 +++++++++++-- 3 files changed, 238 insertions(+), 15 deletions(-) diff --git a/docs/benchmarking/cli.md b/docs/benchmarking/cli.md index 1b598aea38c..6d0b2a01aca 100644 --- a/docs/benchmarking/cli.md +++ b/docs/benchmarking/cli.md @@ -246,6 +246,12 @@ Every image listed in "image_files" is added to the request in the listed order The "image" shorthand accepts the same values as "image_files". The "image_url" field accepts either an OpenAI-style object with a "url" field or a URL string. +By default, image references are sent to the serving endpoint as provided, with local image paths converted to `file://` URLs. + +If the benchmark client should load local and HTTP(S) images before sending requests, pass `--custom-ensure-client-side-data` to encode them as base64 data URLs on the client side. + +Existing `data:image/...` URLs are already self-contained and are kept unchanged. + ```bash # need a model with vision capability here vllm serve Qwen/Qwen2-VL-7B-Instruct @@ -253,13 +259,13 @@ vllm serve Qwen/Qwen2-VL-7B-Instruct ```bash # run benchmarking script -vllm bench serve--save-result --save-detailed \ +vllm bench serve --save-result --save-detailed \ --backend openai-chat \ --model Qwen/Qwen2-VL-7B-Instruct \ --endpoint /v1/chat/completions \ --dataset-name custom_image \ --dataset-path \ - --allowed-local-media-path /path/to/image/folder + --custom-ensure-client-side-data ``` Note that we need to use the `openai-chat` backend and `/v1/chat/completions` endpoint for multimodal inputs. diff --git a/tests/benchmarks/test_custom_image_dataset.py b/tests/benchmarks/test_custom_image_dataset.py index 336bac93d0b..f2a48abe604 100644 --- a/tests/benchmarks/test_custom_image_dataset.py +++ b/tests/benchmarks/test_custom_image_dataset.py @@ -2,11 +2,15 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import json from argparse import Namespace +from io import BytesIO from pathlib import Path from typing import Any +import pybase64 as base64 import pytest +from PIL import Image +import vllm.benchmarks.datasets.datasets as datasets_module from vllm.benchmarks.datasets import CustomImageDataset, get_samples from vllm.benchmarks.lib.endpoint_request_func import ( RequestFuncInput, @@ -33,6 +37,22 @@ def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None: f.write(json.dumps(row) + "\n") +def _write_png(path: Path, color: tuple[int, int, int] = (255, 0, 0)) -> None: + Image.new("RGB", (1, 1), color=color).save(path) + + +def _decode_data_url(data_url: str) -> tuple[str, bytes]: + prefix, image_base64 = data_url.split(",", 1) + return prefix, base64.b64decode(image_base64) + + +def _assert_png_data_url(data_url: str) -> None: + prefix, image_bytes = _decode_data_url(data_url) + assert prefix == "data:image/png;base64" + with Image.open(BytesIO(image_bytes)) as image: + image.verify() + + def _args_for_custom_image(dataset_path: Path) -> Namespace: return Namespace( dataset_name="custom_image", @@ -42,6 +62,7 @@ def _args_for_custom_image(dataset_path: Path) -> Namespace: num_prompts=2, custom_output_len=32, enable_multimodal_chat=False, + custom_ensure_client_side_data=False, request_id_prefix="req-", no_oversample=False, ) @@ -230,6 +251,125 @@ def test_custom_image_dataset_wraps_interleaved_content_for_multimodal_chat( assert _get_chat_messages(request_input) == sample.prompt +@pytest.mark.benchmark +def test_custom_image_dataset_encodes_image_media_when_requested( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + image_a = tmp_path / "chart_a.png" + image_b = tmp_path / "chart b.png" + _write_png(image_a, color=(255, 0, 0)) + _write_png(image_b, color=(0, 255, 0)) + data_url = "data:image/png;base64,Zm9v" + remote_url = "https://example.com/chart.png" + original_fetch_image = datasets_module.fetch_image + + def fake_fetch_image(image_url: str) -> Image.Image: + if image_url == remote_url: + return Image.new("RGB", (1, 1), color=(0, 0, 255)) + return original_fetch_image(image_url) + + monkeypatch.setattr(datasets_module, "fetch_image", fake_fetch_image) + + jsonl = tmp_path / "images.jsonl" + _write_jsonl( + jsonl, + [ + { + "prompt": "Compare the charts.", + "image_files": [ + str(image_a), + image_b.as_uri(), + remote_url, + data_url, + ], + } + ], + ) + + dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True) + samples = dataset.sample( + tokenizer=_Tokenizer(), + num_requests=1, + output_len=32, + ensure_client_side_data=True, + ) + + assert len(samples) == 1 + assert isinstance(samples[0].multi_modal_data, list) + image_urls = [part["image_url"]["url"] for part in samples[0].multi_modal_data] + + _assert_png_data_url(image_urls[0]) + _assert_png_data_url(image_urls[1]) + _assert_png_data_url(image_urls[2]) + assert image_urls[3] == data_url + + +@pytest.mark.benchmark +def test_custom_image_dataset_encodes_interleaved_image_media( + tmp_path: Path, +) -> None: + image_a = tmp_path / "chart_a.png" + image_b = tmp_path / "chart_b.png" + _write_png(image_a, color=(255, 0, 0)) + _write_png(image_b, color=(0, 255, 0)) + jsonl = tmp_path / "images.jsonl" + _write_jsonl( + jsonl, + [ + { + "content": [ + {"type": "text", "text": "Compare "}, + {"type": "image", "image": str(image_a)}, + { + "type": "image_url", + "image_url": { + "url": image_b.as_uri(), + "detail": "low", + }, + }, + ], + } + ], + ) + + dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True) + samples = dataset.sample( + tokenizer=_Tokenizer(), + num_requests=1, + output_len=32, + ensure_client_side_data=True, + ) + + sample = samples[0] + assert isinstance(sample.prompt, list) + _assert_png_data_url(sample.prompt[1]["image_url"]["url"]) + _assert_png_data_url(sample.prompt[2]["image_url"]["url"]) + assert sample.prompt[2]["image_url"]["detail"] == "low" + + +@pytest.mark.benchmark +def test_custom_image_dataset_rejects_invalid_image_media( + tmp_path: Path, +) -> None: + invalid_image = tmp_path / "not_an_image.png" + invalid_image.write_text("not an image") + jsonl = tmp_path / "images.jsonl" + _write_jsonl( + jsonl, + [{"prompt": "Describe the image.", "image_files": [str(invalid_image)]}], + ) + + dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True) + with pytest.raises(ValueError, match="Invalid image URL"): + dataset.sample( + tokenizer=_Tokenizer(), + num_requests=1, + output_len=32, + ensure_client_side_data=True, + ) + + @pytest.mark.benchmark def test_custom_image_dataset_rejects_invalid_content_part( tmp_path: Path, diff --git a/vllm/benchmarks/datasets/datasets.py b/vllm/benchmarks/datasets/datasets.py index af62f049ff3..cc64bd0b94b 100644 --- a/vllm/benchmarks/datasets/datasets.py +++ b/vllm/benchmarks/datasets/datasets.py @@ -44,6 +44,7 @@ from vllm.lora.request import LoRARequest from vllm.lora.utils import get_adapter_absolute_path from vllm.multimodal.audio import get_audio_duration from vllm.multimodal.image import convert_image_mode +from vllm.multimodal.utils import encode_image_url, fetch_image from vllm.tokenizers import TokenizerLike from vllm.transformers_utils.repo_utils import hf_api from vllm.utils.argparse_utils import FlexibleArgumentParser @@ -363,7 +364,11 @@ def lora_path_on_disk(lora_path: str) -> str: lora_tokenizer_cache: dict[int, TokenizerLike] = {} -def process_image(image: Any) -> Mapping[str, Any]: +def process_image( + image: Any, + *, + ensure_client_side_data: bool = False, +) -> Mapping[str, Any]: """ Process a single image input and return a multimedia content dictionary. @@ -380,6 +385,9 @@ def process_image(image: Any) -> Mapping[str, Any]: encoded data. - If string starts with "data:image/", treats as base64. - If string starts with "http://", "https://", or "file://", treats as URL. - Otherwise treats as local file path and prepends "file://". + - If ensure_client_side_data is True, local and HTTP(S) image references + are loaded and encoded as base64 image data URLs. Existing data:image + URLs are kept unchanged. - Returns a dictionary with the image URL or base64 data. Raises: @@ -403,6 +411,13 @@ def process_image(image: Any) -> Mapping[str, Any]: if image.startswith(("http://", "https://", "file://", "data:image/")) else f"file://{image}" ) + + if ensure_client_side_data and not image_url.startswith("data:image/"): + try: + fetched_image = fetch_image(image_url) + image_url = encode_image_url(fetched_image) + except Exception as e: + raise ValueError(f"Invalid image URL: {image_url}") from e return {"type": "image_url", "image_url": {"url": image_url}} raise ValueError( @@ -1645,6 +1660,16 @@ def add_dataset_parser(parser: FlexibleArgumentParser): "value overrides potential output length loaded from the dataset. It is " "used only for custom dataset.", ) + custom_group.add_argument( + "--custom-ensure-client-side-data", + action="store_true", + help=( + "Ensure custom dataset media is sent as client-side data instead " + "of references. For custom_image datasets, this loads local and " + "HTTP(S) images on the benchmark client and encodes them as " + "base64 data URLs. Existing data:image URLs are kept unchanged." + ), + ) spec_bench_group = parser.add_argument_group("spec bench dataset options") spec_bench_group.add_argument( @@ -2075,6 +2100,9 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]: tokenizer=tokenizer, output_len=args.custom_output_len, enable_multimodal_chat=args.enable_multimodal_chat, + ensure_client_side_data=getattr( + args, "custom_ensure_client_side_data", False + ), request_id_prefix=args.request_id_prefix, no_oversample=args.no_oversample, ) @@ -2627,7 +2655,12 @@ class CustomImageDataset(CustomDataset): return parts @classmethod - def _process_content_part(cls, part: dict[str, Any]) -> dict[str, Any]: + def _process_content_part( + cls, + part: dict[str, Any], + *, + ensure_client_side_data: bool = False, + ) -> dict[str, Any]: content_type = part.get("type") if content_type == "text": text = part.get("text") @@ -2638,12 +2671,22 @@ class CustomImageDataset(CustomDataset): if content_type == "image": if "image" not in part: raise ValueError("Image content parts must contain an 'image' field.") - return dict(process_image(part["image"])) + return dict( + process_image( + part["image"], + ensure_client_side_data=ensure_client_side_data, + ) + ) if content_type == "image_url": image_url = part.get("image_url") if isinstance(image_url, str): - return dict(process_image(image_url)) + return dict( + process_image( + image_url, + ensure_client_side_data=ensure_client_side_data, + ) + ) if isinstance(image_url, dict): url = image_url.get("url") @@ -2652,7 +2695,12 @@ class CustomImageDataset(CustomDataset): "Image URL content parts must contain a string 'image_url.url'." ) - processed_part = dict(process_image(url)) + processed_part = dict( + process_image( + url, + ensure_client_side_data=ensure_client_side_data, + ) + ) processed_image_url = dict(processed_part["image_url"]) processed_image_url.update( {key: value for key, value in image_url.items() if key != "url"} @@ -2671,9 +2719,17 @@ class CustomImageDataset(CustomDataset): ) @classmethod - def _process_interleaved_content(cls, content: Any) -> list[dict[str, Any]]: + def _process_interleaved_content( + cls, + content: Any, + *, + ensure_client_side_data: bool = False, + ) -> list[dict[str, Any]]: return [ - cls._process_content_part(part) + cls._process_content_part( + part, + ensure_client_side_data=ensure_client_side_data, + ) for part in cls._validate_content_parts(content) ] @@ -2682,11 +2738,23 @@ class CustomImageDataset(CustomDataset): return "".join(part["text"] for part in content if part.get("type") == "text") @staticmethod - def _process_image_files(images: Any) -> dict[str, Any] | list[dict[str, Any]]: + def _process_image_files( + images: Any, + *, + ensure_client_side_data: bool = False, + ) -> dict[str, Any] | list[dict[str, Any]]: if not isinstance(images, list) or not images: raise ValueError("'image_files' must be a non-empty list.") - mm_content = [dict(process_image(image)) for image in images] + mm_content = [ + dict( + process_image( + image, + ensure_client_side_data=ensure_client_side_data, + ) + ) + for image in images + ] if len(mm_content) == 1: return mm_content[0] @@ -2698,6 +2766,7 @@ class CustomImageDataset(CustomDataset): num_requests: int, output_len: int | None = None, enable_multimodal_chat: bool = False, + ensure_client_side_data: bool = False, request_id_prefix: str = "", no_oversample: bool = False, **kwargs, @@ -2718,9 +2787,14 @@ class CustomImageDataset(CustomDataset): break if "content" in item: - content = self._process_interleaved_content(item["content"]) + content = self._process_interleaved_content( + item["content"], + ensure_client_side_data=ensure_client_side_data, + ) text_prompt = self._get_text_from_content(content) - prompt_len = len(tokenizer(text_prompt).input_ids) + prompt_len = ( + 1 if tokenizer is None else len(tokenizer(text_prompt).input_ids) + ) prompt = ( [{"role": "user", "content": content}] if enable_multimodal_chat @@ -2741,8 +2815,11 @@ class CustomImageDataset(CustomDataset): if not isinstance(prompt, str): raise ValueError("'prompt' must be a string.") - prompt_len = len(tokenizer(prompt).input_ids) - mm_content = self._process_image_files(item["image_files"]) + prompt_len = 1 if tokenizer is None else len(tokenizer(prompt).input_ids) + mm_content = self._process_image_files( + item["image_files"], + ensure_client_side_data=ensure_client_side_data, + ) if enable_multimodal_chat: # Note: when chat is enabled the request prompt_len is no longer # accurate and we will be using request output to count the