[Misc] Support local image encoding in benchmarks (#43843)

Signed-off-by: xiaoz <[email protected]>

docs/benchmarking/cli.md

@@ -246,6 +246,12 @@ Every image listed in "image_files" is added to the request in the listed order
 
 The "image" shorthand accepts the same values as "image_files". The "image_url" field accepts either an OpenAI-style object with a "url" field or a URL string.
 
+By default, image references are sent to the serving endpoint as provided, with local image paths converted to `file://` URLs.
+
+If the benchmark client should load local and HTTP(S) images before sending requests, pass `--custom-ensure-client-side-data` to encode them as base64 data URLs on the client side.
+
+Existing `data:image/...` URLs are already self-contained and are kept unchanged.
+
 ```bash
 # need a model with vision capability here
 vllm serve Qwen/Qwen2-VL-7B-Instruct
@@ -253,13 +259,13 @@ vllm serve Qwen/Qwen2-VL-7B-Instruct
 
 ```bash
 # run benchmarking script
-vllm bench serve--save-result --save-detailed \
+vllm bench serve --save-result --save-detailed \
   --backend openai-chat \
   --model Qwen/Qwen2-VL-7B-Instruct \
   --endpoint /v1/chat/completions \
   --dataset-name custom_image \
   --dataset-path <path-to-your-image-data-jsonl> \
-  --allowed-local-media-path /path/to/image/folder
+  --custom-ensure-client-side-data
 ```
 
 Note that we need to use the `openai-chat` backend and `/v1/chat/completions` endpoint for multimodal inputs.

tests/benchmarks/test_custom_image_dataset.py

@@ -2,11 +2,15 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 import json
 from argparse import Namespace
+from io import BytesIO
 from pathlib import Path
 from typing import Any
 
+import pybase64 as base64
 import pytest
+from PIL import Image
 
+import vllm.benchmarks.datasets.datasets as datasets_module
 from vllm.benchmarks.datasets import CustomImageDataset, get_samples
 from vllm.benchmarks.lib.endpoint_request_func import (
     RequestFuncInput,
@@ -33,6 +37,22 @@ def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
             f.write(json.dumps(row) + "\n")
 
 
+def _write_png(path: Path, color: tuple[int, int, int] = (255, 0, 0)) -> None:
+    Image.new("RGB", (1, 1), color=color).save(path)
+
+
+def _decode_data_url(data_url: str) -> tuple[str, bytes]:
+    prefix, image_base64 = data_url.split(",", 1)
+    return prefix, base64.b64decode(image_base64)
+
+
+def _assert_png_data_url(data_url: str) -> None:
+    prefix, image_bytes = _decode_data_url(data_url)
+    assert prefix == "data:image/png;base64"
+    with Image.open(BytesIO(image_bytes)) as image:
+        image.verify()
+
+
 def _args_for_custom_image(dataset_path: Path) -> Namespace:
     return Namespace(
         dataset_name="custom_image",
@@ -42,6 +62,7 @@ def _args_for_custom_image(dataset_path: Path) -> Namespace:
         num_prompts=2,
         custom_output_len=32,
         enable_multimodal_chat=False,
+        custom_ensure_client_side_data=False,
         request_id_prefix="req-",
         no_oversample=False,
     )
@@ -230,6 +251,125 @@ def test_custom_image_dataset_wraps_interleaved_content_for_multimodal_chat(
     assert _get_chat_messages(request_input) == sample.prompt
 
 
+@pytest.mark.benchmark
+def test_custom_image_dataset_encodes_image_media_when_requested(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    image_a = tmp_path / "chart_a.png"
+    image_b = tmp_path / "chart b.png"
+    _write_png(image_a, color=(255, 0, 0))
+    _write_png(image_b, color=(0, 255, 0))
+    data_url = "data:image/png;base64,Zm9v"
+    remote_url = "https://example.com/chart.png"
+    original_fetch_image = datasets_module.fetch_image
+
+    def fake_fetch_image(image_url: str) -> Image.Image:
+        if image_url == remote_url:
+            return Image.new("RGB", (1, 1), color=(0, 0, 255))
+        return original_fetch_image(image_url)
+
+    monkeypatch.setattr(datasets_module, "fetch_image", fake_fetch_image)
+
+    jsonl = tmp_path / "images.jsonl"
+    _write_jsonl(
+        jsonl,
+        [
+            {
+                "prompt": "Compare the charts.",
+                "image_files": [
+                    str(image_a),
+                    image_b.as_uri(),
+                    remote_url,
+                    data_url,
+                ],
+            }
+        ],
+    )
+
+    dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
+    samples = dataset.sample(
+        tokenizer=_Tokenizer(),
+        num_requests=1,
+        output_len=32,
+        ensure_client_side_data=True,
+    )
+
+    assert len(samples) == 1
+    assert isinstance(samples[0].multi_modal_data, list)
+    image_urls = [part["image_url"]["url"] for part in samples[0].multi_modal_data]
+
+    _assert_png_data_url(image_urls[0])
+    _assert_png_data_url(image_urls[1])
+    _assert_png_data_url(image_urls[2])
+    assert image_urls[3] == data_url
+
+
+@pytest.mark.benchmark
+def test_custom_image_dataset_encodes_interleaved_image_media(
+    tmp_path: Path,
+) -> None:
+    image_a = tmp_path / "chart_a.png"
+    image_b = tmp_path / "chart_b.png"
+    _write_png(image_a, color=(255, 0, 0))
+    _write_png(image_b, color=(0, 255, 0))
+    jsonl = tmp_path / "images.jsonl"
+    _write_jsonl(
+        jsonl,
+        [
+            {
+                "content": [
+                    {"type": "text", "text": "Compare "},
+                    {"type": "image", "image": str(image_a)},
+                    {
+                        "type": "image_url",
+                        "image_url": {
+                            "url": image_b.as_uri(),
+                            "detail": "low",
+                        },
+                    },
+                ],
+            }
+        ],
+    )
+
+    dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
+    samples = dataset.sample(
+        tokenizer=_Tokenizer(),
+        num_requests=1,
+        output_len=32,
+        ensure_client_side_data=True,
+    )
+
+    sample = samples[0]
+    assert isinstance(sample.prompt, list)
+    _assert_png_data_url(sample.prompt[1]["image_url"]["url"])
+    _assert_png_data_url(sample.prompt[2]["image_url"]["url"])
+    assert sample.prompt[2]["image_url"]["detail"] == "low"
+
+
+@pytest.mark.benchmark
+def test_custom_image_dataset_rejects_invalid_image_media(
+    tmp_path: Path,
+) -> None:
+    invalid_image = tmp_path / "not_an_image.png"
+    invalid_image.write_text("not an image")
+    jsonl = tmp_path / "images.jsonl"
+    _write_jsonl(
+        jsonl,
+        [{"prompt": "Describe the image.", "image_files": [str(invalid_image)]}],
+    )
+
+    dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
+    with pytest.raises(ValueError, match="Invalid image URL"):
+        dataset.sample(
+            tokenizer=_Tokenizer(),
+            num_requests=1,
+            output_len=32,
+            ensure_client_side_data=True,
+        )
+
+
 @pytest.mark.benchmark
 def test_custom_image_dataset_rejects_invalid_content_part(
     tmp_path: Path,

vllm/benchmarks/datasets/datasets.py

@@ -44,6 +44,7 @@ from vllm.lora.request import LoRARequest
 from vllm.lora.utils import get_adapter_absolute_path
 from vllm.multimodal.audio import get_audio_duration
 from vllm.multimodal.image import convert_image_mode
+from vllm.multimodal.utils import encode_image_url, fetch_image
 from vllm.tokenizers import TokenizerLike
 from vllm.transformers_utils.repo_utils import hf_api
 from vllm.utils.argparse_utils import FlexibleArgumentParser
@@ -363,7 +364,11 @@ def lora_path_on_disk(lora_path: str) -> str:
 lora_tokenizer_cache: dict[int, TokenizerLike] = {}
 
 
-def process_image(image: Any) -> Mapping[str, Any]:
+def process_image(
+    image: Any,
+    *,
+    ensure_client_side_data: bool = False,
+) -> Mapping[str, Any]:
     """
     Process a single image input and return a multimedia content dictionary.
 
@@ -380,6 +385,9 @@ def process_image(image: Any) -> Mapping[str, Any]:
        encoded data.  - If string starts with "data:image/", treats as base64.
        - If string starts with "http://", "https://", or "file://", treats as URL.
        - Otherwise treats as local file path and prepends "file://".
+       - If ensure_client_side_data is True, local and HTTP(S) image references
+         are loaded and encoded as base64 image data URLs. Existing data:image
+         URLs are kept unchanged.
        - Returns a dictionary with the image URL or base64 data.
 
     Raises:
@@ -403,6 +411,13 @@ def process_image(image: Any) -> Mapping[str, Any]:
             if image.startswith(("http://", "https://", "file://", "data:image/"))
             else f"file://{image}"
         )
+
+        if ensure_client_side_data and not image_url.startswith("data:image/"):
+            try:
+                fetched_image = fetch_image(image_url)
+                image_url = encode_image_url(fetched_image)
+            except Exception as e:
+                raise ValueError(f"Invalid image URL: {image_url}") from e
         return {"type": "image_url", "image_url": {"url": image_url}}
 
     raise ValueError(
@@ -1645,6 +1660,16 @@ def add_dataset_parser(parser: FlexibleArgumentParser):
         "value overrides potential output length loaded from the dataset. It is "
         "used only for custom dataset.",
     )
+    custom_group.add_argument(
+        "--custom-ensure-client-side-data",
+        action="store_true",
+        help=(
+            "Ensure custom dataset media is sent as client-side data instead "
+            "of references. For custom_image datasets, this loads local and "
+            "HTTP(S) images on the benchmark client and encodes them as "
+            "base64 data URLs. Existing data:image URLs are kept unchanged."
+        ),
+    )
 
     spec_bench_group = parser.add_argument_group("spec bench dataset options")
     spec_bench_group.add_argument(
@@ -2075,6 +2100,9 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
             tokenizer=tokenizer,
             output_len=args.custom_output_len,
             enable_multimodal_chat=args.enable_multimodal_chat,
+            ensure_client_side_data=getattr(
+                args, "custom_ensure_client_side_data", False
+            ),
             request_id_prefix=args.request_id_prefix,
             no_oversample=args.no_oversample,
         )
@@ -2627,7 +2655,12 @@ class CustomImageDataset(CustomDataset):
         return parts
 
     @classmethod
-    def _process_content_part(cls, part: dict[str, Any]) -> dict[str, Any]:
+    def _process_content_part(
+        cls,
+        part: dict[str, Any],
+        *,
+        ensure_client_side_data: bool = False,
+    ) -> dict[str, Any]:
         content_type = part.get("type")
         if content_type == "text":
             text = part.get("text")
@@ -2638,12 +2671,22 @@ class CustomImageDataset(CustomDataset):
         if content_type == "image":
             if "image" not in part:
                 raise ValueError("Image content parts must contain an 'image' field.")
-            return dict(process_image(part["image"]))
+            return dict(
+                process_image(
+                    part["image"],
+                    ensure_client_side_data=ensure_client_side_data,
+                )
+            )
 
         if content_type == "image_url":
             image_url = part.get("image_url")
             if isinstance(image_url, str):
-                return dict(process_image(image_url))
+                return dict(
+                    process_image(
+                        image_url,
+                        ensure_client_side_data=ensure_client_side_data,
+                    )
+                )
 
             if isinstance(image_url, dict):
                 url = image_url.get("url")
@@ -2652,7 +2695,12 @@ class CustomImageDataset(CustomDataset):
                         "Image URL content parts must contain a string 'image_url.url'."
                     )
 
-                processed_part = dict(process_image(url))
+                processed_part = dict(
+                    process_image(
+                        url,
+                        ensure_client_side_data=ensure_client_side_data,
+                    )
+                )
                 processed_image_url = dict(processed_part["image_url"])
                 processed_image_url.update(
                     {key: value for key, value in image_url.items() if key != "url"}
@@ -2671,9 +2719,17 @@ class CustomImageDataset(CustomDataset):
         )
 
     @classmethod
-    def _process_interleaved_content(cls, content: Any) -> list[dict[str, Any]]:
+    def _process_interleaved_content(
+        cls,
+        content: Any,
+        *,
+        ensure_client_side_data: bool = False,
+    ) -> list[dict[str, Any]]:
         return [
-            cls._process_content_part(part)
+            cls._process_content_part(
+                part,
+                ensure_client_side_data=ensure_client_side_data,
+            )
             for part in cls._validate_content_parts(content)
         ]
 
@@ -2682,11 +2738,23 @@ class CustomImageDataset(CustomDataset):
         return "".join(part["text"] for part in content if part.get("type") == "text")
 
     @staticmethod
-    def _process_image_files(images: Any) -> dict[str, Any] | list[dict[str, Any]]:
+    def _process_image_files(
+        images: Any,
+        *,
+        ensure_client_side_data: bool = False,
+    ) -> dict[str, Any] | list[dict[str, Any]]:
         if not isinstance(images, list) or not images:
             raise ValueError("'image_files' must be a non-empty list.")
 
-        mm_content = [dict(process_image(image)) for image in images]
+        mm_content = [
+            dict(
+                process_image(
+                    image,
+                    ensure_client_side_data=ensure_client_side_data,
+                )
+            )
+            for image in images
+        ]
         if len(mm_content) == 1:
             return mm_content[0]
 
@@ -2698,6 +2766,7 @@ class CustomImageDataset(CustomDataset):
         num_requests: int,
         output_len: int | None = None,
         enable_multimodal_chat: bool = False,
+        ensure_client_side_data: bool = False,
         request_id_prefix: str = "",
         no_oversample: bool = False,
         **kwargs,
@@ -2718,9 +2787,14 @@ class CustomImageDataset(CustomDataset):
                 break
 
             if "content" in item:
-                content = self._process_interleaved_content(item["content"])
+                content = self._process_interleaved_content(
+                    item["content"],
+                    ensure_client_side_data=ensure_client_side_data,
+                )
                 text_prompt = self._get_text_from_content(content)
-                prompt_len = len(tokenizer(text_prompt).input_ids)
+                prompt_len = (
+                    1 if tokenizer is None else len(tokenizer(text_prompt).input_ids)
+                )
                 prompt = (
                     [{"role": "user", "content": content}]
                     if enable_multimodal_chat
@@ -2741,8 +2815,11 @@ class CustomImageDataset(CustomDataset):
             if not isinstance(prompt, str):
                 raise ValueError("'prompt' must be a string.")
 
-            prompt_len = len(tokenizer(prompt).input_ids)
-            mm_content = self._process_image_files(item["image_files"])
+            prompt_len = 1 if tokenizer is None else len(tokenizer(prompt).input_ids)
+            mm_content = self._process_image_files(
+                item["image_files"],
+                ensure_client_side_data=ensure_client_side_data,
+            )
             if enable_multimodal_chat:
                 # Note: when chat is enabled the request prompt_len is no longer
                 # accurate and we will be using request output to count the