@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 53fa09d085

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 2, 04:45 UTC

[Misc] Support local image encoding in benchmarks (#43843)

Signed-off-by: xiaoz <Sukra1@outlook.com>

XiaoZ committed Jun 2, 2026 at 23:15 UTC 53fa09d0857fb07da061137ea03c932da5e05f5d

3 files changed +238 -15

docs/benchmarking/cli.md

+8 -2

@@ -246,6 +246,12 @@ Every image listed in "image_files" is added to the request in the listed order

246

247 The "image" shorthand accepts the same values as "image_files". The "image_url" field accepts either an OpenAI-style object with a "url" field or a URL string.

248

249 +By default, image references are sent to the serving endpoint as provided, with local image paths converted to `file://` URLs.

250 +

251 +If the benchmark client should load local and HTTP(S) images before sending requests, pass `--custom-ensure-client-side-data` to encode them as base64 data URLs on the client side.

252 +

253 +Existing `data:image/...` URLs are already self-contained and are kept unchanged.

254 +

255 ```bash

256 # need a model with vision capability here

257 vllm serve Qwen/Qwen2-VL-7B-Instruct

@@ -253,13 +259,13 @@ vllm serve Qwen/Qwen2-VL-7B-Instruct

259

260 ```bash

261 # run benchmarking script

256 -vllm bench serve--save-result --save-detailed \

262 +vllm bench serve --save-result --save-detailed \

263 --backend openai-chat \

264 --model Qwen/Qwen2-VL-7B-Instruct \

265 --endpoint /v1/chat/completions \

266 --dataset-name custom_image \

267 --dataset-path <path-to-your-image-data-jsonl> \

262 - --allowed-local-media-path /path/to/image/folder

268 + --custom-ensure-client-side-data

269 ```

270

271 Note that we need to use the `openai-chat` backend and `/v1/chat/completions` endpoint for multimodal inputs.

tests/benchmarks/test_custom_image_dataset.py

+140

@@ -2,11 +2,15 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 import json

4 from argparse import Namespace

5 +from io import BytesIO

6 from pathlib import Path

7 from typing import Any

9 +import pybase64 as base64

10 import pytest

11 +from PIL import Image

13 +import vllm.benchmarks.datasets.datasets as datasets_module

14 from vllm.benchmarks.datasets import CustomImageDataset, get_samples

15 from vllm.benchmarks.lib.endpoint_request_func import (

16 RequestFuncInput,

@@ -33,6 +37,22 @@ def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:

37 f.write(json.dumps(row) + "\n")

40 +def _write_png(path: Path, color: tuple[int, int, int] = (255, 0, 0)) -> None:

41 + Image.new("RGB", (1, 1), color=color).save(path)

42 +

43 +

44 +def _decode_data_url(data_url: str) -> tuple[str, bytes]:

45 + prefix, image_base64 = data_url.split(",", 1)

46 + return prefix, base64.b64decode(image_base64)

47 +

48 +

49 +def _assert_png_data_url(data_url: str) -> None:

50 + prefix, image_bytes = _decode_data_url(data_url)

51 + assert prefix == "data:image/png;base64"

52 + with Image.open(BytesIO(image_bytes)) as image:

53 + image.verify()

54 +

55 +

56 def _args_for_custom_image(dataset_path: Path) -> Namespace:

57 return Namespace(

58 dataset_name="custom_image",

@@ -42,6 +62,7 @@ def _args_for_custom_image(dataset_path: Path) -> Namespace:

62 num_prompts=2,

63 custom_output_len=32,

64 enable_multimodal_chat=False,

65 + custom_ensure_client_side_data=False,

66 request_id_prefix="req-",

67 no_oversample=False,

68 )

@@ -230,6 +251,125 @@ def test_custom_image_dataset_wraps_interleaved_content_for_multimodal_chat(

251 assert _get_chat_messages(request_input) == sample.prompt

252

253

254 +@pytest.mark.benchmark

255 +def test_custom_image_dataset_encodes_image_media_when_requested(

256 + tmp_path: Path,

257 + monkeypatch: pytest.MonkeyPatch,

258 +) -> None:

259 + image_a = tmp_path / "chart_a.png"

260 + image_b = tmp_path / "chart b.png"

261 + _write_png(image_a, color=(255, 0, 0))

262 + _write_png(image_b, color=(0, 255, 0))

263 + data_url = "data:image/png;base64,Zm9v"

264 + remote_url = "https://example.com/chart.png"

265 + original_fetch_image = datasets_module.fetch_image

266 +

267 + def fake_fetch_image(image_url: str) -> Image.Image:

268 + if image_url == remote_url:

269 + return Image.new("RGB", (1, 1), color=(0, 0, 255))

270 + return original_fetch_image(image_url)

271 +

272 + monkeypatch.setattr(datasets_module, "fetch_image", fake_fetch_image)

273 +

274 + jsonl = tmp_path / "images.jsonl"

275 + _write_jsonl(

276 + jsonl,

277 + [

278 + {

279 + "prompt": "Compare the charts.",

280 + "image_files": [

281 + str(image_a),

282 + image_b.as_uri(),

283 + remote_url,

284 + data_url,

285 + ],

286 + }

287 + ],

288 + )

289 +

290 + dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)

291 + samples = dataset.sample(

292 + tokenizer=_Tokenizer(),

293 + num_requests=1,

294 + output_len=32,

295 + ensure_client_side_data=True,

296 + )

297 +

298 + assert len(samples) == 1

299 + assert isinstance(samples[0].multi_modal_data, list)

300 + image_urls = [part["image_url"]["url"] for part in samples[0].multi_modal_data]

301 +

302 + _assert_png_data_url(image_urls[0])

303 + _assert_png_data_url(image_urls[1])

304 + _assert_png_data_url(image_urls[2])

305 + assert image_urls[3] == data_url

306 +

307 +

308 +@pytest.mark.benchmark

309 +def test_custom_image_dataset_encodes_interleaved_image_media(

310 + tmp_path: Path,

311 +) -> None:

312 + image_a = tmp_path / "chart_a.png"

313 + image_b = tmp_path / "chart_b.png"

314 + _write_png(image_a, color=(255, 0, 0))

315 + _write_png(image_b, color=(0, 255, 0))

316 + jsonl = tmp_path / "images.jsonl"

317 + _write_jsonl(

318 + jsonl,

319 + [

320 + {

321 + "content": [

322 + {"type": "text", "text": "Compare "},

323 + {"type": "image", "image": str(image_a)},

324 + {

325 + "type": "image_url",

326 + "image_url": {

327 + "url": image_b.as_uri(),

328 + "detail": "low",

329 + },

330 + },

331 + ],

332 + }

333 + ],

334 + )

335 +

336 + dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)

337 + samples = dataset.sample(

338 + tokenizer=_Tokenizer(),

339 + num_requests=1,

340 + output_len=32,

341 + ensure_client_side_data=True,

342 + )

343 +

344 + sample = samples[0]

345 + assert isinstance(sample.prompt, list)

346 + _assert_png_data_url(sample.prompt[1]["image_url"]["url"])

347 + _assert_png_data_url(sample.prompt[2]["image_url"]["url"])

348 + assert sample.prompt[2]["image_url"]["detail"] == "low"

349 +

350 +

351 +@pytest.mark.benchmark

352 +def test_custom_image_dataset_rejects_invalid_image_media(

353 + tmp_path: Path,

354 +) -> None:

355 + invalid_image = tmp_path / "not_an_image.png"

356 + invalid_image.write_text("not an image")

357 + jsonl = tmp_path / "images.jsonl"

358 + _write_jsonl(

359 + jsonl,

360 + [{"prompt": "Describe the image.", "image_files": [str(invalid_image)]}],

361 + )

362 +

363 + dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)

364 + with pytest.raises(ValueError, match="Invalid image URL"):

365 + dataset.sample(

366 + tokenizer=_Tokenizer(),

367 + num_requests=1,

368 + output_len=32,

369 + ensure_client_side_data=True,

370 + )

371 +

372 +

373 @pytest.mark.benchmark

374 def test_custom_image_dataset_rejects_invalid_content_part(

375 tmp_path: Path,

vllm/benchmarks/datasets/datasets.py

+90 -13

@@ -44,6 +44,7 @@ from vllm.lora.request import LoRARequest

44 from vllm.lora.utils import get_adapter_absolute_path

45 from vllm.multimodal.audio import get_audio_duration

46 from vllm.multimodal.image import convert_image_mode

47 +from vllm.multimodal.utils import encode_image_url, fetch_image

48 from vllm.tokenizers import TokenizerLike

49 from vllm.transformers_utils.repo_utils import hf_api

50 from vllm.utils.argparse_utils import FlexibleArgumentParser

@@ -363,7 +364,11 @@ def lora_path_on_disk(lora_path: str) -> str:

364 lora_tokenizer_cache: dict[int, TokenizerLike] = {}

365

366

366 -def process_image(image: Any) -> Mapping[str, Any]:

367 +def process_image(

368 + image: Any,

369 + *,

370 + ensure_client_side_data: bool = False,

371 +) -> Mapping[str, Any]:

372 """

373 Process a single image input and return a multimedia content dictionary.

374

@@ -380,6 +385,9 @@ def process_image(image: Any) -> Mapping[str, Any]:

385 encoded data. - If string starts with "data:image/", treats as base64.

386 - If string starts with "http://", "https://", or "file://", treats as URL.

387 - Otherwise treats as local file path and prepends "file://".

388 + - If ensure_client_side_data is True, local and HTTP(S) image references

389 + are loaded and encoded as base64 image data URLs. Existing data:image

390 + URLs are kept unchanged.

391 - Returns a dictionary with the image URL or base64 data.

392

393 Raises:

@@ -403,6 +411,13 @@ def process_image(image: Any) -> Mapping[str, Any]:

411 if image.startswith(("http://", "https://", "file://", "data:image/"))

412 else f"file://{image}"

413 )

414 +

415 + if ensure_client_side_data and not image_url.startswith("data:image/"):

416 + try:

417 + fetched_image = fetch_image(image_url)

418 + image_url = encode_image_url(fetched_image)

419 + except Exception as e:

420 + raise ValueError(f"Invalid image URL: {image_url}") from e

421 return {"type": "image_url", "image_url": {"url": image_url}}

422

423 raise ValueError(

@@ -1645,6 +1660,16 @@ def add_dataset_parser(parser: FlexibleArgumentParser):

1660 "value overrides potential output length loaded from the dataset. It is "

1661 "used only for custom dataset.",

1662 )

1663 + custom_group.add_argument(

1664 + "--custom-ensure-client-side-data",

1665 + action="store_true",

1666 + help=(

1667 + "Ensure custom dataset media is sent as client-side data instead "

1668 + "of references. For custom_image datasets, this loads local and "

1669 + "HTTP(S) images on the benchmark client and encodes them as "

1670 + "base64 data URLs. Existing data:image URLs are kept unchanged."

1671 + ),

1672 + )

1673

1674 spec_bench_group = parser.add_argument_group("spec bench dataset options")

1675 spec_bench_group.add_argument(

@@ -2075,6 +2100,9 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:

2100 tokenizer=tokenizer,

2101 output_len=args.custom_output_len,

2102 enable_multimodal_chat=args.enable_multimodal_chat,

2103 + ensure_client_side_data=getattr(

2104 + args, "custom_ensure_client_side_data", False

2105 + ),

2106 request_id_prefix=args.request_id_prefix,

2107 no_oversample=args.no_oversample,

2108 )

@@ -2627,7 +2655,12 @@ class CustomImageDataset(CustomDataset):

2655 return parts

2656

2657 @classmethod

2630 - def _process_content_part(cls, part: dict[str, Any]) -> dict[str, Any]:

2658 + def _process_content_part(

2659 + cls,

2660 + part: dict[str, Any],

2661 + *,

2662 + ensure_client_side_data: bool = False,

2663 + ) -> dict[str, Any]:

2664 content_type = part.get("type")

2665 if content_type == "text":

2666 text = part.get("text")

@@ -2638,12 +2671,22 @@ class CustomImageDataset(CustomDataset):

2671 if content_type == "image":

2672 if "image" not in part:

2673 raise ValueError("Image content parts must contain an 'image' field.")

2641 - return dict(process_image(part["image"]))

2674 + return dict(

2675 + process_image(

2676 + part["image"],

2677 + ensure_client_side_data=ensure_client_side_data,

2678 + )

2679 + )

2680

2681 if content_type == "image_url":

2682 image_url = part.get("image_url")

2683 if isinstance(image_url, str):

2646 - return dict(process_image(image_url))

2684 + return dict(

2685 + process_image(

2686 + image_url,

2687 + ensure_client_side_data=ensure_client_side_data,

2688 + )

2689 + )

2690

2691 if isinstance(image_url, dict):

2692 url = image_url.get("url")

@@ -2652,7 +2695,12 @@ class CustomImageDataset(CustomDataset):

2695 "Image URL content parts must contain a string 'image_url.url'."

2696 )

2697

2655 - processed_part = dict(process_image(url))

2698 + processed_part = dict(

2699 + process_image(

2700 + url,

2701 + ensure_client_side_data=ensure_client_side_data,

2702 + )

2703 + )

2704 processed_image_url = dict(processed_part["image_url"])

2705 processed_image_url.update(

2706 {key: value for key, value in image_url.items() if key != "url"}

@@ -2671,9 +2719,17 @@ class CustomImageDataset(CustomDataset):

2719 )

2720

2721 @classmethod

2674 - def _process_interleaved_content(cls, content: Any) -> list[dict[str, Any]]:

2722 + def _process_interleaved_content(

2723 + cls,

2724 + content: Any,

2725 + *,

2726 + ensure_client_side_data: bool = False,

2727 + ) -> list[dict[str, Any]]:

2728 return [

2676 - cls._process_content_part(part)

2729 + cls._process_content_part(

2730 + part,

2731 + ensure_client_side_data=ensure_client_side_data,

2732 + )

2733 for part in cls._validate_content_parts(content)

2734 ]

2735

@@ -2682,11 +2738,23 @@ class CustomImageDataset(CustomDataset):

2738 return "".join(part["text"] for part in content if part.get("type") == "text")

2739

2740 @staticmethod

2685 - def _process_image_files(images: Any) -> dict[str, Any] | list[dict[str, Any]]:

2741 + def _process_image_files(

2742 + images: Any,

2743 + *,

2744 + ensure_client_side_data: bool = False,

2745 + ) -> dict[str, Any] | list[dict[str, Any]]:

2746 if not isinstance(images, list) or not images:

2747 raise ValueError("'image_files' must be a non-empty list.")

2748

2689 - mm_content = [dict(process_image(image)) for image in images]

2749 + mm_content = [

2750 + dict(

2751 + process_image(

2752 + image,

2753 + ensure_client_side_data=ensure_client_side_data,

2754 + )

2755 + )

2756 + for image in images

2757 + ]

2758 if len(mm_content) == 1:

2759 return mm_content[0]

2760

@@ -2698,6 +2766,7 @@ class CustomImageDataset(CustomDataset):

2766 num_requests: int,

2767 output_len: int | None = None,

2768 enable_multimodal_chat: bool = False,

2769 + ensure_client_side_data: bool = False,

2770 request_id_prefix: str = "",

2771 no_oversample: bool = False,

2772 **kwargs,

@@ -2718,9 +2787,14 @@ class CustomImageDataset(CustomDataset):

2787 break

2788

2789 if "content" in item:

2721 - content = self._process_interleaved_content(item["content"])

2790 + content = self._process_interleaved_content(

2791 + item["content"],

2792 + ensure_client_side_data=ensure_client_side_data,

2793 + )

2794 text_prompt = self._get_text_from_content(content)

2723 - prompt_len = len(tokenizer(text_prompt).input_ids)

2795 + prompt_len = (

2796 + 1 if tokenizer is None else len(tokenizer(text_prompt).input_ids)

2797 + )

2798 prompt = (

2799 [{"role": "user", "content": content}]

2800 if enable_multimodal_chat

@@ -2741,8 +2815,11 @@ class CustomImageDataset(CustomDataset):

2815 if not isinstance(prompt, str):

2816 raise ValueError("'prompt' must be a string.")

2817

2744 - prompt_len = len(tokenizer(prompt).input_ids)

2745 - mm_content = self._process_image_files(item["image_files"])

2818 + prompt_len = 1 if tokenizer is None else len(tokenizer(prompt).input_ids)

2819 + mm_content = self._process_image_files(

2820 + item["image_files"],

2821 + ensure_client_side_data=ensure_client_side_data,

2822 + )

2823 if enable_multimodal_chat:

2824 # Note: when chat is enabled the request prompt_len is no longer

2825 # accurate and we will be using request output to count the