A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 20:45 UTC
[Misc] Support interleaved custom image benchmark datasets (#43636)
Signed-off-by: ThibaultCastells <thib.castells@icloud.com>
Thibault Castells committed
May 26, 2026 at 19:37 UTC
5d09f471f4ec44edfb0aa8917a985bda7c897195
4 files changed
+471
-22
docs/benchmarking/cli.md
+9
-1
@@ -231,13 +231,21 @@ vllm bench serve \
231
232
#### Custom Image Dataset
233
234
-If the image dataset you want to benchmark is not supported yet in vLLM, then you can benchmark on it using `CustomImageDataset`. At inference time, use the option `--dataset-name custom_image`. Your data needs to be in the `.jsonl` format and needs to have "prompt" and "image_files" fields per entry, e.g., `image_data.jsonl`:
234
+If the image dataset you want to benchmark is not supported yet in vLLM, then you can benchmark on it using `CustomImageDataset`. At inference time, use the option `--dataset-name custom_image`. Your data needs to be in the `.jsonl` format and can use "prompt" and "image_files" fields per entry, e.g., `image_data.jsonl`:
235
236
```json
237
{"prompt": "How many animals are present in the given image?", "image_files": ["/path/to/image/folder/horsepony.jpg"]}
238
{"prompt": "What colour is the bird shown in the image?", "image_files": ["/path/to/image/folder/flycatcher.jpeg"]}
239
```
240
241
+Every image listed in "image_files" is added to the request in the listed order after the prompt text. To preserve an interleaved order of text and images, use a "content" field with OpenAI-compatible content parts:
242
+
243
+```json
244
+{"content": [{"type": "text", "text": "Compare "}, {"type": "image", "image": "/path/to/image/folder/chart_a.png"}, {"type": "text", "text": " with "}, {"type": "image_url", "image_url": {"url": "/path/to/image/folder/chart_b.png"}}]}
245
+```
246
+
247
+The "image" shorthand accepts the same values as "image_files". The "image_url" field accepts either an OpenAI-style object with a "url" field or a URL string.
248
+
249
```bash
250
# need a model with vision capability here
251
vllm serve Qwen/Qwen2-VL-7B-Instruct
tests/benchmarks/test_custom_image_dataset.py
new
+246
@@ -0,0 +1,246 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import json
4
+from argparse import Namespace
5
+from pathlib import Path
6
+from typing import Any
7
+
8
+import pytest
9
+
10
+from vllm.benchmarks.datasets import CustomImageDataset, get_samples
11
+from vllm.benchmarks.lib.endpoint_request_func import (
12
+ RequestFuncInput,
13
+ _get_chat_content,
14
+ _get_chat_messages,
15
+)
16
+
17
+pytestmark = pytest.mark.skip_global_cleanup
18
+
19
+
20
+class _TokenizedPrompt:
21
+ def __init__(self, prompt: str) -> None:
22
+ self.input_ids = prompt.split()
23
+
24
+
25
+class _Tokenizer:
26
+ def __call__(self, prompt: str) -> _TokenizedPrompt:
27
+ return _TokenizedPrompt(prompt)
28
+
29
+
30
+def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
31
+ with path.open("w") as f:
32
+ for row in rows:
33
+ f.write(json.dumps(row) + "\n")
34
+
35
+
36
+def _args_for_custom_image(dataset_path: Path) -> Namespace:
37
+ return Namespace(
38
+ dataset_name="custom_image",
39
+ dataset_path=str(dataset_path),
40
+ disable_shuffle=True,
41
+ seed=0,
42
+ num_prompts=2,
43
+ custom_output_len=32,
44
+ enable_multimodal_chat=False,
45
+ request_id_prefix="req-",
46
+ no_oversample=False,
47
+ )
48
+
49
+
50
+@pytest.mark.benchmark
51
+def test_get_samples_custom_image_cli_path_supports_multi_image_and_content(
52
+ tmp_path: Path,
53
+) -> None:
54
+ image_a = tmp_path / "chart_a.png"
55
+ image_b = tmp_path / "chart_b.png"
56
+ image_c = tmp_path / "chart_c.png"
57
+ jsonl = tmp_path / "images.jsonl"
58
+ _write_jsonl(
59
+ jsonl,
60
+ [
61
+ {
62
+ "prompt": "Compare the first two charts.",
63
+ "image_files": [str(image_a), str(image_b)],
64
+ },
65
+ {
66
+ "content": [
67
+ {"type": "text", "text": "Now compare "},
68
+ {"type": "image", "image": str(image_c)},
69
+ ],
70
+ },
71
+ ],
72
+ )
73
+
74
+ samples = get_samples(_args_for_custom_image(jsonl), _Tokenizer())
75
+
76
+ assert len(samples) == 2
77
+ assert samples[0].request_id == "req-0"
78
+ assert isinstance(samples[0].multi_modal_data, list)
79
+ assert [part["image_url"]["url"] for part in samples[0].multi_modal_data] == [
80
+ f"file://{image_a}",
81
+ f"file://{image_b}",
82
+ ]
83
+
84
+ assert samples[1].request_id == "req-1"
85
+ assert samples[1].multi_modal_data is None
86
+ assert isinstance(samples[1].prompt, list)
87
+ assert samples[1].prompt[0] == {"type": "text", "text": "Now compare "}
88
+ assert samples[1].prompt[1]["image_url"]["url"] == f"file://{image_c}"
89
+
90
+
91
+@pytest.mark.benchmark
92
+def test_custom_image_dataset_uses_all_image_files(tmp_path: Path) -> None:
93
+ image_a = tmp_path / "chart_a.png"
94
+ image_b = tmp_path / "chart_b.png"
95
+ jsonl = tmp_path / "images.jsonl"
96
+ _write_jsonl(
97
+ jsonl,
98
+ [
99
+ {
100
+ "prompt": "Compare the charts.",
101
+ "image_files": [str(image_a), str(image_b)],
102
+ }
103
+ ],
104
+ )
105
+
106
+ dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
107
+ samples = dataset.sample(
108
+ tokenizer=_Tokenizer(),
109
+ num_requests=1,
110
+ output_len=32,
111
+ )
112
+
113
+ assert len(samples) == 1
114
+ sample = samples[0]
115
+ assert sample.prompt == "Compare the charts."
116
+ assert sample.prompt_len == 3
117
+ assert isinstance(sample.multi_modal_data, list)
118
+ assert [part["image_url"]["url"] for part in sample.multi_modal_data] == [
119
+ f"file://{image_a}",
120
+ f"file://{image_b}",
121
+ ]
122
+
123
+
124
+@pytest.mark.benchmark
125
+def test_custom_image_dataset_preserves_interleaved_content_order(
126
+ tmp_path: Path,
127
+) -> None:
128
+ image_a = tmp_path / "chart_a.png"
129
+ image_b = tmp_path / "chart_b.png"
130
+ jsonl = tmp_path / "images.jsonl"
131
+ _write_jsonl(
132
+ jsonl,
133
+ [
134
+ {
135
+ "content": [
136
+ {"type": "text", "text": "Compare "},
137
+ {"type": "image", "image": str(image_a)},
138
+ {"type": "text", "text": " with "},
139
+ {
140
+ "type": "image_url",
141
+ "image_url": {
142
+ "url": str(image_b),
143
+ "detail": "low",
144
+ },
145
+ },
146
+ ],
147
+ }
148
+ ],
149
+ )
150
+
151
+ dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
152
+ samples = dataset.sample(
153
+ tokenizer=_Tokenizer(),
154
+ num_requests=1,
155
+ output_len=32,
156
+ )
157
+
158
+ assert len(samples) == 1
159
+ sample = samples[0]
160
+ assert sample.multi_modal_data is None
161
+ assert sample.prompt_len == 2
162
+ assert isinstance(sample.prompt, list)
163
+ assert [part["type"] for part in sample.prompt] == [
164
+ "text",
165
+ "image_url",
166
+ "text",
167
+ "image_url",
168
+ ]
169
+ assert sample.prompt[1]["image_url"]["url"] == f"file://{image_a}"
170
+ assert sample.prompt[3]["image_url"] == {
171
+ "url": f"file://{image_b}",
172
+ "detail": "low",
173
+ }
174
+
175
+ request_input = RequestFuncInput(
176
+ prompt=sample.prompt,
177
+ api_url="http://localhost:8000/v1/chat/completions",
178
+ prompt_len=sample.prompt_len,
179
+ output_len=32,
180
+ model="test-model",
181
+ )
182
+ assert _get_chat_content(request_input) == sample.prompt
183
+
184
+
185
+@pytest.mark.benchmark
186
+def test_custom_image_dataset_wraps_interleaved_content_for_multimodal_chat(
187
+ tmp_path: Path,
188
+) -> None:
189
+ image = tmp_path / "chart.png"
190
+ jsonl = tmp_path / "images.jsonl"
191
+ _write_jsonl(
192
+ jsonl,
193
+ [
194
+ {
195
+ "content": [
196
+ {"type": "text", "text": "Describe "},
197
+ {"type": "image", "image": str(image)},
198
+ ],
199
+ }
200
+ ],
201
+ )
202
+
203
+ dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
204
+ samples = dataset.sample(
205
+ tokenizer=_Tokenizer(),
206
+ num_requests=1,
207
+ output_len=32,
208
+ enable_multimodal_chat=True,
209
+ )
210
+
211
+ sample = samples[0]
212
+ assert sample.multi_modal_data is None
213
+ assert sample.prompt == [
214
+ {
215
+ "role": "user",
216
+ "content": [
217
+ {"type": "text", "text": "Describe "},
218
+ {"type": "image_url", "image_url": {"url": f"file://{image}"}},
219
+ ],
220
+ }
221
+ ]
222
+
223
+ request_input = RequestFuncInput(
224
+ prompt=sample.prompt,
225
+ api_url="http://localhost:8000/v1/chat/completions",
226
+ prompt_len=sample.prompt_len,
227
+ output_len=32,
228
+ model="test-model",
229
+ )
230
+ assert _get_chat_messages(request_input) == sample.prompt
231
+
232
+
233
+@pytest.mark.benchmark
234
+def test_custom_image_dataset_rejects_invalid_content_part(
235
+ tmp_path: Path,
236
+) -> None:
237
+ jsonl = tmp_path / "images.jsonl"
238
+ _write_jsonl(jsonl, [{"content": [{"type": "audio", "audio": "clip.wav"}]}])
239
+
240
+ dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
241
+ with pytest.raises(ValueError, match="type 'text', 'image', or 'image_url'"):
242
+ dataset.sample(
243
+ tokenizer=_Tokenizer(),
244
+ num_requests=1,
245
+ output_len=32,
246
+ )
vllm/benchmarks/datasets/datasets.py
+163
-10
@@ -2323,15 +2323,152 @@ class CustomImageDataset(CustomDataset):
2323
"prompt": "Which country has the most pokemons based on the given graphs?",
2324
"image_files": ["path/to/image.png"],
2325
}
2326
+ {
2327
+ "content": [
2328
+ {"type": "text", "text": "Compare these images: "},
2329
+ {"type": "image", "image": "path/to/image1.png"},
2330
+ {"type": "text", "text": " and "},
2331
+ {"type": "image_url", "image_url": {"url": "path/to/image2.png"}},
2332
+ ],
2333
+ }
2334
```
2335
2328
- NOTE: Only the first image file in "image_files" is used for each sample request.
2329
-
2336
This is used to benchmark multimodal LLMs on arbitrary datasets.
2337
"""
2338
2339
IS_MULTIMODAL = True
2340
2341
+ def load_data(self) -> None:
2342
+ if self.dataset_path is None:
2343
+ raise ValueError("dataset_path must be provided for loading data.")
2344
+
2345
+ self.data: list[dict] = []
2346
+
2347
+ if not self.dataset_path.endswith(".jsonl"):
2348
+ raise NotImplementedError(
2349
+ "Only JSONL format is supported for CustomImageDataset."
2350
+ )
2351
+
2352
+ with open(self.dataset_path, encoding="utf-8") as f:
2353
+ for line_number, line in enumerate(f, start=1):
2354
+ line = line.strip()
2355
+ if not line:
2356
+ continue
2357
+
2358
+ try:
2359
+ item = json.loads(line)
2360
+ except json.JSONDecodeError as e:
2361
+ raise ValueError(
2362
+ f"Invalid JSON in custom image dataset line {line_number}: {e}"
2363
+ ) from e
2364
+
2365
+ if not isinstance(item, dict):
2366
+ raise ValueError(
2367
+ "Each custom image dataset line must contain a JSON object. "
2368
+ f"Found {type(item)} on line {line_number}."
2369
+ )
2370
+
2371
+ has_legacy_fields = "prompt" in item and "image_files" in item
2372
+ has_interleaved_content = "content" in item
2373
+ if not has_legacy_fields and not has_interleaved_content:
2374
+ raise ValueError(
2375
+ "Each custom image dataset line must contain either "
2376
+ "'prompt' and 'image_files' fields, or a 'content' field. "
2377
+ f"Invalid line: {line_number}."
2378
+ )
2379
+
2380
+ self.data.append(item)
2381
+
2382
+ random.seed(self.random_seed)
2383
+ if not getattr(self, "disable_shuffle", False):
2384
+ random.shuffle(self.data)
2385
+
2386
+ @staticmethod
2387
+ def _validate_content_parts(content: Any) -> list[dict[str, Any]]:
2388
+ if not isinstance(content, list):
2389
+ raise ValueError(
2390
+ "'content' must be a list of text and image content dictionaries."
2391
+ )
2392
+
2393
+ if not content:
2394
+ raise ValueError("'content' must contain at least one item.")
2395
+
2396
+ parts: list[dict[str, Any]] = []
2397
+ for part in content:
2398
+ if not isinstance(part, dict):
2399
+ raise ValueError(
2400
+ f"Each item in 'content' must be a dictionary. Found {type(part)}."
2401
+ )
2402
+ parts.append(part)
2403
+
2404
+ return parts
2405
+
2406
+ @classmethod
2407
+ def _process_content_part(cls, part: dict[str, Any]) -> dict[str, Any]:
2408
+ content_type = part.get("type")
2409
+ if content_type == "text":
2410
+ text = part.get("text")
2411
+ if not isinstance(text, str):
2412
+ raise ValueError("Text content parts must contain a string 'text'.")
2413
+ return {"type": "text", "text": text}
2414
+
2415
+ if content_type == "image":
2416
+ if "image" not in part:
2417
+ raise ValueError("Image content parts must contain an 'image' field.")
2418
+ return dict(process_image(part["image"]))
2419
+
2420
+ if content_type == "image_url":
2421
+ image_url = part.get("image_url")
2422
+ if isinstance(image_url, str):
2423
+ return dict(process_image(image_url))
2424
+
2425
+ if isinstance(image_url, dict):
2426
+ url = image_url.get("url")
2427
+ if not isinstance(url, str):
2428
+ raise ValueError(
2429
+ "Image URL content parts must contain a string 'image_url.url'."
2430
+ )
2431
+
2432
+ processed_part = dict(process_image(url))
2433
+ processed_image_url = dict(processed_part["image_url"])
2434
+ processed_image_url.update(
2435
+ {key: value for key, value in image_url.items() if key != "url"}
2436
+ )
2437
+ processed_part["image_url"] = processed_image_url
2438
+ return processed_part
2439
+
2440
+ raise ValueError(
2441
+ "Image URL content parts must contain an 'image_url' string "
2442
+ "or dictionary."
2443
+ )
2444
+
2445
+ raise ValueError(
2446
+ "Content parts must have type 'text', 'image', or 'image_url'. "
2447
+ f"Found: {content_type!r}."
2448
+ )
2449
+
2450
+ @classmethod
2451
+ def _process_interleaved_content(cls, content: Any) -> list[dict[str, Any]]:
2452
+ return [
2453
+ cls._process_content_part(part)
2454
+ for part in cls._validate_content_parts(content)
2455
+ ]
2456
+
2457
+ @staticmethod
2458
+ def _get_text_from_content(content: list[dict[str, Any]]) -> str:
2459
+ return "".join(part["text"] for part in content if part.get("type") == "text")
2460
+
2461
+ @staticmethod
2462
+ def _process_image_files(images: Any) -> dict[str, Any] | list[dict[str, Any]]:
2463
+ if not isinstance(images, list) or not images:
2464
+ raise ValueError("'image_files' must be a non-empty list.")
2465
+
2466
+ mm_content = [dict(process_image(image)) for image in images]
2467
+ if len(mm_content) == 1:
2468
+ return mm_content[0]
2469
+
2470
+ return mm_content
2471
+
2472
def sample(
2473
self,
2474
tokenizer: TokenizerLike,
@@ -2356,17 +2493,33 @@ class CustomImageDataset(CustomDataset):
2493
for i, item in enumerate(self.data):
2494
if len(sampled_requests) >= num_requests:
2495
break
2496
+
2497
+ if "content" in item:
2498
+ content = self._process_interleaved_content(item["content"])
2499
+ text_prompt = self._get_text_from_content(content)
2500
+ prompt_len = len(tokenizer(text_prompt).input_ids)
2501
+ prompt = (
2502
+ [{"role": "user", "content": content}]
2503
+ if enable_multimodal_chat
2504
+ else content
2505
+ )
2506
+ sampled_requests.append(
2507
+ SampleRequest(
2508
+ prompt=prompt,
2509
+ prompt_len=prompt_len,
2510
+ expected_output_len=output_len,
2511
+ multi_modal_data=None,
2512
+ request_id=request_id_prefix + str(i),
2513
+ )
2514
+ )
2515
+ continue
2516
+
2517
prompt = item["prompt"]
2518
+ if not isinstance(prompt, str):
2519
+ raise ValueError("'prompt' must be a string.")
2520
2521
prompt_len = len(tokenizer(prompt).input_ids)
2362
- images = item["image_files"]
2363
- if len(images) > 1:
2364
- logger.warning(
2365
- "Multiple image files found for sample %d. "
2366
- "Only the first image will be used.",
2367
- i,
2368
- )
2369
- mm_content = process_image(images[0])
2522
+ mm_content = self._process_image_files(item["image_files"])
2523
if enable_multimodal_chat:
2524
# Note: when chat is enabled the request prompt_len is no longer
2525
# accurate and we will be using request output to count the
vllm/benchmarks/lib/endpoint_request_func.py
+53
-11
@@ -66,7 +66,7 @@ class StreamedResponseHandler:
66
class RequestFuncInput:
67
"""The input for the request function."""
68
69
- prompt: str | list[str]
69
+ prompt: str | list[str] | list[dict[str, Any]]
70
api_url: str
71
prompt_len: int
72
output_len: int
@@ -268,8 +268,6 @@ def _get_chat_content(
268
request_func_input: RequestFuncInput,
269
mm_position: Literal["first", "last"] = "last",
270
) -> list[dict[str, Any]]:
271
- text_contents = [{"type": "text", "text": request_func_input.prompt}]
272
-
271
mm_contents = []
272
if request_func_input.multi_modal_content:
273
mm_content = request_func_input.multi_modal_content
@@ -282,12 +280,60 @@ def _get_chat_content(
280
"multi_modal_content must be a dict or list[dict] for openai-chat"
281
)
282
283
+ prompt = request_func_input.prompt
284
+ if (
285
+ isinstance(prompt, list)
286
+ and prompt
287
+ and all(
288
+ isinstance(item, dict) and isinstance(item.get("type"), str)
289
+ for item in prompt
290
+ )
291
+ ):
292
+ if mm_position == "first":
293
+ return mm_contents + prompt
294
+
295
+ return prompt + mm_contents
296
+
297
+ text_contents = [{"type": "text", "text": prompt}]
298
+
299
if mm_position == "first":
300
return mm_contents + text_contents
301
302
return text_contents + mm_contents
303
304
305
+def _is_chat_messages(prompt: Any) -> bool:
306
+ return (
307
+ isinstance(prompt, list)
308
+ and prompt
309
+ and all(
310
+ isinstance(item, dict)
311
+ and isinstance(item.get("role"), str)
312
+ and isinstance(item.get("content"), (str, list))
313
+ for item in prompt
314
+ )
315
+ )
316
+
317
+
318
+def _get_chat_messages(
319
+ request_func_input: RequestFuncInput,
320
+ mm_position: Literal["first", "last"] = "last",
321
+) -> list[dict[str, Any]]:
322
+ prompt = request_func_input.prompt
323
+ if _is_chat_messages(prompt):
324
+ return prompt
325
+
326
+ return [
327
+ {
328
+ "role": "user",
329
+ "content": _get_chat_content(
330
+ request_func_input,
331
+ mm_position=mm_position,
332
+ ),
333
+ }
334
+ ]
335
+
336
+
337
async def async_request_openai_chat_completions(
338
request_func_input: RequestFuncInput,
339
session: aiohttp.ClientSession,
@@ -297,15 +343,13 @@ async def async_request_openai_chat_completions(
343
api_url = request_func_input.api_url
344
_validate_api_url(api_url, "OpenAI Chat Completions API", "chat/completions")
345
300
- content = _get_chat_content(request_func_input, mm_position=mm_position)
346
+ messages = _get_chat_messages(request_func_input, mm_position=mm_position)
347
348
payload = {
349
"model": request_func_input.model_name
350
if request_func_input.model_name
351
else request_func_input.model,
306
- "messages": [
307
- {"role": "user", "content": content},
308
- ],
352
+ "messages": messages,
353
"max_completion_tokens": request_func_input.output_len,
354
"stream": True,
355
"stream_options": {
@@ -608,15 +652,13 @@ async def async_request_openai_embeddings_chat(
652
api_url = request_func_input.api_url
653
_validate_api_url(api_url, "OpenAI Embeddings API", "embeddings")
654
611
- content = _get_chat_content(request_func_input, mm_position=mm_position)
655
+ messages = _get_chat_messages(request_func_input, mm_position=mm_position)
656
657
payload = {
658
"model": request_func_input.model_name
659
if request_func_input.model_name
660
else request_func_input.model,
617
- "messages": [
618
- {"role": "user", "content": content},
619
- ],
661
+ "messages": messages,
662
# Many embedding models have short context length,
663
# this is to avoid dropping some of the requests.
664
"truncate_prompt_tokens": -1,