A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Benchmarks] Add video inputs to ShareGPTDataset. (#23199)
Signed-off-by: Chenheli Hua <huachenheli@outlook.com>
Chenheli Hua committed
Aug 19, 2025 at 16:42 UTC
1630cc8d0f5e0ff19d4c5736a4b531dd27a3f4d8
3 files changed
+113
-6
benchmarks/README.md
+38
-3
@@ -32,6 +32,14 @@ become available.
32
<div>Note that the images need to be downloaded separately. For example, to download COCO's 2017 Train images:</div>
33
<code>wget http://images.cocodataset.org/zips/train2017.zip</code>
34
</td>
35
+ </tr>
36
+ <tr>
37
+ <td><strong>ShareGPT4Video (Video)</strong></td>
38
+ <td style="text-align: center;">✅</td>
39
+ <td style="text-align: center;">✅</td>
40
+ <td>
41
+ <code>git clone https://huggingface.co/datasets/ShareGPT4Video/ShareGPT4Video</code>
42
+ </td>
43
</tr>
44
<tr>
45
<td><strong>BurstGPT</strong></td>
@@ -231,7 +239,7 @@ vllm serve Qwen/Qwen2-VL-7B-Instruct
239
```bash
240
vllm bench serve \
241
--backend openai-chat \
234
- --endpoint-type openai-chat \
242
+ --endpoint-type openai-chat \
243
--model Qwen/Qwen2-VL-7B-Instruct \
244
--endpoint /v1/chat/completions \
245
--dataset-name hf \
@@ -246,7 +254,7 @@ vllm bench serve \
254
```bash
255
vllm bench serve \
256
--backend openai-chat \
249
- --endpoint-type openai-chat \
257
+ --endpoint-type openai-chat \
258
--model Qwen/Qwen2-VL-7B-Instruct \
259
--endpoint /v1/chat/completions \
260
--dataset-name hf \
@@ -612,7 +620,7 @@ vllm bench serve \
620
--prefix-repetition-prefix-len 512 \
621
--prefix-repetition-suffix-len 128 \
622
--prefix-repetition-num-prefixes 5 \
615
- --prefix-repetition-output-len 128
623
+ --prefix-repetition-output-len 128
624
```
625
626
</details>
@@ -687,4 +695,31 @@ python benchmarks/benchmark_serving.py \
695
--endpoint /v1/chat/completion
696
```
697
698
+### Videos (ShareGPT4Video)
699
+
700
+Start vLLM:
701
+
702
+```bash
703
+python -m vllm.entrypoints.openai.api_server \
704
+ --model Qwen/Qwen2.5-VL-7B-Instruct \
705
+ --dtype bfloat16 \
706
+ --limit-mm-per-prompt '{"video": 1}' \
707
+ --allowed-local-media-path /path/to/sharegpt4video/videos
708
+```
709
+
710
+Send requests with videos:
711
+
712
+```bash
713
+python benchmarks/benchmark_serving.py \
714
+ --backend openai-chat \
715
+ --model Qwen/Qwen2.5-VL-7B-Instruct \
716
+ --dataset-name sharegpt \
717
+ --dataset-path /path/to/ShareGPT4Video/llava_v1_5_mix665k_with_video_chatgpt72k_share4video28k.json \
718
+ --num-prompts 100 \
719
+ --save-result \
720
+ --result-dir ~/vllm_benchmark_results \
721
+ --save-detailed \
722
+ --endpoint /v1/chat/completion
723
+```
724
+
725
</details>
benchmarks/benchmark_dataset.py
+37
-1
@@ -293,6 +293,41 @@ def process_image(image: Any) -> Mapping[str, Any]:
293
)
294
295
296
+def process_video(video: Any) -> Mapping[str, Any]:
297
+ """
298
+ Process a single video input and return a multimedia content dictionary.
299
+
300
+ Supports the following input types:
301
+
302
+ 1. Dictionary with raw video bytes: - Expects a dict with a 'bytes' key
303
+ containing raw video data.
304
+
305
+ 2. String input: - Treats the string as a URL or local file path. -
306
+ Prepends "file://" if the string doesn't start with "http://" or
307
+ "file://". - Returns a dictionary with the image URL.
308
+
309
+ Raises:
310
+ ValueError: If the input is not a supported type.
311
+ """
312
+ if isinstance(video, dict) and "bytes" in video:
313
+ video_bytes = video["bytes"]
314
+ video_base64 = base64.b64encode(video_bytes).decode("utf-8")
315
+ return {
316
+ "type": "video_url",
317
+ "video_url": {"url": f"data:video/mp4;base64,{video_base64}"},
318
+ }
319
+
320
+ if isinstance(video, str):
321
+ video_url = (
322
+ video if video.startswith(("http://", "file://")) else f"file://{video}"
323
+ )
324
+ return {"type": "video_url", "video_url": {"url": video_url}}
325
+
326
+ raise ValueError(
327
+ f"Invalid video input {video}. Must be a string of local path/remote url, or a dictionary with raw video bytes in the form of `{{'bytes': raw_video_bytes}}`." # noqa: E501
328
+ )
329
+
330
+
331
# -----------------------------------------------------------------------------
332
# Random Dataset Implementation (Synthetic Data)
333
# -----------------------------------------------------------------------------
@@ -451,9 +486,10 @@ class ShareGPTDataset(BenchmarkDataset):
486
skip_min_output_len_check=output_len is not None,
487
):
488
continue
454
- # TODO: Also support ShareGPT4Video.
489
if image_path := entry.get("image"):
490
mm_content = process_image(image_path)
491
+ elif video_path := entry.get("video"):
492
+ mm_content = process_video(video_path)
493
else:
494
mm_content = None
495
if enable_multimodal_chat:
vllm/benchmarks/datasets.py
+38
-2
@@ -281,7 +281,7 @@ def process_image(image: Any) -> Mapping[str, Any]:
281
"""
282
Process a single image input and return a multimedia content dictionary.
283
284
- Supports three input types:
284
+ Supports the following input types:
285
286
1. Dictionary with raw image bytes: - Expects a dict with a 'bytes' key
287
containing raw image data. - Loads the bytes as a PIL.Image.Image.
@@ -321,6 +321,41 @@ def process_image(image: Any) -> Mapping[str, Any]:
321
" or str or dictionary with raw image bytes.")
322
323
324
+def process_video(video: Any) -> Mapping[str, Any]:
325
+ """
326
+ Process a single video input and return a multimedia content dictionary.
327
+
328
+ Supports the following input types:
329
+
330
+ 1. Dictionary with raw video bytes: - Expects a dict with a 'bytes' key
331
+ containing raw video data.
332
+
333
+ 2. String input: - Treats the string as a URL or local file path. -
334
+ Prepends "file://" if the string doesn't start with "http://" or
335
+ "file://". - Returns a dictionary with the image URL.
336
+
337
+ Raises:
338
+ ValueError: If the input is not a supported type.
339
+ """
340
+ if isinstance(video, dict) and 'bytes' in video:
341
+ video_bytes = video['bytes']
342
+ video_base64 = base64.b64encode(video_bytes).decode("utf-8")
343
+ return {
344
+ "type": "video_url",
345
+ "video_url": {
346
+ "url": f"data:video/mp4;base64,{video_base64}"
347
+ },
348
+ }
349
+
350
+ if isinstance(video, str):
351
+ video_url = (video if video.startswith(
352
+ ("http://", "file://")) else f"file://{video}")
353
+ return {"type": "video_url", "video_url": {"url": video_url}}
354
+
355
+ raise ValueError(
356
+ f"Invalid video input {video}. Must be a string of local path/remote url, or a dictionary with raw video bytes in the form of `{{'bytes': raw_video_bytes}}`." # noqa: E501
357
+ )
358
+
359
# -----------------------------------------------------------------------------
360
# Random Dataset Implementation (Synthetic Data)
361
# -----------------------------------------------------------------------------
@@ -474,9 +509,10 @@ class ShareGPTDataset(BenchmarkDataset):
509
skip_min_output_len_check=output_len
510
is not None):
511
continue
477
- # TODO: Also support ShareGPT4Video.
512
if image_path := entry.get("image"):
513
mm_content = process_image(image_path)
514
+ elif video_path := entry.get("video"):
515
+ mm_content = process_video(video_path)
516
else:
517
mm_content = None
518
if enable_multimodal_chat: