A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Benchmarks] Include image data when ShareGPT4V dataset is used. (#22955)
Signed-off-by: Chenheli Hua <huachenheli@outlook.com>
Chenheli Hua committed
Aug 15, 2025 at 11:23 UTC
993d3d122b114cf93bf423fe0b4410ac493d9c45
3 files changed
+63
-2
benchmarks/README.md
+49
@@ -22,6 +22,17 @@ become available.
22
<td style="text-align: center;">✅</td>
23
<td><code>wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json</code></td>
24
</tr>
25
+ <tr>
26
+ <td><strong>ShareGPT4V (Image)</strong></td>
27
+ <td style="text-align: center;">✅</td>
28
+ <td style="text-align: center;">✅</td>
29
+ <td>
30
+ <code>wget https://huggingface.co/datasets/Lin-Chen/ShareGPT4V/blob/main/sharegpt4v_instruct_gpt4-vision_cap100k.json</code>
31
+ <br>
32
+ <div>Note that the images need to be downloaded separately. For example, to download COCO's 2017 Train images:</div>
33
+ <code>wget http://images.cocodataset.org/zips/train2017.zip</code>
34
+ </td>
35
+ </tr>
36
<tr>
37
<td><strong>BurstGPT</strong></td>
38
<td style="text-align: center;">✅</td>
@@ -616,3 +627,41 @@ python3 benchmarks/benchmark_prioritization.py \
627
```
628
629
</details>
630
+
631
+## 👁️ Example - Multi-Modal Benchmark
632
+
633
+<details>
634
+<summary>Show more</summary>
635
+
636
+<br/>
637
+
638
+Benchmark the performance of multi-modal requests in vLLM.
639
+
640
+### Images (ShareGPT4V)
641
+
642
+Start vLLM:
643
+
644
+```bash
645
+python -m vllm.entrypoints.openai.api_server \
646
+ --model Qwen/Qwen2.5-VL-7B-Instruct \
647
+ --dtype bfloat16 \
648
+ --limit-mm-per-prompt '{"image": 1}' \
649
+ --allowed-local-media-path /path/to/sharegpt4v/images
650
+```
651
+
652
+Send requests with images:
653
+
654
+```bash
655
+python benchmarks/benchmark_serving.py \
656
+ --backend openai-chat \
657
+ --model Qwen/Qwen2.5-VL-7B-Instruct \
658
+ --dataset-name sharegpt \
659
+ --dataset-path /path/to/ShareGPT4V/sharegpt4v_instruct_gpt4-vision_cap100k.json \
660
+ --num-prompts 100 \
661
+ --save-result \
662
+ --result-dir ~/vllm_benchmark_results \
663
+ --save-detailed \
664
+ --endpoint /v1/chat/completion
665
+```
666
+
667
+</details>
benchmarks/benchmark_dataset.py
+7
-1
@@ -430,14 +430,20 @@ class ShareGPTDataset(BenchmarkDataset):
430
skip_min_output_len_check=output_len is not None,
431
):
432
continue
433
+ # TODO: Also support ShareGPT4Video.
434
+ if image_path := entry.get("image"):
435
+ mm_content = process_image(image_path)
436
+ else:
437
+ mm_content = None
438
if enable_multimodal_chat:
434
- prompt = self.apply_multimodal_chat_transformation(prompt, None)
439
+ prompt = self.apply_multimodal_chat_transformation(prompt, mm_content)
440
samples.append(
441
SampleRequest(
442
prompt=prompt,
443
prompt_len=prompt_len,
444
expected_output_len=new_output_len,
445
lora_request=lora_request,
446
+ multi_modal_data=mm_content,
447
)
448
)
449
self.maybe_oversample_requests(samples, num_requests)
vllm/benchmarks/datasets.py
+7
-1
@@ -454,15 +454,21 @@ class ShareGPTDataset(BenchmarkDataset):
454
skip_min_output_len_check=output_len
455
is not None):
456
continue
457
+ # TODO: Also support ShareGPT4Video.
458
+ if image_path := entry.get("image"):
459
+ mm_content = process_image(image_path)
460
+ else:
461
+ mm_content = None
462
if enable_multimodal_chat:
463
prompt = self.apply_multimodal_chat_transformation(
459
- prompt, None)
464
+ prompt, mm_content)
465
samples.append(
466
SampleRequest(
467
prompt=prompt,
468
prompt_len=prompt_len,
469
expected_output_len=new_output_len,
470
lora_request=lora_request,
471
+ multi_modal_data=mm_content,
472
))
473
self.maybe_oversample_requests(samples, num_requests)
474
return samples