A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 02:30 UTC
[Misc] support arbitrary MM datasets in spec dec bench (#33486)
Signed-off-by: kkt-cohere <komal@cohere.com> Signed-off-by: Komal Kumar Teru <162363718+kkt-cohere@users.noreply.github.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
Komal Kumar Teru committed
Feb 2, 2026 at 14:19 UTC
ba871fb788f5036b8f55f61ffc7c05e367dacc5a
3 files changed
+156
-18
docs/benchmarking/cli.md
+28
@@ -32,6 +32,7 @@ th {
32
| HuggingFace-Blazedit | ✅ | ✅ | `vdaita/edit_5k_char`, `vdaita/edit_10k_char` |
33
| Spec Bench | ✅ | ✅ | `wget https://raw.githubusercontent.com/hemingkx/Spec-Bench/refs/heads/main/data/spec_bench/question.jsonl` |
34
| Custom | ✅ | ✅ | Local file: `data.jsonl` |
35
+| Custom MM | ✅ | ✅ | Local file: `mm_data.jsonl` |
36
37
Legend:
38
@@ -133,6 +134,33 @@ vllm bench serve --port 9001 --save-result --save-detailed \
134
135
You can skip applying chat template if your data already has it by using `--custom-skip-chat-template`.
136
137
+#### Custom multimodal dataset
138
+
139
+If the multimodal dataset you want to benchmark is not supported yet in vLLM, then you can benchmark on it using `CustomMMDataset`. Your data needs to be in `.jsonl` format and needs to have "prompt" and "image_files" field per entry, e.g., `mm_data.jsonl`:
140
+
141
+```json
142
+{"prompt": "How many animals are present in the given image?", "image_files": ["/path/to/image/folder/horsepony.jpg"]}
143
+{"prompt": "What colour is the bird shown in the image?", "image_files": ["/path/to/image/folder/flycatcher.jpeg"]}
144
+```
145
+
146
+```bash
147
+# need a model with vision capability here
148
+vllm serve Qwen/Qwen2-VL-7B-Instruct
149
+```
150
+
151
+```bash
152
+# run benchmarking script
153
+vllm bench serve--save-result --save-detailed \
154
+ --backend openai-chat \
155
+ --model Qwen/Qwen2-VL-7B-Instruct \
156
+ --endpoint /v1/chat/completions \
157
+ --dataset-name custom_mm \
158
+ --dataset-path <path-to-your-mm-data-jsonl> \
159
+ --allowed-local-media-path /path/to/image/folder
160
+```
161
+
162
+Note that we need to use the `openai-chat` backend and `/v1/chat/completions` endpoint for multimodal inputs.
163
+
164
#### VisionArena Benchmark for Vision Language Models
165
166
```bash
examples/offline_inference/spec_decode.py
+26
-18
@@ -5,7 +5,6 @@ from transformers import AutoTokenizer
5
6
from vllm import LLM, SamplingParams
7
from vllm.benchmarks.datasets import add_dataset_parser, get_samples
8
-from vllm.inputs import TokensPrompt
8
from vllm.v1.metrics.reader import Counter, Vector
9
10
try:
@@ -56,6 +55,7 @@ def parse_args():
55
default="eagle",
56
choices=["ngram", "eagle", "eagle3", "mtp", "draft_model"],
57
)
58
+ parser.add_argument("--backend", type=str, default="openai")
59
parser.add_argument("--num-spec-tokens", type=int, default=2)
60
parser.add_argument("--prompt-lookup-max", type=int, default=5)
61
parser.add_argument("--prompt-lookup-min", type=int, default=2)
@@ -75,12 +75,11 @@ def parse_args():
75
parser.add_argument("--gpu-memory-utilization", type=float, default=0.9)
76
parser.add_argument("--disable-padded-drafter-batch", action="store_true")
77
parser.add_argument("--max-num-seqs", type=int, default=None)
78
+ parser.add_argument("--allowed-local-media-path", type=str, default="")
79
return parser.parse_args()
80
81
82
def main(args):
82
- args.endpoint_type = "openai-chat"
83
-
83
model_dir = args.model_dir
84
if args.model_dir is None:
85
if args.custom_mm_prompts:
@@ -91,19 +90,25 @@ def main(args):
90
)
91
model_dir = "meta-llama/Llama-3.1-8B-Instruct"
92
tokenizer = AutoTokenizer.from_pretrained(model_dir)
94
- args.custom_skip_chat_template = True
93
96
- if not args.custom_mm_prompts:
97
- prompts = get_samples(args, tokenizer)
98
- # add_special_tokens is False to avoid adding bos twice
99
- # when using chat templates
100
- prompt_ids = [
101
- tokenizer.encode(prompt.prompt, add_special_tokens=False)
102
- for prompt in prompts
103
- ]
94
+ if args.custom_mm_prompts:
95
+ prompts = llm_prompts = get_custom_mm_prompts(args.num_prompts)
96
else:
105
- prompts = get_custom_mm_prompts(args.num_prompts)
106
-
97
+ prompts = get_samples(args, tokenizer)
98
+ if args.enable_multimodal_chat:
99
+ llm_prompts = [p.prompt for p in prompts]
100
+ else:
101
+ # add_special_tokens is False to avoid adding bos twice
102
+ # when using chat templates
103
+ llm_prompts = [
104
+ {
105
+ "prompt_token_ids": tokenizer.encode(
106
+ prompt.prompt, add_special_tokens=False
107
+ ),
108
+ "multi_modal_data": prompt.multi_modal_data,
109
+ }
110
+ for prompt in prompts
111
+ ]
112
if args.method == "eagle" or args.method == "eagle3":
113
eagle_dir = args.eagle_dir
114
if args.method == "eagle" and eagle_dir is None:
@@ -154,16 +159,17 @@ def main(args):
159
limit_mm_per_prompt={"image": 5},
160
disable_chunked_mm_input=True,
161
max_num_seqs=args.max_num_seqs,
162
+ allowed_local_media_path=args.allowed_local_media_path,
163
)
164
165
sampling_params = SamplingParams(temperature=args.temp, max_tokens=args.output_len)
160
- if not args.custom_mm_prompts:
166
+ if args.backend == "openai-chat":
167
+ outputs = llm.chat(llm_prompts, sampling_params=sampling_params)
168
+ else:
169
outputs = llm.generate(
162
- [TokensPrompt(prompt_token_ids=x) for x in prompt_ids],
170
+ llm_prompts,
171
sampling_params=sampling_params,
172
)
165
- else:
166
- outputs = llm.chat(prompts, sampling_params=sampling_params)
173
174
# print the generated text
175
if args.print_output:
@@ -219,6 +225,8 @@ def main(args):
225
226
if __name__ == "__main__":
227
args = parse_args()
228
+ args.enable_multimodal_chat = args.backend == "openai-chat"
229
+
230
acceptance_length = main(args)
231
232
if args.test:
vllm/benchmarks/datasets.py
+102
@@ -1335,6 +1335,7 @@ def add_dataset_parser(parser: FlexibleArgumentParser):
1335
"random-rerank",
1336
"hf",
1337
"custom",
1338
+ "custom_mm",
1339
"prefix_repetition",
1340
"spec_bench",
1341
],
@@ -1363,6 +1364,11 @@ def add_dataset_parser(parser: FlexibleArgumentParser):
1364
action="store_true",
1365
help="Skip applying chat template to prompt for datasets that support it.",
1366
)
1367
+ parser.add_argument(
1368
+ "--enable-multimodal-chat",
1369
+ action="store_true",
1370
+ help="Enable multimodal chat transformation for datasets that support it.",
1371
+ )
1372
parser.add_argument(
1373
"--disable-shuffle",
1374
action="store_true",
@@ -1685,6 +1691,19 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
1691
no_oversample=args.no_oversample,
1692
)
1693
1694
+ elif args.dataset_name == "custom_mm":
1695
+ dataset = CustomMMDataset(
1696
+ dataset_path=args.dataset_path, disable_shuffle=args.disable_shuffle
1697
+ )
1698
+ input_requests = dataset.sample(
1699
+ num_requests=args.num_prompts,
1700
+ tokenizer=tokenizer,
1701
+ output_len=args.custom_output_len,
1702
+ enable_multimodal_chat=args.enable_multimodal_chat,
1703
+ request_id_prefix=args.request_id_prefix,
1704
+ no_oversample=args.no_oversample,
1705
+ )
1706
+
1707
elif args.dataset_name == "sonnet":
1708
dataset = SonnetDataset(
1709
dataset_path=args.dataset_path, disable_shuffle=args.disable_shuffle
@@ -1832,6 +1851,7 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
1851
num_requests=args.num_prompts,
1852
tokenizer=tokenizer,
1853
output_len=args.hf_output_len,
1854
+ enable_multimodal_chat=args.enable_multimodal_chat,
1855
request_id_prefix=args.request_id_prefix,
1856
no_oversample=args.no_oversample,
1857
skip_chat_template=args.skip_chat_template,
@@ -1849,6 +1869,7 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
1869
num_requests=args.num_prompts,
1870
tokenizer=tokenizer,
1871
output_len=args.spec_bench_output_len,
1872
+ enable_multimodal_chat=args.enable_multimodal_chat,
1873
request_id_prefix=args.request_id_prefix,
1874
no_oversample=args.no_oversample,
1875
),
@@ -1860,6 +1881,7 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
1881
tokenizer=tokenizer,
1882
num_requests=args.num_prompts,
1883
output_len=args.sharegpt_output_len,
1884
+ enable_multimodal_chat=args.enable_multimodal_chat,
1885
request_id_prefix=args.request_id_prefix,
1886
no_oversample=args.no_oversample,
1887
),
@@ -1903,6 +1925,7 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
1925
limit_mm_per_prompt=args.random_mm_limit_mm_per_prompt,
1926
num_mm_items_range_ratio=args.random_mm_num_mm_items_range_ratio,
1927
bucket_config=args.random_mm_bucket_config,
1928
+ enable_multimodal_chat=args.enable_multimodal_chat,
1929
request_id_prefix=args.request_id_prefix,
1930
no_oversample=args.no_oversample,
1931
),
@@ -2075,6 +2098,85 @@ class CustomDataset(BenchmarkDataset):
2098
return sampled_requests
2099
2100
2101
+class CustomMMDataset(CustomDataset):
2102
+ """
2103
+ Implements the Custom MultiModal dataset. Loads data from a JSONL file and generates
2104
+ sample requests based on conversation turns. E.g.,
2105
+ ```
2106
+ {
2107
+ "prompt": "How many red blocks in the given images?",
2108
+ "image_files": ["path/to/image1.png", "path/to/image2.png"],
2109
+ }
2110
+ {
2111
+ "prompt": "Which country has the most pokemons based on the given graphs?",
2112
+ "image_files": ["path/to/image.png"],
2113
+ }
2114
+ ```
2115
+
2116
+ NOTE: Only the first image file in "image_files" is used for each sample request.
2117
+
2118
+ This is used to benchmark multimodal LLMs on arbitrary datasets.
2119
+ """
2120
+
2121
+ IS_MULTIMODAL = True
2122
+
2123
+ def sample(
2124
+ self,
2125
+ tokenizer: TokenizerLike,
2126
+ num_requests: int,
2127
+ output_len: int | None = None,
2128
+ enable_multimodal_chat: bool = False,
2129
+ request_id_prefix: str = "",
2130
+ no_oversample: bool = False,
2131
+ **kwargs,
2132
+ ) -> list:
2133
+ # load all data if needed
2134
+ self.num_available_samples = len(self.data)
2135
+ if num_requests <= 0:
2136
+ num_requests = self.num_available_samples
2137
+ logger.info(
2138
+ "num_requests is set to 0 or negative, "
2139
+ "so using all available samples: %d",
2140
+ num_requests,
2141
+ )
2142
+
2143
+ sampled_requests = []
2144
+ for i, item in enumerate(self.data):
2145
+ if len(sampled_requests) >= num_requests:
2146
+ break
2147
+ prompt = item["prompt"]
2148
+
2149
+ prompt_len = len(tokenizer(prompt).input_ids)
2150
+ images = item["image_files"]
2151
+ if len(images) > 1:
2152
+ logger.warning(
2153
+ "Multiple image files found for sample %d. "
2154
+ "Only the first image will be used.",
2155
+ i,
2156
+ )
2157
+ mm_content = process_image(images[0])
2158
+ if enable_multimodal_chat:
2159
+ # Note: when chat is enabled the request prompt_len is no longer
2160
+ # accurate and we will be using request output to count the
2161
+ # actual prompt len
2162
+ prompt = self.apply_multimodal_chat_transformation(prompt, mm_content)
2163
+
2164
+ sampled_requests.append(
2165
+ SampleRequest(
2166
+ prompt=prompt,
2167
+ prompt_len=prompt_len,
2168
+ expected_output_len=output_len,
2169
+ multi_modal_data=mm_content,
2170
+ request_id=request_id_prefix + str(i),
2171
+ )
2172
+ )
2173
+ self.maybe_oversample_requests(
2174
+ sampled_requests, num_requests, request_id_prefix, no_oversample
2175
+ )
2176
+
2177
+ return sampled_requests
2178
+
2179
+
2180
# -----------------------------------------------------------------------------
2181
# Spec Bench Dataset Implementation
2182
# -----------------------------------------------------------------------------