A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Fix(benchmarks): allow multiple mm contents in OpenAI Chat Completion Benchmarks (#22534)
Signed-off-by: breno.skuk <breno.skuk@hcompany.ai>
Breno Baldas Skuk committed
Aug 10, 2025 at 18:03 UTC
65a7917be480c1b0e45f12bfad31eb4b25539db9
6 files changed
+49
-10
benchmarks/backend_request_func.py
+14
-3
@@ -31,7 +31,7 @@ class RequestFuncInput:
31
model_name: Optional[str] = None
32
logprobs: Optional[int] = None
33
extra_body: Optional[dict] = None
34
- multi_modal_content: Optional[dict] = None
34
+ multi_modal_content: Optional[dict | list[dict]] = None
35
ignore_eos: bool = False
36
language: Optional[str] = None
37
@@ -364,7 +364,15 @@ async def async_request_openai_chat_completions(
364
) as session:
365
content = [{"type": "text", "text": request_func_input.prompt}]
366
if request_func_input.multi_modal_content:
367
- content.append(request_func_input.multi_modal_content)
367
+ mm_content = request_func_input.multi_modal_content
368
+ if isinstance(mm_content, list):
369
+ content.extend(mm_content)
370
+ elif isinstance(mm_content, dict):
371
+ content.append(mm_content)
372
+ else:
373
+ raise TypeError(
374
+ "multi_modal_content must be a dict or list[dict] for openai-chat"
375
+ )
376
payload = {
377
"model": request_func_input.model_name
378
if request_func_input.model_name
@@ -491,7 +499,10 @@ async def async_request_openai_audio(
499
buffer.seek(0)
500
return buffer
501
494
- with to_bytes(*request_func_input.multi_modal_content["audio"]) as f:
502
+ mm_audio = request_func_input.multi_modal_content
503
+ if not isinstance(mm_audio, dict) or "audio" not in mm_audio:
504
+ raise TypeError("multi_modal_content must be a dict containing 'audio'")
505
+ with to_bytes(*mm_audio["audio"]) as f:
506
form = aiohttp.FormData()
507
form.add_field("file", f, content_type="audio/wav")
508
for key, value in payload.items():
benchmarks/benchmark_dataset.py
+1
-1
@@ -52,7 +52,7 @@ class SampleRequest:
52
prompt: Union[str, Any]
53
prompt_len: int
54
expected_output_len: int
55
- multi_modal_data: Optional[Union[MultiModalDataDict, dict]] = None
55
+ multi_modal_data: Optional[Union[MultiModalDataDict, dict, list[dict]]] = None
56
lora_request: Optional[LoRARequest] = None
57
58
benchmarks/benchmark_serving.py
+8
-1
@@ -263,7 +263,14 @@ async def benchmark(
263
input_requests[0].multi_modal_data,
264
)
265
266
- assert test_mm_content is None or isinstance(test_mm_content, dict)
266
+ assert (
267
+ test_mm_content is None
268
+ or isinstance(test_mm_content, dict)
269
+ or (
270
+ isinstance(test_mm_content, list)
271
+ and all(isinstance(item, dict) for item in test_mm_content)
272
+ )
273
+ ), "multi_modal_data must be a dict or list[dict]"
274
test_input = RequestFuncInput(
275
model=model_id,
276
model_name=model_name,
vllm/benchmarks/datasets.py
+3
-1
@@ -71,7 +71,9 @@ class SampleRequest:
71
prompt: Union[str, Any]
72
prompt_len: int
73
expected_output_len: int
74
- multi_modal_data: Optional[Union[MultiModalDataDict, dict]] = None
74
+ multi_modal_data: Optional[
75
+ Union[MultiModalDataDict, dict, list[dict]]
76
+ ] = None
77
lora_request: Optional[LoRARequest] = None
78
79
vllm/benchmarks/lib/endpoint_request_func.py
+15
-3
@@ -28,7 +28,7 @@ class RequestFuncInput:
28
model_name: Optional[str] = None
29
logprobs: Optional[int] = None
30
extra_body: Optional[dict] = None
31
- multi_modal_content: Optional[dict] = None
31
+ multi_modal_content: Optional[dict | list[dict]] = None
32
ignore_eos: bool = False
33
language: Optional[str] = None
34
@@ -172,7 +172,16 @@ async def async_request_openai_chat_completions(
172
173
content = [{"type": "text", "text": request_func_input.prompt}]
174
if request_func_input.multi_modal_content:
175
- content.append(request_func_input.multi_modal_content)
175
+ mm_content = request_func_input.multi_modal_content
176
+ if isinstance(mm_content, list):
177
+ content.extend(mm_content)
178
+ elif isinstance(mm_content, dict):
179
+ content.append(mm_content)
180
+ else:
181
+ raise TypeError(
182
+ "multi_modal_content must be a dict or list[dict] "
183
+ "for openai-chat"
184
+ )
185
payload = {
186
"model":
187
request_func_input.model_name
@@ -310,7 +319,10 @@ async def async_request_openai_audio(
319
buffer.seek(0)
320
return buffer
321
313
- with to_bytes(*request_func_input.multi_modal_content["audio"]) as f:
322
+ mm_audio = request_func_input.multi_modal_content
323
+ if not isinstance(mm_audio, dict) or "audio" not in mm_audio:
324
+ raise TypeError("multi_modal_content must be a dict containing 'audio'")
325
+ with to_bytes(*mm_audio["audio"]) as f:
326
form = aiohttp.FormData()
327
form.add_field("file", f, content_type="audio/wav")
328
for key, value in payload.items():
vllm/benchmarks/serve.py
+8
-1
@@ -365,7 +365,14 @@ async def benchmark(
365
input_requests[0].multi_modal_data,
366
)
367
368
- assert test_mm_content is None or isinstance(test_mm_content, dict)
368
+ assert (
369
+ test_mm_content is None
370
+ or isinstance(test_mm_content, dict)
371
+ or (
372
+ isinstance(test_mm_content, list)
373
+ and all(isinstance(item, dict) for item in test_mm_content)
374
+ )
375
+ ), "multi_modal_data must be a dict or list[dict]"
376
test_input = RequestFuncInput(
377
model=model_id,
378
model_name=model_name,