A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Incorrect another MM data format in vllm bench throughput (#26462)
Signed-off-by: Huy Do <huydhn@gmail.com>
Huy Do committed
Oct 8, 2025 at 22:58 UTC
8bd696fa535dcacbe8b84fdde21c96af370893e5
1 file changed
+8
-8
vllm/benchmarks/throughput.py
+8
-8
@@ -59,16 +59,16 @@ def run_vllm(
59
prompts: list[Union[TextPrompt, TokensPrompt]] = []
60
sampling_params: list[SamplingParams] = []
61
for request in requests:
62
- prompts.append(
63
- TokensPrompt(
64
- prompt_token_ids=request.prompt["prompt_token_ids"],
65
- multi_modal_data=request.multi_modal_data,
66
- )
62
+ prompt = (
63
+ TokensPrompt(prompt_token_ids=request.prompt["prompt_token_ids"])
64
if "prompt_token_ids" in request.prompt
68
- else TextPrompt(
69
- prompt=request.prompt, multi_modal_data=request.multi_modal_data
70
- )
65
+ else TextPrompt(prompt=request.prompt)
66
)
67
+ if request.multi_modal_data:
68
+ assert isinstance(request.multi_modal_data, dict)
69
+ prompt["multi_modal_data"] = request.multi_modal_data
70
+ prompts.append(prompt)
71
+
72
sampling_params.append(
73
SamplingParams(
74
n=n,