A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Incorrect MM data format in `vllm bench throughput` (#26395)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Oct 8, 2025 at 13:52 UTC
0d4f48fa103e8407ae94eb87fd87d2380fbb1eb7
1 file changed
+8
-8
vllm/benchmarks/throughput.py
+8
-8
@@ -201,16 +201,16 @@ async def run_vllm_async(
201
sampling_params: list[SamplingParams] = []
202
lora_requests: list[Optional[LoRARequest]] = []
203
for request in requests:
204
- prompts.append(
205
- TokensPrompt(
206
- prompt_token_ids=request.prompt["prompt_token_ids"],
207
- multi_modal_data=request.multi_modal_data,
208
- )
204
+ prompt = (
205
+ TokensPrompt(prompt_token_ids=request.prompt["prompt_token_ids"])
206
if "prompt_token_ids" in request.prompt
210
- else TextPrompt(
211
- prompt=request.prompt, multi_modal_data=request.multi_modal_data
212
- )
207
+ else TextPrompt(prompt=request.prompt)
208
)
209
+
210
+ if request.multi_modal_data:
211
+ assert isinstance(request.multi_modal_data, dict)
212
+ prompt["multi_modal_data"] = request.multi_modal_data
213
+
214
sampling_params.append(
215
SamplingParams(
216
n=n,