A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 04:45 UTC
[Bugfix] Fix `vllm bench serve` to count multimodal tokens in "total input tokens" (#38654)
Signed-off-by: Matthias Gehre <matthias.gehre@amd.com>
Matthias Gehre committed
Apr 14, 2026 at 13:00 UTC
1e4748c66a156af3ea7b72a9dbe3fb686c8750eb
2 files changed
+5
-1
vllm/benchmarks/lib/endpoint_request_func.py
+4
@@ -237,6 +237,8 @@ async def async_request_openai_completions(
237
generated_text += text or ""
238
elif usage := data.get("usage"):
239
output.output_tokens = usage.get("completion_tokens")
240
+ if (pt := usage.get("prompt_tokens")) is not None:
241
+ output.prompt_len = pt
242
if first_chunk_received:
243
output.success = True
244
else:
@@ -358,6 +360,8 @@ async def async_request_openai_chat_completions(
360
generated_text += content or ""
361
elif usage := data.get("usage"):
362
output.output_tokens = usage.get("completion_tokens")
363
+ if (pt := usage.get("prompt_tokens")) is not None:
364
+ output.prompt_len = pt
365
366
most_recent_timestamp = timestamp
367
vllm/benchmarks/serve.py
+1
-1
@@ -439,7 +439,7 @@ def calculate_metrics(
439
).input_ids
440
)
441
actual_output_lens.append(output_len)
442
- total_input += input_requests[i].prompt_len
442
+ total_input += outputs[i].prompt_len
443
tpot = 0
444
if output_len > 1:
445
latency_minus_ttft = outputs[i].latency - outputs[i].ttft