A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 16:00 UTC
[Bugfix] Fix E2E latency calculation and add warmup support in mm_processor benchmark (#32646)
Signed-off-by: Hongjian Zhang <zhanghongjian@xiaohongshu.com> Signed-off-by: Xingran Wang <wangxingran123456@outlook.com> Signed-off-by: Hiroken. <105287758+HirokenOvo@users.noreply.github.com> Co-authored-by: Xingran Wang <wangxingran123456@outlook.com>
Hiroken. committed
Jan 24, 2026 at 18:31 UTC
6450b536a63d92d856d368c44c8d4250e31e2ea7
1 file changed
+46
-13
vllm/benchmarks/mm_processor.py
+46
-13
@@ -42,6 +42,7 @@ except ImportError:
42
43
def collect_mm_processor_stats(
44
llm_engine: Any,
45
+ num_warmup_reqs: int = 0,
46
) -> dict[str, list[float]]:
47
"""
48
Collect multimodal processor timing stats.
@@ -60,7 +61,10 @@ def collect_mm_processor_stats(
61
]
62
stats_by_stage = {key: [] for key in stat_keys}
63
63
- for stats_dict in all_stats.values():
64
+ # Skip warmup requests
65
+ stats_list = list(all_stats.values())[num_warmup_reqs:]
66
+
67
+ for stats_dict in stats_list:
68
for key in stat_keys:
69
if key in stats_dict:
70
stats_by_stage[key].append(stats_dict[key])
@@ -178,6 +182,25 @@ def benchmark_multimodal_processor(
182
183
freeze_gc_heap()
184
185
+ num_warmups = getattr(args, "num_warmups", 0)
186
+ if num_warmups > 0:
187
+ print(f"Processing {num_warmups} warmup requests...")
188
+ # Create a temporary args object for warmup requests
189
+ warmup_args = argparse.Namespace(**vars(args))
190
+ warmup_args.num_prompts = num_warmups
191
+ warmup_args.seed += 1
192
+ warmup_requests = get_requests(warmup_args, tokenizer)
193
+ warmup_prompts = [req.prompt for req in warmup_requests]
194
+ warmup_output_lens = [req.expected_output_len for req in warmup_requests]
195
+ warmup_sampling_params = [
196
+ SamplingParams(max_tokens=output_len) for output_len in warmup_output_lens
197
+ ]
198
+ llm.chat(
199
+ warmup_prompts,
200
+ warmup_sampling_params,
201
+ use_tqdm=not getattr(args, "disable_tqdm", False),
202
+ )
203
+
204
print(f"Processing {len(prompts)} requests...")
205
start_time = time.perf_counter()
206
@@ -188,9 +211,7 @@ def benchmark_multimodal_processor(
211
end_time = time.perf_counter()
212
total_time = end_time - start_time
213
191
- mm_stats_by_stage = collect_mm_processor_stats(
192
- llm.llm_engine,
193
- )
214
+ mm_stats_by_stage = collect_mm_processor_stats(llm.llm_engine, num_warmups)
215
216
if not any(mm_stats_by_stage.values()):
217
print(
@@ -212,17 +233,23 @@ def benchmark_multimodal_processor(
233
if not output.finished or output.metrics is None:
234
continue
235
metrics = output.metrics
215
- for attr in ("finished_time", "last_token_time"):
216
- if (
217
- getattr(metrics, attr, None) is not None
218
- and getattr(metrics, "arrival_time", None) is not None
219
- ):
220
- e2el_times.append(
221
- (getattr(metrics, attr) - metrics.arrival_time) * 1000
222
- )
223
- break
236
+ # Calculate E2E latency as: TTFT + (last_token_ts - first_token_ts)
237
+ if (
238
+ getattr(metrics, "first_token_latency", None) is not None
239
+ and getattr(metrics, "last_token_ts", None) is not None
240
+ and getattr(metrics, "first_token_ts", None) is not None
241
+ ):
242
+ ttft = metrics.first_token_latency
243
+ # Decode time is the duration between the first and last token generation
244
+ decode_time = max(0.0, metrics.last_token_ts - metrics.first_token_ts)
245
+ e2el_times.append((ttft + decode_time) * 1000)
246
247
if not e2el_times and completed > 0:
248
+ print(
249
+ "\n⚠️ Warning: Detailed end-to-end latency metrics not available.\n"
250
+ " Falling back to average request latency "
251
+ "(total_time / num_completed_requests).\n"
252
+ )
253
avg_time_per_request = total_time / completed
254
e2el_times = [avg_time_per_request * 1000] * completed
255
@@ -285,6 +312,12 @@ def add_cli_args(parser: argparse.ArgumentParser) -> None:
312
default=10,
313
help="Number of prompts to process.",
314
)
315
+ parser.add_argument(
316
+ "--num-warmups",
317
+ type=int,
318
+ default=1,
319
+ help="Number of warmup prompts to process.",
320
+ )
321
322
from vllm.benchmarks.datasets import (
323
add_random_dataset_base_args,