| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | r"""Benchmark multimodal processor latency. |
| 4 | |
| 5 | This benchmark measures the latency of the mm processor module |
| 6 | using multimodal prompts from datasets. |
| 7 | MM processor stats are automatically enabled. |
| 8 | |
| 9 | Run: |
| 10 | vllm bench mm-processor \ |
| 11 | --model <your_model> \ |
| 12 | --dataset-name random-mm \ |
| 13 | --num-prompts 10 \ |
| 14 | """ |
| 15 | |
| 16 | import argparse |
| 17 | import json |
| 18 | import time |
| 19 | from collections import defaultdict |
| 20 | from datetime import datetime |
| 21 | from typing import TYPE_CHECKING, Any, Literal |
| 22 | |
| 23 | import numpy as np |
| 24 | |
| 25 | from vllm.benchmarks.datasets import ( |
| 26 | MultiModalConversationDataset, |
| 27 | VisionArenaDataset, |
| 28 | ) |
| 29 | from vllm.benchmarks.throughput import get_requests |
| 30 | from vllm.engine.arg_utils import EngineArgs |
| 31 | from vllm.utils.argparse_utils import FlexibleArgumentParser |
| 32 | from vllm.utils.gc_utils import freeze_gc_heap |
| 33 | from vllm.utils.import_utils import PlaceholderModule |
| 34 | |
| 35 | try: |
| 36 | import pandas as pd |
| 37 | except ImportError: |
| 38 | pd = PlaceholderModule("pandas") |
| 39 | |
| 40 | if TYPE_CHECKING: # Avoid having to mock during docs build |
| 41 | from vllm.v1.engine.llm_engine import LLMEngine |
| 42 | else: |
| 43 | LLMEngine = object |
| 44 | |
| 45 | |
| 46 | def get_timing_stats_from_engine(llm_engine: LLMEngine) -> dict[str, dict[str, float]]: |
| 47 | """Get all multimodal timing stats from the LLM engine. |
| 48 | |
| 49 | Collects both preprocessing stats (HF processor, hashing, cache lookup, |
| 50 | prompt update) and encoder forward pass timing, merged by request_id. |
| 51 | |
| 52 | Args: |
| 53 | llm_engine: The LLM engine (has input_processor and workers). |
| 54 | |
| 55 | Returns: |
| 56 | Dictionary mapping request_id to merged stats dict containing |
| 57 | both preprocessing and encoder timing metrics. |
| 58 | |
| 59 | Example: |
| 60 | { |
| 61 | 'request-123': { |
| 62 | 'get_mm_hashes_secs': 0.02, |
| 63 | 'get_cache_missing_items_secs': 0.01, |
| 64 | 'apply_hf_processor_secs': 0.45, |
| 65 | 'merge_mm_kwargs_secs': 0.01, |
| 66 | 'apply_prompt_updates_secs': 0.03, |
| 67 | 'preprocessor_total_secs': 0.51, |
| 68 | 'encoder_forward_secs': 0.23, |
| 69 | 'num_encoder_calls': 1 |
| 70 | } |
| 71 | } |
| 72 | |
| 73 | """ |
| 74 | observability_config = llm_engine.vllm_config.observability_config |
| 75 | if not observability_config or not observability_config.enable_mm_processor_stats: |
| 76 | return {} |
| 77 | |
| 78 | renderer = llm_engine.renderer |
| 79 | mm_processor_stats = renderer._mm_timing_registry.stat() |
| 80 | |
| 81 | encoder_stats = dict[str, dict[str, float]]() |
| 82 | for worker_stats in llm_engine.collective_rpc("get_encoder_timing_stats"): |
| 83 | if not worker_stats: |
| 84 | continue |
| 85 | |
| 86 | for request_id, stats_dict in worker_stats.items(): |
| 87 | if request_id not in encoder_stats: |
| 88 | encoder_stats[request_id] = dict(stats_dict) |
| 89 | else: |
| 90 | # Aggregate timing metrics across workers |
| 91 | current_time = encoder_stats[request_id].get( |
| 92 | "encoder_forward_secs", 0.0 |
| 93 | ) |
| 94 | new_time = stats_dict.get("encoder_forward_secs", 0.0) |
| 95 | encoder_stats[request_id]["encoder_forward_secs"] = max( |
| 96 | current_time, new_time |
| 97 | ) |
| 98 | |
| 99 | current_calls = encoder_stats[request_id].get("num_encoder_calls", 0) |
| 100 | new_calls = stats_dict.get("num_encoder_calls", 0) |
| 101 | encoder_stats[request_id]["num_encoder_calls"] = max( |
| 102 | current_calls, new_calls |
| 103 | ) |
| 104 | |
| 105 | merged_stats = dict[str, dict[str, float]]() |
| 106 | |
| 107 | for request_id, prep_dict in mm_processor_stats.items(): |
| 108 | merged_stats[request_id] = dict(prep_dict) |
| 109 | |
| 110 | for request_id, enc_dict in encoder_stats.items(): |
| 111 | if request_id in merged_stats: |
| 112 | merged_stats[request_id].update(enc_dict) |
| 113 | continue |
| 114 | |
| 115 | # In V1 engine, the request_id in encoder_stats has a suffix |
| 116 | # appended to the original request_id (which is used in |
| 117 | # preprocessing_stats). |
| 118 | # We try to strip the suffix to find the matching request. |
| 119 | possible_original_id = request_id.rpartition("-")[0] |
| 120 | if possible_original_id and possible_original_id in merged_stats: |
| 121 | merged_stats[possible_original_id].update(enc_dict) |
| 122 | else: |
| 123 | merged_stats[request_id] = dict(enc_dict) |
| 124 | |
| 125 | return merged_stats |
| 126 | |
| 127 | |
| 128 | def collect_mm_processor_stats(llm_engine: LLMEngine) -> dict[str, list[float]]: |
| 129 | """Collect multimodal processor timing stats. |
| 130 | Returns a dictionary mapping stage names to lists of timing values (in seconds). |
| 131 | """ |
| 132 | all_stats = get_timing_stats_from_engine(llm_engine) |
| 133 | |
| 134 | stats_by_stage = defaultdict[str, list[float]](list) |
| 135 | |
| 136 | for stats_dict in all_stats.values(): |
| 137 | for stat_key, stat_val in stats_dict.items(): |
| 138 | stats_by_stage[stat_key].append(stat_val) |
| 139 | |
| 140 | return stats_by_stage |
| 141 | |
| 142 | |
| 143 | def calculate_mm_processor_metrics( |
| 144 | stats_by_stage: dict[str, list[float]], |
| 145 | selected_percentiles: list[float], |
| 146 | *, |
| 147 | unit: Literal["us", "ms", "s"] = "ms", |
| 148 | ) -> dict[str, dict[str, float]]: |
| 149 | """Calculate aggregate metrics from stats by stage.""" |
| 150 | unit2mult = {"us": 1000000, "ms": 1000, "s": 1} |
| 151 | unit_mult = unit2mult[unit] |
| 152 | |
| 153 | metrics = {} |
| 154 | |
| 155 | for stage, times in stats_by_stage.items(): |
| 156 | stage_name = stage.replace("_secs", "_" + unit) |
| 157 | |
| 158 | if not times: |
| 159 | metrics[stage_name] = { |
| 160 | "mean": 0.0, |
| 161 | "median": 0.0, |
| 162 | "std": 0.0, |
| 163 | **{f"p{p}": 0.0 for p in selected_percentiles}, |
| 164 | } |
| 165 | continue |
| 166 | |
| 167 | is_count_metric = stage == "num_encoder_calls" |
| 168 | values = times if is_count_metric else [t * unit_mult for t in times] |
| 169 | |
| 170 | metrics[stage_name] = { |
| 171 | "mean": float(np.mean(values)), |
| 172 | "median": float(np.median(values)), |
| 173 | "std": float(np.std(values)), |
| 174 | **{f"p{p}": float(np.percentile(values, p)) for p in selected_percentiles}, |
| 175 | } |
| 176 | |
| 177 | return metrics |
| 178 | |
| 179 | |
| 180 | def validate_args(args): |
| 181 | """Validate command-line arguments for mm_processor benchmark.""" |
| 182 | if not getattr(args, "tokenizer", None): |
| 183 | args.tokenizer = args.model |
| 184 | if not hasattr(args, "dataset_path"): |
| 185 | args.dataset_path = None |
| 186 | if not hasattr(args, "lora_path"): |
| 187 | args.lora_path = None |
| 188 | if not hasattr(args, "max_loras"): |
| 189 | args.max_loras = None |
| 190 | |
| 191 | if args.dataset_name == "hf" and not args.dataset_path: |
| 192 | raise ValueError( |
| 193 | "--dataset-path is required when using --dataset-name hf. " |
| 194 | "For multimodal benchmarking, specify a dataset like " |
| 195 | "'lmarena-ai/VisionArena-Chat'." |
| 196 | ) |
| 197 | if args.dataset_name == "hf": |
| 198 | supported_mm_datasets = ( |
| 199 | VisionArenaDataset.SUPPORTED_DATASET_PATHS.keys() |
| 200 | | MultiModalConversationDataset.SUPPORTED_DATASET_PATHS |
| 201 | ) |
| 202 | if args.dataset_path not in supported_mm_datasets: |
| 203 | raise ValueError( |
| 204 | f"{args.dataset_path} is not a supported multimodal dataset. " |
| 205 | f"Supported multimodal datasets are: {sorted(supported_mm_datasets)}" |
| 206 | ) |
| 207 | |
| 208 | |
| 209 | def benchmark_multimodal_processor( |
| 210 | args: argparse.Namespace, |
| 211 | ) -> dict[str, Any]: |
| 212 | """Run the multimodal processor benchmark.""" |
| 213 | from vllm import LLM, SamplingParams |
| 214 | |
| 215 | validate_args(args) |
| 216 | |
| 217 | if args.seed is None: |
| 218 | args.seed = 0 |
| 219 | |
| 220 | engine_args = EngineArgs.from_cli_args(args) |
| 221 | llm = LLM.from_engine_args(engine_args) |
| 222 | |
| 223 | tokenizer = llm.get_tokenizer() |
| 224 | requests = get_requests(args, tokenizer) |
| 225 | |
| 226 | assert all( |
| 227 | llm.llm_engine.model_config.max_model_len |
| 228 | >= (request.prompt_len + request.expected_output_len) |
| 229 | for request in requests |
| 230 | ), ( |
| 231 | "Please ensure that max_model_len is greater than the sum of " |
| 232 | "prompt_len and expected_output_len for all requests." |
| 233 | ) |
| 234 | |
| 235 | prompts = [request.prompt for request in requests] |
| 236 | expected_output_lens = [request.expected_output_len for request in requests] |
| 237 | |
| 238 | sampling_params = [ |
| 239 | SamplingParams( |
| 240 | n=1, |
| 241 | temperature=0.0, |
| 242 | max_tokens=output_len, |
| 243 | detokenize=True, |
| 244 | ) |
| 245 | for output_len in expected_output_lens |
| 246 | ] |
| 247 | |
| 248 | selected_percentiles = [ |
| 249 | float(p) for p in getattr(args, "metric_percentiles", "99").split(",") |
| 250 | ] |
| 251 | |
| 252 | freeze_gc_heap() |
| 253 | |
| 254 | num_warmups = getattr(args, "num_warmups", 0) |
| 255 | if num_warmups > 0: |
| 256 | print(f"Processing {num_warmups} warmup requests...") |
| 257 | # Create a temporary args object for warmup requests |
| 258 | warmup_args = argparse.Namespace(**vars(args)) |
| 259 | warmup_args.num_prompts = num_warmups |
| 260 | warmup_args.seed += 1 |
| 261 | warmup_requests = get_requests(warmup_args, tokenizer) |
| 262 | warmup_prompts = [req.prompt for req in warmup_requests] |
| 263 | warmup_output_lens = [req.expected_output_len for req in warmup_requests] |
| 264 | warmup_sampling_params = [ |
| 265 | SamplingParams(max_tokens=output_len) for output_len in warmup_output_lens |
| 266 | ] |
| 267 | llm.chat( |
| 268 | warmup_prompts, |
| 269 | warmup_sampling_params, |
| 270 | use_tqdm=not getattr(args, "disable_tqdm", False), |
| 271 | ) |
| 272 | |
| 273 | # Clear stats from warmup requests |
| 274 | collect_mm_processor_stats(llm.llm_engine) |
| 275 | |
| 276 | print(f"Processing {len(prompts)} requests...") |
| 277 | start_time = time.perf_counter() |
| 278 | |
| 279 | outputs = llm.chat( |
| 280 | prompts, sampling_params, use_tqdm=not getattr(args, "disable_tqdm", False) |
| 281 | ) |
| 282 | |
| 283 | end_time = time.perf_counter() |
| 284 | total_time = end_time - start_time |
| 285 | |
| 286 | mm_stats_by_stage = collect_mm_processor_stats(llm.llm_engine) |
| 287 | |
| 288 | if not any(mm_stats_by_stage.values()): |
| 289 | print( |
| 290 | "\n⚠️ Warning: No MM processor stats found in registry.\n" |
| 291 | " This may indicate that:\n" |
| 292 | " - No multimodal requests were processed\n" |
| 293 | " - Stats were already retrieved (registry is cleared after retrieval)\n" |
| 294 | ) |
| 295 | |
| 296 | mm_processor_metrics = calculate_mm_processor_metrics( |
| 297 | mm_stats_by_stage, selected_percentiles |
| 298 | ) |
| 299 | |
| 300 | completed = len([o for o in outputs if o.finished]) |
| 301 | failed = len(outputs) - completed |
| 302 | |
| 303 | e2el_times = [] |
| 304 | for output in outputs: |
| 305 | if not output.finished or output.metrics is None: |
| 306 | continue |
| 307 | metrics = output.metrics |
| 308 | # Calculate E2E latency as: TTFT + (last_token_ts - first_token_ts) |
| 309 | if ( |
| 310 | getattr(metrics, "first_token_latency", None) is not None |
| 311 | and getattr(metrics, "last_token_ts", None) is not None |
| 312 | and getattr(metrics, "first_token_ts", None) is not None |
| 313 | ): |
| 314 | ttft = metrics.first_token_latency |
| 315 | # Decode time is the duration between the first and last token generation |
| 316 | decode_time = max(0.0, metrics.last_token_ts - metrics.first_token_ts) |
| 317 | e2el_times.append((ttft + decode_time) * 1000) |
| 318 | |
| 319 | if not e2el_times and completed > 0: |
| 320 | print( |
| 321 | "\n⚠️ Warning: Detailed end-to-end latency metrics not available.\n" |
| 322 | " Falling back to average request latency " |
| 323 | "(total_time / num_completed_requests).\n" |
| 324 | ) |
| 325 | avg_time_per_request = total_time / completed |
| 326 | e2el_times = [avg_time_per_request * 1000] * completed |
| 327 | |
| 328 | if e2el_times: |
| 329 | mean_e2el_ms = float(np.mean(e2el_times)) |
| 330 | median_e2el_ms = float(np.median(e2el_times)) |
| 331 | std_e2el_ms = float(np.std(e2el_times)) |
| 332 | percentiles_e2el_ms = [ |
| 333 | (p, float(np.percentile(e2el_times, p))) for p in selected_percentiles |
| 334 | ] |
| 335 | else: |
| 336 | mean_e2el_ms = 0.0 |
| 337 | median_e2el_ms = 0.0 |
| 338 | std_e2el_ms = 0.0 |
| 339 | percentiles_e2el_ms = [(p, 0.0) for p in selected_percentiles] |
| 340 | |
| 341 | encoder_summary = {} |
| 342 | if ( |
| 343 | "num_encoder_calls" in mm_stats_by_stage |
| 344 | and mm_stats_by_stage["num_encoder_calls"] |
| 345 | ): |
| 346 | encoder_calls = mm_stats_by_stage["num_encoder_calls"] |
| 347 | encoder_summary = { |
| 348 | "total_encoder_calls": int(sum(encoder_calls)), |
| 349 | "num_requests_with_encoder_calls": len(encoder_calls), |
| 350 | } |
| 351 | |
| 352 | benchmark_result = { |
| 353 | "completed": completed, |
| 354 | "failed": failed, |
| 355 | "mean_e2el_ms": mean_e2el_ms, |
| 356 | "median_e2el_ms": median_e2el_ms, |
| 357 | "std_e2el_ms": std_e2el_ms, |
| 358 | "percentiles_e2el_ms": percentiles_e2el_ms, |
| 359 | "mm_processor_stats": mm_processor_metrics, |
| 360 | "encoder_summary": encoder_summary, |
| 361 | } |
| 362 | |
| 363 | return benchmark_result |
| 364 | |
| 365 | |
| 366 | def add_cli_args(parser: FlexibleArgumentParser) -> None: |
| 367 | """Add CLI arguments for the multimodal processor benchmark.""" |
| 368 | from vllm.engine.arg_utils import EngineArgs |
| 369 | |
| 370 | EngineArgs.add_cli_args(parser) |
| 371 | |
| 372 | parser.set_defaults(enable_mm_processor_stats=True) |
| 373 | |
| 374 | parser.add_argument( |
| 375 | "--dataset-name", |
| 376 | type=str, |
| 377 | default="random-mm", |
| 378 | choices=["random-mm", "hf"], |
| 379 | help="Name of the dataset to benchmark on. Defaults to 'random-mm'.", |
| 380 | ) |
| 381 | parser.add_argument( |
| 382 | "--num-prompts", |
| 383 | type=int, |
| 384 | default=10, |
| 385 | help="Number of prompts to process.", |
| 386 | ) |
| 387 | parser.add_argument( |
| 388 | "--num-warmups", |
| 389 | type=int, |
| 390 | default=1, |
| 391 | help="Number of warmup prompts to process.", |
| 392 | ) |
| 393 | |
| 394 | from vllm.benchmarks.datasets import ( |
| 395 | add_random_dataset_base_args, |
| 396 | add_random_multimodal_dataset_args, |
| 397 | ) |
| 398 | |
| 399 | add_random_dataset_base_args(parser) |
| 400 | add_random_multimodal_dataset_args(parser) |
| 401 | |
| 402 | # HuggingFace dataset arguments |
| 403 | parser.add_argument( |
| 404 | "--dataset-path", |
| 405 | type=str, |
| 406 | default=None, |
| 407 | help="Path to the dataset file or HuggingFace dataset name " |
| 408 | "(e.g., 'yale-nlp/MMVU', 'lmarena-ai/VisionArena-Chat').", |
| 409 | ) |
| 410 | parser.add_argument( |
| 411 | "--hf-subset", |
| 412 | type=str, |
| 413 | default=None, |
| 414 | help="Subset of the HuggingFace dataset (optional).", |
| 415 | ) |
| 416 | parser.add_argument( |
| 417 | "--hf-split", |
| 418 | type=str, |
| 419 | default=None, |
| 420 | help="Split of the HuggingFace dataset (e.g., 'train', 'test', 'validation').", |
| 421 | ) |
| 422 | parser.add_argument( |
| 423 | "--output-len", |
| 424 | type=int, |
| 425 | default=None, |
| 426 | help="Output length for each request. " |
| 427 | "Overrides the default output lengths from the dataset.", |
| 428 | ) |
| 429 | |
| 430 | parser.add_argument( |
| 431 | "--output-json", |
| 432 | type=str, |
| 433 | default=None, |
| 434 | help="Path to save the benchmark results in JSON format.", |
| 435 | ) |
| 436 | parser.add_argument( |
| 437 | "--metric-percentiles", |
| 438 | type=str, |
| 439 | default="99", |
| 440 | help="Comma-separated list of percentiles to calculate (e.g., '50,90,99').", |
| 441 | ) |
| 442 | parser.add_argument( |
| 443 | "--disable-tqdm", |
| 444 | action="store_true", |
| 445 | help="Disable tqdm progress bar.", |
| 446 | ) |
| 447 | |
| 448 | |
| 449 | def main(args: argparse.Namespace) -> None: |
| 450 | """Main entry point for the multimodal processor benchmark.""" |
| 451 | print("Starting multimodal processor benchmark...") |
| 452 | result = benchmark_multimodal_processor(args) |
| 453 | |
| 454 | print("\n" + "=" * 80) |
| 455 | print("Multimodal Processor Benchmark Results") |
| 456 | print("=" * 80) |
| 457 | |
| 458 | if "mm_processor_stats" in result: |
| 459 | print("\nMM Processor Metrics:") |
| 460 | selected_percentiles = [ |
| 461 | float(p) for p in getattr(args, "metric_percentiles", "99").split(",") |
| 462 | ] |
| 463 | mm_data = [] |
| 464 | for stage, metrics in result["mm_processor_stats"].items(): |
| 465 | row = { |
| 466 | "Stage": stage, |
| 467 | "Mean": f"{metrics['mean']:.2f}", |
| 468 | "Median": f"{metrics['median']:.2f}", |
| 469 | "Std": f"{metrics['std']:.2f}", |
| 470 | } |
| 471 | for p in selected_percentiles: |
| 472 | row[f"P{p}"] = f"{metrics.get(f'p{p}', 0.0):.2f}" |
| 473 | mm_data.append(row) |
| 474 | |
| 475 | mm_df = pd.DataFrame(mm_data) |
| 476 | print(mm_df.to_string(index=False)) |
| 477 | |
| 478 | if "encoder_summary" in result and result["encoder_summary"]: |
| 479 | total_calls = result["encoder_summary"]["total_encoder_calls"] |
| 480 | num_requests = result["encoder_summary"]["num_requests_with_encoder_calls"] |
| 481 | print( |
| 482 | f"\nSummary: {total_calls} total encoder calls " |
| 483 | f"across {num_requests} requests." |
| 484 | ) |
| 485 | |
| 486 | if "mean_e2el_ms" in result: |
| 487 | print("\nEnd-to-End Latency (ms):") |
| 488 | selected_percentiles = [ |
| 489 | float(p) for p in getattr(args, "metric_percentiles", "99").split(",") |
| 490 | ] |
| 491 | |
| 492 | e2el_data = [ |
| 493 | {"Metric": "Mean", "Value (ms)": f"{result['mean_e2el_ms']:.2f}"}, |
| 494 | {"Metric": "Median", "Value (ms)": f"{result['median_e2el_ms']:.2f}"}, |
| 495 | {"Metric": "Std", "Value (ms)": f"{result['std_e2el_ms']:.2f}"}, |
| 496 | ] |
| 497 | |
| 498 | for p in selected_percentiles: |
| 499 | percentile_value = next( |
| 500 | (val for pct, val in result["percentiles_e2el_ms"] if pct == p), |
| 501 | 0.0, |
| 502 | ) |
| 503 | e2el_data.append( |
| 504 | { |
| 505 | "Metric": f"P{p}", |
| 506 | "Value (ms)": f"{percentile_value:.2f}", |
| 507 | } |
| 508 | ) |
| 509 | |
| 510 | e2el_df = pd.DataFrame(e2el_data) |
| 511 | print(e2el_df.to_string(index=False)) |
| 512 | |
| 513 | if args.output_json: |
| 514 | result["config"] = { |
| 515 | "model": args.model, |
| 516 | "num_prompts": args.num_prompts, |
| 517 | "input_len": getattr(args, "random_input_len", None), |
| 518 | "output_len": getattr(args, "random_output_len", None), |
| 519 | } |
| 520 | result["timestamp"] = datetime.now().isoformat() |
| 521 | |
| 522 | with open(args.output_json, "w") as f: |
| 523 | json.dump(result, f, indent=2) |
| 524 | print(f"\nResults saved to {args.output_json}") |
| 525 | |
| 526 | |
| 527 | if __name__ == "__main__": |
| 528 | parser = FlexibleArgumentParser(description="Benchmark mm processor latency") |
| 529 | add_cli_args(parser) |
| 530 | args = parser.parse_args() |
| 531 | main(args) |