@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1f5b7c41c3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 10, 20:15 UTC

Add Multimodal Processor Benchmark (#29105)

Signed-off-by: Reagan Lee <reaganjlee@gmail.com> Signed-off-by: Reagan <reaganjlee@gmail.com>

Reagan Lee committed Jan 1, 2026 at 19:26 UTC 1f5b7c41c3e3ed97152d0ae28091141280fff20f
13 files changed +968 -155
docs/cli/bench/mm_processor.md new
+9
@@ -0,0 +1,9 @@
1 +# vllm bench mm-processor
2 +
3 +## JSON CLI Arguments
4 +
5 +--8<-- "docs/cli/json_tip.inc.md"
6 +
7 +## Arguments
8 +
9 +--8<-- "docs/argparse/bench_mm_processor.inc.md"
docs/mkdocs/hooks/generate_argparse.py
+2
@@ -92,6 +92,7 @@ def auto_mock(module_name: str, attr: str, max_mocks: int = 100):
92
93
94 bench_latency = auto_mock("vllm.benchmarks", "latency")
95 +bench_mm_processor = auto_mock("vllm.benchmarks", "mm_processor")
96 bench_serve = auto_mock("vllm.benchmarks", "serve")
97 bench_sweep_plot = auto_mock("vllm.benchmarks.sweep.plot", "SweepPlotArgs")
98 bench_sweep_plot_pareto = auto_mock(
@@ -222,6 +223,7 @@ def on_startup(command: Literal["build", "gh-deploy", "serve"], dirty: bool):
223 "run-batch": create_parser(openai_run_batch.make_arg_parser),
224 # Benchmark CLI
225 "bench_latency": create_parser(bench_latency.add_cli_args),
226 + "bench_mm_processor": create_parser(bench_mm_processor.add_cli_args),
227 "bench_serve": create_parser(bench_serve.add_cli_args),
228 "bench_sweep_plot": create_parser(bench_sweep_plot.add_cli_args),
229 "bench_sweep_plot_pareto": create_parser(bench_sweep_plot_pareto.add_cli_args),
vllm/benchmarks/datasets.py
+100 -71
@@ -1437,19 +1437,97 @@ def add_dataset_parser(parser: FlexibleArgumentParser):
1437 )
1438
1439 random_group = parser.add_argument_group("random dataset options")
1440 - random_group.add_argument(
1440 + add_random_dataset_base_args(random_group)
1441 +
1442 + random_mm_group = parser.add_argument_group(
1443 + "random multimodal dataset options extended from random dataset"
1444 + )
1445 + add_random_multimodal_dataset_args(random_mm_group)
1446 +
1447 + hf_group = parser.add_argument_group("hf dataset options")
1448 + hf_group.add_argument(
1449 + "--hf-subset", type=str, default=None, help="Subset of the HF dataset."
1450 + )
1451 + hf_group.add_argument(
1452 + "--hf-split", type=str, default=None, help="Split of the HF dataset."
1453 + )
1454 + hf_group.add_argument(
1455 + "--hf-name",
1456 + type=str,
1457 + default=None,
1458 + help=(
1459 + "Name of the dataset on HuggingFace "
1460 + "(e.g., 'lmarena-ai/VisionArena-Chat'). "
1461 + "Specify this if your dataset-path is a local path."
1462 + ),
1463 + )
1464 + hf_group.add_argument(
1465 + "--hf-output-len",
1466 + type=int,
1467 + default=None,
1468 + help="Output length for each request. Overrides the output lengths "
1469 + "from the sampled HF dataset.",
1470 + )
1471 +
1472 + prefix_repetition_group = parser.add_argument_group(
1473 + "prefix repetition dataset options"
1474 + )
1475 + prefix_repetition_group.add_argument(
1476 + "--prefix-repetition-prefix-len",
1477 + type=int,
1478 + default=256,
1479 + help="Number of prefix tokens per request, used only for prefix "
1480 + "repetition dataset.",
1481 + )
1482 + prefix_repetition_group.add_argument(
1483 + "--prefix-repetition-suffix-len",
1484 + type=int,
1485 + default=256,
1486 + help="Number of suffix tokens per request, used only for prefix "
1487 + "repetition dataset. Total input length is prefix_len + suffix_len.",
1488 + )
1489 + prefix_repetition_group.add_argument(
1490 + "--prefix-repetition-num-prefixes",
1491 + type=int,
1492 + default=10,
1493 + help="Number of prefixes to generate, used only for prefix repetition "
1494 + "dataset. Prompts per prefix is num_requests // num_prefixes.",
1495 + )
1496 + prefix_repetition_group.add_argument(
1497 + "--prefix-repetition-output-len",
1498 + type=int,
1499 + default=128,
1500 + help="Number of output tokens per request, used only for prefix "
1501 + "repetition dataset.",
1502 + )
1503 +
1504 +
1505 +def add_random_dataset_base_args(
1506 + parser_or_group: FlexibleArgumentParser | argparse._ArgumentGroup,
1507 +) -> None:
1508 + """Add CLI arguments for base random dataset options.
1509 +
1510 + This function adds arguments needed for:
1511 + - random (random dataset)
1512 + - random-mm (random multimodal dataset)
1513 + - random-rerank (random dataset for reranking)
1514 +
1515 + Args:
1516 + parser_or_group: Either a parser or an argument group to add arguments to.
1517 + """
1518 + parser_or_group.add_argument(
1519 "--random-input-len",
1520 type=int,
1521 default=1024,
1522 help="Number of input tokens per request, used only for random sampling.",
1523 )
1446 - random_group.add_argument(
1524 + parser_or_group.add_argument(
1525 "--random-output-len",
1526 type=int,
1527 default=128,
1528 help="Number of output tokens per request, used only for random sampling.",
1529 )
1452 - random_group.add_argument(
1530 + parser_or_group.add_argument(
1531 "--random-range-ratio",
1532 type=float,
1533 default=0.0,
@@ -1458,7 +1536,7 @@ def add_dataset_parser(parser: FlexibleArgumentParser):
1536 "a symmetric sampling range"
1537 "[length * (1 - range_ratio), length * (1 + range_ratio)].",
1538 )
1461 - random_group.add_argument(
1539 + parser_or_group.add_argument(
1540 "--random-prefix-len",
1541 type=int,
1542 default=0,
@@ -1471,13 +1549,13 @@ def add_dataset_parser(parser: FlexibleArgumentParser):
1549 "input_len * (1 + range_ratio)]."
1550 ),
1551 )
1474 - random_group.add_argument(
1552 + parser_or_group.add_argument(
1553 "--random-batch-size",
1554 type=int,
1555 default=1,
1556 help=("Batch size for random sampling. Only used for embeddings benchmark."),
1557 )
1480 - random_group.add_argument(
1558 + parser_or_group.add_argument(
1559 "--no-reranker",
1560 action="store_true",
1561 help=(
@@ -1486,11 +1564,19 @@ def add_dataset_parser(parser: FlexibleArgumentParser):
1564 ),
1565 )
1566
1489 - # random multimodal dataset options
1490 - random_mm_group = parser.add_argument_group(
1491 - "random multimodal dataset options extended from random dataset"
1492 - )
1493 - random_mm_group.add_argument(
1567 +
1568 +def add_random_multimodal_dataset_args(
1569 + parser_or_group: FlexibleArgumentParser | argparse._ArgumentGroup,
1570 +) -> None:
1571 + """Add CLI arguments for random multimodal dataset options.
1572 +
1573 + This function adds arguments needed for:
1574 + - random-mm (random multimodal dataset)
1575 +
1576 + Args:
1577 + parser_or_group: Either a parser or an argument group to add arguments to.
1578 + """
1579 + parser_or_group.add_argument(
1580 "--random-mm-base-items-per-request",
1581 type=int,
1582 default=RandomMultiModalDataset.DEFAULT_BASE_ITEMS_PER_REQUEST,
@@ -1500,7 +1586,7 @@ def add_dataset_parser(parser: FlexibleArgumentParser):
1586 "--random-mm-num-mm-items-range-ratio."
1587 ),
1588 )
1503 - random_mm_group.add_argument(
1589 + parser_or_group.add_argument(
1590 "--random-mm-num-mm-items-range-ratio",
1591 type=float,
1592 default=RandomMultiModalDataset.DEFAULT_NUM_MM_ITEMS_RANGE_RATIO,
@@ -1515,7 +1601,7 @@ def add_dataset_parser(parser: FlexibleArgumentParser):
1601 "An error is raised if the computed min exceeds the max."
1602 ),
1603 )
1518 - random_mm_group.add_argument(
1604 + parser_or_group.add_argument(
1605 "--random-mm-limit-mm-per-prompt",
1606 type=json.loads,
1607 default=RandomMultiModalDataset.DEFAULT_LIMIT_MM_PER_PROMPT,
@@ -1559,7 +1645,7 @@ def add_dataset_parser(parser: FlexibleArgumentParser):
1645 return normalize(parsed)
1646 raise ValueError("Unsupported value for --random-mm-bucket-config.")
1647
1562 - random_mm_group.add_argument(
1648 + parser_or_group.add_argument(
1649 "--random-mm-bucket-config",
1650 type=_parse_mm_bucket_config,
1651 default=RandomMultiModalDataset.DEFAULT_MM_ITEM_BUCKET_CONFIG,
@@ -1580,63 +1666,6 @@ def add_dataset_parser(parser: FlexibleArgumentParser):
1666 ),
1667 )
1668
1583 - hf_group = parser.add_argument_group("hf dataset options")
1584 - hf_group.add_argument(
1585 - "--hf-subset", type=str, default=None, help="Subset of the HF dataset."
1586 - )
1587 - hf_group.add_argument(
1588 - "--hf-split", type=str, default=None, help="Split of the HF dataset."
1589 - )
1590 - hf_group.add_argument(
1591 - "--hf-name",
1592 - type=str,
1593 - default=None,
1594 - help=(
1595 - "Name of the dataset on HuggingFace "
1596 - "(e.g., 'lmarena-ai/VisionArena-Chat'). "
1597 - "Specify this if your dataset-path is a local path."
1598 - ),
1599 - )
1600 - hf_group.add_argument(
1601 - "--hf-output-len",
1602 - type=int,
1603 - default=None,
1604 - help="Output length for each request. Overrides the output lengths "
1605 - "from the sampled HF dataset.",
1606 - )
1607 -
1608 - prefix_repetition_group = parser.add_argument_group(
1609 - "prefix repetition dataset options"
1610 - )
1611 - prefix_repetition_group.add_argument(
1612 - "--prefix-repetition-prefix-len",
1613 - type=int,
1614 - default=256,
1615 - help="Number of prefix tokens per request, used only for prefix "
1616 - "repetition dataset.",
1617 - )
1618 - prefix_repetition_group.add_argument(
1619 - "--prefix-repetition-suffix-len",
1620 - type=int,
1621 - default=256,
1622 - help="Number of suffix tokens per request, used only for prefix "
1623 - "repetition dataset. Total input length is prefix_len + suffix_len.",
1624 - )
1625 - prefix_repetition_group.add_argument(
1626 - "--prefix-repetition-num-prefixes",
1627 - type=int,
1628 - default=10,
1629 - help="Number of prefixes to generate, used only for prefix repetition "
1630 - "dataset. Prompts per prefix is num_requests // num_prefixes.",
1631 - )
1632 - prefix_repetition_group.add_argument(
1633 - "--prefix-repetition-output-len",
1634 - type=int,
1635 - default=128,
1636 - help="Number of output tokens per request, used only for prefix "
1637 - "repetition dataset.",
1638 - )
1639 -
1669
1670 def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
1671 if not hasattr(args, "request_id_prefix"):
vllm/benchmarks/mm_processor.py new
+363
@@ -0,0 +1,363 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +r"""Benchmark multimodal processor latency.
4 +
5 +This benchmark measures the latency of the mm processor module
6 +using multimodal prompts from datasets.
7 +MM processor stats are automatically enabled.
8 +
9 +Run:
10 + vllm bench mm-processor \
11 + --model <your_model> \
12 + --dataset-name random-mm \
13 + --num-prompts 10 \
14 +"""
15 +
16 +import argparse
17 +import dataclasses
18 +import json
19 +import time
20 +from datetime import datetime
21 +from typing import Any
22 +
23 +import numpy as np
24 +
25 +from vllm.benchmarks.throughput import get_requests
26 +from vllm.engine.arg_utils import EngineArgs
27 +from vllm.multimodal.processing import (
28 + get_timing_stats_from_engine_client,
29 +)
30 +from vllm.utils.gc_utils import freeze_gc_heap
31 +from vllm.utils.import_utils import PlaceholderModule
32 +
33 +try:
34 + import pandas as pd
35 +except ImportError:
36 + pd = PlaceholderModule("pandas")
37 +
38 +
39 +def collect_mm_processor_stats(
40 + llm_engine: Any,
41 +) -> dict[str, list[float]]:
42 + """
43 + Collect multimodal processor timing stats.
44 + Returns a dictionary mapping stage names to lists of timing values (in seconds).
45 + """
46 + all_stats = get_timing_stats_from_engine_client(llm_engine)
47 +
48 + stats_by_stage = {
49 + "hf_processor_time": [],
50 + "hashing_time": [],
51 + "cache_lookup_time": [],
52 + "prompt_update_time": [],
53 + "total_time": [],
54 + }
55 +
56 + for stats_dict in all_stats.values():
57 + stats_by_stage["hf_processor_time"].append(
58 + stats_dict.get("hf_processor_time", 0.0)
59 + )
60 + stats_by_stage["hashing_time"].append(stats_dict.get("hashing_time", 0.0))
61 + stats_by_stage["cache_lookup_time"].append(
62 + stats_dict.get("cache_lookup_time", 0.0)
63 + )
64 + stats_by_stage["prompt_update_time"].append(
65 + stats_dict.get("prompt_update_time", 0.0)
66 + )
67 + stats_by_stage["total_time"].append(stats_dict.get("total_time", 0.0))
68 +
69 + return stats_by_stage
70 +
71 +
72 +def calculate_mm_processor_metrics(
73 + stats_by_stage: dict[str, list[float]],
74 + selected_percentiles: list[float],
75 +) -> dict[str, dict[str, float]]:
76 + """
77 + Calculate aggregate metrics from stats by stage.
78 + """
79 + metrics = {}
80 +
81 + for stage_name, times in stats_by_stage.items():
82 + if not times:
83 + metrics[stage_name] = {
84 + "mean": 0.0,
85 + "median": 0.0,
86 + "std": 0.0,
87 + **{f"p{p}": 0.0 for p in selected_percentiles},
88 + }
89 + continue
90 +
91 + times_ms = [t * 1000 for t in times]
92 + metrics[stage_name] = {
93 + "mean": float(np.mean(times_ms)),
94 + "median": float(np.median(times_ms)),
95 + "std": float(np.std(times_ms)),
96 + **{
97 + f"p{p}": float(np.percentile(times_ms, p)) for p in selected_percentiles
98 + },
99 + }
100 +
101 + return metrics
102 +
103 +
104 +def validate_args(args):
105 + """
106 + Validate command-line arguments for mm_processor benchmark.
107 + """
108 + if not getattr(args, "tokenizer", None):
109 + args.tokenizer = args.model
110 + if not hasattr(args, "dataset_path"):
111 + args.dataset_path = None
112 + if not hasattr(args, "lora_path"):
113 + args.lora_path = None
114 + if not hasattr(args, "max_loras"):
115 + args.max_loras = None
116 +
117 +
118 +def benchmark_multimodal_processor(
119 + args: argparse.Namespace,
120 +) -> dict[str, Any]:
121 + """
122 + Run the multimodal processor benchmark.
123 + """
124 + from vllm import LLM, SamplingParams
125 +
126 + validate_args(args)
127 +
128 + if args.seed is None:
129 + args.seed = 0
130 +
131 + engine_args = EngineArgs.from_cli_args(args)
132 + llm = LLM(**dataclasses.asdict(engine_args))
133 +
134 + tokenizer = llm.get_tokenizer()
135 + requests = get_requests(args, tokenizer)
136 +
137 + assert all(
138 + llm.llm_engine.model_config.max_model_len
139 + >= (request.prompt_len + request.expected_output_len)
140 + for request in requests
141 + ), (
142 + "Please ensure that max_model_len is greater than the sum of "
143 + "prompt_len and expected_output_len for all requests."
144 + )
145 +
146 + prompts = [request.prompt for request in requests]
147 + expected_output_lens = [request.expected_output_len for request in requests]
148 +
149 + sampling_params = [
150 + SamplingParams(
151 + n=1,
152 + temperature=0.0,
153 + max_tokens=output_len,
154 + detokenize=True,
155 + )
156 + for output_len in expected_output_lens
157 + ]
158 +
159 + selected_percentiles = [
160 + float(p) for p in getattr(args, "metric_percentiles", "99").split(",")
161 + ]
162 +
163 + freeze_gc_heap()
164 +
165 + print(f"Processing {len(prompts)} requests...")
166 + start_time = time.perf_counter()
167 +
168 + outputs = llm.chat(
169 + prompts, sampling_params, use_tqdm=not getattr(args, "disable_tqdm", False)
170 + )
171 +
172 + end_time = time.perf_counter()
173 + total_time = end_time - start_time
174 +
175 + mm_stats_by_stage = collect_mm_processor_stats(
176 + llm.llm_engine,
177 + )
178 +
179 + if not any(mm_stats_by_stage.values()):
180 + print(
181 + "\n⚠️ Warning: No MM processor stats found in registry.\n"
182 + " This may indicate that:\n"
183 + " - No multimodal requests were processed\n"
184 + " - Stats were already retrieved (registry is cleared after retrieval)\n"
185 + )
186 +
187 + mm_processor_metrics = calculate_mm_processor_metrics(
188 + mm_stats_by_stage, selected_percentiles
189 + )
190 +
191 + completed = len([o for o in outputs if o.finished])
192 + failed = len(outputs) - completed
193 +
194 + e2el_times = []
195 + for output in outputs:
196 + if not output.finished or output.metrics is None:
197 + continue
198 + metrics = output.metrics
199 + for attr in ("finished_time", "last_token_time"):
200 + if (
201 + getattr(metrics, attr, None) is not None
202 + and getattr(metrics, "arrival_time", None) is not None
203 + ):
204 + e2el_times.append(
205 + (getattr(metrics, attr) - metrics.arrival_time) * 1000
206 + )
207 + break
208 +
209 + if not e2el_times and completed > 0:
210 + avg_time_per_request = total_time / completed
211 + e2el_times = [avg_time_per_request * 1000] * completed
212 +
213 + if e2el_times:
214 + mean_e2el_ms = float(np.mean(e2el_times))
215 + median_e2el_ms = float(np.median(e2el_times))
216 + std_e2el_ms = float(np.std(e2el_times))
217 + percentiles_e2el_ms = [
218 + (p, float(np.percentile(e2el_times, p))) for p in selected_percentiles
219 + ]
220 + else:
221 + mean_e2el_ms = 0.0
222 + median_e2el_ms = 0.0
223 + std_e2el_ms = 0.0
224 + percentiles_e2el_ms = [(p, 0.0) for p in selected_percentiles]
225 +
226 + benchmark_result = {
227 + "completed": completed,
228 + "failed": failed,
229 + "mean_e2el_ms": mean_e2el_ms,
230 + "median_e2el_ms": median_e2el_ms,
231 + "std_e2el_ms": std_e2el_ms,
232 + "percentiles_e2el_ms": percentiles_e2el_ms,
233 + "mm_processor_stats": mm_processor_metrics,
234 + }
235 +
236 + return benchmark_result
237 +
238 +
239 +def add_cli_args(parser: argparse.ArgumentParser) -> None:
240 + """Add CLI arguments for the multimodal processor benchmark."""
241 + from vllm.engine.arg_utils import EngineArgs
242 +
243 + EngineArgs.add_cli_args(parser)
244 +
245 + parser.set_defaults(enable_mm_processor_stats=True)
246 +
247 + parser.add_argument(
248 + "--dataset-name",
249 + type=str,
250 + default="random-mm",
251 + choices=["random-mm", "random-rerank"],
252 + help="Name of the dataset to benchmark on. Defaults to 'random-mm'.",
253 + )
254 + parser.add_argument(
255 + "--num-prompts",
256 + type=int,
257 + default=10,
258 + help="Number of prompts to process.",
259 + )
260 +
261 + from vllm.benchmarks.datasets import (
262 + add_random_dataset_base_args,
263 + add_random_multimodal_dataset_args,
264 + )
265 +
266 + add_random_dataset_base_args(parser)
267 + add_random_multimodal_dataset_args(parser)
268 +
269 + parser.add_argument(
270 + "--output-json",
271 + type=str,
272 + default=None,
273 + help="Path to save the benchmark results in JSON format.",
274 + )
275 + parser.add_argument(
276 + "--metric-percentiles",
277 + type=str,
278 + default="99",
279 + help="Comma-separated list of percentiles to calculate (e.g., '50,90,99').",
280 + )
281 + parser.add_argument(
282 + "--disable-tqdm",
283 + action="store_true",
284 + help="Disable tqdm progress bar.",
285 + )
286 +
287 +
288 +def main(args: argparse.Namespace) -> None:
289 + """Main entry point for the multimodal processor benchmark."""
290 +
291 + print("Starting multimodal processor benchmark...")
292 + result = benchmark_multimodal_processor(args)
293 +
294 + print("\n" + "=" * 80)
295 + print("Multimodal Processor Benchmark Results")
296 + print("=" * 80)
297 +
298 + if "mm_processor_stats" in result:
299 + print("\nMM Processor Timing (ms):")
300 + selected_percentiles = [
301 + float(p) for p in getattr(args, "metric_percentiles", "99").split(",")
302 + ]
303 + mm_data = []
304 + for stage, metrics in result["mm_processor_stats"].items():
305 + row = {
306 + "Stage": stage,
307 + "Mean": f"{metrics['mean']:.2f}",
308 + "Median": f"{metrics['median']:.2f}",
309 + "Std": f"{metrics['std']:.2f}",
310 + }
311 + for p in selected_percentiles:
312 + row[f"P{p}"] = f"{metrics.get(f'p{p}', 0.0):.2f}"
313 + mm_data.append(row)
314 +
315 + mm_df = pd.DataFrame(mm_data)
316 + print(mm_df.to_string(index=False))
317 +
318 + if "mean_e2el_ms" in result:
319 + print("\nEnd-to-End Latency (ms):")
320 + selected_percentiles = [
321 + float(p) for p in getattr(args, "metric_percentiles", "99").split(",")
322 + ]
323 +
324 + e2el_data = [
325 + {"Metric": "Mean", "Value (ms)": f"{result['mean_e2el_ms']:.2f}"},
326 + {"Metric": "Median", "Value (ms)": f"{result['median_e2el_ms']:.2f}"},
327 + {"Metric": "Std", "Value (ms)": f"{result['std_e2el_ms']:.2f}"},
328 + ]
329 +
330 + for p in selected_percentiles:
331 + percentile_value = next(
332 + (val for pct, val in result["percentiles_e2el_ms"] if pct == p),
333 + 0.0,
334 + )
335 + e2el_data.append(
336 + {
337 + "Metric": f"P{p}",
338 + "Value (ms)": f"{percentile_value:.2f}",
339 + }
340 + )
341 +
342 + e2el_df = pd.DataFrame(e2el_data)
343 + print(e2el_df.to_string(index=False))
344 +
345 + if args.output_json:
346 + result["config"] = {
347 + "model": args.model,
348 + "num_prompts": args.num_prompts,
349 + "input_len": getattr(args, "random_input_len", None),
350 + "output_len": getattr(args, "random_output_len", None),
351 + }
352 + result["timestamp"] = datetime.now().isoformat()
353 +
354 + with open(args.output_json, "w") as f:
355 + json.dump(result, f, indent=2)
356 + print(f"\nResults saved to {args.output_json}")
357 +
358 +
359 +if __name__ == "__main__":
360 + parser = argparse.ArgumentParser(description="Benchmark mm processor latency")
361 + add_cli_args(parser)
362 + args = parser.parse_args()
363 + main(args)
vllm/benchmarks/throughput.py
+167 -32
@@ -24,10 +24,14 @@ from vllm.benchmarks.datasets import (
24 MultiModalConversationDataset,
25 PrefixRepetitionRandomDataset,
26 RandomDataset,
27 + RandomDatasetForReranking,
28 + RandomMultiModalDataset,
29 SampleRequest,
30 ShareGPTDataset,
31 SonnetDataset,
32 VisionArenaDataset,
33 + add_random_dataset_base_args,
34 + add_random_multimodal_dataset_args,
35 )
36 from vllm.benchmarks.lib.utils import convert_to_pytorch_benchmark_format, write_to_json
37 from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
@@ -342,8 +346,6 @@ def get_requests(args, tokenizer):
346 "lora_path": args.lora_path,
347 "max_loras": args.max_loras,
348 "num_requests": args.num_prompts,
345 - "input_len": args.input_len,
346 - "output_len": args.output_len,
349 }
350
351 if args.dataset_name == "random" or (
@@ -351,12 +353,26 @@ def get_requests(args, tokenizer):
353 and args.dataset_name not in {"prefix_repetition", "random-mm", "random-rerank"}
354 ):
355 sample_kwargs["range_ratio"] = args.random_range_ratio
354 - sample_kwargs["prefix_len"] = args.prefix_len
356 + # prefer random_* arguments, fall back to regular arguments
357 + random_prefix_len = getattr(args, "random_prefix_len", None)
358 + sample_kwargs["prefix_len"] = (
359 + random_prefix_len if random_prefix_len is not None else args.prefix_len
360 + )
361 + random_input_len = getattr(args, "random_input_len", None)
362 + sample_kwargs["input_len"] = (
363 + random_input_len if random_input_len is not None else args.input_len
364 + )
365 + random_output_len = getattr(args, "random_output_len", None)
366 + sample_kwargs["output_len"] = (
367 + random_output_len if random_output_len is not None else args.output_len
368 + )
369 dataset_cls = RandomDataset
370 elif args.dataset_name == "sharegpt":
371 dataset_cls = ShareGPTDataset
372 if args.backend == "vllm-chat":
373 sample_kwargs["enable_multimodal_chat"] = True
374 + if args.output_len is not None:
375 + sample_kwargs["output_len"] = args.output_len
376 elif args.dataset_name == "sonnet":
377 assert tokenizer.chat_template or tokenizer.default_chat_template, (
378 "Tokenizer/model must have chat template for sonnet dataset."
@@ -364,9 +380,15 @@ def get_requests(args, tokenizer):
380 dataset_cls = SonnetDataset
381 sample_kwargs["prefix_len"] = args.prefix_len
382 sample_kwargs["return_prompt_formatted"] = True
383 + if args.input_len is not None:
384 + sample_kwargs["input_len"] = args.input_len
385 + if args.output_len is not None:
386 + sample_kwargs["output_len"] = args.output_len
387 elif args.dataset_name == "burstgpt":
388 dataset_cls = BurstGPTDataset
389 elif args.dataset_name == "hf":
390 + if args.output_len is not None:
391 + sample_kwargs["output_len"] = args.output_len
392 if args.dataset_path in VisionArenaDataset.SUPPORTED_DATASET_PATHS:
393 dataset_cls = VisionArenaDataset
394 common_kwargs["dataset_subset"] = None
@@ -395,6 +417,56 @@ def get_requests(args, tokenizer):
417 sample_kwargs["suffix_len"] = args.prefix_repetition_suffix_len
418 sample_kwargs["num_prefixes"] = args.prefix_repetition_num_prefixes
419 sample_kwargs["output_len"] = args.prefix_repetition_output_len
420 + elif args.dataset_name == "random-mm":
421 + dataset_cls = RandomMultiModalDataset
422 + # prefer random_* arguments, fall back to regular arguments
423 + random_input_len = getattr(args, "random_input_len", None)
424 + sample_kwargs["input_len"] = (
425 + random_input_len
426 + if random_input_len is not None
427 + else getattr(args, "input_len", None)
428 + )
429 + random_output_len = getattr(args, "random_output_len", None)
430 + sample_kwargs["output_len"] = (
431 + random_output_len
432 + if random_output_len is not None
433 + else getattr(args, "output_len", None)
434 + )
435 + sample_kwargs["base_items_per_request"] = getattr(
436 + args, "random_mm_base_items_per_request", None
437 + )
438 + sample_kwargs["num_mm_items_range_ratio"] = getattr(
439 + args, "random_mm_num_mm_items_range_ratio", None
440 + )
441 + sample_kwargs["limit_mm_per_prompt"] = getattr(
442 + args, "random_mm_limit_mm_per_prompt", None
443 + )
444 + sample_kwargs["bucket_config"] = getattr(args, "random_mm_bucket_config", None)
445 + sample_kwargs["enable_multimodal_chat"] = True
446 + random_prefix_len = getattr(args, "random_prefix_len", None)
447 + prefix_len = getattr(args, "prefix_len", None)
448 + sample_kwargs["prefix_len"] = (
449 + random_prefix_len if random_prefix_len is not None else prefix_len
450 + )
451 + sample_kwargs["range_ratio"] = args.random_range_ratio
452 + elif args.dataset_name == "random-rerank":
453 + dataset_cls = RandomDatasetForReranking
454 + # prefer random_* arguments, fall back to regular arguments
455 + random_input_len = getattr(args, "random_input_len", None)
456 + sample_kwargs["input_len"] = (
457 + random_input_len
458 + if random_input_len is not None
459 + else getattr(args, "input_len", None)
460 + )
461 + random_output_len = getattr(args, "random_output_len", None)
462 + sample_kwargs["output_len"] = (
463 + random_output_len
464 + if random_output_len is not None
465 + else getattr(args, "output_len", None)
466 + )
467 + sample_kwargs["batchsize"] = getattr(args, "random_batch_size", 1)
468 + sample_kwargs["is_reranker"] = not getattr(args, "no_reranker", False)
469 + sample_kwargs["range_ratio"] = args.random_range_ratio
470 else:
471 raise ValueError(f"Unknown dataset name: {args.dataset_name}")
472 # Remove None values
@@ -451,8 +523,12 @@ def validate_args(args):
523 ):
524 print("When dataset path is not set, it will default to random dataset")
525 args.dataset_name = "random"
454 - if args.input_len is None:
455 - raise ValueError("input_len must be provided for a random dataset")
526 + random_input_len = getattr(args, "random_input_len", None)
527 + if args.input_len is None and random_input_len is None:
528 + raise ValueError(
529 + "Either --input-len or --random-input-len must be provided "
530 + "for a random dataset"
531 + )
532
533 # === Dataset Name Specific Checks ===
534 # --hf-subset and --hf-split: only used
@@ -485,26 +561,79 @@ def validate_args(args):
561 else:
562 raise ValueError(f"{args.dataset_path} is not supported by hf dataset.")
563
488 - # --random-range-ratio: only used when dataset_name is 'random'
489 - if args.dataset_name != "random" and args.random_range_ratio is not None:
564 + # --random-range-ratio: only used when dataset_name is 'random',
565 + # 'random-mm', or 'random-rerank'
566 + if (
567 + args.dataset_name not in {"random", "random-mm", "random-rerank"}
568 + and args.random_range_ratio is not None
569 + ):
570 warnings.warn(
571 "--random-range-ratio will be ignored since \
492 - --dataset-name is not 'random'.",
572 + --dataset-name is not 'random', 'random-mm', or 'random-rerank'.",
573 + stacklevel=2,
574 + )
575 +
576 + # --random-batch-size: only used when dataset_name is 'random-rerank'
577 + if (
578 + args.dataset_name != "random-rerank"
579 + and getattr(args, "random_batch_size", None) is not None
580 + ) and args.random_batch_size != 1:
581 + warnings.warn(
582 + "--random-batch-size will be ignored since \
583 + --dataset-name is not 'random-rerank'.",
584 + stacklevel=2,
585 + )
586 +
587 + # --no-reranker: only used when dataset_name is 'random-rerank'
588 + if args.dataset_name != "random-rerank" and getattr(args, "no_reranker", False):
589 + warnings.warn(
590 + "--no-reranker will be ignored since \
591 + --dataset-name is not 'random-rerank'.",
592 stacklevel=2,
593 )
594
496 - # --prefix-len: only used when dataset_name is 'random', 'sonnet', or not
497 - # set.
595 + # --prefix-len: only used when dataset_name is 'random', 'random-mm',
596 + # 'sonnet', or not set.
597 if (
499 - args.dataset_name not in {"random", "sonnet", None}
598 + args.dataset_name not in {"random", "random-mm", "sonnet", None}
599 and args.prefix_len is not None
600 ):
601 warnings.warn(
602 "--prefix-len will be ignored since --dataset-name\
504 - is not 'random', 'sonnet', or not set.",
603 + is not 'random', 'random-mm', 'sonnet', or not set.",
604 stacklevel=2,
605 )
606
607 + # === Random Dataset Argument Conflict Detection ===
608 + # Check for conflicts between regular and random arguments when using
609 + # random datasets
610 + if args.dataset_name in {"random", "random-mm", "random-rerank"}:
611 + random_input_len = getattr(args, "random_input_len", None)
612 + random_output_len = getattr(args, "random_output_len", None)
613 + random_prefix_len = getattr(args, "random_prefix_len", None)
614 +
615 + if args.input_len is not None and random_input_len is not None:
616 + warnings.warn(
617 + "Both --input-len and --random-input-len are specified. "
618 + "The random version (--random-input-len) will be preferred "
619 + "in this run.",
620 + stacklevel=2,
621 + )
622 + if args.output_len is not None and random_output_len is not None:
623 + warnings.warn(
624 + "Both --output-len and --random-output-len are specified. "
625 + "The random version (--random-output-len) will be preferred "
626 + "in this run.",
627 + stacklevel=2,
628 + )
629 + if args.prefix_len is not None and random_prefix_len is not None:
630 + warnings.warn(
631 + "Both --prefix-len and --random-prefix-len are specified. "
632 + "The random version (--random-prefix-len) will be preferred "
633 + "in this run.",
634 + stacklevel=2,
635 + )
636 +
637 # === LoRA Settings ===
638 if getattr(args, "enable_lora", False) and args.backend != "vllm":
639 raise ValueError("LoRA benchmarking is only supported for vLLM backend")
@@ -554,7 +683,16 @@ def add_cli_args(parser: argparse.ArgumentParser):
683 parser.add_argument(
684 "--dataset-name",
685 type=str,
557 - choices=["sharegpt", "random", "sonnet", "burstgpt", "hf", "prefix_repetition"],
686 + choices=[
687 + "sharegpt",
688 + "random",
689 + "sonnet",
690 + "burstgpt",
691 + "hf",
692 + "prefix_repetition",
693 + "random-mm",
694 + "random-rerank",
695 + ],
696 help="Name of the dataset to benchmark on.",
697 default="sharegpt",
698 )
@@ -636,23 +774,19 @@ def add_cli_args(parser: argparse.ArgumentParser):
774 help="Number of fixed prefix tokens before the random "
775 "context in a request (default: 0).",
776 )
639 - # random dataset
640 - parser.add_argument(
641 - "--random-range-ratio",
642 - type=float,
643 - default=0.0,
644 - help="Range ratio for sampling input/output length, "
645 - "used only for RandomDataset. Must be in the range [0, 1) to define "
646 - "a symmetric sampling range "
647 - "[length * (1 - range_ratio), length * (1 + range_ratio)].",
648 - )
777
778 # hf dtaset
779 parser.add_argument(
652 - "--hf-subset", type=str, default=None, help="Subset of the HF dataset."
780 + "--hf-subset",
781 + type=str,
782 + default=None,
783 + help="Subset of the HF dataset.",
784 )
785 parser.add_argument(
655 - "--hf-split", type=str, default=None, help="Split of the HF dataset."
786 + "--hf-split",
787 + type=str,
788 + default=None,
789 + help="Split of the HF dataset.",
790 )
791 parser.add_argument(
792 "--profile",
@@ -662,31 +796,28 @@ def add_cli_args(parser: argparse.ArgumentParser):
796 )
797
798 # prefix repetition dataset
665 - prefix_repetition_group = parser.add_argument_group(
666 - "prefix repetition dataset options"
667 - )
668 - prefix_repetition_group.add_argument(
799 + parser.add_argument(
800 "--prefix-repetition-prefix-len",
801 type=int,
802 default=None,
803 help="Number of prefix tokens per request, used only for prefix "
804 "repetition dataset.",
805 )
675 - prefix_repetition_group.add_argument(
806 + parser.add_argument(
807 "--prefix-repetition-suffix-len",
808 type=int,
809 default=None,
810 help="Number of suffix tokens per request, used only for prefix "
811 "repetition dataset. Total input length is prefix_len + suffix_len.",
812 )
682 - prefix_repetition_group.add_argument(
813 + parser.add_argument(
814 "--prefix-repetition-num-prefixes",
815 type=int,
816 default=None,
817 help="Number of prefixes to generate, used only for prefix repetition "
818 "dataset. Prompts per prefix is num_requests // num_prefixes.",
819 )
689 - prefix_repetition_group.add_argument(
820 + parser.add_argument(
821 "--prefix-repetition-output-len",
822 type=int,
823 default=None,
@@ -694,6 +825,10 @@ def add_cli_args(parser: argparse.ArgumentParser):
825 "repetition dataset.",
826 )
827
828 + # (random, random-mm, random-rerank)
829 + add_random_dataset_base_args(parser)
830 + add_random_multimodal_dataset_args(parser)
831 +
832 parser = AsyncEngineArgs.add_cli_args(parser)
833
834
vllm/config/observability.py
+8
@@ -67,6 +67,14 @@ class ObservabilityConfig:
67 enable_mfu_metrics: bool = False
68 """Enable Model FLOPs Utilization (MFU) metrics."""
69
70 + enable_mm_processor_stats: bool = False
71 + """Enable collection of timing statistics for multimodal processor operations.
72 + This is for internal use only (e.g., benchmarks) and is not exposed as a CLI
73 + argument."""
74 +
75 + enable_mfu_metrics: bool = False
76 + """Enable Model FLOPs Utilization (MFU) metrics."""
77 +
78 @cached_property
79 def collect_model_forward_time(self) -> bool:
80 """Whether to collect model forward time for the request."""
vllm/engine/arg_utils.py
+2
@@ -523,6 +523,7 @@ class EngineArgs:
523 ObservabilityConfig.enable_layerwise_nvtx_tracing
524 )
525 enable_mfu_metrics: bool = ObservabilityConfig.enable_mfu_metrics
526 + enable_mm_processor_stats: bool = ObservabilityConfig.enable_mm_processor_stats
527 scheduling_policy: SchedulerPolicy = SchedulerConfig.policy
528 scheduler_cls: str | type[object] | None = SchedulerConfig.scheduler_cls
529
@@ -1712,6 +1713,7 @@ class EngineArgs:
1713 cudagraph_metrics=self.cudagraph_metrics,
1714 enable_layerwise_nvtx_tracing=self.enable_layerwise_nvtx_tracing,
1715 enable_mfu_metrics=self.enable_mfu_metrics,
1716 + enable_mm_processor_stats=self.enable_mm_processor_stats,
1717 )
1718
1719 # Compilation config overrides
vllm/entrypoints/cli/__init__.py
+4
@@ -1,6 +1,9 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 from vllm.entrypoints.cli.benchmark.latency import BenchmarkLatencySubcommand
4 +from vllm.entrypoints.cli.benchmark.mm_processor import (
5 + BenchmarkMMProcessorSubcommand,
6 +)
7 from vllm.entrypoints.cli.benchmark.serve import BenchmarkServingSubcommand
8 from vllm.entrypoints.cli.benchmark.startup import BenchmarkStartupSubcommand
9 from vllm.entrypoints.cli.benchmark.sweep import BenchmarkSweepSubcommand
@@ -8,6 +11,7 @@ from vllm.entrypoints.cli.benchmark.throughput import BenchmarkThroughputSubcomm
11
12 __all__: list[str] = [
13 "BenchmarkLatencySubcommand",
14 + "BenchmarkMMProcessorSubcommand",
15 "BenchmarkServingSubcommand",
16 "BenchmarkStartupSubcommand",
17 "BenchmarkSweepSubcommand",
vllm/entrypoints/cli/benchmark/mm_processor.py new
+21
@@ -0,0 +1,21 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import argparse
4 +
5 +from vllm.benchmarks.mm_processor import add_cli_args, main
6 +from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
7 +
8 +
9 +class BenchmarkMMProcessorSubcommand(BenchmarkSubcommandBase):
10 + """The `mm-processor` subcommand for `vllm bench`."""
11 +
12 + name = "mm-processor"
13 + help = "Benchmark multimodal processor latency across different configurations."
14 +
15 + @classmethod
16 + def add_cli_args(cls, parser: argparse.ArgumentParser) -> None:
17 + add_cli_args(parser)
18 +
19 + @staticmethod
20 + def cmd(args: argparse.Namespace) -> None:
21 + main(args)
vllm/inputs/preprocess.py
+4 -1
@@ -6,7 +6,7 @@ from typing import Any, cast
6
7 from typing_extensions import assert_never
8
9 -from vllm.config import ModelConfig
9 +from vllm.config import ModelConfig, ObservabilityConfig
10 from vllm.logger import init_logger
11 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
12 from vllm.multimodal.cache import BaseMultiModalProcessorCache
@@ -47,6 +47,7 @@ class InputPreprocessor:
47 self,
48 model_config: ModelConfig,
49 tokenizer: TokenizerLike | None,
50 + observability_config: ObservabilityConfig | None = None,
51 mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
52 mm_processor_cache: BaseMultiModalProcessorCache | None = None,
53 ) -> None:
@@ -54,6 +55,7 @@ class InputPreprocessor:
55
56 self.model_config = model_config
57 self.tokenizer = tokenizer
58 + self.observability_config = observability_config
59 self.mm_registry = mm_registry
60 self.mm_processor_cache = mm_processor_cache
61
@@ -232,6 +234,7 @@ class InputPreprocessor:
234 if not hasattr(self, "_mm_processor"):
235 self._mm_processor = self.mm_registry.create_processor(
236 self.model_config,
237 + self.observability_config,
238 tokenizer=self.tokenizer,
239 cache=self.mm_processor_cache,
240 )
vllm/multimodal/processing.py
+252 -37
@@ -1,9 +1,12 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import contextvars
4 +import threading
5 import time
6 from abc import ABC, abstractmethod
7 from collections import defaultdict
8 from collections.abc import Callable, Generator, ItemsView, Iterable, Mapping, Sequence
9 +from contextlib import contextmanager
10 from dataclasses import dataclass, field, replace
11 from enum import Enum
12 from functools import lru_cache
@@ -53,7 +56,7 @@ if TYPE_CHECKING:
56 from transformers.feature_extraction_utils import BatchFeature
57 from transformers.processing_utils import ProcessorMixin
58
56 - from vllm.config import ModelConfig
59 + from vllm.config import ModelConfig, ObservabilityConfig
60
61 from .cache import BaseMultiModalProcessorCache
62 from .profiling import BaseDummyInputsBuilder
@@ -63,6 +66,7 @@ else:
66 ProcessorMixin = object
67
68 ModelConfig = object
69 + ObservabilityConfig = object
70
71 BaseMultiModalProcessorCache = object
72
@@ -70,6 +74,127 @@ logger = init_logger(__name__)
74
75 _S = TypeVar("_S", str, list[int])
76
77 +_request_id_context: contextvars.ContextVar[str | None] = contextvars.ContextVar(
78 + "_request_id_context", default=None
79 +)
80 +
81 +
82 +def get_current_request_id() -> str | None:
83 + """Get the current request_id from the context, if available."""
84 + return _request_id_context.get()
85 +
86 +
87 +@contextmanager
88 +def set_request_id(request_id: str) -> Generator[None, None, None]:
89 + """Context manager to set the request_id for the current context."""
90 + token = _request_id_context.set(request_id)
91 + try:
92 + yield
93 + finally:
94 + _request_id_context.reset(token)
95 +
96 +
97 +@dataclass
98 +class MultiModalProcessorTimingStats:
99 + """Per-request timing statistics for multimodal processor stages."""
100 +
101 + hf_processor_time: float = 0.0
102 + """Time spent in HuggingFace processor calls (seconds)."""
103 +
104 + hashing_time: float = 0.0
105 + """Time spent computing multimodal item hashes (seconds)."""
106 +
107 + cache_lookup_time: float = 0.0
108 + """Time spent in cache lookups and merges (seconds)."""
109 +
110 + prompt_update_time: float = 0.0
111 + """Time spent applying prompt updates and finding placeholders (seconds)."""
112 +
113 + total_time: float = 0.0
114 + """Total processing time (seconds)."""
115 +
116 + def to_dict(self) -> dict[str, float]:
117 + """Convert stats to a dictionary for JSON serialization."""
118 + return {
119 + "hf_processor_time": self.hf_processor_time,
120 + "hashing_time": self.hashing_time,
121 + "cache_lookup_time": self.cache_lookup_time,
122 + "prompt_update_time": self.prompt_update_time,
123 + "total_time": self.total_time,
124 + }
125 +
126 +
127 +def get_timing_stats_from_engine_client(
128 + engine_client: Any,
129 +) -> dict[str, dict[str, float]]:
130 + """
131 + Get all timing stats from the context associated with the engine client.
132 +
133 + Args:
134 + engine_client: The engine client that has input_processor.
135 +
136 + Returns:
137 + A dictionary mapping request_id to stats dict.
138 + """
139 + try:
140 + if not engine_client.vllm_config.observability_config.enable_mm_processor_stats:
141 + return {}
142 + except (AttributeError, RuntimeError):
143 + return {}
144 +
145 + try:
146 + input_processor = engine_client.input_processor
147 + input_preprocessor = input_processor.input_preprocessor
148 +
149 + if hasattr(input_preprocessor, "_get_mm_processor"):
150 + mm_processor = input_preprocessor._get_mm_processor()
151 + if mm_processor is not None and hasattr(mm_processor, "info"):
152 + ctx = mm_processor.info.ctx
153 + return ctx.get_all_timing_stats()
154 + except (AttributeError, RuntimeError):
155 + pass
156 +
157 + return {}
158 +
159 +
160 +@contextmanager
161 +def _timed_operation(ctx: "InputProcessingContext", stage_name: str):
162 + """
163 + Context manager to time an operation using the context's timing stats.
164 +
165 + The request_id is automatically retrieved from the context variable,
166 + so it doesn't need to be passed as a parameter.
167 +
168 + Args:
169 + ctx: The InputProcessingContext containing the timing stats registry.
170 + stage_name: Name of the stage being timed.
171 + """
172 + request_id = get_current_request_id()
173 + if ctx is None or request_id is None:
174 + yield
175 + return
176 +
177 + stats = ctx.get_timing_stats(request_id)
178 + if stats is None:
179 + yield
180 + return
181 +
182 + start_time = time.perf_counter()
183 + try:
184 + yield
185 + finally:
186 + elapsed = time.perf_counter() - start_time
187 + if stage_name == "hf_processor":
188 + stats.hf_processor_time += elapsed
189 + elif stage_name == "hashing":
190 + stats.hashing_time += elapsed
191 + elif stage_name == "cache_lookup":
192 + stats.cache_lookup_time += elapsed
193 + elif stage_name == "prompt_update":
194 + stats.prompt_update_time += elapsed
195 + stats.total_time += elapsed
196 +
197 +
198 PromptSeq: TypeAlias = str | list[int]
199 """A token sequence (list of token IDs) or text."""
200
@@ -951,6 +1076,21 @@ class InputProcessingContext:
1076 tokenizer: TokenizerLike | None
1077 """The tokenizer used to tokenize the inputs."""
1078
1079 + observability_config: "ObservabilityConfig | None" = field(
1080 + default=None, compare=False, repr=False
1081 + )
1082 + """Configuration for observability features."""
1083 +
1084 + timing_stats_registry: dict[str, MultiModalProcessorTimingStats] = field(
1085 + default_factory=dict, compare=False, repr=False
1086 + )
1087 + """Registry for storing timing stats keyed by request_id."""
1088 +
1089 + _timing_stats_registry_lock: threading.Lock = field(
1090 + default_factory=threading.Lock, compare=False, repr=False
1091 + )
1092 + """Lock for thread-safe access to timing_stats_registry."""
1093 +
1094 def get_tokenizer(self) -> TokenizerLike:
1095 if self.tokenizer is None:
1096 raise ValueError(
@@ -1159,6 +1299,71 @@ class InputProcessingContext:
1299
1300 return self._postprocess_output(output)
1301
1302 + def get_timing_stats(
1303 + self, request_id: str
1304 + ) -> MultiModalProcessorTimingStats | None:
1305 + """
1306 + Get timing stats for a request.
1307 + """
1308 + if (
1309 + self.observability_config is None
1310 + or not self.observability_config.enable_mm_processor_stats
1311 + ):
1312 + return None
1313 + with self._timing_stats_registry_lock:
1314 + return self.timing_stats_registry.get(request_id)
1315 +
1316 + def create_timing_stats(self, request_id: str) -> MultiModalProcessorTimingStats:
1317 + """
1318 + Create and store timing stats in the registry for a request.
1319 +
1320 + This should be called at the start of processing for a request.
1321 + The stats object is created immediately and stored in the registry.
1322 + """
1323 + if (
1324 + self.observability_config is None
1325 + or not self.observability_config.enable_mm_processor_stats
1326 + ):
1327 + return MultiModalProcessorTimingStats()
1328 +
1329 + with self._timing_stats_registry_lock:
1330 + if request_id in self.timing_stats_registry:
1331 + raise ValueError(
1332 + f"Timing stats already exist for request_id: {request_id}"
1333 + )
1334 + stats = MultiModalProcessorTimingStats()
1335 + self.timing_stats_registry[request_id] = stats
1336 + return stats
1337 +
1338 + def clear_timing_stats_registry(self) -> int:
1339 + """
1340 + Clear all stats from the registry. Returns the number of stats cleared.
1341 + """
1342 + if (
1343 + self.observability_config is None
1344 + or not self.observability_config.enable_mm_processor_stats
1345 + ):
1346 + return 0
1347 + with self._timing_stats_registry_lock:
1348 + count = len(self.timing_stats_registry)
1349 + self.timing_stats_registry.clear()
1350 + return count
1351 +
1352 + def get_all_timing_stats(self) -> dict[str, dict[str, float]]:
1353 + """
1354 + Get all timing stats as a dictionary for API endpoints.
1355 + """
1356 + if (
1357 + self.observability_config is None
1358 + or not self.observability_config.enable_mm_processor_stats
1359 + ):
1360 + return {}
1361 + with self._timing_stats_registry_lock:
1362 + return {
1363 + rid: stats.to_dict()
1364 + for rid, stats in self.timing_stats_registry.items()
1365 + }
1366 +
1367
1368 class BaseProcessingInfo:
1369 """Base class to provide the information necessary for data processing."""
@@ -1502,11 +1707,12 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1707 Call the HF processor on the prompt text and
1708 associated multi-modal data.
1709 """
1505 - return self.info.ctx.call_hf_processor(
1506 - self.info.get_hf_processor(**mm_kwargs),
1507 - dict(text=prompt, **mm_data),
1508 - dict(**mm_kwargs, **tok_kwargs),
1509 - )
1710 + with _timed_operation(self.info.ctx, "hf_processor"):
1711 + return self.info.ctx.call_hf_processor(
1712 + self.info.get_hf_processor(**mm_kwargs),
1713 + dict(text=prompt, **mm_data),
1714 + dict(**mm_kwargs, **tok_kwargs),
1715 + )
1716
1717 def _hf_processor_applies_updates(
1718 self,
@@ -1854,12 +2060,13 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
2060 )
2061
2062 # Use overrides if provided; fallback to data-dependent hashing.
1857 - mm_hashes = self._hash_mm_items(
1858 - mm_data_items,
1859 - hf_processor_mm_kwargs,
1860 - tokenization_kwargs,
1861 - mm_uuids=mm_uuids,
1862 - )
2063 + with _timed_operation(self.info.ctx, "hashing"):
2064 + mm_hashes = self._hash_mm_items(
2065 + mm_data_items,
2066 + hf_processor_mm_kwargs,
2067 + tokenization_kwargs,
2068 + mm_uuids=mm_uuids,
2069 + )
2070
2071 mm_prompt_updates = self._get_mm_prompt_updates(
2072 mm_data_items,
@@ -1900,18 +2107,20 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
2107 mm_uuids=mm_uuids,
2108 )
2109
1903 - mm_hashes = self._hash_mm_items(
1904 - mm_data_items,
1905 - hf_processor_mm_kwargs,
1906 - tokenization_kwargs,
1907 - mm_uuids=mm_uuids,
1908 - )
2110 + with _timed_operation(self.info.ctx, "hashing"):
2111 + mm_hashes = self._hash_mm_items(
2112 + mm_data_items,
2113 + hf_processor_mm_kwargs,
2114 + tokenization_kwargs,
2115 + mm_uuids=mm_uuids,
2116 + )
2117
1910 - mm_is_cached, mm_missing_data_items = self._get_cache_missing_items(
1911 - cache=cache,
1912 - mm_data_items=mm_data_items,
1913 - mm_hashes=mm_hashes,
1914 - )
2118 + with _timed_operation(self.info.ctx, "cache_lookup"):
2119 + mm_is_cached, mm_missing_data_items = self._get_cache_missing_items(
2120 + cache=cache,
2121 + mm_data_items=mm_data_items,
2122 + mm_hashes=mm_hashes,
2123 + )
2124
2125 # NOTE: `prompt` does not correspond to `mm_missing_data_items`,
2126 # so we can't apply prompt updates until the new multimodal
@@ -1941,13 +2150,14 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
2150 mm_missing_kwargs,
2151 )
2152
1944 - mm_kwargs, mm_prompt_updates = self._merge_mm_kwargs(
1945 - cache,
1946 - mm_hashes=mm_hashes,
1947 - mm_is_cached=mm_is_cached,
1948 - mm_missing_kwargs=mm_missing_kwargs,
1949 - mm_missing_prompt_updates=mm_missing_prompt_updates,
1950 - )
2153 + with _timed_operation(self.info.ctx, "cache_lookup"):
2154 + mm_kwargs, mm_prompt_updates = self._merge_mm_kwargs(
2155 + cache,
2156 + mm_hashes=mm_hashes,
2157 + mm_is_cached=mm_is_cached,
2158 + mm_missing_kwargs=mm_missing_kwargs,
2159 + mm_missing_prompt_updates=mm_missing_prompt_updates,
2160 + )
2161
2162 mm_info = MultiModalProcessingInfo(
2163 kwargs=mm_kwargs,
@@ -2129,6 +2339,10 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
2339 3. Extract information about the placeholder tokens from the
2340 processed token IDs.
2341 """
2342 + request_id = get_current_request_id()
2343 + if request_id is not None:
2344 + self.info.ctx.create_timing_stats(request_id)
2345 +
2346 mm_items = self._to_mm_items(mm_data)
2347
2348 if tokenization_kwargs is None:
@@ -2147,13 +2361,14 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
2361 )
2362
2363 # NOTE: tokenization_kwargs are not required to init processor
2150 - prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
2151 - mm_items=mm_items,
2152 - prompt_ids=prompt_ids,
2153 - mm_kwargs=mm_info.kwargs,
2154 - mm_prompt_updates=mm_info.prompt_updates,
2155 - is_update_applied=is_update_applied,
2156 - )
2364 + with _timed_operation(self.info.ctx, "prompt_update"):
2365 + prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
2366 + mm_items=mm_items,
2367 + prompt_ids=prompt_ids,
2368 + mm_kwargs=mm_info.kwargs,
2369 + mm_prompt_updates=mm_info.prompt_updates,
2370 + is_update_applied=is_update_applied,
2371 + )
2372
2373 mm_placeholder_ranges = {
2374 modality: [item.to_range() for item in placeholders]
vllm/multimodal/registry.py
+26 -8
@@ -5,6 +5,7 @@ from dataclasses import dataclass
5 from typing import TYPE_CHECKING, Generic, Protocol, TypeVar, cast
6
7 from vllm.config.multimodal import BaseDummyOptions
8 +from vllm.config.observability import ObservabilityConfig
9 from vllm.logger import init_logger
10 from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
11
@@ -22,7 +23,7 @@ from .profiling import (
23 )
24
25 if TYPE_CHECKING:
25 - from vllm.config import ModelConfig
26 + from vllm.config import ModelConfig, ObservabilityConfig
27 from vllm.model_executor.models.interfaces import SupportsMultiModal
28
29 logger = init_logger(__name__)
@@ -148,6 +149,7 @@ class MultiModalRegistry:
149 *,
150 cache: BaseMultiModalProcessorCache | None = None,
151 profiler_limits: Mapping[str, int] | None = None,
152 + observability_config: ObservabilityConfig | None = None,
153 ) -> Mapping[str, int]:
154 """
155 Get the maximum number of tokens per data item from each modality based
@@ -156,7 +158,9 @@ class MultiModalRegistry:
158 if not model_config.is_multimodal_model:
159 return {}
160
159 - processor = self.create_processor(model_config, cache=cache)
161 + processor = self.create_processor(
162 + model_config, observability_config, cache=cache
163 + )
164 profiler: MultiModalProfiler = MultiModalProfiler(processor)
165
166 seq_len = model_config.max_model_len
@@ -174,6 +178,7 @@ class MultiModalRegistry:
178 model_config: "ModelConfig",
179 *,
180 cache: BaseMultiModalProcessorCache | None = None,
181 + observability_config: ObservabilityConfig | None = None,
182 ) -> Mapping[str, int]:
183 """
184 Get the maximum number of multi-modal input instances for each modality
@@ -182,7 +187,9 @@ class MultiModalRegistry:
187 if not model_config.is_multimodal_model:
188 return {}
189
185 - processor = self.create_processor(model_config, cache=cache)
190 + processor = self.create_processor(
191 + model_config, observability_config, cache=cache
192 + )
193 profiler: MultiModalProfiler = MultiModalProfiler(processor)
194 return profiler.get_mm_limits()
195
@@ -231,27 +238,32 @@ class MultiModalRegistry:
238 def _create_processing_ctx(
239 self,
240 model_config: "ModelConfig",
241 + observability_config: "ObservabilityConfig | None" = None,
242 tokenizer: TokenizerLike | None = None,
243 ) -> InputProcessingContext:
244 if tokenizer is None and not model_config.skip_tokenizer_init:
245 tokenizer = cached_tokenizer_from_config(model_config)
246
239 - return InputProcessingContext(model_config, tokenizer)
247 + return InputProcessingContext(
248 + model_config, tokenizer, observability_config=observability_config
249 + )
250
251 def _create_processing_info(
252 self,
253 model_config: "ModelConfig",
254 + observability_config: "ObservabilityConfig | None" = None,
255 *,
256 tokenizer: TokenizerLike | None = None,
257 ) -> BaseProcessingInfo:
258 model_cls = self._get_model_cls(model_config)
259 factories = model_cls._processor_factory
249 - ctx = self._create_processing_ctx(model_config, tokenizer)
260 + ctx = self._create_processing_ctx(model_config, observability_config, tokenizer)
261 return factories.info(ctx)
262
263 def create_processor(
264 self,
265 model_config: "ModelConfig",
266 + observability_config: "ObservabilityConfig | None" = None,
267 *,
268 tokenizer: TokenizerLike | None = None,
269 cache: BaseMultiModalProcessorCache | None = None,
@@ -265,7 +277,7 @@ class MultiModalRegistry:
277 model_cls = self._get_model_cls(model_config)
278 factories = model_cls._processor_factory
279
268 - ctx = self._create_processing_ctx(model_config, tokenizer)
280 + ctx = self._create_processing_ctx(model_config, observability_config, tokenizer)
281
282 return factories.build_processor(ctx, cache=cache)
283
@@ -276,13 +288,16 @@ class MultiModalRegistry:
288 mm_counts: Mapping[str, int] | None = None,
289 *,
290 cache: BaseMultiModalProcessorCache | None = None,
291 + observability_config: ObservabilityConfig | None = None,
292 ) -> DummyDecoderData:
293 """
294 Create dummy data for profiling the memory usage of a model.
295
296 The model is identified by `model_config`.
297 """
285 - processor = self.create_processor(model_config, cache=cache)
298 + processor = self.create_processor(
299 + model_config, observability_config, cache=cache
300 + )
301 profiler: MultiModalProfiler = MultiModalProfiler(processor)
302
303 # Extract configurable options from multimodal config.
@@ -309,13 +324,16 @@ class MultiModalRegistry:
324 mm_counts: Mapping[str, int] | None = None,
325 *,
326 cache: BaseMultiModalProcessorCache | None = None,
327 + observability_config: ObservabilityConfig | None = None,
328 ) -> DummyEncoderData:
329 """
330 Create dummy data for profiling the memory usage of a model.
331
332 The model is identified by `model_config`.
333 """
318 - processor = self.create_processor(model_config, cache=cache)
334 + processor = self.create_processor(
335 + model_config, observability_config, cache=cache
336 + )
337 profiler: MultiModalProfiler = MultiModalProfiler(processor)
338
339 # Extract configurable options from multimodal config.
vllm/v1/engine/input_processor.py
+10 -6
@@ -15,7 +15,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
15 from vllm.multimodal.cache import processor_cache_from_config
16 from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalUUIDDict
17 from vllm.multimodal.parse import MultiModalDataParser
18 -from vllm.multimodal.processing import EncDecMultiModalProcessor
18 +from vllm.multimodal.processing import EncDecMultiModalProcessor, set_request_id
19 from vllm.multimodal.utils import argsort_mm_positions
20 from vllm.pooling_params import PoolingParams
21 from vllm.sampling_params import SamplingParams
@@ -60,6 +60,7 @@ class InputProcessor:
60 self.input_preprocessor = InputPreprocessor(
61 self.model_config,
62 tokenizer,
63 + self.vllm_config.observability_config,
64 mm_registry,
65 mm_processor_cache=self.mm_processor_cache,
66 )
@@ -493,11 +494,13 @@ class InputProcessor:
494 # 1. Tokenize text prompt, with LoRA request if one exists.
495 # 2. For multimodal models with a merged preprocessor, preprocess
496 # multimodal data and expand prompt token ids accordingly.
496 - processed_inputs: ProcessorInputs = self.input_preprocessor.preprocess(
497 - prompt,
498 - tokenization_kwargs=tokenization_kwargs,
499 - mm_uuids=mm_uuids,
500 - )
497 + with set_request_id(request_id):
498 + processed_inputs: ProcessorInputs = self.input_preprocessor.preprocess(
499 + prompt,
500 + tokenization_kwargs=tokenization_kwargs,
501 + mm_uuids=mm_uuids,
502 + )
503 +
504 from vllm.platforms import current_platform
505
506 current_platform.validate_request(
@@ -641,6 +644,7 @@ class InputProcessor:
644 mm_registry = self.input_preprocessor.mm_registry
645 mm_processor = mm_registry.create_processor(
646 model_config,
647 + self.vllm_config.observability_config,
648 tokenizer=tokenizer,
649 )
650 assert isinstance(mm_processor, EncDecMultiModalProcessor)