A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 17:06 UTC
feat(benchmark): add encoder forward pass benchmarking to mm-processor (#31655)
Signed-off-by: Reagan <reaganjlee@gmail.com> Signed-off-by: Reagan Lee <96998476+reaganjlee@users.noreply.github.com> Co-authored-by: Hiroken. <105287758+HirokenOvo@users.noreply.github.com>
Reagan Lee committed
Jan 24, 2026 at 00:24 UTC
06b557ecd9881289afcfa43458fb07329c955921
5 files changed
+303
-59
vllm/benchmarks/mm_processor.py
+95
-28
@@ -22,6 +22,10 @@ from typing import Any
22
23
import numpy as np
24
25
+from vllm.benchmarks.datasets import (
26
+ MultiModalConversationDataset,
27
+ VisionArenaDataset,
28
+)
29
from vllm.benchmarks.throughput import get_requests
30
from vllm.engine.arg_utils import EngineArgs
31
from vllm.multimodal.processing.context import (
@@ -45,26 +49,21 @@ def collect_mm_processor_stats(
49
"""
50
all_stats = get_timing_stats_from_engine_client(llm_engine)
51
48
- stats_by_stage = {
49
- "hf_processor_time": [],
50
- "hashing_time": [],
51
- "cache_lookup_time": [],
52
- "prompt_update_time": [],
53
- "total_time": [],
54
- }
52
+ stat_keys = [
53
+ "hf_processor_time",
54
+ "hashing_time",
55
+ "cache_lookup_time",
56
+ "prompt_update_time",
57
+ "preprocessor_total_time",
58
+ "encoder_forward_time",
59
+ "num_encoder_calls",
60
+ ]
61
+ stats_by_stage = {key: [] for key in stat_keys}
62
63
for stats_dict in all_stats.values():
57
- stats_by_stage["hf_processor_time"].append(
58
- stats_dict.get("hf_processor_time", 0.0)
59
- )
60
- stats_by_stage["hashing_time"].append(stats_dict.get("hashing_time", 0.0))
61
- stats_by_stage["cache_lookup_time"].append(
62
- stats_dict.get("cache_lookup_time", 0.0)
63
- )
64
- stats_by_stage["prompt_update_time"].append(
65
- stats_dict.get("prompt_update_time", 0.0)
66
- )
67
- stats_by_stage["total_time"].append(stats_dict.get("total_time", 0.0))
64
+ for key in stat_keys:
65
+ if key in stats_dict:
66
+ stats_by_stage[key].append(stats_dict[key])
67
68
return stats_by_stage
69
@@ -88,14 +87,14 @@ def calculate_mm_processor_metrics(
87
}
88
continue
89
91
- times_ms = [t * 1000 for t in times]
90
+ is_count_metric = stage_name == "num_encoder_calls"
91
+ values = times if is_count_metric else [t * 1000 for t in times]
92
+
93
metrics[stage_name] = {
93
- "mean": float(np.mean(times_ms)),
94
- "median": float(np.median(times_ms)),
95
- "std": float(np.std(times_ms)),
96
- **{
97
- f"p{p}": float(np.percentile(times_ms, p)) for p in selected_percentiles
98
- },
94
+ "mean": float(np.mean(values)),
95
+ "median": float(np.median(values)),
96
+ "std": float(np.std(values)),
97
+ **{f"p{p}": float(np.percentile(values, p)) for p in selected_percentiles},
98
}
99
100
return metrics
@@ -114,6 +113,23 @@ def validate_args(args):
113
if not hasattr(args, "max_loras"):
114
args.max_loras = None
115
116
+ if args.dataset_name == "hf" and not args.dataset_path:
117
+ raise ValueError(
118
+ "--dataset-path is required when using --dataset-name hf. "
119
+ "For multimodal benchmarking, specify a dataset like "
120
+ "'lmarena-ai/VisionArena-Chat'."
121
+ )
122
+ if args.dataset_name == "hf":
123
+ supported_mm_datasets = (
124
+ VisionArenaDataset.SUPPORTED_DATASET_PATHS.keys()
125
+ | MultiModalConversationDataset.SUPPORTED_DATASET_PATHS
126
+ )
127
+ if args.dataset_path not in supported_mm_datasets:
128
+ raise ValueError(
129
+ f"{args.dataset_path} is not a supported multimodal dataset. "
130
+ f"Supported multimodal datasets are: {sorted(supported_mm_datasets)}"
131
+ )
132
+
133
134
def benchmark_multimodal_processor(
135
args: argparse.Namespace,
@@ -223,6 +239,17 @@ def benchmark_multimodal_processor(
239
std_e2el_ms = 0.0
240
percentiles_e2el_ms = [(p, 0.0) for p in selected_percentiles]
241
242
+ encoder_summary = {}
243
+ if (
244
+ "num_encoder_calls" in mm_stats_by_stage
245
+ and mm_stats_by_stage["num_encoder_calls"]
246
+ ):
247
+ encoder_calls = mm_stats_by_stage["num_encoder_calls"]
248
+ encoder_summary = {
249
+ "total_encoder_calls": int(sum(encoder_calls)),
250
+ "num_requests_with_encoder_calls": len(encoder_calls),
251
+ }
252
+
253
benchmark_result = {
254
"completed": completed,
255
"failed": failed,
@@ -231,6 +258,7 @@ def benchmark_multimodal_processor(
258
"std_e2el_ms": std_e2el_ms,
259
"percentiles_e2el_ms": percentiles_e2el_ms,
260
"mm_processor_stats": mm_processor_metrics,
261
+ "encoder_summary": encoder_summary,
262
}
263
264
return benchmark_result
@@ -248,7 +276,7 @@ def add_cli_args(parser: argparse.ArgumentParser) -> None:
276
"--dataset-name",
277
type=str,
278
default="random-mm",
251
- choices=["random-mm", "random-rerank"],
279
+ choices=["random-mm", "hf"],
280
help="Name of the dataset to benchmark on. Defaults to 'random-mm'.",
281
)
282
parser.add_argument(
@@ -266,6 +294,34 @@ def add_cli_args(parser: argparse.ArgumentParser) -> None:
294
add_random_dataset_base_args(parser)
295
add_random_multimodal_dataset_args(parser)
296
297
+ # HuggingFace dataset arguments
298
+ parser.add_argument(
299
+ "--dataset-path",
300
+ type=str,
301
+ default=None,
302
+ help="Path to the dataset file or HuggingFace dataset name "
303
+ "(e.g., 'yale-nlp/MMVU', 'lmarena-ai/VisionArena-Chat').",
304
+ )
305
+ parser.add_argument(
306
+ "--hf-subset",
307
+ type=str,
308
+ default=None,
309
+ help="Subset of the HuggingFace dataset (optional).",
310
+ )
311
+ parser.add_argument(
312
+ "--hf-split",
313
+ type=str,
314
+ default=None,
315
+ help="Split of the HuggingFace dataset (e.g., 'train', 'test', 'validation').",
316
+ )
317
+ parser.add_argument(
318
+ "--output-len",
319
+ type=int,
320
+ default=None,
321
+ help="Output length for each request. "
322
+ "Overrides the default output lengths from the dataset.",
323
+ )
324
+
325
parser.add_argument(
326
"--output-json",
327
type=str,
@@ -296,14 +352,17 @@ def main(args: argparse.Namespace) -> None:
352
print("=" * 80)
353
354
if "mm_processor_stats" in result:
299
- print("\nMM Processor Timing (ms):")
355
+ print("\nMM Processor Metrics:")
356
selected_percentiles = [
357
float(p) for p in getattr(args, "metric_percentiles", "99").split(",")
358
]
359
mm_data = []
360
for stage, metrics in result["mm_processor_stats"].items():
361
+ is_count = stage == "num_encoder_calls"
362
+ unit = "" if is_count else " (ms)"
363
+
364
row = {
306
- "Stage": stage,
365
+ "Stage": stage + unit,
366
"Mean": f"{metrics['mean']:.2f}",
367
"Median": f"{metrics['median']:.2f}",
368
"Std": f"{metrics['std']:.2f}",
@@ -315,6 +374,14 @@ def main(args: argparse.Namespace) -> None:
374
mm_df = pd.DataFrame(mm_data)
375
print(mm_df.to_string(index=False))
376
377
+ if "encoder_summary" in result and result["encoder_summary"]:
378
+ total_calls = result["encoder_summary"]["total_encoder_calls"]
379
+ num_requests = result["encoder_summary"]["num_requests_with_encoder_calls"]
380
+ print(
381
+ f"\nSummary: {total_calls} total encoder calls "
382
+ f"across {num_requests} requests."
383
+ )
384
+
385
if "mean_e2el_ms" in result:
386
print("\nEnd-to-End Latency (ms):")
387
selected_percentiles = [
vllm/multimodal/processing/context.py
+81
-10
@@ -75,8 +75,8 @@ class MultiModalProcessorTimingStats:
75
prompt_update_time: float = 0.0
76
"""Time spent applying prompt updates and finding placeholders (seconds)."""
77
78
- total_time: float = 0.0
79
- """Total processing time (seconds)."""
78
+ preprocessor_total_time: float = 0.0
79
+ """Total preprocessing time (seconds)."""
80
81
def to_dict(self) -> dict[str, float]:
82
"""Convert stats to a dictionary for JSON serialization."""
@@ -85,7 +85,7 @@ class MultiModalProcessorTimingStats:
85
"hashing_time": self.hashing_time,
86
"cache_lookup_time": self.cache_lookup_time,
87
"prompt_update_time": self.prompt_update_time,
88
- "total_time": self.total_time,
88
+ "preprocessor_total_time": self.preprocessor_total_time,
89
}
90
91
@@ -93,13 +93,30 @@ def get_timing_stats_from_engine_client(
93
engine_client: Any,
94
) -> dict[str, dict[str, float]]:
95
"""
96
- Get all timing stats from the context associated with the engine client.
96
+ Get all multimodal timing stats from the engine client.
97
+
98
+ Collects both preprocessing stats (HF processor, hashing, cache lookup,
99
+ prompt update) and encoder forward pass timing, merged by request_id.
100
101
Args:
99
- engine_client: The engine client that has input_processor.
102
+ engine_client: The engine client (has input_processor and workers).
103
104
Returns:
102
- A dictionary mapping request_id to stats dict.
105
+ Dictionary mapping request_id to merged stats dict containing
106
+ both preprocessing and encoder timing metrics.
107
+
108
+ Example:
109
+ {
110
+ 'request-123': {
111
+ 'hf_processor_time': 0.45,
112
+ 'hashing_time': 0.02,
113
+ 'cache_lookup_time': 0.01,
114
+ 'prompt_update_time': 0.03,
115
+ 'preprocessor_total_time': 0.51,
116
+ 'encoder_forward_time': 0.23,
117
+ 'num_encoder_calls': 1
118
+ }
119
+ }
120
"""
121
try:
122
if not engine_client.vllm_config.observability_config.enable_mm_processor_stats:
@@ -107,6 +124,7 @@ def get_timing_stats_from_engine_client(
124
except (AttributeError, RuntimeError):
125
return {}
126
127
+ preprocessing_stats = {}
128
try:
129
input_processor = engine_client.input_processor
130
input_preprocessor = input_processor.input_preprocessor
@@ -115,15 +133,68 @@ def get_timing_stats_from_engine_client(
133
mm_processor = input_preprocessor._get_mm_processor()
134
if mm_processor is not None and hasattr(mm_processor, "info"):
135
ctx = mm_processor.info.ctx
118
- return ctx.get_all_timing_stats()
136
+ preprocessing_stats = ctx.get_all_timing_stats()
137
+ except (AttributeError, RuntimeError):
138
+ pass
139
+
140
+ encoder_stats = {}
141
+ try:
142
+ if hasattr(engine_client, "collective_rpc"):
143
+ encoder_stats_results = engine_client.collective_rpc(
144
+ "get_encoder_timing_stats"
145
+ )
146
+ if encoder_stats_results and len(encoder_stats_results) > 0:
147
+ for worker_stats in encoder_stats_results:
148
+ if not worker_stats:
149
+ continue
150
+ for request_id, stats_dict in worker_stats.items():
151
+ if request_id not in encoder_stats:
152
+ encoder_stats[request_id] = dict(stats_dict)
153
+ else:
154
+ # Aggregate timing metrics across workers
155
+ current_time = encoder_stats[request_id].get(
156
+ "encoder_forward_time", 0.0
157
+ )
158
+ new_time = stats_dict.get("encoder_forward_time", 0.0)
159
+ encoder_stats[request_id]["encoder_forward_time"] = max(
160
+ current_time, new_time
161
+ )
162
+
163
+ current_calls = encoder_stats[request_id].get(
164
+ "num_encoder_calls", 0
165
+ )
166
+ new_calls = stats_dict.get("num_encoder_calls", 0)
167
+ encoder_stats[request_id]["num_encoder_calls"] = max(
168
+ current_calls, new_calls
169
+ )
170
except (AttributeError, RuntimeError):
171
pass
172
122
- return {}
173
+ merged_stats = {}
174
+
175
+ for request_id, prep_dict in preprocessing_stats.items():
176
+ merged_stats[request_id] = dict(prep_dict)
177
+
178
+ for request_id, enc_dict in encoder_stats.items():
179
+ if request_id in merged_stats:
180
+ merged_stats[request_id].update(enc_dict)
181
+ continue
182
+
183
+ # In V1 engine, the request_id in encoder_stats has a suffix
184
+ # appended to the original request_id (which is used in
185
+ # preprocessing_stats).
186
+ # We try to strip the suffix to find the matching request.
187
+ possible_original_id = request_id.rpartition("-")[0]
188
+ if possible_original_id and possible_original_id in merged_stats:
189
+ merged_stats[possible_original_id].update(enc_dict)
190
+ else:
191
+ merged_stats[request_id] = dict(enc_dict)
192
+
193
+ return merged_stats
194
195
196
@contextmanager
126
-def timed_operation(ctx: "InputProcessingContext", stage_name: str):
197
+def timed_preprocessor_operation(ctx: "InputProcessingContext", stage_name: str):
198
"""
199
Context manager to time an operation using the context's timing stats.
200
@@ -157,7 +228,7 @@ def timed_operation(ctx: "InputProcessingContext", stage_name: str):
228
stats.cache_lookup_time += elapsed
229
elif stage_name == "prompt_update":
230
stats.prompt_update_time += elapsed
160
- stats.total_time += elapsed
231
+ stats.preprocessor_total_time += elapsed
232
233
234
_T = TypeVar("_T")
vllm/multimodal/processing/processor.py
+11
-7
@@ -42,7 +42,11 @@ from ..parse import (
42
MultiModalDataItems,
43
MultiModalDataParser,
44
)
45
-from .context import BaseProcessingInfo, get_current_request_id, timed_operation
45
+from .context import (
46
+ BaseProcessingInfo,
47
+ get_current_request_id,
48
+ timed_preprocessor_operation,
49
+)
50
from .dummy_inputs import BaseDummyInputsBuilder
51
52
if TYPE_CHECKING:
@@ -1192,7 +1196,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1196
Call the HF processor on the prompt text and
1197
associated multi-modal data.
1198
"""
1195
- with timed_operation(self.info.ctx, "hf_processor"):
1199
+ with timed_preprocessor_operation(self.info.ctx, "hf_processor"):
1200
return self.info.ctx.call_hf_processor(
1201
self.info.get_hf_processor(**mm_kwargs),
1202
dict(text=prompt, **mm_data),
@@ -1545,7 +1549,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1549
)
1550
1551
# Use overrides if provided; fallback to data-dependent hashing.
1548
- with timed_operation(self.info.ctx, "hashing"):
1552
+ with timed_preprocessor_operation(self.info.ctx, "hashing"):
1553
mm_hashes = self._hash_mm_items(
1554
mm_data_items,
1555
hf_processor_mm_kwargs,
@@ -1592,7 +1596,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1596
mm_uuids=mm_uuids,
1597
)
1598
1595
- with timed_operation(self.info.ctx, "hashing"):
1599
+ with timed_preprocessor_operation(self.info.ctx, "hashing"):
1600
mm_hashes = self._hash_mm_items(
1601
mm_data_items,
1602
hf_processor_mm_kwargs,
@@ -1600,7 +1604,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1604
mm_uuids=mm_uuids,
1605
)
1606
1603
- with timed_operation(self.info.ctx, "cache_lookup"):
1607
+ with timed_preprocessor_operation(self.info.ctx, "cache_lookup"):
1608
mm_is_cached, mm_missing_data_items = self._get_cache_missing_items(
1609
cache=cache,
1610
mm_data_items=mm_data_items,
@@ -1635,7 +1639,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1639
mm_missing_kwargs,
1640
)
1641
1638
- with timed_operation(self.info.ctx, "cache_lookup"):
1642
+ with timed_preprocessor_operation(self.info.ctx, "cache_lookup"):
1643
mm_kwargs, mm_prompt_updates = self._merge_mm_kwargs(
1644
cache,
1645
mm_hashes=mm_hashes,
@@ -1846,7 +1850,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1850
)
1851
1852
# NOTE: tokenization_kwargs are not required to init processor
1849
- with timed_operation(self.info.ctx, "prompt_update"):
1853
+ with timed_preprocessor_operation(self.info.ctx, "prompt_update"):
1854
prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
1855
mm_items=mm_items,
1856
prompt_ids=prompt_ids,
vllm/v1/worker/gpu_model_runner.py
+112
-14
@@ -4,11 +4,13 @@
4
import functools
5
import gc
6
import itertools
7
+import threading
8
import time
9
from collections import defaultdict
10
from collections.abc import Iterator, Sequence
11
from contextlib import contextmanager
12
from copy import copy, deepcopy
13
+from dataclasses import dataclass
14
from functools import reduce
15
from typing import TYPE_CHECKING, Any, NamedTuple, TypeAlias, cast
16
@@ -546,6 +548,10 @@ class GPUModelRunner(
548
# Cache the device properties.
549
self._init_device_properties()
550
551
+ # Encoder timing registry for observability
552
+ self.encoder_timing_registry: dict[str, EncoderTimingStats] = {}
553
+ self._encoder_timing_lock = threading.Lock()
554
+
555
# Persistent buffers for CUDA graphs.
556
self.input_ids = self._make_buffer(self.max_num_tokens, dtype=torch.int32)
557
self.positions = self._make_buffer(self.max_num_tokens, dtype=torch.int64)
@@ -2212,6 +2218,12 @@ class GPUModelRunner(
2218
if not mm_kwargs:
2219
return []
2220
2221
+ should_time = bool(
2222
+ self.observability_config
2223
+ and self.observability_config.enable_mm_processor_stats
2224
+ and scheduler_output.scheduled_encoder_inputs
2225
+ )
2226
+
2227
# Batch mm inputs as much as we can: if a request in the batch has
2228
# multiple modalities or a different modality than the previous one,
2229
# we process it separately to preserve item order.
@@ -2278,6 +2290,8 @@ class GPUModelRunner(
2290
)
2291
2292
encoder_outputs: list[torch.Tensor] = []
2293
+ # Track the current index in mm_kwargs/mm_lora_refs to map groups to request IDs
2294
+ current_item_idx = 0
2295
for modality, num_items, mm_kwargs_group in group_mm_kwargs_by_modality(
2296
mm_kwargs,
2297
device=self.device,
@@ -2300,22 +2314,24 @@ class GPUModelRunner(
2314
and num_items > 1
2315
):
2316
curr_group_outputs_lst = list[torch.Tensor]()
2303
- for video_mm_kwargs_item in filter(
2304
- lambda item: item.modality == "video", mm_kwargs
2305
- ):
2306
- _, _, micro_batch_mm_inputs = next(
2307
- group_mm_kwargs_by_modality(
2308
- [video_mm_kwargs_item],
2309
- device=self.device,
2310
- pin_memory=self.pin_memory,
2317
+ for video_idx in range(num_items):
2318
+ video_mm_kwargs_item = mm_kwargs[current_item_idx + video_idx]
2319
+ with self.timed_encoder_operation(
2320
+ should_time, mm_lora_refs, current_item_idx + video_idx, 1
2321
+ ):
2322
+ _, _, micro_batch_mm_inputs = next(
2323
+ group_mm_kwargs_by_modality(
2324
+ [video_mm_kwargs_item],
2325
+ device=self.device,
2326
+ pin_memory=self.pin_memory,
2327
+ )
2328
)
2312
- )
2329
2314
- micro_batch_outputs = model.embed_multimodal(
2315
- **micro_batch_mm_inputs
2316
- )
2330
+ micro_batch_outputs = model.embed_multimodal(
2331
+ **micro_batch_mm_inputs
2332
+ )
2333
2318
- curr_group_outputs_lst.extend(micro_batch_outputs)
2334
+ curr_group_outputs_lst.extend(micro_batch_outputs)
2335
2336
curr_group_outputs = curr_group_outputs_lst
2337
else:
@@ -2326,7 +2342,11 @@ class GPUModelRunner(
2342
# 2. A list or tuple (length: num_items) of tensors,
2343
# each of shape (feature_size, hidden_size) in case the feature
2344
# size is dynamic depending on the input multimodal items.
2329
- curr_group_outputs = model.embed_multimodal(**mm_kwargs_group)
2345
+
2346
+ with self.timed_encoder_operation(
2347
+ should_time, mm_lora_refs, current_item_idx, num_items
2348
+ ):
2349
+ curr_group_outputs = model.embed_multimodal(**mm_kwargs_group)
2350
2351
sanity_check_mm_encoder_outputs(
2352
curr_group_outputs,
@@ -2334,6 +2354,8 @@ class GPUModelRunner(
2354
)
2355
encoder_outputs.extend(curr_group_outputs)
2356
2357
+ current_item_idx += num_items
2358
+
2359
# Cache the encoder outputs by mm_hash
2360
for mm_hash, output in zip(mm_hashes, encoder_outputs):
2361
self.encoder_cache[mm_hash] = output
@@ -5919,3 +5941,79 @@ class GPUModelRunner(
5941
self.transfer_event.record()
5942
self.transfer_event.synchronize()
5943
return pinned.tolist()
5944
+
5945
+ def get_encoder_timing_stats(self) -> dict[str, dict[str, float | int]]:
5946
+ """
5947
+ Get encoder timing stats for all requests and clear the registry.
5948
+
5949
+ Returns:
5950
+ Dictionary mapping request_id to stats dict.
5951
+ """
5952
+ with self._encoder_timing_lock:
5953
+ stats = {
5954
+ req_id: stats_obj.to_dict()
5955
+ for req_id, stats_obj in self.encoder_timing_registry.items()
5956
+ }
5957
+ self.encoder_timing_registry.clear()
5958
+ return stats
5959
+
5960
+ @contextmanager
5961
+ def timed_encoder_operation(
5962
+ self,
5963
+ should_time: bool,
5964
+ group_lora_refs: list[tuple[str, Any]],
5965
+ current_item_idx: int,
5966
+ num_items: int,
5967
+ ):
5968
+ """
5969
+ Context manager to time encoder forward operations.
5970
+
5971
+ Args:
5972
+ should_time: Whether timing is enabled
5973
+ group_lora_refs: Full list of (request_id, pos_info) tuples
5974
+ current_item_idx: Starting index for this group
5975
+ num_items: Number of items in this group
5976
+ """
5977
+ if not should_time:
5978
+ yield
5979
+ return
5980
+
5981
+ group_refs = group_lora_refs[current_item_idx : current_item_idx + num_items]
5982
+ group_request_ids = {req_id for req_id, _ in group_refs}
5983
+
5984
+ torch.cuda.synchronize()
5985
+ start_time = time.perf_counter()
5986
+
5987
+ try:
5988
+ yield
5989
+ finally:
5990
+ torch.cuda.synchronize()
5991
+ elapsed = time.perf_counter() - start_time
5992
+
5993
+ per_request_time = elapsed / max(len(group_request_ids), 1)
5994
+
5995
+ with self._encoder_timing_lock:
5996
+ for req_id in group_request_ids:
5997
+ if req_id not in self.encoder_timing_registry:
5998
+ self.encoder_timing_registry[req_id] = EncoderTimingStats()
5999
+
6000
+ stats = self.encoder_timing_registry[req_id]
6001
+ stats.encoder_forward_time += per_request_time
6002
+ stats.num_encoder_calls += 1
6003
+
6004
+
6005
+@dataclass
6006
+class EncoderTimingStats:
6007
+ """Per-request timing statistics for encoder forward pass."""
6008
+
6009
+ encoder_forward_time: float = 0.0
6010
+ """Time spent in vision encoder forward pass (seconds)."""
6011
+
6012
+ num_encoder_calls: int = 0
6013
+ """Number of times encoder was called for this request."""
6014
+
6015
+ def to_dict(self) -> dict[str, float | int]:
6016
+ return {
6017
+ "encoder_forward_time": self.encoder_forward_time,
6018
+ "num_encoder_calls": self.num_encoder_calls,
6019
+ }
vllm/v1/worker/gpu_worker.py
+4
@@ -542,6 +542,10 @@ class Worker(WorkerBase):
542
def get_supported_tasks(self) -> tuple[SupportedTask, ...]:
543
return self.model_runner.get_supported_tasks()
544
545
+ def get_encoder_timing_stats(self) -> dict[str, dict[str, float | int]]:
546
+ """Get encoder timing stats from model runner."""
547
+ return self.model_runner.get_encoder_timing_stats()
548
+
549
def annotate_profile(self, scheduler_output):
550
# add trace annotation so that we can easily distinguish
551
# context/generation request numbers in each iteration.