A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 04:00 UTC
[Refactor] Move top-level dummy data generation to registry (#32310)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 14, 2026 at 18:17 UTC
90db5b31e4eeb8fc8bfb3260791bf795fda4f633
6 files changed
+57
-132
tests/models/multimodal/processing/test_mllama4.py
+5
-18
@@ -24,32 +24,20 @@ def test_profiling(model_id: str, max_model_len: int):
24
limit_mm_per_prompt=mm_counts,
25
)
26
27
- processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
28
- decoder_dummy_data = processor.dummy_inputs.get_decoder_dummy_data(
29
- processor,
30
- max_model_len,
31
- mm_counts=mm_counts,
32
- )
33
- dummy_mm_data = processor.dummy_inputs.get_dummy_processor_inputs(
34
- max_model_len,
27
+ mm_inputs = MULTIMODAL_REGISTRY.get_dummy_mm_inputs(
28
+ ctx.model_config,
29
mm_counts=mm_counts,
30
)
31
32
hf_config = ctx.get_hf_config(Llama4Config)
39
-
40
- mm_inputs = processor.apply(
41
- prompt=dummy_mm_data.prompt,
42
- mm_data=dummy_mm_data.mm_data,
43
- hf_processor_mm_kwargs=dict(),
44
- )
45
- mm_data = mm_inputs["mm_kwargs"].get_data()
46
-
33
image_size = hf_config.vision_config.image_size
34
patch_size = hf_config.vision_config.patch_size
35
downsample_ratio = int(
36
round(1.0 / (hf_config.vision_config.pixel_shuffle_ratio**2))
37
)
38
tokens_per_patch = ((image_size // patch_size) ** 2) // downsample_ratio
39
+
40
+ mm_data = mm_inputs["mm_kwargs"].get_data()
41
chunks_per_image = prod(mm_data["patches_per_image"])
42
total_num_patches = chunks_per_image * tokens_per_patch
43
num_tiles = (
@@ -63,6 +51,5 @@ def test_profiling(model_id: str, max_model_len: int):
51
item.get_num_embeds for item in mm_inputs["mm_placeholders"]["image"]
52
)
53
assert total_tokens == sum(
66
- placeholder.length
67
- for placeholder in decoder_dummy_data.multi_modal_placeholders["image"]
54
+ placeholder.length for placeholder in mm_inputs["mm_placeholders"]["image"]
55
)
tests/multimodal/test_processing.py
+3
-3
@@ -926,10 +926,10 @@ def test_limit_mm_per_prompt_dummy(model_id, limit, num_supported, is_valid):
926
exc_ctx = nullcontext() if is_valid else pytest.raises(ValueError, match="At most")
927
928
with exc_ctx:
929
- processor.dummy_inputs.get_decoder_dummy_data(
930
- processor,
931
- model_config.max_model_len,
929
+ MULTIMODAL_REGISTRY.get_dummy_mm_inputs(
930
+ model_config,
931
mm_counts=limit_mm_per_prompt,
932
+ processor=processor,
933
)
934
935
vllm/multimodal/processing.py
+1
-1
@@ -50,6 +50,7 @@ from .parse import (
50
MultiModalDataItems,
51
MultiModalDataParser,
52
)
53
+from .profiling import BaseDummyInputsBuilder
54
55
if TYPE_CHECKING:
56
from transformers.configuration_utils import PretrainedConfig
@@ -59,7 +60,6 @@ if TYPE_CHECKING:
60
from vllm.config import ModelConfig, ObservabilityConfig
61
62
from .cache import BaseMultiModalProcessorCache
62
- from .profiling import BaseDummyInputsBuilder
63
else:
64
PretrainedConfig = object
65
BatchFeature = object
vllm/multimodal/profiling.py
+9
-77
@@ -3,7 +3,7 @@
3
from abc import ABC, abstractmethod
4
from collections.abc import Mapping
5
from dataclasses import dataclass, field
6
-from typing import Generic, NamedTuple, TypeVar
6
+from typing import TYPE_CHECKING, Generic
7
8
import numpy as np
9
import numpy.typing as npt
@@ -17,16 +17,14 @@ from vllm.config.multimodal import (
17
)
18
from vllm.logger import init_logger
19
20
-from .inputs import (
21
- MultiModalDataDict,
22
- MultiModalInputs,
23
- MultiModalKwargsItems,
24
- MultiModalPlaceholderDict,
25
-)
26
-from .processing import (
27
- BaseMultiModalProcessor,
28
- BaseProcessingInfo,
29
-)
20
+from .inputs import MultiModalDataDict
21
+
22
+if TYPE_CHECKING:
23
+ from .processing import _I
24
+else:
25
+ from typing import TypeVar
26
+
27
+ _I = TypeVar("_I")
28
29
logger = init_logger(__name__)
30
@@ -44,23 +42,6 @@ class ProcessorInputs:
42
tokenization_kwargs: Mapping[str, object] = field(default_factory=dict)
43
44
47
-class DummyEncoderData(NamedTuple):
48
- """Dummy data used for profiling."""
49
-
50
- prompt_token_ids: list[int]
51
-
52
-
53
-class DummyDecoderData(NamedTuple):
54
- """Dummy data used for profiling."""
55
-
56
- prompt_token_ids: list[int]
57
- multi_modal_data: MultiModalKwargsItems
58
- multi_modal_placeholders: MultiModalPlaceholderDict
59
-
60
-
61
-_I = TypeVar("_I", bound=BaseProcessingInfo)
62
-
63
-
45
class BaseDummyInputsBuilder(ABC, Generic[_I]):
46
"""
47
Abstract base class that constructs the dummy data to profile
@@ -222,52 +203,3 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
203
height = min(height, overrides.height)
204
video = np.full((num_frames, width, height, 3), 255, dtype=np.uint8)
205
return [video] * num_videos
225
-
226
- def get_dummy_mm_inputs(
227
- self,
228
- processor: BaseMultiModalProcessor[_I],
229
- seq_len: int,
230
- mm_counts: Mapping[str, int] | None = None,
231
- mm_options: Mapping[str, BaseDummyOptions] | None = None,
232
- ) -> MultiModalInputs:
233
- if mm_counts is None:
234
- mm_counts = processor.allowed_mm_limits
235
-
236
- processor_inputs = self.get_dummy_processor_inputs(
237
- seq_len,
238
- mm_counts=mm_counts,
239
- mm_options=mm_options,
240
- )
241
-
242
- return processor.apply(
243
- prompt=processor_inputs.prompt,
244
- mm_data=processor_inputs.mm_data,
245
- hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
246
- tokenization_kwargs=processor_inputs.tokenization_kwargs,
247
- )
248
-
249
- def get_decoder_dummy_data(
250
- self,
251
- processor: BaseMultiModalProcessor[_I],
252
- seq_len: int,
253
- mm_counts: Mapping[str, int] | None = None,
254
- mm_options: Mapping[str, BaseDummyOptions] | None = None,
255
- ) -> DummyDecoderData:
256
- mm_inputs = self.get_dummy_mm_inputs(
257
- processor,
258
- seq_len,
259
- mm_counts=mm_counts,
260
- mm_options=mm_options,
261
- )
262
-
263
- prompt_token_ids = mm_inputs["prompt_token_ids"]
264
- total_len = len(prompt_token_ids)
265
-
266
- if total_len < seq_len:
267
- prompt_token_ids.extend([0] * (seq_len - total_len))
268
-
269
- return DummyDecoderData(
270
- prompt_token_ids=prompt_token_ids,
271
- multi_modal_data=mm_inputs["mm_kwargs"].require_data(),
272
- multi_modal_placeholders=mm_inputs["mm_placeholders"],
273
- )
vllm/multimodal/registry.py
+31
-27
@@ -10,15 +10,13 @@ from vllm.logger import init_logger
10
from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
11
12
from .cache import BaseMultiModalProcessorCache
13
+from .inputs import MultiModalInputs
14
from .processing import (
15
BaseMultiModalProcessor,
16
BaseProcessingInfo,
17
InputProcessingContext,
18
)
18
-from .profiling import (
19
- BaseDummyInputsBuilder,
20
- DummyDecoderData,
21
-)
19
+from .profiling import BaseDummyInputsBuilder
20
21
if TYPE_CHECKING:
22
from vllm.config import ModelConfig, ObservabilityConfig
@@ -160,7 +158,6 @@ class MultiModalRegistry:
158
model_config, observability_config, cache=cache
159
)
160
163
- seq_len = model_config.max_model_len
161
if profiler_limits is None:
162
profiler_limits = processor.allowed_mm_limits
163
@@ -169,7 +166,7 @@ class MultiModalRegistry:
166
}
167
168
max_tokens_per_item = processor.info.get_mm_max_tokens_per_item(
172
- seq_len=seq_len,
169
+ seq_len=model_config.max_model_len,
170
mm_counts=mm_counts,
171
)
172
if max_tokens_per_item is not None:
@@ -179,11 +176,10 @@ class MultiModalRegistry:
176
if mm_counts.get(modality, 0) > 0
177
}
178
182
- mm_inputs = processor.dummy_inputs.get_dummy_mm_inputs(
183
- processor,
184
- seq_len,
179
+ mm_inputs = self.get_dummy_mm_inputs(
180
+ model_config,
181
mm_counts=mm_counts,
186
- mm_options=self._extract_mm_options(model_config),
182
+ processor=processor,
183
)
184
185
return {
@@ -298,39 +294,47 @@ class MultiModalRegistry:
294
295
return factories.build_processor(ctx, cache=cache)
296
301
- def get_decoder_dummy_data(
297
+ def get_dummy_mm_inputs(
298
self,
299
model_config: "ModelConfig",
304
- seq_len: int,
300
mm_counts: Mapping[str, int] | None = None,
301
*,
302
cache: BaseMultiModalProcessorCache | None = None,
303
observability_config: ObservabilityConfig | None = None,
309
- ) -> DummyDecoderData:
304
+ processor: BaseMultiModalProcessor | None = None,
305
+ ) -> MultiModalInputs:
306
"""
307
Create dummy data for profiling the memory usage of a model.
308
309
The model is identified by `model_config`.
310
"""
315
- processor = self.create_processor(
316
- model_config, observability_config, cache=cache
317
- )
318
- dummy_data = processor.dummy_inputs.get_decoder_dummy_data(
319
- processor,
320
- seq_len,
311
+ seq_len = model_config.max_model_len
312
+
313
+ if processor is None:
314
+ processor = self.create_processor(
315
+ model_config, observability_config, cache=cache
316
+ )
317
+ if mm_counts is None:
318
+ mm_counts = processor.allowed_mm_limits
319
+
320
+ processor_inputs = processor.dummy_inputs.get_dummy_processor_inputs(
321
+ seq_len=seq_len,
322
mm_counts=mm_counts,
323
mm_options=self._extract_mm_options(model_config),
324
)
325
+ mm_inputs = processor.apply(
326
+ prompt=processor_inputs.prompt,
327
+ mm_data=processor_inputs.mm_data,
328
+ hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
329
+ tokenization_kwargs=processor_inputs.tokenization_kwargs,
330
+ )
331
325
- # Having more tokens is over-conservative but otherwise fine
326
- token_ids = dummy_data.prompt_token_ids
327
- if len(token_ids) < seq_len:
328
- raise AssertionError(
329
- f"Expected at least {seq_len} dummy tokens for profiling, "
330
- f"but found {len(token_ids)} tokens instead."
331
- )
332
+ prompt_token_ids = mm_inputs["prompt_token_ids"]
333
+ total_len = len(prompt_token_ids)
334
+ if total_len < seq_len:
335
+ prompt_token_ids.extend([0] * (seq_len - total_len))
336
333
- return dummy_data
337
+ return mm_inputs
338
339
def get_encdec_max_encoder_len(self, model_config: "ModelConfig") -> int:
340
"""
vllm/v1/worker/gpu_model_runner.py
+8
-6
@@ -4192,16 +4192,18 @@ class GPUModelRunner(
4192
"""Dummy data for profiling and precompiling multimodal models."""
4193
assert self.mm_budget is not None
4194
4195
- dummy_decoder_data = self.mm_registry.get_decoder_dummy_data(
4196
- model_config=self.model_config,
4197
- seq_len=self.max_model_len,
4195
+ # Don't use `max_items_per_batch` here to avoid redundant computation
4196
+ dummy_mm_inputs = self.mm_registry.get_dummy_mm_inputs(
4197
+ self.model_config,
4198
mm_counts={modality: 1},
4199
cache=self.mm_budget.cache,
4200
)
4201
- dummy_mm_data = dummy_decoder_data.multi_modal_data
4201
+ dummy_mm_item = dummy_mm_inputs["mm_kwargs"][modality][0]
4202
+
4203
+ # We use the cache so that the item is saved to the cache,
4204
+ # but not read from the cache
4205
+ assert dummy_mm_item is not None, "Item should not already be cached"
4206
4203
- # Result in the maximum GPU consumption of the model
4204
- dummy_mm_item = dummy_mm_data[modality][0]
4207
dummy_mm_items = [dummy_mm_item] * max_items_per_batch
4208
4209
return next(