A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 05:15 UTC
[Refactor] Remove `MultiModalProfiler` (#32254)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 13, 2026 at 23:10 UTC
252c011012658490c2e970c5bc739a84bf5759c0
4 files changed
+56
-100
tests/models/multimodal/processing/test_mllama4.py
+7
-11
@@ -7,7 +7,6 @@ from torch import prod
7
from transformers import Llama4Config
8
9
from vllm.multimodal import MULTIMODAL_REGISTRY
10
-from vllm.multimodal.profiling import MultiModalProfiler
10
11
from ...utils import build_model_context
12
@@ -26,9 +25,8 @@ def test_profiling(model_id: str, max_model_len: int):
25
)
26
27
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
29
- profiler = MultiModalProfiler(processor)
30
-
31
- decoder_dummy_data = profiler.get_decoder_dummy_data(
28
+ decoder_dummy_data = processor.dummy_inputs.get_decoder_dummy_data(
29
+ processor,
30
max_model_len,
31
mm_counts=mm_counts,
32
)
@@ -39,11 +37,12 @@ def test_profiling(model_id: str, max_model_len: int):
37
38
hf_config = ctx.get_hf_config(Llama4Config)
39
42
- mm_data = processor.apply(
40
+ mm_inputs = processor.apply(
41
prompt=dummy_mm_data.prompt,
42
mm_data=dummy_mm_data.mm_data,
43
hf_processor_mm_kwargs=dict(),
46
- )["mm_kwargs"].get_data()
44
+ )
45
+ mm_data = mm_inputs["mm_kwargs"].get_data()
46
47
image_size = hf_config.vision_config.image_size
48
patch_size = hf_config.vision_config.patch_size
@@ -60,12 +59,9 @@ def test_profiling(model_id: str, max_model_len: int):
59
total_num_patches.item() + num_tiles.item() + 3
60
) # image start, image, image end
61
63
- profiled_tokens = profiler.get_mm_max_tokens(
64
- max_model_len,
65
- mm_counts=mm_counts,
62
+ assert total_num_patches == sum(
63
+ item.get_num_embeds for item in mm_inputs["mm_placeholders"]["image"]
64
)
67
-
68
- assert total_num_patches == profiled_tokens["image"]
65
assert total_tokens == sum(
66
placeholder.length
67
for placeholder in decoder_dummy_data.multi_modal_placeholders["image"]
tests/multimodal/test_processing.py
+2
-4
@@ -22,7 +22,6 @@ from vllm.multimodal.processing import (
22
iter_token_matches,
23
replace_token_matches,
24
)
25
-from vllm.multimodal.profiling import MultiModalProfiler
25
26
from .utils import random_image
27
@@ -924,12 +923,11 @@ def test_limit_mm_per_prompt_dummy(model_id, limit, num_supported, is_valid):
923
processor = MULTIMODAL_REGISTRY.create_processor(model_config)
924
processor._supported_mm_limits = {"image": num_supported}
925
927
- profiler = MultiModalProfiler(processor)
928
-
926
exc_ctx = nullcontext() if is_valid else pytest.raises(ValueError, match="At most")
927
928
with exc_ctx:
932
- profiler.get_decoder_dummy_data(
929
+ processor.dummy_inputs.get_decoder_dummy_data(
930
+ processor,
931
model_config.max_model_len,
932
mm_counts=limit_mm_per_prompt,
933
)
vllm/multimodal/profiling.py
+14
-68
@@ -223,70 +223,42 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
223
video = np.full((num_frames, width, height, 3), 255, dtype=np.uint8)
224
return [video] * num_videos
225
226
-
227
-class MultiModalProfiler(Generic[_I]):
228
- """
229
- Contains code for running memory profiling for multi-modal models.
230
- """
231
-
232
- def __init__(
226
+ def get_dummy_mm_inputs(
227
self,
228
processor: BaseMultiModalProcessor[_I],
235
- ) -> None:
236
- super().__init__()
237
-
238
- self.processor = processor
239
-
240
- @property
241
- def processing_info(self) -> BaseProcessingInfo:
242
- return self.processor.info
243
-
244
- @property
245
- def dummy_inputs(self) -> BaseDummyInputsBuilder[_I]:
246
- return self.processor.dummy_inputs
247
-
248
- def get_mm_limits(self) -> Mapping[str, int]:
249
- return self.processor.allowed_mm_limits
250
-
251
- def _get_dummy_mm_inputs(
252
- self,
229
seq_len: int,
230
mm_counts: Mapping[str, int] | None = None,
231
mm_options: Mapping[str, BaseDummyOptions] | None = None,
232
) -> MultiModalInputs:
233
if mm_counts is None:
258
- mm_counts = self.get_mm_limits()
234
+ mm_counts = processor.allowed_mm_limits
235
260
- factory = self.dummy_inputs
261
- processor_inputs = factory.get_dummy_processor_inputs(
262
- seq_len, mm_counts, mm_options
236
+ processor_inputs = self.get_dummy_processor_inputs(
237
+ seq_len,
238
+ mm_counts=mm_counts,
239
+ mm_options=mm_options,
240
)
241
265
- return self.processor.apply(
242
+ return processor.apply(
243
prompt=processor_inputs.prompt,
244
mm_data=processor_inputs.mm_data,
245
hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
246
tokenization_kwargs=processor_inputs.tokenization_kwargs,
247
)
248
272
- def _get_mm_num_tokens(
273
- self,
274
- mm_inputs: MultiModalInputs,
275
- ) -> Mapping[str, int]:
276
- placeholders_by_modality = mm_inputs["mm_placeholders"]
277
-
278
- return {
279
- modality: sum(item.get_num_embeds for item in placeholders)
280
- for modality, placeholders in placeholders_by_modality.items()
281
- }
282
-
249
def get_decoder_dummy_data(
250
self,
251
+ processor: BaseMultiModalProcessor[_I],
252
seq_len: int,
253
mm_counts: Mapping[str, int] | None = None,
254
mm_options: Mapping[str, BaseDummyOptions] | None = None,
255
) -> DummyDecoderData:
289
- mm_inputs = self._get_dummy_mm_inputs(seq_len, mm_counts, mm_options)
256
+ mm_inputs = self.get_dummy_mm_inputs(
257
+ processor,
258
+ seq_len,
259
+ mm_counts=mm_counts,
260
+ mm_options=mm_options,
261
+ )
262
263
prompt_token_ids = mm_inputs["prompt_token_ids"]
264
total_len = len(prompt_token_ids)
@@ -299,29 +271,3 @@ class MultiModalProfiler(Generic[_I]):
271
multi_modal_data=mm_inputs["mm_kwargs"].require_data(),
272
multi_modal_placeholders=mm_inputs["mm_placeholders"],
273
)
302
-
303
- def get_mm_max_tokens(
304
- self,
305
- seq_len: int,
306
- mm_counts: Mapping[str, int] | None = None,
307
- ) -> Mapping[str, int]:
308
- """
309
- Returns the maximum number of embeddings per item of each modality, excluding
310
- any break/text tokens in-between multimodal embeddings/encoder outputs.
311
- """
312
- if mm_counts is None:
313
- mm_counts = self.get_mm_limits()
314
-
315
- max_tokens_per_item = self.processing_info.get_mm_max_tokens_per_item(
316
- seq_len=seq_len,
317
- mm_counts=mm_counts,
318
- )
319
- if max_tokens_per_item is not None:
320
- return {
321
- modality: max_tokens
322
- for modality, max_tokens in max_tokens_per_item.items()
323
- if mm_counts.get(modality, 0) > 0
324
- }
325
-
326
- mm_inputs = self._get_dummy_mm_inputs(seq_len, mm_counts)
327
- return self._get_mm_num_tokens(mm_inputs)
vllm/multimodal/registry.py
+33
-17
@@ -18,7 +18,6 @@ from .processing import (
18
from .profiling import (
19
BaseDummyInputsBuilder,
20
DummyDecoderData,
21
- MultiModalProfiler,
21
)
22
23
if TYPE_CHECKING:
@@ -160,18 +159,38 @@ class MultiModalRegistry:
159
processor = self.create_processor(
160
model_config, observability_config, cache=cache
161
)
163
- profiler: MultiModalProfiler = MultiModalProfiler(processor)
162
163
seq_len = model_config.max_model_len
166
- profiler_limits = (
167
- profiler.get_mm_limits() if profiler_limits is None else profiler_limits
168
- )
164
+ if profiler_limits is None:
165
+ profiler_limits = processor.allowed_mm_limits
166
+
167
+ mm_counts = {
168
+ modality: 1 for modality, limit in profiler_limits.items() if limit > 0
169
+ }
170
170
- return profiler.get_mm_max_tokens(
171
+ max_tokens_per_item = processor.info.get_mm_max_tokens_per_item(
172
+ seq_len=seq_len,
173
+ mm_counts=mm_counts,
174
+ )
175
+ if max_tokens_per_item is not None:
176
+ return {
177
+ modality: max_tokens
178
+ for modality, max_tokens in max_tokens_per_item.items()
179
+ if mm_counts.get(modality, 0) > 0
180
+ }
181
+
182
+ mm_inputs = processor.dummy_inputs.get_dummy_mm_inputs(
183
+ processor,
184
seq_len,
172
- {modality: 1 for modality, limit in profiler_limits.items() if limit > 0},
185
+ mm_counts=mm_counts,
186
+ mm_options=self._extract_mm_options(model_config),
187
)
188
189
+ return {
190
+ modality: sum(item.get_num_embeds for item in placeholders)
191
+ for modality, placeholders in mm_inputs["mm_placeholders"].items()
192
+ }
193
+
194
def get_mm_limits_per_prompt(
195
self,
196
model_config: "ModelConfig",
@@ -189,8 +208,7 @@ class MultiModalRegistry:
208
processor = self.create_processor(
209
model_config, observability_config, cache=cache
210
)
192
- profiler: MultiModalProfiler = MultiModalProfiler(processor)
193
- return profiler.get_mm_limits()
211
+ return processor.allowed_mm_limits
212
213
def register_processor(
214
self,
@@ -297,14 +315,12 @@ class MultiModalRegistry:
315
processor = self.create_processor(
316
model_config, observability_config, cache=cache
317
)
300
- profiler: MultiModalProfiler = MultiModalProfiler(processor)
301
-
302
- # Extract configurable options from multimodal config.
303
- # Only include modalities that use advanced option types so legacy
304
- # count-only behavior remains unchanged.
305
- mm_options = self._extract_mm_options(model_config)
306
-
307
- dummy_data = profiler.get_decoder_dummy_data(seq_len, mm_counts, mm_options)
318
+ dummy_data = processor.dummy_inputs.get_decoder_dummy_data(
319
+ processor,
320
+ seq_len,
321
+ mm_counts=mm_counts,
322
+ mm_options=self._extract_mm_options(model_config),
323
+ )
324
325
# Having more tokens is over-conservative but otherwise fine
326
token_ids = dummy_data.prompt_token_ids