A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 07:30 UTC
[Refactor] Move MM item count validation outside of processor (#33396)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 30, 2026 at 17:27 UTC
c87eac18f7d0169e527631772fd081cc08aad01a
7 files changed
+46
-51
tests/multimodal/test_processing.py
+1
-1
@@ -921,7 +921,7 @@ def test_limit_mm_per_prompt_dummy(model_id, limit, num_supported, is_valid):
921
)
922
923
processor = MULTIMODAL_REGISTRY.create_processor(model_config)
924
- processor._supported_mm_limits = {"image": num_supported}
924
+ processor.info.get_supported_mm_limits = lambda: {"image": num_supported}
925
926
exc_ctx = nullcontext() if is_valid else pytest.raises(ValueError, match="At most")
927
vllm/entrypoints/chat_utils.py
+1
-1
@@ -528,7 +528,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
528
else:
529
num_items = len(self._items_by_modality[original_modality]) + 1
530
531
- self.mm_processor.validate_num_items(input_modality, num_items)
531
+ self.mm_processor.info.validate_num_items(input_modality, num_items)
532
533
# Track original modality for vision_chunk items
534
if use_vision_chunk:
vllm/lora/model_manager.py
+1
-3
@@ -176,9 +176,7 @@ class LoRAModelManager:
176
)
177
178
mm_budget = MultiModalBudget(vllm_config, mm_registry)
179
- limit_per_prompt: int = max(
180
- self.mm_processor_info.get_allowed_mm_limits().values()
181
- )
179
+ limit_per_prompt = max(self.mm_processor_info.allowed_mm_limits.values())
180
num_encoder_tokens = self.model.get_num_mm_encoder_tokens(
181
mm_budget.get_encoder_budget()
182
)
vllm/multimodal/processing/context.py
+32
-9
@@ -7,11 +7,8 @@ from abc import abstractmethod
7
from collections.abc import Generator, Mapping
8
from contextlib import contextmanager
9
from dataclasses import dataclass, field
10
-from typing import (
11
- TYPE_CHECKING,
12
- Any,
13
- overload,
14
-)
10
+from functools import cached_property
11
+from typing import TYPE_CHECKING, Any, overload
12
13
import torch
14
from typing_extensions import TypeVar
@@ -615,13 +612,18 @@ class BaseProcessingInfo:
612
"""
613
raise NotImplementedError
614
618
- def get_allowed_mm_limits(self) -> Mapping[str, int]:
619
- """Return the maximum allowed number of items for each modality."""
620
- supported_mm_limits = self.get_supported_mm_limits()
615
+ @cached_property
616
+ def supported_mm_limits(self) -> Mapping[str, int | None]:
617
+ """The maximum supported number of items for each modality."""
618
+ return self.get_supported_mm_limits()
619
+
620
+ @cached_property
621
+ def allowed_mm_limits(self) -> Mapping[str, int]:
622
+ """The maximum allowed number of items for each modality."""
623
mm_config = self.ctx.get_mm_config()
624
625
allowed_limits = dict[str, int]()
624
- for modality, supported_limit in supported_mm_limits.items():
626
+ for modality, supported_limit in self.supported_mm_limits.items():
627
user_limit = mm_config.get_limit_per_prompt(modality)
628
629
allowed_limits[modality] = (
@@ -632,6 +634,27 @@ class BaseProcessingInfo:
634
635
return allowed_limits
636
637
+ def validate_num_items(self, modality: str, num_items: int) -> None:
638
+ """
639
+ Raise `ValueError` if the number of input items for the given modality
640
+ is invalid.
641
+ """
642
+ supported_limit = self.supported_mm_limits.get(modality, 0)
643
+ allowed_limit = self.allowed_mm_limits.get(modality, 0)
644
+
645
+ if supported_limit is None:
646
+ supported_limit = allowed_limit
647
+
648
+ limit = min(supported_limit, allowed_limit)
649
+
650
+ if num_items > limit:
651
+ msg = f"At most {limit} {modality}(s) may be provided in one prompt."
652
+
653
+ if num_items <= supported_limit:
654
+ msg += " Set `--limit-mm-per-prompt` to increase this limit."
655
+
656
+ raise ValueError(msg)
657
+
658
def get_mm_max_tokens_per_item(
659
self,
660
seq_len: int,
vllm/multimodal/processing/processor.py
+6
-29
@@ -17,7 +17,7 @@ from typing import (
17
18
import regex as re
19
import torch
20
-from typing_extensions import TypeVar, assert_never
20
+from typing_extensions import TypeVar, assert_never, deprecated
21
22
from vllm.logger import init_logger
23
from vllm.tokenizers import TokenizerLike
@@ -1000,17 +1000,15 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1000
else:
1001
self.data_parser = self.info.get_data_parser()
1002
1003
- # Avoid unnecessary recomputation
1004
- self._supported_mm_limits = self.info.get_supported_mm_limits()
1005
- self._allowed_mm_limits = self.info.get_allowed_mm_limits()
1006
-
1003
@property
1004
+ @deprecated("Will be removed in v0.17. Use `info.supported_mm_limits` instead.")
1005
def supported_mm_limits(self):
1009
- return self._supported_mm_limits
1006
+ return self.info.supported_mm_limits
1007
1008
@property
1009
+ @deprecated("Will be removed in v0.17. Use `info.allowed_mm_limits` instead.")
1010
def allowed_mm_limits(self):
1013
- return self._allowed_mm_limits
1011
+ return self.info.allowed_mm_limits
1012
1013
def __call__(
1014
self,
@@ -1022,27 +1020,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1020
) -> MultiModalInputs:
1021
return self.apply(prompt, mm_data, hf_processor_mm_kwargs, mm_uuids=mm_uuids)
1022
1025
- def validate_num_items(
1026
- self,
1027
- modality: str,
1028
- num_items: int,
1029
- ) -> None:
1030
- supported_limit = self.supported_mm_limits.get(modality, 0)
1031
- allowed_limit = self.allowed_mm_limits.get(modality, 0)
1032
-
1033
- if supported_limit is None:
1034
- supported_limit = allowed_limit
1035
-
1036
- limit = min(supported_limit, allowed_limit)
1037
-
1038
- if num_items > limit:
1039
- msg = f"At most {limit} {modality}(s) may be provided in one prompt."
1040
-
1041
- if num_items <= supported_limit:
1042
- msg += " Set `--limit-mm-per-prompt` to increase this limit."
1043
-
1044
- raise ValueError(msg)
1045
-
1023
def _to_mm_items(
1024
self,
1025
mm_data: MultiModalDataDict,
@@ -1066,7 +1043,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1043
)
1044
1045
for modality, items in mm_items.items():
1069
- self.validate_num_items(modality, len(items))
1046
+ self.info.validate_num_items(modality, len(items))
1047
1048
return mm_items
1049
vllm/multimodal/registry.py
+4
-7
@@ -168,7 +168,7 @@ class MultiModalRegistry:
168
)
169
170
if profiler_limits is None:
171
- profiler_limits = processor.allowed_mm_limits
171
+ profiler_limits = processor.info.allowed_mm_limits
172
173
mm_counts = {
174
modality: 1 for modality, limit in profiler_limits.items() if limit > 0
@@ -200,7 +200,6 @@ class MultiModalRegistry:
200
self,
201
model_config: "ModelConfig",
202
*,
203
- cache: BaseMultiModalProcessorCache | None = None,
203
observability_config: ObservabilityConfig | None = None,
204
) -> Mapping[str, int]:
205
"""
@@ -210,10 +209,8 @@ class MultiModalRegistry:
209
if not model_config.is_multimodal_model:
210
return {}
211
213
- processor = self.create_processor(
214
- model_config, observability_config, cache=cache
215
- )
216
- return processor.allowed_mm_limits
212
+ info = self._create_processing_info(model_config, observability_config)
213
+ return info.allowed_mm_limits
214
215
def register_processor(
216
self,
@@ -324,7 +321,7 @@ class MultiModalRegistry:
321
model_config, observability_config, cache=cache
322
)
323
if mm_counts is None:
327
- mm_counts = processor.allowed_mm_limits
324
+ mm_counts = processor.info.allowed_mm_limits
325
326
processor_inputs = processor.dummy_inputs.get_dummy_processor_inputs(
327
seq_len=seq_len,
vllm/v1/worker/utils.py
+1
-1
@@ -40,7 +40,7 @@ class MultiModalBudget:
40
self.max_model_len = model_config.max_model_len
41
self.max_num_reqs = scheduler_config.max_num_seqs
42
43
- self.mm_limits = mm_registry.get_mm_limits_per_prompt(model_config, cache=cache)
43
+ self.mm_limits = mm_registry.get_mm_limits_per_prompt(model_config)
44
45
max_tokens_by_modality = mm_registry.get_max_tokens_per_item_by_modality(
46
model_config,