A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Model] Merge `SupportsMultiModalWithRawInput` with `SupportsMultiModal` (#23749)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Aug 28, 2025 at 01:01 UTC
52883ed08461943ff55d5dd3cf12a28c00902fa7
5 files changed
+30
-50
vllm/config/__init__.py
+4
-4
@@ -1698,6 +1698,10 @@ class ModelConfig:
1698
def is_multimodal_model(self) -> bool:
1699
return self.multimodal_config is not None
1700
1701
+ @property
1702
+ def is_multimodal_raw_input_only_model(self) -> bool:
1703
+ return self._model_info.supports_multimodal_raw_input_only
1704
+
1705
@property
1706
def is_cross_encoder(self) -> bool:
1707
return (self._model_info.supports_cross_encoding
@@ -1707,10 +1711,6 @@ class ModelConfig:
1711
def is_pp_supported(self) -> bool:
1712
return self._model_info.supports_pp
1713
1710
- @property
1711
- def is_multimodal_raw_input_supported(self) -> bool:
1712
- return self._model_info.supports_multimodal_raw_input
1713
-
1714
@property
1715
def is_attention_free(self) -> bool:
1716
return self._model_info.is_attention_free
vllm/model_executor/models/interfaces.py
+11
-34
@@ -52,6 +52,12 @@ class SupportsMultiModal(Protocol):
52
MRO of your model class.
53
"""
54
55
+ supports_multimodal_raw_input_only: ClassVar[bool] = False
56
+ """
57
+ A flag that indicates this model supports multi-modal inputs and processes
58
+ them in their raw form and not embeddings.
59
+ """
60
+
61
supports_encoder_tp_data: ClassVar[bool] = False
62
"""
63
A flag that indicates whether this model supports
@@ -143,43 +149,14 @@ def supports_multimodal(
149
return getattr(model, "supports_multimodal", False)
150
151
146
-def supports_multimodal_encoder_tp_data(
152
+def supports_multimodal_raw_input_only(
153
model: Union[type[object], object]) -> bool:
148
- return getattr(model, "supports_encoder_tp_data", False)
149
-
150
-
151
-@runtime_checkable
152
-class SupportsMultiModalWithRawInput(SupportsMultiModal, Protocol):
153
- """The interface required for all multi-modal models."""
154
-
155
- supports_multimodal_raw_input: ClassVar[Literal[True]] = True
156
- """
157
- A flag that indicates this model supports multi-modal inputs and processes
158
- them in their raw form and not embeddings.
159
-
160
- Note:
161
- There is no need to redefine this flag if this class is in the
162
- MRO of your model class.
163
- """
164
-
165
-
166
-@overload
167
-def supports_multimodal_raw_input(
168
- model: object) -> TypeIs[SupportsMultiModalWithRawInput]:
169
- ...
154
+ return getattr(model, "supports_multimodal_raw_input_only", False)
155
156
172
-@overload
173
-def supports_multimodal_raw_input(
174
- model: type[object]) -> TypeIs[type[SupportsMultiModalWithRawInput]]:
175
- ...
176
-
177
-
178
-def supports_multimodal_raw_input(
179
- model: Union[type[object], object]
180
-) -> Union[TypeIs[type[SupportsMultiModalWithRawInput]],
181
- TypeIs[SupportsMultiModalWithRawInput]]:
182
- return getattr(model, "supports_multimodal_raw_input", False)
157
+def supports_multimodal_encoder_tp_data(
158
+ model: Union[type[object], object]) -> bool:
159
+ return getattr(model, "supports_encoder_tp_data", False)
160
161
162
@runtime_checkable
vllm/model_executor/models/prithvi_geospatial_mae.py
+3
-3
@@ -41,7 +41,7 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder
41
from vllm.sequence import IntermediateTensors
42
43
from .interfaces import (IsAttentionFree, MultiModalEmbeddings,
44
- SupportsMultiModalWithRawInput)
44
+ SupportsMultiModal)
45
from .interfaces_base import default_pooling_type
46
47
@@ -174,10 +174,10 @@ class PrithviGeoSpatialMAEMultiModalProcessor(BaseMultiModalProcessor):
174
info=PrithviGeoSpatialMAEProcessingInfo,
175
dummy_inputs=PrithviGeoSpatialMAEInputBuilder,
176
)
177
-class PrithviGeoSpatialMAE(nn.Module, IsAttentionFree,
178
- SupportsMultiModalWithRawInput):
177
+class PrithviGeoSpatialMAE(nn.Module, IsAttentionFree, SupportsMultiModal):
178
"""Prithvi Masked Autoencoder"""
179
180
+ supports_multimodal_raw_input_only = True
181
is_pooling_model = True
182
183
@classmethod
vllm/model_executor/models/registry.py
+6
-5
@@ -29,7 +29,7 @@ from .interfaces import (has_inner_state, has_noops, is_attention_free,
29
is_hybrid, supports_cross_encoding,
30
supports_multimodal,
31
supports_multimodal_encoder_tp_data,
32
- supports_multimodal_raw_input, supports_pp,
32
+ supports_multimodal_raw_input_only, supports_pp,
33
supports_transcription, supports_v0_only)
34
from .interfaces_base import (get_default_pooling_type, is_pooling_model,
35
is_text_generation_model)
@@ -326,7 +326,7 @@ class _ModelInfo:
326
default_pooling_type: str
327
supports_cross_encoding: bool
328
supports_multimodal: bool
329
- supports_multimodal_raw_input: bool
329
+ supports_multimodal_raw_input_only: bool
330
supports_multimodal_encoder_tp_data: bool
331
supports_pp: bool
332
has_inner_state: bool
@@ -346,7 +346,8 @@ class _ModelInfo:
346
default_pooling_type=get_default_pooling_type(model),
347
supports_cross_encoding=supports_cross_encoding(model),
348
supports_multimodal=supports_multimodal(model),
349
- supports_multimodal_raw_input=supports_multimodal_raw_input(model),
349
+ supports_multimodal_raw_input_only=
350
+ supports_multimodal_raw_input_only(model),
351
supports_multimodal_encoder_tp_data=
352
supports_multimodal_encoder_tp_data(model),
353
supports_pp=supports_pp(model),
@@ -743,13 +744,13 @@ class _ModelRegistry:
744
model_cls, _ = self.inspect_model_cls(architectures, model_config)
745
return model_cls.supports_multimodal
746
746
- def supports_multimodal_raw_input(
747
+ def is_multimodal_raw_input_only_model(
748
self,
749
architectures: Union[str, list[str]],
750
model_config: ModelConfig,
751
) -> bool:
752
model_cls, _ = self.inspect_model_cls(architectures, model_config)
752
- return model_cls.supports_multimodal_raw_input
753
+ return model_cls.supports_multimodal_raw_input_only
754
755
def is_pp_supported_model(
756
self,
vllm/v1/worker/gpu_model_runner.py
+6
-4
@@ -139,8 +139,9 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
139
cache_config.cache_dtype]
140
141
self.is_pooling_model = model_config.pooler_config is not None
142
- self.is_multimodal_raw_input_supported = (
143
- model_config.is_multimodal_raw_input_supported)
142
+ self.is_multimodal_raw_input_only_model = (
143
+ model_config.is_multimodal_raw_input_only_model)
144
+
145
self.max_model_len = model_config.max_model_len
146
self.max_num_tokens = scheduler_config.max_num_batched_tokens
147
self.max_num_reqs = scheduler_config.max_num_seqs
@@ -612,7 +613,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
613
self,
614
scheduler_output: "SchedulerOutput",
615
) -> BatchedTensorInputs:
615
- if not self.is_multimodal_raw_input_supported or not scheduler_output: # noqa: SIM102
616
+ if not scheduler_output or not self.is_multimodal_raw_input_only_model:
617
return {}
618
619
mm_kwargs = list[MultiModalKwargsItem]()
@@ -631,8 +632,9 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
632
return mm_kwargs_combined
633
634
def _dummy_mm_kwargs(self, num_seqs: int) -> BatchedTensorInputs:
634
- if not self.is_multimodal_raw_input_supported:
635
+ if not self.is_multimodal_raw_input_only_model:
636
return {}
637
+
638
mm_budget = self.mm_budget
639
assert mm_budget is not None
640