A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Implicit language-model-only mode via limit-mm-per-prompt (#22299)
Signed-off-by: Roger Wang <hey@rogerw.me> Signed-off-by: Andy Xie <andy.xning@gmail.com> Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com> Signed-off-by: Andrew Sansom <andrew@protopia.ai> Signed-off-by: Zhiyu Cheng <zhiyuc@nvidia.com> Signed-off-by: Shu Wang <shuw@nvidia.com> Signed-off-by: Po-Han Huang <pohanh@nvidia.com> Signed-off-by: Shu Wang. <shuw@nvidia.com> Signed-off-by: XIn Li <xinli@nvidia.com> Signed-off-by: Junhao Li <junhao@ubicloud.com> Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Signed-off-by: zRzRzRzRzRzRzR <2448370773@qq.com> Signed-off-by: zitian.zhao <zitian.zhao@tencentmusic.com> Signed-off-by: zitian zhao <zitian.zhao@tencentmusic.com> Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: iAmir97 <Amir.balwel@embeddedllm.com> Signed-off-by: iAmir97 <71513472+iAmir97@users.noreply.github.com> Signed-off-by: Linkun <github@lkchen.net> Co-authored-by: Ning Xie <andy.xning@gmail.com> Co-authored-by: TJian <tunjian.tan@embeddedllm.com> Co-authored-by: Andrew Sansom <andrew@protopia.ai> Co-authored-by: Zhiyu <zhiyuc@nvidia.com> Co-authored-by: Shu Wang <shuw@nvidia.com> Co-authored-by: XIn Li <xinli@nvidia.com> Co-authored-by: Junhao Li <streaver91@gmail.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com> Co-authored-by: Yuxuan Zhang <2448370773@qq.com> Co-authored-by: ZiTian Zhao <zitian.zhao@tencentmusic.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk> Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com> Co-authored-by: iAmir97 <71513472+iAmir97@users.noreply.github.com> Co-authored-by: iAmir97 <Amir.balwel@embeddedllm.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: Hong Hanh <hanh.usth@gmail.com> Co-authored-by: youkaichao <youkaichao@gmail.com> Co-authored-by: lkchen <github@lkchen.net>
Roger Wang committed
Aug 8, 2025 at 22:21 UTC
08b751ba749541259e5450d6371d822fdf769b8a
16 files changed
+271
-116
tests/multimodal/test_registry.py
new
+38
@@ -0,0 +1,38 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""
4
+Unit tests for MultiModalRegistry.supports_multimodal_inputs and
5
+Qwen2.5-VL visual component loading behavior.
6
+"""
7
+
8
+import pytest
9
+
10
+from vllm.multimodal import MULTIMODAL_REGISTRY
11
+
12
+from ..models.utils import build_model_context
13
+
14
+
15
+@pytest.mark.parametrize(
16
+ "model_id,limit_mm_per_prompt,expected",
17
+ [
18
+ ("Qwen/Qwen2-0.5B-Instruct", {}, False),
19
+ ("Qwen/Qwen2.5-VL-3B-Instruct", {}, True),
20
+ ("Qwen/Qwen2.5-VL-3B-Instruct", {
21
+ "image": 0,
22
+ "video": 0
23
+ }, False),
24
+ ("Qwen/Qwen2.5-VL-3B-Instruct", {
25
+ "image": 0
26
+ }, True),
27
+ ],
28
+)
29
+@pytest.mark.core_model
30
+def test_supports_multimodal_inputs(model_id, limit_mm_per_prompt, expected):
31
+ """Test supports_multimodal_inputs returns correct boolean for various
32
+ configs."""
33
+ ctx = build_model_context(
34
+ model_id,
35
+ limit_mm_per_prompt=limit_mm_per_prompt,
36
+ )
37
+ assert MULTIMODAL_REGISTRY.supports_multimodal_inputs(
38
+ ctx.model_config) is expected
\ No newline at end of file
vllm/config/__init__.py
-9
@@ -1695,15 +1695,6 @@ class ModelConfig:
1695
1696
return mm_config.mm_processor_cache_gb > 0
1697
1698
- @property
1699
- def enable_mm_input_cache(self) -> bool:
1700
- """Whether the multi-modal input cache should be enabled."""
1701
- mm_config = self.multimodal_config
1702
- if mm_config is None:
1703
- return False
1704
-
1705
- return mm_config.mm_processor_cache_gb > 0
1706
-
1698
def get_mm_input_cache_gb(self) -> int:
1699
mm_config = self.multimodal_config
1700
if mm_config is None:
vllm/model_executor/models/llava.py
+21
-13
@@ -521,18 +521,22 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
521
config.projector_hidden_act = "gelu"
522
523
# TODO: Optionally initializes this for supporting embeddings.
524
- self.vision_tower = init_vision_tower_for_llava(
525
- config,
526
- quant_config,
527
- require_post_norm=False,
528
- prefix=maybe_prefix(prefix, "vision_tower"))
529
- self.multi_modal_projector = LlavaMultiModalProjector(
530
- vision_hidden_size=config.vision_config.hidden_size,
531
- text_hidden_size=config.text_config.hidden_size,
532
- projector_hidden_act=config.projector_hidden_act,
533
- multimodal_projector_bias=config.multimodal_projector_bias,
534
- quant_config=quant_config,
535
- prefix=maybe_prefix(prefix, "multi_modal_projector"))
524
+ if multimodal_config.get_limit_per_prompt("image"):
525
+ self.vision_tower = init_vision_tower_for_llava(
526
+ config,
527
+ quant_config,
528
+ require_post_norm=False,
529
+ prefix=maybe_prefix(prefix, "vision_tower"))
530
+ self.multi_modal_projector = LlavaMultiModalProjector(
531
+ vision_hidden_size=config.vision_config.hidden_size,
532
+ text_hidden_size=config.text_config.hidden_size,
533
+ projector_hidden_act=config.projector_hidden_act,
534
+ multimodal_projector_bias=config.multimodal_projector_bias,
535
+ quant_config=quant_config,
536
+ prefix=maybe_prefix(prefix, "multi_modal_projector"))
537
+ else:
538
+ self.vision_tower = None
539
+ self.multi_modal_projector = None
540
541
self.language_model = init_vllm_registered_model(
542
vllm_config=vllm_config,
@@ -756,7 +760,11 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
760
761
def load_weights(self, weights: Iterable[tuple[str,
762
torch.Tensor]]) -> set[str]:
759
- loader = AutoWeightsLoader(self)
763
+ skip_prefixes = []
764
+ if self.vision_tower is None and self.multi_modal_projector is None:
765
+ skip_prefixes.extend(["vision_tower.", "multi_modal_projector."])
766
+
767
+ loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
768
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
769
770
vllm/model_executor/models/mistral3.py
+23
-15
@@ -428,20 +428,24 @@ class Mistral3ForConditionalGeneration(nn.Module, SupportsLoRA,
428
config.projector_hidden_act = "gelu"
429
430
# TODO: Optionally initializes this for supporting embeddings.
431
- self.vision_tower = init_vision_tower_for_llava(
432
- config,
433
- quant_config,
434
- require_post_norm=False,
435
- prefix=maybe_prefix(prefix, "vision_tower"))
436
- self.multi_modal_projector = Mistral3MultiModalProjector(
437
- vision_hidden_size=config.vision_config.hidden_size,
438
- text_hidden_size=config.text_config.hidden_size,
439
- projector_hidden_act=config.projector_hidden_act,
440
- spatial_merge_size=config.spatial_merge_size,
441
- patch_size=config.vision_config.patch_size,
442
- multimodal_projector_bias=config.multimodal_projector_bias,
443
- quant_config=quant_config,
444
- prefix=maybe_prefix(prefix, "multi_modal_projector"))
431
+ if multimodal_config.get_limit_per_prompt("image"):
432
+ self.vision_tower = init_vision_tower_for_llava(
433
+ config,
434
+ quant_config,
435
+ require_post_norm=False,
436
+ prefix=maybe_prefix(prefix, "vision_tower"))
437
+ self.multi_modal_projector = Mistral3MultiModalProjector(
438
+ vision_hidden_size=config.vision_config.hidden_size,
439
+ text_hidden_size=config.text_config.hidden_size,
440
+ projector_hidden_act=config.projector_hidden_act,
441
+ spatial_merge_size=config.spatial_merge_size,
442
+ patch_size=config.vision_config.patch_size,
443
+ multimodal_projector_bias=config.multimodal_projector_bias,
444
+ quant_config=quant_config,
445
+ prefix=maybe_prefix(prefix, "multi_modal_projector"))
446
+ else:
447
+ self.vision_tower = None
448
+ self.multi_modal_projector = None
449
450
self.language_model = init_vllm_registered_model(
451
vllm_config=vllm_config,
@@ -611,7 +615,11 @@ class Mistral3ForConditionalGeneration(nn.Module, SupportsLoRA,
615
616
def load_weights(self, weights: Iterable[tuple[str,
617
torch.Tensor]]) -> set[str]:
614
- loader = AutoWeightsLoader(self)
618
+ skip_prefixes = []
619
+ if self.vision_tower is None and self.multi_modal_projector is None:
620
+ skip_prefixes = ["vision_tower.", "multi_modal_projector."]
621
+
622
+ loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
623
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
624
625
def get_mm_mapping(self) -> MultiModelKeys:
vllm/model_executor/models/mllama4.py
+20
-10
@@ -737,16 +737,20 @@ class Llama4ForConditionalGeneration(nn.Module, SupportsMultiModal,
737
self.config = config
738
self.quant_config = quant_config
739
self.multimodal_config = multimodal_config
740
- self.vision_model = Llama4VisionModel(
741
- config.vision_config,
742
- None,
743
- prefix=maybe_prefix(prefix, "vision_model"),
744
- use_data_parallel=self.use_data_parallel,
745
- )
746
- self.multi_modal_projector = Llama4MultiModalProjector(
747
- self.config,
748
- None,
749
- prefix=maybe_prefix(prefix, "multi_modal_projector"))
740
+ if multimodal_config.get_limit_per_prompt("image"):
741
+ self.vision_model = Llama4VisionModel(
742
+ config.vision_config,
743
+ None,
744
+ prefix=maybe_prefix(prefix, "vision_model"),
745
+ use_data_parallel=self.use_data_parallel,
746
+ )
747
+ self.multi_modal_projector = Llama4MultiModalProjector(
748
+ self.config,
749
+ None,
750
+ prefix=maybe_prefix(prefix, "multi_modal_projector"))
751
+ else:
752
+ self.vision_model = None
753
+ self.multi_modal_projector = None
754
self.language_model = initialize_model(
755
vllm_config=vllm_config.with_hf_config(config.text_config,
756
["LlamaForCausalLM"]),
@@ -783,6 +787,8 @@ class Llama4ForConditionalGeneration(nn.Module, SupportsMultiModal,
787
788
def _process_image_input(
789
self, image_input: Llama4ImagePatchInputs) -> MultiModalEmbeddings:
790
+
791
+ assert self.vision_model and self.multi_modal_projector
792
flat_data = image_input["flat_data"]
793
patches_per_image = image_input["patches_per_image"].tolist()
794
@@ -1048,6 +1054,10 @@ class Llama4ForConditionalGeneration(nn.Module, SupportsMultiModal,
1054
language_model_weights, other_weights = (
1055
self._separate_and_rename_weights(weights))
1056
1057
+ # Skip loading vision model and projector if they're not initialized.
1058
+ if self.vision_model is None and self.multi_modal_projector is None:
1059
+ other_weights = []
1060
+
1061
# Handle expert scale parameters
1062
regular_weights, expert_scale_weights, updated_params_from_experts = (
1063
self._handle_expert_scale_broadcasting(language_model_weights,
vllm/model_executor/models/qwen2_5_omni_thinker.py
+25
-8
@@ -722,13 +722,24 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
722
"exactly same result as the transformers implementation "
723
"in the audio tower part.")
724
725
- self.audio_tower = Qwen2_5OmniAudioEncoder(thinker_config.audio_config)
726
- self.visual = Qwen2_5_VisionTransformer(
727
- vision_config=thinker_config.vision_config,
728
- norm_eps=getattr(thinker_config.text_config, "rms_norm_eps", 1e-6),
729
- quant_config=quant_config,
730
- prefix=maybe_prefix(prefix, "visual"),
731
- )
725
+ if multimodal_config.get_limit_per_prompt("audio"):
726
+ self.audio_tower = Qwen2_5OmniAudioEncoder(
727
+ thinker_config.audio_config)
728
+ else:
729
+ self.audio_tower = None
730
+
731
+ if multimodal_config.get_limit_per_prompt(
732
+ "image") or multimodal_config.get_limit_per_prompt("video"):
733
+ self.visual = Qwen2_5_VisionTransformer(
734
+ vision_config=thinker_config.vision_config,
735
+ norm_eps=getattr(thinker_config.text_config, "rms_norm_eps",
736
+ 1e-6),
737
+ quant_config=quant_config,
738
+ prefix=maybe_prefix(prefix, "visual"),
739
+ )
740
+ else:
741
+ self.visual = None
742
+
743
self.quant_config = quant_config
744
self.language_model = init_vllm_registered_model(
745
vllm_config=vllm_config,
@@ -886,9 +897,15 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
897
898
def load_weights(self, weights: Iterable[tuple[str,
899
torch.Tensor]]) -> set[str]:
900
+ skip_prefixes = ["talker.", "token2wav."]
901
+ if self.audio_tower is None:
902
+ skip_prefixes.extend(["audio_tower."])
903
+ if self.visual is None:
904
+ skip_prefixes.extend(["visual."])
905
+
906
loader = AutoWeightsLoader(
907
self,
891
- skip_prefixes=["talker.", "token2wav."],
908
+ skip_prefixes=skip_prefixes,
909
)
910
loaded_weights = loader.load_weights(weights,
911
mapper=self.hf_to_vllm_mapper)
vllm/model_executor/models/qwen2_5_vl.py
+15
-7
@@ -843,12 +843,17 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,
843
self.config = config
844
self.multimodal_config = multimodal_config
845
846
- self.visual = Qwen2_5_VisionTransformer(
847
- config.vision_config,
848
- norm_eps=getattr(config, "rms_norm_eps", 1e-6),
849
- quant_config=self._maybe_ignore_quant_config(self.quant_config),
850
- prefix=maybe_prefix(prefix, "visual"),
851
- )
846
+ if multimodal_config.get_limit_per_prompt("image") or \
847
+ multimodal_config.get_limit_per_prompt("video"):
848
+ self.visual = Qwen2_5_VisionTransformer(
849
+ config.vision_config,
850
+ norm_eps=getattr(config, "rms_norm_eps", 1e-6),
851
+ quant_config=self._maybe_ignore_quant_config(
852
+ self.quant_config),
853
+ prefix=maybe_prefix(prefix, "visual"),
854
+ )
855
+ else:
856
+ self.visual = None
857
858
self.language_model = init_vllm_registered_model(
859
vllm_config=vllm_config,
@@ -1152,7 +1157,10 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1157
def load_weights(self, weights: Iterable[tuple[str,
1158
torch.Tensor]]) -> set[str]:
1159
1155
- loader = AutoWeightsLoader(self)
1160
+ skip_prefixes = []
1161
+ if self.visual is None:
1162
+ skip_prefixes.extend(["visual."])
1163
+ loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
1164
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
1165
1166
def get_mm_mapping(self) -> MultiModelKeys:
vllm/model_executor/models/qwen2_vl.py
+18
-8
@@ -1049,12 +1049,16 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1049
self.config = config
1050
self.multimodal_config = multimodal_config
1051
1052
- self.visual = Qwen2VisionTransformer(
1053
- config.vision_config,
1054
- norm_eps=getattr(config, "rms_norm_eps", 1e-6),
1055
- quant_config=self._maybe_ignore_quant_config(quant_config),
1056
- prefix=maybe_prefix(prefix, "visual"),
1057
- )
1052
+ if multimodal_config.get_limit_per_prompt("image") or \
1053
+ multimodal_config.get_limit_per_prompt("video"):
1054
+ self.visual = Qwen2VisionTransformer(
1055
+ config.vision_config,
1056
+ norm_eps=getattr(config, "rms_norm_eps", 1e-6),
1057
+ quant_config=self._maybe_ignore_quant_config(quant_config),
1058
+ prefix=maybe_prefix(prefix, "visual"),
1059
+ )
1060
+ else:
1061
+ self.visual = None
1062
1063
self.language_model = init_vllm_registered_model(
1064
vllm_config=vllm_config,
@@ -1350,7 +1354,10 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1354
def load_weights(self, weights: Iterable[tuple[str,
1355
torch.Tensor]]) -> set[str]:
1356
1353
- loader = AutoWeightsLoader(self)
1357
+ skip_prefixes = []
1358
+ if self.visual is None:
1359
+ skip_prefixes.extend(["visual."])
1360
+ loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
1361
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
1362
1363
def get_mm_mapping(self) -> MultiModelKeys:
@@ -1445,5 +1452,8 @@ class Tarsier2ForConditionalGeneration(Qwen2VLForConditionalGeneration):
1452
def load_weights(self, weights: Iterable[tuple[str,
1453
torch.Tensor]]) -> set[str]:
1454
1448
- loader = AutoWeightsLoader(self)
1455
+ skip_prefixes = []
1456
+ if self.visual is None:
1457
+ skip_prefixes.extend(["visual."])
1458
+ loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
1459
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
vllm/model_executor/models/step3_vl.py
+38
-22
@@ -837,27 +837,35 @@ class Step3VLForConditionalGeneration(nn.Module, SupportsMultiModal,
837
self.config = config
838
self.multimodal_config = multimodal_config
839
840
- self.vision_model = Step3VisionTransformer(config.vision_config,
841
- None,
842
- prefix=maybe_prefix(
843
- prefix, "vision_model"))
844
- self.vit_downsampler = nn.Conv2d(
845
- config.vision_config.hidden_size,
846
- config.vision_config.output_hidden_size,
847
- kernel_size=2,
848
- stride=config.understand_projector_stride)
849
- self.vit_downsampler2 = nn.Conv2d(
850
- config.vision_config.output_hidden_size,
851
- config.vision_config.output_hidden_size * 2,
852
- kernel_size=3,
853
- stride=2,
854
- padding=1,
855
- )
856
- self.vit_large_projector = nn.Linear(
857
- config.vision_config.output_hidden_size * 2,
858
- config.hidden_size,
859
- bias=config.projector_bias,
860
- )
840
+ if multimodal_config.get_limit_per_prompt("image"):
841
+ self.vision_model = Step3VisionTransformer(config.vision_config,
842
+ None,
843
+ prefix=maybe_prefix(
844
+ prefix,
845
+ "vision_model"))
846
+ self.vit_downsampler = nn.Conv2d(
847
+ config.vision_config.hidden_size,
848
+ config.vision_config.output_hidden_size,
849
+ kernel_size=2,
850
+ stride=config.understand_projector_stride)
851
+ self.vit_downsampler2 = nn.Conv2d(
852
+ config.vision_config.output_hidden_size,
853
+ config.vision_config.output_hidden_size * 2,
854
+ kernel_size=3,
855
+ stride=2,
856
+ padding=1,
857
+ )
858
+ self.vit_large_projector = nn.Linear(
859
+ config.vision_config.output_hidden_size * 2,
860
+ config.hidden_size,
861
+ bias=config.projector_bias,
862
+ )
863
+ else:
864
+ self.vision_model = None
865
+ self.vit_downsampler = None
866
+ self.vit_downsampler2 = None
867
+ self.vit_large_projector = None
868
+
869
self.language_model = init_vllm_registered_model(
870
vllm_config=vllm_config,
871
hf_config=config.text_config,
@@ -1046,7 +1054,15 @@ class Step3VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1054
return self.language_model.sample(logits, sampling_metadata)
1055
1056
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
1049
- loader = AutoWeightsLoader(self)
1057
+
1058
+ skip_prefixes = []
1059
+ if self.vision_model is None and self.vit_large_projector is None:
1060
+ skip_prefixes = [
1061
+ "vision_model.", "vit_downsampler.", "vit_downsampler2.",
1062
+ "vit_large_projector."
1063
+ ]
1064
+
1065
+ loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
1066
loaded_weights = loader.load_weights(weights,
1067
mapper=self.hf_to_vllm_mapper)
1068
return loaded_weights
vllm/multimodal/registry.py
+39
@@ -115,6 +115,45 @@ class MultiModalRegistry:
115
116
return True # Success
117
118
+ def enable_mm_input_cache(self, model_config: "ModelConfig") -> bool:
119
+ """Whether the multi-modal input cache should be enabled.
120
+ NOTE: This is put under MultiModalRegistry on purpose to respect
121
+ text-only mode for multimodal models.
122
+ """
123
+
124
+ if not self.supports_multimodal_inputs(model_config):
125
+ return False
126
+
127
+ mm_config = model_config.get_multimodal_config()
128
+
129
+ return mm_config.mm_processor_cache_gb > 0
130
+
131
+ def supports_multimodal_inputs(self, model_config: "ModelConfig") -> bool:
132
+ """
133
+ Checks if the model supports multimodal inputs.
134
+ Returns True if the model is multimodal with any non-zero supported
135
+ modalities, otherwise returns False, effectively running in
136
+ text-only mode.
137
+ """
138
+ if not model_config.is_multimodal_model:
139
+ return False
140
+
141
+ processor = self.create_processor(model_config, disable_cache=False)
142
+ supported_modalities = processor.info.get_supported_mm_limits()
143
+
144
+ mm_config = model_config.get_multimodal_config()
145
+
146
+ # Check if all supported modalities have limit == 0
147
+ if all(
148
+ mm_config.get_limit_per_prompt(modality) == 0
149
+ for modality in supported_modalities):
150
+ logger.info_once(
151
+ "All limits of multimodal modalities supported by the model "
152
+ "are set to 0, running in text-only mode.")
153
+ return False
154
+
155
+ return True
156
+
157
def get_max_tokens_per_item_by_modality(
158
self,
159
model_config: "ModelConfig",
vllm/v1/core/encoder_cache_manager.py
+1
-1
@@ -189,7 +189,7 @@ def compute_encoder_budget(
189
in the input sequence.
190
"""
191
192
- if not model_config.is_multimodal_model:
192
+ if not mm_registry.supports_multimodal_inputs(model_config):
193
return 0, 0
194
195
# TODO: handle encoder-decoder models once we support them.
vllm/v1/engine/core.py
+2
-1
@@ -21,6 +21,7 @@ from vllm.distributed import stateless_destroy_torch_distributed_process_group
21
from vllm.logger import init_logger
22
from vllm.logging_utils.dump_input import dump_engine_exception
23
from vllm.lora.request import LoRARequest
24
+from vllm.multimodal import MULTIMODAL_REGISTRY
25
from vllm.tasks import POOLING_TASKS, SupportedTask
26
from vllm.transformers_utils.config import (
27
maybe_register_config_serialize_by_value)
@@ -125,7 +126,7 @@ class EngineCore:
126
)
127
128
self.mm_input_cache_server = MultiModalInputCacheServer(
128
- vllm_config.model_config)
129
+ vllm_config.model_config, MULTIMODAL_REGISTRY)
130
131
# Setup batch queue for pipeline parallelism.
132
# Batch queue for scheduled batches. This enables us to asynchronously
vllm/v1/engine/mm_input_cache.py
+7
-5
@@ -3,7 +3,7 @@
3
from collections.abc import Sequence
4
from typing import TYPE_CHECKING, Optional
5
6
-from vllm.multimodal import MultiModalKwargs
6
+from vllm.multimodal import MultiModalKwargs, MultiModalRegistry
7
from vllm.multimodal.cache import MultiModalCache, MultiModalCacheItemMetadata
8
from vllm.utils import is_list_of
9
@@ -46,10 +46,11 @@ if TYPE_CHECKING:
46
class MultiModalInputCacheClient:
47
"""Used by P0 to check whether multi-modal kwargs are cached in P1."""
48
49
- def __init__(self, model_config: "ModelConfig") -> None:
49
+ def __init__(self, model_config: "ModelConfig",
50
+ mm_registry: MultiModalRegistry) -> None:
51
super().__init__()
52
52
- self.enabled = model_config.enable_mm_input_cache
53
+ self.enabled = mm_registry.enable_mm_input_cache(model_config)
54
self.mm_cache = MultiModalCache.get_lru_cache(
55
model_config.get_mm_input_cache_gb(),
56
MultiModalCacheItemMetadata,
@@ -85,10 +86,11 @@ class MultiModalInputCacheClient:
86
class MultiModalInputCacheServer:
87
"""Used by P1 to avoid requiring past multi-modal kwargs from P0."""
88
88
- def __init__(self, model_config: "ModelConfig") -> None:
89
+ def __init__(self, model_config: "ModelConfig",
90
+ mm_registry: MultiModalRegistry) -> None:
91
super().__init__()
92
91
- self.enabled = model_config.enable_mm_input_cache
93
+ self.enabled = mm_registry.enable_mm_input_cache(model_config)
94
self.mm_cache = MultiModalCache.get_lru_cache(
95
model_config.get_mm_input_cache_gb(),
96
MultiModalKwargs,
vllm/v1/engine/processor.py
+1
-1
@@ -51,7 +51,7 @@ class Processor:
51
mm_registry)
52
53
self.mm_input_cache_client = MultiModalInputCacheClient(
54
- self.model_config)
54
+ self.model_config, mm_registry)
55
56
@property
57
def mm_registry(self):
vllm/v1/worker/gpu_model_runner.py
+9
-7
@@ -129,7 +129,6 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
129
self.kv_cache_dtype = STR_DTYPE_TO_TORCH_DTYPE[
130
cache_config.cache_dtype]
131
132
- self.is_multimodal_model = model_config.is_multimodal_model
132
self.is_pooling_model = model_config.pooler_config is not None
133
self.is_encoder_only_model = False
134
self.is_multimodal_raw_input_supported = (
@@ -149,6 +148,8 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
148
# Multi-modal data support
149
self.mm_registry = MULTIMODAL_REGISTRY
150
self.uses_mrope = model_config.uses_mrope
151
+ self.supports_mm_inputs = self.mm_registry.supports_multimodal_inputs(
152
+ model_config)
153
154
# Sampler
155
self.sampler = Sampler(logprobs_mode=self.model_config.logprobs_mode)
@@ -330,7 +331,8 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
331
self.mm_registry,
332
max_model_len=self.max_model_len,
333
max_num_reqs=self.max_num_reqs,
333
- ) if self.is_multimodal_model else None)
334
+ ) if self.supports_mm_inputs \
335
+ else None)
336
337
self.reorder_batch_threshold: Optional[int] = None
338
@@ -1479,14 +1481,14 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
1481
1482
# _prepare_inputs may reorder the batch, so we must gather multi
1483
# modal outputs after that to ensure the correct order
1482
- if self.is_multimodal_model:
1484
+ if self.supports_mm_inputs:
1485
# Run the multimodal encoder if any.
1486
self._execute_mm_encoder(scheduler_output)
1487
mm_embeds = self._gather_mm_embeddings(scheduler_output)
1488
else:
1489
mm_embeds = []
1490
1489
- if self.is_multimodal_model and get_pp_group().is_first_rank:
1491
+ if self.supports_mm_inputs and get_pp_group().is_first_rank:
1492
# NOTE(woosuk): To unify token ids and soft tokens (vision
1493
# embeddings), we always use embeddings (rather than token ids)
1494
# as input to the multimodal model, even when the input is text.
@@ -1817,7 +1819,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
1819
else:
1820
target_hidden_states = hidden_states[token_indices]
1821
mm_embeds = None
1820
- if self.is_multimodal_model:
1822
+ if self.supports_mm_inputs:
1823
mm_embeds = self._gather_mm_embeddings(scheduler_output,
1824
shift_computed_tokens=1)
1825
@@ -2209,7 +2211,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
2211
2212
with self.maybe_dummy_run_with_lora(self.lora_config,
2213
num_scheduled_tokens):
2212
- if self.is_multimodal_model:
2214
+ if self.supports_mm_inputs:
2215
input_ids = None
2216
inputs_embeds = self.inputs_embeds[:num_tokens]
2217
model_mm_kwargs = self._dummy_mm_kwargs(num_reqs)
@@ -2417,7 +2419,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
2419
2420
def profile_run(self) -> None:
2421
# Profile with multimodal encoder & encoder cache.
2420
- if self.is_multimodal_model:
2422
+ if self.supports_mm_inputs:
2423
mm_budget = self.mm_budget
2424
assert mm_budget is not None
2425
vllm/v1/worker/tpu_model_runner.py
+14
-9
@@ -157,7 +157,6 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
157
cache_config.cache_dtype]
158
self._hidden_states_dtype = self.dtype
159
160
- self.is_multimodal_model = model_config.is_multimodal_model
160
self.sliding_window = model_config.get_sliding_window()
161
self.block_size = cache_config.block_size
162
self.max_model_len = model_config.max_model_len
@@ -193,6 +192,8 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
192
# Multi-modal data support
193
self.mm_registry = MULTIMODAL_REGISTRY
194
self.uses_mrope = model_config.uses_mrope
195
+ self.supports_mm_inputs = self.mm_registry.supports_multimodal_inputs(
196
+ model_config)
197
# TODO: Support M-RoPE (e.g, Qwen2-VL)
198
assert not self.uses_mrope, "TPU does not support M-RoPE yet."
199
@@ -293,7 +294,7 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
294
self.mm_registry,
295
max_model_len=self.max_model_len,
296
max_num_reqs=self.max_num_reqs,
296
- ) if self.is_multimodal_model else None)
297
+ ) if self.supports_mm_inputs else None)
298
299
if not self.use_spmd:
300
self.sample_from_logits_func = torch.compile(
@@ -947,7 +948,7 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
948
949
def _get_model_inputs(self, input_ids: torch.Tensor,
950
mm_embeds: list[torch.Tensor]):
950
- if self.is_multimodal_model:
951
+ if self.supports_mm_inputs:
952
# NOTE(woosuk): To unify token ids and soft tokens (vision
953
# embeddings), we always use embeddings (rather than token ids)
954
# as input to the multimodal model, even when the input is text.
@@ -979,7 +980,7 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
980
return self.kv_connector_no_forward(scheduler_output,
981
self.vllm_config)
982
982
- if self.is_multimodal_model:
983
+ if self.supports_mm_inputs:
984
# Run the multimodal encoder if any.
985
self._execute_mm_encoder(scheduler_output)
986
mm_embeds = self._gather_mm_embeddings(scheduler_output)
@@ -1230,7 +1231,7 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
1231
@torch.no_grad()
1232
def _dummy_run(self, num_tokens: int, num_reqs: int,
1233
num_blocks: int) -> None:
1233
- if self.is_multimodal_model:
1234
+ if self.supports_mm_inputs:
1235
input_ids = None
1236
inputs_embeds = torch.zeros((num_tokens, self.hidden_size),
1237
dtype=self.dtype,
@@ -1271,7 +1272,7 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
1272
_num_slices_per_kv_cache_update_block,
1273
)
1274
1274
- if self.is_multimodal_model:
1275
+ if self.supports_mm_inputs:
1276
torch._dynamo.mark_dynamic(inputs_embeds, 0)
1277
else:
1278
torch._dynamo.mark_dynamic(input_ids, 0)
@@ -1305,7 +1306,7 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
1306
xm.mark_step() # Captures metadata updates
1307
1308
def _precompile_mm_encoder(self) -> None:
1308
- if not self.is_multimodal_model:
1309
+ if not self.supports_mm_inputs:
1310
return
1311
1312
# Pre-compile MM encoder for all supported data modalities.
@@ -1527,7 +1528,7 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
1528
num_tokens: int,
1529
) -> None:
1530
# Profile with multimodal encoder & encoder cache.
1530
- if self.is_multimodal_model:
1531
+ if self.supports_mm_inputs:
1532
mm_budget = self.mm_budget
1533
assert mm_budget is not None
1534
@@ -1684,7 +1685,11 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
1685
get_kv_transfer_group().set_host_xfer_buffer_ops(copy_kv_blocks)
1686
1687
def reset_dynamo_cache(self):
1687
- if self.is_multimodal_model:
1688
+
1689
+ # NOTE: We check `is_multimodal_model` instead of `supports_mm_inputs`
1690
+ # since the compiled model object of the language backbone of a
1691
+ # multimodal model needs to be extracted via `get_language_model`.
1692
+ if self.model_config.is_multimodal_model:
1693
compiled_model = self.model.get_language_model().model
1694
else:
1695
compiled_model = self.model.model