@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fda3f03eb2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 9, 23:00 UTC

[4/N] Initialize MM components in context managers (M-P) (#32663)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 20, 2026 at 22:06 UTC fda3f03eb28c3b0daf73394219b8ad73adfc53e8
24 files changed +289 -352
vllm/model_executor/models/aria.py
+2 -8
@@ -621,14 +621,8 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
621 inputs_embeds: torch.Tensor | None = None,
622 **kwargs: object,
623 ) -> torch.Tensor | IntermediateTensors:
624 - if inputs_embeds is None:
625 - multimodal_embeddings = self.embed_multimodal(**kwargs)
626 - inputs_embeds = self.embed_input_ids(
627 - input_ids,
628 - multimodal_embeddings,
629 - is_multimodal=input_ids == self.config.image_token_index,
630 - )
631 - input_ids = None
624 + if intermediate_tensors is not None:
625 + inputs_embeds = None
626
627 hidden_states = self.language_model(
628 input_ids,
vllm/model_executor/models/dots_ocr.py
-8
@@ -791,14 +791,6 @@ class DotsOCRForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA
791 ) -> torch.Tensor | IntermediateTensors:
792 if intermediate_tensors is not None:
793 inputs_embeds = None
794 - elif inputs_embeds is None:
795 - vision_embeddings = self.embed_multimodal(**kwargs)
796 - inputs_embeds = self.embed_input_ids(
797 - input_ids,
798 - vision_embeddings,
799 - is_multimodal=input_ids == self.config.image_token_id,
800 - )
801 - input_ids = None
794
795 hidden_states = self.language_model(
796 input_ids=input_ids,
vllm/model_executor/models/interfaces.py
+4 -5
@@ -71,8 +71,6 @@ def _require_is_multimodal(is_multimodal: Tensor | None) -> Tensor:
71
72
73 class LMMissingLayer(nn.Module):
74 - packed_modules_mapping: dict[str, list[str]] = {}
75 -
74 def make_empty_intermediate_tensors(self, *args, **kwargs):
75 raise RuntimeError("This module should not be called in MM encoder-only mode")
76
@@ -81,8 +79,6 @@ class LMMissingLayer(nn.Module):
79
80
81 class TowerMissingLayer(nn.Module):
84 - packed_modules_mapping: dict[str, list[str]] = {}
85 -
82 def __init__(self, modalities: set[str] | str) -> None:
83 if isinstance(modalities, str):
84 modalities = {modalities}
@@ -92,7 +88,10 @@ class TowerMissingLayer(nn.Module):
88 self.modalities = modalities
89
90 def __call__(self, *args, **kwargs):
95 - raise RuntimeError(f"The following modalities are disabled: {self.modalities}")
91 + raise RuntimeError(
92 + f"This module should not be called when the following "
93 + f"modalities are disabled: {self.modalities}"
94 + )
95
96
97 @contextmanager
vllm/model_executor/models/interns1.py
-1
@@ -789,7 +789,6 @@ class InternS1ForConditionalGeneration(
789 **kwargs: object,
790 ) -> IntermediateTensors:
791 if intermediate_tensors is not None:
792 - input_ids = None
792 inputs_embeds = None
793
794 forward_kwargs = {
vllm/model_executor/models/internvl.py
-1
@@ -1379,7 +1379,6 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA)
1379 **kwargs: object,
1380 ) -> IntermediateTensors:
1381 if intermediate_tensors is not None:
1382 - input_ids = None
1382 inputs_embeds = None
1383
1384 forward_kwargs = {
vllm/model_executor/models/midashenglm.py
+21 -24
@@ -707,30 +707,30 @@ class MiDashengLMModel(nn.Module, SupportsMultiModal, SupportsPP):
707 config = vllm_config.model_config.hf_config
708 quant_config = vllm_config.quant_config
709 self.config = config
710 + self.quant_config = quant_config
711
711 - # Initialize audio components
712 - self.audio_encoder = DashengAudioTransformer(
713 - config.audio_encoder_config,
714 - quant_config=quant_config,
715 - prefix=maybe_prefix(prefix, "audio_encoder"),
716 - )
717 - self.audio_projector = AudioProjectorSubsample(
718 - in_dim=config.audio_encoder_config.embed_dim,
719 - out_dim=config.text_config.hidden_size,
720 - downsample_rate=config.subsample_factor,
721 - quant_config=quant_config,
722 - prefix=maybe_prefix(prefix, "audio_projector"),
723 - )
712 + with self._mark_tower_model(vllm_config, "audio"):
713 + self.audio_encoder = DashengAudioTransformer(
714 + config.audio_encoder_config,
715 + quant_config=quant_config,
716 + prefix=maybe_prefix(prefix, "audio_encoder"),
717 + )
718 + self.audio_projector = AudioProjectorSubsample(
719 + in_dim=config.audio_encoder_config.embed_dim,
720 + out_dim=config.text_config.hidden_size,
721 + downsample_rate=config.subsample_factor,
722 + quant_config=quant_config,
723 + prefix=maybe_prefix(prefix, "audio_projector"),
724 + )
725
725 - # Initialize language model (decoder)
726 - self.decoder = init_vllm_registered_model(
727 - vllm_config=vllm_config,
728 - hf_config=config.text_config,
729 - prefix=maybe_prefix(prefix, "decoder"),
730 - architectures=["Qwen2ForCausalLM"],
731 - )
726 + with self._mark_language_model(vllm_config):
727 + self.decoder = init_vllm_registered_model(
728 + vllm_config=vllm_config,
729 + hf_config=config.text_config,
730 + prefix=maybe_prefix(prefix, "decoder"),
731 + architectures=["Qwen2ForCausalLM"],
732 + )
733
733 - self.quant_config = quant_config
734 self.make_empty_intermediate_tensors = (
735 self.decoder.make_empty_intermediate_tensors
736 )
@@ -787,9 +787,6 @@ class MiDashengLMModel(nn.Module, SupportsMultiModal, SupportsPP):
787
788 return torch.split(masked_audio_features, audio_output_lengths.tolist())
789
790 - def get_language_model(self) -> torch.nn.Module:
791 - return self.decoder
792 -
790 def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
791 audio_input = self._parse_and_validate_audio_input(**kwargs)
792
vllm/model_executor/models/minicpmo.py
+5 -3
@@ -553,9 +553,11 @@ class MiniCPMO(MiniCPMV2_6):
553
554 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
555 super().__init__(vllm_config=vllm_config, prefix=prefix)
556 - self.apm = self.init_audio_module(
557 - vllm_config=vllm_config, prefix=maybe_prefix(prefix, "apm")
558 - )
556 +
557 + with self._mark_tower_model(vllm_config, "audio"):
558 + self.apm = self.init_audio_module(
559 + vllm_config=vllm_config, prefix=maybe_prefix(prefix, "apm")
560 + )
561
562 def init_audio_module(self, *, vllm_config: VllmConfig, prefix: str = ""):
563 # Do not use parameters temporarily
vllm/model_executor/models/minicpmv.py
+20 -21
@@ -1028,25 +1028,27 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
1028 self.multimodal_config = multimodal_config
1029
1030 self.version = get_version_by_config(self.config)
1031 - self.llm = self.init_llm(
1032 - vllm_config=vllm_config, prefix=maybe_prefix(prefix, "llm")
1033 - )
1034 - self.vpm = self.init_vision_module(
1035 - config, quant_config, prefix=maybe_prefix(prefix, "vpm")
1036 - )
1037 - self.vision_dim = (
1038 - self.vpm.embed_dim
1039 - if self.version == (2, 0)
1040 - else self.vpm.embeddings.embed_dim
1041 - )
1042 - self.embed_dim = self.config.hidden_size
1031
1044 - self.resampler = self.init_resampler(
1045 - self.embed_dim,
1046 - self.vision_dim,
1047 - quant_config=quant_config,
1048 - prefix=maybe_prefix(prefix, "resampler"),
1049 - )
1032 + with self._mark_language_model(vllm_config):
1033 + self.llm = self.init_llm(
1034 + vllm_config=vllm_config, prefix=maybe_prefix(prefix, "llm")
1035 + )
1036 +
1037 + with self._mark_tower_model(vllm_config, {"image", "video"}):
1038 + self.vpm = vpm = self.init_vision_module(
1039 + config, quant_config, prefix=maybe_prefix(prefix, "vpm")
1040 + )
1041 + self.vision_dim = (
1042 + vpm.embed_dim if self.version == (2, 0) else vpm.embeddings.embed_dim
1043 + )
1044 + self.embed_dim = self.config.hidden_size
1045 +
1046 + self.resampler = self.init_resampler(
1047 + self.embed_dim,
1048 + self.vision_dim,
1049 + quant_config=quant_config,
1050 + prefix=maybe_prefix(prefix, "resampler"),
1051 + )
1052
1053 self.make_empty_intermediate_tensors = self.llm.make_empty_intermediate_tensors
1054
@@ -1134,9 +1136,6 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
1136
1137 return multimodal_embeddings
1138
1137 - def get_language_model(self) -> torch.nn.Module:
1138 - return self.llm
1139 -
1139 def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
1140 modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
1141 if not modalities:
vllm/model_executor/models/minimax_vl_01.py
+27 -36
@@ -201,28 +201,33 @@ class MiniMaxVL01ForConditionalGeneration(nn.Module, SupportsMultiModal, Support
201 self.config = config
202 self.multimodal_config = multimodal_config
203
204 - # TODO: Optionally initializes this for supporting embeddings.
205 - self.vision_tower = init_vision_tower_for_llava(
206 - config,
207 - quant_config=quant_config,
208 - multimodal_config=multimodal_config,
209 - require_post_norm=False,
210 - prefix=maybe_prefix(prefix, "vision_tower"),
211 - )
212 - self.multi_modal_projector = MiniMaxVL01MultiModalProjector(
213 - vision_hidden_size=config.vision_config.hidden_size,
214 - text_hidden_size=config.text_config.hidden_size,
215 - projector_hidden_act=config.projector_hidden_act,
216 - multimodal_projector_bias=True,
217 - quant_config=quant_config,
218 - prefix=maybe_prefix(prefix, "multi_modal_projector"),
219 - )
220 - self.image_newline = nn.Parameter(torch.empty(config.text_config.hidden_size))
221 - self.language_model = init_vllm_registered_model(
222 - vllm_config=vllm_config,
223 - hf_config=config.text_config,
224 - prefix=maybe_prefix(prefix, "language_model"),
225 - )
204 + with self._mark_tower_model(vllm_config, "image"):
205 + self.vision_tower = init_vision_tower_for_llava(
206 + config,
207 + quant_config=quant_config,
208 + multimodal_config=multimodal_config,
209 + require_post_norm=False,
210 + prefix=maybe_prefix(prefix, "vision_tower"),
211 + )
212 + self.multi_modal_projector = MiniMaxVL01MultiModalProjector(
213 + vision_hidden_size=config.vision_config.hidden_size,
214 + text_hidden_size=config.text_config.hidden_size,
215 + projector_hidden_act=config.projector_hidden_act,
216 + multimodal_projector_bias=True,
217 + quant_config=quant_config,
218 + prefix=maybe_prefix(prefix, "multi_modal_projector"),
219 + )
220 + self.image_newline = nn.Parameter(
221 + torch.empty(config.text_config.hidden_size)
222 + )
223 +
224 + with self._mark_language_model(vllm_config):
225 + self.language_model = init_vllm_registered_model(
226 + vllm_config=vllm_config,
227 + hf_config=config.text_config,
228 + prefix=maybe_prefix(prefix, "language_model"),
229 + )
230 +
231 self.vision_feature_layer = config.vision_feature_layer
232 self.vocab_size = config.text_config.vocab_size
233 self.pad_token_id = -1
@@ -233,9 +238,6 @@ class MiniMaxVL01ForConditionalGeneration(nn.Module, SupportsMultiModal, Support
238 self.language_model.make_empty_intermediate_tensors
239 )
240
236 - def get_language_model(self) -> torch.nn.Module:
237 - return self.language_model
238 -
241 def _image_pixels_to_features(
242 self,
243 vision_tower: CLIPVisionModel | SiglipVisionModel | PixtralHFVisionModel,
@@ -302,8 +304,6 @@ class MiniMaxVL01ForConditionalGeneration(nn.Module, SupportsMultiModal, Support
304 self,
305 inputs: MiniMaxVL01ImagePixelInputs,
306 ) -> torch.Tensor | tuple[torch.Tensor, ...]:
305 - assert self.vision_tower is not None
306 -
307 pixel_values = inputs["pixel_values"]
308 return self._image_pixels_to_features(self.vision_tower, pixel_values)
309
@@ -314,7 +314,6 @@ class MiniMaxVL01ForConditionalGeneration(nn.Module, SupportsMultiModal, Support
314 if image_input["type"] == "image_embeds":
315 return image_input["data"]
316
317 - assert self.vision_tower is not None
317 image_features = self._process_image_pixels(image_input)
318
319 if isinstance(image_features, torch.Tensor):
@@ -369,14 +368,6 @@ class MiniMaxVL01ForConditionalGeneration(nn.Module, SupportsMultiModal, Support
368 ) -> torch.Tensor | IntermediateTensors:
369 if intermediate_tensors is not None:
370 inputs_embeds = None
372 - elif inputs_embeds is None:
373 - vision_embeddings = self.embed_multimodal(**kwargs)
374 - inputs_embeds = self.embed_input_ids(
375 - input_ids,
376 - vision_embeddings,
377 - is_multimodal=input_ids == self.config.image_token_index,
378 - )
379 - input_ids = None
371
372 hidden_states = self.language_model.model(
373 input_ids, positions, intermediate_tensors, inputs_embeds=inputs_embeds
vllm/model_executor/models/molmo.py
+14 -12
@@ -1441,15 +1441,20 @@ class MolmoForCausalLM(
1441 self.multimodal_config = multimodal_config
1442
1443 vision_config = VisionBackboneConfig()
1444 - self.vision_backbone = MolmoVisionBackbone(
1445 - config,
1446 - vision_config,
1447 - quant_config,
1448 - prefix=maybe_prefix(prefix, "vision_backbone"),
1449 - )
1450 - self.model = MolmoModel(
1451 - vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
1452 - )
1444 +
1445 + with self._mark_tower_model(vllm_config, "image"):
1446 + self.vision_backbone = MolmoVisionBackbone(
1447 + config,
1448 + vision_config,
1449 + quant_config,
1450 + prefix=maybe_prefix(prefix, "vision_backbone"),
1451 + )
1452 +
1453 + with self._mark_language_model(vllm_config):
1454 + self.model = MolmoModel(
1455 + vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
1456 + )
1457 +
1458 self.img_patch_id = None
1459
1460 if self.config.weight_tying:
@@ -1525,9 +1530,6 @@ class MolmoForCausalLM(
1530 results.append(feats[is_valid][order])
1531 return results
1532
1528 - def get_language_model(self) -> torch.nn.Module:
1529 - return self.model
1530 -
1533 def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
1534 image_input = self._parse_and_validate_image_input(**kwargs)
1535 if image_input is None:
vllm/model_executor/models/molmo2.py
+13 -13
@@ -2514,16 +2514,19 @@ class Molmo2ForConditionalGeneration(
2514 kwargs[field.name] = getattr(config.adapter_config, field.name)
2515 adapter_config = AdapterConfig(**kwargs)
2516
2517 - self.vision_backbone = Molmo2VisionBackbone(
2518 - vit_config,
2519 - adapter_config,
2520 - quant_config,
2521 - prefix=maybe_prefix(prefix, "vision_backbone"),
2522 - )
2523 - self.model = Molmo2TextModel(
2524 - vllm_config=vllm_config,
2525 - prefix=maybe_prefix(prefix, "model"),
2526 - )
2517 + with self._mark_tower_model(vllm_config, {"image", "video"}):
2518 + self.vision_backbone = Molmo2VisionBackbone(
2519 + vit_config,
2520 + adapter_config,
2521 + quant_config,
2522 + prefix=maybe_prefix(prefix, "vision_backbone"),
2523 + )
2524 +
2525 + with self._mark_language_model(vllm_config):
2526 + self.model = Molmo2TextModel(
2527 + vllm_config=vllm_config,
2528 + prefix=maybe_prefix(prefix, "model"),
2529 + )
2530
2531 self.img_patch_id = config.image_patch_id
2532
@@ -2687,9 +2690,6 @@ class Molmo2ForConditionalGeneration(
2690 out.append(out_features)
2691 return tuple(out)
2692
2690 - def get_language_model(self) -> torch.nn.Module:
2691 - return self.model
2692 -
2693 def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings | None:
2694 modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
2695 if not modalities:
vllm/model_executor/models/nano_nemotron_vl.py
+31 -31
@@ -1511,34 +1511,38 @@ class NemotronH_Nano_VL_V2(
1511 self.ps_version = config.ps_version
1512 self.image_tag_type = config.image_tag_type
1513 self.video_pruning_rate = multimodal_config.video_pruning_rate
1514 - self.language_model = init_vllm_registered_model(
1515 - vllm_config=vllm_config,
1516 - hf_config=config.text_config,
1517 - prefix=maybe_prefix(prefix, "language_model"),
1518 - )
1519 - self.vision_model = self.get_vit_model_from_radio_config(config).to(
1520 - self.language_model.config.dtype
1521 - )
1514
1523 - # Construct the vision projection.
1524 - vit_hidden_size = config.vit_hidden_size
1525 - vision_projection_hidden_size = config.projector_hidden_size
1526 - llm_hidden_size = config.text_config.hidden_size
1527 -
1528 - self.mlp1 = nn.Sequential(
1529 - RMSNorm(
1530 - hidden_size=vit_hidden_size * int(1 / self.downsample_ratio) ** 2,
1531 - eps=1e-5,
1532 - ),
1533 - nn.Linear(
1534 - vit_hidden_size * int(1 / self.downsample_ratio) ** 2,
1535 - vision_projection_hidden_size,
1536 - bias=False,
1537 - ),
1538 - ReLUSquaredActivation(),
1539 - nn.Linear(vision_projection_hidden_size, llm_hidden_size, bias=False),
1540 - )
1541 - self.mlp1 = self.mlp1.to(self.language_model.config.dtype)
1515 + with self._mark_language_model(vllm_config):
1516 + self.language_model = language_model = init_vllm_registered_model(
1517 + vllm_config=vllm_config,
1518 + hf_config=config.text_config,
1519 + prefix=maybe_prefix(prefix, "language_model"),
1520 + )
1521 +
1522 + with self._mark_tower_model(vllm_config, {"image", "video"}):
1523 + self.vision_model = self.get_vit_model_from_radio_config(config).to(
1524 + self.language_model.config.dtype
1525 + )
1526 +
1527 + # Construct the vision projection.
1528 + vit_hidden_size = config.vit_hidden_size
1529 + vision_projection_hidden_size = config.projector_hidden_size
1530 + llm_hidden_size = config.text_config.hidden_size
1531 +
1532 + mlp1 = nn.Sequential(
1533 + RMSNorm(
1534 + hidden_size=vit_hidden_size * int(1 / self.downsample_ratio) ** 2,
1535 + eps=1e-5,
1536 + ),
1537 + nn.Linear(
1538 + vit_hidden_size * int(1 / self.downsample_ratio) ** 2,
1539 + vision_projection_hidden_size,
1540 + bias=False,
1541 + ),
1542 + ReLUSquaredActivation(),
1543 + nn.Linear(vision_projection_hidden_size, llm_hidden_size, bias=False),
1544 + )
1545 + self.mlp1 = mlp1.to(language_model.config.dtype)
1546
1547 self.config = config
1548 self.model_config = vllm_config.model_config
@@ -1909,9 +1913,6 @@ class NemotronH_Nano_VL_V2(
1913
1914 return multimodal_embeddings
1915
1912 - def get_language_model(self) -> torch.nn.Module:
1913 - return self.language_model
1914 -
1916 def forward(
1917 self,
1918 input_ids: torch.Tensor,
@@ -1921,7 +1922,6 @@ class NemotronH_Nano_VL_V2(
1922 **kwargs: object,
1923 ) -> torch.Tensor | IntermediateTensors:
1924 if intermediate_tensors is not None:
1924 - input_ids = None
1925 inputs_embeds = None
1926
1927 hidden_states = self.language_model(
vllm/model_executor/models/nemotron_parse.py
+11 -12
@@ -820,16 +820,18 @@ class NemotronParseForConditionalGeneration(nn.Module, SupportsMultiModal):
820 cache_config = vllm_config.cache_config
821 quant_config = vllm_config.quant_config
822
823 - self.encoder = RadioWithNeck(
824 - config=config, quant_config=quant_config, prefix=f"{prefix}.encoder"
825 - )
823 + with self._mark_tower_model(vllm_config, "image"):
824 + self.encoder = RadioWithNeck(
825 + config=config, quant_config=quant_config, prefix=f"{prefix}.encoder"
826 + )
827
827 - self.decoder = MBartDecoderNoPos(
828 - config.decoder,
829 - cache_config=cache_config,
830 - quant_config=quant_config,
831 - prefix=f"{prefix}.decoder",
832 - )
828 + with self._mark_language_model(vllm_config):
829 + self.decoder = MBartDecoderNoPos(
830 + config.decoder,
831 + cache_config=cache_config,
832 + quant_config=quant_config,
833 + prefix=f"{prefix}.decoder",
834 + )
835
836 self.vocab_size = config.decoder.vocab_size
837 self.lm_head = ParallelLMHead(
@@ -883,9 +885,6 @@ class NemotronParseForConditionalGeneration(nn.Module, SupportsMultiModal):
885 pixel_values = pixel_values.to(dtype)
886 return self.encoder(pixel_values)
887
886 - def get_language_model(self) -> torch.nn.Module:
887 - return self.decoder
888 -
888 def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings | None:
889 image_input = self._parse_and_validate_image_input(**kwargs)
890 if image_input is None:
vllm/model_executor/models/nemotron_vl.py
+13 -19
@@ -385,20 +385,20 @@ class LlamaNemotronVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, Suppor
385 self.downsample_ratio = config.downsample_ratio
386 self.ps_version = config.ps_version
387
388 - self.llm_arch_name = config.text_config.architectures[0]
389 - self.vision_model = self._init_vision_model(
390 - config,
391 - quant_config=quant_config,
392 - prefix=maybe_prefix(prefix, "vision_model"),
393 - )
394 -
395 - self.language_model = init_vllm_registered_model(
396 - vllm_config=vllm_config,
397 - hf_config=config.text_config,
398 - prefix=maybe_prefix(prefix, "language_model"),
399 - )
388 + with self._mark_tower_model(vllm_config, "image"):
389 + self.vision_model = self._init_vision_model(
390 + config,
391 + quant_config=quant_config,
392 + prefix=maybe_prefix(prefix, "vision_model"),
393 + )
394 + self.mlp1 = self._init_mlp1(config)
395
401 - self.mlp1 = self._init_mlp1(config)
396 + with self._mark_language_model(vllm_config):
397 + self.language_model = init_vllm_registered_model(
398 + vllm_config=vllm_config,
399 + hf_config=config.text_config,
400 + prefix=maybe_prefix(prefix, "language_model"),
401 + )
402
403 self.img_context_token_id = None
404
@@ -520,8 +520,6 @@ class LlamaNemotronVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, Suppor
520 if image_input["type"] == "image_embeds":
521 return image_input["data"]
522
523 - assert self.vision_model is not None
524 -
523 image_embeds = self.extract_feature(image_input["pixel_values_flat"])
524
525 num_patches = image_input["num_patches"]
@@ -556,9 +554,6 @@ class LlamaNemotronVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, Suppor
554 def _set_visual_token_mask(self, input_ids: torch.Tensor) -> None:
555 self.visual_token_mask = None
556
559 - def get_language_model(self) -> torch.nn.Module:
560 - return self.language_model
561 -
557 def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
558 modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
559 if not modalities:
@@ -609,7 +604,6 @@ class LlamaNemotronVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, Suppor
604 **kwargs: object,
605 ) -> IntermediateTensors:
606 if intermediate_tensors is not None:
612 - input_ids = None
607 inputs_embeds = None
608
609 forward_kwargs = {
vllm/model_executor/models/ovis.py
+16 -18
@@ -417,7 +417,7 @@ class Ovis(nn.Module, SupportsMultiModal, SupportsPP):
417 @classmethod
418 def get_placeholder_str(cls, modality: str, i: int) -> str | None:
419 if modality.startswith("image"):
420 - return "<image>"
420 + return IMAGE_TOKEN
421
422 raise ValueError("Only image modality is supported")
423
@@ -427,20 +427,22 @@ class Ovis(nn.Module, SupportsMultiModal, SupportsPP):
427 quant_config = vllm_config.quant_config
428
429 self.config: PretrainedConfig = config
430 - self.llm = init_vllm_registered_model(
431 - vllm_config=vllm_config.with_hf_config(config.get_text_config()),
432 - prefix=maybe_prefix(prefix, "llm"),
433 - )
430
435 - self.visual_tokenizer = VisualTokenizer(
436 - config=config.visual_tokenizer_config,
437 - quant_config=quant_config,
438 - prefix=f"{prefix}.visual_tokenizer",
439 - )
431 + with self._mark_language_model(vllm_config):
432 + self.llm = init_vllm_registered_model(
433 + vllm_config=vllm_config.with_hf_config(config.get_text_config()),
434 + prefix=maybe_prefix(prefix, "llm"),
435 + )
436
441 - self.vte = VisualEmbedding(
442 - self.config.visual_tokenizer_config.vocab_size, self.config.hidden_size
443 - )
437 + with self._mark_tower_model(vllm_config, "image"):
438 + self.visual_tokenizer = VisualTokenizer(
439 + config=config.visual_tokenizer_config,
440 + quant_config=quant_config,
441 + prefix=f"{prefix}.visual_tokenizer",
442 + )
443 + self.vte = VisualEmbedding(
444 + self.config.visual_tokenizer_config.vocab_size, self.config.hidden_size
445 + )
446
447 text_model_type = self.config.get_text_config().model_type
448 self.image_pad_token_id = IMAGE_PAD_TOKEN_ID_MAP[text_model_type]
@@ -546,12 +548,8 @@ class Ovis(nn.Module, SupportsMultiModal, SupportsPP):
548 self,
549 hidden_states: torch.Tensor,
550 ) -> torch.Tensor | None:
549 - logits = self.llm.compute_logits(hidden_states)
550 - return logits
551 + return self.llm.compute_logits(hidden_states)
552
553 def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
554 loader = AutoWeightsLoader(self)
555 return loader.load_weights(weights)
555 -
556 - def get_language_model(self) -> torch.nn.Module:
557 - return self.llm
vllm/model_executor/models/ovis2_5.py
+24 -17
@@ -451,6 +451,15 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo])
451 dummy_inputs=Ovis2_5DummyInputsBuilder,
452 )
453 class Ovis2_5(nn.Module, SupportsMultiModal, SupportsPP):
454 + @classmethod
455 + def get_placeholder_str(cls, modality: str, i: int) -> str | None:
456 + if modality.startswith("image"):
457 + return IMAGE_TOKEN
458 + if modality.startswith("video"):
459 + return VIDEO_TOKEN
460 +
461 + raise ValueError("Only image or video modality is supported")
462 +
463 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
464 super().__init__()
465 config = vllm_config.model_config.hf_config
@@ -458,20 +467,22 @@ class Ovis2_5(nn.Module, SupportsMultiModal, SupportsPP):
467 multimodal_config = vllm_config.model_config.multimodal_config
468
469 self.config: PretrainedConfig = config
461 - self.llm = init_vllm_registered_model(
462 - vllm_config=vllm_config.with_hf_config(config.text_config),
463 - prefix=maybe_prefix(prefix, "llm"),
464 - )
470
466 - self.visual_tokenizer = VisualTokenizer(
467 - config=config.vit_config,
468 - visual_vocab_size=config.visual_vocab_size,
469 - multimodal_config=multimodal_config,
470 - quant_config=quant_config,
471 - prefix=f"{prefix}.visual_tokenizer",
472 - )
471 + with self._mark_language_model(vllm_config):
472 + self.llm = init_vllm_registered_model(
473 + vllm_config=vllm_config.with_hf_config(config.text_config),
474 + prefix=maybe_prefix(prefix, "llm"),
475 + )
476
474 - self.vte = VisualEmbedding(config.visual_vocab_size, config.hidden_size)
477 + with self._mark_tower_model(vllm_config, {"image", "video"}):
478 + self.visual_tokenizer = VisualTokenizer(
479 + config=config.vit_config,
480 + visual_vocab_size=config.visual_vocab_size,
481 + multimodal_config=multimodal_config,
482 + quant_config=quant_config,
483 + prefix=f"{prefix}.visual_tokenizer",
484 + )
485 + self.vte = VisualEmbedding(config.visual_vocab_size, config.hidden_size)
486
487 text_model_type = self.config.get_text_config().model_type
488 self.image_pad_token_id = IMAGE_PAD_TOKEN_ID_MAP[text_model_type]
@@ -650,12 +661,8 @@ class Ovis2_5(nn.Module, SupportsMultiModal, SupportsPP):
661 self,
662 hidden_states: torch.Tensor,
663 ) -> torch.Tensor | None:
653 - logits = self.llm.compute_logits(hidden_states)
654 - return logits
664 + return self.llm.compute_logits(hidden_states)
665
666 def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
667 loader = AutoWeightsLoader(self)
668 return loader.load_weights(weights)
659 -
660 - def get_language_model(self) -> torch.nn.Module:
661 - return self.llm
vllm/model_executor/models/paddleocr_vl.py
+23 -36
@@ -999,6 +999,13 @@ class PaddleOCRVLForConditionalGeneration(nn.Module, SupportsMultiModal, Support
999 }
1000 )
1001
1002 + @classmethod
1003 + def get_placeholder_str(cls, modality: str, i: int) -> str | None:
1004 + if modality.startswith("image"):
1005 + return "<|IMAGE_START|><|IMAGE_PLACEHOLDER|><|IMAGE_END|>"
1006 +
1007 + raise ValueError("Only image modality is supported")
1008 +
1009 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1010 super().__init__()
1011 config = vllm_config.model_config.hf_config
@@ -1008,22 +1015,24 @@ class PaddleOCRVLForConditionalGeneration(nn.Module, SupportsMultiModal, Support
1015 self.config = config
1016 self.multimodal_config = multimodal_config
1017
1011 - self.visual = SiglipVisionModel(
1012 - config=config.vision_config,
1013 - quant_config=quant_config,
1014 - multimodal_config=multimodal_config,
1015 - prefix=maybe_prefix(prefix, "visual"),
1016 - )
1017 - self.mlp_AR = Projector(config, config.vision_config)
1018 + with self._mark_tower_model(vllm_config, "image"):
1019 + self.visual = SiglipVisionModel(
1020 + config=config.vision_config,
1021 + quant_config=quant_config,
1022 + multimodal_config=multimodal_config,
1023 + prefix=maybe_prefix(prefix, "visual"),
1024 + )
1025 + self.mlp_AR = Projector(config, config.vision_config)
1026
1019 - self.language_model = Ernie4_5ForCausalLM(
1020 - vllm_config=vllm_config,
1021 - prefix=maybe_prefix(prefix, "language_model"),
1022 - )
1027 + with self._mark_language_model(vllm_config):
1028 + self.language_model = language_model = Ernie4_5ForCausalLM(
1029 + vllm_config=vllm_config,
1030 + prefix=maybe_prefix(prefix, "language_model"),
1031 + )
1032
1024 - for layer in self.language_model.model.layers:
1025 - if not isinstance(layer, PPMissingLayer):
1026 - layer.self_attn.rotary_emb.is_neox_style = True
1033 + for layer in language_model.model.layers:
1034 + if not isinstance(layer, PPMissingLayer):
1035 + layer.self_attn.rotary_emb.is_neox_style = True
1036
1037 self.make_empty_intermediate_tensors = (
1038 self.language_model.make_empty_intermediate_tensors
@@ -1151,9 +1160,6 @@ class PaddleOCRVLForConditionalGeneration(nn.Module, SupportsMultiModal, Support
1160
1161 return llm_positions, mrope_position_delta
1162
1154 - def get_language_model(self) -> nn.Module:
1155 - return self.language_model
1156 -
1163 def _parse_and_validate_image_input(
1164 self, **kwargs: object
1165 ) -> PaddleOCRImagePixelInputs | None:
@@ -1180,29 +1186,10 @@ class PaddleOCRVLForConditionalGeneration(nn.Module, SupportsMultiModal, Support
1186 if intermediate_tensors is not None:
1187 inputs_embeds = None
1188
1183 - elif inputs_embeds is None:
1184 - vision_embeddings = self.embed_multimodal(**kwargs)
1185 - is_multimodal = kwargs.pop("is_multimodal", None)
1186 - handle_oov_mm_token = kwargs.pop("handle_oov_mm_token", False)
1187 - inputs_embeds = self.embed_input_ids(
1188 - input_ids,
1189 - vision_embeddings,
1190 - is_multimodal=is_multimodal,
1191 - handle_oov_mm_token=handle_oov_mm_token,
1192 - )
1193 - input_ids = None
1194 -
1189 return self.language_model(
1190 input_ids, positions, intermediate_tensors, inputs_embeds
1191 )
1192
1199 - @classmethod
1200 - def get_placeholder_str(cls, modality: str, i: int) -> str | None:
1201 - if modality.startswith("image"):
1202 - return "<|IMAGE_START|><|IMAGE_PLACEHOLDER|><|IMAGE_END|>"
1203 -
1204 - raise ValueError("Only image modality is supported")
1205 -
1193 def encode_image(
1194 self, pixel_values: torch.Tensor, image_grid_thw: torch.Tensor
1195 ) -> torch.Tensor:
vllm/model_executor/models/paligemma.py
+20 -22
@@ -295,30 +295,32 @@ class PaliGemmaForConditionalGeneration(
295 multimodal_config = vllm_config.model_config.multimodal_config
296 self.config = config
297 self.multimodal_config = multimodal_config
298 -
299 - self.vision_tower = SiglipVisionModel(
300 - config.vision_config,
301 - quant_config,
302 - prefix=maybe_prefix(prefix, "vision_tower"),
303 - )
304 - self.multi_modal_projector = PaliGemmaMultiModalProjector(
305 - vision_hidden_size=config.vision_config.hidden_size,
306 - projection_dim=config.vision_config.projection_dim,
307 - )
308 -
298 self.quant_config = quant_config
299
300 + with self._mark_tower_model(vllm_config, "image"):
301 + self.vision_tower = SiglipVisionModel(
302 + config.vision_config,
303 + quant_config,
304 + prefix=maybe_prefix(prefix, "vision_tower"),
305 + )
306 + self.multi_modal_projector = PaliGemmaMultiModalProjector(
307 + vision_hidden_size=config.vision_config.hidden_size,
308 + projection_dim=config.vision_config.projection_dim,
309 + )
310 +
311 if config.text_config.model_type == "gemma":
312 config.text_config.architectures = ["GemmaForCausalLM"]
313 else:
314 config.text_config.architectures = ["Gemma2ForCausalLM"]
315 - self.language_model = init_vllm_registered_model(
316 - vllm_config=vllm_config,
317 - hf_config=config.text_config,
318 - prefix=maybe_prefix(prefix, "language_model"),
319 - )
320 - logit_scale = getattr(config, "logit_scale", 1.0)
321 - self.language_model.logits_processor.scale *= logit_scale
315 +
316 + with self._mark_language_model(vllm_config):
317 + self.language_model = language_model = init_vllm_registered_model(
318 + vllm_config=vllm_config,
319 + hf_config=config.text_config,
320 + prefix=maybe_prefix(prefix, "language_model"),
321 + )
322 + logit_scale = getattr(config, "logit_scale", 1.0)
323 + language_model.logits_processor.scale *= logit_scale
324
325 self.make_empty_intermediate_tensors = (
326 self.language_model.make_empty_intermediate_tensors
@@ -367,7 +369,6 @@ class PaliGemmaForConditionalGeneration(
369 if image_input["type"] == "image_embeds":
370 return image_input["data"]
371
370 - assert self.vision_tower is not None
372 pixel_values = image_input["data"]
373 image_features = self._image_pixels_to_features(
374 self.vision_tower,
@@ -376,9 +377,6 @@ class PaliGemmaForConditionalGeneration(
377
378 return self.multi_modal_projector(image_features)
379
379 - def get_language_model(self) -> torch.nn.Module:
380 - return self.language_model
381 -
380 def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
381 image_input = self._parse_and_validate_image_input(**kwargs)
382 if image_input is None:
vllm/model_executor/models/phi3v.py
+24 -29
@@ -586,31 +586,31 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsQuant)
586 self.multimodal_config = multimodal_config
587 self.image_token_id = _IMAGE_TOKEN_ID
588
589 - self.embed_tokens = VocabParallelEmbedding(
590 - config.vocab_size,
591 - config.hidden_size,
592 - quant_config=quant_config,
593 - prefix=maybe_prefix(prefix, "model.embed_tokens"),
594 - )
595 -
596 - # TODO: Optionally initializes this for supporting input embeddings.
597 - self.vision_embed_tokens = Phi3HDImageEmbedding(
598 - config,
599 - quant_config=quant_config,
600 - multimodal_config=multimodal_config,
601 - prefix=maybe_prefix(prefix, "model.vision_embed_tokens"),
602 - )
589 + with self._mark_tower_model(vllm_config, "image"):
590 + self.embed_tokens = VocabParallelEmbedding(
591 + config.vocab_size,
592 + config.hidden_size,
593 + quant_config=quant_config,
594 + prefix=maybe_prefix(prefix, "model.embed_tokens"),
595 + )
596 + self.vision_embed_tokens = Phi3HDImageEmbedding(
597 + config,
598 + quant_config=quant_config,
599 + multimodal_config=multimodal_config,
600 + prefix=maybe_prefix(prefix, "model.vision_embed_tokens"),
601 + )
602
604 - self.language_model = init_vllm_registered_model(
605 - vllm_config=vllm_config,
606 - # The prefix is empty intentionally because default prefix of
607 - # LlamaForCausalLM is "model"
608 - prefix="",
609 - # We don't directly initialize vLLM's LlamaForCausalLM so we
610 - # can automatically apply embedding wrapper if this model is
611 - # initialized as an embedding model
612 - architectures=["LlamaForCausalLM"],
613 - )
603 + with self._mark_language_model(vllm_config):
604 + self.language_model = init_vllm_registered_model(
605 + vllm_config=vllm_config,
606 + # The prefix is empty intentionally because default prefix of
607 + # LlamaForCausalLM is "model"
608 + prefix="",
609 + # We don't directly initialize vLLM's LlamaForCausalLM so we
610 + # can automatically apply embedding wrapper if this model is
611 + # initialized as an embedding model
612 + architectures=["LlamaForCausalLM"],
613 + )
614
615 self.make_empty_intermediate_tensors = (
616 self.language_model.make_empty_intermediate_tensors
@@ -652,17 +652,12 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsQuant)
652 if image_input["type"] == "image_embeds":
653 return image_input["data"]
654
655 - assert self.vision_embed_tokens is not None
656 -
655 image_embeds = self.vision_embed_tokens(
656 image_input["pixel_values"], image_input["image_sizes"]
657 )
658
659 return image_embeds
660
663 - def get_language_model(self) -> torch.nn.Module:
664 - return self.language_model
665 -
661 def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
662 image_input = self._parse_and_validate_image_input(**kwargs)
663 if image_input is None:
vllm/model_executor/models/phi4mm.py
+14 -13
@@ -1027,12 +1027,13 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):
1027 # Tensor/Pipeline parallel not supported for now.
1028 assert get_pp_group().world_size == 1, "pipeline parallel is not supported"
1029
1030 - self.vision_encoder = Phi4MMImageEncoder(
1031 - config,
1032 - quant_config,
1033 - prefix="model.vision_embed_tokens",
1034 - model_dir=config._name_or_path,
1035 - )
1030 + with self._mark_tower_model(vllm_config, {"image", "video"}):
1031 + self.vision_encoder = Phi4MMImageEncoder(
1032 + config,
1033 + quant_config,
1034 + prefix="model.vision_embed_tokens",
1035 + model_dir=config._name_or_path,
1036 + )
1037
1038 if isinstance(config.embd_layer["audio_embd_layer"], dict):
1039 embedding_config = {
@@ -1044,10 +1045,13 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):
1045 "embedding_cls": self.config.embd_layer["embedding_cls"]
1046 }
1047
1047 - self.embed_tokens_extend = AudioEmbedding(config, **embedding_config)
1048 - self.model = LlamaModel(
1049 - vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
1050 - )
1048 + with self._mark_tower_model(vllm_config, "audio"):
1049 + self.embed_tokens_extend = AudioEmbedding(config, **embedding_config)
1050 +
1051 + with self._mark_language_model(vllm_config):
1052 + self.model = LlamaModel(
1053 + vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
1054 + )
1055
1056 self.lm_head = ParallelLMHead(
1057 config.vocab_size,
@@ -1245,6 +1249,3 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):
1249 connector=["audio_projection_for_vision", "audio_projection"],
1250 tower_model=["vision_encoder", "embed_tokens_extend"],
1251 )
1248 -
1249 - def get_language_model(self) -> torch.nn.Module:
1250 - return self.model
vllm/model_executor/models/qwen3_vl_moe.py
+6 -6
@@ -453,15 +453,15 @@ class Qwen3VLMoeForConditionalGeneration(
453 ]
454
455 with self._mark_language_model(vllm_config):
456 - self.language_model = Qwen3MoeLLMForCausalLM(
456 + self.language_model = language_model = Qwen3MoeLLMForCausalLM(
457 vllm_config=vllm_config, prefix=maybe_prefix(prefix, "language_model")
458 )
459
460 - # Whether to include the gate_up_proj mapping is determined by
461 - # the language model.
462 - self.packed_modules_mapping = (
463 - self.packed_modules_mapping | self.language_model.packed_modules_mapping
464 - )
460 + # Whether to include the gate_up_proj mapping is determined by
461 + # the language model.
462 + self.packed_modules_mapping = (
463 + self.packed_modules_mapping | language_model.packed_modules_mapping
464 + )
465
466 self.make_empty_intermediate_tensors = (
467 self.language_model.make_empty_intermediate_tensors
vllm/model_executor/models/skyworkr1v.py
-1
@@ -908,7 +908,6 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):
908 **kwargs: object,
909 ) -> IntermediateTensors:
910 if intermediate_tensors is not None:
911 - input_ids = None
911 inputs_embeds = None
912
913 forward_kwargs = {
vllm/model_executor/models/step3_vl.py
-8
@@ -1104,14 +1104,6 @@ class Step3VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)
1104 ) -> torch.Tensor | IntermediateTensors:
1105 if intermediate_tensors is not None:
1106 inputs_embeds = None
1107 - elif inputs_embeds is None:
1108 - vision_embeddings = self.embed_multimodal(**kwargs)
1109 - inputs_embeds = self.embed_input_ids(
1110 - input_ids,
1111 - vision_embeddings,
1112 - is_multimodal=input_ids == self.config.image_token_id,
1113 - )
1114 - input_ids = None
1107
1108 hidden_states = self.language_model(
1109 input_ids, positions, intermediate_tensors, inputs_embeds=inputs_embeds
vllm/model_executor/models/tarsier.py
+1 -8
@@ -597,14 +597,7 @@ class TarsierForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)
597 ) -> torch.Tensor | IntermediateTensors:
598 if intermediate_tensors is not None:
599 inputs_embeds = None
600 - elif inputs_embeds is None:
601 - vision_embeddings = self.embed_multimodal(**kwargs)
602 - inputs_embeds = self.embed_input_ids(
603 - input_ids,
604 - vision_embeddings,
605 - is_multimodal=input_ids == self.config.image_token_index,
606 - )
607 - input_ids = None
600 +
601 hidden_states = self.language_model.model(
602 input_ids=input_ids,
603 positions=positions,