@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4753f3bf69
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 9, 23:00 UTC

[Model] Use context managers for encoder- and LM-only mode (#32605)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 20, 2026 at 11:43 UTC 4753f3bf69a2b975361afa7c49e8d948047613f6
21 files changed +292 -355
examples/online_serving/disaggregated_encoder/README.md
+1 -1
@@ -38,7 +38,7 @@ Encoder engines should be launched with the following flags:
38
39 - `--max-num-batched-tokens=<large value>` **(default: 2048)** – This flag controls the token scheduling budget per decoding step and is irrelevant to encoder-only instances. **Set it to a very high value (effectively unlimited) to bypass scheduler limitations.** The actual token budget is managed by the encoder cache manager.
40
41 -- `--convert "mm_encoder_only"` **(Optional)** - The language model is skipped during initialization to reduce device memory usage. **Models using this option must implement the `get_language_model_spec` interface.**
41 +- `--mm-encoder-only` **(Optional)** - The language model is skipped during initialization to reduce device memory usage. **Models using this option must initialize the language component inside the context of `SupportsMultiModal._mark_language_model`.**
42
43 ## Local media inputs
44
vllm/config/model.py
+12
@@ -306,6 +306,7 @@ class ModelConfig:
306 mm_processor_cache_gb: InitVar[float | None] = None
307 mm_processor_cache_type: InitVar[MMCacheType | None] = None
308 mm_shm_cache_max_object_size_mb: InitVar[int | None] = None
309 + mm_encoder_only: InitVar[bool | None] = None
310 mm_encoder_tp_mode: InitVar[MMEncoderTPMode | None] = None
311 mm_encoder_attn_backend: InitVar[AttentionBackendEnum | str | None] = None
312 interleave_mm_strings: InitVar[bool | None] = None
@@ -420,6 +421,7 @@ class ModelConfig:
421 mm_processor_cache_gb: float | None,
422 mm_processor_cache_type: MMCacheType | None,
423 mm_shm_cache_max_object_size_mb: int | None,
424 + mm_encoder_only: bool | None,
425 mm_encoder_tp_mode: MMEncoderTPMode | None,
426 mm_encoder_attn_backend: AttentionBackendEnum | str | None,
427 interleave_mm_strings: bool | None,
@@ -497,6 +499,15 @@ class ModelConfig:
499 )
500 self.model_arch_config = self.get_model_arch_config()
501
502 + if self.convert == "mm_encoder_only":
503 + logger.warning_once(
504 + "`--convert mm_encoder_only` is deprecated and "
505 + "will be removed in v0.15. "
506 + "Please use --mm-encoder-only` instead."
507 + )
508 + mm_encoder_only = True
509 + self.convert = "none"
510 +
511 architectures = self.architectures
512 registry = self.registry
513 is_generative_model = registry.is_text_generation_model(architectures, self)
@@ -583,6 +594,7 @@ class ModelConfig:
594 mm_processor_cache_gb=mm_processor_cache_gb,
595 mm_processor_cache_type=mm_processor_cache_type,
596 mm_shm_cache_max_object_size_mb=mm_shm_cache_max_object_size_mb,
597 + mm_encoder_only=mm_encoder_only,
598 mm_encoder_tp_mode=mm_encoder_tp_mode,
599 mm_encoder_attn_backend=mm_encoder_attn_backend,
600 interleave_mm_strings=interleave_mm_strings,
vllm/config/multimodal.py
+6
@@ -108,6 +108,12 @@ class MultiModalConfig:
108 """Size limit (in MiB) for each object stored in the multi-modal processor
109 shared memory cache. Only effective when `mm_processor_cache_type` is
110 `"shm"`."""
111 + mm_encoder_only: bool = False
112 + """
113 + When enabled, skips the language component of the model.
114 +
115 + This is usually only valid in disaggregated Encoder process.
116 + """
117 mm_encoder_tp_mode: MMEncoderTPMode = "weights"
118 """Indicates how to optimize multi-modal encoder inference using tensor
119 parallelism (TP).
vllm/engine/arg_utils.py
+5
@@ -467,6 +467,7 @@ class EngineArgs:
467 mm_shm_cache_max_object_size_mb: int = (
468 MultiModalConfig.mm_shm_cache_max_object_size_mb
469 )
470 + mm_encoder_only: bool = MultiModalConfig.mm_encoder_only
471 mm_encoder_tp_mode: MMEncoderTPMode = MultiModalConfig.mm_encoder_tp_mode
472 mm_encoder_attn_backend: AttentionBackendEnum | str | None = (
473 MultiModalConfig.mm_encoder_attn_backend
@@ -973,6 +974,9 @@ class EngineArgs:
974 "--mm-shm-cache-max-object-size-mb",
975 **multimodal_kwargs["mm_shm_cache_max_object_size_mb"],
976 )
977 + multimodal_group.add_argument(
978 + "--mm-encoder-only", **multimodal_kwargs["mm_encoder_only"]
979 + )
980 multimodal_group.add_argument(
981 "--mm-encoder-tp-mode", **multimodal_kwargs["mm_encoder_tp_mode"]
982 )
@@ -1256,6 +1260,7 @@ class EngineArgs:
1260 mm_processor_cache_gb=self.mm_processor_cache_gb,
1261 mm_processor_cache_type=self.mm_processor_cache_type,
1262 mm_shm_cache_max_object_size_mb=self.mm_shm_cache_max_object_size_mb,
1263 + mm_encoder_only=self.mm_encoder_only,
1264 mm_encoder_tp_mode=self.mm_encoder_tp_mode,
1265 mm_encoder_attn_backend=self.mm_encoder_attn_backend,
1266 pooler_config=self.pooler_config,
vllm/model_executor/model_loader/utils.py
+1 -8
@@ -189,9 +189,7 @@ def _get_model_architecture(model_config: ModelConfig) -> tuple[type[nn.Module],
189 )
190
191 convert_type = model_config.convert_type
192 - if convert_type not in ["none", "mm_encoder_only"] and supports_multimodal(
193 - model_cls
194 - ):
192 + if convert_type != "none" and supports_multimodal(model_cls):
193 logger.debug_once("Detected conversion of Multi Modal model.")
194 converted = try_create_mm_pooling_model_cls(model_cls)
195 if converted is not None:
@@ -202,11 +200,6 @@ def _get_model_architecture(model_config: ModelConfig) -> tuple[type[nn.Module],
200
201 if convert_type == "none":
202 pass
205 - elif convert_type == "mm_encoder_only":
206 - logger.debug_once("Converting to mm encoder only model.")
207 - from vllm.model_executor.models.adapters import as_mm_encoder_only_model
208 -
209 - model_cls = as_mm_encoder_only_model(model_cls)
203 elif convert_type == "embed":
204 logger.debug_once("Converting to embedding model.")
205 model_cls = as_embedding_model(model_cls)
vllm/model_executor/models/adapters.py
-61
@@ -529,64 +529,3 @@ def seq_cls_model_loader(model, weights: Iterable[tuple[str, torch.Tensor]]):
529 method = getattr(hf_config, "method", getattr(text_config, "method", None))
530 assert method in SEQ_CLS_LOAD_METHODS, f"method {method} not supported"
531 return SEQ_CLS_LOAD_METHODS[method](model, weights)
532 -
533 -
534 -def as_mm_encoder_only_model(cls: _T) -> _T:
535 - """
536 - Subclass an existing vLLM vl model to support mm encoder only for
537 - EPD encoder instances.
538 - """
539 - if not hasattr(cls, "embed_multimodal"):
540 - # Submodel case: return the original class.
541 - return cls
542 -
543 - if not hasattr(cls, "get_language_model_spec"):
544 - raise TypeError(f"{cls} need to implement `get_language_model_spec` method.")
545 -
546 - lm_model_cls, lm_attr = cls.get_language_model_spec()
547 -
548 - if lm_model_cls is None or lm_attr is None:
549 - raise TypeError(
550 - f"{cls}.get_language_model_spec() must return (lm_model_cls, lm_attr)"
551 - )
552 -
553 - class DummyLM(nn.Module):
554 - def __init__(self, *args, **kwargs):
555 - self.make_empty_intermediate_tensors = None
556 -
557 - class ModelForMMEncoderOnly(cls):
558 - def __init__(
559 - self,
560 - *,
561 - vllm_config: "VllmConfig",
562 - prefix: str = "",
563 - **kwargs: Any,
564 - ) -> None:
565 - self.is_mm_encoder_only_model = True
566 - origin_init = lm_model_cls.__init__
567 - try:
568 - lm_model_cls.__init__ = DummyLM.__init__
569 - super().__init__(vllm_config=vllm_config, prefix=prefix, **kwargs)
570 -
571 - if hasattr(self, lm_attr):
572 - delattr(self, lm_attr)
573 - finally:
574 - lm_model_cls.__init__ = origin_init
575 -
576 - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
577 - from .utils import AutoWeightsLoader
578 -
579 - origin_init_ = AutoWeightsLoader.__init__
580 -
581 - def _new_init_(self, *args, **kwargs):
582 - origin_init_(self, *args, **kwargs)
583 - self.skip_prefixes = (self.skip_prefixes or []) + [f"{lm_attr}."]
584 -
585 - try:
586 - AutoWeightsLoader.__init__ = _new_init_
587 - result = super().load_weights(weights)
588 - finally:
589 - AutoWeightsLoader.__init__ = origin_init_
590 - return result
591 -
592 - return ModelForMMEncoderOnly # type: ignore
vllm/model_executor/models/interfaces.py
+109 -13
@@ -2,6 +2,7 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 from collections.abc import Callable, Iterable, Mapping, MutableSequence
5 +from contextlib import contextmanager, nullcontext
6 from typing import (
7 TYPE_CHECKING,
8 ClassVar,
@@ -69,6 +70,46 @@ def _require_is_multimodal(is_multimodal: Tensor | None) -> Tensor:
70 return is_multimodal
71
72
73 +class LMMissingLayer(nn.Module):
74 + packed_modules_mapping: dict[str, list[str]] = {}
75 +
76 + def make_empty_intermediate_tensors(self, *args, **kwargs):
77 + raise RuntimeError("This module should not be called in MM encoder-only mode")
78 +
79 + def __call__(self, *args, **kwargs):
80 + raise RuntimeError("This module should not be called in MM encoder-only mode")
81 +
82 +
83 +class TowerMissingLayer(nn.Module):
84 + packed_modules_mapping: dict[str, list[str]] = {}
85 +
86 + def __init__(self, modalities: set[str]) -> None:
87 + super().__init__()
88 +
89 + self.modalities = modalities
90 +
91 + def __call__(self, *args, **kwargs):
92 + raise RuntimeError(f"The following modalities are disabled: {self.modalities}")
93 +
94 +
95 +@contextmanager
96 +def _no_init_weights(module: nn.Module, placeholder: Callable[[], nn.Module]):
97 + """
98 + Within this context, prevent weight initialization from using device memory and
99 + replace direct child assignments to `module` with the result of `placeholder()`.
100 + """
101 +
102 + def callback(module_, name, submodule):
103 + if module_ is module:
104 + return placeholder()
105 +
106 + return submodule
107 +
108 + with torch.nn.modules.module.register_module_module_registration_hook(callback): # noqa: E501,SIM117
109 + with torch.device("meta"):
110 + yield
111 +
112 +
113 @runtime_checkable
114 class SupportsMultiModal(Protocol):
115 """The interface required for all multi-modal models."""
@@ -105,6 +146,16 @@ class SupportsMultiModal(Protocol):
146 Set internally by `MultiModalRegistry.register_processor`.
147 """
148
149 + _language_model_names: list[str] = []
150 + """
151 + Set internally by `_mark_language_model`.
152 + """
153 +
154 + _tower_model_names: list[str] = []
155 + """
156 + Set internally by `_mark_tower_model`.
157 + """
158 +
159 @classmethod
160 def get_placeholder_str(cls, modality: str, i: int) -> str | None:
161 """
@@ -134,7 +185,64 @@ class SupportsMultiModal(Protocol):
185 Returns:
186 torch.nn.Module: The core language model component.
187 """
137 - ...
188 + if self._language_model_names:
189 + return getattr(self, self._language_model_names[0])
190 +
191 + raise NotImplementedError(
192 + f"No language model found in {type(self).__name__}! "
193 + "You should initialize it inside `_mark_language_model`."
194 + )
195 +
196 + @contextmanager
197 + def _mark_language_model(self, vllm_config: VllmConfig):
198 + """
199 + Mark each child module that was assigned to this model
200 + during this context as a language model component.
201 + """
202 + mm_config = vllm_config.model_config.multimodal_config
203 +
204 + children_names = list[str]()
205 +
206 + def callback(module_, name, submodule):
207 + if module_ is self:
208 + children_names.append(name)
209 +
210 + with torch.nn.modules.module.register_module_module_registration_hook(callback): # noqa: E501,SIM117
211 + with (
212 + _no_init_weights(self, LMMissingLayer)
213 + if mm_config.mm_encoder_only
214 + else nullcontext()
215 + ):
216 + yield
217 +
218 + self._language_model_names = children_names
219 +
220 + @contextmanager
221 + def _mark_tower_model(self, vllm_config: VllmConfig, modalities: set[str] | str):
222 + """
223 + Mark each child module that was assigned to this model
224 + during this context as a tower model component.
225 + """
226 + if isinstance(modalities, str):
227 + modalities = {modalities}
228 +
229 + mm_config = vllm_config.model_config.multimodal_config
230 +
231 + children_names = list[str]()
232 +
233 + def callback(module_, name, submodule):
234 + if module_ is self:
235 + children_names.append(name)
236 +
237 + with torch.nn.modules.module.register_module_module_registration_hook(callback): # noqa: E501,SIM117
238 + with (
239 + _no_init_weights(self, lambda: TowerMissingLayer(modalities))
240 + if all(mm_config.get_limit_per_prompt(m) == 0 for m in modalities)
241 + else nullcontext()
242 + ):
243 + yield
244 +
245 + self._tower_model_names = children_names
246
247 def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
248 """
@@ -154,14 +262,6 @@ class SupportsMultiModal(Protocol):
262 """
263 ...
264
157 - @classmethod
158 - def get_language_model_spec(cls) -> tuple[nn.Module | None, str | None]:
159 - """
160 - Return the language model spec:
161 - (language model class, language model attr)
162 - """
163 - return None, None
164 -
265 @overload
266 def embed_input_ids(self, input_ids: Tensor) -> Tensor: ...
267
@@ -299,10 +399,6 @@ def supports_multimodal_encoder_tp_data(model: type[object] | object) -> bool:
399 return getattr(model, "supports_encoder_tp_data", False)
400
401
302 -def supports_mm_encoder_only(model: type[object] | object) -> bool:
303 - return getattr(model, "is_mm_encoder_only_model", False)
304 -
305 -
402 @overload
403 def supports_multimodal_pruning(
404 model: type[object],
vllm/model_executor/models/llava.py
+9 -22
@@ -550,8 +550,7 @@ class LlavaForConditionalGeneration(
550 ):
551 config.projector_hidden_act = "gelu"
552
553 - # TODO: Optionally initializes this for supporting embeddings.
554 - if multimodal_config.get_limit_per_prompt("image"):
553 + with self._mark_tower_model(vllm_config, "image"):
554 self.vision_tower = init_vision_tower_for_llava(
555 config,
556 quant_config=quant_config,
@@ -567,15 +566,13 @@ class LlavaForConditionalGeneration(
566 quant_config=quant_config,
567 prefix=maybe_prefix(prefix, "multi_modal_projector"),
568 )
570 - else:
571 - self.vision_tower = None
572 - self.multi_modal_projector = None
573 -
574 - self.language_model = init_vllm_registered_model(
575 - vllm_config=vllm_config,
576 - hf_config=config.text_config,
577 - prefix=maybe_prefix(prefix, "language_model"),
578 - )
569 +
570 + with self._mark_language_model(vllm_config):
571 + self.language_model = init_vllm_registered_model(
572 + vllm_config=vllm_config,
573 + hf_config=config.text_config,
574 + prefix=maybe_prefix(prefix, "language_model"),
575 + )
576
577 self.make_empty_intermediate_tensors = (
578 self.language_model.make_empty_intermediate_tensors
@@ -631,8 +628,6 @@ class LlavaForConditionalGeneration(
628 self,
629 inputs: LlavaImagePixelInputs | PixtralHFImagePixelInputs,
630 ) -> torch.Tensor | tuple[torch.Tensor, ...]:
634 - assert self.vision_tower is not None
635 -
631 pixel_values = inputs["pixel_values"]
632
633 return self._image_pixels_to_features(self.vision_tower, pixel_values)
@@ -644,7 +639,6 @@ class LlavaForConditionalGeneration(
639 if image_input["type"] == "image_embeds":
640 return image_input["data"]
641
647 - assert self.vision_tower is not None
642 image_features = self._process_image_pixels(image_input)
643
644 if isinstance(image_features, torch.Tensor):
@@ -656,9 +650,6 @@ class LlavaForConditionalGeneration(
650 image_embeds = torch.split(image_embeds, feature_sizes)
651 return image_embeds
652
659 - def get_language_model(self) -> torch.nn.Module:
660 - return self.language_model
661 -
653 def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
654 image_input = self._parse_and_validate_image_input(**kwargs)
655 if image_input is None:
@@ -727,11 +718,7 @@ class LlavaForConditionalGeneration(
718 return self.language_model.compute_logits(hidden_states)
719
720 def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
730 - skip_prefixes = []
731 - if self.vision_tower is None and self.multi_modal_projector is None:
732 - skip_prefixes.extend(["vision_tower.", "multi_modal_projector."])
733 -
734 - loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
721 + loader = AutoWeightsLoader(self)
722 return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
723
724 def get_mm_mapping(self) -> MultiModelKeys:
vllm/model_executor/models/mistral3.py
+8 -18
@@ -457,8 +457,7 @@ class Mistral3ForConditionalGeneration(
457 ):
458 config.projector_hidden_act = "gelu"
459
460 - # TODO: Optionally initializes this for supporting embeddings.
461 - if multimodal_config.get_limit_per_prompt("image"):
460 + with self._mark_tower_model(vllm_config, "image"):
461 self.vision_tower = init_vision_tower_for_llava(
462 config,
463 quant_config=quant_config,
@@ -476,15 +475,13 @@ class Mistral3ForConditionalGeneration(
475 quant_config=quant_config,
476 prefix=maybe_prefix(prefix, "multi_modal_projector"),
477 )
479 - else:
480 - self.vision_tower = None
481 - self.multi_modal_projector = None
478
483 - self.language_model = init_vllm_registered_model(
484 - vllm_config=vllm_config,
485 - hf_config=config.text_config,
486 - prefix=maybe_prefix(prefix, "language_model"),
487 - )
479 + with self._mark_language_model(vllm_config):
480 + self.language_model = init_vllm_registered_model(
481 + vllm_config=vllm_config,
482 + hf_config=config.text_config,
483 + prefix=maybe_prefix(prefix, "language_model"),
484 + )
485
486 self.make_empty_intermediate_tensors = (
487 self.language_model.make_empty_intermediate_tensors
@@ -534,9 +531,6 @@ class Mistral3ForConditionalGeneration(
531 image_embeds = (image_embeds,)
532 return image_embeds
533
537 - def get_language_model(self) -> torch.nn.Module:
538 - return self.language_model
539 -
534 def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
535 image_input = self._parse_and_validate_image_input(**kwargs)
536 if image_input is None:
@@ -607,11 +601,7 @@ class Mistral3ForConditionalGeneration(
601 return self.language_model.compute_logits(hidden_states)
602
603 def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
610 - skip_prefixes = []
611 - if self.vision_tower is None and self.multi_modal_projector is None:
612 - skip_prefixes = ["vision_tower.", "multi_modal_projector."]
613 -
614 - loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
604 + loader = AutoWeightsLoader(self)
605 return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
606
607 def get_mm_mapping(self) -> MultiModelKeys:
vllm/model_executor/models/mllama4.py
+17 -18
@@ -70,12 +70,14 @@ from vllm.sequence import IntermediateTensors
70 from vllm.utils.tensor_schema import TensorSchema, TensorShape
71
72 from .interfaces import (
73 + LMMissingLayer,
74 MixtureOfExperts,
75 MultiModalEmbeddings,
76 SupportsEagle3,
77 SupportsLoRA,
78 SupportsMultiModal,
79 SupportsPP,
80 + TowerMissingLayer,
81 )
82 from .llama4 import Llama4ForCausalLM
83 from .utils import (
@@ -773,7 +775,8 @@ class Llama4ForConditionalGeneration(
775 self.config = config
776 self.quant_config = quant_config
777 self.multimodal_config = multimodal_config
776 - if multimodal_config.get_limit_per_prompt("image"):
778 +
779 + with self._mark_tower_model(vllm_config, "image"):
780 from vllm.compilation.backends import set_model_tag
781
782 with (
@@ -792,16 +795,15 @@ class Llama4ForConditionalGeneration(
795 quant_config=None,
796 prefix=maybe_prefix(prefix, "multi_modal_projector"),
797 )
795 - else:
796 - self.vision_model = None
797 - self.multi_modal_projector = None
798 - self.language_model = initialize_model(
799 - vllm_config=vllm_config.with_hf_config(
800 - config.text_config, ["LlamaForCausalLM"]
801 - ),
802 - prefix=maybe_prefix(prefix, "language_model"),
803 - model_class=Llama4ForCausalLM,
804 - )
798 +
799 + with self._mark_language_model(vllm_config):
800 + self.language_model = initialize_model(
801 + vllm_config=vllm_config.with_hf_config(
802 + config.text_config, ["LlamaForCausalLM"]
803 + ),
804 + prefix=maybe_prefix(prefix, "language_model"),
805 + model_class=Llama4ForCausalLM,
806 + )
807
808 self.make_empty_intermediate_tensors = (
809 self.language_model.make_empty_intermediate_tensors
@@ -892,9 +894,6 @@ class Llama4ForConditionalGeneration(
894 for img in vision_embeddings_flat.split(patches_per_image, dim=0)
895 ]
896
895 - def get_language_model(self) -> torch.nn.Module:
896 - return self.language_model
897 -
897 def embed_multimodal(self, **kwargs) -> MultiModalEmbeddings:
898 image_input = self._parse_and_validate_image_input(**kwargs)
899 if image_input is None:
@@ -1024,6 +1023,10 @@ class Llama4ForConditionalGeneration(
1023 for name, weight in weights:
1024 renamed = self._rename_weight_for_modelopt_checkpoint(name)
1025
1026 + attr = renamed.split(".", 1)[0]
1027 + if isinstance(getattr(self, attr), (LMMissingLayer, TowerMissingLayer)):
1028 + continue
1029 +
1030 if renamed.startswith("language_model."):
1031 language_model_weights.append((renamed, weight))
1032 else:
@@ -1133,10 +1136,6 @@ class Llama4ForConditionalGeneration(
1136 weights
1137 )
1138
1136 - # Skip loading vision model and projector if they're not initialized.
1137 - if self.vision_model is None and self.multi_modal_projector is None:
1138 - other_weights = []
1139 -
1139 # Handle expert scale parameters
1140 regular_weights, expert_scale_weights, updated_params_from_experts = (
1141 self._handle_expert_scale_broadcasting(language_model_weights, params_dict)
vllm/model_executor/models/opencua.py
+8 -9
@@ -239,7 +239,7 @@ class OpenCUAForConditionalGeneration(Qwen2_5_VLForConditionalGeneration):
239 multimodal_config.is_multimodal_pruning_enabled()
240 )
241
242 - if multimodal_config.get_limit_per_prompt("image"):
242 + with self._mark_tower_model(vllm_config, "image"):
243 self.visual = OpenCUAVisionTransformer(
244 vision_config=config.vision_config,
245 norm_eps=getattr(config, "rms_norm_eps", 1e-6),
@@ -247,15 +247,14 @@ class OpenCUAForConditionalGeneration(Qwen2_5_VLForConditionalGeneration):
247 multimodal_config=self.multimodal_config,
248 prefix=maybe_prefix(prefix, "visual"),
249 )
250 - else:
251 - self.visual = None
250
253 - self.language_model = init_vllm_registered_model(
254 - vllm_config=vllm_config,
255 - hf_config=config.text_config,
256 - prefix=maybe_prefix(prefix, "language_model"),
257 - architectures=["Qwen2ForCausalLM"],
258 - )
251 + with self._mark_language_model(vllm_config):
252 + self.language_model = init_vllm_registered_model(
253 + vllm_config=vllm_config,
254 + hf_config=config.text_config,
255 + prefix=maybe_prefix(prefix, "language_model"),
256 + architectures=["Qwen2ForCausalLM"],
257 + )
258
259 self.make_empty_intermediate_tensors = (
260 self.language_model.make_empty_intermediate_tensors
vllm/model_executor/models/pixtral.py
+7 -18
@@ -398,13 +398,14 @@ class PixtralForConditionalGeneration(
398 self.vision_args = VisionEncoderArgs(**vision_args)
399
400 # init MistralForCausalLM
401 - self.language_model = init_vllm_registered_model(
402 - vllm_config=vllm_config,
403 - hf_config=config.text_config,
404 - prefix=maybe_prefix(prefix, "language_model"),
405 - )
401 + with self._mark_language_model(vllm_config):
402 + self.language_model = init_vllm_registered_model(
403 + vllm_config=vllm_config,
404 + hf_config=config.text_config,
405 + prefix=maybe_prefix(prefix, "language_model"),
406 + )
407
407 - if multimodal_config.get_limit_per_prompt("image"):
408 + with self._mark_tower_model(vllm_config, "image"):
409 self.vision_encoder = VisionTransformer(self.vision_args)
410 self.pre_mm_projector_norm = (
411 RMSNorm(self.vision_args.hidden_size, eps=1e-5)
@@ -423,11 +424,6 @@ class PixtralForConditionalGeneration(
424 self.vision_language_adapter = VisionLanguageAdapter(
425 self.vision_args, dim=config.text_config.hidden_size
426 )
426 - else:
427 - self.vision_encoder = None
428 - self.pre_mm_projector_norm = None
429 - self.patch_merger = None
430 - self.vision_language_adapter = None
427
428 self.make_empty_intermediate_tensors = (
429 self.language_model.make_empty_intermediate_tensors
@@ -449,10 +445,6 @@ class PixtralForConditionalGeneration(
445 self,
446 image_input: PixtralImagePixelInputs,
447 ) -> tuple[torch.Tensor, ...]:
452 - assert (
453 - self.vision_encoder is not None and self.vision_language_adapter is not None
454 - )
455 -
448 images = image_input["images"]
449 image_features = self.vision_encoder(images)
450 feature_sizes = [image_feature.shape[0] for image_feature in image_features]
@@ -477,9 +469,6 @@ class PixtralForConditionalGeneration(
469 image_embeds = torch.split(image_embeds, feature_sizes)
470 return image_embeds
471
480 - def get_language_model(self) -> torch.nn.Module:
481 - return self.language_model
482 -
472 def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
473 image_input = self._parse_and_validate_image_input(**kwargs)
474 if image_input is None:
vllm/model_executor/models/qwen2_5_omni_thinker.py
+12 -31
@@ -822,6 +822,7 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
822 multimodal_config = vllm_config.model_config.multimodal_config
823 self.config = thinker_config
824 self.multimodal_config = multimodal_config
825 + self.quant_config = quant_config
826
827 # force "use_flash_attention_2=True" to audio tower to align
828 # the results.
@@ -836,14 +837,10 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
837 "in the audio tower part."
838 )
839
839 - if multimodal_config.get_limit_per_prompt("audio"):
840 + with self._mark_tower_model(vllm_config, "audio"):
841 self.audio_tower = Qwen2_5OmniAudioEncoder(thinker_config.audio_config)
841 - else:
842 - self.audio_tower = None
842
844 - if multimodal_config.get_limit_per_prompt(
845 - "image"
846 - ) or multimodal_config.get_limit_per_prompt("video"):
843 + with self._mark_tower_model(vllm_config, {"image", "video"}):
844 self.visual = Qwen2_5_VisionTransformer(
845 vision_config=thinker_config.vision_config,
846 norm_eps=getattr(thinker_config.text_config, "rms_norm_eps", 1e-6),
@@ -851,16 +848,14 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
848 prefix=maybe_prefix(prefix, "visual"),
849 multimodal_config=multimodal_config,
850 )
854 - else:
855 - self.visual = None
851
857 - self.quant_config = quant_config
858 - self.language_model = init_vllm_registered_model(
859 - vllm_config=vllm_config,
860 - prefix=maybe_prefix(prefix, "language_model"),
861 - hf_config=thinker_config.text_config,
862 - architectures=["Qwen2ForCausalLM"],
863 - )
852 + with self._mark_language_model(vllm_config):
853 + self.language_model = init_vllm_registered_model(
854 + vllm_config=vllm_config,
855 + prefix=maybe_prefix(prefix, "language_model"),
856 + hf_config=thinker_config.text_config,
857 + architectures=["Qwen2ForCausalLM"],
858 + )
859
860 self.make_empty_intermediate_tensors = (
861 self.language_model.make_empty_intermediate_tensors
@@ -895,9 +890,6 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
890 )
891 return mm_input_by_modality
892
898 - def get_language_model(self) -> torch.nn.Module:
899 - return self.language_model
900 -
893 def get_mrope_input_positions(
894 self,
895 input_tokens: list[int],
@@ -1175,19 +1167,8 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
1167 return self.language_model.compute_logits(hidden_states)
1168
1169 def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
1178 - skip_prefixes = ["talker.", "token2wav."]
1179 - if self.audio_tower is None:
1180 - skip_prefixes.extend(["audio_tower."])
1181 - if self.visual is None:
1182 - skip_prefixes.extend(["visual."])
1183 -
1184 - loader = AutoWeightsLoader(
1185 - self,
1186 - skip_prefixes=skip_prefixes,
1187 - )
1188 - loaded_weights = loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
1189 -
1190 - return loaded_weights
1170 + loader = AutoWeightsLoader(self, skip_prefixes=["talker.", "token2wav."])
1171 + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
1172
1173 def get_mm_mapping(self) -> MultiModelKeys:
1174 """
vllm/model_executor/models/qwen2_5_vl.py
+9 -26
@@ -35,7 +35,7 @@ import numpy as np
35 import torch
36 import torch.nn as nn
37 import torch.nn.functional as F
38 -from transformers import BatchFeature, Qwen2ForCausalLM
38 +from transformers import BatchFeature
39 from transformers.models.qwen2_5_vl import Qwen2_5_VLProcessor
40 from transformers.models.qwen2_5_vl.configuration_qwen2_5_vl import (
41 Qwen2_5_VLConfig,
@@ -1145,9 +1145,7 @@ class Qwen2_5_VLForConditionalGeneration(
1145 multimodal_config.is_multimodal_pruning_enabled()
1146 )
1147
1148 - if multimodal_config.get_limit_per_prompt(
1149 - "image"
1150 - ) or multimodal_config.get_limit_per_prompt("video"):
1148 + with self._mark_tower_model(vllm_config, {"image", "video"}):
1149 self.visual = Qwen2_5_VisionTransformer(
1150 vision_config=config.vision_config,
1151 norm_eps=getattr(config, "rms_norm_eps", 1e-6),
@@ -1155,14 +1153,13 @@ class Qwen2_5_VLForConditionalGeneration(
1153 prefix=maybe_prefix(prefix, "visual"),
1154 multimodal_config=multimodal_config,
1155 )
1158 - else:
1159 - self.visual = None
1156
1161 - self.language_model = init_vllm_registered_model(
1162 - vllm_config=vllm_config,
1163 - prefix=maybe_prefix(prefix, "language_model"),
1164 - architectures=["Qwen2ForCausalLM"],
1165 - )
1157 + with self._mark_language_model(vllm_config):
1158 + self.language_model = init_vllm_registered_model(
1159 + vllm_config=vllm_config,
1160 + prefix=maybe_prefix(prefix, "language_model"),
1161 + architectures=["Qwen2ForCausalLM"],
1162 + )
1163
1164 self.make_empty_intermediate_tensors = (
1165 self.language_model.make_empty_intermediate_tensors
@@ -1447,9 +1444,6 @@ class Qwen2_5_VLForConditionalGeneration(
1444 )
1445 return mm_input_by_modality
1446
1450 - def get_language_model(self) -> torch.nn.Module:
1451 - return self.language_model
1452 -
1447 def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
1448 mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)
1449 if not mm_input_by_modality:
@@ -1516,10 +1510,7 @@ class Qwen2_5_VLForConditionalGeneration(
1510 return self.language_model.compute_logits(hidden_states)
1511
1512 def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
1519 - skip_prefixes = []
1520 - if self.visual is None:
1521 - skip_prefixes.extend(["visual."])
1522 - loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
1513 + loader = AutoWeightsLoader(self)
1514 return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
1515
1516 def get_mm_mapping(self) -> MultiModelKeys:
@@ -1550,11 +1541,3 @@ class Qwen2_5_VLForConditionalGeneration(
1541 vision_config = hf_config.vision_config
1542 merge_size = vision_config.spatial_merge_size
1543 return num_vision_tokens // merge_size**2
1553 -
1554 - @classmethod
1555 - def get_language_model_spec(cls) -> tuple[nn.Module | None, str | None]:
1556 - """
1557 - Return the language model spec:
1558 - (language model class, language model attr)
1559 - """
1560 - return Qwen2ForCausalLM, "language_model"
vllm/model_executor/models/qwen2_vl.py
+8 -17
@@ -1233,9 +1233,7 @@ class Qwen2VLForConditionalGeneration(
1233 self.config = config
1234 self.multimodal_config = multimodal_config
1235
1236 - if multimodal_config.get_limit_per_prompt(
1237 - "image"
1238 - ) or multimodal_config.get_limit_per_prompt("video"):
1236 + with self._mark_tower_model(vllm_config, {"image", "video"}):
1237 self.visual = Qwen2VisionTransformer(
1238 config.vision_config,
1239 norm_eps=getattr(config, "rms_norm_eps", 1e-6),
@@ -1243,14 +1241,13 @@ class Qwen2VLForConditionalGeneration(
1241 multimodal_config=multimodal_config,
1242 prefix=maybe_prefix(prefix, "visual"),
1243 )
1246 - else:
1247 - self.visual = None
1244
1249 - self.language_model = init_vllm_registered_model(
1250 - vllm_config=vllm_config,
1251 - prefix=maybe_prefix(prefix, "language_model"),
1252 - architectures=["Qwen2ForCausalLM"],
1253 - )
1245 + with self._mark_language_model(vllm_config):
1246 + self.language_model = init_vllm_registered_model(
1247 + vllm_config=vllm_config,
1248 + prefix=maybe_prefix(prefix, "language_model"),
1249 + architectures=["Qwen2ForCausalLM"],
1250 + )
1251
1252 self.make_empty_intermediate_tensors = (
1253 self.language_model.make_empty_intermediate_tensors
@@ -1371,9 +1368,6 @@ class Qwen2VLForConditionalGeneration(
1368
1369 return modalities
1370
1374 - def get_language_model(self) -> torch.nn.Module:
1375 - return self.language_model
1376 -
1371 def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
1372 modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
1373 if not modalities:
@@ -1437,10 +1431,7 @@ class Qwen2VLForConditionalGeneration(
1431 return self.language_model.compute_logits(hidden_states)
1432
1433 def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
1440 - skip_prefixes = []
1441 - if self.visual is None:
1442 - skip_prefixes.extend(["visual."])
1443 - loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
1434 + loader = AutoWeightsLoader(self)
1435 return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
1436
1437 def get_mm_mapping(self) -> MultiModelKeys:
vllm/model_executor/models/qwen3_vl.py
+25 -43
@@ -1277,11 +1277,16 @@ class Qwen3VLForConditionalGeneration(
1277 multimodal_config.is_multimodal_pruning_enabled()
1278 )
1279
1280 - if not multimodal_config.get_limit_per_prompt(
1281 - "image"
1282 - ) and not multimodal_config.get_limit_per_prompt("video"):
1283 - self.visual = None
1284 - else:
1280 + self.use_deepstack = hasattr(config.vision_config, "deepstack_visual_indexes")
1281 + self.deepstack_num_level = (
1282 + len(config.vision_config.deepstack_visual_indexes)
1283 + if self.use_deepstack
1284 + else 0
1285 + )
1286 + self.visual_dim = config.vision_config.out_hidden_size
1287 + self.multiscale_dim = self.visual_dim * self.deepstack_num_level
1288 +
1289 + with self._mark_tower_model(vllm_config, {"image", "video"}):
1290 self.visual = Qwen3_VisionTransformer(
1291 config.vision_config,
1292 norm_eps=getattr(config, "rms_norm_eps", 1e-6),
@@ -1290,34 +1295,25 @@ class Qwen3VLForConditionalGeneration(
1295 prefix=maybe_prefix(prefix, "visual"),
1296 )
1297
1293 - self.language_model = Qwen3LLMForCausalLM(
1294 - vllm_config=vllm_config, prefix=maybe_prefix(prefix, "language_model")
1295 - )
1298 + # register buffer for deepstack
1299 + if self.use_deepstack:
1300 + self.deepstack_input_embeds = [
1301 + torch.zeros(
1302 + vllm_config.scheduler_config.max_num_batched_tokens,
1303 + config.text_config.hidden_size,
1304 + )
1305 + for _ in range(self.deepstack_num_level)
1306 + ]
1307 +
1308 + with self._mark_language_model(vllm_config):
1309 + self.language_model = Qwen3LLMForCausalLM(
1310 + vllm_config=vllm_config, prefix=maybe_prefix(prefix, "language_model")
1311 + )
1312
1313 self.make_empty_intermediate_tensors = (
1314 self.language_model.make_empty_intermediate_tensors
1315 )
1316
1301 - self.use_deepstack = hasattr(config.vision_config, "deepstack_visual_indexes")
1302 - self.deepstack_num_level = (
1303 - len(config.vision_config.deepstack_visual_indexes)
1304 - if self.use_deepstack
1305 - else 0
1306 - )
1307 - # register buffer for deepstack
1308 - if self.use_deepstack and self.visual is not None:
1309 - self.deepstack_input_embeds = [
1310 - torch.zeros(
1311 - vllm_config.scheduler_config.max_num_batched_tokens,
1312 - config.text_config.hidden_size,
1313 - )
1314 - for _ in range(self.deepstack_num_level)
1315 - ]
1316 - else:
1317 - self.deepstack_input_embeds = None
1318 - self.visual_dim = config.vision_config.out_hidden_size
1319 - self.multiscale_dim = self.visual_dim * self.deepstack_num_level
1320 -
1317 def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None:
1318 self.language_model.model.aux_hidden_state_layers = layers
1319
@@ -1893,9 +1889,6 @@ class Qwen3VLForConditionalGeneration(
1889
1890 return torch.from_numpy(llm_positions), mrope_position_delta
1891
1896 - def get_language_model(self) -> torch.nn.Module:
1897 - return self.language_model
1898 -
1892 def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings | None:
1893 mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)
1894 if not mm_input_by_modality:
@@ -2076,10 +2069,7 @@ class Qwen3VLForConditionalGeneration(
2069 return self.language_model.compute_logits(hidden_states)
2070
2071 def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
2079 - skip_prefixes = []
2080 - if self.visual is None:
2081 - skip_prefixes.extend(["visual."])
2082 - loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
2072 + loader = AutoWeightsLoader(self)
2073 return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
2074
2075 def get_mm_mapping(self) -> MultiModelKeys:
@@ -2110,11 +2100,3 @@ class Qwen3VLForConditionalGeneration(
2100 vision_config = hf_config.vision_config
2101 merge_size = vision_config.spatial_merge_size
2102 return num_vision_tokens // merge_size**2
2113 -
2114 - @classmethod
2115 - def get_language_model_spec(cls) -> tuple[nn.Module | None, str | None]:
2116 - """
2117 - Return the language model spec:
2118 - (language model class, language model attr)
2119 - """
2120 - return Qwen3LLMForCausalLM, "language_model"
vllm/model_executor/models/qwen3_vl_moe.py
+25 -28
@@ -424,11 +424,16 @@ class Qwen3VLMoeForConditionalGeneration(
424 multimodal_config.is_multimodal_pruning_enabled()
425 )
426
427 - if not multimodal_config.get_limit_per_prompt(
428 - "image"
429 - ) and not multimodal_config.get_limit_per_prompt("video"):
430 - self.visual = None
431 - else:
427 + self.use_deepstack = hasattr(config.vision_config, "deepstack_visual_indexes")
428 + self.deepstack_num_level = (
429 + len(config.vision_config.deepstack_visual_indexes)
430 + if self.use_deepstack
431 + else 0
432 + )
433 + self.visual_dim = config.vision_config.out_hidden_size
434 + self.multiscale_dim = self.visual_dim * self.deepstack_num_level
435 +
436 + with self._mark_tower_model(vllm_config, {"image", "video"}):
437 self.visual = Qwen3_VisionTransformer(
438 config.vision_config,
439 norm_eps=getattr(config, "rms_norm_eps", 1e-6),
@@ -437,9 +442,21 @@ class Qwen3VLMoeForConditionalGeneration(
442 prefix=maybe_prefix(prefix, "visual"),
443 )
444
440 - self.language_model = Qwen3MoeLLMForCausalLM(
441 - vllm_config=vllm_config, prefix=maybe_prefix(prefix, "language_model")
442 - )
445 + # register buffer for deepstack
446 + if self.use_deepstack:
447 + self.deepstack_input_embeds = [
448 + torch.zeros(
449 + vllm_config.scheduler_config.max_num_batched_tokens,
450 + config.text_config.hidden_size,
451 + )
452 + for _ in range(self.deepstack_num_level)
453 + ]
454 +
455 + with self._mark_language_model(vllm_config):
456 + self.language_model = Qwen3MoeLLMForCausalLM(
457 + vllm_config=vllm_config, prefix=maybe_prefix(prefix, "language_model")
458 + )
459 +
460 # Whether to include the gate_up_proj mapping is determined by
461 # the language model.
462 self.packed_modules_mapping = (
@@ -450,25 +467,5 @@ class Qwen3VLMoeForConditionalGeneration(
467 self.language_model.make_empty_intermediate_tensors
468 )
469
453 - self.use_deepstack = hasattr(config.vision_config, "deepstack_visual_indexes")
454 - self.deepstack_num_level = (
455 - len(config.vision_config.deepstack_visual_indexes)
456 - if self.use_deepstack
457 - else 0
458 - )
459 - # register buffer for deepstack
460 - if self.use_deepstack and self.visual is not None:
461 - self.deepstack_input_embeds = [
462 - torch.zeros(
463 - vllm_config.scheduler_config.max_num_batched_tokens,
464 - config.text_config.hidden_size,
465 - )
466 - for _ in range(self.deepstack_num_level)
467 - ]
468 - else:
469 - self.deepstack_input_embeds = None
470 - self.visual_dim = config.vision_config.out_hidden_size
471 - self.multiscale_dim = self.visual_dim * self.deepstack_num_level
472 -
470 # Set MoE hyperparameters
471 self.set_moe_parameters()
vllm/model_executor/models/step3_vl.py
+10 -27
@@ -942,7 +942,7 @@ class Step3VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)
942 self.multimodal_config = multimodal_config
943 self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
944
945 - if multimodal_config.get_limit_per_prompt("image"):
945 + with self._mark_tower_model(vllm_config, "image"):
946 self.vision_model = Step3VisionTransformer(
947 config.vision_config,
948 None,
@@ -967,17 +967,13 @@ class Step3VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)
967 config.hidden_size,
968 bias=config.projector_bias,
969 )
970 - else:
971 - self.vision_model = None
972 - self.vit_downsampler = None
973 - self.vit_downsampler2 = None
974 - self.vit_large_projector = None
975 -
976 - self.language_model = init_vllm_registered_model(
977 - vllm_config=vllm_config,
978 - hf_config=config.text_config,
979 - prefix=maybe_prefix(prefix, "language_model"),
980 - )
970 +
971 + with self._mark_language_model(vllm_config):
972 + self.language_model = init_vllm_registered_model(
973 + vllm_config=vllm_config,
974 + hf_config=config.text_config,
975 + prefix=maybe_prefix(prefix, "language_model"),
976 + )
977
978 self.make_empty_intermediate_tensors = (
979 self.language_model.make_empty_intermediate_tensors
@@ -1071,9 +1067,6 @@ class Step3VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)
1067 )
1068 return merged_image_features
1069
1074 - def get_language_model(self) -> torch.nn.Module:
1075 - return self.language_model
1076 -
1070 def embed_multimodal(self, **kwargs) -> MultiModalEmbeddings:
1071 image_input = self._parse_and_validate_image_input(**kwargs)
1072 if image_input is None:
@@ -1133,15 +1126,5 @@ class Step3VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)
1126 return self.language_model.compute_logits(hidden_states)
1127
1128 def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
1136 - skip_prefixes = []
1137 - if self.vision_model is None and self.vit_large_projector is None:
1138 - skip_prefixes = [
1139 - "vision_model.",
1140 - "vit_downsampler.",
1141 - "vit_downsampler2.",
1142 - "vit_large_projector.",
1143 - ]
1144 -
1145 - loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
1146 - loaded_weights = loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
1147 - return loaded_weights
1129 + loader = AutoWeightsLoader(self)
1130 + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
vllm/model_executor/models/step_vl.py
+8 -9
@@ -504,7 +504,8 @@ class StepVLForConditionalGeneration(Step3VLForConditionalGeneration):
504 self.config = config
505 self.multimodal_config = multimodal_config
506 self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
507 - if multimodal_config.get_limit_per_prompt("image"):
507 +
508 + with self._mark_tower_model(vllm_config, "image"):
509 self.vision_model = PerceptionEncoder(
510 config.vision_config,
511 get_act_fn(config.vision_config.hidden_act),
@@ -521,15 +522,13 @@ class StepVLForConditionalGeneration(Step3VLForConditionalGeneration):
522 prefix=maybe_prefix(prefix, "vit_large_projector"),
523 disable_tp=self.use_data_parallel,
524 )
524 - else:
525 - self.vision_model = None
526 - self.vit_large_projector = None
525
528 - self.language_model = init_vllm_registered_model(
529 - vllm_config=vllm_config,
530 - hf_config=config.text_config,
531 - prefix=maybe_prefix(prefix, "language_model"),
532 - )
526 + with self._mark_language_model(vllm_config):
527 + self.language_model = init_vllm_registered_model(
528 + vllm_config=vllm_config,
529 + hf_config=config.text_config,
530 + prefix=maybe_prefix(prefix, "language_model"),
531 + )
532
533 self.make_empty_intermediate_tensors = (
534 self.language_model.make_empty_intermediate_tensors
vllm/model_executor/models/utils.py
+6 -2
@@ -24,7 +24,11 @@ from vllm.model_executor.model_loader.online_quantization import (
24 support_quantized_model_reload_from_hp_weights,
25 )
26 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
27 -from vllm.model_executor.models.interfaces import supports_any_eagle
27 +from vllm.model_executor.models.interfaces import (
28 + LMMissingLayer,
29 + TowerMissingLayer,
30 + supports_any_eagle,
31 +)
32 from vllm.multimodal import NestedTensors
33 from vllm.sequence import IntermediateTensors
34 from vllm.utils.math_utils import cdiv
@@ -250,7 +254,7 @@ class AutoWeightsLoader:
254 module: nn.Module,
255 weights: Iterable[tuple[str, torch.Tensor]],
256 ) -> Iterable[str]:
253 - if isinstance(module, PPMissingLayer):
257 + if isinstance(module, (LMMissingLayer, TowerMissingLayer, PPMissingLayer)):
258 return
259
260 # Avoid infinite recursion since this function is typically
vllm/v1/worker/gpu_model_runner.py
+6 -4
@@ -65,7 +65,6 @@ from vllm.model_executor.models.interfaces import (
65 SupportsXDRoPE,
66 is_mixture_of_experts,
67 supports_eagle3,
68 - supports_mm_encoder_only,
68 supports_mrope,
69 supports_multimodal_pruning,
70 supports_transcription,
@@ -4271,7 +4270,8 @@ class GPUModelRunner(
4270 remove_lora: If False, dummy LoRAs are not destroyed after the run
4271 activate_lora: If False, dummy_run is performed without LoRAs.
4272 """
4274 - if supports_mm_encoder_only(self.model):
4273 + mm_config = self.vllm_config.model_config.multimodal_config
4274 + if mm_config and mm_config.mm_encoder_only:
4275 # The current dummy run only covers LM execution, so we can skip it.
4276 # mm encoder dummy run may need to add in the future.
4277 return torch.tensor([]), torch.tensor([])
@@ -4558,7 +4558,8 @@ class GPUModelRunner(
4558 # like `inf` or `nan`.
4559 # To avoid breaking the sampler, we use a random tensor here instead.
4560
4561 - if supports_mm_encoder_only(self.model):
4561 + mm_config = self.vllm_config.model_config.multimodal_config
4562 + if mm_config and mm_config.mm_encoder_only:
4563 # MM Encoder only model no need to run sampler.
4564 return torch.tensor([])
4565
@@ -4687,7 +4688,8 @@ class GPUModelRunner(
4688 self,
4689 hidden_states: torch.Tensor,
4690 ) -> PoolerOutput:
4690 - if supports_mm_encoder_only(self.model):
4691 + mm_config = self.vllm_config.model_config.multimodal_config
4692 + if mm_config and mm_config.mm_encoder_only:
4693 # MM Encoder only model not need to run pooler.
4694 return torch.tensor([])
4695