@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8f8f469b1b

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 11, 01:15 UTC

[BugFix] skip language model in Encoder (#30242)

Signed-off-by: dengyunyang <584797741@qq.com>

dengyunyang committed Dec 22, 2025 at 21:25 UTC 8f8f469b1b8c4ef44a0a8a5f0bc6ddf86acbc9db

8 files changed +116 -3

examples/online_serving/disaggregated_encoder/README.md

@@ -38,6 +38,8 @@ Encoder engines should be launched with the following flags:

39 - `--max-num-batched-tokens=<large value>` **(default: 2048)** – This flag controls the token scheduling budget per decoding step and is irrelevant to encoder-only instances. **Set it to a very high value (effectively unlimited) to bypass scheduler limitations.** The actual token budget is managed by the encoder cache manager.

41 +- `--convert "mm_encoder_only"` **(Optional)** - The language model is skipped during initialization to reduce device memory usage. **Models using this option must implement the `get_language_model_spec` interface.**

42 +

43 ## Local media inputs

45 To support local image inputs (from your ```MEDIA_PATH``` directory), add the following flag to the encoder instance:

vllm/config/model.py

+1 -1

@@ -71,7 +71,7 @@ else:

71 logger = init_logger(__name__)

73 RunnerOption = Literal["auto", RunnerType]

74 -ConvertType = Literal["none", "embed", "classify", "reward"]

74 +ConvertType = Literal["none", "embed", "classify", "reward", "mm_encoder_only"]

75 ConvertOption = Literal["auto", ConvertType]

76 TokenizerMode = Literal["auto", "hf", "slow", "mistral", "deepseek_v32"]

77 ModelDType = Literal["auto", "half", "float16", "bfloat16", "float", "float32"]

vllm/model_executor/model_loader/utils.py

+8 -1

@@ -189,7 +189,9 @@ def _get_model_architecture(model_config: ModelConfig) -> tuple[type[nn.Module],

189 )

190

191 convert_type = model_config.convert_type

192 - if convert_type != "none" and supports_multimodal(model_cls):

192 + if convert_type not in ["none", "mm_encoder_only"] and supports_multimodal(

193 + model_cls

194 + ):

195 logger.debug_once("Detected conversion of Multi Modal model.")

196 converted = try_create_mm_pooling_model_cls(model_cls)

197 if converted is not None:

@@ -200,6 +202,11 @@ def _get_model_architecture(model_config: ModelConfig) -> tuple[type[nn.Module],

202

203 if convert_type == "none":

204 pass

205 + elif convert_type == "mm_encoder_only":

206 + logger.debug_once("Converting to mm encoder only model.")

207 + from vllm.model_executor.models.adapters import as_mm_encoder_only_model

208 +

209 + model_cls = as_mm_encoder_only_model(model_cls)

210 elif convert_type == "embed":

211 logger.debug_once("Converting to embedding model.")

212 model_cls = as_embedding_model(model_cls)

vllm/model_executor/models/adapters.py

+61

@@ -520,3 +520,64 @@ def seq_cls_model_loader(model, weights: Iterable[tuple[str, torch.Tensor]]):

520 method = getattr(text_config, "method", None)

521 assert method in SEQ_CLS_LOAD_METHODS, f"method {method} not supported"

522 return SEQ_CLS_LOAD_METHODS[method](model, weights)

523 +

524 +

525 +def as_mm_encoder_only_model(cls: _T) -> _T:

526 + """

527 + Subclass an existing vLLM vl model to support mm encoder only for

528 + EPD encoder instances.

529 + """

530 + if not hasattr(cls, "embed_multimodal"):

531 + # Submodel case: return the original class.

532 + return cls

533 +

534 + if not hasattr(cls, "get_language_model_spec"):

535 + raise TypeError(f"{cls} need to implement `get_language_model_spec` method.")

536 +

537 + lm_model_cls, lm_attr = cls.get_language_model_spec()

538 +

539 + if lm_model_cls is None or lm_attr is None:

540 + raise TypeError(

541 + f"{cls}.get_language_model_spec() must return (lm_model_cls, lm_attr)"

542 + )

543 +

544 + class DummyLM(nn.Module):

545 + def __init__(self, *args, **kwargs):

546 + self.make_empty_intermediate_tensors = None

547 +

548 + class ModelForMMEncoderOnly(cls):

549 + def __init__(

550 + self,

551 + *,

552 + vllm_config: "VllmConfig",

553 + prefix: str = "",

554 + **kwargs: Any,

555 + ) -> None:

556 + self.is_mm_encoder_only_model = True

557 + origin_init = lm_model_cls.__init__

558 + try:

559 + lm_model_cls.__init__ = DummyLM.__init__

560 + super().__init__(vllm_config=vllm_config, prefix=prefix, **kwargs)

561 +

562 + if hasattr(self, lm_attr):

563 + delattr(self, lm_attr)

564 + finally:

565 + lm_model_cls.__init__ = origin_init

566 +

567 + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:

568 + from .utils import AutoWeightsLoader

569 +

570 + origin_init_ = AutoWeightsLoader.__init__

571 +

572 + def _new_init_(self, *args, **kwargs):

573 + origin_init_(self, *args, **kwargs)

574 + self.skip_prefixes = (self.skip_prefixes or []) + [f"{lm_attr}."]

575 +

576 + try:

577 + AutoWeightsLoader.__init__ = _new_init_

578 + result = super().load_weights(weights)

579 + finally:

580 + AutoWeightsLoader.__init__ = origin_init_

581 + return result

582 +

583 + return ModelForMMEncoderOnly # type: ignore

vllm/model_executor/models/interfaces.py

+12

@@ -141,6 +141,14 @@ class SupportsMultiModal(Protocol):

141 """

142 ...

143

144 + @classmethod

145 + def get_language_model_spec(cls) -> tuple[nn.Module | None, str | None]:

146 + """

147 + Return the language model spec:

148 + (language model class, language model attr)

149 + """

150 + return None, None

151 +

152 @overload

153 def embed_input_ids(self, input_ids: Tensor) -> Tensor: ...

154

@@ -302,6 +310,10 @@ def supports_multimodal_encoder_tp_data(model: type[object] | object) -> bool:

310 return getattr(model, "supports_encoder_tp_data", False)

311

312

313 +def supports_mm_encoder_only(model: type[object] | object) -> bool:

314 + return getattr(model, "is_mm_encoder_only_model", False)

315 +

316 +

317 @overload

318 def supports_multimodal_pruning(

319 model: type[object],

vllm/model_executor/models/qwen2_5_vl.py

+9 -1

@@ -34,7 +34,7 @@ import einops

34 import torch

35 import torch.nn as nn

36 import torch.nn.functional as F

37 -from transformers import BatchFeature

37 +from transformers import BatchFeature, Qwen2ForCausalLM

38 from transformers.models.qwen2_5_vl import Qwen2_5_VLProcessor

39 from transformers.models.qwen2_5_vl.configuration_qwen2_5_vl import (

40 Qwen2_5_VLConfig,

@@ -1567,3 +1567,11 @@ class Qwen2_5_VLForConditionalGeneration(

1567 connector="visual.merger.",

1568 tower_model="visual.",

1569 )

1570 +

1571 + @classmethod

1572 + def get_language_model_spec(cls) -> tuple[nn.Module | None, str | None]:

1573 + """

1574 + Return the language model spec:

1575 + (language model class, language model attr)

1576 + """

1577 + return Qwen2ForCausalLM, "language_model"

vllm/model_executor/models/qwen3_vl.py

@@ -2090,3 +2090,11 @@ class Qwen3VLForConditionalGeneration(

2090 connector="visual.merger",

2091 tower_model="visual.",

2092 )

2093 +

2094 + @classmethod

2095 + def get_language_model_spec(cls) -> tuple[nn.Module | None, str | None]:

2096 + """

2097 + Return the language model spec:

2098 + (language model class, language model attr)

2099 + """

2100 + return Qwen3LLMForCausalLM, "language_model"

vllm/v1/worker/gpu_model_runner.py

+15

@@ -66,6 +66,7 @@ from vllm.model_executor.models.interfaces import (

66 SupportsXDRoPE,

67 is_mixture_of_experts,

68 supports_eagle3,

69 + supports_mm_encoder_only,

70 supports_mrope,

71 supports_multimodal_pruning,

72 supports_transcription,

@@ -4067,6 +4068,11 @@ class GPUModelRunner(

4068 remove_lora: If False, dummy LoRAs are not destroyed after the run

4069 activate_lora: If False, dummy_run is performed without LoRAs.

4070 """

4071 + if supports_mm_encoder_only(self.model):

4072 + # The current dummy run only covers LM execution, so we can skip it.

4073 + # mm encoder dummy run may need to add in the future.

4074 + return torch.tensor([]), torch.tensor([])

4075 +

4076 assert (

4077 cudagraph_runtime_mode is None

4078 or cudagraph_runtime_mode.valid_runtime_modes()

@@ -4344,6 +4350,11 @@ class GPUModelRunner(

4350 # The dummy hidden states may contain special values,

4351 # like `inf` or `nan`.

4352 # To avoid breaking the sampler, we use a random tensor here instead.

4353 +

4354 + if supports_mm_encoder_only(self.model):

4355 + # MM Encoder only model no need to run sampler.

4356 + return torch.tensor([])

4357 +

4358 hidden_states = torch.rand_like(hidden_states)

4359

4360 logits = self.model.compute_logits(hidden_states)

@@ -4472,6 +4483,10 @@ class GPUModelRunner(

4483 self,

4484 hidden_states: torch.Tensor,

4485 ) -> PoolerOutput:

4486 + if supports_mm_encoder_only(self.model):

4487 + # MM Encoder only model not need to run pooler.

4488 + return torch.tensor([])

4489 +

4490 # Find the task that has the largest output for subsequent steps

4491 supported_pooling_tasks = self.get_supported_pooling_tasks()

4492