A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Model] Interface to enable batch-level DP support (#23733)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Cyrus Leung committed
Aug 27, 2025 at 21:41 UTC
fe8d7b6f03e7d8a36ffb6931397fc81ee594dd64
8 files changed
+38
-4
docs/configuration/optimization.md
+5
-2
@@ -168,8 +168,11 @@ llm = LLM(
168
Batch-level DP is not to be confused with API request-level DP
169
(which is instead controlled by `data_parallel_size`).
170
171
-The availability of batch-level DP is based on model implementation.
172
-Currently, the following models support `mm_encoder_tp_mode="data"`:
171
+Batch-level DP needs to be implemented on a per-model basis,
172
+and enabled by setting `supports_encoder_tp_data = True` in the model class.
173
+Regardless, you need to set `mm_encoder_tp_mode="data"` in engine arguments to use this feature.
174
+
175
+Known supported models:
176
177
- Llama4 (<gh-pr:18368>)
178
- MiniCPM-V-4 (<gh-pr:23327>)
vllm/config/__init__.py
+7
@@ -872,6 +872,13 @@ class ModelConfig:
872
873
def _init_multimodal_config(self) -> Optional["MultiModalConfig"]:
874
if self._model_info.supports_multimodal:
875
+ if (self.mm_encoder_tp_mode == "data" and
876
+ not self._model_info.supports_multimodal_encoder_tp_data):
877
+ logger.warning_once(
878
+ "This model does not support `--mm-encoder-tp-mode data`. "
879
+ "Falling back to `--mm-encoder-tp-mode weights`.")
880
+ self.mm_encoder_tp_mode = "weights"
881
+
882
return MultiModalConfig(
883
limit_per_prompt=self.limit_mm_per_prompt,
884
media_io_kwargs=self.media_io_kwargs,
vllm/model_executor/models/interfaces.py
+11
@@ -52,6 +52,12 @@ class SupportsMultiModal(Protocol):
52
MRO of your model class.
53
"""
54
55
+ supports_encoder_tp_data: ClassVar[bool] = False
56
+ """
57
+ A flag that indicates whether this model supports
58
+ `multimodal_config.mm_encoder_tp_mode="data"`.
59
+ """
60
+
61
@classmethod
62
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
63
"""
@@ -137,6 +143,11 @@ def supports_multimodal(
143
return getattr(model, "supports_multimodal", False)
144
145
146
+def supports_multimodal_encoder_tp_data(
147
+ model: Union[type[object], object]) -> bool:
148
+ return getattr(model, "supports_encoder_tp_data", False)
149
+
150
+
151
@runtime_checkable
152
class SupportsMultiModalWithRawInput(SupportsMultiModal, Protocol):
153
"""The interface required for all multi-modal models."""
vllm/model_executor/models/minicpmv.py
+2
@@ -1521,6 +1521,8 @@ class MiniCPMV4_0(MiniCPMVBaseModel, SupportsLoRA):
1521
],
1522
}
1523
1524
+ supports_encoder_tp_data = True
1525
+
1526
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1527
super().__init__(vllm_config=vllm_config, prefix=prefix)
1528
assert self.version == (4, 0)
vllm/model_executor/models/mllama4.py
+2
@@ -716,6 +716,8 @@ class Llama4ForConditionalGeneration(nn.Module, SupportsMultiModal,
716
"gate_up_proj": ["gate_proj", "up_proj"],
717
}
718
719
+ supports_encoder_tp_data = True
720
+
721
@classmethod
722
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
723
if modality.startswith("image"):
vllm/model_executor/models/qwen2_5_vl.py
+2
@@ -868,6 +868,8 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,
868
"model.": "language_model.model.",
869
})
870
871
+ supports_encoder_tp_data = True
872
+
873
@classmethod
874
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
875
if modality.startswith("image"):
vllm/model_executor/models/registry.py
+7
-2
@@ -27,8 +27,10 @@ from vllm.transformers_utils.dynamic_module import (
27
28
from .interfaces import (has_inner_state, has_noops, is_attention_free,
29
is_hybrid, supports_cross_encoding,
30
- supports_multimodal, supports_multimodal_raw_input,
31
- supports_pp, supports_transcription, supports_v0_only)
30
+ supports_multimodal,
31
+ supports_multimodal_encoder_tp_data,
32
+ supports_multimodal_raw_input, supports_pp,
33
+ supports_transcription, supports_v0_only)
34
from .interfaces_base import (get_default_pooling_type, is_pooling_model,
35
is_text_generation_model)
36
@@ -324,6 +326,7 @@ class _ModelInfo:
326
supports_cross_encoding: bool
327
supports_multimodal: bool
328
supports_multimodal_raw_input: bool
329
+ supports_multimodal_encoder_tp_data: bool
330
supports_pp: bool
331
has_inner_state: bool
332
is_attention_free: bool
@@ -343,6 +346,8 @@ class _ModelInfo:
346
supports_cross_encoding=supports_cross_encoding(model),
347
supports_multimodal=supports_multimodal(model),
348
supports_multimodal_raw_input=supports_multimodal_raw_input(model),
349
+ supports_multimodal_encoder_tp_data=
350
+ supports_multimodal_encoder_tp_data(model),
351
supports_pp=supports_pp(model),
352
has_inner_state=has_inner_state(model),
353
is_attention_free=is_attention_free(model),
vllm/model_executor/models/step3_vl.py
+2
@@ -867,6 +867,8 @@ class Step3VLForConditionalGeneration(nn.Module, SupportsMultiModal,
867
"lm_head.": "language_model.lm_head.",
868
})
869
870
+ supports_encoder_tp_data = True
871
+
872
@classmethod
873
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
874
if modality.startswith("image"):