A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 07:30 UTC
[Bug] Fix status update address for non-MOE model within external dp mode (#40839)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
Wentao Ye committed
May 4, 2026 at 19:37 UTC
577b9623e6f8801698d411f4b04269326f5afbe2
5 files changed
+19
-7
docs/serving/data_parallel_deployment.md
+1
-1
@@ -98,7 +98,7 @@ For larger scale deployments especially, it can make sense to handle the orchest
98
99
In this case, it's more convenient to treat each DP rank like a separate vLLM deployment, with its own endpoint, and have an external router balance HTTP requests between them, making use of appropriate real-time telemetry from each server for routing decisions.
100
101
-This can already be done trivially for non-MoE models, since each deployed server is fully independent. No data parallel CLI options need to be used for this.
101
+This can already be done trivially for non-MoE models, since each deployed server is fully independent. In that case, launch independent vLLM instances without any `--data-parallel-*` arguments; external DP CLI options are only supported for MoE deployments.
102
103
We support an equivalent topology for MoE DP+EP which can be configured via the following CLI arguments.
104
tests/test_config.py
-2
@@ -1215,8 +1215,6 @@ def test_scheduler_config_init():
1215
("facebook/opt-125m", 1, False, False),
1216
# Non-MoE model with DP>1 internal LB should need coordinator
1217
("facebook/opt-125m", 2, False, True),
1218
- # Non-MoE model with DP>1 external LB should not need coordinator
1219
- ("facebook/opt-125m", 2, True, False),
1218
# MoE model with DP=1 should not need coordinator
1219
("mistralai/Mixtral-8x7B-Instruct-v0.1", 1, False, False),
1220
# MoE model with DP>1 internal LB should need both coordinator
tests/v1/distributed/test_external_lb_dp.py
+1
-1
@@ -14,7 +14,7 @@ import requests
14
from tests.utils import RemoteOpenAIServer
15
from vllm.platforms import current_platform
16
17
-MODEL_NAME = "ibm-research/PowerMoE-3b"
17
+MODEL_NAME = os.getenv("MODEL_NAME", "ibm-research/PowerMoE-3b")
18
19
# Number of data parallel ranks for external LB testing
20
DP_SIZE = int(os.getenv("DP_SIZE", "2"))
vllm/config/parallel.py
+4
-2
@@ -135,8 +135,10 @@ class ParallelConfig:
135
data_parallel_external_lb: bool = False
136
"""Whether to use "external" DP LB mode. Applies only to online serving
137
and when data_parallel_size > 0. This is useful for a "one-pod-per-rank"
138
- wide-EP setup in Kubernetes. Set implicitly when --data-parallel-rank
139
- is provided explicitly to vllm serve."""
138
+ wide-EP setup in Kubernetes. Supported only for MoE deployments; non-MoE
139
+ models should use independent vLLM instances without --data-parallel-*
140
+ arguments. Set implicitly when --data-parallel-rank is provided explicitly
141
+ to vllm serve."""
142
data_parallel_hybrid_lb: bool = False
143
"""Whether to use "hybrid" DP LB mode. Applies only to online serving
144
and when data_parallel_size > 0. Enables running an AsyncLLM
vllm/engine/arg_utils.py
+13
-1
@@ -962,7 +962,9 @@ class EngineArgs:
962
"-dpn",
963
type=int,
964
help="Data parallel rank of this instance. "
965
- "When set, enables external load balancer mode.",
965
+ "When set, enables external load balancer mode for MoE "
966
+ "data-parallel deployments. Unsupported for non-MoE models; "
967
+ "launch independent vLLM instances instead.",
968
)
969
parallel_group.add_argument(
970
"--data-parallel-start-rank",
@@ -1793,6 +1795,16 @@ class EngineArgs:
1795
data_parallel_external_lb = (
1796
self.data_parallel_external_lb or self.data_parallel_rank is not None
1797
)
1798
+ if (
1799
+ self.data_parallel_size > 1
1800
+ and data_parallel_external_lb
1801
+ and not model_config.is_moe
1802
+ ):
1803
+ raise ValueError(
1804
+ "Non-MoE models do not support external data parallel mode. "
1805
+ "For external load balancing, launch independent vLLM "
1806
+ "instances without --data-parallel-* arguments."
1807
+ )
1808
# Local DP rank = 1, use pure-external LB.
1809
if data_parallel_external_lb:
1810
assert self.data_parallel_rank is not None, (