[Feature] Support sequence parallel without the need for DP, 1.9%~5.0% E2E Throughput Improvement (#47070)
Signed-off-by: yewentao256 <zhyanwentao@126.com> Signed-off-by: gcanlin <canlinguosdu@gmail.com> Co-authored-by: Canlin Guo <canlinguosdu@gmail.com>
Wentao Ye committed
Jul 5, 2026 at 08:41 UTC
b6cc46ec3b903c71405f4355c1e9ecb47ae54bb2
7 files changed
+33
-22
vllm/config/parallel.py
+2
-3
@@ -632,8 +632,8 @@ class ParallelConfig:
632
633
# The all_reduce at the end of attention (during o_proj) means that
634
# inputs are replicated across each rank of the tensor parallel group.
635
- # If using expert-parallelism with DeepEP All2All ops, replicated
636
- # tokens results in useless duplicate computation and communication.
635
+ # If using expert-parallelism, replicated tokens results in useless
636
+ # duplicate computation and communication.
637
#
638
# In this case, ensure the input to the experts is sequence parallel
639
# to avoid the excess work.
@@ -652,7 +652,6 @@ class ParallelConfig:
652
)
653
and self.enable_expert_parallel
654
and self.tensor_parallel_size > 1
655
- and self.data_parallel_size > 1
655
)
656
657
@property
vllm/distributed/device_communicators/base_device_communicator.py
+5
-4
@@ -175,10 +175,11 @@ class DeviceCommunicatorBase:
175
176
config = get_current_vllm_config_or_none()
177
if config is not None:
178
- # as long as we use data parallel (coupled data parallel
179
- # where all data parallel ranks execute forward together),
180
- # we initialize the all2all manager used in expert parallel.
181
- use_ep = config.parallel_config.data_parallel_size > 1
178
+ # initialize the all2all manager for DP or sequence-parallel EP.
179
+ use_ep = (
180
+ config.parallel_config.data_parallel_size > 1
181
+ or config.parallel_config.use_sequence_parallel_moe
182
+ )
183
all2all_backend = config.parallel_config.all2all_backend
184
185
self.is_ep_communicator = unique_name.split(":")[0] == "ep"
vllm/forward_context.py
+15
-3
@@ -83,7 +83,10 @@ class DPMetadata:
83
num_tokens_across_dp_cpu: torch.Tensor,
84
) -> "DPMetadata":
85
assert num_tokens_across_dp_cpu is not None
86
- assert parallel_config.data_parallel_size > 1
86
+ assert (
87
+ parallel_config.data_parallel_size > 1
88
+ or parallel_config.use_sequence_parallel_moe
89
+ )
90
assert parallel_config.is_moe_model is not False
91
dp_rank = parallel_config.data_parallel_rank
92
batchsize = num_tokens
@@ -277,14 +280,20 @@ def set_forward_context(
280
281
dp_metadata: DPMetadata | None = None
282
if (
280
- vllm_config.parallel_config.data_parallel_size > 1
283
+ (
284
+ vllm_config.parallel_config.data_parallel_size > 1
285
+ or vllm_config.parallel_config.use_sequence_parallel_moe
286
+ )
287
and vllm_config.parallel_config.is_moe_model is not False
288
and (attn_metadata is not None or num_tokens is not None)
289
):
290
# If num_tokens_across_dp hasn't already been initialized, then
291
# initialize it here. Both DP padding and Microbatching will be
292
# disabled.
287
- if num_tokens_across_dp is None:
293
+ if (
294
+ num_tokens_across_dp is None
295
+ and vllm_config.parallel_config.data_parallel_size > 1
296
+ ):
297
assert ubatch_slices is None
298
assert num_tokens is not None
299
_, num_tokens_across_dp, _ = coordinate_batch_across_dp(
@@ -293,6 +302,9 @@ def set_forward_context(
302
allow_microbatching=False,
303
)
304
assert num_tokens_across_dp is not None
305
+ elif num_tokens_across_dp is None:
306
+ assert num_tokens is not None
307
+ num_tokens_across_dp = torch.tensor([num_tokens], dtype=torch.int32)
308
dp_metadata = DPMetadata.make(
309
vllm_config.parallel_config, num_tokens or 0, num_tokens_across_dp
310
)
vllm/model_executor/layers/fused_moe/config.py
+1
-1
@@ -1038,7 +1038,7 @@ class FusedMoEParallelConfig:
1038
1039
@property
1040
def use_all2all_kernels(self):
1041
- return self.dp_size > 1 and self.use_ep
1041
+ return self.use_ep and (self.dp_size > 1 or self.is_sequence_parallel)
1042
1043
@property
1044
def use_deepep_ht_kernels(self):
vllm/model_executor/layers/fused_moe/runner/moe_runner.py
+2
-2
@@ -725,8 +725,8 @@ class MoERunner(MoERunnerInterface):
725
@property
726
def do_naive_dispatch_combine(self) -> bool:
727
return (
728
- self.moe_config.dp_size > 1 and not self._quant_method.supports_internal_mk
729
- )
728
+ self.moe_config.dp_size > 1 or self.moe_config.is_sequence_parallel
729
+ ) and not self._quant_method.supports_internal_mk
730
731
def _maybe_dispatch(
732
self,
vllm/model_executor/models/deepseek_v2.py
+4
-8
@@ -1305,15 +1305,11 @@ class DeepseekV2DecoderLayer(nn.Module):
1305
residual *= 1.0 / self.routed_scaling_factor
1306
1307
if self.use_sequence_parallel_moe:
1308
- sp_remainder = (
1309
- hidden_states.shape[0] % get_tensor_model_parallel_world_size()
1310
- )
1308
+ tp_world_size = get_tensor_model_parallel_world_size()
1309
+ # small trick using minus, eg. -17 % 8 = 7
1310
+ sp_pad = (-hidden_states.shape[0]) % tp_world_size
1311
# pad if not divisible by world size
1312
- if sp_remainder:
1313
- sp_pad = get_tensor_model_parallel_world_size() - sp_remainder
1314
- hidden_states = torch.nn.functional.pad(
1315
- hidden_states, (0, 0, 0, sp_pad)
1316
- )
1312
+ hidden_states = torch.nn.functional.pad(hidden_states, (0, 0, 0, sp_pad))
1313
hidden_states = tensor_model_parallel_reduce_scatter(hidden_states, 0)
1314
if not input_is_sequence_parallel:
1315
residual = sequence_parallel_chunk(residual)
vllm/model_executor/models/gpt_oss.py
+4
-1
@@ -192,7 +192,10 @@ class MLPBlock(torch.nn.Module):
192
quant_config = vllm_config.quant_config
193
parallel_config = vllm_config.parallel_config
194
195
- self.is_sequence_parallel = parallel_config.use_sequence_parallel_moe
195
+ self.is_sequence_parallel = (
196
+ parallel_config.use_sequence_parallel_moe
197
+ and vllm_config.lora_config is None
198
+ )
199
200
self.layer_idx = layer_idx
201
self.num_experts = config.num_local_experts