A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 16:00 UTC
[Bugfix] fix use-ep bug to enable ep by dp/tp size > 1 (#16161)
zxfan-cpu committed
Apr 8, 2025 at 11:48 UTC
ad971af8c7dc2c006b58853207b36942569a10ee
1 file changed
+1
-1
vllm/model_executor/layers/fused_moe/layer.py
+1
-1
@@ -437,7 +437,7 @@ class FusedMoE(torch.nn.Module):
437
# Use expert parallelism instead of tensor parallelism?
438
vllm_config = get_current_vllm_config()
439
use_ep = (vllm_config.parallel_config.enable_expert_parallel
440
- and self.tp_size > 1)
440
+ and self.tp_size * self.dp_size > 1)
441
442
# For smuggling this layer into the fused moe custom op
443
self.use_direct_call = self.dp_size == 1