A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 12:15 UTC
[Bugfix][MoE] Fix 6-8% decode regression: prefer multi-stream shared expert overlap (#38990)
Signed-off-by: Martin Vit <martin@voipmonitor.org> Signed-off-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
Martin Vit committed
Apr 5, 2026 at 16:28 UTC
228023b3a58f78ed028cb4e5fb4e078bb1574262
1 file changed
+9
-9
vllm/model_executor/layers/fused_moe/runner/shared_experts.py
+9
-9
@@ -93,24 +93,24 @@ class SharedExperts:
93
)
94
95
@property
96
- def _has_external_experts(self) -> bool:
96
+ def _use_external_experts(self) -> bool:
97
+ if self._use_dp_chunking:
98
+ return False
99
+
100
# Disable shared expert overlap if:
101
# - we are using eplb with non-default backend, because of correctness issues
102
# - we are using flashinfer with DP, since there nothing to gain
103
backend = self._moe_config.moe_parallel_config.all2all_backend
101
- return not (
102
- (
103
- self._moe_config.moe_parallel_config.enable_eplb
104
- and backend != "allgather_reducescatter"
105
- )
106
- or self._moe_config.moe_parallel_config.use_fi_nvl_two_sided_kernels
107
- )
104
+ return (
105
+ self._moe_config.moe_parallel_config.enable_eplb
106
+ and backend != "allgather_reducescatter"
107
+ ) or self._moe_config.moe_parallel_config.use_fi_nvl_two_sided_kernels
108
109
def _determine_shared_experts_order(
110
self,
111
hidden_states: torch.Tensor,
112
) -> SharedExpertsOrder:
113
- if self._has_external_experts and not self._use_dp_chunking:
113
+ if self._use_external_experts:
114
return SharedExpertsOrder.EXTERNAL
115
116
if self._quant_method.mk_owns_shared_expert: