A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 19:00 UTC
[FIX] fused moe with lora shared expert dual stream (1.07x otps) (#34933)
Signed-off-by: Jhao-Ting Chen <jhaotingc@nvidia.com> Co-authored-by: Michael Goin <mgoin64@gmail.com>
Jhao-Ting Chen committed
Feb 24, 2026 at 20:40 UTC
c2c4c4611a6d92f3006dfdedf8598a70d39002b3
1 file changed
+6
-2
vllm/lora/layers/fused_moe.py
+6
-2
@@ -133,15 +133,19 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
133
if getattr(self.base_layer.quant_method, "supports_internal_mk", False):
134
# Use the existing modular kernel from the quant method
135
m_fused_moe_fn = self.base_layer.quant_method.moe_mk
136
+ # Don't let the kernel own shared experts so the runner can
137
+ # overlap them with routed experts via a separate CUDA stream.
138
+ m_fused_moe_fn.shared_experts = None
139
else:
137
- # Create a new modular kernel via select_gemm_impl
140
+ # Create a new modular kernel via select_gemm_impl.
141
+ # Don't pass shared_experts to the kernel so the runner can
142
+ # overlap them with routed experts via a separate CUDA stream.
143
prepare_finalize = MoEPrepareAndFinalizeNoEP()
144
m_fused_moe_fn = FusedMoEModularKernel(
145
prepare_finalize,
146
self.base_layer.quant_method.select_gemm_impl(
147
prepare_finalize, self.base_layer
148
),
144
- self.base_layer.shared_experts,
149
)
150
151
if quant_config.use_mxfp4_w4a16: