A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 14:45 UTC
[Perf] Disable DeepGEMM MoE by default when TP=8 is used (#29346)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Nov 24, 2025 at 21:01 UTC
6f1355a1b74e4502e6a4e6ba9a811cc50729ee1f
1 file changed
+20
-4
vllm/model_executor/layers/quantization/fp8.py
+20
-4
@@ -28,6 +28,7 @@ from vllm.model_executor.layers.fused_moe import (
28
FusedMoeWeightScaleSupported,
29
)
30
from vllm.model_executor.layers.fused_moe.config import (
31
+ FusedMoEParallelConfig,
32
FusedMoEQuantConfig,
33
RoutingMethodType,
34
fp8_w8a8_moe_quant_config,
@@ -118,7 +119,9 @@ class Fp8MoeBackend(Enum):
119
TRITON = 6
120
121
121
-def get_fp8_moe_backend(block_quant: bool) -> Fp8MoeBackend:
122
+def get_fp8_moe_backend(
123
+ block_quant: bool, moe_parallel_config: FusedMoEParallelConfig
124
+) -> Fp8MoeBackend:
125
"""
126
Select the primary FP8 MoE backend
127
Note: Shape-specific fallbacks may still occur at runtime.
@@ -159,8 +162,19 @@ def get_fp8_moe_backend(block_quant: bool) -> Fp8MoeBackend:
162
logger.info_once("Using Marlin backend for FP8 MoE")
163
return Fp8MoeBackend.MARLIN
164
162
- # deepGEMM on supported platforms with block-quantized weights
163
- if envs.VLLM_USE_DEEP_GEMM and envs.VLLM_MOE_USE_DEEP_GEMM and block_quant:
165
+ # Determine if we should use DeepGEMM with block-quantized weights:
166
+ # - If explicitly set by user, respect their choice
167
+ # - If not explicitly set (default), disable when TP size is >= 8
168
+ moe_use_deep_gemm = envs.VLLM_MOE_USE_DEEP_GEMM
169
+ if not envs.is_set("VLLM_MOE_USE_DEEP_GEMM") and moe_parallel_config.tp_size >= 8:
170
+ moe_use_deep_gemm = False
171
+ logger.info_once(
172
+ "DeepGEMM MoE is disabled by default when TP size is >= 8. "
173
+ "Set VLLM_MOE_USE_DEEP_GEMM=1 to enable it.",
174
+ scope="local",
175
+ )
176
+
177
+ if envs.VLLM_USE_DEEP_GEMM and moe_use_deep_gemm and block_quant:
178
if not has_deep_gemm():
179
logger.warning_once(
180
"DeepGEMM backend requested but not available.", scope="local"
@@ -641,7 +655,9 @@ class Fp8MoEMethod(FusedMoEMethodBase):
655
self.quant_config = quant_config
656
self.weight_block_size = self.quant_config.weight_block_size
657
self.block_quant: bool = self.weight_block_size is not None
644
- self.fp8_backend = get_fp8_moe_backend(self.block_quant)
658
+ self.fp8_backend = get_fp8_moe_backend(
659
+ self.block_quant, layer.moe_parallel_config
660
+ )
661
662
self.use_marlin = self.fp8_backend == Fp8MoeBackend.MARLIN
663
self.flashinfer_moe_backend: FlashinferMoeBackend | None = None