A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 02:15 UTC
[Bugfix] Fix incorrect dispatch for CutlassBlockScaledGroupedGemm and DeepGEMM (#20933)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Jul 15, 2025 at 10:42 UTC
bcdfb2a3308e14fbf46da6d6d41747f289af9300
1 file changed
+10
-5
vllm/model_executor/layers/quantization/fp8.py
+10
-5
@@ -488,11 +488,16 @@ class Fp8MoEMethod(FusedMoEMethodBase):
488
logger.warning_once("Failed to import DeepGemm kernels.")
489
elif not self.block_quant:
490
logger.warning_once("Model is not block quantized. Not using "
491
- " DeepGemm kernels")
491
+ "DeepGemm kernels")
492
elif (current_platform.is_cuda()
493
- and current_platform.has_device_capability(90)):
493
+ and current_platform.is_device_capability(90)):
494
logger.info_once("Using DeepGemm kernels for Fp8MoEMethod.")
495
self.allow_deep_gemm = True
496
+ elif (current_platform.is_cuda()
497
+ and is_blackwell_deep_gemm_used()):
498
+ logger.info_once("Using DeepGemm SM100 kernels for "
499
+ "Fp8MoEMethod.")
500
+ self.allow_deep_gemm = True
501
else:
502
logger.warning_once(
503
"DeepGemm not supported on the current platform.")
@@ -500,10 +505,10 @@ class Fp8MoEMethod(FusedMoEMethodBase):
505
# Check for CutlassBlockScaledGroupedGemm support.
506
self.allow_cutlass_block_scaled_grouped_gemm = False
507
if not self.block_quant:
503
- logger.warning_once("Model is not block quantized. Not using "
504
- "CutlassBlockScaledGroupedGemm kernels")
508
+ logger.debug_once("Model is not block quantized. Not using "
509
+ "CutlassBlockScaledGroupedGemm kernels")
510
elif (current_platform.is_cuda()
506
- and current_platform.has_device_capability(100)):
511
+ and current_platform.is_device_capability(100)):
512
logger.info_once(
513
"Using CutlassBlockScaledGroupedGemm kernels for Fp8MoEMethod."
514
)