A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 17:30 UTC
[Bugfix] Fix fp8 DeepGemm compilation issues (#30336)
ElizaWszola committed
Dec 10, 2025 at 02:17 UTC
2e7035dd8cc2e6c907873462b4ac0bb9f08e0abb
1 file changed
+2
-2
vllm/model_executor/layers/quantization/utils/fp8_utils.py
+2
-2
@@ -31,7 +31,6 @@ from vllm.model_executor.utils import replace_parameter
31
from vllm.platforms import current_platform
32
from vllm.triton_utils import tl, triton
33
from vllm.utils.deep_gemm import (
34
- DeepGemmQuantScaleFMT,
34
fp8_gemm_nt,
35
is_deep_gemm_e8m0_used,
36
is_deep_gemm_supported,
@@ -248,6 +247,7 @@ class W8A8BlockFp8LinearOp:
247
self.act_quant_group_shape = act_quant_group_shape
248
self.is_deep_gemm_supported = is_deep_gemm_supported()
249
self.is_hopper = current_platform.is_device_capability(90)
250
+ self.is_blackwell = current_platform.is_device_capability(100)
251
self.use_deep_gemm_e8m0 = is_deep_gemm_e8m0_used()
252
253
# Get the correct blockscale mul and input quant operations.
@@ -303,7 +303,7 @@ class W8A8BlockFp8LinearOp:
303
weight: torch.Tensor,
304
weight_scale: torch.Tensor,
305
) -> torch.Tensor:
306
- if DeepGemmQuantScaleFMT.from_oracle() == DeepGemmQuantScaleFMT.UE8M0:
306
+ if self.use_deep_gemm_e8m0 and self.is_blackwell:
307
q_input, input_scale = per_token_group_quant_fp8_packed_for_deepgemm(
308
input_2d,
309
group_size=self.act_quant_group_shape.col,