A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 20:00 UTC
[Bugfix] Fix DeepGEMM after #29546 (#30267)
Signed-off-by: zhewenli <zhewenli@meta.com> Signed-off-by: Zhewen Li <zhewenli@meta.com>
Zhewen Li committed
Dec 8, 2025 at 17:05 UTC
ae339b1a67ac6651dae926aae2afa0c168c1ddb5
2 files changed
+11
-6
vllm/model_executor/layers/quantization/utils/fp8_utils.py
+10
-6
@@ -30,6 +30,7 @@ from vllm.model_executor.parameter import (
30
from vllm.platforms import current_platform
31
from vllm.triton_utils import tl, triton
32
from vllm.utils.deep_gemm import (
33
+ DeepGemmQuantScaleFMT,
34
fp8_gemm_nt,
35
is_deep_gemm_e8m0_used,
36
is_deep_gemm_supported,
@@ -268,12 +269,15 @@ class W8A8BlockFp8LinearOp:
269
weight: torch.Tensor,
270
weight_scale: torch.Tensor,
271
) -> torch.Tensor:
271
- assert self.deepgemm_input_quant_op is not None
272
- q_input, input_scale = per_token_group_quant_fp8_packed_for_deepgemm(
273
- input_2d,
274
- group_size=self.act_quant_group_shape.col,
275
- use_ue8m0=True,
276
- )
272
+ if DeepGemmQuantScaleFMT.from_oracle() == DeepGemmQuantScaleFMT.UE8M0:
273
+ q_input, input_scale = per_token_group_quant_fp8_packed_for_deepgemm(
274
+ input_2d,
275
+ group_size=self.act_quant_group_shape.col,
276
+ use_ue8m0=True,
277
+ )
278
+ else:
279
+ assert self.deepgemm_input_quant_op is not None
280
+ q_input, input_scale = self.deepgemm_input_quant_op(input_2d)
281
output = torch.empty(
282
(q_input.shape[0], weight.shape[0]),
283
dtype=torch.bfloat16,
vllm/utils/deep_gemm.py
+1
@@ -399,6 +399,7 @@ def should_use_deepgemm_for_fp8_linear_for_nk(
399
400
__all__ = [
401
"calc_diff",
402
+ "DeepGemmQuantScaleFMT",
403
"fp8_gemm_nt",
404
"m_grouped_fp8_gemm_nt_contiguous",
405
"fp8_m_grouped_gemm_nt_masked",