A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 23:45 UTC
[Bug] Fix DeepGemm Init Error (#21554)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
Wentao Ye committed
Jul 24, 2025 at 23:07 UTC
633f6e804ba8d362afedc15798bfa7dd1e61282f
1 file changed
+5
-1
vllm/model_executor/layers/quantization/utils/fp8_utils.py
+5
-1
@@ -366,7 +366,7 @@ def per_token_group_quant_fp8(
366
dtype: Optional[torch.dtype] = None,
367
column_major_scales: bool = False,
368
out_q: Optional[torch.Tensor] = None,
369
- use_ue8m0: bool = is_blackwell_deep_gemm_used(),
369
+ use_ue8m0: Optional[bool] = None,
370
) -> tuple[torch.Tensor, torch.Tensor]:
371
"""Function to perform per-token-group quantization on an input tensor `x`.
372
It converts the tensor values into signed float8 values and returns the
@@ -383,6 +383,10 @@ def per_token_group_quant_fp8(
383
tuple[torch.Tensor, torch.Tensor]: The quantized tensor and the
384
scaling factor.
385
"""
386
+ # TODO(wentao): refactor this
387
+ # use_ue8m0 should be a global flag that could be set by user
388
+ if use_ue8m0 is None:
389
+ use_ue8m0 = is_blackwell_deep_gemm_used()
390
dtype = current_platform.fp8_dtype() if dtype is None else dtype
391
assert (x.shape[-1] % group_size == 0), (
392
f"the last dimension of `x` {x.shape[-1]} must be divisible "