A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Bug Fix] Fix the support check for FP8 CUTLASS (#5352)
Bug description: With torch 2.4.0.dev20240603+cu121, cutlass_fp8_supported outputs False, and the (capability, version) before the comparison is (90, 11111111112) This PR fixes the support check for FP8 CUTLASS ( cutlass_fp8_supported) which was introduced in https://github.com/vllm-project/vllm/pull/5183.
Cheng Li committed
Jun 7, 2024 at 17:42 UTC
e69ded7d1c8a4f6ed26e64090bdc050c06cde3b9
1 file changed
+7
-7
vllm/model_executor/layers/quantization/fp8.py
+7
-7
@@ -20,16 +20,16 @@ logger = init_logger(__name__)
20
def cutlass_fp8_supported() -> bool:
21
capability = torch.cuda.get_device_capability()
22
capability = capability[0] * 10 + capability[1]
23
- version = torch.version.cuda
24
- version = version[0] * 10 + version[1]
23
+ major, minor = torch.version.cuda.split(".")
24
+ version = int(major) * 10 + int(minor)
25
26
# CUTLASS FP8 kernels need at least
27
# CUDA 12.0 on SM90 systems (Hopper)
28
# CUDA 12.4 on SM89 systems (Lovelace)
29
gpu_is_supported = False
30
- if capability >= 900:
30
+ if capability >= 90:
31
gpu_is_supported = version > 120
32
- elif capability >= 890:
32
+ elif capability >= 89:
33
gpu_is_supported = version > 124
34
35
return gpu_is_supported
@@ -103,7 +103,7 @@ class Fp8LinearMethod(LinearMethodBase):
103
1. Only support per-tensor quantization due to torch._scaled_mm support.
104
2. Only support float8_e4m3fn data type due to the limitation of
105
torch._scaled_mm (https://github.com/pytorch/pytorch/blob/2e48b39603411a41c5025efbe52f89560b827825/aten/src/ATen/native/cuda/Blas.cpp#L854-L856)
106
-
106
+
107
Args:
108
quant_config: The quantization config.
109
"""
@@ -298,8 +298,8 @@ class Fp8KVCacheMethod(QuantizeMethodBase):
298
self.quant_config = quant_config
299
300
def create_weights(self, layer: torch.nn.Module):
301
- """Create "weight" (aka kv_scale) for an attention layer.
302
-
301
+ """Create "weight" (aka kv_scale) for an attention layer.
302
+
303
Args:
304
layer: The layer that is using the QuantizeMethodBase factory.
305
"""