@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e69ded7d1c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Bug Fix] Fix the support check for FP8 CUTLASS (#5352)

Bug description: With torch 2.4.0.dev20240603+cu121, cutlass_fp8_supported outputs False, and the (capability, version) before the comparison is (90, 11111111112) This PR fixes the support check for FP8 CUTLASS ( cutlass_fp8_supported) which was introduced in https://github.com/vllm-project/vllm/pull/5183.

Cheng Li committed Jun 7, 2024 at 17:42 UTC e69ded7d1c8a4f6ed26e64090bdc050c06cde3b9
1 file changed +7 -7
vllm/model_executor/layers/quantization/fp8.py
+7 -7
@@ -20,16 +20,16 @@ logger = init_logger(__name__)
20 def cutlass_fp8_supported() -> bool:
21 capability = torch.cuda.get_device_capability()
22 capability = capability[0] * 10 + capability[1]
23 - version = torch.version.cuda
24 - version = version[0] * 10 + version[1]
23 + major, minor = torch.version.cuda.split(".")
24 + version = int(major) * 10 + int(minor)
25
26 # CUTLASS FP8 kernels need at least
27 # CUDA 12.0 on SM90 systems (Hopper)
28 # CUDA 12.4 on SM89 systems (Lovelace)
29 gpu_is_supported = False
30 - if capability >= 900:
30 + if capability >= 90:
31 gpu_is_supported = version > 120
32 - elif capability >= 890:
32 + elif capability >= 89:
33 gpu_is_supported = version > 124
34
35 return gpu_is_supported
@@ -103,7 +103,7 @@ class Fp8LinearMethod(LinearMethodBase):
103 1. Only support per-tensor quantization due to torch._scaled_mm support.
104 2. Only support float8_e4m3fn data type due to the limitation of
105 torch._scaled_mm (https://github.com/pytorch/pytorch/blob/2e48b39603411a41c5025efbe52f89560b827825/aten/src/ATen/native/cuda/Blas.cpp#L854-L856)
106 -
106 +
107 Args:
108 quant_config: The quantization config.
109 """
@@ -298,8 +298,8 @@ class Fp8KVCacheMethod(QuantizeMethodBase):
298 self.quant_config = quant_config
299
300 def create_weights(self, layer: torch.nn.Module):
301 - """Create "weight" (aka kv_scale) for an attention layer.
302 -
301 + """Create "weight" (aka kv_scale) for an attention layer.
302 +
303 Args:
304 layer: The layer that is using the QuantizeMethodBase factory.
305 """