@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3f3b6b2150
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Bugfix] Fix the CUDA version check for FP8 support in the CUTLASS kernels (#5715)

Tyler Michael Smith committed Jun 20, 2024 at 14:36 UTC 3f3b6b21500bce2061cae33706bd47c8b6663771
5 files changed +30 -13
csrc/ops.h
+2
@@ -92,6 +92,8 @@ torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
92 int64_t size_k, int64_t size_n,
93 int64_t num_bits);
94
95 +bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability);
96 +
97 void cutlass_scaled_mm(torch::Tensor& out, torch::Tensor const& a,
98 torch::Tensor const& b, torch::Tensor const& a_scales,
99 torch::Tensor const& b_scales);
csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu
+16
@@ -25,6 +25,22 @@ void cutlass_scaled_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
25 torch::Tensor const& b_scales);
26 #endif
27
28 +bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability) {
29 + // CUTLASS FP8 kernels need at least
30 + // CUDA 12.0 on SM90 systems (Hopper)
31 + // CUDA 12.4 on SM89 systems (Lovelace)
32 +
33 +#if defined CUDA_VERSION
34 + if (cuda_device_capability >= 90) {
35 + return CUDA_VERSION >= 12000;
36 + } else if (cuda_device_capability >= 89) {
37 + return CUDA_VERSION >= 12040;
38 + }
39 +#endif
40 +
41 + return false;
42 +}
43 +
44 void cutlass_scaled_mm(torch::Tensor& c, torch::Tensor const& a,
45 torch::Tensor const& b, torch::Tensor const& a_scales,
46 torch::Tensor const& b_scales) {
csrc/torch_bindings.cpp
+6
@@ -144,6 +144,12 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
144 " Tensor b, Tensor a_scales,"
145 " Tensor b_scales) -> ()");
146 ops.impl("cutlass_scaled_mm", torch::kCUDA, &cutlass_scaled_mm);
147 +
148 + // Check if cutlass scaled_mm is supported for CUDA devices of the given
149 + // capability
150 + ops.def("cutlass_scaled_mm_supports_fp8", &cutlass_scaled_mm_supports_fp8);
151 + ops.impl("cutlass_scaled_mm_supports_fp8", torch::kCUDA,
152 + &cutlass_scaled_mm_supports_fp8);
153 #endif
154
155 // Quantized GEMM for GPTQ.
vllm/_custom_ops.py
+4
@@ -216,6 +216,10 @@ def gptq_marlin_24_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
216
217
218 # cutlass
219 +def cutlass_scaled_mm_supports_fp8(cuda_device_capability: int) -> bool:
220 + return torch.ops._C.cutlass_scaled_mm_supports_fp8(cuda_device_capability)
221 +
222 +
223 def cutlass_scaled_mm(a: torch.Tensor, b: torch.Tensor, scale_a: torch.Tensor,
224 scale_b: torch.Tensor,
225 out_dtype: Type[torch.dtype]) -> torch.Tensor:
vllm/model_executor/layers/quantization/fp8.py
+2 -13
@@ -20,19 +20,8 @@ logger = init_logger(__name__)
20 def cutlass_fp8_supported() -> bool:
21 capability = torch.cuda.get_device_capability()
22 capability = capability[0] * 10 + capability[1]
23 - major, minor = torch.version.cuda.split(".")
24 - version = int(major) * 10 + int(minor)
25 -
26 - # CUTLASS FP8 kernels need at least
27 - # CUDA 12.0 on SM90 systems (Hopper)
28 - # CUDA 12.4 on SM89 systems (Lovelace)
29 - gpu_is_supported = False
30 - if capability >= 90:
31 - gpu_is_supported = version > 120
32 - elif capability >= 89:
33 - gpu_is_supported = version > 124
34 -
35 - return gpu_is_supported
23 +
24 + return ops.cutlass_scaled_mm_supports_fp8(capability)
25
26
27 class Fp8Config(QuantizationConfig):