A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Kernel] Enable FP8 Cutlass for Ada Lovelace (#6950)
Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
Varun Sundar Rabindranath committed
Jul 31, 2024 at 17:40 UTC
93548eb37e952a0af035dc524a3826cdcd78d6cf
1 file changed
+1
-7
csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu
+1
-7
@@ -38,13 +38,7 @@ bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability) {
38
if (cuda_device_capability >= 90) {
39
return CUDA_VERSION >= 12000;
40
} else if (cuda_device_capability >= 89) {
41
- // CUTLASS Kernels have not been tuned for Ada Lovelace systems
42
- // and are slower than torch.mm. Return false unconditionally in this case.
43
- return false;
44
-
45
- // Once the CUTLASS kernels have been optimized for Lovelace systems,
46
- // use the following check:
47
- // return CUDA_VERSION >= 12040;
41
+ return CUDA_VERSION >= 12040;
42
}
43
#endif
44