@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1197e02141
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Build] Guard against older CUDA versions when building CUTLASS 3.x kernels (#5168)

Tyler Michael Smith committed May 31, 2024 at 20:21 UTC 1197e02141df1a7442f21ff6922c98ec0bba153e
2 files changed +18 -3
csrc/quantization/cutlass_w8a8/scaled_mm_dq_c3x.cu
+8 -2
@@ -1,3 +1,9 @@
1 +// clang-format will break include orders
2 +// clang-format off
3 +#include <cudaTypedefs.h>
4 +
5 +#if defined CUDA_VERSION && CUDA_VERSION >= 12000
6 +
7 #include <torch/extension.h>
8
9 #include <ATen/cuda/CUDAContext.h>
@@ -6,8 +12,6 @@
12 #include <sstream>
13 #include <vector>
14
9 -// clang-format will break include orders
10 -// clang-format off
15 #include "cutlass/cutlass.h"
16
17 #include "cute/tensor.hpp"
@@ -241,3 +245,5 @@ void cutlass_scaled_mm_dq_sm90(torch::Tensor& out, torch::Tensor const& a,
245 }
246 }
247 }
248 +
249 +#endif
csrc/quantization/cutlass_w8a8/scaled_mm_dq_entry.cu
+10 -1
@@ -1,5 +1,6 @@
1 +#include <cudaTypedefs.h>
2 +
3 #include <c10/cuda/CUDAGuard.h>
2 -#include <cuda_runtime.h>
4 #include <torch/extension.h>
5
6 void cutlass_scaled_mm_dq_sm75(torch::Tensor& c, torch::Tensor const& a,
@@ -17,10 +18,12 @@ void cutlass_scaled_mm_dq_sm89(torch::Tensor& c, torch::Tensor const& a,
18 torch::Tensor const& a_scales,
19 torch::Tensor const& b_scales);
20
21 +#if defined CUDA_VERSION && CUDA_VERSION >= 12000
22 void cutlass_scaled_mm_dq_sm90(torch::Tensor& c, torch::Tensor const& a,
23 torch::Tensor const& b,
24 torch::Tensor const& a_scales,
25 torch::Tensor const& b_scales);
26 +#endif
27
28 void cutlass_scaled_mm_dq(torch::Tensor& c, torch::Tensor const& a,
29 torch::Tensor const& b, torch::Tensor const& a_scales,
@@ -51,7 +54,13 @@ void cutlass_scaled_mm_dq(torch::Tensor& c, torch::Tensor const& a,
54
55 if (version_num >= 90) {
56 // Hopper
57 +
58 + // Guard against compilation issues for sm90 kernels
59 +#if defined CUDA_VERSION && CUDA_VERSION >= 12000
60 cutlass_scaled_mm_dq_sm90(c, a, b, a_scales, b_scales);
61 +#else
62 + cutlass_scaled_mm_dq_sm80(c, a, b, a_scales, b_scales);
63 +#endif
64 } else if (version_num == 89) {
65 // Ada Lovelace
66 cutlass_scaled_mm_dq_sm89(c, a, b, a_scales, b_scales);