A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Build] Guard against older CUDA versions when building CUTLASS 3.x kernels (#5168)
Tyler Michael Smith committed
May 31, 2024 at 20:21 UTC
1197e02141df1a7442f21ff6922c98ec0bba153e
2 files changed
+18
-3
csrc/quantization/cutlass_w8a8/scaled_mm_dq_c3x.cu
+8
-2
@@ -1,3 +1,9 @@
1
+// clang-format will break include orders
2
+// clang-format off
3
+#include <cudaTypedefs.h>
4
+
5
+#if defined CUDA_VERSION && CUDA_VERSION >= 12000
6
+
7
#include <torch/extension.h>
8
9
#include <ATen/cuda/CUDAContext.h>
@@ -6,8 +12,6 @@
12
#include <sstream>
13
#include <vector>
14
9
-// clang-format will break include orders
10
-// clang-format off
15
#include "cutlass/cutlass.h"
16
17
#include "cute/tensor.hpp"
@@ -241,3 +245,5 @@ void cutlass_scaled_mm_dq_sm90(torch::Tensor& out, torch::Tensor const& a,
245
}
246
}
247
}
248
+
249
+#endif
csrc/quantization/cutlass_w8a8/scaled_mm_dq_entry.cu
+10
-1
@@ -1,5 +1,6 @@
1
+#include <cudaTypedefs.h>
2
+
3
#include <c10/cuda/CUDAGuard.h>
2
-#include <cuda_runtime.h>
4
#include <torch/extension.h>
5
6
void cutlass_scaled_mm_dq_sm75(torch::Tensor& c, torch::Tensor const& a,
@@ -17,10 +18,12 @@ void cutlass_scaled_mm_dq_sm89(torch::Tensor& c, torch::Tensor const& a,
18
torch::Tensor const& a_scales,
19
torch::Tensor const& b_scales);
20
21
+#if defined CUDA_VERSION && CUDA_VERSION >= 12000
22
void cutlass_scaled_mm_dq_sm90(torch::Tensor& c, torch::Tensor const& a,
23
torch::Tensor const& b,
24
torch::Tensor const& a_scales,
25
torch::Tensor const& b_scales);
26
+#endif
27
28
void cutlass_scaled_mm_dq(torch::Tensor& c, torch::Tensor const& a,
29
torch::Tensor const& b, torch::Tensor const& a_scales,
@@ -51,7 +54,13 @@ void cutlass_scaled_mm_dq(torch::Tensor& c, torch::Tensor const& a,
54
55
if (version_num >= 90) {
56
// Hopper
57
+
58
+ // Guard against compilation issues for sm90 kernels
59
+#if defined CUDA_VERSION && CUDA_VERSION >= 12000
60
cutlass_scaled_mm_dq_sm90(c, a, b, a_scales, b_scales);
61
+#else
62
+ cutlass_scaled_mm_dq_sm80(c, a, b, a_scales, b_scales);
63
+#endif
64
} else if (version_num == 89) {
65
// Ada Lovelace
66
cutlass_scaled_mm_dq_sm89(c, a, b, a_scales, b_scales);