A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Kernel] Add GPU architecture guards to the CUTLASS w8a8 kernels to reduce binary size (#5157)
Co-authored-by: Cody Yu <hao.yu.cody@gmail.com>
Tyler Michael Smith committed
Jun 5, 2024 at 13:44 UTC
ccd4f129e8ad95191b3c8d6d0e935382b10c5164
2 files changed
+87
-37
csrc/quantization/cutlass_w8a8/scaled_mm_dq_c2x.cu
+70
-35
@@ -48,9 +48,44 @@ using namespace cute;
48
49
namespace {
50
51
-template <typename Arch, typename ElementAB_, typename ElementD_,
52
- typename TileShape, typename WarpShape, typename InstructionShape,
53
- int32_t MainLoopStages>
51
+// Wrappers for the GEMM kernel that is used to guard against compilation on
52
+// architectures that will never use the kernel. The purpose of this is to
53
+// reduce the size of the compiled binary.
54
+// __CUDA_ARCH__ is not defined in host code, so this lets us smuggle the ifdef
55
+// into code that will be executed on the device where it is defined.
56
+template <typename Kernel>
57
+struct enable_sm75_to_sm80 : Kernel {
58
+ template <typename... Args>
59
+ CUTLASS_DEVICE static void invoke(Args&&... args) {
60
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 750 && __CUDA_ARCH__ < 800
61
+ Kernel::invoke(std::forward<Args>(args)...);
62
+#endif
63
+ }
64
+};
65
+
66
+template <typename Kernel>
67
+struct enable_sm80_to_sm89 : Kernel {
68
+ template <typename... Args>
69
+ CUTLASS_DEVICE static void invoke(Args&&... args) {
70
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 800 && __CUDA_ARCH__ < 890
71
+ Kernel::invoke(std::forward<Args>(args)...);
72
+#endif
73
+ }
74
+};
75
+
76
+template <typename Kernel>
77
+struct enable_sm89_to_sm90 : Kernel {
78
+ template <typename... Args>
79
+ CUTLASS_DEVICE static void invoke(Args&&... args) {
80
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 890 && __CUDA_ARCH__ < 900
81
+ Kernel::invoke(std::forward<Args>(args)...);
82
+#endif
83
+ }
84
+};
85
+
86
+template <typename Arch, template <typename> typename ArchGuard,
87
+ typename ElementAB_, typename ElementD_, typename TileShape,
88
+ typename WarpShape, typename InstructionShape, int32_t MainLoopStages>
89
struct cutlass_2x_gemm {
90
using ElementAB = ElementAB_;
91
using ElementD = ElementD_;
@@ -101,7 +136,7 @@ struct cutlass_2x_gemm {
136
using RowMajor = typename cutlass::layout::RowMajor;
137
using ColumnMajor = typename cutlass::layout::ColumnMajor;
138
using KernelType =
104
- typename cutlass::gemm::kernel::DefaultGemmWithVisitor<
139
+ ArchGuard<typename cutlass::gemm::kernel::DefaultGemmWithVisitor<
140
ElementAB, RowMajor, cutlass::ComplexTransform::kNone, 16,
141
ElementAB, ColumnMajor, cutlass::ComplexTransform::kNone, 16,
142
float, cutlass::layout::RowMajor, 4,
@@ -112,7 +147,7 @@ struct cutlass_2x_gemm {
147
cutlass::gemm::threadblock::ThreadblockSwizzleStreamK,
148
MainLoopStages, Operator,
149
1 /* epilogue stages */
115
- >::GemmKernel;
150
+ >::GemmKernel>;
151
// clang-format on
152
153
using Op = cutlass::gemm::device::GemmUniversalAdapter<KernelType>;
@@ -208,16 +243,16 @@ void cutlass_scaled_mm_dq_sm75(torch::Tensor& out, torch::Tensor const& a,
243
using InstructionShape = typename cutlass::gemm::GemmShape<8, 8, 16>;
244
245
if (out.dtype() == torch::kBFloat16) {
211
- return cutlass_scaled_mm_dq_dispatcher<
212
- cutlass_2x_gemm<cutlass::arch::Sm75, int8_t, cutlass::bfloat16_t,
213
- TileShape, WarpShape, InstructionShape, 2>>(
214
- out, a, b, a_scales, b_scales);
246
+ return cutlass_scaled_mm_dq_dispatcher<cutlass_2x_gemm<
247
+ cutlass::arch::Sm75, enable_sm75_to_sm80, int8_t, cutlass::bfloat16_t,
248
+ TileShape, WarpShape, InstructionShape, 2>>(out, a, b, a_scales,
249
+ b_scales);
250
} else {
251
TORCH_CHECK(out.dtype() == torch::kFloat16);
217
- return cutlass_scaled_mm_dq_dispatcher<
218
- cutlass_2x_gemm<cutlass::arch::Sm75, int8_t, cutlass::half_t, TileShape,
219
- WarpShape, InstructionShape, 2>>(out, a, b, a_scales,
220
- b_scales);
252
+ return cutlass_scaled_mm_dq_dispatcher<cutlass_2x_gemm<
253
+ cutlass::arch::Sm75, enable_sm75_to_sm80, int8_t, cutlass::half_t,
254
+ TileShape, WarpShape, InstructionShape, 2>>(out, a, b, a_scales,
255
+ b_scales);
256
}
257
}
258
@@ -235,16 +270,16 @@ void cutlass_scaled_mm_dq_sm80(torch::Tensor& out, torch::Tensor const& a,
270
using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
271
272
if (out.dtype() == torch::kBFloat16) {
238
- return cutlass_scaled_mm_dq_dispatcher<
239
- cutlass_2x_gemm<cutlass::arch::Sm80, int8_t, cutlass::bfloat16_t,
240
- TileShape, WarpShape, InstructionShape, 5>>(
241
- out, a, b, a_scales, b_scales);
273
+ return cutlass_scaled_mm_dq_dispatcher<cutlass_2x_gemm<
274
+ cutlass::arch::Sm80, enable_sm80_to_sm89, int8_t, cutlass::bfloat16_t,
275
+ TileShape, WarpShape, InstructionShape, 5>>(out, a, b, a_scales,
276
+ b_scales);
277
} else {
278
TORCH_CHECK(out.dtype() == torch::kFloat16);
244
- return cutlass_scaled_mm_dq_dispatcher<
245
- cutlass_2x_gemm<cutlass::arch::Sm80, int8_t, cutlass::half_t, TileShape,
246
- WarpShape, InstructionShape, 5>>(out, a, b, a_scales,
247
- b_scales);
279
+ return cutlass_scaled_mm_dq_dispatcher<cutlass_2x_gemm<
280
+ cutlass::arch::Sm80, enable_sm80_to_sm89, int8_t, cutlass::half_t,
281
+ TileShape, WarpShape, InstructionShape, 5>>(out, a, b, a_scales,
282
+ b_scales);
283
}
284
}
285
@@ -263,16 +298,16 @@ void cutlass_scaled_mm_dq_sm89(torch::Tensor& out, torch::Tensor const& a,
298
TORCH_CHECK(b.dtype() == torch::kInt8);
299
300
if (out.dtype() == torch::kBFloat16) {
266
- return cutlass_scaled_mm_dq_dispatcher<
267
- cutlass_2x_gemm<cutlass::arch::Sm89, int8_t, cutlass::bfloat16_t,
268
- TileShape, WarpShape, InstructionShape, 5>>(
269
- out, a, b, a_scales, b_scales);
301
+ return cutlass_scaled_mm_dq_dispatcher<cutlass_2x_gemm<
302
+ cutlass::arch::Sm89, enable_sm89_to_sm90, int8_t, cutlass::bfloat16_t,
303
+ TileShape, WarpShape, InstructionShape, 5>>(out, a, b, a_scales,
304
+ b_scales);
305
} else {
306
assert(out.dtype() == torch::kFloat16);
272
- return cutlass_scaled_mm_dq_dispatcher<
273
- cutlass_2x_gemm<cutlass::arch::Sm89, int8_t, cutlass::half_t,
274
- TileShape, WarpShape, InstructionShape, 5>>(
275
- out, a, b, a_scales, b_scales);
307
+ return cutlass_scaled_mm_dq_dispatcher<cutlass_2x_gemm<
308
+ cutlass::arch::Sm89, enable_sm89_to_sm90, int8_t, cutlass::half_t,
309
+ TileShape, WarpShape, InstructionShape, 5>>(out, a, b, a_scales,
310
+ b_scales);
311
}
312
} else {
313
TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
@@ -280,15 +315,15 @@ void cutlass_scaled_mm_dq_sm89(torch::Tensor& out, torch::Tensor const& a,
315
316
if (out.dtype() == torch::kBFloat16) {
317
return cutlass_scaled_mm_dq_dispatcher<cutlass_2x_gemm<
283
- cutlass::arch::Sm89, cutlass::float_e4m3_t, cutlass::bfloat16_t,
284
- TileShape, WarpShape, InstructionShape, 5>>(out, a, b, a_scales,
285
- b_scales);
318
+ cutlass::arch::Sm89, enable_sm89_to_sm90, cutlass::float_e4m3_t,
319
+ cutlass::bfloat16_t, TileShape, WarpShape, InstructionShape, 5>>(
320
+ out, a, b, a_scales, b_scales);
321
} else {
322
TORCH_CHECK(out.dtype() == torch::kFloat16);
323
return cutlass_scaled_mm_dq_dispatcher<cutlass_2x_gemm<
289
- cutlass::arch::Sm89, cutlass::float_e4m3_t, cutlass::half_t,
290
- TileShape, WarpShape, InstructionShape, 5>>(out, a, b, a_scales,
291
- b_scales);
324
+ cutlass::arch::Sm89, enable_sm89_to_sm90, cutlass::float_e4m3_t,
325
+ cutlass::half_t, TileShape, WarpShape, InstructionShape, 5>>(
326
+ out, a, b, a_scales, b_scales);
327
}
328
}
329
}
csrc/quantization/cutlass_w8a8/scaled_mm_dq_c3x.cu
+17
-2
@@ -56,6 +56,21 @@ uint32_t next_pow_2(uint32_t const num) {
56
return 1 << (CHAR_BIT * sizeof(num) - __builtin_clz(num - 1));
57
}
58
59
+// A wrapper for the GEMM kernel that is used to guard against compilation on
60
+// architectures that will never use the kernel. The purpose of this is to
61
+// reduce the size of the compiled binary.
62
+// __CUDA_ARCH__ is not defined in host code, so this lets us smuggle the ifdef
63
+// into code that will be executed on the device where it is defined.
64
+template <typename Kernel>
65
+struct enable_sm90_or_later : Kernel {
66
+ template <typename... Args>
67
+ CUTLASS_DEVICE void operator()(Args&&... args) {
68
+ #if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 900
69
+ Kernel::operator()(std::forward<Args>(args)...);
70
+ #endif
71
+ }
72
+};
73
+
74
template <typename ElementAB_, typename ElementD_, typename TileShape,
75
typename ClusterShape, typename KernelSchedule,
76
typename EpilogueSchedule>
@@ -126,9 +141,9 @@ struct cutlass_3x_gemm {
141
KernelSchedule>::CollectiveOp;
142
// clang-format on
143
129
- using KernelType = cutlass::gemm::kernel::GemmUniversal<
144
+ using KernelType = enable_sm90_or_later<cutlass::gemm::kernel::GemmUniversal<
145
cute::Shape<int, int, int, int>, CollectiveMainloop, CollectiveEpilogue,
131
- cutlass::gemm::PersistentScheduler>;
146
+ cutlass::gemm::PersistentScheduler>>;
147
148
struct GemmKernel : public KernelType {};
149
};