A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Kernel] Update Cutlass int8 kernel configs for SM80 (#5275)
Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
Varun Sundar Rabindranath committed
Jun 20, 2024 at 19:03 UTC
a7dcc62086ea751b46b4821c2811cf8ac83711bf
3 files changed
+123
-16
csrc/quantization/cutlass_w8a8/common.hpp
+7
@@ -1,6 +1,7 @@
1
#pragma once
2
3
#include "cutlass/cutlass.h"
4
+#include <climits>
5
6
/**
7
* Helper function for checking CUTLASS errors
@@ -10,3 +11,9 @@
11
TORCH_CHECK(status == cutlass::Status::kSuccess, \
12
cutlassGetStatusString(status)) \
13
}
14
+
15
+inline uint32_t next_pow_2(uint32_t const num) {
16
+ if (num <= 1) return num;
17
+ return 1 << (CHAR_BIT * sizeof(num) - __builtin_clz(num - 1));
18
+}
19
+
csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cu
+116
-11
@@ -250,8 +250,120 @@ void cutlass_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
250
CUTLASS_CHECK(status);
251
}
252
253
+template <typename InType, typename OutType,
254
+ template <typename, typename> typename Epilogue>
255
+struct sm80_config_default {
256
+ // This config is used in 2 cases,
257
+ // - M in (128, inf)
258
+ // - M in (64, 128] and N >= 8192
259
+ static_assert(std::is_same<InType, int8_t>());
260
+ using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
261
+ using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
262
+ using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
263
+ using Cutlass2xGemm =
264
+ cutlass_2x_gemm<cutlass::arch::Sm80, enable_sm80_to_sm89, InType, OutType,
265
+ Epilogue, TileShape, WarpShape, InstructionShape, 5>;
266
+};
267
+
268
+template <typename InType, typename OutType,
269
+ template <typename, typename> typename Epilogue>
270
+struct sm80_config_M64 {
271
+ // This config is used in 2 cases,
272
+ // - M in (32, 64]
273
+ // - M in (64, 128] and N < 8192
274
+ static_assert(std::is_same<InType, int8_t>());
275
+ using TileShape = typename cutlass::gemm::GemmShape<64, 128, 128>;
276
+ using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
277
+ using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
278
+ using Cutlass2xGemm =
279
+ cutlass_2x_gemm<cutlass::arch::Sm80, enable_sm80_to_sm89, InType, OutType,
280
+ Epilogue, TileShape, WarpShape, InstructionShape, 5>;
281
+};
282
+
283
+template <typename InType, typename OutType,
284
+ template <typename, typename> typename Epilogue>
285
+struct sm80_config_M32 {
286
+ // M in (16, 32]
287
+ static_assert(std::is_same<InType, int8_t>());
288
+ using TileShape = typename cutlass::gemm::GemmShape<32, 64, 128>;
289
+ using WarpShape = typename cutlass::gemm::GemmShape<32, 64, 64>;
290
+ using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
291
+ using Cutlass2xGemm =
292
+ cutlass_2x_gemm<cutlass::arch::Sm80, enable_sm80_to_sm89, InType, OutType,
293
+ Epilogue, TileShape, WarpShape, InstructionShape, 5>;
294
+};
295
+
296
+template <typename InType, typename OutType,
297
+ template <typename, typename> typename Epilogue>
298
+struct sm80_config_M16 {
299
+ // M in [1, 16]
300
+ static_assert(std::is_same<InType, int8_t>());
301
+ using TileShape = typename cutlass::gemm::GemmShape<16, 64, 128>;
302
+ using WarpShape = typename cutlass::gemm::GemmShape<16, 64, 64>;
303
+ using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
304
+ using Cutlass2xGemm =
305
+ cutlass_2x_gemm<cutlass::arch::Sm80, enable_sm80_to_sm89, InType, OutType,
306
+ Epilogue, TileShape, WarpShape, InstructionShape, 5>;
307
+};
308
+
309
} // namespace
310
311
+template <typename InType, typename OutType,
312
+ template <typename, typename> typename Epilogue,
313
+ typename... EpilogueArgs>
314
+void cutlass_gemm_sm80_dispatch(torch::Tensor& out, torch::Tensor const& a,
315
+ torch::Tensor const& b,
316
+ EpilogueArgs&&... args) {
317
+ static_assert(std::is_same<InType, int8_t>());
318
+ TORCH_CHECK(a.dtype() == torch::kInt8);
319
+ TORCH_CHECK(b.dtype() == torch::kInt8);
320
+
321
+ using Cutlass2xGemmDefault =
322
+ typename sm80_config_default<InType, OutType, Epilogue>::Cutlass2xGemm;
323
+ using Cutlass2xGemmM128BigN =
324
+ typename sm80_config_default<InType, OutType, Epilogue>::Cutlass2xGemm;
325
+ using Cutlass2xGemmM128SmallN =
326
+ typename sm80_config_M64<InType, OutType, Epilogue>::Cutlass2xGemm;
327
+ using Cutlass2xGemmM64 =
328
+ typename sm80_config_M64<InType, OutType, Epilogue>::Cutlass2xGemm;
329
+ using Cutlass2xGemmM32 =
330
+ typename sm80_config_M32<InType, OutType, Epilogue>::Cutlass2xGemm;
331
+ using Cutlass2xGemmM16 =
332
+ typename sm80_config_M16<InType, OutType, Epilogue>::Cutlass2xGemm;
333
+
334
+ uint32_t const m = a.size(0);
335
+ uint32_t const mp2 =
336
+ std::max(static_cast<uint32_t>(16), next_pow_2(m)); // next power of 2
337
+ if (mp2 <= 16) {
338
+ // M in [1, 16]
339
+ return cutlass_gemm_caller<Cutlass2xGemmM16>(
340
+ out, a, b, std::forward<EpilogueArgs>(args)...);
341
+ } else if (mp2 <= 32) {
342
+ // M in (16, 32]
343
+ return cutlass_gemm_caller<Cutlass2xGemmM32>(
344
+ out, a, b, std::forward<EpilogueArgs>(args)...);
345
+ } else if (mp2 <= 64) {
346
+ // M in (32, 64]
347
+ return cutlass_gemm_caller<Cutlass2xGemmM64>(
348
+ out, a, b, std::forward<EpilogueArgs>(args)...);
349
+ } else if (mp2 <= 128) {
350
+ // M in (64, 128]
351
+ uint32_t const n = out.size(1);
352
+ bool const small_n = n < 8192;
353
+ if (small_n) {
354
+ return cutlass_gemm_caller<Cutlass2xGemmM128SmallN>(
355
+ out, a, b, std::forward<EpilogueArgs>(args)...);
356
+ } else {
357
+ return cutlass_gemm_caller<Cutlass2xGemmM128BigN>(
358
+ out, a, b, std::forward<EpilogueArgs>(args)...);
359
+ }
360
+ } else {
361
+ // M in (128, inf)
362
+ return cutlass_gemm_caller<Cutlass2xGemmDefault>(
363
+ out, a, b, std::forward<EpilogueArgs>(args)...);
364
+ }
365
+}
366
+
367
void cutlass_scaled_mm_sm75(torch::Tensor& out, torch::Tensor const& a,
368
torch::Tensor const& b,
369
torch::Tensor const& a_scales,
@@ -288,20 +400,13 @@ void cutlass_scaled_mm_sm80(torch::Tensor& out, torch::Tensor const& a,
400
TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
401
TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
402
291
- using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
292
- using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
293
- using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
294
-
403
if (out.dtype() == torch::kBFloat16) {
296
- return cutlass_gemm_caller<cutlass_2x_gemm<
297
- cutlass::arch::Sm80, enable_sm80_to_sm89, int8_t, cutlass::bfloat16_t,
298
- ScaledEpilogue, TileShape, WarpShape, InstructionShape, 5>>(
299
- out, a, b, a_scales, b_scales);
404
+ return cutlass_gemm_sm80_dispatch<int8_t, cutlass::bfloat16_t,
405
+ ScaledEpilogue>(out, a, b, a_scales,
406
+ b_scales);
407
} else {
408
TORCH_CHECK(out.dtype() == torch::kFloat16);
302
- return cutlass_gemm_caller<cutlass_2x_gemm<
303
- cutlass::arch::Sm80, enable_sm80_to_sm89, int8_t, cutlass::half_t,
304
- ScaledEpilogue, TileShape, WarpShape, InstructionShape, 5>>(
409
+ return cutlass_gemm_sm80_dispatch<int8_t, cutlass::half_t, ScaledEpilogue>(
410
out, a, b, a_scales, b_scales);
411
}
412
}
csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu
-5
@@ -44,11 +44,6 @@ using namespace cute;
44
45
namespace {
46
47
-uint32_t next_pow_2(uint32_t const num) {
48
- if (num <= 1) return num;
49
- return 1 << (CHAR_BIT * sizeof(num) - __builtin_clz(num - 1));
50
-}
51
-
47
// A wrapper for the GEMM kernel that is used to guard against compilation on
48
// architectures that will never use the kernel. The purpose of this is to
49
// reduce the size of the compiled binary.