A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Kernel] Tuned int8 Cutlass Kernels for SM75 (T4) (#6996)
Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
Varun Sundar Rabindranath committed
Jul 31, 2024 at 17:40 UTC
35e9c12bfaf8f273281af897b7208dfba53f103c
3 files changed
+135
-12
benchmarks/cutlass_benchmarks/w8a8_benchmarks.py
+8
-1
@@ -112,13 +112,20 @@ def bench_int8(dtype: torch.dtype, m: int, k: int, n: int, label: str,
112
scale_b = torch.tensor(1.0, device="cuda", dtype=torch.float32)
113
114
timers = []
115
- # pytorch impl
115
+ # pytorch impl - bfloat16
116
timers.append(
117
bench_fn(a.to(dtype=torch.bfloat16, device="cuda"),
118
b.to(dtype=torch.bfloat16, device="cuda"), scale_a, scale_b,
119
torch.bfloat16, label, sub_label, pytorch_mm_impl,
120
"pytorch_bf16_bf16_bf16_matmul-no-scales"))
121
122
+ # pytorch impl - float16
123
+ timers.append(
124
+ bench_fn(a.to(dtype=torch.float16, device="cuda"),
125
+ b.to(dtype=torch.float16, device="cuda"), scale_a, scale_b,
126
+ torch.float16, label, sub_label, pytorch_mm_impl,
127
+ "pytorch_fp16_fp16_fp16_matmul-no-scales"))
128
+
129
# cutlass impl
130
timers.append(
131
bench_fn(a, b, scale_a, scale_b, torch.bfloat16, label, sub_label,
csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cu
+4
-11
@@ -3,6 +3,7 @@
3
#include "cutlass/cutlass.h"
4
5
#include "scaled_mm_c2x.cuh"
6
+#include "scaled_mm_c2x_sm75_dispatch.cuh"
7
#include "scaled_mm_c2x_sm80_dispatch.cuh"
8
#include "scaled_mm_c2x_sm89_fp8_dispatch.cuh"
9
#include "scaled_mm_c2x_sm89_int8_dispatch.cuh"
@@ -20,21 +21,13 @@ void cutlass_scaled_mm_sm75_epilogue(torch::Tensor& out, torch::Tensor const& a,
21
TORCH_CHECK(a.dtype() == torch::kInt8);
22
TORCH_CHECK(b.dtype() == torch::kInt8);
23
23
- using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
24
- using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
25
- using InstructionShape = typename cutlass::gemm::GemmShape<8, 8, 16>;
26
-
24
if (out.dtype() == torch::kBFloat16) {
28
- return vllm::cutlass_gemm_caller<
29
- vllm::cutlass_2x_gemm<cutlass::arch::Sm75, vllm::enable_sm75_to_sm80,
30
- int8_t, cutlass::bfloat16_t, Epilogue, TileShape,
31
- WarpShape, InstructionShape, 2>>(
25
+ return vllm::cutlass_gemm_sm75_dispatch<int8_t, cutlass::bfloat16_t,
26
+ Epilogue>(
27
out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
28
} else {
29
TORCH_CHECK(out.dtype() == torch::kFloat16);
35
- return vllm::cutlass_gemm_caller<vllm::cutlass_2x_gemm<
36
- cutlass::arch::Sm75, vllm::enable_sm75_to_sm80, int8_t, cutlass::half_t,
37
- Epilogue, TileShape, WarpShape, InstructionShape, 2>>(
30
+ return vllm::cutlass_gemm_sm75_dispatch<int8_t, cutlass::half_t, Epilogue>(
31
out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
32
}
33
}
csrc/quantization/cutlass_w8a8/scaled_mm_c2x_sm75_dispatch.cuh
new
+123
@@ -0,0 +1,123 @@
1
+#pragma once
2
+
3
+#include "scaled_mm_c2x.cuh"
4
+
5
+/**
6
+ * This file defines Gemm kernel configurations for SM75 based on the Gemm
7
+ * shape.
8
+ */
9
+
10
+namespace vllm {
11
+
12
+template <typename InType, typename OutType,
13
+ template <typename, typename> typename Epilogue>
14
+struct sm75_config_default {
15
+ // This config is used in 2 cases,
16
+ // - M in (256, inf]
17
+ // - M in (64, 128]
18
+ // Shared memory required by this Gemm 32768
19
+ static_assert(std::is_same<InType, int8_t>());
20
+ using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
21
+ using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
22
+ using InstructionShape = typename cutlass::gemm::GemmShape<8, 8, 16>;
23
+ using Cutlass2xGemm =
24
+ cutlass_2x_gemm<cutlass::arch::Sm75, enable_sm75_to_sm80, InType, OutType,
25
+ Epilogue, TileShape, WarpShape, InstructionShape, 2>;
26
+};
27
+
28
+template <typename InType, typename OutType,
29
+ template <typename, typename> typename Epilogue>
30
+struct sm75_config_M256 {
31
+ // M in (128, 256]
32
+ // Shared memory required by this Gemm 65536
33
+ static_assert(std::is_same<InType, int8_t>());
34
+ using TileShape = typename cutlass::gemm::GemmShape<128, 128, 128>;
35
+ using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
36
+ using InstructionShape = typename cutlass::gemm::GemmShape<8, 8, 16>;
37
+ using Cutlass2xGemm =
38
+ cutlass_2x_gemm<cutlass::arch::Sm75, enable_sm75_to_sm80, InType, OutType,
39
+ Epilogue, TileShape, WarpShape, InstructionShape, 2>;
40
+};
41
+
42
+template <typename InType, typename OutType,
43
+ template <typename, typename> typename Epilogue>
44
+struct sm75_config_M64 {
45
+ // M in (32, 64]
46
+ // Shared memory required by this Gemm 49152
47
+ static_assert(std::is_same<InType, int8_t>());
48
+ using TileShape = typename cutlass::gemm::GemmShape<64, 128, 128>;
49
+ using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
50
+ using InstructionShape = typename cutlass::gemm::GemmShape<8, 8, 16>;
51
+ using Cutlass2xGemm =
52
+ cutlass_2x_gemm<cutlass::arch::Sm75, enable_sm75_to_sm80, InType, OutType,
53
+ Epilogue, TileShape, WarpShape, InstructionShape, 2>;
54
+};
55
+
56
+template <typename InType, typename OutType,
57
+ template <typename, typename> typename Epilogue>
58
+struct sm75_config_M32 {
59
+ // M in [1, 32]
60
+ // Shared memory required by this Gemm 49152
61
+ static_assert(std::is_same<InType, int8_t>());
62
+ using TileShape = typename cutlass::gemm::GemmShape<32, 128, 64>;
63
+ using WarpShape = typename cutlass::gemm::GemmShape<32, 64, 64>;
64
+ using InstructionShape = typename cutlass::gemm::GemmShape<8, 8, 16>;
65
+ using Cutlass2xGemm =
66
+ cutlass_2x_gemm<cutlass::arch::Sm75, enable_sm75_to_sm80, InType, OutType,
67
+ Epilogue, TileShape, WarpShape, InstructionShape, 2>;
68
+};
69
+
70
+template <typename InType, typename OutType,
71
+ template <typename, typename> typename Epilogue,
72
+ typename... EpilogueArgs>
73
+inline void cutlass_gemm_sm75_dispatch(torch::Tensor& out,
74
+ torch::Tensor const& a,
75
+ torch::Tensor const& b,
76
+ EpilogueArgs&&... args) {
77
+ static_assert(std::is_same<InType, int8_t>());
78
+ TORCH_CHECK(a.dtype() == torch::kInt8);
79
+ TORCH_CHECK(b.dtype() == torch::kInt8);
80
+
81
+ using Cutlass2xGemmDefault =
82
+ typename sm75_config_default<InType, OutType, Epilogue>::Cutlass2xGemm;
83
+ using Cutlass2xGemmM256 =
84
+ typename sm75_config_M256<InType, OutType, Epilogue>::Cutlass2xGemm;
85
+ using Cutlass2xGemmM128 = Cutlass2xGemmDefault;
86
+ using Cutlass2xGemmM64 =
87
+ typename sm75_config_M64<InType, OutType, Epilogue>::Cutlass2xGemm;
88
+ using Cutlass2xGemmM32 =
89
+ typename sm75_config_M32<InType, OutType, Epilogue>::Cutlass2xGemm;
90
+
91
+ // Due to shared memory requirements, some Gemms may fail to run on some
92
+ // GPUs. As the name indicates, the Fallback Gemm is used as an alternative
93
+ // in such cases.
94
+ // sm75_config_default has the least shared-memory requirements.
95
+ using FallbackGemm = Cutlass2xGemmDefault;
96
+
97
+ uint32_t const m = a.size(0);
98
+ uint32_t const mp2 =
99
+ std::max(static_cast<uint32_t>(32), next_pow_2(m)); // next power of 2
100
+ if (mp2 <= 32) {
101
+ // M in [1, 32]
102
+ return fallback_cutlass_gemm_caller<Cutlass2xGemmM32, FallbackGemm>(
103
+ out, a, b, std::forward<EpilogueArgs>(args)...);
104
+ } else if (mp2 <= 64) {
105
+ // M in (32, 64]
106
+ return fallback_cutlass_gemm_caller<Cutlass2xGemmM64, FallbackGemm>(
107
+ out, a, b, std::forward<EpilogueArgs>(args)...);
108
+ } else if (mp2 <= 128) {
109
+ // M in (64, 128]
110
+ return fallback_cutlass_gemm_caller<Cutlass2xGemmM128, FallbackGemm>(
111
+ out, a, b, std::forward<EpilogueArgs>(args)...);
112
+ } else if (mp2 <= 256) {
113
+ // M in (128, 256]
114
+ return fallback_cutlass_gemm_caller<Cutlass2xGemmM256, FallbackGemm>(
115
+ out, a, b, std::forward<EpilogueArgs>(args)...);
116
+ } else {
117
+ // M in (256, inf)
118
+ return fallback_cutlass_gemm_caller<Cutlass2xGemmDefault, FallbackGemm>(
119
+ out, a, b, std::forward<EpilogueArgs>(args)...);
120
+ }
121
+}
122
+
123
+} // namespace vllm