A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Kernel] Tuned int8 kernels for Ada Lovelace (#6848)
Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
Varun Sundar Rabindranath committed
Jul 29, 2024 at 22:24 UTC
af647fb8b3ea9d910f7d1bc104af8986d048a8e2
4 files changed
+395
-43
csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cu
+10
-17
@@ -4,7 +4,8 @@
4
5
#include "scaled_mm_c2x.cuh"
6
#include "scaled_mm_c2x_sm80_dispatch.cuh"
7
-#include "scaled_mm_c2x_sm89_dispatch.cuh"
7
+#include "scaled_mm_c2x_sm89_fp8_dispatch.cuh"
8
+#include "scaled_mm_c2x_sm89_int8_dispatch.cuh"
9
10
/*
11
This file defines quantized GEMM operations using the CUTLASS 2.x API, for
@@ -98,25 +99,17 @@ template <template <typename, typename> typename Epilogue,
99
void cutlass_scaled_mm_sm89_epilogue(torch::Tensor& out, torch::Tensor const& a,
100
torch::Tensor const& b,
101
EpilogueArgs&&... epilogue_args) {
101
- using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
102
- using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
103
- using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
104
-
102
if (a.dtype() == torch::kInt8) {
103
TORCH_CHECK(b.dtype() == torch::kInt8);
104
105
if (out.dtype() == torch::kBFloat16) {
109
- return vllm::cutlass_gemm_caller<
110
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
111
- int8_t, cutlass::bfloat16_t, Epilogue,
112
- TileShape, WarpShape, InstructionShape, 5>>(
106
+ return vllm::cutlass_gemm_sm89_int8_dispatch<int8_t, cutlass::bfloat16_t,
107
+ Epilogue>(
108
out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
109
} else {
110
assert(out.dtype() == torch::kFloat16);
116
- return vllm::cutlass_gemm_caller<
117
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
118
- int8_t, cutlass::half_t, Epilogue, TileShape,
119
- WarpShape, InstructionShape, 5>>(
111
+ return vllm::cutlass_gemm_sm89_int8_dispatch<int8_t, cutlass::half_t,
112
+ Epilogue>(
113
out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
114
}
115
} else {
@@ -124,13 +117,13 @@ void cutlass_scaled_mm_sm89_epilogue(torch::Tensor& out, torch::Tensor const& a,
117
TORCH_CHECK(b.dtype() == torch::kFloat8_e4m3fn);
118
119
if (out.dtype() == torch::kBFloat16) {
127
- return vllm::cutlass_gemm_sm89_dispatch<cutlass::float_e4m3_t,
128
- cutlass::bfloat16_t, Epilogue>(
120
+ return vllm::cutlass_gemm_sm89_fp8_dispatch<
121
+ cutlass::float_e4m3_t, cutlass::bfloat16_t, Epilogue>(
122
out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
123
} else {
124
TORCH_CHECK(out.dtype() == torch::kFloat16);
132
- return vllm::cutlass_gemm_sm89_dispatch<cutlass::float_e4m3_t,
133
- cutlass::half_t, Epilogue>(
125
+ return vllm::cutlass_gemm_sm89_fp8_dispatch<cutlass::float_e4m3_t,
126
+ cutlass::half_t, Epilogue>(
127
out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
128
}
129
}
csrc/quantization/cutlass_w8a8/scaled_mm_c2x_sm89_fp8_dispatch.cuh
renamed
+30
-24
@@ -4,7 +4,7 @@
4
#include "cutlass/float8.h"
5
6
/**
7
- * This file defines Gemm kernel configurations for SM89 based on the Gemm
7
+ * This file defines Gemm kernel configurations for SM89 (FP8) based on the Gemm
8
* shape.
9
*/
10
@@ -12,7 +12,7 @@ namespace vllm {
12
13
template <typename InType, typename OutType,
14
template <typename, typename> typename Epilogue>
15
-struct sm89_fallback_gemm {
15
+struct sm89_fp8_fallback_gemm {
16
// Shared Memory required by this Gemm - 61440 bytes
17
static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
18
using TileShape = typename cutlass::gemm::GemmShape<64, 128, 64>;
@@ -25,7 +25,7 @@ struct sm89_fallback_gemm {
25
FP8MathOperator>;
26
};
27
28
-struct sm89_config_default {
28
+struct sm89_fp8_config_default {
29
// M in (256, inf)
30
using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
31
using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
@@ -40,7 +40,8 @@ struct sm89_config_default {
40
TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
41
42
using FallbackGemm =
43
- typename sm89_fallback_gemm<InType, OutType, Epilogue>::Cutlass2xGemm;
43
+ typename sm89_fp8_fallback_gemm<InType, OutType,
44
+ Epilogue>::Cutlass2xGemm;
45
46
uint32_t const n = out.size(1);
47
uint32_t const np2 = next_pow_2(n);
@@ -74,7 +75,7 @@ struct sm89_config_default {
75
}
76
};
77
77
-struct sm89_config_M256 {
78
+struct sm89_fp8_config_M256 {
79
// M in (128, 256]
80
using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
81
using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
@@ -89,7 +90,8 @@ struct sm89_config_M256 {
90
TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
91
92
using FallbackGemm =
92
- typename sm89_fallback_gemm<InType, OutType, Epilogue>::Cutlass2xGemm;
93
+ typename sm89_fp8_fallback_gemm<InType, OutType,
94
+ Epilogue>::Cutlass2xGemm;
95
96
uint32_t const n = out.size(1);
97
uint32_t const np2 = next_pow_2(n);
@@ -114,7 +116,7 @@ struct sm89_config_M256 {
116
}
117
};
118
117
-struct sm89_config_M128 {
119
+struct sm89_fp8_config_M128 {
120
// M in (64, 128]
121
using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
122
using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
@@ -129,7 +131,8 @@ struct sm89_config_M128 {
131
TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
132
133
using FallbackGemm =
132
- typename sm89_fallback_gemm<InType, OutType, Epilogue>::Cutlass2xGemm;
134
+ typename sm89_fp8_fallback_gemm<InType, OutType,
135
+ Epilogue>::Cutlass2xGemm;
136
137
uint32_t const n = out.size(1);
138
uint32_t const np2 = next_pow_2(n);
@@ -163,7 +166,7 @@ struct sm89_config_M128 {
166
}
167
};
168
166
-struct sm89_config_M64 {
169
+struct sm89_fp8_config_M64 {
170
// M in (32, 64]
171
using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
172
@@ -176,7 +179,8 @@ struct sm89_config_M64 {
179
TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
180
181
using FallbackGemm =
179
- typename sm89_fallback_gemm<InType, OutType, Epilogue>::Cutlass2xGemm;
182
+ typename sm89_fp8_fallback_gemm<InType, OutType,
183
+ Epilogue>::Cutlass2xGemm;
184
185
uint32_t const n = out.size(1);
186
uint32_t const np2 = next_pow_2(n);
@@ -215,7 +219,7 @@ struct sm89_config_M64 {
219
}
220
};
221
218
-struct sm89_config_M32 {
222
+struct sm89_fp8_config_M32 {
223
// M in (16, 32]
224
using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
225
using FP8MathOperator = typename cutlass::arch::OpMultiplyAddFastAccum;
@@ -229,7 +233,8 @@ struct sm89_config_M32 {
233
TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
234
235
using FallbackGemm =
232
- typename sm89_fallback_gemm<InType, OutType, Epilogue>::Cutlass2xGemm;
236
+ typename sm89_fp8_fallback_gemm<InType, OutType,
237
+ Epilogue>::Cutlass2xGemm;
238
239
uint32_t const n = out.size(1);
240
uint32_t const np2 = next_pow_2(n);
@@ -265,7 +270,7 @@ struct sm89_config_M32 {
270
}
271
};
272
268
-struct sm89_config_M16 {
273
+struct sm89_fp8_config_M16 {
274
// M in [1, 16]
275
using WarpShape = typename cutlass::gemm::GemmShape<16, 64, 64>;
276
using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
@@ -281,7 +286,8 @@ struct sm89_config_M16 {
286
TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
287
288
using FallbackGemm =
284
- typename sm89_fallback_gemm<InType, OutType, Epilogue>::Cutlass2xGemm;
289
+ typename sm89_fp8_fallback_gemm<InType, OutType,
290
+ Epilogue>::Cutlass2xGemm;
291
292
uint32_t const n = out.size(1);
293
uint32_t const np2 = next_pow_2(n);
@@ -320,10 +326,10 @@ struct sm89_config_M16 {
326
template <typename InType, typename OutType,
327
template <typename, typename> typename Epilogue,
328
typename... EpilogueArgs>
323
-inline void cutlass_gemm_sm89_dispatch(torch::Tensor& out,
324
- torch::Tensor const& a,
325
- torch::Tensor const& b,
326
- EpilogueArgs&&... args) {
329
+inline void cutlass_gemm_sm89_fp8_dispatch(torch::Tensor& out,
330
+ torch::Tensor const& a,
331
+ torch::Tensor const& b,
332
+ EpilogueArgs&&... args) {
333
static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
334
TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
335
TORCH_CHECK(b.dtype() == torch::kFloat8_e4m3fn);
@@ -334,27 +340,27 @@ inline void cutlass_gemm_sm89_dispatch(torch::Tensor& out,
340
341
if (mp2 <= 16) {
342
// M in [1, 16]
337
- return sm89_config_M16::dispatch<InType, OutType, Epilogue>(
343
+ return sm89_fp8_config_M16::dispatch<InType, OutType, Epilogue>(
344
out, a, b, std::forward<EpilogueArgs>(args)...);
345
} else if (mp2 <= 32) {
346
// M in (16, 32]
341
- return sm89_config_M32::dispatch<InType, OutType, Epilogue>(
347
+ return sm89_fp8_config_M32::dispatch<InType, OutType, Epilogue>(
348
out, a, b, std::forward<EpilogueArgs>(args)...);
349
} else if (mp2 <= 64) {
350
// M in (32, 64]
345
- return sm89_config_M64::dispatch<InType, OutType, Epilogue>(
351
+ return sm89_fp8_config_M64::dispatch<InType, OutType, Epilogue>(
352
out, a, b, std::forward<EpilogueArgs>(args)...);
353
} else if (mp2 <= 128) {
354
// M in (64, 128]
349
- return sm89_config_M128::dispatch<InType, OutType, Epilogue>(
355
+ return sm89_fp8_config_M128::dispatch<InType, OutType, Epilogue>(
356
out, a, b, std::forward<EpilogueArgs>(args)...);
357
} else if (mp2 <= 256) {
358
// M in (128, 256]
353
- return sm89_config_M256::dispatch<InType, OutType, Epilogue>(
359
+ return sm89_fp8_config_M256::dispatch<InType, OutType, Epilogue>(
360
out, a, b, std::forward<EpilogueArgs>(args)...);
361
} else {
362
// M in (256, inf)
357
- return sm89_config_default::dispatch<InType, OutType, Epilogue>(
363
+ return sm89_fp8_config_default::dispatch<InType, OutType, Epilogue>(
364
out, a, b, std::forward<EpilogueArgs>(args)...);
365
}
366
}
csrc/quantization/cutlass_w8a8/scaled_mm_c2x_sm89_int8_dispatch.cuh
new
+353
@@ -0,0 +1,353 @@
1
+#pragma once
2
+
3
+#include "scaled_mm_c2x.cuh"
4
+
5
+/**
6
+ * This file defines Gemm kernel configurations for SM89 (int8) based on the
7
+ * Gemm shape.
8
+ */
9
+
10
+namespace vllm {
11
+
12
+template <typename InType, typename OutType,
13
+ template <typename, typename> typename Epilogue>
14
+struct sm89_int8_fallback_gemm {
15
+ // Shared mem requirement : 61440
16
+ static_assert(std::is_same<InType, int8_t>());
17
+ using TileShape = cutlass::gemm::GemmShape<32, 64, 128>;
18
+ using WarpShape = cutlass::gemm::GemmShape<16, 64, 64>;
19
+ using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
20
+ static int32_t const MainLoopStages = 5;
21
+
22
+ using Cutlass2xGemm =
23
+ cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90, InType, OutType,
24
+ Epilogue, TileShape, WarpShape, InstructionShape, 5>;
25
+};
26
+
27
+struct sm89_int8_config_default {
28
+ // M in (256, inf)
29
+ using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
30
+ using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
31
+
32
+ template <typename InType, typename OutType,
33
+ template <typename, typename> typename Epilogue,
34
+ typename... EpilogueArgs>
35
+ static void dispatch(torch::Tensor& out, torch::Tensor const& a,
36
+ torch::Tensor const& b, EpilogueArgs&&... args) {
37
+ static_assert(std::is_same<InType, int8_t>());
38
+ TORCH_CHECK(a.dtype() == torch::kInt8);
39
+
40
+ using FallbackGemm =
41
+ typename sm89_int8_fallback_gemm<InType, OutType,
42
+ Epilogue>::Cutlass2xGemm;
43
+
44
+ uint32_t const n = out.size(1);
45
+ uint32_t const np2 = next_pow_2(n);
46
+
47
+ if (np2 <= 4096) {
48
+ using TileShape = cutlass::gemm::GemmShape<128, 128, 64>;
49
+
50
+ return vllm::fallback_cutlass_gemm_caller<
51
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
52
+ InType, OutType, Epilogue, TileShape, WarpShape,
53
+ InstructionShape, 5>,
54
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
55
+ } else if (np2 <= 8192) {
56
+ using TileShape = cutlass::gemm::GemmShape<256, 128, 64>;
57
+
58
+ return vllm::fallback_cutlass_gemm_caller<
59
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
60
+ InType, OutType, Epilogue, TileShape, WarpShape,
61
+ InstructionShape, 3>,
62
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
63
+ } else if (np2 <= 16384) {
64
+ using TileShape = cutlass::gemm::GemmShape<128, 128, 64>;
65
+
66
+ return vllm::fallback_cutlass_gemm_caller<
67
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
68
+ InType, OutType, Epilogue, TileShape, WarpShape,
69
+ InstructionShape, 5>,
70
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
71
+ } else {
72
+ using TileShape = cutlass::gemm::GemmShape<256, 128, 64>;
73
+
74
+ return vllm::fallback_cutlass_gemm_caller<
75
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
76
+ InType, OutType, Epilogue, TileShape, WarpShape,
77
+ InstructionShape, 3>,
78
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
79
+ }
80
+ }
81
+};
82
+
83
+struct sm89_int8_config_M256 {
84
+ // M in (128, 256]
85
+ using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
86
+ using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
87
+
88
+ template <typename InType, typename OutType,
89
+ template <typename, typename> typename Epilogue,
90
+ typename... EpilogueArgs>
91
+ static void dispatch(torch::Tensor& out, torch::Tensor const& a,
92
+ torch::Tensor const& b, EpilogueArgs&&... args) {
93
+ static_assert(std::is_same<InType, int8_t>());
94
+ TORCH_CHECK(a.dtype() == torch::kInt8);
95
+
96
+ using FallbackGemm =
97
+ typename sm89_int8_fallback_gemm<InType, OutType,
98
+ Epilogue>::Cutlass2xGemm;
99
+
100
+ uint32_t const n = out.size(1);
101
+ uint32_t const np2 = next_pow_2(n);
102
+
103
+ if (np2 <= 4096) {
104
+ using TileShape = cutlass::gemm::GemmShape<64, 128, 128>;
105
+
106
+ return vllm::fallback_cutlass_gemm_caller<
107
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
108
+ InType, OutType, Epilogue, TileShape, WarpShape,
109
+ InstructionShape, 3>,
110
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
111
+ } else if (np2 <= 8192) {
112
+ using TileShape = cutlass::gemm::GemmShape<128, 128, 64>;
113
+
114
+ return vllm::fallback_cutlass_gemm_caller<
115
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
116
+ InType, OutType, Epilogue, TileShape, WarpShape,
117
+ InstructionShape, 5>,
118
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
119
+ } else if (np2 <= 16384) {
120
+ using TileShape = cutlass::gemm::GemmShape<256, 128, 64>;
121
+
122
+ return vllm::fallback_cutlass_gemm_caller<
123
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
124
+ InType, OutType, Epilogue, TileShape, WarpShape,
125
+ InstructionShape, 3>,
126
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
127
+ } else {
128
+ using TileShape = cutlass::gemm::GemmShape<128, 128, 64>;
129
+
130
+ return vllm::fallback_cutlass_gemm_caller<
131
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
132
+ InType, OutType, Epilogue, TileShape, WarpShape,
133
+ InstructionShape, 5>,
134
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
135
+ }
136
+ }
137
+};
138
+
139
+struct sm89_int8_config_M128 {
140
+ // M in (64, 128]
141
+ using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
142
+
143
+ template <typename InType, typename OutType,
144
+ template <typename, typename> typename Epilogue,
145
+ typename... EpilogueArgs>
146
+ static void dispatch(torch::Tensor& out, torch::Tensor const& a,
147
+ torch::Tensor const& b, EpilogueArgs&&... args) {
148
+ static_assert(std::is_same<InType, int8_t>());
149
+ TORCH_CHECK(a.dtype() == torch::kInt8);
150
+
151
+ using FallbackGemm =
152
+ typename sm89_int8_fallback_gemm<InType, OutType,
153
+ Epilogue>::Cutlass2xGemm;
154
+
155
+ uint32_t const n = out.size(1);
156
+ uint32_t const np2 = next_pow_2(n);
157
+
158
+ if (np2 <= 8192) {
159
+ using TileShape = cutlass::gemm::GemmShape<64, 128, 128>;
160
+ using WarpShape = cutlass::gemm::GemmShape<64, 64, 64>;
161
+
162
+ return vllm::fallback_cutlass_gemm_caller<
163
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
164
+ InType, OutType, Epilogue, TileShape, WarpShape,
165
+ InstructionShape, 3>,
166
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
167
+ } else if (np2 <= 16384) {
168
+ using TileShape = cutlass::gemm::GemmShape<128, 128, 64>;
169
+ using WarpShape = cutlass::gemm::GemmShape<64, 64, 64>;
170
+
171
+ return vllm::fallback_cutlass_gemm_caller<
172
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
173
+ InType, OutType, Epilogue, TileShape, WarpShape,
174
+ InstructionShape, 5>,
175
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
176
+ } else {
177
+ using TileShape = cutlass::gemm::GemmShape<64, 64, 128>;
178
+ using WarpShape = cutlass::gemm::GemmShape<32, 64, 64>;
179
+
180
+ return vllm::fallback_cutlass_gemm_caller<
181
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
182
+ InType, OutType, Epilogue, TileShape, WarpShape,
183
+ InstructionShape, 5>,
184
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
185
+ }
186
+ }
187
+};
188
+
189
+struct sm89_int8_config_M64 {
190
+ // M in (32, 64]
191
+ using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
192
+
193
+ template <typename InType, typename OutType,
194
+ template <typename, typename> typename Epilogue,
195
+ typename... EpilogueArgs>
196
+ static void dispatch(torch::Tensor& out, torch::Tensor const& a,
197
+ torch::Tensor const& b, EpilogueArgs&&... args) {
198
+ static_assert(std::is_same<InType, int8_t>());
199
+ TORCH_CHECK(a.dtype() == torch::kInt8);
200
+
201
+ using FallbackGemm =
202
+ typename sm89_int8_fallback_gemm<InType, OutType,
203
+ Epilogue>::Cutlass2xGemm;
204
+
205
+ uint32_t const n = out.size(1);
206
+ uint32_t const np2 = next_pow_2(n);
207
+
208
+ if (np2 <= 8192) {
209
+ using TileShape = cutlass::gemm::GemmShape<64, 64, 128>;
210
+ using WarpShape = cutlass::gemm::GemmShape<32, 64, 64>;
211
+
212
+ return vllm::fallback_cutlass_gemm_caller<
213
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
214
+ InType, OutType, Epilogue, TileShape, WarpShape,
215
+ InstructionShape, 5>,
216
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
217
+ } else {
218
+ using TileShape = cutlass::gemm::GemmShape<64, 128, 128>;
219
+ using WarpShape = cutlass::gemm::GemmShape<64, 64, 64>;
220
+
221
+ return vllm::fallback_cutlass_gemm_caller<
222
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
223
+ InType, OutType, Epilogue, TileShape, WarpShape,
224
+ InstructionShape, 3>,
225
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
226
+ }
227
+ }
228
+};
229
+
230
+struct sm89_int8_config_M32 {
231
+ // M in (16, 32]
232
+ using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
233
+
234
+ template <typename InType, typename OutType,
235
+ template <typename, typename> typename Epilogue,
236
+ typename... EpilogueArgs>
237
+ static void dispatch(torch::Tensor& out, torch::Tensor const& a,
238
+ torch::Tensor const& b, EpilogueArgs&&... args) {
239
+ static_assert(std::is_same<InType, int8_t>());
240
+ TORCH_CHECK(a.dtype() == torch::kInt8);
241
+
242
+ using FallbackGemm =
243
+ typename sm89_int8_fallback_gemm<InType, OutType,
244
+ Epilogue>::Cutlass2xGemm;
245
+
246
+ uint32_t const n = out.size(1);
247
+ uint32_t const np2 = next_pow_2(n);
248
+
249
+ if (np2 <= 8192) {
250
+ using TileShape = cutlass::gemm::GemmShape<32, 64, 128>;
251
+ using WarpShape = cutlass::gemm::GemmShape<16, 64, 64>;
252
+
253
+ return vllm::fallback_cutlass_gemm_caller<
254
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
255
+ InType, OutType, Epilogue, TileShape, WarpShape,
256
+ InstructionShape, 5>,
257
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
258
+ } else {
259
+ using TileShape = cutlass::gemm::GemmShape<32, 128, 128>;
260
+ using WarpShape = cutlass::gemm::GemmShape<32, 64, 64>;
261
+
262
+ return vllm::fallback_cutlass_gemm_caller<
263
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
264
+ InType, OutType, Epilogue, TileShape, WarpShape,
265
+ InstructionShape, 4>,
266
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
267
+ }
268
+ }
269
+};
270
+
271
+struct sm89_int8_config_M16 {
272
+ // M in [1, 16]
273
+ using WarpShape = typename cutlass::gemm::GemmShape<16, 64, 64>;
274
+ using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
275
+
276
+ template <typename InType, typename OutType,
277
+ template <typename, typename> typename Epilogue,
278
+ typename... EpilogueArgs>
279
+ static void dispatch(torch::Tensor& out, torch::Tensor const& a,
280
+ torch::Tensor const& b, EpilogueArgs&&... args) {
281
+ static_assert(std::is_same<InType, int8_t>());
282
+ TORCH_CHECK(a.dtype() == torch::kInt8);
283
+
284
+ using FallbackGemm =
285
+ typename sm89_int8_fallback_gemm<InType, OutType,
286
+ Epilogue>::Cutlass2xGemm;
287
+
288
+ uint32_t const n = out.size(1);
289
+ uint32_t const np2 = next_pow_2(n);
290
+
291
+ if (np2 <= 8192) {
292
+ using TileShape = cutlass::gemm::GemmShape<16, 64, 128>;
293
+
294
+ return vllm::fallback_cutlass_gemm_caller<
295
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
296
+ InType, OutType, Epilogue, TileShape, WarpShape,
297
+ InstructionShape, 5>,
298
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
299
+ } else {
300
+ using TileShape = cutlass::gemm::GemmShape<16, 128, 128>;
301
+
302
+ return vllm::fallback_cutlass_gemm_caller<
303
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
304
+ InType, OutType, Epilogue, TileShape, WarpShape,
305
+ InstructionShape, 4>,
306
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
307
+ }
308
+ }
309
+};
310
+
311
+template <typename InType, typename OutType,
312
+ template <typename, typename> typename Epilogue,
313
+ typename... EpilogueArgs>
314
+inline void cutlass_gemm_sm89_int8_dispatch(torch::Tensor& out,
315
+ torch::Tensor const& a,
316
+ torch::Tensor const& b,
317
+ EpilogueArgs&&... args) {
318
+ static_assert(std::is_same<InType, int8_t>());
319
+ TORCH_CHECK(a.dtype() == torch::kInt8);
320
+ TORCH_CHECK(b.dtype() == torch::kInt8);
321
+
322
+ uint32_t const m = a.size(0);
323
+ uint32_t const mp2 =
324
+ std::max(static_cast<uint32_t>(32), next_pow_2(m)); // next power of 2
325
+
326
+ if (mp2 <= 16) {
327
+ // M in [1, 16]
328
+ return sm89_int8_config_M16::dispatch<InType, OutType, Epilogue>(
329
+ out, a, b, std::forward<EpilogueArgs>(args)...);
330
+ } else if (mp2 <= 32) {
331
+ // M in (16, 32]
332
+ return sm89_int8_config_M32::dispatch<InType, OutType, Epilogue>(
333
+ out, a, b, std::forward<EpilogueArgs>(args)...);
334
+ } else if (mp2 <= 64) {
335
+ // M in (32, 64]
336
+ return sm89_int8_config_M64::dispatch<InType, OutType, Epilogue>(
337
+ out, a, b, std::forward<EpilogueArgs>(args)...);
338
+ } else if (mp2 <= 128) {
339
+ // M in (64, 128]
340
+ return sm89_int8_config_M128::dispatch<InType, OutType, Epilogue>(
341
+ out, a, b, std::forward<EpilogueArgs>(args)...);
342
+ } else if (mp2 <= 256) {
343
+ // M in (128, 256]
344
+ return sm89_int8_config_M256::dispatch<InType, OutType, Epilogue>(
345
+ out, a, b, std::forward<EpilogueArgs>(args)...);
346
+ } else {
347
+ // M in (256, inf)
348
+ return sm89_int8_config_default::dispatch<InType, OutType, Epilogue>(
349
+ out, a, b, std::forward<EpilogueArgs>(args)...);
350
+ }
351
+}
352
+
353
+} // namespace vllm
tests/kernels/test_cutlass.py
+2
-2
@@ -119,8 +119,8 @@ def test_cutlass_fp8_gemm(m: int, n: int, k: int, per_act_token: bool,
119
cutlass_fp8_gemm_helper(m, n, k, per_act_token, per_out_ch, use_bias)
120
121
122
-@pytest.mark.parametrize("m", [512, 222, 33, 1])
123
-@pytest.mark.parametrize("n", [2048, 256, 1024])
122
+@pytest.mark.parametrize("m", [1, 16, 32, 64, 128, 256, 512, 222, 33, 1])
123
+@pytest.mark.parametrize("n", [2048, 8192, 16384, 256, 1024])
124
@pytest.mark.parametrize("k", [128, 496, 1024])
125
@pytest.mark.parametrize("per_act_token", [True, False])
126
@pytest.mark.parametrize("per_out_ch", [True, False])