A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Kernel] Refactor Cutlass c3x (#10049)
Signed-off-by: Varun Sundar Rabindranath <varun@neuralmagic.com> Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
Varun Sundar Rabindranath committed
Dec 19, 2024 at 02:00 UTC
8936316d587ca0afb5ef058584c407d404c0ffb0
7 files changed
+406
-373
csrc/cutlass_extensions/epilogue/scaled_mm_epilogues_c2x.hpp
+2
@@ -1,3 +1,5 @@
1
+#pragma once
2
+
3
#include "cutlass_extensions/epilogue/broadcast_load_epilogue_c2x.hpp"
4
5
/*
csrc/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp
+2
@@ -1,3 +1,5 @@
1
+#pragma once
2
+
3
#include "cutlass_extensions/epilogue/broadcast_load_epilogue_c3x.hpp"
4
5
/*
csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cuh
+3
-3
@@ -28,9 +28,9 @@
28
using namespace cute;
29
30
/*
31
- Epilogue functions can be defined to post-process the output before it is
32
- written to GPU memory.
33
- Epilogues must contain a public type named EVTCompute of type Sm80EVT,
31
+ Epilogues defined in,
32
+ csrc/cutlass_extensions/epilogue/scaled_mm_epilogues_c2x.hpp
33
+ must contain a public type named EVTCompute of type Sm80EVT,
34
as well as a static prepare_args function that constructs an
35
EVTCompute::Arguments struct.
36
*/
csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu
+3
-370
@@ -1,385 +1,18 @@
1
-// clang-format will break include orders
2
-// clang-format off
1
#include <cudaTypedefs.h>
2
3
#if defined CUDA_VERSION && CUDA_VERSION >= 12000
4
7
-#include <torch/all.h>
5
+ #include "scaled_mm_c3x_sm90_fp8_dispatch.cuh"
6
+ #include "scaled_mm_c3x_sm90_int8_dispatch.cuh"
7
9
-#include <ATen/cuda/CUDAContext.h>
10
-
11
-#include <iostream>
12
-#include <sstream>
13
-#include <vector>
14
-
15
-#include "cutlass/cutlass.h"
16
-
17
-#include "cute/tensor.hpp"
18
-#include "cute/atom/mma_atom.hpp"
19
-#include "cutlass/numeric_types.h"
20
-
21
-#include "cutlass/gemm/device/gemm_universal_adapter.h"
22
-#include "cutlass/gemm/kernel/gemm_universal.hpp"
23
-#include "cutlass/epilogue/collective/collective_builder.hpp"
24
-#include "cutlass/gemm/collective/collective_builder.hpp"
25
-
26
-#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
27
-#include "core/math.hpp"
28
-#include "cutlass_extensions/common.hpp"
29
-// clang-format on
30
-
31
-using namespace cute;
8
+ #include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
9
using namespace vllm;
10
11
/*
12
This file defines quantized GEMM operations using the CUTLASS 3.x API, for
13
NVIDIA GPUs with sm90a (Hopper) or later.
37
-
38
- Epilogue functions can be defined to post-process the output before it is
39
- written to GPU memory.
40
- Epilogues must contain a public type named EVTCompute of type Sm90EVT,
41
- as well as a static prepare_args function that constructs an
42
- EVTCompute::Arguments struct.
14
*/
15
45
-namespace {
46
-
47
-// A wrapper for the GEMM kernel that is used to guard against compilation on
48
-// architectures that will never use the kernel. The purpose of this is to
49
-// reduce the size of the compiled binary.
50
-// __CUDA_ARCH__ is not defined in host code, so this lets us smuggle the ifdef
51
-// into code that will be executed on the device where it is defined.
52
-template <typename Kernel>
53
-struct enable_sm90_or_later : Kernel {
54
- template <typename... Args>
55
- CUTLASS_DEVICE void operator()(Args&&... args) {
56
- #if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 900
57
- Kernel::operator()(std::forward<Args>(args)...);
58
- #endif
59
- }
60
-};
61
-template <typename ElementAB_, typename ElementD_,
62
- template <typename, typename, typename> typename Epilogue_,
63
- typename TileShape, typename ClusterShape, typename KernelSchedule,
64
- typename EpilogueSchedule>
65
-struct cutlass_3x_gemm {
66
- using ElementAB = ElementAB_;
67
- using ElementD = ElementD_;
68
- using ElementAcc =
69
- typename std::conditional<std::is_same_v<ElementAB, int8_t>, int32_t,
70
- float>::type;
71
-
72
- using EpilogueDescriptor =
73
- cutlass::epilogue::collective::detail::EpilogueDescriptor<
74
- TileShape, cutlass::epilogue::collective::EpilogueTileAuto, ElementD,
75
- ElementD, EpilogueSchedule>;
76
-
77
- using Epilogue = Epilogue_<ElementAcc, ElementD, EpilogueDescriptor>;
78
-
79
- using StrideD = Stride<int64_t, Int<1>, Int<0>>;
80
- using ElementC = void;
81
- using StrideC = StrideD;
82
-
83
- using EVTCompute = typename Epilogue::EVTCompute;
84
-
85
- using CollectiveEpilogue =
86
- typename cutlass::epilogue::collective::CollectiveBuilder<
87
- cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp, TileShape,
88
- ClusterShape, cutlass::epilogue::collective::EpilogueTileAuto,
89
- ElementAcc, float, ElementC, StrideC, 4, ElementD, StrideD, 4,
90
- EpilogueSchedule, EVTCompute>::CollectiveOp;
91
-
92
- static constexpr size_t CEStorageSize =
93
- sizeof(typename CollectiveEpilogue::SharedStorage);
94
- using Stages = typename cutlass::gemm::collective::StageCountAutoCarveout<
95
- static_cast<int>(CEStorageSize)>;
96
-
97
- // clang-format off
98
- using CollectiveMainloop =
99
- typename cutlass::gemm::collective::CollectiveBuilder<
100
- cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
101
- ElementAB, cutlass::layout::RowMajor, 16,
102
- ElementAB, cutlass::layout::ColumnMajor, 16,
103
- ElementAcc, TileShape, ClusterShape,
104
- Stages,
105
- KernelSchedule>::CollectiveOp;
106
- // clang-format on
107
-
108
- using KernelType = enable_sm90_or_later<cutlass::gemm::kernel::GemmUniversal<
109
- cute::Shape<int, int, int, int>, CollectiveMainloop, CollectiveEpilogue,
110
- cutlass::gemm::PersistentScheduler>>;
111
-
112
- struct GemmKernel : public KernelType {};
113
-};
114
-
115
-template <typename Gemm, typename... EpilogueArgs>
116
-void cutlass_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
117
- torch::Tensor const& b,
118
- EpilogueArgs&&... epilogue_params) {
119
- using ElementAB = typename Gemm::ElementAB;
120
- using ElementD = typename Gemm::ElementD;
121
-
122
- int32_t m = a.size(0);
123
- int32_t n = b.size(1);
124
- int32_t k = a.size(1);
125
-
126
- int64_t lda = a.stride(0);
127
- int64_t ldb = b.stride(1);
128
- int64_t ldc = out.stride(0);
129
-
130
- using StrideA = Stride<int64_t, Int<1>, int64_t>;
131
- using StrideB = Stride<int64_t, Int<1>, int64_t>;
132
- using StrideC = typename Gemm::StrideC;
133
-
134
- StrideA a_stride{lda, Int<1>{}, 0};
135
- StrideB b_stride{ldb, Int<1>{}, 0};
136
- StrideC c_stride{ldc, Int<1>{}, Int<0>{}};
137
-
138
- using GemmKernel = typename Gemm::GemmKernel;
139
- typename GemmKernel::ProblemShape prob_shape{m, n, k, 1};
140
-
141
- auto a_ptr = static_cast<ElementAB*>(a.data_ptr());
142
- auto b_ptr = static_cast<ElementAB*>(b.data_ptr());
143
- typename GemmKernel::MainloopArguments mainloop_args{a_ptr, a_stride, b_ptr,
144
- b_stride};
145
-
146
- auto c_ptr = static_cast<ElementD*>(out.data_ptr());
147
- typename GemmKernel::EpilogueArguments epilogue_args{
148
- Gemm::Epilogue::prepare_args(
149
- std::forward<EpilogueArgs>(epilogue_params)...),
150
- c_ptr, c_stride, c_ptr, c_stride};
151
-
152
- typename GemmKernel::Arguments args{cutlass::gemm::GemmUniversalMode::kGemm,
153
- prob_shape, mainloop_args, epilogue_args};
154
-
155
- // Launch the CUTLASS GEMM kernel.
156
- using GemmOp = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
157
- GemmOp gemm_op;
158
- CUTLASS_CHECK(gemm_op.can_implement(args));
159
-
160
- size_t workspace_size = gemm_op.get_workspace_size(args);
161
- auto const workspace_options =
162
- torch::TensorOptions().dtype(torch::kUInt8).device(a.device());
163
- auto workspace = torch::empty(workspace_size, workspace_options);
164
-
165
- auto stream = at::cuda::getCurrentCUDAStream(a.get_device());
166
-
167
- cutlass::Status status = gemm_op.run(args, workspace.data_ptr(), stream);
168
- CUTLASS_CHECK(status);
169
-}
170
-
171
-template <typename InType, typename OutType,
172
- template <typename, typename, typename> typename Epilogue>
173
-struct sm90_fp8_config_default {
174
- // M in (128, inf)
175
- static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
176
- using KernelSchedule =
177
- cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum;
178
- using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
179
- using TileShape = Shape<_128, _128, _128>;
180
- using ClusterShape = Shape<_2, _1, _1>;
181
- using Cutlass3xGemm =
182
- cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
183
- KernelSchedule, EpilogueSchedule>;
184
-};
185
-
186
-template <typename InType, typename OutType,
187
- template <typename, typename, typename> typename Epilogue>
188
-struct sm90_fp8_config_M128 {
189
- // M in (64, 128]
190
- static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
191
- using KernelSchedule =
192
- cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum;
193
- using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
194
- using TileShape = Shape<_64, _128, _128>;
195
- using ClusterShape = Shape<_2, _1, _1>;
196
- using Cutlass3xGemm =
197
- cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
198
- KernelSchedule, EpilogueSchedule>;
199
-};
200
-
201
-template <typename InType, typename OutType,
202
- template <typename, typename, typename> typename Epilogue>
203
-struct sm90_fp8_config_M64 {
204
- // M in [1, 64]
205
- static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
206
- using KernelSchedule =
207
- cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum;
208
- using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
209
- using TileShape = Shape<_64, _64, _128>;
210
- using ClusterShape = Shape<_1, _8, _1>;
211
-
212
- using Cutlass3xGemm =
213
- cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
214
- KernelSchedule, EpilogueSchedule>;
215
-};
216
-
217
-template <typename InType, typename OutType,
218
- template <typename, typename, typename> typename Epilogue>
219
-struct sm90_int8_config_default {
220
- // For M > 128 and any N
221
- static_assert(std::is_same<InType, int8_t>());
222
- using KernelSchedule =
223
- typename cutlass::gemm::KernelTmaWarpSpecializedPingpong;
224
- using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
225
- using TileShape = Shape<_128, _128, _128>;
226
- using ClusterShape = Shape<_2, _1, _1>;
227
- using Cutlass3xGemm =
228
- cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
229
- KernelSchedule, EpilogueSchedule>;
230
-};
231
-
232
-template <typename InType, typename OutType,
233
- template <typename, typename, typename> typename Epilogue>
234
-struct sm90_int8_config_M128 {
235
- // For M in (64, 128] and any N
236
- static_assert(std::is_same<InType, int8_t>());
237
- using KernelSchedule =
238
- typename cutlass::gemm::KernelTmaWarpSpecializedPingpong;
239
- using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
240
- using TileShape = Shape<_64, _128, _128>;
241
- using ClusterShape = Shape<_2, _1, _1>;
242
- using Cutlass3xGemm =
243
- cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
244
- KernelSchedule, EpilogueSchedule>;
245
-};
246
-
247
-template <typename InType, typename OutType,
248
- template <typename, typename, typename> typename Epilogue>
249
-struct sm90_int8_config_M64 {
250
- // For M in (32, 64] and any N
251
- static_assert(std::is_same<InType, int8_t>());
252
- using KernelSchedule = typename cutlass::gemm::KernelTmaWarpSpecialized;
253
- using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
254
- using TileShape = Shape<_64, _64, _256>;
255
- using ClusterShape = Shape<_1, _1, _1>;
256
- using Cutlass3xGemm =
257
- cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
258
- KernelSchedule, EpilogueSchedule>;
259
-};
260
-
261
-template <typename InType, typename OutType,
262
- template <typename, typename, typename> typename Epilogue>
263
-struct sm90_int8_config_M32_NBig {
264
- // For M in [1, 32] and N >= 8192
265
- static_assert(std::is_same<InType, int8_t>());
266
- using KernelSchedule = typename cutlass::gemm::KernelTmaWarpSpecialized;
267
- using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
268
- using TileShape = Shape<_64, _128, _256>;
269
- using ClusterShape = Shape<_1, _4, _1>;
270
- using Cutlass3xGemm =
271
- cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
272
- KernelSchedule, EpilogueSchedule>;
273
-};
274
-
275
-template <typename InType, typename OutType,
276
- template <typename, typename, typename> typename Epilogue>
277
-struct sm90_int8_config_M32_NSmall {
278
- // For M in [1, 32] and N < 8192
279
- static_assert(std::is_same<InType, int8_t>());
280
- using KernelSchedule = typename cutlass::gemm::KernelTmaWarpSpecialized;
281
- using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
282
- using TileShape = Shape<_64, _64, _256>;
283
- using ClusterShape = Shape<_1, _8, _1>;
284
- using Cutlass3xGemm =
285
- cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
286
- KernelSchedule, EpilogueSchedule>;
287
-};
288
-
289
-} // namespace
290
-
291
-template <typename InType, typename OutType,
292
- template <typename, typename, typename> typename Epilogue,
293
- typename... EpilogueArgs>
294
-void cutlass_gemm_sm90_fp8_dispatch(torch::Tensor& out, torch::Tensor const& a,
295
- torch::Tensor const& b,
296
- EpilogueArgs&&... args) {
297
- static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
298
- TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
299
- TORCH_CHECK(b.dtype() == torch::kFloat8_e4m3fn);
300
-
301
- using Cutlass3xGemmDefault =
302
- typename sm90_fp8_config_default<InType, OutType,
303
- Epilogue>::Cutlass3xGemm;
304
- using Cutlass3xGemmM64 =
305
- typename sm90_fp8_config_M64<InType, OutType, Epilogue>::Cutlass3xGemm;
306
- using Cutlass3xGemmM128 =
307
- typename sm90_fp8_config_M128<InType, OutType, Epilogue>::Cutlass3xGemm;
308
-
309
- uint32_t const m = a.size(0);
310
- uint32_t const mp2 =
311
- std::max(static_cast<uint32_t>(64), next_pow_2(m)); // next power of 2
312
-
313
- if (mp2 <= 64) {
314
- // m in [1, 64]
315
- return cutlass_gemm_caller<Cutlass3xGemmM64>(
316
- out, a, b, std::forward<EpilogueArgs>(args)...);
317
- } else if (mp2 <= 128) {
318
- // m in (64, 128]
319
- return cutlass_gemm_caller<Cutlass3xGemmM128>(
320
- out, a, b, std::forward<EpilogueArgs>(args)...);
321
- } else {
322
- // m in (128, inf)
323
- return cutlass_gemm_caller<Cutlass3xGemmDefault>(
324
- out, a, b, std::forward<EpilogueArgs>(args)...);
325
- }
326
-}
327
-
328
-template <typename InType, typename OutType,
329
- template <typename, typename, typename> typename Epilogue,
330
- typename... EpilogueArgs>
331
-void cutlass_gemm_sm90_int8_dispatch(torch::Tensor& out, torch::Tensor const& a,
332
- torch::Tensor const& b,
333
- EpilogueArgs&&... args) {
334
- static_assert(std::is_same<InType, int8_t>());
335
- TORCH_CHECK(a.dtype() == torch::kInt8);
336
- TORCH_CHECK(b.dtype() == torch::kInt8);
337
-
338
- using Cutlass3xGemmDefault =
339
- typename sm90_int8_config_default<InType, OutType,
340
- Epilogue>::Cutlass3xGemm;
341
- using Cutlass3xGemmM128 =
342
- typename sm90_int8_config_M128<InType, OutType, Epilogue>::Cutlass3xGemm;
343
- using Cutlass3xGemmM64 =
344
- typename sm90_int8_config_M64<InType, OutType, Epilogue>::Cutlass3xGemm;
345
- using Cutlass3xGemmM32NBig =
346
- typename sm90_int8_config_M32_NBig<InType, OutType,
347
- Epilogue>::Cutlass3xGemm;
348
- using Cutlass3xGemmM32NSmall =
349
- typename sm90_int8_config_M32_NSmall<InType, OutType,
350
- Epilogue>::Cutlass3xGemm;
351
-
352
- uint32_t const n = out.size(1);
353
- bool const is_small_n = n < 8192;
354
-
355
- uint32_t const m = a.size(0);
356
- uint32_t const mp2 =
357
- std::max(static_cast<uint32_t>(32), next_pow_2(m)); // next power of 2
358
-
359
- if (mp2 <= 32) {
360
- // m in [1, 32]
361
- if (is_small_n) {
362
- return cutlass_gemm_caller<Cutlass3xGemmM32NSmall>(
363
- out, a, b, std::forward<EpilogueArgs>(args)...);
364
- } else {
365
- return cutlass_gemm_caller<Cutlass3xGemmM32NBig>(
366
- out, a, b, std::forward<EpilogueArgs>(args)...);
367
- }
368
- } else if (mp2 <= 64) {
369
- // m in (32, 64]
370
- return cutlass_gemm_caller<Cutlass3xGemmM64>(
371
- out, a, b, std::forward<EpilogueArgs>(args)...);
372
- } else if (mp2 <= 128) {
373
- // m in (64, 128]
374
- return cutlass_gemm_caller<Cutlass3xGemmM128>(
375
- out, a, b, std::forward<EpilogueArgs>(args)...);
376
- } else {
377
- // m in (128, inf)
378
- return cutlass_gemm_caller<Cutlass3xGemmDefault>(
379
- out, a, b, std::forward<EpilogueArgs>(args)...);
380
- }
381
-}
382
-
16
template <template <typename, typename, typename> typename Epilogue,
17
typename... EpilogueArgs>
18
void cutlass_scaled_mm_sm90_epilogue(torch::Tensor& out, torch::Tensor const& a,
csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cuh
new
+160
@@ -0,0 +1,160 @@
1
+#pragma once
2
+
3
+// clang-format will break include orders
4
+// clang-format off
5
+#include <torch/all.h>
6
+
7
+#include <ATen/cuda/CUDAContext.h>
8
+
9
+#include "cutlass/cutlass.h"
10
+
11
+#include "cute/tensor.hpp"
12
+#include "cute/atom/mma_atom.hpp"
13
+#include "cutlass/numeric_types.h"
14
+
15
+#include "cutlass/gemm/device/gemm_universal_adapter.h"
16
+#include "cutlass/gemm/kernel/gemm_universal.hpp"
17
+#include "cutlass/epilogue/collective/collective_builder.hpp"
18
+#include "cutlass/gemm/collective/collective_builder.hpp"
19
+
20
+#include "core/math.hpp"
21
+#include "cutlass_extensions/common.hpp"
22
+// clang-format on
23
+
24
+/*
25
+ Epilogues defined in,
26
+ csrc/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp,
27
+ must contain a public type named EVTCompute of type Sm90EVT, as well as a
28
+ static prepare_args function that constructs an EVTCompute::Arguments struct.
29
+*/
30
+
31
+using namespace cute;
32
+
33
+namespace vllm {
34
+
35
+// A wrapper for the GEMM kernel that is used to guard against compilation on
36
+// architectures that will never use the kernel. The purpose of this is to
37
+// reduce the size of the compiled binary.
38
+// __CUDA_ARCH__ is not defined in host code, so this lets us smuggle the ifdef
39
+// into code that will be executed on the device where it is defined.
40
+template <typename Kernel>
41
+struct enable_sm90_or_later : Kernel {
42
+ template <typename... Args>
43
+ CUTLASS_DEVICE void operator()(Args&&... args) {
44
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 900
45
+ Kernel::operator()(std::forward<Args>(args)...);
46
+#endif
47
+ }
48
+};
49
+
50
+template <typename ElementAB_, typename ElementD_,
51
+ template <typename, typename, typename> typename Epilogue_,
52
+ typename TileShape, typename ClusterShape, typename KernelSchedule,
53
+ typename EpilogueSchedule>
54
+struct cutlass_3x_gemm {
55
+ using ElementAB = ElementAB_;
56
+ using ElementD = ElementD_;
57
+ using ElementAcc =
58
+ typename std::conditional<std::is_same_v<ElementAB, int8_t>, int32_t,
59
+ float>::type;
60
+
61
+ using EpilogueDescriptor =
62
+ cutlass::epilogue::collective::detail::EpilogueDescriptor<
63
+ TileShape, cutlass::epilogue::collective::EpilogueTileAuto, ElementD,
64
+ ElementD, EpilogueSchedule>;
65
+
66
+ using Epilogue = Epilogue_<ElementAcc, ElementD, EpilogueDescriptor>;
67
+
68
+ using StrideD = Stride<int64_t, Int<1>, Int<0>>;
69
+ using ElementC = void;
70
+ using StrideC = StrideD;
71
+
72
+ using EVTCompute = typename Epilogue::EVTCompute;
73
+
74
+ using CollectiveEpilogue =
75
+ typename cutlass::epilogue::collective::CollectiveBuilder<
76
+ cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp, TileShape,
77
+ ClusterShape, cutlass::epilogue::collective::EpilogueTileAuto,
78
+ ElementAcc, float, ElementC, StrideC, 4, ElementD, StrideD, 4,
79
+ EpilogueSchedule, EVTCompute>::CollectiveOp;
80
+
81
+ static constexpr size_t CEStorageSize =
82
+ sizeof(typename CollectiveEpilogue::SharedStorage);
83
+ using Stages = typename cutlass::gemm::collective::StageCountAutoCarveout<
84
+ static_cast<int>(CEStorageSize)>;
85
+
86
+ // clang-format off
87
+ using CollectiveMainloop =
88
+ typename cutlass::gemm::collective::CollectiveBuilder<
89
+ cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
90
+ ElementAB, cutlass::layout::RowMajor, 16,
91
+ ElementAB, cutlass::layout::ColumnMajor, 16,
92
+ ElementAcc, TileShape, ClusterShape,
93
+ Stages,
94
+ KernelSchedule>::CollectiveOp;
95
+ // clang-format on
96
+
97
+ using KernelType = enable_sm90_or_later<cutlass::gemm::kernel::GemmUniversal<
98
+ cute::Shape<int, int, int, int>, CollectiveMainloop, CollectiveEpilogue,
99
+ cutlass::gemm::PersistentScheduler>>;
100
+
101
+ struct GemmKernel : public KernelType {};
102
+};
103
+
104
+template <typename Gemm, typename... EpilogueArgs>
105
+void cutlass_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
106
+ torch::Tensor const& b,
107
+ EpilogueArgs&&... epilogue_params) {
108
+ using ElementAB = typename Gemm::ElementAB;
109
+ using ElementD = typename Gemm::ElementD;
110
+
111
+ int32_t m = a.size(0);
112
+ int32_t n = b.size(1);
113
+ int32_t k = a.size(1);
114
+
115
+ int64_t lda = a.stride(0);
116
+ int64_t ldb = b.stride(1);
117
+ int64_t ldc = out.stride(0);
118
+
119
+ using StrideA = Stride<int64_t, Int<1>, int64_t>;
120
+ using StrideB = Stride<int64_t, Int<1>, int64_t>;
121
+ using StrideC = typename Gemm::StrideC;
122
+
123
+ StrideA a_stride{lda, Int<1>{}, 0};
124
+ StrideB b_stride{ldb, Int<1>{}, 0};
125
+ StrideC c_stride{ldc, Int<1>{}, Int<0>{}};
126
+
127
+ using GemmKernel = typename Gemm::GemmKernel;
128
+ typename GemmKernel::ProblemShape prob_shape{m, n, k, 1};
129
+
130
+ auto a_ptr = static_cast<ElementAB*>(a.data_ptr());
131
+ auto b_ptr = static_cast<ElementAB*>(b.data_ptr());
132
+ typename GemmKernel::MainloopArguments mainloop_args{a_ptr, a_stride, b_ptr,
133
+ b_stride};
134
+
135
+ auto c_ptr = static_cast<ElementD*>(out.data_ptr());
136
+ typename GemmKernel::EpilogueArguments epilogue_args{
137
+ Gemm::Epilogue::prepare_args(
138
+ std::forward<EpilogueArgs>(epilogue_params)...),
139
+ c_ptr, c_stride, c_ptr, c_stride};
140
+
141
+ typename GemmKernel::Arguments args{cutlass::gemm::GemmUniversalMode::kGemm,
142
+ prob_shape, mainloop_args, epilogue_args};
143
+
144
+ // Launch the CUTLASS GEMM kernel.
145
+ using GemmOp = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
146
+ GemmOp gemm_op;
147
+ CUTLASS_CHECK(gemm_op.can_implement(args));
148
+
149
+ size_t workspace_size = gemm_op.get_workspace_size(args);
150
+ auto const workspace_options =
151
+ torch::TensorOptions().dtype(torch::kUInt8).device(a.device());
152
+ auto workspace = torch::empty(workspace_size, workspace_options);
153
+
154
+ auto stream = at::cuda::getCurrentCUDAStream(a.get_device());
155
+
156
+ cutlass::Status status = gemm_op.run(args, workspace.data_ptr(), stream);
157
+ CUTLASS_CHECK(status);
158
+}
159
+
160
+} // namespace vllm
csrc/quantization/cutlass_w8a8/scaled_mm_c3x_sm90_fp8_dispatch.cuh
new
+96
@@ -0,0 +1,96 @@
1
+#pragma once
2
+
3
+#include "scaled_mm_c3x.cuh"
4
+
5
+/**
6
+ * This file defines Gemm kernel configurations for SM90 (fp8) based on the Gemm
7
+ * shape.
8
+ */
9
+
10
+namespace vllm {
11
+
12
+template <typename InType, typename OutType,
13
+ template <typename, typename, typename> typename Epilogue>
14
+struct sm90_fp8_config_default {
15
+ // M in (128, inf)
16
+ static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
17
+ using KernelSchedule =
18
+ cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum;
19
+ using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
20
+ using TileShape = Shape<_128, _128, _128>;
21
+ using ClusterShape = Shape<_2, _1, _1>;
22
+ using Cutlass3xGemm =
23
+ cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
24
+ KernelSchedule, EpilogueSchedule>;
25
+};
26
+
27
+template <typename InType, typename OutType,
28
+ template <typename, typename, typename> typename Epilogue>
29
+struct sm90_fp8_config_M128 {
30
+ // M in (64, 128]
31
+ static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
32
+ using KernelSchedule =
33
+ cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum;
34
+ using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
35
+ using TileShape = Shape<_64, _128, _128>;
36
+ using ClusterShape = Shape<_2, _1, _1>;
37
+ using Cutlass3xGemm =
38
+ cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
39
+ KernelSchedule, EpilogueSchedule>;
40
+};
41
+
42
+template <typename InType, typename OutType,
43
+ template <typename, typename, typename> typename Epilogue>
44
+struct sm90_fp8_config_M64 {
45
+ // M in [1, 64]
46
+ static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
47
+ using KernelSchedule =
48
+ cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum;
49
+ using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
50
+ using TileShape = Shape<_64, _64, _128>;
51
+ using ClusterShape = Shape<_1, _8, _1>;
52
+
53
+ using Cutlass3xGemm =
54
+ cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
55
+ KernelSchedule, EpilogueSchedule>;
56
+};
57
+
58
+template <typename InType, typename OutType,
59
+ template <typename, typename, typename> typename Epilogue,
60
+ typename... EpilogueArgs>
61
+inline void cutlass_gemm_sm90_fp8_dispatch(torch::Tensor& out,
62
+ torch::Tensor const& a,
63
+ torch::Tensor const& b,
64
+ EpilogueArgs&&... args) {
65
+ static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
66
+ TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
67
+ TORCH_CHECK(b.dtype() == torch::kFloat8_e4m3fn);
68
+
69
+ using Cutlass3xGemmDefault =
70
+ typename sm90_fp8_config_default<InType, OutType,
71
+ Epilogue>::Cutlass3xGemm;
72
+ using Cutlass3xGemmM64 =
73
+ typename sm90_fp8_config_M64<InType, OutType, Epilogue>::Cutlass3xGemm;
74
+ using Cutlass3xGemmM128 =
75
+ typename sm90_fp8_config_M128<InType, OutType, Epilogue>::Cutlass3xGemm;
76
+
77
+ uint32_t const m = a.size(0);
78
+ uint32_t const mp2 =
79
+ std::max(static_cast<uint32_t>(64), next_pow_2(m)); // next power of 2
80
+
81
+ if (mp2 <= 64) {
82
+ // m in [1, 64]
83
+ return cutlass_gemm_caller<Cutlass3xGemmM64>(
84
+ out, a, b, std::forward<EpilogueArgs>(args)...);
85
+ } else if (mp2 <= 128) {
86
+ // m in (64, 128]
87
+ return cutlass_gemm_caller<Cutlass3xGemmM128>(
88
+ out, a, b, std::forward<EpilogueArgs>(args)...);
89
+ } else {
90
+ // m in (128, inf)
91
+ return cutlass_gemm_caller<Cutlass3xGemmDefault>(
92
+ out, a, b, std::forward<EpilogueArgs>(args)...);
93
+ }
94
+}
95
+
96
+} // namespace vllm
\ No newline at end of file
csrc/quantization/cutlass_w8a8/scaled_mm_c3x_sm90_int8_dispatch.cuh
new
+140
@@ -0,0 +1,140 @@
1
+#pragma once
2
+
3
+#include "scaled_mm_c3x.cuh"
4
+
5
+/**
6
+ * This file defines Gemm kernel configurations for SM90 (int8) based on the
7
+ * Gemm shape.
8
+ */
9
+
10
+namespace vllm {
11
+
12
+template <typename InType, typename OutType,
13
+ template <typename, typename, typename> typename Epilogue>
14
+struct sm90_int8_config_default {
15
+ // For M > 128 and any N
16
+ static_assert(std::is_same<InType, int8_t>());
17
+ using KernelSchedule =
18
+ typename cutlass::gemm::KernelTmaWarpSpecializedPingpong;
19
+ using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
20
+ using TileShape = Shape<_128, _128, _128>;
21
+ using ClusterShape = Shape<_2, _1, _1>;
22
+ using Cutlass3xGemm =
23
+ cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
24
+ KernelSchedule, EpilogueSchedule>;
25
+};
26
+
27
+template <typename InType, typename OutType,
28
+ template <typename, typename, typename> typename Epilogue>
29
+struct sm90_int8_config_M128 {
30
+ // For M in (64, 128] and any N
31
+ static_assert(std::is_same<InType, int8_t>());
32
+ using KernelSchedule =
33
+ typename cutlass::gemm::KernelTmaWarpSpecializedPingpong;
34
+ using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
35
+ using TileShape = Shape<_64, _128, _128>;
36
+ using ClusterShape = Shape<_2, _1, _1>;
37
+ using Cutlass3xGemm =
38
+ cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
39
+ KernelSchedule, EpilogueSchedule>;
40
+};
41
+
42
+template <typename InType, typename OutType,
43
+ template <typename, typename, typename> typename Epilogue>
44
+struct sm90_int8_config_M64 {
45
+ // For M in (32, 64] and any N
46
+ static_assert(std::is_same<InType, int8_t>());
47
+ using KernelSchedule = typename cutlass::gemm::KernelTmaWarpSpecialized;
48
+ using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
49
+ using TileShape = Shape<_64, _64, _256>;
50
+ using ClusterShape = Shape<_1, _1, _1>;
51
+ using Cutlass3xGemm =
52
+ cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
53
+ KernelSchedule, EpilogueSchedule>;
54
+};
55
+
56
+template <typename InType, typename OutType,
57
+ template <typename, typename, typename> typename Epilogue>
58
+struct sm90_int8_config_M32_NBig {
59
+ // For M in [1, 32] and N >= 8192
60
+ static_assert(std::is_same<InType, int8_t>());
61
+ using KernelSchedule = typename cutlass::gemm::KernelTmaWarpSpecialized;
62
+ using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
63
+ using TileShape = Shape<_64, _128, _256>;
64
+ using ClusterShape = Shape<_1, _4, _1>;
65
+ using Cutlass3xGemm =
66
+ cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
67
+ KernelSchedule, EpilogueSchedule>;
68
+};
69
+
70
+template <typename InType, typename OutType,
71
+ template <typename, typename, typename> typename Epilogue>
72
+struct sm90_int8_config_M32_NSmall {
73
+ // For M in [1, 32] and N < 8192
74
+ static_assert(std::is_same<InType, int8_t>());
75
+ using KernelSchedule = typename cutlass::gemm::KernelTmaWarpSpecialized;
76
+ using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
77
+ using TileShape = Shape<_64, _64, _256>;
78
+ using ClusterShape = Shape<_1, _8, _1>;
79
+ using Cutlass3xGemm =
80
+ cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
81
+ KernelSchedule, EpilogueSchedule>;
82
+};
83
+
84
+template <typename InType, typename OutType,
85
+ template <typename, typename, typename> typename Epilogue,
86
+ typename... EpilogueArgs>
87
+inline void cutlass_gemm_sm90_int8_dispatch(torch::Tensor& out,
88
+ torch::Tensor const& a,
89
+ torch::Tensor const& b,
90
+ EpilogueArgs&&... args) {
91
+ static_assert(std::is_same<InType, int8_t>());
92
+ TORCH_CHECK(a.dtype() == torch::kInt8);
93
+ TORCH_CHECK(b.dtype() == torch::kInt8);
94
+
95
+ using Cutlass3xGemmDefault =
96
+ typename sm90_int8_config_default<InType, OutType,
97
+ Epilogue>::Cutlass3xGemm;
98
+ using Cutlass3xGemmM128 =
99
+ typename sm90_int8_config_M128<InType, OutType, Epilogue>::Cutlass3xGemm;
100
+ using Cutlass3xGemmM64 =
101
+ typename sm90_int8_config_M64<InType, OutType, Epilogue>::Cutlass3xGemm;
102
+ using Cutlass3xGemmM32NBig =
103
+ typename sm90_int8_config_M32_NBig<InType, OutType,
104
+ Epilogue>::Cutlass3xGemm;
105
+ using Cutlass3xGemmM32NSmall =
106
+ typename sm90_int8_config_M32_NSmall<InType, OutType,
107
+ Epilogue>::Cutlass3xGemm;
108
+
109
+ uint32_t const n = out.size(1);
110
+ bool const is_small_n = n < 8192;
111
+
112
+ uint32_t const m = a.size(0);
113
+ uint32_t const mp2 =
114
+ std::max(static_cast<uint32_t>(32), next_pow_2(m)); // next power of 2
115
+
116
+ if (mp2 <= 32) {
117
+ // m in [1, 32]
118
+ if (is_small_n) {
119
+ return cutlass_gemm_caller<Cutlass3xGemmM32NSmall>(
120
+ out, a, b, std::forward<EpilogueArgs>(args)...);
121
+ } else {
122
+ return cutlass_gemm_caller<Cutlass3xGemmM32NBig>(
123
+ out, a, b, std::forward<EpilogueArgs>(args)...);
124
+ }
125
+ } else if (mp2 <= 64) {
126
+ // m in (32, 64]
127
+ return cutlass_gemm_caller<Cutlass3xGemmM64>(
128
+ out, a, b, std::forward<EpilogueArgs>(args)...);
129
+ } else if (mp2 <= 128) {
130
+ // m in (64, 128]
131
+ return cutlass_gemm_caller<Cutlass3xGemmM128>(
132
+ out, a, b, std::forward<EpilogueArgs>(args)...);
133
+ } else {
134
+ // m in (128, inf)
135
+ return cutlass_gemm_caller<Cutlass3xGemmDefault>(
136
+ out, a, b, std::forward<EpilogueArgs>(args)...);
137
+ }
138
+}
139
+
140
+} // namespace vllm
\ No newline at end of file