@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8936316d58
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Kernel] Refactor Cutlass c3x (#10049)

Signed-off-by: Varun Sundar Rabindranath <varun@neuralmagic.com> Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>

Varun Sundar Rabindranath committed Dec 19, 2024 at 02:00 UTC 8936316d587ca0afb5ef058584c407d404c0ffb0
7 files changed +406 -373
csrc/cutlass_extensions/epilogue/scaled_mm_epilogues_c2x.hpp
+2
@@ -1,3 +1,5 @@
1 +#pragma once
2 +
3 #include "cutlass_extensions/epilogue/broadcast_load_epilogue_c2x.hpp"
4
5 /*
csrc/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp
+2
@@ -1,3 +1,5 @@
1 +#pragma once
2 +
3 #include "cutlass_extensions/epilogue/broadcast_load_epilogue_c3x.hpp"
4
5 /*
csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cuh
+3 -3
@@ -28,9 +28,9 @@
28 using namespace cute;
29
30 /*
31 - Epilogue functions can be defined to post-process the output before it is
32 - written to GPU memory.
33 - Epilogues must contain a public type named EVTCompute of type Sm80EVT,
31 + Epilogues defined in,
32 + csrc/cutlass_extensions/epilogue/scaled_mm_epilogues_c2x.hpp
33 + must contain a public type named EVTCompute of type Sm80EVT,
34 as well as a static prepare_args function that constructs an
35 EVTCompute::Arguments struct.
36 */
csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu
+3 -370
@@ -1,385 +1,18 @@
1 -// clang-format will break include orders
2 -// clang-format off
1 #include <cudaTypedefs.h>
2
3 #if defined CUDA_VERSION && CUDA_VERSION >= 12000
4
7 -#include <torch/all.h>
5 + #include "scaled_mm_c3x_sm90_fp8_dispatch.cuh"
6 + #include "scaled_mm_c3x_sm90_int8_dispatch.cuh"
7
9 -#include <ATen/cuda/CUDAContext.h>
10 -
11 -#include <iostream>
12 -#include <sstream>
13 -#include <vector>
14 -
15 -#include "cutlass/cutlass.h"
16 -
17 -#include "cute/tensor.hpp"
18 -#include "cute/atom/mma_atom.hpp"
19 -#include "cutlass/numeric_types.h"
20 -
21 -#include "cutlass/gemm/device/gemm_universal_adapter.h"
22 -#include "cutlass/gemm/kernel/gemm_universal.hpp"
23 -#include "cutlass/epilogue/collective/collective_builder.hpp"
24 -#include "cutlass/gemm/collective/collective_builder.hpp"
25 -
26 -#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
27 -#include "core/math.hpp"
28 -#include "cutlass_extensions/common.hpp"
29 -// clang-format on
30 -
31 -using namespace cute;
8 + #include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
9 using namespace vllm;
10
11 /*
12 This file defines quantized GEMM operations using the CUTLASS 3.x API, for
13 NVIDIA GPUs with sm90a (Hopper) or later.
37 -
38 - Epilogue functions can be defined to post-process the output before it is
39 - written to GPU memory.
40 - Epilogues must contain a public type named EVTCompute of type Sm90EVT,
41 - as well as a static prepare_args function that constructs an
42 - EVTCompute::Arguments struct.
14 */
15
45 -namespace {
46 -
47 -// A wrapper for the GEMM kernel that is used to guard against compilation on
48 -// architectures that will never use the kernel. The purpose of this is to
49 -// reduce the size of the compiled binary.
50 -// __CUDA_ARCH__ is not defined in host code, so this lets us smuggle the ifdef
51 -// into code that will be executed on the device where it is defined.
52 -template <typename Kernel>
53 -struct enable_sm90_or_later : Kernel {
54 - template <typename... Args>
55 - CUTLASS_DEVICE void operator()(Args&&... args) {
56 - #if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 900
57 - Kernel::operator()(std::forward<Args>(args)...);
58 - #endif
59 - }
60 -};
61 -template <typename ElementAB_, typename ElementD_,
62 - template <typename, typename, typename> typename Epilogue_,
63 - typename TileShape, typename ClusterShape, typename KernelSchedule,
64 - typename EpilogueSchedule>
65 -struct cutlass_3x_gemm {
66 - using ElementAB = ElementAB_;
67 - using ElementD = ElementD_;
68 - using ElementAcc =
69 - typename std::conditional<std::is_same_v<ElementAB, int8_t>, int32_t,
70 - float>::type;
71 -
72 - using EpilogueDescriptor =
73 - cutlass::epilogue::collective::detail::EpilogueDescriptor<
74 - TileShape, cutlass::epilogue::collective::EpilogueTileAuto, ElementD,
75 - ElementD, EpilogueSchedule>;
76 -
77 - using Epilogue = Epilogue_<ElementAcc, ElementD, EpilogueDescriptor>;
78 -
79 - using StrideD = Stride<int64_t, Int<1>, Int<0>>;
80 - using ElementC = void;
81 - using StrideC = StrideD;
82 -
83 - using EVTCompute = typename Epilogue::EVTCompute;
84 -
85 - using CollectiveEpilogue =
86 - typename cutlass::epilogue::collective::CollectiveBuilder<
87 - cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp, TileShape,
88 - ClusterShape, cutlass::epilogue::collective::EpilogueTileAuto,
89 - ElementAcc, float, ElementC, StrideC, 4, ElementD, StrideD, 4,
90 - EpilogueSchedule, EVTCompute>::CollectiveOp;
91 -
92 - static constexpr size_t CEStorageSize =
93 - sizeof(typename CollectiveEpilogue::SharedStorage);
94 - using Stages = typename cutlass::gemm::collective::StageCountAutoCarveout<
95 - static_cast<int>(CEStorageSize)>;
96 -
97 - // clang-format off
98 - using CollectiveMainloop =
99 - typename cutlass::gemm::collective::CollectiveBuilder<
100 - cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
101 - ElementAB, cutlass::layout::RowMajor, 16,
102 - ElementAB, cutlass::layout::ColumnMajor, 16,
103 - ElementAcc, TileShape, ClusterShape,
104 - Stages,
105 - KernelSchedule>::CollectiveOp;
106 - // clang-format on
107 -
108 - using KernelType = enable_sm90_or_later<cutlass::gemm::kernel::GemmUniversal<
109 - cute::Shape<int, int, int, int>, CollectiveMainloop, CollectiveEpilogue,
110 - cutlass::gemm::PersistentScheduler>>;
111 -
112 - struct GemmKernel : public KernelType {};
113 -};
114 -
115 -template <typename Gemm, typename... EpilogueArgs>
116 -void cutlass_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
117 - torch::Tensor const& b,
118 - EpilogueArgs&&... epilogue_params) {
119 - using ElementAB = typename Gemm::ElementAB;
120 - using ElementD = typename Gemm::ElementD;
121 -
122 - int32_t m = a.size(0);
123 - int32_t n = b.size(1);
124 - int32_t k = a.size(1);
125 -
126 - int64_t lda = a.stride(0);
127 - int64_t ldb = b.stride(1);
128 - int64_t ldc = out.stride(0);
129 -
130 - using StrideA = Stride<int64_t, Int<1>, int64_t>;
131 - using StrideB = Stride<int64_t, Int<1>, int64_t>;
132 - using StrideC = typename Gemm::StrideC;
133 -
134 - StrideA a_stride{lda, Int<1>{}, 0};
135 - StrideB b_stride{ldb, Int<1>{}, 0};
136 - StrideC c_stride{ldc, Int<1>{}, Int<0>{}};
137 -
138 - using GemmKernel = typename Gemm::GemmKernel;
139 - typename GemmKernel::ProblemShape prob_shape{m, n, k, 1};
140 -
141 - auto a_ptr = static_cast<ElementAB*>(a.data_ptr());
142 - auto b_ptr = static_cast<ElementAB*>(b.data_ptr());
143 - typename GemmKernel::MainloopArguments mainloop_args{a_ptr, a_stride, b_ptr,
144 - b_stride};
145 -
146 - auto c_ptr = static_cast<ElementD*>(out.data_ptr());
147 - typename GemmKernel::EpilogueArguments epilogue_args{
148 - Gemm::Epilogue::prepare_args(
149 - std::forward<EpilogueArgs>(epilogue_params)...),
150 - c_ptr, c_stride, c_ptr, c_stride};
151 -
152 - typename GemmKernel::Arguments args{cutlass::gemm::GemmUniversalMode::kGemm,
153 - prob_shape, mainloop_args, epilogue_args};
154 -
155 - // Launch the CUTLASS GEMM kernel.
156 - using GemmOp = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
157 - GemmOp gemm_op;
158 - CUTLASS_CHECK(gemm_op.can_implement(args));
159 -
160 - size_t workspace_size = gemm_op.get_workspace_size(args);
161 - auto const workspace_options =
162 - torch::TensorOptions().dtype(torch::kUInt8).device(a.device());
163 - auto workspace = torch::empty(workspace_size, workspace_options);
164 -
165 - auto stream = at::cuda::getCurrentCUDAStream(a.get_device());
166 -
167 - cutlass::Status status = gemm_op.run(args, workspace.data_ptr(), stream);
168 - CUTLASS_CHECK(status);
169 -}
170 -
171 -template <typename InType, typename OutType,
172 - template <typename, typename, typename> typename Epilogue>
173 -struct sm90_fp8_config_default {
174 - // M in (128, inf)
175 - static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
176 - using KernelSchedule =
177 - cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum;
178 - using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
179 - using TileShape = Shape<_128, _128, _128>;
180 - using ClusterShape = Shape<_2, _1, _1>;
181 - using Cutlass3xGemm =
182 - cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
183 - KernelSchedule, EpilogueSchedule>;
184 -};
185 -
186 -template <typename InType, typename OutType,
187 - template <typename, typename, typename> typename Epilogue>
188 -struct sm90_fp8_config_M128 {
189 - // M in (64, 128]
190 - static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
191 - using KernelSchedule =
192 - cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum;
193 - using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
194 - using TileShape = Shape<_64, _128, _128>;
195 - using ClusterShape = Shape<_2, _1, _1>;
196 - using Cutlass3xGemm =
197 - cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
198 - KernelSchedule, EpilogueSchedule>;
199 -};
200 -
201 -template <typename InType, typename OutType,
202 - template <typename, typename, typename> typename Epilogue>
203 -struct sm90_fp8_config_M64 {
204 - // M in [1, 64]
205 - static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
206 - using KernelSchedule =
207 - cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum;
208 - using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
209 - using TileShape = Shape<_64, _64, _128>;
210 - using ClusterShape = Shape<_1, _8, _1>;
211 -
212 - using Cutlass3xGemm =
213 - cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
214 - KernelSchedule, EpilogueSchedule>;
215 -};
216 -
217 -template <typename InType, typename OutType,
218 - template <typename, typename, typename> typename Epilogue>
219 -struct sm90_int8_config_default {
220 - // For M > 128 and any N
221 - static_assert(std::is_same<InType, int8_t>());
222 - using KernelSchedule =
223 - typename cutlass::gemm::KernelTmaWarpSpecializedPingpong;
224 - using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
225 - using TileShape = Shape<_128, _128, _128>;
226 - using ClusterShape = Shape<_2, _1, _1>;
227 - using Cutlass3xGemm =
228 - cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
229 - KernelSchedule, EpilogueSchedule>;
230 -};
231 -
232 -template <typename InType, typename OutType,
233 - template <typename, typename, typename> typename Epilogue>
234 -struct sm90_int8_config_M128 {
235 - // For M in (64, 128] and any N
236 - static_assert(std::is_same<InType, int8_t>());
237 - using KernelSchedule =
238 - typename cutlass::gemm::KernelTmaWarpSpecializedPingpong;
239 - using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
240 - using TileShape = Shape<_64, _128, _128>;
241 - using ClusterShape = Shape<_2, _1, _1>;
242 - using Cutlass3xGemm =
243 - cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
244 - KernelSchedule, EpilogueSchedule>;
245 -};
246 -
247 -template <typename InType, typename OutType,
248 - template <typename, typename, typename> typename Epilogue>
249 -struct sm90_int8_config_M64 {
250 - // For M in (32, 64] and any N
251 - static_assert(std::is_same<InType, int8_t>());
252 - using KernelSchedule = typename cutlass::gemm::KernelTmaWarpSpecialized;
253 - using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
254 - using TileShape = Shape<_64, _64, _256>;
255 - using ClusterShape = Shape<_1, _1, _1>;
256 - using Cutlass3xGemm =
257 - cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
258 - KernelSchedule, EpilogueSchedule>;
259 -};
260 -
261 -template <typename InType, typename OutType,
262 - template <typename, typename, typename> typename Epilogue>
263 -struct sm90_int8_config_M32_NBig {
264 - // For M in [1, 32] and N >= 8192
265 - static_assert(std::is_same<InType, int8_t>());
266 - using KernelSchedule = typename cutlass::gemm::KernelTmaWarpSpecialized;
267 - using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
268 - using TileShape = Shape<_64, _128, _256>;
269 - using ClusterShape = Shape<_1, _4, _1>;
270 - using Cutlass3xGemm =
271 - cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
272 - KernelSchedule, EpilogueSchedule>;
273 -};
274 -
275 -template <typename InType, typename OutType,
276 - template <typename, typename, typename> typename Epilogue>
277 -struct sm90_int8_config_M32_NSmall {
278 - // For M in [1, 32] and N < 8192
279 - static_assert(std::is_same<InType, int8_t>());
280 - using KernelSchedule = typename cutlass::gemm::KernelTmaWarpSpecialized;
281 - using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
282 - using TileShape = Shape<_64, _64, _256>;
283 - using ClusterShape = Shape<_1, _8, _1>;
284 - using Cutlass3xGemm =
285 - cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
286 - KernelSchedule, EpilogueSchedule>;
287 -};
288 -
289 -} // namespace
290 -
291 -template <typename InType, typename OutType,
292 - template <typename, typename, typename> typename Epilogue,
293 - typename... EpilogueArgs>
294 -void cutlass_gemm_sm90_fp8_dispatch(torch::Tensor& out, torch::Tensor const& a,
295 - torch::Tensor const& b,
296 - EpilogueArgs&&... args) {
297 - static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
298 - TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
299 - TORCH_CHECK(b.dtype() == torch::kFloat8_e4m3fn);
300 -
301 - using Cutlass3xGemmDefault =
302 - typename sm90_fp8_config_default<InType, OutType,
303 - Epilogue>::Cutlass3xGemm;
304 - using Cutlass3xGemmM64 =
305 - typename sm90_fp8_config_M64<InType, OutType, Epilogue>::Cutlass3xGemm;
306 - using Cutlass3xGemmM128 =
307 - typename sm90_fp8_config_M128<InType, OutType, Epilogue>::Cutlass3xGemm;
308 -
309 - uint32_t const m = a.size(0);
310 - uint32_t const mp2 =
311 - std::max(static_cast<uint32_t>(64), next_pow_2(m)); // next power of 2
312 -
313 - if (mp2 <= 64) {
314 - // m in [1, 64]
315 - return cutlass_gemm_caller<Cutlass3xGemmM64>(
316 - out, a, b, std::forward<EpilogueArgs>(args)...);
317 - } else if (mp2 <= 128) {
318 - // m in (64, 128]
319 - return cutlass_gemm_caller<Cutlass3xGemmM128>(
320 - out, a, b, std::forward<EpilogueArgs>(args)...);
321 - } else {
322 - // m in (128, inf)
323 - return cutlass_gemm_caller<Cutlass3xGemmDefault>(
324 - out, a, b, std::forward<EpilogueArgs>(args)...);
325 - }
326 -}
327 -
328 -template <typename InType, typename OutType,
329 - template <typename, typename, typename> typename Epilogue,
330 - typename... EpilogueArgs>
331 -void cutlass_gemm_sm90_int8_dispatch(torch::Tensor& out, torch::Tensor const& a,
332 - torch::Tensor const& b,
333 - EpilogueArgs&&... args) {
334 - static_assert(std::is_same<InType, int8_t>());
335 - TORCH_CHECK(a.dtype() == torch::kInt8);
336 - TORCH_CHECK(b.dtype() == torch::kInt8);
337 -
338 - using Cutlass3xGemmDefault =
339 - typename sm90_int8_config_default<InType, OutType,
340 - Epilogue>::Cutlass3xGemm;
341 - using Cutlass3xGemmM128 =
342 - typename sm90_int8_config_M128<InType, OutType, Epilogue>::Cutlass3xGemm;
343 - using Cutlass3xGemmM64 =
344 - typename sm90_int8_config_M64<InType, OutType, Epilogue>::Cutlass3xGemm;
345 - using Cutlass3xGemmM32NBig =
346 - typename sm90_int8_config_M32_NBig<InType, OutType,
347 - Epilogue>::Cutlass3xGemm;
348 - using Cutlass3xGemmM32NSmall =
349 - typename sm90_int8_config_M32_NSmall<InType, OutType,
350 - Epilogue>::Cutlass3xGemm;
351 -
352 - uint32_t const n = out.size(1);
353 - bool const is_small_n = n < 8192;
354 -
355 - uint32_t const m = a.size(0);
356 - uint32_t const mp2 =
357 - std::max(static_cast<uint32_t>(32), next_pow_2(m)); // next power of 2
358 -
359 - if (mp2 <= 32) {
360 - // m in [1, 32]
361 - if (is_small_n) {
362 - return cutlass_gemm_caller<Cutlass3xGemmM32NSmall>(
363 - out, a, b, std::forward<EpilogueArgs>(args)...);
364 - } else {
365 - return cutlass_gemm_caller<Cutlass3xGemmM32NBig>(
366 - out, a, b, std::forward<EpilogueArgs>(args)...);
367 - }
368 - } else if (mp2 <= 64) {
369 - // m in (32, 64]
370 - return cutlass_gemm_caller<Cutlass3xGemmM64>(
371 - out, a, b, std::forward<EpilogueArgs>(args)...);
372 - } else if (mp2 <= 128) {
373 - // m in (64, 128]
374 - return cutlass_gemm_caller<Cutlass3xGemmM128>(
375 - out, a, b, std::forward<EpilogueArgs>(args)...);
376 - } else {
377 - // m in (128, inf)
378 - return cutlass_gemm_caller<Cutlass3xGemmDefault>(
379 - out, a, b, std::forward<EpilogueArgs>(args)...);
380 - }
381 -}
382 -
16 template <template <typename, typename, typename> typename Epilogue,
17 typename... EpilogueArgs>
18 void cutlass_scaled_mm_sm90_epilogue(torch::Tensor& out, torch::Tensor const& a,
csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cuh new
+160
@@ -0,0 +1,160 @@
1 +#pragma once
2 +
3 +// clang-format will break include orders
4 +// clang-format off
5 +#include <torch/all.h>
6 +
7 +#include <ATen/cuda/CUDAContext.h>
8 +
9 +#include "cutlass/cutlass.h"
10 +
11 +#include "cute/tensor.hpp"
12 +#include "cute/atom/mma_atom.hpp"
13 +#include "cutlass/numeric_types.h"
14 +
15 +#include "cutlass/gemm/device/gemm_universal_adapter.h"
16 +#include "cutlass/gemm/kernel/gemm_universal.hpp"
17 +#include "cutlass/epilogue/collective/collective_builder.hpp"
18 +#include "cutlass/gemm/collective/collective_builder.hpp"
19 +
20 +#include "core/math.hpp"
21 +#include "cutlass_extensions/common.hpp"
22 +// clang-format on
23 +
24 +/*
25 + Epilogues defined in,
26 + csrc/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp,
27 + must contain a public type named EVTCompute of type Sm90EVT, as well as a
28 + static prepare_args function that constructs an EVTCompute::Arguments struct.
29 +*/
30 +
31 +using namespace cute;
32 +
33 +namespace vllm {
34 +
35 +// A wrapper for the GEMM kernel that is used to guard against compilation on
36 +// architectures that will never use the kernel. The purpose of this is to
37 +// reduce the size of the compiled binary.
38 +// __CUDA_ARCH__ is not defined in host code, so this lets us smuggle the ifdef
39 +// into code that will be executed on the device where it is defined.
40 +template <typename Kernel>
41 +struct enable_sm90_or_later : Kernel {
42 + template <typename... Args>
43 + CUTLASS_DEVICE void operator()(Args&&... args) {
44 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 900
45 + Kernel::operator()(std::forward<Args>(args)...);
46 +#endif
47 + }
48 +};
49 +
50 +template <typename ElementAB_, typename ElementD_,
51 + template <typename, typename, typename> typename Epilogue_,
52 + typename TileShape, typename ClusterShape, typename KernelSchedule,
53 + typename EpilogueSchedule>
54 +struct cutlass_3x_gemm {
55 + using ElementAB = ElementAB_;
56 + using ElementD = ElementD_;
57 + using ElementAcc =
58 + typename std::conditional<std::is_same_v<ElementAB, int8_t>, int32_t,
59 + float>::type;
60 +
61 + using EpilogueDescriptor =
62 + cutlass::epilogue::collective::detail::EpilogueDescriptor<
63 + TileShape, cutlass::epilogue::collective::EpilogueTileAuto, ElementD,
64 + ElementD, EpilogueSchedule>;
65 +
66 + using Epilogue = Epilogue_<ElementAcc, ElementD, EpilogueDescriptor>;
67 +
68 + using StrideD = Stride<int64_t, Int<1>, Int<0>>;
69 + using ElementC = void;
70 + using StrideC = StrideD;
71 +
72 + using EVTCompute = typename Epilogue::EVTCompute;
73 +
74 + using CollectiveEpilogue =
75 + typename cutlass::epilogue::collective::CollectiveBuilder<
76 + cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp, TileShape,
77 + ClusterShape, cutlass::epilogue::collective::EpilogueTileAuto,
78 + ElementAcc, float, ElementC, StrideC, 4, ElementD, StrideD, 4,
79 + EpilogueSchedule, EVTCompute>::CollectiveOp;
80 +
81 + static constexpr size_t CEStorageSize =
82 + sizeof(typename CollectiveEpilogue::SharedStorage);
83 + using Stages = typename cutlass::gemm::collective::StageCountAutoCarveout<
84 + static_cast<int>(CEStorageSize)>;
85 +
86 + // clang-format off
87 + using CollectiveMainloop =
88 + typename cutlass::gemm::collective::CollectiveBuilder<
89 + cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
90 + ElementAB, cutlass::layout::RowMajor, 16,
91 + ElementAB, cutlass::layout::ColumnMajor, 16,
92 + ElementAcc, TileShape, ClusterShape,
93 + Stages,
94 + KernelSchedule>::CollectiveOp;
95 + // clang-format on
96 +
97 + using KernelType = enable_sm90_or_later<cutlass::gemm::kernel::GemmUniversal<
98 + cute::Shape<int, int, int, int>, CollectiveMainloop, CollectiveEpilogue,
99 + cutlass::gemm::PersistentScheduler>>;
100 +
101 + struct GemmKernel : public KernelType {};
102 +};
103 +
104 +template <typename Gemm, typename... EpilogueArgs>
105 +void cutlass_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
106 + torch::Tensor const& b,
107 + EpilogueArgs&&... epilogue_params) {
108 + using ElementAB = typename Gemm::ElementAB;
109 + using ElementD = typename Gemm::ElementD;
110 +
111 + int32_t m = a.size(0);
112 + int32_t n = b.size(1);
113 + int32_t k = a.size(1);
114 +
115 + int64_t lda = a.stride(0);
116 + int64_t ldb = b.stride(1);
117 + int64_t ldc = out.stride(0);
118 +
119 + using StrideA = Stride<int64_t, Int<1>, int64_t>;
120 + using StrideB = Stride<int64_t, Int<1>, int64_t>;
121 + using StrideC = typename Gemm::StrideC;
122 +
123 + StrideA a_stride{lda, Int<1>{}, 0};
124 + StrideB b_stride{ldb, Int<1>{}, 0};
125 + StrideC c_stride{ldc, Int<1>{}, Int<0>{}};
126 +
127 + using GemmKernel = typename Gemm::GemmKernel;
128 + typename GemmKernel::ProblemShape prob_shape{m, n, k, 1};
129 +
130 + auto a_ptr = static_cast<ElementAB*>(a.data_ptr());
131 + auto b_ptr = static_cast<ElementAB*>(b.data_ptr());
132 + typename GemmKernel::MainloopArguments mainloop_args{a_ptr, a_stride, b_ptr,
133 + b_stride};
134 +
135 + auto c_ptr = static_cast<ElementD*>(out.data_ptr());
136 + typename GemmKernel::EpilogueArguments epilogue_args{
137 + Gemm::Epilogue::prepare_args(
138 + std::forward<EpilogueArgs>(epilogue_params)...),
139 + c_ptr, c_stride, c_ptr, c_stride};
140 +
141 + typename GemmKernel::Arguments args{cutlass::gemm::GemmUniversalMode::kGemm,
142 + prob_shape, mainloop_args, epilogue_args};
143 +
144 + // Launch the CUTLASS GEMM kernel.
145 + using GemmOp = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
146 + GemmOp gemm_op;
147 + CUTLASS_CHECK(gemm_op.can_implement(args));
148 +
149 + size_t workspace_size = gemm_op.get_workspace_size(args);
150 + auto const workspace_options =
151 + torch::TensorOptions().dtype(torch::kUInt8).device(a.device());
152 + auto workspace = torch::empty(workspace_size, workspace_options);
153 +
154 + auto stream = at::cuda::getCurrentCUDAStream(a.get_device());
155 +
156 + cutlass::Status status = gemm_op.run(args, workspace.data_ptr(), stream);
157 + CUTLASS_CHECK(status);
158 +}
159 +
160 +} // namespace vllm
csrc/quantization/cutlass_w8a8/scaled_mm_c3x_sm90_fp8_dispatch.cuh new
+96
@@ -0,0 +1,96 @@
1 +#pragma once
2 +
3 +#include "scaled_mm_c3x.cuh"
4 +
5 +/**
6 + * This file defines Gemm kernel configurations for SM90 (fp8) based on the Gemm
7 + * shape.
8 + */
9 +
10 +namespace vllm {
11 +
12 +template <typename InType, typename OutType,
13 + template <typename, typename, typename> typename Epilogue>
14 +struct sm90_fp8_config_default {
15 + // M in (128, inf)
16 + static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
17 + using KernelSchedule =
18 + cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum;
19 + using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
20 + using TileShape = Shape<_128, _128, _128>;
21 + using ClusterShape = Shape<_2, _1, _1>;
22 + using Cutlass3xGemm =
23 + cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
24 + KernelSchedule, EpilogueSchedule>;
25 +};
26 +
27 +template <typename InType, typename OutType,
28 + template <typename, typename, typename> typename Epilogue>
29 +struct sm90_fp8_config_M128 {
30 + // M in (64, 128]
31 + static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
32 + using KernelSchedule =
33 + cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum;
34 + using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
35 + using TileShape = Shape<_64, _128, _128>;
36 + using ClusterShape = Shape<_2, _1, _1>;
37 + using Cutlass3xGemm =
38 + cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
39 + KernelSchedule, EpilogueSchedule>;
40 +};
41 +
42 +template <typename InType, typename OutType,
43 + template <typename, typename, typename> typename Epilogue>
44 +struct sm90_fp8_config_M64 {
45 + // M in [1, 64]
46 + static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
47 + using KernelSchedule =
48 + cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum;
49 + using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
50 + using TileShape = Shape<_64, _64, _128>;
51 + using ClusterShape = Shape<_1, _8, _1>;
52 +
53 + using Cutlass3xGemm =
54 + cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
55 + KernelSchedule, EpilogueSchedule>;
56 +};
57 +
58 +template <typename InType, typename OutType,
59 + template <typename, typename, typename> typename Epilogue,
60 + typename... EpilogueArgs>
61 +inline void cutlass_gemm_sm90_fp8_dispatch(torch::Tensor& out,
62 + torch::Tensor const& a,
63 + torch::Tensor const& b,
64 + EpilogueArgs&&... args) {
65 + static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
66 + TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
67 + TORCH_CHECK(b.dtype() == torch::kFloat8_e4m3fn);
68 +
69 + using Cutlass3xGemmDefault =
70 + typename sm90_fp8_config_default<InType, OutType,
71 + Epilogue>::Cutlass3xGemm;
72 + using Cutlass3xGemmM64 =
73 + typename sm90_fp8_config_M64<InType, OutType, Epilogue>::Cutlass3xGemm;
74 + using Cutlass3xGemmM128 =
75 + typename sm90_fp8_config_M128<InType, OutType, Epilogue>::Cutlass3xGemm;
76 +
77 + uint32_t const m = a.size(0);
78 + uint32_t const mp2 =
79 + std::max(static_cast<uint32_t>(64), next_pow_2(m)); // next power of 2
80 +
81 + if (mp2 <= 64) {
82 + // m in [1, 64]
83 + return cutlass_gemm_caller<Cutlass3xGemmM64>(
84 + out, a, b, std::forward<EpilogueArgs>(args)...);
85 + } else if (mp2 <= 128) {
86 + // m in (64, 128]
87 + return cutlass_gemm_caller<Cutlass3xGemmM128>(
88 + out, a, b, std::forward<EpilogueArgs>(args)...);
89 + } else {
90 + // m in (128, inf)
91 + return cutlass_gemm_caller<Cutlass3xGemmDefault>(
92 + out, a, b, std::forward<EpilogueArgs>(args)...);
93 + }
94 +}
95 +
96 +} // namespace vllm
\ No newline at end of file
csrc/quantization/cutlass_w8a8/scaled_mm_c3x_sm90_int8_dispatch.cuh new
+140
@@ -0,0 +1,140 @@
1 +#pragma once
2 +
3 +#include "scaled_mm_c3x.cuh"
4 +
5 +/**
6 + * This file defines Gemm kernel configurations for SM90 (int8) based on the
7 + * Gemm shape.
8 + */
9 +
10 +namespace vllm {
11 +
12 +template <typename InType, typename OutType,
13 + template <typename, typename, typename> typename Epilogue>
14 +struct sm90_int8_config_default {
15 + // For M > 128 and any N
16 + static_assert(std::is_same<InType, int8_t>());
17 + using KernelSchedule =
18 + typename cutlass::gemm::KernelTmaWarpSpecializedPingpong;
19 + using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
20 + using TileShape = Shape<_128, _128, _128>;
21 + using ClusterShape = Shape<_2, _1, _1>;
22 + using Cutlass3xGemm =
23 + cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
24 + KernelSchedule, EpilogueSchedule>;
25 +};
26 +
27 +template <typename InType, typename OutType,
28 + template <typename, typename, typename> typename Epilogue>
29 +struct sm90_int8_config_M128 {
30 + // For M in (64, 128] and any N
31 + static_assert(std::is_same<InType, int8_t>());
32 + using KernelSchedule =
33 + typename cutlass::gemm::KernelTmaWarpSpecializedPingpong;
34 + using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
35 + using TileShape = Shape<_64, _128, _128>;
36 + using ClusterShape = Shape<_2, _1, _1>;
37 + using Cutlass3xGemm =
38 + cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
39 + KernelSchedule, EpilogueSchedule>;
40 +};
41 +
42 +template <typename InType, typename OutType,
43 + template <typename, typename, typename> typename Epilogue>
44 +struct sm90_int8_config_M64 {
45 + // For M in (32, 64] and any N
46 + static_assert(std::is_same<InType, int8_t>());
47 + using KernelSchedule = typename cutlass::gemm::KernelTmaWarpSpecialized;
48 + using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
49 + using TileShape = Shape<_64, _64, _256>;
50 + using ClusterShape = Shape<_1, _1, _1>;
51 + using Cutlass3xGemm =
52 + cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
53 + KernelSchedule, EpilogueSchedule>;
54 +};
55 +
56 +template <typename InType, typename OutType,
57 + template <typename, typename, typename> typename Epilogue>
58 +struct sm90_int8_config_M32_NBig {
59 + // For M in [1, 32] and N >= 8192
60 + static_assert(std::is_same<InType, int8_t>());
61 + using KernelSchedule = typename cutlass::gemm::KernelTmaWarpSpecialized;
62 + using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
63 + using TileShape = Shape<_64, _128, _256>;
64 + using ClusterShape = Shape<_1, _4, _1>;
65 + using Cutlass3xGemm =
66 + cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
67 + KernelSchedule, EpilogueSchedule>;
68 +};
69 +
70 +template <typename InType, typename OutType,
71 + template <typename, typename, typename> typename Epilogue>
72 +struct sm90_int8_config_M32_NSmall {
73 + // For M in [1, 32] and N < 8192
74 + static_assert(std::is_same<InType, int8_t>());
75 + using KernelSchedule = typename cutlass::gemm::KernelTmaWarpSpecialized;
76 + using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
77 + using TileShape = Shape<_64, _64, _256>;
78 + using ClusterShape = Shape<_1, _8, _1>;
79 + using Cutlass3xGemm =
80 + cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
81 + KernelSchedule, EpilogueSchedule>;
82 +};
83 +
84 +template <typename InType, typename OutType,
85 + template <typename, typename, typename> typename Epilogue,
86 + typename... EpilogueArgs>
87 +inline void cutlass_gemm_sm90_int8_dispatch(torch::Tensor& out,
88 + torch::Tensor const& a,
89 + torch::Tensor const& b,
90 + EpilogueArgs&&... args) {
91 + static_assert(std::is_same<InType, int8_t>());
92 + TORCH_CHECK(a.dtype() == torch::kInt8);
93 + TORCH_CHECK(b.dtype() == torch::kInt8);
94 +
95 + using Cutlass3xGemmDefault =
96 + typename sm90_int8_config_default<InType, OutType,
97 + Epilogue>::Cutlass3xGemm;
98 + using Cutlass3xGemmM128 =
99 + typename sm90_int8_config_M128<InType, OutType, Epilogue>::Cutlass3xGemm;
100 + using Cutlass3xGemmM64 =
101 + typename sm90_int8_config_M64<InType, OutType, Epilogue>::Cutlass3xGemm;
102 + using Cutlass3xGemmM32NBig =
103 + typename sm90_int8_config_M32_NBig<InType, OutType,
104 + Epilogue>::Cutlass3xGemm;
105 + using Cutlass3xGemmM32NSmall =
106 + typename sm90_int8_config_M32_NSmall<InType, OutType,
107 + Epilogue>::Cutlass3xGemm;
108 +
109 + uint32_t const n = out.size(1);
110 + bool const is_small_n = n < 8192;
111 +
112 + uint32_t const m = a.size(0);
113 + uint32_t const mp2 =
114 + std::max(static_cast<uint32_t>(32), next_pow_2(m)); // next power of 2
115 +
116 + if (mp2 <= 32) {
117 + // m in [1, 32]
118 + if (is_small_n) {
119 + return cutlass_gemm_caller<Cutlass3xGemmM32NSmall>(
120 + out, a, b, std::forward<EpilogueArgs>(args)...);
121 + } else {
122 + return cutlass_gemm_caller<Cutlass3xGemmM32NBig>(
123 + out, a, b, std::forward<EpilogueArgs>(args)...);
124 + }
125 + } else if (mp2 <= 64) {
126 + // m in (32, 64]
127 + return cutlass_gemm_caller<Cutlass3xGemmM64>(
128 + out, a, b, std::forward<EpilogueArgs>(args)...);
129 + } else if (mp2 <= 128) {
130 + // m in (64, 128]
131 + return cutlass_gemm_caller<Cutlass3xGemmM128>(
132 + out, a, b, std::forward<EpilogueArgs>(args)...);
133 + } else {
134 + // m in (128, inf)
135 + return cutlass_gemm_caller<Cutlass3xGemmDefault>(
136 + out, a, b, std::forward<EpilogueArgs>(args)...);
137 + }
138 +}
139 +
140 +} // namespace vllm
\ No newline at end of file