A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Kernel] Tuned FP8 Kernels for Ada Lovelace (#6677)
Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
Varun Sundar Rabindranath committed
Jul 29, 2024 at 11:42 UTC
766435e660a786933392eb8ef0a873bc38cf0c8b
6 files changed
+877
-490
benchmarks/cutlass_benchmarks/w8a8_benchmarks.py
+1
-1
@@ -13,7 +13,7 @@ from weight_shapes import WEIGHT_SHAPES
13
from vllm import _custom_ops as ops
14
from vllm.utils import FlexibleArgumentParser
15
16
-DEFAULT_MODELS = list(WEIGHT_SHAPES.keys())[1:]
16
+DEFAULT_MODELS = list(WEIGHT_SHAPES.keys())
17
DEFAULT_BATCH_SIZES = [1, 16, 32, 64, 128, 256, 512]
18
DEFAULT_TP_SIZES = [1]
19
csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cu
+33
-487
@@ -1,470 +1,16 @@
1
#include <stddef.h>
2
#include <torch/all.h>
3
-
4
-#include <ATen/cuda/CUDAContext.h>
5
-
6
-// clang-format will break include orders
7
-// clang-format off
8
-#include "cute/tensor.hpp"
9
-#include "cute/atom/mma_atom.hpp"
10
-#include "cutlass/numeric_types.h"
11
-
12
-#include "cutlass/util/device_memory.h"
13
-
3
#include "cutlass/cutlass.h"
15
-#include "cutlass/gemm_coord.h"
16
-#include "cutlass/arch/mma_sm75.h"
17
-#include "cutlass/arch/arch.h"
18
-#include "cutlass/arch/mma.h"
19
-#include "cutlass/gemm/device/gemm.h"
20
-#include "cutlass/gemm/device/gemm_universal_adapter.h"
4
22
-#include "cutlass/epilogue/threadblock/fusion/visitors.hpp"
23
-#include "cutlass/gemm/kernel/default_gemm_universal_with_visitor.h"
24
-
25
-#include "broadcast_load_epilogue_c2x.hpp"
26
-#include "common.hpp"
27
-// clang-format on
28
-
29
-using namespace cute;
5
+#include "scaled_mm_c2x.cuh"
6
+#include "scaled_mm_c2x_sm80_dispatch.cuh"
7
+#include "scaled_mm_c2x_sm89_dispatch.cuh"
8
9
/*
10
This file defines quantized GEMM operations using the CUTLASS 2.x API, for
11
NVIDIA GPUs with SM versions prior to sm90 (Hopper).
34
-
35
- Epilogue functions can be defined to post-process the output before it is
36
- written to GPU memory.
37
- Epilogues must contain a public type named EVTCompute of type Sm80EVT,
38
- as well as a static prepare_args function that constructs an
39
- EVTCompute::Arguments struct.
12
*/
13
42
-namespace {
43
-
44
-// Wrappers for the GEMM kernel that is used to guard against compilation on
45
-// architectures that will never use the kernel. The purpose of this is to
46
-// reduce the size of the compiled binary.
47
-// __CUDA_ARCH__ is not defined in host code, so this lets us smuggle the ifdef
48
-// into code that will be executed on the device where it is defined.
49
-template <typename Kernel>
50
-struct enable_sm75_to_sm80 : Kernel {
51
- template <typename... Args>
52
- CUTLASS_DEVICE static void invoke(Args&&... args) {
53
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 750 && __CUDA_ARCH__ < 800
54
- Kernel::invoke(std::forward<Args>(args)...);
55
-#endif
56
- }
57
-};
58
-
59
-template <typename Kernel>
60
-struct enable_sm80_to_sm89 : Kernel {
61
- template <typename... Args>
62
- CUTLASS_DEVICE static void invoke(Args&&... args) {
63
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 800 && __CUDA_ARCH__ < 890
64
- Kernel::invoke(std::forward<Args>(args)...);
65
-#endif
66
- }
67
-};
68
-
69
-template <typename Kernel>
70
-struct enable_sm89_to_sm90 : Kernel {
71
- template <typename... Args>
72
- CUTLASS_DEVICE static void invoke(Args&&... args) {
73
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 890 && __CUDA_ARCH__ < 900
74
- Kernel::invoke(std::forward<Args>(args)...);
75
-#endif
76
- }
77
-};
78
-
79
-/*
80
- * This class provides the common ScaleA and ScaleB descriptors for the
81
- * ScaledEpilogue and ScaledEpilogueBias classes.
82
- */
83
-template <typename ElementD, typename OutputTileThreadMap>
84
-struct ScaledEpilogueBase {
85
- protected:
86
- using Accum = cutlass::epilogue::threadblock::VisitorAccFetch;
87
-
88
- using ScaleA = cutlass::epilogue::threadblock::VisitorColOrScalarBroadcast<
89
- OutputTileThreadMap, float, Stride<Int<1>, Int<0>, Int<0>>>;
90
-
91
- using ScaleB = cutlass::epilogue::threadblock::VisitorRowOrScalarBroadcast<
92
- OutputTileThreadMap, float, Stride<Int<0>, Int<1>, Int<0>>>;
93
-};
94
-
95
-/*
96
- This epilogue function defines a quantized GEMM operation similar to
97
- torch._scaled_mm.
98
-
99
- A and B may be both either int8 or fp8_e4m3. A can be quantized per-tensor or
100
- per-row. B can be quantized per-tensor or per-column.
101
- Any combination of per-tensor and per-row or column is supported.
102
- A and B must have symmetric quantization (zero point == 0).
103
-
104
- So the GEMM operation is D = (a_scales * A) (b_scales * B), where the
105
- scales are applied elementwise with numpy-style broadcasting.
106
-
107
- ScaleA and ScaleB define the epilogue functions that apply the scales for
108
- the A and B operands respectively. These scales may be either per-tensor or
109
- per row or column.
110
-*/
111
-template <typename ElementD, typename OutputTileThreadMap>
112
-struct ScaledEpilogue
113
- : private ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
114
- private:
115
- using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
116
- using Accum = typename SUPER::Accum;
117
- using ScaleA = typename SUPER::ScaleA;
118
- using ScaleB = typename SUPER::ScaleB;
119
-
120
- using Compute0 = cutlass::epilogue::threadblock::VisitorCompute<
121
- cutlass::multiplies, float, float,
122
- cutlass::FloatRoundStyle::round_to_nearest>;
123
-
124
- using EVTCompute0 =
125
- cutlass::epilogue::threadblock::Sm80EVT<Compute0, ScaleB, Accum>;
126
-
127
- using Compute1 = cutlass::epilogue::threadblock::VisitorCompute<
128
- cutlass::multiplies, ElementD, float,
129
- cutlass::FloatRoundStyle::round_to_nearest>;
130
-
131
- public:
132
- using EVTCompute =
133
- cutlass::epilogue::threadblock::Sm80EVT<Compute1, ScaleA, EVTCompute0>;
134
- using ArgumentType = typename EVTCompute::Arguments;
135
-
136
- static ArgumentType prepare_args(torch::Tensor const& a_scales,
137
- torch::Tensor const& b_scales) {
138
- using ScaleAArgs = typename ScaleA::Arguments;
139
- using ScaleBArgs = typename ScaleB::Arguments;
140
-
141
- ScaleBArgs b_args{b_scales.data_ptr<float>(), b_scales.numel() != 1, {}};
142
- ScaleAArgs a_args{a_scales.data_ptr<float>(), a_scales.numel() != 1, {}};
143
-
144
- typename EVTCompute0::Arguments evt0_compute_args{b_args};
145
-
146
- typename EVTCompute::Arguments evt_compute_args{a_args, evt0_compute_args};
147
- return evt_compute_args;
148
- }
149
-};
150
-
151
-template <typename ElementD, typename OutputTileThreadMap>
152
-struct ScaledEpilogueBias
153
- : private ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
154
- private:
155
- using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
156
- using Accum = typename SUPER::Accum;
157
- using ScaleA = typename SUPER::ScaleA;
158
- using ScaleB = typename SUPER::ScaleB;
159
-
160
- using Compute0 = cutlass::epilogue::threadblock::VisitorCompute<
161
- cutlass::multiplies, float, float,
162
- cutlass::FloatRoundStyle::round_to_nearest>;
163
-
164
- using EVTCompute0 =
165
- cutlass::epilogue::threadblock::Sm80EVT<Compute0, ScaleB, Accum>;
166
-
167
- using Compute1 = cutlass::epilogue::threadblock::VisitorCompute<
168
- cutlass::multiply_add, ElementD, float,
169
- cutlass::FloatRoundStyle::round_to_nearest>;
170
-
171
- using Bias = cutlass::epilogue::threadblock::VisitorRowBroadcast<
172
- OutputTileThreadMap, ElementD, Stride<Int<0>, Int<1>, Int<0>>>;
173
-
174
- public:
175
- using EVTCompute = cutlass::epilogue::threadblock::Sm80EVT<Compute1, ScaleA,
176
- EVTCompute0, Bias>;
177
- using ArgumentType = typename EVTCompute::Arguments;
178
-
179
- static ArgumentType prepare_args(torch::Tensor const& a_scales,
180
- torch::Tensor const& b_scales,
181
- torch::Tensor const& bias) {
182
- using ScaleAArgs = typename ScaleA::Arguments;
183
- using ScaleBArgs = typename ScaleB::Arguments;
184
- using BiasArgs = typename Bias::Arguments;
185
-
186
- ScaleBArgs b_args{b_scales.data_ptr<float>(), b_scales.numel() != 1, {}};
187
- ScaleAArgs a_args{a_scales.data_ptr<float>(), a_scales.numel() != 1, {}};
188
- BiasArgs bias_args{static_cast<ElementD*>(bias.data_ptr()), {}};
189
-
190
- typename EVTCompute0::Arguments evt0_compute_args{b_args};
191
-
192
- typename EVTCompute::Arguments evt_compute_args{a_args, evt0_compute_args,
193
- bias_args};
194
- return evt_compute_args;
195
- }
196
-};
197
-
198
-template <typename Arch, template <typename> typename ArchGuard,
199
- typename ElementAB_, typename ElementD_,
200
- template <typename, typename> typename Epilogue_, typename TileShape,
201
- typename WarpShape, typename InstructionShape, int32_t MainLoopStages>
202
-struct cutlass_2x_gemm {
203
- using ElementAB = ElementAB_;
204
- using ElementD = ElementD_;
205
-
206
- using ElementAcc =
207
- typename std::conditional<std::is_same_v<ElementAB, int8_t>, int32_t,
208
- float>::type;
209
-
210
- using Operator =
211
- typename std::conditional<std::is_same_v<ElementAB, int8_t>,
212
- cutlass::arch::OpMultiplyAddSaturate,
213
- cutlass::arch::OpMultiplyAdd>::type;
214
-
215
- using OutputTileThreadMap =
216
- cutlass::epilogue::threadblock::OutputTileThreadLayout<
217
- TileShape, WarpShape, float, 4, 1 /* epilogue stages */
218
- >;
219
-
220
- using Epilogue = Epilogue_<ElementD, OutputTileThreadMap>;
221
- using EVTCompute = typename Epilogue::EVTCompute;
222
-
223
- using D = cutlass::epilogue::threadblock::VisitorAuxStore<
224
- OutputTileThreadMap, ElementD, cutlass::FloatRoundStyle::round_to_nearest,
225
- Stride<int64_t, Int<1>, Int<0>>>;
226
-
227
- using EVTD = cutlass::epilogue::threadblock::Sm80EVT<D, EVTCompute>;
228
-
229
- // clang-format off
230
- using RowMajor = typename cutlass::layout::RowMajor;
231
- using ColumnMajor = typename cutlass::layout::ColumnMajor;
232
- using KernelType =
233
- ArchGuard<typename cutlass::gemm::kernel::DefaultGemmWithVisitor<
234
- ElementAB, RowMajor, cutlass::ComplexTransform::kNone, 16,
235
- ElementAB, ColumnMajor, cutlass::ComplexTransform::kNone, 16,
236
- float, cutlass::layout::RowMajor, 4,
237
- ElementAcc, float, cutlass::arch::OpClassTensorOp,
238
- Arch,
239
- TileShape, WarpShape, InstructionShape,
240
- EVTD,
241
- cutlass::gemm::threadblock::ThreadblockSwizzleStreamK,
242
- MainLoopStages, Operator,
243
- 1 /* epilogue stages */
244
- >::GemmKernel>;
245
- // clang-format on
246
-
247
- using Op = cutlass::gemm::device::GemmUniversalAdapter<KernelType>;
248
-};
249
-
250
-template <typename Gemm, typename... EpilogueArgs>
251
-void cutlass_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
252
- torch::Tensor const& b,
253
- EpilogueArgs&&... epilogue_params) {
254
- using ElementAB = typename Gemm::ElementAB;
255
- using ElementD = typename Gemm::ElementD;
256
-
257
- int32_t m = a.size(0);
258
- int32_t n = b.size(1);
259
- int32_t k = a.size(1);
260
- cutlass::gemm::GemmCoord problem_size{m, n, k};
261
-
262
- int64_t lda = a.stride(0);
263
- int64_t ldb = b.stride(1);
264
- int64_t ldc = out.stride(0);
265
-
266
- using StrideC = Stride<int64_t, Int<1>, Int<0>>;
267
- StrideC c_stride{ldc, Int<1>{}, Int<0>{}};
268
-
269
- auto a_ptr = static_cast<ElementAB const*>(a.data_ptr());
270
- auto b_ptr = static_cast<ElementAB const*>(b.data_ptr());
271
- auto c_ptr = static_cast<ElementD*>(out.data_ptr());
272
-
273
- typename Gemm::D::Arguments d_args{c_ptr, c_stride};
274
-
275
- using Epilogue = typename Gemm::Epilogue;
276
- auto evt_args =
277
- Epilogue::prepare_args(std::forward<EpilogueArgs>(epilogue_params)...);
278
-
279
- typename Gemm::EVTD::Arguments epilogue_args{
280
- evt_args,
281
- d_args,
282
- };
283
-
284
- typename Gemm::Op::Arguments args{
285
- cutlass::gemm::GemmUniversalMode::kGemmSplitKParallel, // universal mode
286
- problem_size, // problem size
287
- 1, // batch count
288
- epilogue_args,
289
- a_ptr,
290
- b_ptr,
291
- nullptr,
292
- nullptr,
293
- 0,
294
- 0,
295
- 0,
296
- 0,
297
- lda,
298
- ldb,
299
- ldc,
300
- ldc};
301
-
302
- // Launch the CUTLASS GEMM kernel.
303
- typename Gemm::Op gemm_op;
304
- size_t workspace_size = gemm_op.get_workspace_size(args);
305
- cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
306
-
307
- auto stream = at::cuda::getCurrentCUDAStream(a.get_device());
308
-
309
- CUTLASS_CHECK(gemm_op.can_implement(args));
310
- cutlass::Status status = gemm_op(args, workspace.get(), stream);
311
- CUTLASS_CHECK(status);
312
-}
313
-
314
-template <typename Gemm, typename FallbackGemm, typename... EpilogueArgs>
315
-void fallback_cutlass_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
316
- torch::Tensor const& b,
317
- EpilogueArgs&&... args) {
318
- // In some cases, the GPU isn't able to accommodate the
319
- // shared memory requirements of the Gemm. In such cases, use
320
- // the FallbackGemm instead.
321
- static const int max_shared_mem_per_block_opt_in =
322
- get_cuda_max_shared_memory_per_block_opt_in(0);
323
-
324
- size_t const gemm_shared_mem_size =
325
- sizeof(typename Gemm::KernelType::SharedStorage);
326
- size_t const fallback_gemm_shared_mem_size =
327
- sizeof(typename FallbackGemm::KernelType::SharedStorage);
328
-
329
- if (gemm_shared_mem_size <= max_shared_mem_per_block_opt_in) {
330
- return cutlass_gemm_caller<Gemm>(out, a, b,
331
- std::forward<EpilogueArgs>(args)...);
332
- } else {
333
- TORCH_CHECK(fallback_gemm_shared_mem_size <=
334
- max_shared_mem_per_block_opt_in);
335
- return cutlass_gemm_caller<FallbackGemm>(
336
- out, a, b, std::forward<EpilogueArgs>(args)...);
337
- }
338
-}
339
-
340
-template <typename InType, typename OutType,
341
- template <typename, typename> typename Epilogue>
342
-struct sm80_config_default {
343
- // This config is used in 2 cases,
344
- // - M in (128, inf)
345
- // - M in (64, 128] and N >= 8192
346
- // Shared Memory required by this Gemm - 81920 bytes
347
- static_assert(std::is_same<InType, int8_t>());
348
- using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
349
- using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
350
- using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
351
- using Cutlass2xGemm =
352
- cutlass_2x_gemm<cutlass::arch::Sm80, enable_sm80_to_sm89, InType, OutType,
353
- Epilogue, TileShape, WarpShape, InstructionShape, 5>;
354
-};
355
-
356
-template <typename InType, typename OutType,
357
- template <typename, typename> typename Epilogue>
358
-struct sm80_config_M64 {
359
- // This config is used in 2 cases,
360
- // - M in (32, 64]
361
- // - M in (64, 128] and N < 8192
362
- // Shared Memory required by this Gemm - 122880 bytes
363
- static_assert(std::is_same<InType, int8_t>());
364
- using TileShape = typename cutlass::gemm::GemmShape<64, 128, 128>;
365
- using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
366
- using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
367
- using Cutlass2xGemm =
368
- cutlass_2x_gemm<cutlass::arch::Sm80, enable_sm80_to_sm89, InType, OutType,
369
- Epilogue, TileShape, WarpShape, InstructionShape, 5>;
370
-};
371
-
372
-template <typename InType, typename OutType,
373
- template <typename, typename> typename Epilogue>
374
-struct sm80_config_M32 {
375
- // M in (16, 32]
376
- // Shared Memory required by this Gemm - 61440 bytes
377
- static_assert(std::is_same<InType, int8_t>());
378
- using TileShape = typename cutlass::gemm::GemmShape<32, 64, 128>;
379
- using WarpShape = typename cutlass::gemm::GemmShape<32, 64, 64>;
380
- using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
381
- using Cutlass2xGemm =
382
- cutlass_2x_gemm<cutlass::arch::Sm80, enable_sm80_to_sm89, InType, OutType,
383
- Epilogue, TileShape, WarpShape, InstructionShape, 5>;
384
-};
385
-
386
-template <typename InType, typename OutType,
387
- template <typename, typename> typename Epilogue>
388
-struct sm80_config_M16 {
389
- // M in [1, 16]
390
- // Shared Memory required by this Gemm - 51200 bytes
391
- static_assert(std::is_same<InType, int8_t>());
392
- using TileShape = typename cutlass::gemm::GemmShape<16, 64, 128>;
393
- using WarpShape = typename cutlass::gemm::GemmShape<16, 64, 64>;
394
- using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
395
- using Cutlass2xGemm =
396
- cutlass_2x_gemm<cutlass::arch::Sm80, enable_sm80_to_sm89, InType, OutType,
397
- Epilogue, TileShape, WarpShape, InstructionShape, 5>;
398
-};
399
-
400
-} // namespace
401
-
402
-template <typename InType, typename OutType,
403
- template <typename, typename> typename Epilogue,
404
- typename... EpilogueArgs>
405
-void cutlass_gemm_sm80_dispatch(torch::Tensor& out, torch::Tensor const& a,
406
- torch::Tensor const& b,
407
- EpilogueArgs&&... args) {
408
- static_assert(std::is_same<InType, int8_t>());
409
- TORCH_CHECK(a.dtype() == torch::kInt8);
410
- TORCH_CHECK(b.dtype() == torch::kInt8);
411
-
412
- using Cutlass2xGemmDefault =
413
- typename sm80_config_default<InType, OutType, Epilogue>::Cutlass2xGemm;
414
- using Cutlass2xGemmM128BigN =
415
- typename sm80_config_default<InType, OutType, Epilogue>::Cutlass2xGemm;
416
- using Cutlass2xGemmM128SmallN =
417
- typename sm80_config_M64<InType, OutType, Epilogue>::Cutlass2xGemm;
418
- using Cutlass2xGemmM64 =
419
- typename sm80_config_M64<InType, OutType, Epilogue>::Cutlass2xGemm;
420
- using Cutlass2xGemmM32 =
421
- typename sm80_config_M32<InType, OutType, Epilogue>::Cutlass2xGemm;
422
- using Cutlass2xGemmM16 =
423
- typename sm80_config_M16<InType, OutType, Epilogue>::Cutlass2xGemm;
424
-
425
- // Due to shared memory requirements, some Gemms may fail to run on some
426
- // GPUs. As the name indicates, the Fallback Gemm is used as an alternative
427
- // in such cases.
428
- // sm80_config_M16 has the least shared-memory requirement. However,
429
- // based on some profiling, we select sm80_config_M32 as a better alternative
430
- // performance wise.
431
- using FallbackGemm =
432
- typename sm80_config_M32<InType, OutType, Epilogue>::Cutlass2xGemm;
433
-
434
- uint32_t const m = a.size(0);
435
- uint32_t const mp2 =
436
- std::max(static_cast<uint32_t>(16), next_pow_2(m)); // next power of 2
437
- if (mp2 <= 16) {
438
- // M in [1, 16]
439
- return fallback_cutlass_gemm_caller<Cutlass2xGemmM16, FallbackGemm>(
440
- out, a, b, std::forward<EpilogueArgs>(args)...);
441
- } else if (mp2 <= 32) {
442
- // M in (16, 32]
443
- return fallback_cutlass_gemm_caller<Cutlass2xGemmM32, FallbackGemm>(
444
- out, a, b, std::forward<EpilogueArgs>(args)...);
445
- } else if (mp2 <= 64) {
446
- // M in (32, 64]
447
- return fallback_cutlass_gemm_caller<Cutlass2xGemmM64, FallbackGemm>(
448
- out, a, b, std::forward<EpilogueArgs>(args)...);
449
- } else if (mp2 <= 128) {
450
- // M in (64, 128]
451
- uint32_t const n = out.size(1);
452
- bool const small_n = n < 8192;
453
- if (small_n) {
454
- return fallback_cutlass_gemm_caller<Cutlass2xGemmM128SmallN,
455
- FallbackGemm>(
456
- out, a, b, std::forward<EpilogueArgs>(args)...);
457
- } else {
458
- return fallback_cutlass_gemm_caller<Cutlass2xGemmM128BigN, FallbackGemm>(
459
- out, a, b, std::forward<EpilogueArgs>(args)...);
460
- }
461
- } else {
462
- // M in (128, inf)
463
- return fallback_cutlass_gemm_caller<Cutlass2xGemmDefault, FallbackGemm>(
464
- out, a, b, std::forward<EpilogueArgs>(args)...);
465
- }
466
-}
467
-
14
template <template <typename, typename> typename Epilogue,
15
typename... EpilogueArgs>
16
void cutlass_scaled_mm_sm75_epilogue(torch::Tensor& out, torch::Tensor const& a,
@@ -478,14 +24,15 @@ void cutlass_scaled_mm_sm75_epilogue(torch::Tensor& out, torch::Tensor const& a,
24
using InstructionShape = typename cutlass::gemm::GemmShape<8, 8, 16>;
25
26
if (out.dtype() == torch::kBFloat16) {
481
- return cutlass_gemm_caller<cutlass_2x_gemm<
482
- cutlass::arch::Sm75, enable_sm75_to_sm80, int8_t, cutlass::bfloat16_t,
483
- Epilogue, TileShape, WarpShape, InstructionShape, 2>>(
27
+ return vllm::cutlass_gemm_caller<
28
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm75, vllm::enable_sm75_to_sm80,
29
+ int8_t, cutlass::bfloat16_t, Epilogue, TileShape,
30
+ WarpShape, InstructionShape, 2>>(
31
out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
32
} else {
33
TORCH_CHECK(out.dtype() == torch::kFloat16);
487
- return cutlass_gemm_caller<cutlass_2x_gemm<
488
- cutlass::arch::Sm75, enable_sm75_to_sm80, int8_t, cutlass::half_t,
34
+ return vllm::cutlass_gemm_caller<vllm::cutlass_2x_gemm<
35
+ cutlass::arch::Sm75, vllm::enable_sm75_to_sm80, int8_t, cutlass::half_t,
36
Epilogue, TileShape, WarpShape, InstructionShape, 2>>(
37
out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
38
}
@@ -501,11 +48,11 @@ void cutlass_scaled_mm_sm75(torch::Tensor& out, torch::Tensor const& a,
48
if (bias) {
49
TORCH_CHECK(bias->dtype() == out.dtype(),
50
"currently bias dtype must match output dtype ", out.dtype());
504
- return cutlass_scaled_mm_sm75_epilogue<ScaledEpilogueBias>(
51
+ return cutlass_scaled_mm_sm75_epilogue<vllm::ScaledEpilogueBias>(
52
out, a, b, a_scales, b_scales, *bias);
53
} else {
507
- return cutlass_scaled_mm_sm75_epilogue<ScaledEpilogue>(out, a, b, a_scales,
508
- b_scales);
54
+ return cutlass_scaled_mm_sm75_epilogue<vllm::ScaledEpilogue>(
55
+ out, a, b, a_scales, b_scales);
56
}
57
}
58
@@ -518,11 +65,12 @@ void cutlass_scaled_mm_sm80_epilogue(torch::Tensor& out, torch::Tensor const& a,
65
TORCH_CHECK(b.dtype() == torch::kInt8);
66
67
if (out.dtype() == torch::kBFloat16) {
521
- return cutlass_gemm_sm80_dispatch<int8_t, cutlass::bfloat16_t, Epilogue>(
68
+ return vllm::cutlass_gemm_sm80_dispatch<int8_t, cutlass::bfloat16_t,
69
+ Epilogue>(
70
out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
71
} else {
72
TORCH_CHECK(out.dtype() == torch::kFloat16);
525
- return cutlass_gemm_sm80_dispatch<int8_t, cutlass::half_t, Epilogue>(
73
+ return vllm::cutlass_gemm_sm80_dispatch<int8_t, cutlass::half_t, Epilogue>(
74
out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
75
}
76
}
@@ -537,11 +85,11 @@ void cutlass_scaled_mm_sm80(torch::Tensor& out, torch::Tensor const& a,
85
if (bias) {
86
TORCH_CHECK(bias->dtype() == out.dtype(),
87
"currently bias dtype must match output dtype ", out.dtype());
540
- return cutlass_scaled_mm_sm80_epilogue<ScaledEpilogueBias>(
88
+ return cutlass_scaled_mm_sm80_epilogue<vllm::ScaledEpilogueBias>(
89
out, a, b, a_scales, b_scales, *bias);
90
} else {
543
- return cutlass_scaled_mm_sm80_epilogue<ScaledEpilogue>(out, a, b, a_scales,
544
- b_scales);
91
+ return cutlass_scaled_mm_sm80_epilogue<vllm::ScaledEpilogue>(
92
+ out, a, b, a_scales, b_scales);
93
}
94
}
95
@@ -558,15 +106,17 @@ void cutlass_scaled_mm_sm89_epilogue(torch::Tensor& out, torch::Tensor const& a,
106
TORCH_CHECK(b.dtype() == torch::kInt8);
107
108
if (out.dtype() == torch::kBFloat16) {
561
- return cutlass_gemm_caller<cutlass_2x_gemm<
562
- cutlass::arch::Sm89, enable_sm89_to_sm90, int8_t, cutlass::bfloat16_t,
563
- Epilogue, TileShape, WarpShape, InstructionShape, 5>>(
109
+ return vllm::cutlass_gemm_caller<
110
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
111
+ int8_t, cutlass::bfloat16_t, Epilogue,
112
+ TileShape, WarpShape, InstructionShape, 5>>(
113
out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
114
} else {
115
assert(out.dtype() == torch::kFloat16);
567
- return cutlass_gemm_caller<cutlass_2x_gemm<
568
- cutlass::arch::Sm89, enable_sm89_to_sm90, int8_t, cutlass::half_t,
569
- Epilogue, TileShape, WarpShape, InstructionShape, 5>>(
116
+ return vllm::cutlass_gemm_caller<
117
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
118
+ int8_t, cutlass::half_t, Epilogue, TileShape,
119
+ WarpShape, InstructionShape, 5>>(
120
out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
121
}
122
} else {
@@ -574,17 +124,13 @@ void cutlass_scaled_mm_sm89_epilogue(torch::Tensor& out, torch::Tensor const& a,
124
TORCH_CHECK(b.dtype() == torch::kFloat8_e4m3fn);
125
126
if (out.dtype() == torch::kBFloat16) {
577
- return cutlass_gemm_caller<
578
- cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
579
- cutlass::float_e4m3_t, cutlass::bfloat16_t, Epilogue,
580
- TileShape, WarpShape, InstructionShape, 5>>(
127
+ return vllm::cutlass_gemm_sm89_dispatch<cutlass::float_e4m3_t,
128
+ cutlass::bfloat16_t, Epilogue>(
129
out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
130
} else {
131
TORCH_CHECK(out.dtype() == torch::kFloat16);
584
- return cutlass_gemm_caller<
585
- cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
586
- cutlass::float_e4m3_t, cutlass::half_t, Epilogue,
587
- TileShape, WarpShape, InstructionShape, 5>>(
132
+ return vllm::cutlass_gemm_sm89_dispatch<cutlass::float_e4m3_t,
133
+ cutlass::half_t, Epilogue>(
134
out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
135
}
136
}
@@ -600,10 +146,10 @@ void cutlass_scaled_mm_sm89(torch::Tensor& out, torch::Tensor const& a,
146
if (bias) {
147
TORCH_CHECK(bias->dtype() == out.dtype(),
148
"currently bias dtype must match output dtype ", out.dtype());
603
- return cutlass_scaled_mm_sm89_epilogue<ScaledEpilogueBias>(
149
+ return cutlass_scaled_mm_sm89_epilogue<vllm::ScaledEpilogueBias>(
150
out, a, b, a_scales, b_scales, *bias);
151
} else {
606
- return cutlass_scaled_mm_sm89_epilogue<ScaledEpilogue>(out, a, b, a_scales,
607
- b_scales);
152
+ return cutlass_scaled_mm_sm89_epilogue<vllm::ScaledEpilogue>(
153
+ out, a, b, a_scales, b_scales);
154
}
155
}
csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cuh
new
+340
@@ -0,0 +1,340 @@
1
+#pragma once
2
+#include <stddef.h>
3
+#include <torch/all.h>
4
+
5
+#include <ATen/cuda/CUDAContext.h>
6
+
7
+// clang-format will break include orders
8
+// clang-format off
9
+#include "cute/tensor.hpp"
10
+#include "cute/atom/mma_atom.hpp"
11
+#include "cutlass/numeric_types.h"
12
+
13
+#include "cutlass/util/device_memory.h"
14
+
15
+#include "cutlass/cutlass.h"
16
+#include "cutlass/gemm_coord.h"
17
+#include "cutlass/arch/mma_sm75.h"
18
+#include "cutlass/arch/arch.h"
19
+#include "cutlass/arch/mma.h"
20
+#include "cutlass/gemm/device/gemm.h"
21
+#include "cutlass/gemm/device/gemm_universal_adapter.h"
22
+
23
+#include "cutlass/epilogue/threadblock/fusion/visitors.hpp"
24
+#include "cutlass/gemm/kernel/default_gemm_universal_with_visitor.h"
25
+
26
+#include "broadcast_load_epilogue_c2x.hpp"
27
+#include "common.hpp"
28
+// clang-format on
29
+
30
+using namespace cute;
31
+
32
+/*
33
+ Epilogue functions can be defined to post-process the output before it is
34
+ written to GPU memory.
35
+ Epilogues must contain a public type named EVTCompute of type Sm80EVT,
36
+ as well as a static prepare_args function that constructs an
37
+ EVTCompute::Arguments struct.
38
+*/
39
+
40
+namespace vllm {
41
+
42
+// Wrappers for the GEMM kernel that is used to guard against compilation on
43
+// architectures that will never use the kernel. The purpose of this is to
44
+// reduce the size of the compiled binary.
45
+// __CUDA_ARCH__ is not defined in host code, so this lets us smuggle the ifdef
46
+// into code that will be executed on the device where it is defined.
47
+template <typename Kernel>
48
+struct enable_sm75_to_sm80 : Kernel {
49
+ template <typename... Args>
50
+ CUTLASS_DEVICE static void invoke(Args&&... args) {
51
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 750 && __CUDA_ARCH__ < 800
52
+ Kernel::invoke(std::forward<Args>(args)...);
53
+#endif
54
+ }
55
+};
56
+
57
+template <typename Kernel>
58
+struct enable_sm80_to_sm89 : Kernel {
59
+ template <typename... Args>
60
+ CUTLASS_DEVICE static void invoke(Args&&... args) {
61
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 800 && __CUDA_ARCH__ < 890
62
+ Kernel::invoke(std::forward<Args>(args)...);
63
+#endif
64
+ }
65
+};
66
+
67
+template <typename Kernel>
68
+struct enable_sm89_to_sm90 : Kernel {
69
+ template <typename... Args>
70
+ CUTLASS_DEVICE static void invoke(Args&&... args) {
71
+#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 890 && __CUDA_ARCH__ < 900
72
+ Kernel::invoke(std::forward<Args>(args)...);
73
+#endif
74
+ }
75
+};
76
+
77
+/*
78
+ * This class provides the common ScaleA and ScaleB descriptors for the
79
+ * ScaledEpilogue and ScaledEpilogueBias classes.
80
+ */
81
+template <typename ElementD, typename OutputTileThreadMap>
82
+struct ScaledEpilogueBase {
83
+ protected:
84
+ using Accum = cutlass::epilogue::threadblock::VisitorAccFetch;
85
+
86
+ using ScaleA = cutlass::epilogue::threadblock::VisitorColOrScalarBroadcast<
87
+ OutputTileThreadMap, float, Stride<Int<1>, Int<0>, Int<0>>>;
88
+
89
+ using ScaleB = cutlass::epilogue::threadblock::VisitorRowOrScalarBroadcast<
90
+ OutputTileThreadMap, float, Stride<Int<0>, Int<1>, Int<0>>>;
91
+};
92
+
93
+/*
94
+ This epilogue function defines a quantized GEMM operation similar to
95
+ torch._scaled_mm.
96
+
97
+ A and B may be both either int8 or fp8_e4m3. A can be quantized per-tensor or
98
+ per-row. B can be quantized per-tensor or per-column.
99
+ Any combination of per-tensor and per-row or column is supported.
100
+ A and B must have symmetric quantization (zero point == 0).
101
+
102
+ So the GEMM operation is D = (a_scales * A) (b_scales * B), where the
103
+ scales are applied elementwise with numpy-style broadcasting.
104
+
105
+ ScaleA and ScaleB define the epilogue functions that apply the scales for
106
+ the A and B operands respectively. These scales may be either per-tensor or
107
+ per row or column.
108
+*/
109
+template <typename ElementD, typename OutputTileThreadMap>
110
+struct ScaledEpilogue
111
+ : private ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
112
+ private:
113
+ using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
114
+ using Accum = typename SUPER::Accum;
115
+ using ScaleA = typename SUPER::ScaleA;
116
+ using ScaleB = typename SUPER::ScaleB;
117
+
118
+ using Compute0 = cutlass::epilogue::threadblock::VisitorCompute<
119
+ cutlass::multiplies, float, float,
120
+ cutlass::FloatRoundStyle::round_to_nearest>;
121
+
122
+ using EVTCompute0 =
123
+ cutlass::epilogue::threadblock::Sm80EVT<Compute0, ScaleB, Accum>;
124
+
125
+ using Compute1 = cutlass::epilogue::threadblock::VisitorCompute<
126
+ cutlass::multiplies, ElementD, float,
127
+ cutlass::FloatRoundStyle::round_to_nearest>;
128
+
129
+ public:
130
+ using EVTCompute =
131
+ cutlass::epilogue::threadblock::Sm80EVT<Compute1, ScaleA, EVTCompute0>;
132
+ using ArgumentType = typename EVTCompute::Arguments;
133
+
134
+ static ArgumentType prepare_args(torch::Tensor const& a_scales,
135
+ torch::Tensor const& b_scales) {
136
+ using ScaleAArgs = typename ScaleA::Arguments;
137
+ using ScaleBArgs = typename ScaleB::Arguments;
138
+
139
+ ScaleBArgs b_args{b_scales.data_ptr<float>(), b_scales.numel() != 1, {}};
140
+ ScaleAArgs a_args{a_scales.data_ptr<float>(), a_scales.numel() != 1, {}};
141
+
142
+ typename EVTCompute0::Arguments evt0_compute_args{b_args};
143
+
144
+ typename EVTCompute::Arguments evt_compute_args{a_args, evt0_compute_args};
145
+ return evt_compute_args;
146
+ }
147
+};
148
+
149
+template <typename ElementD, typename OutputTileThreadMap>
150
+struct ScaledEpilogueBias
151
+ : private ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
152
+ private:
153
+ using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
154
+ using Accum = typename SUPER::Accum;
155
+ using ScaleA = typename SUPER::ScaleA;
156
+ using ScaleB = typename SUPER::ScaleB;
157
+
158
+ using Compute0 = cutlass::epilogue::threadblock::VisitorCompute<
159
+ cutlass::multiplies, float, float,
160
+ cutlass::FloatRoundStyle::round_to_nearest>;
161
+
162
+ using EVTCompute0 =
163
+ cutlass::epilogue::threadblock::Sm80EVT<Compute0, ScaleB, Accum>;
164
+
165
+ using Compute1 = cutlass::epilogue::threadblock::VisitorCompute<
166
+ cutlass::multiply_add, ElementD, float,
167
+ cutlass::FloatRoundStyle::round_to_nearest>;
168
+
169
+ using Bias = cutlass::epilogue::threadblock::VisitorRowBroadcast<
170
+ OutputTileThreadMap, ElementD, Stride<Int<0>, Int<1>, Int<0>>>;
171
+
172
+ public:
173
+ using EVTCompute = cutlass::epilogue::threadblock::Sm80EVT<Compute1, ScaleA,
174
+ EVTCompute0, Bias>;
175
+ using ArgumentType = typename EVTCompute::Arguments;
176
+
177
+ static ArgumentType prepare_args(torch::Tensor const& a_scales,
178
+ torch::Tensor const& b_scales,
179
+ torch::Tensor const& bias) {
180
+ using ScaleAArgs = typename ScaleA::Arguments;
181
+ using ScaleBArgs = typename ScaleB::Arguments;
182
+ using BiasArgs = typename Bias::Arguments;
183
+
184
+ ScaleBArgs b_args{b_scales.data_ptr<float>(), b_scales.numel() != 1, {}};
185
+ ScaleAArgs a_args{a_scales.data_ptr<float>(), a_scales.numel() != 1, {}};
186
+ BiasArgs bias_args{static_cast<ElementD*>(bias.data_ptr()), {}};
187
+
188
+ typename EVTCompute0::Arguments evt0_compute_args{b_args};
189
+
190
+ typename EVTCompute::Arguments evt_compute_args{a_args, evt0_compute_args,
191
+ bias_args};
192
+ return evt_compute_args;
193
+ }
194
+};
195
+
196
+template <typename Arch, template <typename> typename ArchGuard,
197
+ typename ElementAB_, typename ElementD_,
198
+ template <typename, typename> typename Epilogue_, typename TileShape,
199
+ typename WarpShape, typename InstructionShape, int32_t MainLoopStages,
200
+ typename FP8MathOperator = cutlass::arch::OpMultiplyAdd>
201
+struct cutlass_2x_gemm {
202
+ using ElementAB = ElementAB_;
203
+ using ElementD = ElementD_;
204
+
205
+ using ElementAcc =
206
+ typename std::conditional<std::is_same_v<ElementAB, int8_t>, int32_t,
207
+ float>::type;
208
+
209
+ using Operator =
210
+ typename std::conditional<std::is_same_v<ElementAB, int8_t>,
211
+ cutlass::arch::OpMultiplyAddSaturate,
212
+ FP8MathOperator>::type;
213
+
214
+ using OutputTileThreadMap =
215
+ cutlass::epilogue::threadblock::OutputTileThreadLayout<
216
+ TileShape, WarpShape, float, 4, 1 /* epilogue stages */
217
+ >;
218
+
219
+ using Epilogue = Epilogue_<ElementD, OutputTileThreadMap>;
220
+ using EVTCompute = typename Epilogue::EVTCompute;
221
+
222
+ using D = cutlass::epilogue::threadblock::VisitorAuxStore<
223
+ OutputTileThreadMap, ElementD, cutlass::FloatRoundStyle::round_to_nearest,
224
+ Stride<int64_t, Int<1>, Int<0>>>;
225
+
226
+ using EVTD = cutlass::epilogue::threadblock::Sm80EVT<D, EVTCompute>;
227
+
228
+ // clang-format off
229
+ using RowMajor = typename cutlass::layout::RowMajor;
230
+ using ColumnMajor = typename cutlass::layout::ColumnMajor;
231
+ using KernelType =
232
+ ArchGuard<typename cutlass::gemm::kernel::DefaultGemmWithVisitor<
233
+ ElementAB, RowMajor, cutlass::ComplexTransform::kNone, 16,
234
+ ElementAB, ColumnMajor, cutlass::ComplexTransform::kNone, 16,
235
+ float, cutlass::layout::RowMajor, 4,
236
+ ElementAcc, float, cutlass::arch::OpClassTensorOp,
237
+ Arch,
238
+ TileShape, WarpShape, InstructionShape,
239
+ EVTD,
240
+ cutlass::gemm::threadblock::ThreadblockSwizzleStreamK,
241
+ MainLoopStages, Operator,
242
+ 1 /* epilogue stages */
243
+ >::GemmKernel>;
244
+ // clang-format on
245
+
246
+ using Op = cutlass::gemm::device::GemmUniversalAdapter<KernelType>;
247
+};
248
+
249
+template <typename Gemm, typename... EpilogueArgs>
250
+inline void cutlass_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
251
+ torch::Tensor const& b,
252
+ EpilogueArgs&&... epilogue_params) {
253
+ using ElementAB = typename Gemm::ElementAB;
254
+ using ElementD = typename Gemm::ElementD;
255
+
256
+ int32_t m = a.size(0);
257
+ int32_t n = b.size(1);
258
+ int32_t k = a.size(1);
259
+ cutlass::gemm::GemmCoord problem_size{m, n, k};
260
+
261
+ int64_t lda = a.stride(0);
262
+ int64_t ldb = b.stride(1);
263
+ int64_t ldc = out.stride(0);
264
+
265
+ using StrideC = Stride<int64_t, Int<1>, Int<0>>;
266
+ StrideC c_stride{ldc, Int<1>{}, Int<0>{}};
267
+
268
+ auto a_ptr = static_cast<ElementAB const*>(a.data_ptr());
269
+ auto b_ptr = static_cast<ElementAB const*>(b.data_ptr());
270
+ auto c_ptr = static_cast<ElementD*>(out.data_ptr());
271
+
272
+ typename Gemm::D::Arguments d_args{c_ptr, c_stride};
273
+
274
+ using Epilogue = typename Gemm::Epilogue;
275
+ auto evt_args =
276
+ Epilogue::prepare_args(std::forward<EpilogueArgs>(epilogue_params)...);
277
+
278
+ typename Gemm::EVTD::Arguments epilogue_args{
279
+ evt_args,
280
+ d_args,
281
+ };
282
+
283
+ typename Gemm::Op::Arguments args{
284
+ cutlass::gemm::GemmUniversalMode::kGemmSplitKParallel, // universal mode
285
+ problem_size, // problem size
286
+ 1, // batch count
287
+ epilogue_args,
288
+ a_ptr,
289
+ b_ptr,
290
+ nullptr,
291
+ nullptr,
292
+ 0,
293
+ 0,
294
+ 0,
295
+ 0,
296
+ lda,
297
+ ldb,
298
+ ldc,
299
+ ldc};
300
+
301
+ // Launch the CUTLASS GEMM kernel.
302
+ typename Gemm::Op gemm_op;
303
+ size_t workspace_size = gemm_op.get_workspace_size(args);
304
+ cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
305
+
306
+ auto stream = at::cuda::getCurrentCUDAStream(a.get_device());
307
+
308
+ CUTLASS_CHECK(gemm_op.can_implement(args));
309
+ cutlass::Status status = gemm_op(args, workspace.get(), stream);
310
+ CUTLASS_CHECK(status);
311
+}
312
+
313
+template <typename Gemm, typename FallbackGemm, typename... EpilogueArgs>
314
+inline void fallback_cutlass_gemm_caller(torch::Tensor& out,
315
+ torch::Tensor const& a,
316
+ torch::Tensor const& b,
317
+ EpilogueArgs&&... args) {
318
+ // In some cases, the GPU isn't able to accommodate the
319
+ // shared memory requirements of the Gemm. In such cases, use
320
+ // the FallbackGemm instead.
321
+ static const int max_shared_mem_per_block_opt_in =
322
+ get_cuda_max_shared_memory_per_block_opt_in(0);
323
+
324
+ size_t const gemm_shared_mem_size =
325
+ sizeof(typename Gemm::KernelType::SharedStorage);
326
+ size_t const fallback_gemm_shared_mem_size =
327
+ sizeof(typename FallbackGemm::KernelType::SharedStorage);
328
+
329
+ if (gemm_shared_mem_size <= max_shared_mem_per_block_opt_in) {
330
+ return cutlass_gemm_caller<Gemm>(out, a, b,
331
+ std::forward<EpilogueArgs>(args)...);
332
+ } else {
333
+ TORCH_CHECK(fallback_gemm_shared_mem_size <=
334
+ max_shared_mem_per_block_opt_in);
335
+ return cutlass_gemm_caller<FallbackGemm>(
336
+ out, a, b, std::forward<EpilogueArgs>(args)...);
337
+ }
338
+}
339
+
340
+} // namespace vllm
csrc/quantization/cutlass_w8a8/scaled_mm_c2x_sm80_dispatch.cuh
new
+139
@@ -0,0 +1,139 @@
1
+#pragma once
2
+
3
+#include "scaled_mm_c2x.cuh"
4
+
5
+/**
6
+ * This file defines Gemm kernel configurations for SM80 based on the Gemm
7
+ * shape.
8
+ */
9
+
10
+namespace vllm {
11
+
12
+template <typename InType, typename OutType,
13
+ template <typename, typename> typename Epilogue>
14
+struct sm80_config_default {
15
+ // This config is used in 2 cases,
16
+ // - M in (128, inf)
17
+ // - M in (64, 128] and N >= 8192
18
+ // Shared Memory required by this Gemm - 81920 bytes
19
+ static_assert(std::is_same<InType, int8_t>());
20
+ using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
21
+ using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
22
+ using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
23
+ using Cutlass2xGemm =
24
+ cutlass_2x_gemm<cutlass::arch::Sm80, enable_sm80_to_sm89, InType, OutType,
25
+ Epilogue, TileShape, WarpShape, InstructionShape, 5>;
26
+};
27
+
28
+template <typename InType, typename OutType,
29
+ template <typename, typename> typename Epilogue>
30
+struct sm80_config_M64 {
31
+ // This config is used in 2 cases,
32
+ // - M in (32, 64]
33
+ // - M in (64, 128] and N < 8192
34
+ // Shared Memory required by this Gemm - 122880 bytes
35
+ static_assert(std::is_same<InType, int8_t>());
36
+ using TileShape = typename cutlass::gemm::GemmShape<64, 128, 128>;
37
+ using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
38
+ using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
39
+ using Cutlass2xGemm =
40
+ cutlass_2x_gemm<cutlass::arch::Sm80, enable_sm80_to_sm89, InType, OutType,
41
+ Epilogue, TileShape, WarpShape, InstructionShape, 5>;
42
+};
43
+
44
+template <typename InType, typename OutType,
45
+ template <typename, typename> typename Epilogue>
46
+struct sm80_config_M32 {
47
+ // M in (16, 32]
48
+ // Shared Memory required by this Gemm - 61440 bytes
49
+ static_assert(std::is_same<InType, int8_t>());
50
+ using TileShape = typename cutlass::gemm::GemmShape<32, 64, 128>;
51
+ using WarpShape = typename cutlass::gemm::GemmShape<32, 64, 64>;
52
+ using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
53
+ using Cutlass2xGemm =
54
+ cutlass_2x_gemm<cutlass::arch::Sm80, enable_sm80_to_sm89, InType, OutType,
55
+ Epilogue, TileShape, WarpShape, InstructionShape, 5>;
56
+};
57
+
58
+template <typename InType, typename OutType,
59
+ template <typename, typename> typename Epilogue>
60
+struct sm80_config_M16 {
61
+ // M in [1, 16]
62
+ // Shared Memory required by this Gemm - 51200 bytes
63
+ static_assert(std::is_same<InType, int8_t>());
64
+ using TileShape = typename cutlass::gemm::GemmShape<16, 64, 128>;
65
+ using WarpShape = typename cutlass::gemm::GemmShape<16, 64, 64>;
66
+ using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
67
+ using Cutlass2xGemm =
68
+ cutlass_2x_gemm<cutlass::arch::Sm80, enable_sm80_to_sm89, InType, OutType,
69
+ Epilogue, TileShape, WarpShape, InstructionShape, 5>;
70
+};
71
+
72
+template <typename InType, typename OutType,
73
+ template <typename, typename> typename Epilogue,
74
+ typename... EpilogueArgs>
75
+inline void cutlass_gemm_sm80_dispatch(torch::Tensor& out,
76
+ torch::Tensor const& a,
77
+ torch::Tensor const& b,
78
+ EpilogueArgs&&... args) {
79
+ static_assert(std::is_same<InType, int8_t>());
80
+ TORCH_CHECK(a.dtype() == torch::kInt8);
81
+ TORCH_CHECK(b.dtype() == torch::kInt8);
82
+
83
+ using Cutlass2xGemmDefault =
84
+ typename sm80_config_default<InType, OutType, Epilogue>::Cutlass2xGemm;
85
+ using Cutlass2xGemmM128BigN =
86
+ typename sm80_config_default<InType, OutType, Epilogue>::Cutlass2xGemm;
87
+ using Cutlass2xGemmM128SmallN =
88
+ typename sm80_config_M64<InType, OutType, Epilogue>::Cutlass2xGemm;
89
+ using Cutlass2xGemmM64 =
90
+ typename sm80_config_M64<InType, OutType, Epilogue>::Cutlass2xGemm;
91
+ using Cutlass2xGemmM32 =
92
+ typename sm80_config_M32<InType, OutType, Epilogue>::Cutlass2xGemm;
93
+ using Cutlass2xGemmM16 =
94
+ typename sm80_config_M16<InType, OutType, Epilogue>::Cutlass2xGemm;
95
+
96
+ // Due to shared memory requirements, some Gemms may fail to run on some
97
+ // GPUs. As the name indicates, the Fallback Gemm is used as an alternative
98
+ // in such cases.
99
+ // sm80_config_M16 has the least shared-memory requirement. However,
100
+ // based on some profiling, we select sm80_config_M32 as a better alternative
101
+ // performance wise.
102
+ using FallbackGemm =
103
+ typename sm80_config_M32<InType, OutType, Epilogue>::Cutlass2xGemm;
104
+
105
+ uint32_t const m = a.size(0);
106
+ uint32_t const mp2 =
107
+ std::max(static_cast<uint32_t>(16), next_pow_2(m)); // next power of 2
108
+ if (mp2 <= 16) {
109
+ // M in [1, 16]
110
+ return fallback_cutlass_gemm_caller<Cutlass2xGemmM16, FallbackGemm>(
111
+ out, a, b, std::forward<EpilogueArgs>(args)...);
112
+ } else if (mp2 <= 32) {
113
+ // M in (16, 32]
114
+ return fallback_cutlass_gemm_caller<Cutlass2xGemmM32, FallbackGemm>(
115
+ out, a, b, std::forward<EpilogueArgs>(args)...);
116
+ } else if (mp2 <= 64) {
117
+ // M in (32, 64]
118
+ return fallback_cutlass_gemm_caller<Cutlass2xGemmM64, FallbackGemm>(
119
+ out, a, b, std::forward<EpilogueArgs>(args)...);
120
+ } else if (mp2 <= 128) {
121
+ // M in (64, 128]
122
+ uint32_t const n = out.size(1);
123
+ bool const small_n = n < 8192;
124
+ if (small_n) {
125
+ return fallback_cutlass_gemm_caller<Cutlass2xGemmM128SmallN,
126
+ FallbackGemm>(
127
+ out, a, b, std::forward<EpilogueArgs>(args)...);
128
+ } else {
129
+ return fallback_cutlass_gemm_caller<Cutlass2xGemmM128BigN, FallbackGemm>(
130
+ out, a, b, std::forward<EpilogueArgs>(args)...);
131
+ }
132
+ } else {
133
+ // M in (128, inf)
134
+ return fallback_cutlass_gemm_caller<Cutlass2xGemmDefault, FallbackGemm>(
135
+ out, a, b, std::forward<EpilogueArgs>(args)...);
136
+ }
137
+}
138
+
139
+} // namespace vllm
csrc/quantization/cutlass_w8a8/scaled_mm_c2x_sm89_dispatch.cuh
new
+362
@@ -0,0 +1,362 @@
1
+#pragma once
2
+
3
+#include "scaled_mm_c2x.cuh"
4
+#include "cutlass/float8.h"
5
+
6
+/**
7
+ * This file defines Gemm kernel configurations for SM89 based on the Gemm
8
+ * shape.
9
+ */
10
+
11
+namespace vllm {
12
+
13
+template <typename InType, typename OutType,
14
+ template <typename, typename> typename Epilogue>
15
+struct sm89_fallback_gemm {
16
+ // Shared Memory required by this Gemm - 61440 bytes
17
+ static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
18
+ using TileShape = typename cutlass::gemm::GemmShape<64, 128, 64>;
19
+ using WarpShape = typename cutlass::gemm::GemmShape<32, 64, 64>;
20
+ using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
21
+ using FP8MathOperator = typename cutlass::arch::OpMultiplyAdd;
22
+ using Cutlass2xGemm =
23
+ cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90, InType, OutType,
24
+ Epilogue, TileShape, WarpShape, InstructionShape, 5,
25
+ FP8MathOperator>;
26
+};
27
+
28
+struct sm89_config_default {
29
+ // M in (256, inf)
30
+ using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
31
+ using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
32
+ using FP8MathOperator = typename cutlass::arch::OpMultiplyAddFastAccum;
33
+
34
+ template <typename InType, typename OutType,
35
+ template <typename, typename> typename Epilogue,
36
+ typename... EpilogueArgs>
37
+ static void dispatch(torch::Tensor& out, torch::Tensor const& a,
38
+ torch::Tensor const& b, EpilogueArgs&&... args) {
39
+ static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
40
+ TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
41
+
42
+ using FallbackGemm =
43
+ typename sm89_fallback_gemm<InType, OutType, Epilogue>::Cutlass2xGemm;
44
+
45
+ uint32_t const n = out.size(1);
46
+ uint32_t const np2 = next_pow_2(n);
47
+
48
+ if (np2 <= 4096) {
49
+ using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
50
+
51
+ return vllm::fallback_cutlass_gemm_caller<
52
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
53
+ InType, OutType, Epilogue, TileShape, WarpShape,
54
+ InstructionShape, 5, FP8MathOperator>,
55
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
56
+ } else if (np2 <= 8192) {
57
+ using TileShape = typename cutlass::gemm::GemmShape<256, 128, 64>;
58
+
59
+ return vllm::fallback_cutlass_gemm_caller<
60
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
61
+ InType, OutType, Epilogue, TileShape, WarpShape,
62
+ InstructionShape, 3, FP8MathOperator>,
63
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
64
+
65
+ } else {
66
+ using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
67
+
68
+ return vllm::fallback_cutlass_gemm_caller<
69
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
70
+ InType, OutType, Epilogue, TileShape, WarpShape,
71
+ InstructionShape, 5, FP8MathOperator>,
72
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
73
+ }
74
+ }
75
+};
76
+
77
+struct sm89_config_M256 {
78
+ // M in (128, 256]
79
+ using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
80
+ using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
81
+ using FP8MathOperator = typename cutlass::arch::OpMultiplyAddFastAccum;
82
+
83
+ template <typename InType, typename OutType,
84
+ template <typename, typename> typename Epilogue,
85
+ typename... EpilogueArgs>
86
+ static void dispatch(torch::Tensor& out, torch::Tensor const& a,
87
+ torch::Tensor const& b, EpilogueArgs&&... args) {
88
+ static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
89
+ TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
90
+
91
+ using FallbackGemm =
92
+ typename sm89_fallback_gemm<InType, OutType, Epilogue>::Cutlass2xGemm;
93
+
94
+ uint32_t const n = out.size(1);
95
+ uint32_t const np2 = next_pow_2(n);
96
+
97
+ if (np2 <= 4096) {
98
+ using TileShape = typename cutlass::gemm::GemmShape<64, 128, 128>;
99
+
100
+ return vllm::fallback_cutlass_gemm_caller<
101
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
102
+ InType, OutType, Epilogue, TileShape, WarpShape,
103
+ InstructionShape, 3, FP8MathOperator>,
104
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
105
+ } else {
106
+ using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
107
+
108
+ return vllm::fallback_cutlass_gemm_caller<
109
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
110
+ InType, OutType, Epilogue, TileShape, WarpShape,
111
+ InstructionShape, 5, FP8MathOperator>,
112
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
113
+ }
114
+ }
115
+};
116
+
117
+struct sm89_config_M128 {
118
+ // M in (64, 128]
119
+ using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
120
+ using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
121
+ using FP8MathOperator = typename cutlass::arch::OpMultiplyAddFastAccum;
122
+
123
+ template <typename InType, typename OutType,
124
+ template <typename, typename> typename Epilogue,
125
+ typename... EpilogueArgs>
126
+ static void dispatch(torch::Tensor& out, torch::Tensor const& a,
127
+ torch::Tensor const& b, EpilogueArgs&&... args) {
128
+ static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
129
+ TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
130
+
131
+ using FallbackGemm =
132
+ typename sm89_fallback_gemm<InType, OutType, Epilogue>::Cutlass2xGemm;
133
+
134
+ uint32_t const n = out.size(1);
135
+ uint32_t const np2 = next_pow_2(n);
136
+
137
+ if (np2 <= 8192) {
138
+ using TileShape = typename cutlass::gemm::GemmShape<64, 128, 128>;
139
+
140
+ return vllm::fallback_cutlass_gemm_caller<
141
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
142
+ InType, OutType, Epilogue, TileShape, WarpShape,
143
+ InstructionShape, 3, FP8MathOperator>,
144
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
145
+
146
+ } else if (np2 <= 16384) {
147
+ using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
148
+
149
+ return vllm::fallback_cutlass_gemm_caller<
150
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
151
+ InType, OutType, Epilogue, TileShape, WarpShape,
152
+ InstructionShape, 5, FP8MathOperator>,
153
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
154
+ } else {
155
+ using TileShape = typename cutlass::gemm::GemmShape<128, 64, 128>;
156
+
157
+ return vllm::fallback_cutlass_gemm_caller<
158
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
159
+ InType, OutType, Epilogue, TileShape, WarpShape,
160
+ InstructionShape, 3, FP8MathOperator>,
161
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
162
+ }
163
+ }
164
+};
165
+
166
+struct sm89_config_M64 {
167
+ // M in (32, 64]
168
+ using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
169
+
170
+ template <typename InType, typename OutType,
171
+ template <typename, typename> typename Epilogue,
172
+ typename... EpilogueArgs>
173
+ static void dispatch(torch::Tensor& out, torch::Tensor const& a,
174
+ torch::Tensor const& b, EpilogueArgs&&... args) {
175
+ static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
176
+ TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
177
+
178
+ using FallbackGemm =
179
+ typename sm89_fallback_gemm<InType, OutType, Epilogue>::Cutlass2xGemm;
180
+
181
+ uint32_t const n = out.size(1);
182
+ uint32_t const np2 = next_pow_2(n);
183
+
184
+ if (np2 <= 8196) {
185
+ using TileShape = typename cutlass::gemm::GemmShape<64, 64, 128>;
186
+ using WarpShape = typename cutlass::gemm::GemmShape<32, 64, 64>;
187
+ using FP8MathOperator = typename cutlass::arch::OpMultiplyAdd;
188
+
189
+ return vllm::fallback_cutlass_gemm_caller<
190
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
191
+ InType, OutType, Epilogue, TileShape, WarpShape,
192
+ InstructionShape, 5, FP8MathOperator>,
193
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
194
+ } else if (np2 <= 16384) {
195
+ using TileShape = typename cutlass::gemm::GemmShape<64, 128, 128>;
196
+ using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
197
+ using FP8MathOperator = typename cutlass::arch::OpMultiplyAddFastAccum;
198
+
199
+ return vllm::fallback_cutlass_gemm_caller<
200
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
201
+ InType, OutType, Epilogue, TileShape, WarpShape,
202
+ InstructionShape, 3, FP8MathOperator>,
203
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
204
+ } else {
205
+ using TileShape = typename cutlass::gemm::GemmShape<64, 64, 128>;
206
+ using WarpShape = typename cutlass::gemm::GemmShape<32, 64, 64>;
207
+ using FP8MathOperator = typename cutlass::arch::OpMultiplyAdd;
208
+
209
+ return vllm::fallback_cutlass_gemm_caller<
210
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
211
+ InType, OutType, Epilogue, TileShape, WarpShape,
212
+ InstructionShape, 5, FP8MathOperator>,
213
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
214
+ }
215
+ }
216
+};
217
+
218
+struct sm89_config_M32 {
219
+ // M in (16, 32]
220
+ using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
221
+ using FP8MathOperator = typename cutlass::arch::OpMultiplyAddFastAccum;
222
+
223
+ template <typename InType, typename OutType,
224
+ template <typename, typename> typename Epilogue,
225
+ typename... EpilogueArgs>
226
+ static void dispatch(torch::Tensor& out, torch::Tensor const& a,
227
+ torch::Tensor const& b, EpilogueArgs&&... args) {
228
+ static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
229
+ TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
230
+
231
+ using FallbackGemm =
232
+ typename sm89_fallback_gemm<InType, OutType, Epilogue>::Cutlass2xGemm;
233
+
234
+ uint32_t const n = out.size(1);
235
+ uint32_t const np2 = next_pow_2(n);
236
+
237
+ if (np2 <= 8192) {
238
+ using TileShape = typename cutlass::gemm::GemmShape<32, 64, 128>;
239
+ using WarpShape = typename cutlass::gemm::GemmShape<16, 64, 64>;
240
+
241
+ return vllm::fallback_cutlass_gemm_caller<
242
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
243
+ InType, OutType, Epilogue, TileShape, WarpShape,
244
+ InstructionShape, 5, FP8MathOperator>,
245
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
246
+ } else if (np2 <= 16384) {
247
+ using TileShape = typename cutlass::gemm::GemmShape<32, 128, 128>;
248
+ using WarpShape = typename cutlass::gemm::GemmShape<32, 64, 64>;
249
+
250
+ return vllm::fallback_cutlass_gemm_caller<
251
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
252
+ InType, OutType, Epilogue, TileShape, WarpShape,
253
+ InstructionShape, 4, FP8MathOperator>,
254
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
255
+ } else {
256
+ using TileShape = typename cutlass::gemm::GemmShape<32, 64, 128>;
257
+ using WarpShape = typename cutlass::gemm::GemmShape<16, 64, 64>;
258
+
259
+ return vllm::fallback_cutlass_gemm_caller<
260
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
261
+ InType, OutType, Epilogue, TileShape, WarpShape,
262
+ InstructionShape, 5, FP8MathOperator>,
263
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
264
+ }
265
+ }
266
+};
267
+
268
+struct sm89_config_M16 {
269
+ // M in [1, 16]
270
+ using WarpShape = typename cutlass::gemm::GemmShape<16, 64, 64>;
271
+ using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
272
+ using FP8MathOperator = typename cutlass::arch::OpMultiplyAddFastAccum;
273
+ static const int32_t MainLoopStages = 5;
274
+
275
+ template <typename InType, typename OutType,
276
+ template <typename, typename> typename Epilogue,
277
+ typename... EpilogueArgs>
278
+ static void dispatch(torch::Tensor& out, torch::Tensor const& a,
279
+ torch::Tensor const& b, EpilogueArgs&&... args) {
280
+ static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
281
+ TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
282
+
283
+ using FallbackGemm =
284
+ typename sm89_fallback_gemm<InType, OutType, Epilogue>::Cutlass2xGemm;
285
+
286
+ uint32_t const n = out.size(1);
287
+ uint32_t const np2 = next_pow_2(n);
288
+
289
+ if (np2 <= 8192) {
290
+ using TileShape = typename cutlass::gemm::GemmShape<16, 64, 128>;
291
+
292
+ return vllm::fallback_cutlass_gemm_caller<
293
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
294
+ InType, OutType, Epilogue, TileShape, WarpShape,
295
+ InstructionShape, MainLoopStages,
296
+ FP8MathOperator>,
297
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
298
+ } else if (np2 <= 24576) {
299
+ using TileShape = typename cutlass::gemm::GemmShape<16, 128, 64>;
300
+
301
+ return vllm::fallback_cutlass_gemm_caller<
302
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
303
+ InType, OutType, Epilogue, TileShape, WarpShape,
304
+ InstructionShape, MainLoopStages,
305
+ FP8MathOperator>,
306
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
307
+ } else {
308
+ using TileShape = typename cutlass::gemm::GemmShape<32, 64, 128>;
309
+
310
+ return vllm::fallback_cutlass_gemm_caller<
311
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
312
+ InType, OutType, Epilogue, TileShape, WarpShape,
313
+ InstructionShape, MainLoopStages,
314
+ FP8MathOperator>,
315
+ FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
316
+ }
317
+ }
318
+};
319
+
320
+template <typename InType, typename OutType,
321
+ template <typename, typename> typename Epilogue,
322
+ typename... EpilogueArgs>
323
+inline void cutlass_gemm_sm89_dispatch(torch::Tensor& out,
324
+ torch::Tensor const& a,
325
+ torch::Tensor const& b,
326
+ EpilogueArgs&&... args) {
327
+ static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
328
+ TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
329
+ TORCH_CHECK(b.dtype() == torch::kFloat8_e4m3fn);
330
+
331
+ uint32_t const m = a.size(0);
332
+ uint32_t const mp2 =
333
+ std::max(static_cast<uint32_t>(32), next_pow_2(m)); // next power of 2
334
+
335
+ if (mp2 <= 16) {
336
+ // M in [1, 16]
337
+ return sm89_config_M16::dispatch<InType, OutType, Epilogue>(
338
+ out, a, b, std::forward<EpilogueArgs>(args)...);
339
+ } else if (mp2 <= 32) {
340
+ // M in (16, 32]
341
+ return sm89_config_M32::dispatch<InType, OutType, Epilogue>(
342
+ out, a, b, std::forward<EpilogueArgs>(args)...);
343
+ } else if (mp2 <= 64) {
344
+ // M in (32, 64]
345
+ return sm89_config_M64::dispatch<InType, OutType, Epilogue>(
346
+ out, a, b, std::forward<EpilogueArgs>(args)...);
347
+ } else if (mp2 <= 128) {
348
+ // M in (64, 128]
349
+ return sm89_config_M128::dispatch<InType, OutType, Epilogue>(
350
+ out, a, b, std::forward<EpilogueArgs>(args)...);
351
+ } else if (mp2 <= 256) {
352
+ // M in (128, 256]
353
+ return sm89_config_M256::dispatch<InType, OutType, Epilogue>(
354
+ out, a, b, std::forward<EpilogueArgs>(args)...);
355
+ } else {
356
+ // M in (256, inf)
357
+ return sm89_config_default::dispatch<InType, OutType, Epilogue>(
358
+ out, a, b, std::forward<EpilogueArgs>(args)...);
359
+ }
360
+}
361
+
362
+} // namespace vllm
tests/kernels/test_cutlass.py
+2
-2
@@ -106,8 +106,8 @@ def cutlass_int8_gemm_helper(m: int,
106
assert torch.allclose(out, baseline, rtol=1e-1, atol=1e0)
107
108
109
-@pytest.mark.parametrize("m", [512, 222, 100, 33, 1])
110
-@pytest.mark.parametrize("n", [2048, 256, 1024])
109
+@pytest.mark.parametrize("m", [1, 16, 32, 64, 128, 256, 512, 222, 100, 33])
110
+@pytest.mark.parametrize("n", [2048, 4096, 8192, 16384, 24576, 256, 1024])
111
@pytest.mark.parametrize("k", [128, 496, 1024])
112
@pytest.mark.parametrize("per_act_token", [True, False])
113
@pytest.mark.parametrize("per_out_ch", [True, False])