@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 766435e660
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Kernel] Tuned FP8 Kernels for Ada Lovelace (#6677)

Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>

Varun Sundar Rabindranath committed Jul 29, 2024 at 11:42 UTC 766435e660a786933392eb8ef0a873bc38cf0c8b
6 files changed +877 -490
benchmarks/cutlass_benchmarks/w8a8_benchmarks.py
+1 -1
@@ -13,7 +13,7 @@ from weight_shapes import WEIGHT_SHAPES
13 from vllm import _custom_ops as ops
14 from vllm.utils import FlexibleArgumentParser
15
16 -DEFAULT_MODELS = list(WEIGHT_SHAPES.keys())[1:]
16 +DEFAULT_MODELS = list(WEIGHT_SHAPES.keys())
17 DEFAULT_BATCH_SIZES = [1, 16, 32, 64, 128, 256, 512]
18 DEFAULT_TP_SIZES = [1]
19
csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cu
+33 -487
@@ -1,470 +1,16 @@
1 #include <stddef.h>
2 #include <torch/all.h>
3 -
4 -#include <ATen/cuda/CUDAContext.h>
5 -
6 -// clang-format will break include orders
7 -// clang-format off
8 -#include "cute/tensor.hpp"
9 -#include "cute/atom/mma_atom.hpp"
10 -#include "cutlass/numeric_types.h"
11 -
12 -#include "cutlass/util/device_memory.h"
13 -
3 #include "cutlass/cutlass.h"
15 -#include "cutlass/gemm_coord.h"
16 -#include "cutlass/arch/mma_sm75.h"
17 -#include "cutlass/arch/arch.h"
18 -#include "cutlass/arch/mma.h"
19 -#include "cutlass/gemm/device/gemm.h"
20 -#include "cutlass/gemm/device/gemm_universal_adapter.h"
4
22 -#include "cutlass/epilogue/threadblock/fusion/visitors.hpp"
23 -#include "cutlass/gemm/kernel/default_gemm_universal_with_visitor.h"
24 -
25 -#include "broadcast_load_epilogue_c2x.hpp"
26 -#include "common.hpp"
27 -// clang-format on
28 -
29 -using namespace cute;
5 +#include "scaled_mm_c2x.cuh"
6 +#include "scaled_mm_c2x_sm80_dispatch.cuh"
7 +#include "scaled_mm_c2x_sm89_dispatch.cuh"
8
9 /*
10 This file defines quantized GEMM operations using the CUTLASS 2.x API, for
11 NVIDIA GPUs with SM versions prior to sm90 (Hopper).
34 -
35 - Epilogue functions can be defined to post-process the output before it is
36 - written to GPU memory.
37 - Epilogues must contain a public type named EVTCompute of type Sm80EVT,
38 - as well as a static prepare_args function that constructs an
39 - EVTCompute::Arguments struct.
12 */
13
42 -namespace {
43 -
44 -// Wrappers for the GEMM kernel that is used to guard against compilation on
45 -// architectures that will never use the kernel. The purpose of this is to
46 -// reduce the size of the compiled binary.
47 -// __CUDA_ARCH__ is not defined in host code, so this lets us smuggle the ifdef
48 -// into code that will be executed on the device where it is defined.
49 -template <typename Kernel>
50 -struct enable_sm75_to_sm80 : Kernel {
51 - template <typename... Args>
52 - CUTLASS_DEVICE static void invoke(Args&&... args) {
53 -#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 750 && __CUDA_ARCH__ < 800
54 - Kernel::invoke(std::forward<Args>(args)...);
55 -#endif
56 - }
57 -};
58 -
59 -template <typename Kernel>
60 -struct enable_sm80_to_sm89 : Kernel {
61 - template <typename... Args>
62 - CUTLASS_DEVICE static void invoke(Args&&... args) {
63 -#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 800 && __CUDA_ARCH__ < 890
64 - Kernel::invoke(std::forward<Args>(args)...);
65 -#endif
66 - }
67 -};
68 -
69 -template <typename Kernel>
70 -struct enable_sm89_to_sm90 : Kernel {
71 - template <typename... Args>
72 - CUTLASS_DEVICE static void invoke(Args&&... args) {
73 -#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 890 && __CUDA_ARCH__ < 900
74 - Kernel::invoke(std::forward<Args>(args)...);
75 -#endif
76 - }
77 -};
78 -
79 -/*
80 - * This class provides the common ScaleA and ScaleB descriptors for the
81 - * ScaledEpilogue and ScaledEpilogueBias classes.
82 - */
83 -template <typename ElementD, typename OutputTileThreadMap>
84 -struct ScaledEpilogueBase {
85 - protected:
86 - using Accum = cutlass::epilogue::threadblock::VisitorAccFetch;
87 -
88 - using ScaleA = cutlass::epilogue::threadblock::VisitorColOrScalarBroadcast<
89 - OutputTileThreadMap, float, Stride<Int<1>, Int<0>, Int<0>>>;
90 -
91 - using ScaleB = cutlass::epilogue::threadblock::VisitorRowOrScalarBroadcast<
92 - OutputTileThreadMap, float, Stride<Int<0>, Int<1>, Int<0>>>;
93 -};
94 -
95 -/*
96 - This epilogue function defines a quantized GEMM operation similar to
97 - torch._scaled_mm.
98 -
99 - A and B may be both either int8 or fp8_e4m3. A can be quantized per-tensor or
100 - per-row. B can be quantized per-tensor or per-column.
101 - Any combination of per-tensor and per-row or column is supported.
102 - A and B must have symmetric quantization (zero point == 0).
103 -
104 - So the GEMM operation is D = (a_scales * A) (b_scales * B), where the
105 - scales are applied elementwise with numpy-style broadcasting.
106 -
107 - ScaleA and ScaleB define the epilogue functions that apply the scales for
108 - the A and B operands respectively. These scales may be either per-tensor or
109 - per row or column.
110 -*/
111 -template <typename ElementD, typename OutputTileThreadMap>
112 -struct ScaledEpilogue
113 - : private ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
114 - private:
115 - using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
116 - using Accum = typename SUPER::Accum;
117 - using ScaleA = typename SUPER::ScaleA;
118 - using ScaleB = typename SUPER::ScaleB;
119 -
120 - using Compute0 = cutlass::epilogue::threadblock::VisitorCompute<
121 - cutlass::multiplies, float, float,
122 - cutlass::FloatRoundStyle::round_to_nearest>;
123 -
124 - using EVTCompute0 =
125 - cutlass::epilogue::threadblock::Sm80EVT<Compute0, ScaleB, Accum>;
126 -
127 - using Compute1 = cutlass::epilogue::threadblock::VisitorCompute<
128 - cutlass::multiplies, ElementD, float,
129 - cutlass::FloatRoundStyle::round_to_nearest>;
130 -
131 - public:
132 - using EVTCompute =
133 - cutlass::epilogue::threadblock::Sm80EVT<Compute1, ScaleA, EVTCompute0>;
134 - using ArgumentType = typename EVTCompute::Arguments;
135 -
136 - static ArgumentType prepare_args(torch::Tensor const& a_scales,
137 - torch::Tensor const& b_scales) {
138 - using ScaleAArgs = typename ScaleA::Arguments;
139 - using ScaleBArgs = typename ScaleB::Arguments;
140 -
141 - ScaleBArgs b_args{b_scales.data_ptr<float>(), b_scales.numel() != 1, {}};
142 - ScaleAArgs a_args{a_scales.data_ptr<float>(), a_scales.numel() != 1, {}};
143 -
144 - typename EVTCompute0::Arguments evt0_compute_args{b_args};
145 -
146 - typename EVTCompute::Arguments evt_compute_args{a_args, evt0_compute_args};
147 - return evt_compute_args;
148 - }
149 -};
150 -
151 -template <typename ElementD, typename OutputTileThreadMap>
152 -struct ScaledEpilogueBias
153 - : private ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
154 - private:
155 - using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
156 - using Accum = typename SUPER::Accum;
157 - using ScaleA = typename SUPER::ScaleA;
158 - using ScaleB = typename SUPER::ScaleB;
159 -
160 - using Compute0 = cutlass::epilogue::threadblock::VisitorCompute<
161 - cutlass::multiplies, float, float,
162 - cutlass::FloatRoundStyle::round_to_nearest>;
163 -
164 - using EVTCompute0 =
165 - cutlass::epilogue::threadblock::Sm80EVT<Compute0, ScaleB, Accum>;
166 -
167 - using Compute1 = cutlass::epilogue::threadblock::VisitorCompute<
168 - cutlass::multiply_add, ElementD, float,
169 - cutlass::FloatRoundStyle::round_to_nearest>;
170 -
171 - using Bias = cutlass::epilogue::threadblock::VisitorRowBroadcast<
172 - OutputTileThreadMap, ElementD, Stride<Int<0>, Int<1>, Int<0>>>;
173 -
174 - public:
175 - using EVTCompute = cutlass::epilogue::threadblock::Sm80EVT<Compute1, ScaleA,
176 - EVTCompute0, Bias>;
177 - using ArgumentType = typename EVTCompute::Arguments;
178 -
179 - static ArgumentType prepare_args(torch::Tensor const& a_scales,
180 - torch::Tensor const& b_scales,
181 - torch::Tensor const& bias) {
182 - using ScaleAArgs = typename ScaleA::Arguments;
183 - using ScaleBArgs = typename ScaleB::Arguments;
184 - using BiasArgs = typename Bias::Arguments;
185 -
186 - ScaleBArgs b_args{b_scales.data_ptr<float>(), b_scales.numel() != 1, {}};
187 - ScaleAArgs a_args{a_scales.data_ptr<float>(), a_scales.numel() != 1, {}};
188 - BiasArgs bias_args{static_cast<ElementD*>(bias.data_ptr()), {}};
189 -
190 - typename EVTCompute0::Arguments evt0_compute_args{b_args};
191 -
192 - typename EVTCompute::Arguments evt_compute_args{a_args, evt0_compute_args,
193 - bias_args};
194 - return evt_compute_args;
195 - }
196 -};
197 -
198 -template <typename Arch, template <typename> typename ArchGuard,
199 - typename ElementAB_, typename ElementD_,
200 - template <typename, typename> typename Epilogue_, typename TileShape,
201 - typename WarpShape, typename InstructionShape, int32_t MainLoopStages>
202 -struct cutlass_2x_gemm {
203 - using ElementAB = ElementAB_;
204 - using ElementD = ElementD_;
205 -
206 - using ElementAcc =
207 - typename std::conditional<std::is_same_v<ElementAB, int8_t>, int32_t,
208 - float>::type;
209 -
210 - using Operator =
211 - typename std::conditional<std::is_same_v<ElementAB, int8_t>,
212 - cutlass::arch::OpMultiplyAddSaturate,
213 - cutlass::arch::OpMultiplyAdd>::type;
214 -
215 - using OutputTileThreadMap =
216 - cutlass::epilogue::threadblock::OutputTileThreadLayout<
217 - TileShape, WarpShape, float, 4, 1 /* epilogue stages */
218 - >;
219 -
220 - using Epilogue = Epilogue_<ElementD, OutputTileThreadMap>;
221 - using EVTCompute = typename Epilogue::EVTCompute;
222 -
223 - using D = cutlass::epilogue::threadblock::VisitorAuxStore<
224 - OutputTileThreadMap, ElementD, cutlass::FloatRoundStyle::round_to_nearest,
225 - Stride<int64_t, Int<1>, Int<0>>>;
226 -
227 - using EVTD = cutlass::epilogue::threadblock::Sm80EVT<D, EVTCompute>;
228 -
229 - // clang-format off
230 - using RowMajor = typename cutlass::layout::RowMajor;
231 - using ColumnMajor = typename cutlass::layout::ColumnMajor;
232 - using KernelType =
233 - ArchGuard<typename cutlass::gemm::kernel::DefaultGemmWithVisitor<
234 - ElementAB, RowMajor, cutlass::ComplexTransform::kNone, 16,
235 - ElementAB, ColumnMajor, cutlass::ComplexTransform::kNone, 16,
236 - float, cutlass::layout::RowMajor, 4,
237 - ElementAcc, float, cutlass::arch::OpClassTensorOp,
238 - Arch,
239 - TileShape, WarpShape, InstructionShape,
240 - EVTD,
241 - cutlass::gemm::threadblock::ThreadblockSwizzleStreamK,
242 - MainLoopStages, Operator,
243 - 1 /* epilogue stages */
244 - >::GemmKernel>;
245 - // clang-format on
246 -
247 - using Op = cutlass::gemm::device::GemmUniversalAdapter<KernelType>;
248 -};
249 -
250 -template <typename Gemm, typename... EpilogueArgs>
251 -void cutlass_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
252 - torch::Tensor const& b,
253 - EpilogueArgs&&... epilogue_params) {
254 - using ElementAB = typename Gemm::ElementAB;
255 - using ElementD = typename Gemm::ElementD;
256 -
257 - int32_t m = a.size(0);
258 - int32_t n = b.size(1);
259 - int32_t k = a.size(1);
260 - cutlass::gemm::GemmCoord problem_size{m, n, k};
261 -
262 - int64_t lda = a.stride(0);
263 - int64_t ldb = b.stride(1);
264 - int64_t ldc = out.stride(0);
265 -
266 - using StrideC = Stride<int64_t, Int<1>, Int<0>>;
267 - StrideC c_stride{ldc, Int<1>{}, Int<0>{}};
268 -
269 - auto a_ptr = static_cast<ElementAB const*>(a.data_ptr());
270 - auto b_ptr = static_cast<ElementAB const*>(b.data_ptr());
271 - auto c_ptr = static_cast<ElementD*>(out.data_ptr());
272 -
273 - typename Gemm::D::Arguments d_args{c_ptr, c_stride};
274 -
275 - using Epilogue = typename Gemm::Epilogue;
276 - auto evt_args =
277 - Epilogue::prepare_args(std::forward<EpilogueArgs>(epilogue_params)...);
278 -
279 - typename Gemm::EVTD::Arguments epilogue_args{
280 - evt_args,
281 - d_args,
282 - };
283 -
284 - typename Gemm::Op::Arguments args{
285 - cutlass::gemm::GemmUniversalMode::kGemmSplitKParallel, // universal mode
286 - problem_size, // problem size
287 - 1, // batch count
288 - epilogue_args,
289 - a_ptr,
290 - b_ptr,
291 - nullptr,
292 - nullptr,
293 - 0,
294 - 0,
295 - 0,
296 - 0,
297 - lda,
298 - ldb,
299 - ldc,
300 - ldc};
301 -
302 - // Launch the CUTLASS GEMM kernel.
303 - typename Gemm::Op gemm_op;
304 - size_t workspace_size = gemm_op.get_workspace_size(args);
305 - cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
306 -
307 - auto stream = at::cuda::getCurrentCUDAStream(a.get_device());
308 -
309 - CUTLASS_CHECK(gemm_op.can_implement(args));
310 - cutlass::Status status = gemm_op(args, workspace.get(), stream);
311 - CUTLASS_CHECK(status);
312 -}
313 -
314 -template <typename Gemm, typename FallbackGemm, typename... EpilogueArgs>
315 -void fallback_cutlass_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
316 - torch::Tensor const& b,
317 - EpilogueArgs&&... args) {
318 - // In some cases, the GPU isn't able to accommodate the
319 - // shared memory requirements of the Gemm. In such cases, use
320 - // the FallbackGemm instead.
321 - static const int max_shared_mem_per_block_opt_in =
322 - get_cuda_max_shared_memory_per_block_opt_in(0);
323 -
324 - size_t const gemm_shared_mem_size =
325 - sizeof(typename Gemm::KernelType::SharedStorage);
326 - size_t const fallback_gemm_shared_mem_size =
327 - sizeof(typename FallbackGemm::KernelType::SharedStorage);
328 -
329 - if (gemm_shared_mem_size <= max_shared_mem_per_block_opt_in) {
330 - return cutlass_gemm_caller<Gemm>(out, a, b,
331 - std::forward<EpilogueArgs>(args)...);
332 - } else {
333 - TORCH_CHECK(fallback_gemm_shared_mem_size <=
334 - max_shared_mem_per_block_opt_in);
335 - return cutlass_gemm_caller<FallbackGemm>(
336 - out, a, b, std::forward<EpilogueArgs>(args)...);
337 - }
338 -}
339 -
340 -template <typename InType, typename OutType,
341 - template <typename, typename> typename Epilogue>
342 -struct sm80_config_default {
343 - // This config is used in 2 cases,
344 - // - M in (128, inf)
345 - // - M in (64, 128] and N >= 8192
346 - // Shared Memory required by this Gemm - 81920 bytes
347 - static_assert(std::is_same<InType, int8_t>());
348 - using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
349 - using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
350 - using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
351 - using Cutlass2xGemm =
352 - cutlass_2x_gemm<cutlass::arch::Sm80, enable_sm80_to_sm89, InType, OutType,
353 - Epilogue, TileShape, WarpShape, InstructionShape, 5>;
354 -};
355 -
356 -template <typename InType, typename OutType,
357 - template <typename, typename> typename Epilogue>
358 -struct sm80_config_M64 {
359 - // This config is used in 2 cases,
360 - // - M in (32, 64]
361 - // - M in (64, 128] and N < 8192
362 - // Shared Memory required by this Gemm - 122880 bytes
363 - static_assert(std::is_same<InType, int8_t>());
364 - using TileShape = typename cutlass::gemm::GemmShape<64, 128, 128>;
365 - using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
366 - using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
367 - using Cutlass2xGemm =
368 - cutlass_2x_gemm<cutlass::arch::Sm80, enable_sm80_to_sm89, InType, OutType,
369 - Epilogue, TileShape, WarpShape, InstructionShape, 5>;
370 -};
371 -
372 -template <typename InType, typename OutType,
373 - template <typename, typename> typename Epilogue>
374 -struct sm80_config_M32 {
375 - // M in (16, 32]
376 - // Shared Memory required by this Gemm - 61440 bytes
377 - static_assert(std::is_same<InType, int8_t>());
378 - using TileShape = typename cutlass::gemm::GemmShape<32, 64, 128>;
379 - using WarpShape = typename cutlass::gemm::GemmShape<32, 64, 64>;
380 - using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
381 - using Cutlass2xGemm =
382 - cutlass_2x_gemm<cutlass::arch::Sm80, enable_sm80_to_sm89, InType, OutType,
383 - Epilogue, TileShape, WarpShape, InstructionShape, 5>;
384 -};
385 -
386 -template <typename InType, typename OutType,
387 - template <typename, typename> typename Epilogue>
388 -struct sm80_config_M16 {
389 - // M in [1, 16]
390 - // Shared Memory required by this Gemm - 51200 bytes
391 - static_assert(std::is_same<InType, int8_t>());
392 - using TileShape = typename cutlass::gemm::GemmShape<16, 64, 128>;
393 - using WarpShape = typename cutlass::gemm::GemmShape<16, 64, 64>;
394 - using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
395 - using Cutlass2xGemm =
396 - cutlass_2x_gemm<cutlass::arch::Sm80, enable_sm80_to_sm89, InType, OutType,
397 - Epilogue, TileShape, WarpShape, InstructionShape, 5>;
398 -};
399 -
400 -} // namespace
401 -
402 -template <typename InType, typename OutType,
403 - template <typename, typename> typename Epilogue,
404 - typename... EpilogueArgs>
405 -void cutlass_gemm_sm80_dispatch(torch::Tensor& out, torch::Tensor const& a,
406 - torch::Tensor const& b,
407 - EpilogueArgs&&... args) {
408 - static_assert(std::is_same<InType, int8_t>());
409 - TORCH_CHECK(a.dtype() == torch::kInt8);
410 - TORCH_CHECK(b.dtype() == torch::kInt8);
411 -
412 - using Cutlass2xGemmDefault =
413 - typename sm80_config_default<InType, OutType, Epilogue>::Cutlass2xGemm;
414 - using Cutlass2xGemmM128BigN =
415 - typename sm80_config_default<InType, OutType, Epilogue>::Cutlass2xGemm;
416 - using Cutlass2xGemmM128SmallN =
417 - typename sm80_config_M64<InType, OutType, Epilogue>::Cutlass2xGemm;
418 - using Cutlass2xGemmM64 =
419 - typename sm80_config_M64<InType, OutType, Epilogue>::Cutlass2xGemm;
420 - using Cutlass2xGemmM32 =
421 - typename sm80_config_M32<InType, OutType, Epilogue>::Cutlass2xGemm;
422 - using Cutlass2xGemmM16 =
423 - typename sm80_config_M16<InType, OutType, Epilogue>::Cutlass2xGemm;
424 -
425 - // Due to shared memory requirements, some Gemms may fail to run on some
426 - // GPUs. As the name indicates, the Fallback Gemm is used as an alternative
427 - // in such cases.
428 - // sm80_config_M16 has the least shared-memory requirement. However,
429 - // based on some profiling, we select sm80_config_M32 as a better alternative
430 - // performance wise.
431 - using FallbackGemm =
432 - typename sm80_config_M32<InType, OutType, Epilogue>::Cutlass2xGemm;
433 -
434 - uint32_t const m = a.size(0);
435 - uint32_t const mp2 =
436 - std::max(static_cast<uint32_t>(16), next_pow_2(m)); // next power of 2
437 - if (mp2 <= 16) {
438 - // M in [1, 16]
439 - return fallback_cutlass_gemm_caller<Cutlass2xGemmM16, FallbackGemm>(
440 - out, a, b, std::forward<EpilogueArgs>(args)...);
441 - } else if (mp2 <= 32) {
442 - // M in (16, 32]
443 - return fallback_cutlass_gemm_caller<Cutlass2xGemmM32, FallbackGemm>(
444 - out, a, b, std::forward<EpilogueArgs>(args)...);
445 - } else if (mp2 <= 64) {
446 - // M in (32, 64]
447 - return fallback_cutlass_gemm_caller<Cutlass2xGemmM64, FallbackGemm>(
448 - out, a, b, std::forward<EpilogueArgs>(args)...);
449 - } else if (mp2 <= 128) {
450 - // M in (64, 128]
451 - uint32_t const n = out.size(1);
452 - bool const small_n = n < 8192;
453 - if (small_n) {
454 - return fallback_cutlass_gemm_caller<Cutlass2xGemmM128SmallN,
455 - FallbackGemm>(
456 - out, a, b, std::forward<EpilogueArgs>(args)...);
457 - } else {
458 - return fallback_cutlass_gemm_caller<Cutlass2xGemmM128BigN, FallbackGemm>(
459 - out, a, b, std::forward<EpilogueArgs>(args)...);
460 - }
461 - } else {
462 - // M in (128, inf)
463 - return fallback_cutlass_gemm_caller<Cutlass2xGemmDefault, FallbackGemm>(
464 - out, a, b, std::forward<EpilogueArgs>(args)...);
465 - }
466 -}
467 -
14 template <template <typename, typename> typename Epilogue,
15 typename... EpilogueArgs>
16 void cutlass_scaled_mm_sm75_epilogue(torch::Tensor& out, torch::Tensor const& a,
@@ -478,14 +24,15 @@ void cutlass_scaled_mm_sm75_epilogue(torch::Tensor& out, torch::Tensor const& a,
24 using InstructionShape = typename cutlass::gemm::GemmShape<8, 8, 16>;
25
26 if (out.dtype() == torch::kBFloat16) {
481 - return cutlass_gemm_caller<cutlass_2x_gemm<
482 - cutlass::arch::Sm75, enable_sm75_to_sm80, int8_t, cutlass::bfloat16_t,
483 - Epilogue, TileShape, WarpShape, InstructionShape, 2>>(
27 + return vllm::cutlass_gemm_caller<
28 + vllm::cutlass_2x_gemm<cutlass::arch::Sm75, vllm::enable_sm75_to_sm80,
29 + int8_t, cutlass::bfloat16_t, Epilogue, TileShape,
30 + WarpShape, InstructionShape, 2>>(
31 out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
32 } else {
33 TORCH_CHECK(out.dtype() == torch::kFloat16);
487 - return cutlass_gemm_caller<cutlass_2x_gemm<
488 - cutlass::arch::Sm75, enable_sm75_to_sm80, int8_t, cutlass::half_t,
34 + return vllm::cutlass_gemm_caller<vllm::cutlass_2x_gemm<
35 + cutlass::arch::Sm75, vllm::enable_sm75_to_sm80, int8_t, cutlass::half_t,
36 Epilogue, TileShape, WarpShape, InstructionShape, 2>>(
37 out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
38 }
@@ -501,11 +48,11 @@ void cutlass_scaled_mm_sm75(torch::Tensor& out, torch::Tensor const& a,
48 if (bias) {
49 TORCH_CHECK(bias->dtype() == out.dtype(),
50 "currently bias dtype must match output dtype ", out.dtype());
504 - return cutlass_scaled_mm_sm75_epilogue<ScaledEpilogueBias>(
51 + return cutlass_scaled_mm_sm75_epilogue<vllm::ScaledEpilogueBias>(
52 out, a, b, a_scales, b_scales, *bias);
53 } else {
507 - return cutlass_scaled_mm_sm75_epilogue<ScaledEpilogue>(out, a, b, a_scales,
508 - b_scales);
54 + return cutlass_scaled_mm_sm75_epilogue<vllm::ScaledEpilogue>(
55 + out, a, b, a_scales, b_scales);
56 }
57 }
58
@@ -518,11 +65,12 @@ void cutlass_scaled_mm_sm80_epilogue(torch::Tensor& out, torch::Tensor const& a,
65 TORCH_CHECK(b.dtype() == torch::kInt8);
66
67 if (out.dtype() == torch::kBFloat16) {
521 - return cutlass_gemm_sm80_dispatch<int8_t, cutlass::bfloat16_t, Epilogue>(
68 + return vllm::cutlass_gemm_sm80_dispatch<int8_t, cutlass::bfloat16_t,
69 + Epilogue>(
70 out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
71 } else {
72 TORCH_CHECK(out.dtype() == torch::kFloat16);
525 - return cutlass_gemm_sm80_dispatch<int8_t, cutlass::half_t, Epilogue>(
73 + return vllm::cutlass_gemm_sm80_dispatch<int8_t, cutlass::half_t, Epilogue>(
74 out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
75 }
76 }
@@ -537,11 +85,11 @@ void cutlass_scaled_mm_sm80(torch::Tensor& out, torch::Tensor const& a,
85 if (bias) {
86 TORCH_CHECK(bias->dtype() == out.dtype(),
87 "currently bias dtype must match output dtype ", out.dtype());
540 - return cutlass_scaled_mm_sm80_epilogue<ScaledEpilogueBias>(
88 + return cutlass_scaled_mm_sm80_epilogue<vllm::ScaledEpilogueBias>(
89 out, a, b, a_scales, b_scales, *bias);
90 } else {
543 - return cutlass_scaled_mm_sm80_epilogue<ScaledEpilogue>(out, a, b, a_scales,
544 - b_scales);
91 + return cutlass_scaled_mm_sm80_epilogue<vllm::ScaledEpilogue>(
92 + out, a, b, a_scales, b_scales);
93 }
94 }
95
@@ -558,15 +106,17 @@ void cutlass_scaled_mm_sm89_epilogue(torch::Tensor& out, torch::Tensor const& a,
106 TORCH_CHECK(b.dtype() == torch::kInt8);
107
108 if (out.dtype() == torch::kBFloat16) {
561 - return cutlass_gemm_caller<cutlass_2x_gemm<
562 - cutlass::arch::Sm89, enable_sm89_to_sm90, int8_t, cutlass::bfloat16_t,
563 - Epilogue, TileShape, WarpShape, InstructionShape, 5>>(
109 + return vllm::cutlass_gemm_caller<
110 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
111 + int8_t, cutlass::bfloat16_t, Epilogue,
112 + TileShape, WarpShape, InstructionShape, 5>>(
113 out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
114 } else {
115 assert(out.dtype() == torch::kFloat16);
567 - return cutlass_gemm_caller<cutlass_2x_gemm<
568 - cutlass::arch::Sm89, enable_sm89_to_sm90, int8_t, cutlass::half_t,
569 - Epilogue, TileShape, WarpShape, InstructionShape, 5>>(
116 + return vllm::cutlass_gemm_caller<
117 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
118 + int8_t, cutlass::half_t, Epilogue, TileShape,
119 + WarpShape, InstructionShape, 5>>(
120 out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
121 }
122 } else {
@@ -574,17 +124,13 @@ void cutlass_scaled_mm_sm89_epilogue(torch::Tensor& out, torch::Tensor const& a,
124 TORCH_CHECK(b.dtype() == torch::kFloat8_e4m3fn);
125
126 if (out.dtype() == torch::kBFloat16) {
577 - return cutlass_gemm_caller<
578 - cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
579 - cutlass::float_e4m3_t, cutlass::bfloat16_t, Epilogue,
580 - TileShape, WarpShape, InstructionShape, 5>>(
127 + return vllm::cutlass_gemm_sm89_dispatch<cutlass::float_e4m3_t,
128 + cutlass::bfloat16_t, Epilogue>(
129 out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
130 } else {
131 TORCH_CHECK(out.dtype() == torch::kFloat16);
584 - return cutlass_gemm_caller<
585 - cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
586 - cutlass::float_e4m3_t, cutlass::half_t, Epilogue,
587 - TileShape, WarpShape, InstructionShape, 5>>(
132 + return vllm::cutlass_gemm_sm89_dispatch<cutlass::float_e4m3_t,
133 + cutlass::half_t, Epilogue>(
134 out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
135 }
136 }
@@ -600,10 +146,10 @@ void cutlass_scaled_mm_sm89(torch::Tensor& out, torch::Tensor const& a,
146 if (bias) {
147 TORCH_CHECK(bias->dtype() == out.dtype(),
148 "currently bias dtype must match output dtype ", out.dtype());
603 - return cutlass_scaled_mm_sm89_epilogue<ScaledEpilogueBias>(
149 + return cutlass_scaled_mm_sm89_epilogue<vllm::ScaledEpilogueBias>(
150 out, a, b, a_scales, b_scales, *bias);
151 } else {
606 - return cutlass_scaled_mm_sm89_epilogue<ScaledEpilogue>(out, a, b, a_scales,
607 - b_scales);
152 + return cutlass_scaled_mm_sm89_epilogue<vllm::ScaledEpilogue>(
153 + out, a, b, a_scales, b_scales);
154 }
155 }
csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cuh new
+340
@@ -0,0 +1,340 @@
1 +#pragma once
2 +#include <stddef.h>
3 +#include <torch/all.h>
4 +
5 +#include <ATen/cuda/CUDAContext.h>
6 +
7 +// clang-format will break include orders
8 +// clang-format off
9 +#include "cute/tensor.hpp"
10 +#include "cute/atom/mma_atom.hpp"
11 +#include "cutlass/numeric_types.h"
12 +
13 +#include "cutlass/util/device_memory.h"
14 +
15 +#include "cutlass/cutlass.h"
16 +#include "cutlass/gemm_coord.h"
17 +#include "cutlass/arch/mma_sm75.h"
18 +#include "cutlass/arch/arch.h"
19 +#include "cutlass/arch/mma.h"
20 +#include "cutlass/gemm/device/gemm.h"
21 +#include "cutlass/gemm/device/gemm_universal_adapter.h"
22 +
23 +#include "cutlass/epilogue/threadblock/fusion/visitors.hpp"
24 +#include "cutlass/gemm/kernel/default_gemm_universal_with_visitor.h"
25 +
26 +#include "broadcast_load_epilogue_c2x.hpp"
27 +#include "common.hpp"
28 +// clang-format on
29 +
30 +using namespace cute;
31 +
32 +/*
33 + Epilogue functions can be defined to post-process the output before it is
34 + written to GPU memory.
35 + Epilogues must contain a public type named EVTCompute of type Sm80EVT,
36 + as well as a static prepare_args function that constructs an
37 + EVTCompute::Arguments struct.
38 +*/
39 +
40 +namespace vllm {
41 +
42 +// Wrappers for the GEMM kernel that is used to guard against compilation on
43 +// architectures that will never use the kernel. The purpose of this is to
44 +// reduce the size of the compiled binary.
45 +// __CUDA_ARCH__ is not defined in host code, so this lets us smuggle the ifdef
46 +// into code that will be executed on the device where it is defined.
47 +template <typename Kernel>
48 +struct enable_sm75_to_sm80 : Kernel {
49 + template <typename... Args>
50 + CUTLASS_DEVICE static void invoke(Args&&... args) {
51 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 750 && __CUDA_ARCH__ < 800
52 + Kernel::invoke(std::forward<Args>(args)...);
53 +#endif
54 + }
55 +};
56 +
57 +template <typename Kernel>
58 +struct enable_sm80_to_sm89 : Kernel {
59 + template <typename... Args>
60 + CUTLASS_DEVICE static void invoke(Args&&... args) {
61 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 800 && __CUDA_ARCH__ < 890
62 + Kernel::invoke(std::forward<Args>(args)...);
63 +#endif
64 + }
65 +};
66 +
67 +template <typename Kernel>
68 +struct enable_sm89_to_sm90 : Kernel {
69 + template <typename... Args>
70 + CUTLASS_DEVICE static void invoke(Args&&... args) {
71 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 890 && __CUDA_ARCH__ < 900
72 + Kernel::invoke(std::forward<Args>(args)...);
73 +#endif
74 + }
75 +};
76 +
77 +/*
78 + * This class provides the common ScaleA and ScaleB descriptors for the
79 + * ScaledEpilogue and ScaledEpilogueBias classes.
80 + */
81 +template <typename ElementD, typename OutputTileThreadMap>
82 +struct ScaledEpilogueBase {
83 + protected:
84 + using Accum = cutlass::epilogue::threadblock::VisitorAccFetch;
85 +
86 + using ScaleA = cutlass::epilogue::threadblock::VisitorColOrScalarBroadcast<
87 + OutputTileThreadMap, float, Stride<Int<1>, Int<0>, Int<0>>>;
88 +
89 + using ScaleB = cutlass::epilogue::threadblock::VisitorRowOrScalarBroadcast<
90 + OutputTileThreadMap, float, Stride<Int<0>, Int<1>, Int<0>>>;
91 +};
92 +
93 +/*
94 + This epilogue function defines a quantized GEMM operation similar to
95 + torch._scaled_mm.
96 +
97 + A and B may be both either int8 or fp8_e4m3. A can be quantized per-tensor or
98 + per-row. B can be quantized per-tensor or per-column.
99 + Any combination of per-tensor and per-row or column is supported.
100 + A and B must have symmetric quantization (zero point == 0).
101 +
102 + So the GEMM operation is D = (a_scales * A) (b_scales * B), where the
103 + scales are applied elementwise with numpy-style broadcasting.
104 +
105 + ScaleA and ScaleB define the epilogue functions that apply the scales for
106 + the A and B operands respectively. These scales may be either per-tensor or
107 + per row or column.
108 +*/
109 +template <typename ElementD, typename OutputTileThreadMap>
110 +struct ScaledEpilogue
111 + : private ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
112 + private:
113 + using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
114 + using Accum = typename SUPER::Accum;
115 + using ScaleA = typename SUPER::ScaleA;
116 + using ScaleB = typename SUPER::ScaleB;
117 +
118 + using Compute0 = cutlass::epilogue::threadblock::VisitorCompute<
119 + cutlass::multiplies, float, float,
120 + cutlass::FloatRoundStyle::round_to_nearest>;
121 +
122 + using EVTCompute0 =
123 + cutlass::epilogue::threadblock::Sm80EVT<Compute0, ScaleB, Accum>;
124 +
125 + using Compute1 = cutlass::epilogue::threadblock::VisitorCompute<
126 + cutlass::multiplies, ElementD, float,
127 + cutlass::FloatRoundStyle::round_to_nearest>;
128 +
129 + public:
130 + using EVTCompute =
131 + cutlass::epilogue::threadblock::Sm80EVT<Compute1, ScaleA, EVTCompute0>;
132 + using ArgumentType = typename EVTCompute::Arguments;
133 +
134 + static ArgumentType prepare_args(torch::Tensor const& a_scales,
135 + torch::Tensor const& b_scales) {
136 + using ScaleAArgs = typename ScaleA::Arguments;
137 + using ScaleBArgs = typename ScaleB::Arguments;
138 +
139 + ScaleBArgs b_args{b_scales.data_ptr<float>(), b_scales.numel() != 1, {}};
140 + ScaleAArgs a_args{a_scales.data_ptr<float>(), a_scales.numel() != 1, {}};
141 +
142 + typename EVTCompute0::Arguments evt0_compute_args{b_args};
143 +
144 + typename EVTCompute::Arguments evt_compute_args{a_args, evt0_compute_args};
145 + return evt_compute_args;
146 + }
147 +};
148 +
149 +template <typename ElementD, typename OutputTileThreadMap>
150 +struct ScaledEpilogueBias
151 + : private ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
152 + private:
153 + using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
154 + using Accum = typename SUPER::Accum;
155 + using ScaleA = typename SUPER::ScaleA;
156 + using ScaleB = typename SUPER::ScaleB;
157 +
158 + using Compute0 = cutlass::epilogue::threadblock::VisitorCompute<
159 + cutlass::multiplies, float, float,
160 + cutlass::FloatRoundStyle::round_to_nearest>;
161 +
162 + using EVTCompute0 =
163 + cutlass::epilogue::threadblock::Sm80EVT<Compute0, ScaleB, Accum>;
164 +
165 + using Compute1 = cutlass::epilogue::threadblock::VisitorCompute<
166 + cutlass::multiply_add, ElementD, float,
167 + cutlass::FloatRoundStyle::round_to_nearest>;
168 +
169 + using Bias = cutlass::epilogue::threadblock::VisitorRowBroadcast<
170 + OutputTileThreadMap, ElementD, Stride<Int<0>, Int<1>, Int<0>>>;
171 +
172 + public:
173 + using EVTCompute = cutlass::epilogue::threadblock::Sm80EVT<Compute1, ScaleA,
174 + EVTCompute0, Bias>;
175 + using ArgumentType = typename EVTCompute::Arguments;
176 +
177 + static ArgumentType prepare_args(torch::Tensor const& a_scales,
178 + torch::Tensor const& b_scales,
179 + torch::Tensor const& bias) {
180 + using ScaleAArgs = typename ScaleA::Arguments;
181 + using ScaleBArgs = typename ScaleB::Arguments;
182 + using BiasArgs = typename Bias::Arguments;
183 +
184 + ScaleBArgs b_args{b_scales.data_ptr<float>(), b_scales.numel() != 1, {}};
185 + ScaleAArgs a_args{a_scales.data_ptr<float>(), a_scales.numel() != 1, {}};
186 + BiasArgs bias_args{static_cast<ElementD*>(bias.data_ptr()), {}};
187 +
188 + typename EVTCompute0::Arguments evt0_compute_args{b_args};
189 +
190 + typename EVTCompute::Arguments evt_compute_args{a_args, evt0_compute_args,
191 + bias_args};
192 + return evt_compute_args;
193 + }
194 +};
195 +
196 +template <typename Arch, template <typename> typename ArchGuard,
197 + typename ElementAB_, typename ElementD_,
198 + template <typename, typename> typename Epilogue_, typename TileShape,
199 + typename WarpShape, typename InstructionShape, int32_t MainLoopStages,
200 + typename FP8MathOperator = cutlass::arch::OpMultiplyAdd>
201 +struct cutlass_2x_gemm {
202 + using ElementAB = ElementAB_;
203 + using ElementD = ElementD_;
204 +
205 + using ElementAcc =
206 + typename std::conditional<std::is_same_v<ElementAB, int8_t>, int32_t,
207 + float>::type;
208 +
209 + using Operator =
210 + typename std::conditional<std::is_same_v<ElementAB, int8_t>,
211 + cutlass::arch::OpMultiplyAddSaturate,
212 + FP8MathOperator>::type;
213 +
214 + using OutputTileThreadMap =
215 + cutlass::epilogue::threadblock::OutputTileThreadLayout<
216 + TileShape, WarpShape, float, 4, 1 /* epilogue stages */
217 + >;
218 +
219 + using Epilogue = Epilogue_<ElementD, OutputTileThreadMap>;
220 + using EVTCompute = typename Epilogue::EVTCompute;
221 +
222 + using D = cutlass::epilogue::threadblock::VisitorAuxStore<
223 + OutputTileThreadMap, ElementD, cutlass::FloatRoundStyle::round_to_nearest,
224 + Stride<int64_t, Int<1>, Int<0>>>;
225 +
226 + using EVTD = cutlass::epilogue::threadblock::Sm80EVT<D, EVTCompute>;
227 +
228 + // clang-format off
229 + using RowMajor = typename cutlass::layout::RowMajor;
230 + using ColumnMajor = typename cutlass::layout::ColumnMajor;
231 + using KernelType =
232 + ArchGuard<typename cutlass::gemm::kernel::DefaultGemmWithVisitor<
233 + ElementAB, RowMajor, cutlass::ComplexTransform::kNone, 16,
234 + ElementAB, ColumnMajor, cutlass::ComplexTransform::kNone, 16,
235 + float, cutlass::layout::RowMajor, 4,
236 + ElementAcc, float, cutlass::arch::OpClassTensorOp,
237 + Arch,
238 + TileShape, WarpShape, InstructionShape,
239 + EVTD,
240 + cutlass::gemm::threadblock::ThreadblockSwizzleStreamK,
241 + MainLoopStages, Operator,
242 + 1 /* epilogue stages */
243 + >::GemmKernel>;
244 + // clang-format on
245 +
246 + using Op = cutlass::gemm::device::GemmUniversalAdapter<KernelType>;
247 +};
248 +
249 +template <typename Gemm, typename... EpilogueArgs>
250 +inline void cutlass_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
251 + torch::Tensor const& b,
252 + EpilogueArgs&&... epilogue_params) {
253 + using ElementAB = typename Gemm::ElementAB;
254 + using ElementD = typename Gemm::ElementD;
255 +
256 + int32_t m = a.size(0);
257 + int32_t n = b.size(1);
258 + int32_t k = a.size(1);
259 + cutlass::gemm::GemmCoord problem_size{m, n, k};
260 +
261 + int64_t lda = a.stride(0);
262 + int64_t ldb = b.stride(1);
263 + int64_t ldc = out.stride(0);
264 +
265 + using StrideC = Stride<int64_t, Int<1>, Int<0>>;
266 + StrideC c_stride{ldc, Int<1>{}, Int<0>{}};
267 +
268 + auto a_ptr = static_cast<ElementAB const*>(a.data_ptr());
269 + auto b_ptr = static_cast<ElementAB const*>(b.data_ptr());
270 + auto c_ptr = static_cast<ElementD*>(out.data_ptr());
271 +
272 + typename Gemm::D::Arguments d_args{c_ptr, c_stride};
273 +
274 + using Epilogue = typename Gemm::Epilogue;
275 + auto evt_args =
276 + Epilogue::prepare_args(std::forward<EpilogueArgs>(epilogue_params)...);
277 +
278 + typename Gemm::EVTD::Arguments epilogue_args{
279 + evt_args,
280 + d_args,
281 + };
282 +
283 + typename Gemm::Op::Arguments args{
284 + cutlass::gemm::GemmUniversalMode::kGemmSplitKParallel, // universal mode
285 + problem_size, // problem size
286 + 1, // batch count
287 + epilogue_args,
288 + a_ptr,
289 + b_ptr,
290 + nullptr,
291 + nullptr,
292 + 0,
293 + 0,
294 + 0,
295 + 0,
296 + lda,
297 + ldb,
298 + ldc,
299 + ldc};
300 +
301 + // Launch the CUTLASS GEMM kernel.
302 + typename Gemm::Op gemm_op;
303 + size_t workspace_size = gemm_op.get_workspace_size(args);
304 + cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
305 +
306 + auto stream = at::cuda::getCurrentCUDAStream(a.get_device());
307 +
308 + CUTLASS_CHECK(gemm_op.can_implement(args));
309 + cutlass::Status status = gemm_op(args, workspace.get(), stream);
310 + CUTLASS_CHECK(status);
311 +}
312 +
313 +template <typename Gemm, typename FallbackGemm, typename... EpilogueArgs>
314 +inline void fallback_cutlass_gemm_caller(torch::Tensor& out,
315 + torch::Tensor const& a,
316 + torch::Tensor const& b,
317 + EpilogueArgs&&... args) {
318 + // In some cases, the GPU isn't able to accommodate the
319 + // shared memory requirements of the Gemm. In such cases, use
320 + // the FallbackGemm instead.
321 + static const int max_shared_mem_per_block_opt_in =
322 + get_cuda_max_shared_memory_per_block_opt_in(0);
323 +
324 + size_t const gemm_shared_mem_size =
325 + sizeof(typename Gemm::KernelType::SharedStorage);
326 + size_t const fallback_gemm_shared_mem_size =
327 + sizeof(typename FallbackGemm::KernelType::SharedStorage);
328 +
329 + if (gemm_shared_mem_size <= max_shared_mem_per_block_opt_in) {
330 + return cutlass_gemm_caller<Gemm>(out, a, b,
331 + std::forward<EpilogueArgs>(args)...);
332 + } else {
333 + TORCH_CHECK(fallback_gemm_shared_mem_size <=
334 + max_shared_mem_per_block_opt_in);
335 + return cutlass_gemm_caller<FallbackGemm>(
336 + out, a, b, std::forward<EpilogueArgs>(args)...);
337 + }
338 +}
339 +
340 +} // namespace vllm
csrc/quantization/cutlass_w8a8/scaled_mm_c2x_sm80_dispatch.cuh new
+139
@@ -0,0 +1,139 @@
1 +#pragma once
2 +
3 +#include "scaled_mm_c2x.cuh"
4 +
5 +/**
6 + * This file defines Gemm kernel configurations for SM80 based on the Gemm
7 + * shape.
8 + */
9 +
10 +namespace vllm {
11 +
12 +template <typename InType, typename OutType,
13 + template <typename, typename> typename Epilogue>
14 +struct sm80_config_default {
15 + // This config is used in 2 cases,
16 + // - M in (128, inf)
17 + // - M in (64, 128] and N >= 8192
18 + // Shared Memory required by this Gemm - 81920 bytes
19 + static_assert(std::is_same<InType, int8_t>());
20 + using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
21 + using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
22 + using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
23 + using Cutlass2xGemm =
24 + cutlass_2x_gemm<cutlass::arch::Sm80, enable_sm80_to_sm89, InType, OutType,
25 + Epilogue, TileShape, WarpShape, InstructionShape, 5>;
26 +};
27 +
28 +template <typename InType, typename OutType,
29 + template <typename, typename> typename Epilogue>
30 +struct sm80_config_M64 {
31 + // This config is used in 2 cases,
32 + // - M in (32, 64]
33 + // - M in (64, 128] and N < 8192
34 + // Shared Memory required by this Gemm - 122880 bytes
35 + static_assert(std::is_same<InType, int8_t>());
36 + using TileShape = typename cutlass::gemm::GemmShape<64, 128, 128>;
37 + using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
38 + using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
39 + using Cutlass2xGemm =
40 + cutlass_2x_gemm<cutlass::arch::Sm80, enable_sm80_to_sm89, InType, OutType,
41 + Epilogue, TileShape, WarpShape, InstructionShape, 5>;
42 +};
43 +
44 +template <typename InType, typename OutType,
45 + template <typename, typename> typename Epilogue>
46 +struct sm80_config_M32 {
47 + // M in (16, 32]
48 + // Shared Memory required by this Gemm - 61440 bytes
49 + static_assert(std::is_same<InType, int8_t>());
50 + using TileShape = typename cutlass::gemm::GemmShape<32, 64, 128>;
51 + using WarpShape = typename cutlass::gemm::GemmShape<32, 64, 64>;
52 + using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
53 + using Cutlass2xGemm =
54 + cutlass_2x_gemm<cutlass::arch::Sm80, enable_sm80_to_sm89, InType, OutType,
55 + Epilogue, TileShape, WarpShape, InstructionShape, 5>;
56 +};
57 +
58 +template <typename InType, typename OutType,
59 + template <typename, typename> typename Epilogue>
60 +struct sm80_config_M16 {
61 + // M in [1, 16]
62 + // Shared Memory required by this Gemm - 51200 bytes
63 + static_assert(std::is_same<InType, int8_t>());
64 + using TileShape = typename cutlass::gemm::GemmShape<16, 64, 128>;
65 + using WarpShape = typename cutlass::gemm::GemmShape<16, 64, 64>;
66 + using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
67 + using Cutlass2xGemm =
68 + cutlass_2x_gemm<cutlass::arch::Sm80, enable_sm80_to_sm89, InType, OutType,
69 + Epilogue, TileShape, WarpShape, InstructionShape, 5>;
70 +};
71 +
72 +template <typename InType, typename OutType,
73 + template <typename, typename> typename Epilogue,
74 + typename... EpilogueArgs>
75 +inline void cutlass_gemm_sm80_dispatch(torch::Tensor& out,
76 + torch::Tensor const& a,
77 + torch::Tensor const& b,
78 + EpilogueArgs&&... args) {
79 + static_assert(std::is_same<InType, int8_t>());
80 + TORCH_CHECK(a.dtype() == torch::kInt8);
81 + TORCH_CHECK(b.dtype() == torch::kInt8);
82 +
83 + using Cutlass2xGemmDefault =
84 + typename sm80_config_default<InType, OutType, Epilogue>::Cutlass2xGemm;
85 + using Cutlass2xGemmM128BigN =
86 + typename sm80_config_default<InType, OutType, Epilogue>::Cutlass2xGemm;
87 + using Cutlass2xGemmM128SmallN =
88 + typename sm80_config_M64<InType, OutType, Epilogue>::Cutlass2xGemm;
89 + using Cutlass2xGemmM64 =
90 + typename sm80_config_M64<InType, OutType, Epilogue>::Cutlass2xGemm;
91 + using Cutlass2xGemmM32 =
92 + typename sm80_config_M32<InType, OutType, Epilogue>::Cutlass2xGemm;
93 + using Cutlass2xGemmM16 =
94 + typename sm80_config_M16<InType, OutType, Epilogue>::Cutlass2xGemm;
95 +
96 + // Due to shared memory requirements, some Gemms may fail to run on some
97 + // GPUs. As the name indicates, the Fallback Gemm is used as an alternative
98 + // in such cases.
99 + // sm80_config_M16 has the least shared-memory requirement. However,
100 + // based on some profiling, we select sm80_config_M32 as a better alternative
101 + // performance wise.
102 + using FallbackGemm =
103 + typename sm80_config_M32<InType, OutType, Epilogue>::Cutlass2xGemm;
104 +
105 + uint32_t const m = a.size(0);
106 + uint32_t const mp2 =
107 + std::max(static_cast<uint32_t>(16), next_pow_2(m)); // next power of 2
108 + if (mp2 <= 16) {
109 + // M in [1, 16]
110 + return fallback_cutlass_gemm_caller<Cutlass2xGemmM16, FallbackGemm>(
111 + out, a, b, std::forward<EpilogueArgs>(args)...);
112 + } else if (mp2 <= 32) {
113 + // M in (16, 32]
114 + return fallback_cutlass_gemm_caller<Cutlass2xGemmM32, FallbackGemm>(
115 + out, a, b, std::forward<EpilogueArgs>(args)...);
116 + } else if (mp2 <= 64) {
117 + // M in (32, 64]
118 + return fallback_cutlass_gemm_caller<Cutlass2xGemmM64, FallbackGemm>(
119 + out, a, b, std::forward<EpilogueArgs>(args)...);
120 + } else if (mp2 <= 128) {
121 + // M in (64, 128]
122 + uint32_t const n = out.size(1);
123 + bool const small_n = n < 8192;
124 + if (small_n) {
125 + return fallback_cutlass_gemm_caller<Cutlass2xGemmM128SmallN,
126 + FallbackGemm>(
127 + out, a, b, std::forward<EpilogueArgs>(args)...);
128 + } else {
129 + return fallback_cutlass_gemm_caller<Cutlass2xGemmM128BigN, FallbackGemm>(
130 + out, a, b, std::forward<EpilogueArgs>(args)...);
131 + }
132 + } else {
133 + // M in (128, inf)
134 + return fallback_cutlass_gemm_caller<Cutlass2xGemmDefault, FallbackGemm>(
135 + out, a, b, std::forward<EpilogueArgs>(args)...);
136 + }
137 +}
138 +
139 +} // namespace vllm
csrc/quantization/cutlass_w8a8/scaled_mm_c2x_sm89_dispatch.cuh new
+362
@@ -0,0 +1,362 @@
1 +#pragma once
2 +
3 +#include "scaled_mm_c2x.cuh"
4 +#include "cutlass/float8.h"
5 +
6 +/**
7 + * This file defines Gemm kernel configurations for SM89 based on the Gemm
8 + * shape.
9 + */
10 +
11 +namespace vllm {
12 +
13 +template <typename InType, typename OutType,
14 + template <typename, typename> typename Epilogue>
15 +struct sm89_fallback_gemm {
16 + // Shared Memory required by this Gemm - 61440 bytes
17 + static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
18 + using TileShape = typename cutlass::gemm::GemmShape<64, 128, 64>;
19 + using WarpShape = typename cutlass::gemm::GemmShape<32, 64, 64>;
20 + using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
21 + using FP8MathOperator = typename cutlass::arch::OpMultiplyAdd;
22 + using Cutlass2xGemm =
23 + cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90, InType, OutType,
24 + Epilogue, TileShape, WarpShape, InstructionShape, 5,
25 + FP8MathOperator>;
26 +};
27 +
28 +struct sm89_config_default {
29 + // M in (256, inf)
30 + using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
31 + using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
32 + using FP8MathOperator = typename cutlass::arch::OpMultiplyAddFastAccum;
33 +
34 + template <typename InType, typename OutType,
35 + template <typename, typename> typename Epilogue,
36 + typename... EpilogueArgs>
37 + static void dispatch(torch::Tensor& out, torch::Tensor const& a,
38 + torch::Tensor const& b, EpilogueArgs&&... args) {
39 + static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
40 + TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
41 +
42 + using FallbackGemm =
43 + typename sm89_fallback_gemm<InType, OutType, Epilogue>::Cutlass2xGemm;
44 +
45 + uint32_t const n = out.size(1);
46 + uint32_t const np2 = next_pow_2(n);
47 +
48 + if (np2 <= 4096) {
49 + using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
50 +
51 + return vllm::fallback_cutlass_gemm_caller<
52 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
53 + InType, OutType, Epilogue, TileShape, WarpShape,
54 + InstructionShape, 5, FP8MathOperator>,
55 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
56 + } else if (np2 <= 8192) {
57 + using TileShape = typename cutlass::gemm::GemmShape<256, 128, 64>;
58 +
59 + return vllm::fallback_cutlass_gemm_caller<
60 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
61 + InType, OutType, Epilogue, TileShape, WarpShape,
62 + InstructionShape, 3, FP8MathOperator>,
63 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
64 +
65 + } else {
66 + using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
67 +
68 + return vllm::fallback_cutlass_gemm_caller<
69 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
70 + InType, OutType, Epilogue, TileShape, WarpShape,
71 + InstructionShape, 5, FP8MathOperator>,
72 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
73 + }
74 + }
75 +};
76 +
77 +struct sm89_config_M256 {
78 + // M in (128, 256]
79 + using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
80 + using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
81 + using FP8MathOperator = typename cutlass::arch::OpMultiplyAddFastAccum;
82 +
83 + template <typename InType, typename OutType,
84 + template <typename, typename> typename Epilogue,
85 + typename... EpilogueArgs>
86 + static void dispatch(torch::Tensor& out, torch::Tensor const& a,
87 + torch::Tensor const& b, EpilogueArgs&&... args) {
88 + static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
89 + TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
90 +
91 + using FallbackGemm =
92 + typename sm89_fallback_gemm<InType, OutType, Epilogue>::Cutlass2xGemm;
93 +
94 + uint32_t const n = out.size(1);
95 + uint32_t const np2 = next_pow_2(n);
96 +
97 + if (np2 <= 4096) {
98 + using TileShape = typename cutlass::gemm::GemmShape<64, 128, 128>;
99 +
100 + return vllm::fallback_cutlass_gemm_caller<
101 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
102 + InType, OutType, Epilogue, TileShape, WarpShape,
103 + InstructionShape, 3, FP8MathOperator>,
104 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
105 + } else {
106 + using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
107 +
108 + return vllm::fallback_cutlass_gemm_caller<
109 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
110 + InType, OutType, Epilogue, TileShape, WarpShape,
111 + InstructionShape, 5, FP8MathOperator>,
112 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
113 + }
114 + }
115 +};
116 +
117 +struct sm89_config_M128 {
118 + // M in (64, 128]
119 + using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
120 + using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
121 + using FP8MathOperator = typename cutlass::arch::OpMultiplyAddFastAccum;
122 +
123 + template <typename InType, typename OutType,
124 + template <typename, typename> typename Epilogue,
125 + typename... EpilogueArgs>
126 + static void dispatch(torch::Tensor& out, torch::Tensor const& a,
127 + torch::Tensor const& b, EpilogueArgs&&... args) {
128 + static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
129 + TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
130 +
131 + using FallbackGemm =
132 + typename sm89_fallback_gemm<InType, OutType, Epilogue>::Cutlass2xGemm;
133 +
134 + uint32_t const n = out.size(1);
135 + uint32_t const np2 = next_pow_2(n);
136 +
137 + if (np2 <= 8192) {
138 + using TileShape = typename cutlass::gemm::GemmShape<64, 128, 128>;
139 +
140 + return vllm::fallback_cutlass_gemm_caller<
141 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
142 + InType, OutType, Epilogue, TileShape, WarpShape,
143 + InstructionShape, 3, FP8MathOperator>,
144 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
145 +
146 + } else if (np2 <= 16384) {
147 + using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
148 +
149 + return vllm::fallback_cutlass_gemm_caller<
150 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
151 + InType, OutType, Epilogue, TileShape, WarpShape,
152 + InstructionShape, 5, FP8MathOperator>,
153 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
154 + } else {
155 + using TileShape = typename cutlass::gemm::GemmShape<128, 64, 128>;
156 +
157 + return vllm::fallback_cutlass_gemm_caller<
158 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
159 + InType, OutType, Epilogue, TileShape, WarpShape,
160 + InstructionShape, 3, FP8MathOperator>,
161 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
162 + }
163 + }
164 +};
165 +
166 +struct sm89_config_M64 {
167 + // M in (32, 64]
168 + using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
169 +
170 + template <typename InType, typename OutType,
171 + template <typename, typename> typename Epilogue,
172 + typename... EpilogueArgs>
173 + static void dispatch(torch::Tensor& out, torch::Tensor const& a,
174 + torch::Tensor const& b, EpilogueArgs&&... args) {
175 + static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
176 + TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
177 +
178 + using FallbackGemm =
179 + typename sm89_fallback_gemm<InType, OutType, Epilogue>::Cutlass2xGemm;
180 +
181 + uint32_t const n = out.size(1);
182 + uint32_t const np2 = next_pow_2(n);
183 +
184 + if (np2 <= 8196) {
185 + using TileShape = typename cutlass::gemm::GemmShape<64, 64, 128>;
186 + using WarpShape = typename cutlass::gemm::GemmShape<32, 64, 64>;
187 + using FP8MathOperator = typename cutlass::arch::OpMultiplyAdd;
188 +
189 + return vllm::fallback_cutlass_gemm_caller<
190 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
191 + InType, OutType, Epilogue, TileShape, WarpShape,
192 + InstructionShape, 5, FP8MathOperator>,
193 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
194 + } else if (np2 <= 16384) {
195 + using TileShape = typename cutlass::gemm::GemmShape<64, 128, 128>;
196 + using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
197 + using FP8MathOperator = typename cutlass::arch::OpMultiplyAddFastAccum;
198 +
199 + return vllm::fallback_cutlass_gemm_caller<
200 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
201 + InType, OutType, Epilogue, TileShape, WarpShape,
202 + InstructionShape, 3, FP8MathOperator>,
203 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
204 + } else {
205 + using TileShape = typename cutlass::gemm::GemmShape<64, 64, 128>;
206 + using WarpShape = typename cutlass::gemm::GemmShape<32, 64, 64>;
207 + using FP8MathOperator = typename cutlass::arch::OpMultiplyAdd;
208 +
209 + return vllm::fallback_cutlass_gemm_caller<
210 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
211 + InType, OutType, Epilogue, TileShape, WarpShape,
212 + InstructionShape, 5, FP8MathOperator>,
213 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
214 + }
215 + }
216 +};
217 +
218 +struct sm89_config_M32 {
219 + // M in (16, 32]
220 + using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
221 + using FP8MathOperator = typename cutlass::arch::OpMultiplyAddFastAccum;
222 +
223 + template <typename InType, typename OutType,
224 + template <typename, typename> typename Epilogue,
225 + typename... EpilogueArgs>
226 + static void dispatch(torch::Tensor& out, torch::Tensor const& a,
227 + torch::Tensor const& b, EpilogueArgs&&... args) {
228 + static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
229 + TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
230 +
231 + using FallbackGemm =
232 + typename sm89_fallback_gemm<InType, OutType, Epilogue>::Cutlass2xGemm;
233 +
234 + uint32_t const n = out.size(1);
235 + uint32_t const np2 = next_pow_2(n);
236 +
237 + if (np2 <= 8192) {
238 + using TileShape = typename cutlass::gemm::GemmShape<32, 64, 128>;
239 + using WarpShape = typename cutlass::gemm::GemmShape<16, 64, 64>;
240 +
241 + return vllm::fallback_cutlass_gemm_caller<
242 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
243 + InType, OutType, Epilogue, TileShape, WarpShape,
244 + InstructionShape, 5, FP8MathOperator>,
245 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
246 + } else if (np2 <= 16384) {
247 + using TileShape = typename cutlass::gemm::GemmShape<32, 128, 128>;
248 + using WarpShape = typename cutlass::gemm::GemmShape<32, 64, 64>;
249 +
250 + return vllm::fallback_cutlass_gemm_caller<
251 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
252 + InType, OutType, Epilogue, TileShape, WarpShape,
253 + InstructionShape, 4, FP8MathOperator>,
254 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
255 + } else {
256 + using TileShape = typename cutlass::gemm::GemmShape<32, 64, 128>;
257 + using WarpShape = typename cutlass::gemm::GemmShape<16, 64, 64>;
258 +
259 + return vllm::fallback_cutlass_gemm_caller<
260 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
261 + InType, OutType, Epilogue, TileShape, WarpShape,
262 + InstructionShape, 5, FP8MathOperator>,
263 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
264 + }
265 + }
266 +};
267 +
268 +struct sm89_config_M16 {
269 + // M in [1, 16]
270 + using WarpShape = typename cutlass::gemm::GemmShape<16, 64, 64>;
271 + using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
272 + using FP8MathOperator = typename cutlass::arch::OpMultiplyAddFastAccum;
273 + static const int32_t MainLoopStages = 5;
274 +
275 + template <typename InType, typename OutType,
276 + template <typename, typename> typename Epilogue,
277 + typename... EpilogueArgs>
278 + static void dispatch(torch::Tensor& out, torch::Tensor const& a,
279 + torch::Tensor const& b, EpilogueArgs&&... args) {
280 + static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
281 + TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
282 +
283 + using FallbackGemm =
284 + typename sm89_fallback_gemm<InType, OutType, Epilogue>::Cutlass2xGemm;
285 +
286 + uint32_t const n = out.size(1);
287 + uint32_t const np2 = next_pow_2(n);
288 +
289 + if (np2 <= 8192) {
290 + using TileShape = typename cutlass::gemm::GemmShape<16, 64, 128>;
291 +
292 + return vllm::fallback_cutlass_gemm_caller<
293 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
294 + InType, OutType, Epilogue, TileShape, WarpShape,
295 + InstructionShape, MainLoopStages,
296 + FP8MathOperator>,
297 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
298 + } else if (np2 <= 24576) {
299 + using TileShape = typename cutlass::gemm::GemmShape<16, 128, 64>;
300 +
301 + return vllm::fallback_cutlass_gemm_caller<
302 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
303 + InType, OutType, Epilogue, TileShape, WarpShape,
304 + InstructionShape, MainLoopStages,
305 + FP8MathOperator>,
306 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
307 + } else {
308 + using TileShape = typename cutlass::gemm::GemmShape<32, 64, 128>;
309 +
310 + return vllm::fallback_cutlass_gemm_caller<
311 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
312 + InType, OutType, Epilogue, TileShape, WarpShape,
313 + InstructionShape, MainLoopStages,
314 + FP8MathOperator>,
315 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
316 + }
317 + }
318 +};
319 +
320 +template <typename InType, typename OutType,
321 + template <typename, typename> typename Epilogue,
322 + typename... EpilogueArgs>
323 +inline void cutlass_gemm_sm89_dispatch(torch::Tensor& out,
324 + torch::Tensor const& a,
325 + torch::Tensor const& b,
326 + EpilogueArgs&&... args) {
327 + static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
328 + TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
329 + TORCH_CHECK(b.dtype() == torch::kFloat8_e4m3fn);
330 +
331 + uint32_t const m = a.size(0);
332 + uint32_t const mp2 =
333 + std::max(static_cast<uint32_t>(32), next_pow_2(m)); // next power of 2
334 +
335 + if (mp2 <= 16) {
336 + // M in [1, 16]
337 + return sm89_config_M16::dispatch<InType, OutType, Epilogue>(
338 + out, a, b, std::forward<EpilogueArgs>(args)...);
339 + } else if (mp2 <= 32) {
340 + // M in (16, 32]
341 + return sm89_config_M32::dispatch<InType, OutType, Epilogue>(
342 + out, a, b, std::forward<EpilogueArgs>(args)...);
343 + } else if (mp2 <= 64) {
344 + // M in (32, 64]
345 + return sm89_config_M64::dispatch<InType, OutType, Epilogue>(
346 + out, a, b, std::forward<EpilogueArgs>(args)...);
347 + } else if (mp2 <= 128) {
348 + // M in (64, 128]
349 + return sm89_config_M128::dispatch<InType, OutType, Epilogue>(
350 + out, a, b, std::forward<EpilogueArgs>(args)...);
351 + } else if (mp2 <= 256) {
352 + // M in (128, 256]
353 + return sm89_config_M256::dispatch<InType, OutType, Epilogue>(
354 + out, a, b, std::forward<EpilogueArgs>(args)...);
355 + } else {
356 + // M in (256, inf)
357 + return sm89_config_default::dispatch<InType, OutType, Epilogue>(
358 + out, a, b, std::forward<EpilogueArgs>(args)...);
359 + }
360 +}
361 +
362 +} // namespace vllm
tests/kernels/test_cutlass.py
+2 -2
@@ -106,8 +106,8 @@ def cutlass_int8_gemm_helper(m: int,
106 assert torch.allclose(out, baseline, rtol=1e-1, atol=1e0)
107
108
109 -@pytest.mark.parametrize("m", [512, 222, 100, 33, 1])
110 -@pytest.mark.parametrize("n", [2048, 256, 1024])
109 +@pytest.mark.parametrize("m", [1, 16, 32, 64, 128, 256, 512, 222, 100, 33])
110 +@pytest.mark.parametrize("n", [2048, 4096, 8192, 16384, 24576, 256, 1024])
111 @pytest.mark.parametrize("k", [128, 496, 1024])
112 @pytest.mark.parametrize("per_act_token", [True, False])
113 @pytest.mark.parametrize("per_out_ch", [True, False])