@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / af647fb8b3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Kernel] Tuned int8 kernels for Ada Lovelace (#6848)

Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>

Varun Sundar Rabindranath committed Jul 29, 2024 at 22:24 UTC af647fb8b3ea9d910f7d1bc104af8986d048a8e2
4 files changed +395 -43
csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cu
+10 -17
@@ -4,7 +4,8 @@
4
5 #include "scaled_mm_c2x.cuh"
6 #include "scaled_mm_c2x_sm80_dispatch.cuh"
7 -#include "scaled_mm_c2x_sm89_dispatch.cuh"
7 +#include "scaled_mm_c2x_sm89_fp8_dispatch.cuh"
8 +#include "scaled_mm_c2x_sm89_int8_dispatch.cuh"
9
10 /*
11 This file defines quantized GEMM operations using the CUTLASS 2.x API, for
@@ -98,25 +99,17 @@ template <template <typename, typename> typename Epilogue,
99 void cutlass_scaled_mm_sm89_epilogue(torch::Tensor& out, torch::Tensor const& a,
100 torch::Tensor const& b,
101 EpilogueArgs&&... epilogue_args) {
101 - using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
102 - using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
103 - using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
104 -
102 if (a.dtype() == torch::kInt8) {
103 TORCH_CHECK(b.dtype() == torch::kInt8);
104
105 if (out.dtype() == torch::kBFloat16) {
109 - return vllm::cutlass_gemm_caller<
110 - vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
111 - int8_t, cutlass::bfloat16_t, Epilogue,
112 - TileShape, WarpShape, InstructionShape, 5>>(
106 + return vllm::cutlass_gemm_sm89_int8_dispatch<int8_t, cutlass::bfloat16_t,
107 + Epilogue>(
108 out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
109 } else {
110 assert(out.dtype() == torch::kFloat16);
116 - return vllm::cutlass_gemm_caller<
117 - vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
118 - int8_t, cutlass::half_t, Epilogue, TileShape,
119 - WarpShape, InstructionShape, 5>>(
111 + return vllm::cutlass_gemm_sm89_int8_dispatch<int8_t, cutlass::half_t,
112 + Epilogue>(
113 out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
114 }
115 } else {
@@ -124,13 +117,13 @@ void cutlass_scaled_mm_sm89_epilogue(torch::Tensor& out, torch::Tensor const& a,
117 TORCH_CHECK(b.dtype() == torch::kFloat8_e4m3fn);
118
119 if (out.dtype() == torch::kBFloat16) {
127 - return vllm::cutlass_gemm_sm89_dispatch<cutlass::float_e4m3_t,
128 - cutlass::bfloat16_t, Epilogue>(
120 + return vllm::cutlass_gemm_sm89_fp8_dispatch<
121 + cutlass::float_e4m3_t, cutlass::bfloat16_t, Epilogue>(
122 out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
123 } else {
124 TORCH_CHECK(out.dtype() == torch::kFloat16);
132 - return vllm::cutlass_gemm_sm89_dispatch<cutlass::float_e4m3_t,
133 - cutlass::half_t, Epilogue>(
125 + return vllm::cutlass_gemm_sm89_fp8_dispatch<cutlass::float_e4m3_t,
126 + cutlass::half_t, Epilogue>(
127 out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
128 }
129 }
csrc/quantization/cutlass_w8a8/scaled_mm_c2x_sm89_fp8_dispatch.cuh renamed
+30 -24
@@ -4,7 +4,7 @@
4 #include "cutlass/float8.h"
5
6 /**
7 - * This file defines Gemm kernel configurations for SM89 based on the Gemm
7 + * This file defines Gemm kernel configurations for SM89 (FP8) based on the Gemm
8 * shape.
9 */
10
@@ -12,7 +12,7 @@ namespace vllm {
12
13 template <typename InType, typename OutType,
14 template <typename, typename> typename Epilogue>
15 -struct sm89_fallback_gemm {
15 +struct sm89_fp8_fallback_gemm {
16 // Shared Memory required by this Gemm - 61440 bytes
17 static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
18 using TileShape = typename cutlass::gemm::GemmShape<64, 128, 64>;
@@ -25,7 +25,7 @@ struct sm89_fallback_gemm {
25 FP8MathOperator>;
26 };
27
28 -struct sm89_config_default {
28 +struct sm89_fp8_config_default {
29 // M in (256, inf)
30 using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
31 using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
@@ -40,7 +40,8 @@ struct sm89_config_default {
40 TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
41
42 using FallbackGemm =
43 - typename sm89_fallback_gemm<InType, OutType, Epilogue>::Cutlass2xGemm;
43 + typename sm89_fp8_fallback_gemm<InType, OutType,
44 + Epilogue>::Cutlass2xGemm;
45
46 uint32_t const n = out.size(1);
47 uint32_t const np2 = next_pow_2(n);
@@ -74,7 +75,7 @@ struct sm89_config_default {
75 }
76 };
77
77 -struct sm89_config_M256 {
78 +struct sm89_fp8_config_M256 {
79 // M in (128, 256]
80 using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
81 using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
@@ -89,7 +90,8 @@ struct sm89_config_M256 {
90 TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
91
92 using FallbackGemm =
92 - typename sm89_fallback_gemm<InType, OutType, Epilogue>::Cutlass2xGemm;
93 + typename sm89_fp8_fallback_gemm<InType, OutType,
94 + Epilogue>::Cutlass2xGemm;
95
96 uint32_t const n = out.size(1);
97 uint32_t const np2 = next_pow_2(n);
@@ -114,7 +116,7 @@ struct sm89_config_M256 {
116 }
117 };
118
117 -struct sm89_config_M128 {
119 +struct sm89_fp8_config_M128 {
120 // M in (64, 128]
121 using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
122 using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
@@ -129,7 +131,8 @@ struct sm89_config_M128 {
131 TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
132
133 using FallbackGemm =
132 - typename sm89_fallback_gemm<InType, OutType, Epilogue>::Cutlass2xGemm;
134 + typename sm89_fp8_fallback_gemm<InType, OutType,
135 + Epilogue>::Cutlass2xGemm;
136
137 uint32_t const n = out.size(1);
138 uint32_t const np2 = next_pow_2(n);
@@ -163,7 +166,7 @@ struct sm89_config_M128 {
166 }
167 };
168
166 -struct sm89_config_M64 {
169 +struct sm89_fp8_config_M64 {
170 // M in (32, 64]
171 using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
172
@@ -176,7 +179,8 @@ struct sm89_config_M64 {
179 TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
180
181 using FallbackGemm =
179 - typename sm89_fallback_gemm<InType, OutType, Epilogue>::Cutlass2xGemm;
182 + typename sm89_fp8_fallback_gemm<InType, OutType,
183 + Epilogue>::Cutlass2xGemm;
184
185 uint32_t const n = out.size(1);
186 uint32_t const np2 = next_pow_2(n);
@@ -215,7 +219,7 @@ struct sm89_config_M64 {
219 }
220 };
221
218 -struct sm89_config_M32 {
222 +struct sm89_fp8_config_M32 {
223 // M in (16, 32]
224 using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
225 using FP8MathOperator = typename cutlass::arch::OpMultiplyAddFastAccum;
@@ -229,7 +233,8 @@ struct sm89_config_M32 {
233 TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
234
235 using FallbackGemm =
232 - typename sm89_fallback_gemm<InType, OutType, Epilogue>::Cutlass2xGemm;
236 + typename sm89_fp8_fallback_gemm<InType, OutType,
237 + Epilogue>::Cutlass2xGemm;
238
239 uint32_t const n = out.size(1);
240 uint32_t const np2 = next_pow_2(n);
@@ -265,7 +270,7 @@ struct sm89_config_M32 {
270 }
271 };
272
268 -struct sm89_config_M16 {
273 +struct sm89_fp8_config_M16 {
274 // M in [1, 16]
275 using WarpShape = typename cutlass::gemm::GemmShape<16, 64, 64>;
276 using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
@@ -281,7 +286,8 @@ struct sm89_config_M16 {
286 TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
287
288 using FallbackGemm =
284 - typename sm89_fallback_gemm<InType, OutType, Epilogue>::Cutlass2xGemm;
289 + typename sm89_fp8_fallback_gemm<InType, OutType,
290 + Epilogue>::Cutlass2xGemm;
291
292 uint32_t const n = out.size(1);
293 uint32_t const np2 = next_pow_2(n);
@@ -320,10 +326,10 @@ struct sm89_config_M16 {
326 template <typename InType, typename OutType,
327 template <typename, typename> typename Epilogue,
328 typename... EpilogueArgs>
323 -inline void cutlass_gemm_sm89_dispatch(torch::Tensor& out,
324 - torch::Tensor const& a,
325 - torch::Tensor const& b,
326 - EpilogueArgs&&... args) {
329 +inline void cutlass_gemm_sm89_fp8_dispatch(torch::Tensor& out,
330 + torch::Tensor const& a,
331 + torch::Tensor const& b,
332 + EpilogueArgs&&... args) {
333 static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
334 TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
335 TORCH_CHECK(b.dtype() == torch::kFloat8_e4m3fn);
@@ -334,27 +340,27 @@ inline void cutlass_gemm_sm89_dispatch(torch::Tensor& out,
340
341 if (mp2 <= 16) {
342 // M in [1, 16]
337 - return sm89_config_M16::dispatch<InType, OutType, Epilogue>(
343 + return sm89_fp8_config_M16::dispatch<InType, OutType, Epilogue>(
344 out, a, b, std::forward<EpilogueArgs>(args)...);
345 } else if (mp2 <= 32) {
346 // M in (16, 32]
341 - return sm89_config_M32::dispatch<InType, OutType, Epilogue>(
347 + return sm89_fp8_config_M32::dispatch<InType, OutType, Epilogue>(
348 out, a, b, std::forward<EpilogueArgs>(args)...);
349 } else if (mp2 <= 64) {
350 // M in (32, 64]
345 - return sm89_config_M64::dispatch<InType, OutType, Epilogue>(
351 + return sm89_fp8_config_M64::dispatch<InType, OutType, Epilogue>(
352 out, a, b, std::forward<EpilogueArgs>(args)...);
353 } else if (mp2 <= 128) {
354 // M in (64, 128]
349 - return sm89_config_M128::dispatch<InType, OutType, Epilogue>(
355 + return sm89_fp8_config_M128::dispatch<InType, OutType, Epilogue>(
356 out, a, b, std::forward<EpilogueArgs>(args)...);
357 } else if (mp2 <= 256) {
358 // M in (128, 256]
353 - return sm89_config_M256::dispatch<InType, OutType, Epilogue>(
359 + return sm89_fp8_config_M256::dispatch<InType, OutType, Epilogue>(
360 out, a, b, std::forward<EpilogueArgs>(args)...);
361 } else {
362 // M in (256, inf)
357 - return sm89_config_default::dispatch<InType, OutType, Epilogue>(
363 + return sm89_fp8_config_default::dispatch<InType, OutType, Epilogue>(
364 out, a, b, std::forward<EpilogueArgs>(args)...);
365 }
366 }
csrc/quantization/cutlass_w8a8/scaled_mm_c2x_sm89_int8_dispatch.cuh new
+353
@@ -0,0 +1,353 @@
1 +#pragma once
2 +
3 +#include "scaled_mm_c2x.cuh"
4 +
5 +/**
6 + * This file defines Gemm kernel configurations for SM89 (int8) based on the
7 + * Gemm shape.
8 + */
9 +
10 +namespace vllm {
11 +
12 +template <typename InType, typename OutType,
13 + template <typename, typename> typename Epilogue>
14 +struct sm89_int8_fallback_gemm {
15 + // Shared mem requirement : 61440
16 + static_assert(std::is_same<InType, int8_t>());
17 + using TileShape = cutlass::gemm::GemmShape<32, 64, 128>;
18 + using WarpShape = cutlass::gemm::GemmShape<16, 64, 64>;
19 + using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
20 + static int32_t const MainLoopStages = 5;
21 +
22 + using Cutlass2xGemm =
23 + cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90, InType, OutType,
24 + Epilogue, TileShape, WarpShape, InstructionShape, 5>;
25 +};
26 +
27 +struct sm89_int8_config_default {
28 + // M in (256, inf)
29 + using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
30 + using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
31 +
32 + template <typename InType, typename OutType,
33 + template <typename, typename> typename Epilogue,
34 + typename... EpilogueArgs>
35 + static void dispatch(torch::Tensor& out, torch::Tensor const& a,
36 + torch::Tensor const& b, EpilogueArgs&&... args) {
37 + static_assert(std::is_same<InType, int8_t>());
38 + TORCH_CHECK(a.dtype() == torch::kInt8);
39 +
40 + using FallbackGemm =
41 + typename sm89_int8_fallback_gemm<InType, OutType,
42 + Epilogue>::Cutlass2xGemm;
43 +
44 + uint32_t const n = out.size(1);
45 + uint32_t const np2 = next_pow_2(n);
46 +
47 + if (np2 <= 4096) {
48 + using TileShape = cutlass::gemm::GemmShape<128, 128, 64>;
49 +
50 + return vllm::fallback_cutlass_gemm_caller<
51 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
52 + InType, OutType, Epilogue, TileShape, WarpShape,
53 + InstructionShape, 5>,
54 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
55 + } else if (np2 <= 8192) {
56 + using TileShape = cutlass::gemm::GemmShape<256, 128, 64>;
57 +
58 + return vllm::fallback_cutlass_gemm_caller<
59 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
60 + InType, OutType, Epilogue, TileShape, WarpShape,
61 + InstructionShape, 3>,
62 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
63 + } else if (np2 <= 16384) {
64 + using TileShape = cutlass::gemm::GemmShape<128, 128, 64>;
65 +
66 + return vllm::fallback_cutlass_gemm_caller<
67 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
68 + InType, OutType, Epilogue, TileShape, WarpShape,
69 + InstructionShape, 5>,
70 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
71 + } else {
72 + using TileShape = cutlass::gemm::GemmShape<256, 128, 64>;
73 +
74 + return vllm::fallback_cutlass_gemm_caller<
75 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
76 + InType, OutType, Epilogue, TileShape, WarpShape,
77 + InstructionShape, 3>,
78 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
79 + }
80 + }
81 +};
82 +
83 +struct sm89_int8_config_M256 {
84 + // M in (128, 256]
85 + using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
86 + using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
87 +
88 + template <typename InType, typename OutType,
89 + template <typename, typename> typename Epilogue,
90 + typename... EpilogueArgs>
91 + static void dispatch(torch::Tensor& out, torch::Tensor const& a,
92 + torch::Tensor const& b, EpilogueArgs&&... args) {
93 + static_assert(std::is_same<InType, int8_t>());
94 + TORCH_CHECK(a.dtype() == torch::kInt8);
95 +
96 + using FallbackGemm =
97 + typename sm89_int8_fallback_gemm<InType, OutType,
98 + Epilogue>::Cutlass2xGemm;
99 +
100 + uint32_t const n = out.size(1);
101 + uint32_t const np2 = next_pow_2(n);
102 +
103 + if (np2 <= 4096) {
104 + using TileShape = cutlass::gemm::GemmShape<64, 128, 128>;
105 +
106 + return vllm::fallback_cutlass_gemm_caller<
107 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
108 + InType, OutType, Epilogue, TileShape, WarpShape,
109 + InstructionShape, 3>,
110 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
111 + } else if (np2 <= 8192) {
112 + using TileShape = cutlass::gemm::GemmShape<128, 128, 64>;
113 +
114 + return vllm::fallback_cutlass_gemm_caller<
115 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
116 + InType, OutType, Epilogue, TileShape, WarpShape,
117 + InstructionShape, 5>,
118 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
119 + } else if (np2 <= 16384) {
120 + using TileShape = cutlass::gemm::GemmShape<256, 128, 64>;
121 +
122 + return vllm::fallback_cutlass_gemm_caller<
123 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
124 + InType, OutType, Epilogue, TileShape, WarpShape,
125 + InstructionShape, 3>,
126 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
127 + } else {
128 + using TileShape = cutlass::gemm::GemmShape<128, 128, 64>;
129 +
130 + return vllm::fallback_cutlass_gemm_caller<
131 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
132 + InType, OutType, Epilogue, TileShape, WarpShape,
133 + InstructionShape, 5>,
134 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
135 + }
136 + }
137 +};
138 +
139 +struct sm89_int8_config_M128 {
140 + // M in (64, 128]
141 + using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
142 +
143 + template <typename InType, typename OutType,
144 + template <typename, typename> typename Epilogue,
145 + typename... EpilogueArgs>
146 + static void dispatch(torch::Tensor& out, torch::Tensor const& a,
147 + torch::Tensor const& b, EpilogueArgs&&... args) {
148 + static_assert(std::is_same<InType, int8_t>());
149 + TORCH_CHECK(a.dtype() == torch::kInt8);
150 +
151 + using FallbackGemm =
152 + typename sm89_int8_fallback_gemm<InType, OutType,
153 + Epilogue>::Cutlass2xGemm;
154 +
155 + uint32_t const n = out.size(1);
156 + uint32_t const np2 = next_pow_2(n);
157 +
158 + if (np2 <= 8192) {
159 + using TileShape = cutlass::gemm::GemmShape<64, 128, 128>;
160 + using WarpShape = cutlass::gemm::GemmShape<64, 64, 64>;
161 +
162 + return vllm::fallback_cutlass_gemm_caller<
163 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
164 + InType, OutType, Epilogue, TileShape, WarpShape,
165 + InstructionShape, 3>,
166 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
167 + } else if (np2 <= 16384) {
168 + using TileShape = cutlass::gemm::GemmShape<128, 128, 64>;
169 + using WarpShape = cutlass::gemm::GemmShape<64, 64, 64>;
170 +
171 + return vllm::fallback_cutlass_gemm_caller<
172 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
173 + InType, OutType, Epilogue, TileShape, WarpShape,
174 + InstructionShape, 5>,
175 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
176 + } else {
177 + using TileShape = cutlass::gemm::GemmShape<64, 64, 128>;
178 + using WarpShape = cutlass::gemm::GemmShape<32, 64, 64>;
179 +
180 + return vllm::fallback_cutlass_gemm_caller<
181 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
182 + InType, OutType, Epilogue, TileShape, WarpShape,
183 + InstructionShape, 5>,
184 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
185 + }
186 + }
187 +};
188 +
189 +struct sm89_int8_config_M64 {
190 + // M in (32, 64]
191 + using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
192 +
193 + template <typename InType, typename OutType,
194 + template <typename, typename> typename Epilogue,
195 + typename... EpilogueArgs>
196 + static void dispatch(torch::Tensor& out, torch::Tensor const& a,
197 + torch::Tensor const& b, EpilogueArgs&&... args) {
198 + static_assert(std::is_same<InType, int8_t>());
199 + TORCH_CHECK(a.dtype() == torch::kInt8);
200 +
201 + using FallbackGemm =
202 + typename sm89_int8_fallback_gemm<InType, OutType,
203 + Epilogue>::Cutlass2xGemm;
204 +
205 + uint32_t const n = out.size(1);
206 + uint32_t const np2 = next_pow_2(n);
207 +
208 + if (np2 <= 8192) {
209 + using TileShape = cutlass::gemm::GemmShape<64, 64, 128>;
210 + using WarpShape = cutlass::gemm::GemmShape<32, 64, 64>;
211 +
212 + return vllm::fallback_cutlass_gemm_caller<
213 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
214 + InType, OutType, Epilogue, TileShape, WarpShape,
215 + InstructionShape, 5>,
216 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
217 + } else {
218 + using TileShape = cutlass::gemm::GemmShape<64, 128, 128>;
219 + using WarpShape = cutlass::gemm::GemmShape<64, 64, 64>;
220 +
221 + return vllm::fallback_cutlass_gemm_caller<
222 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
223 + InType, OutType, Epilogue, TileShape, WarpShape,
224 + InstructionShape, 3>,
225 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
226 + }
227 + }
228 +};
229 +
230 +struct sm89_int8_config_M32 {
231 + // M in (16, 32]
232 + using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
233 +
234 + template <typename InType, typename OutType,
235 + template <typename, typename> typename Epilogue,
236 + typename... EpilogueArgs>
237 + static void dispatch(torch::Tensor& out, torch::Tensor const& a,
238 + torch::Tensor const& b, EpilogueArgs&&... args) {
239 + static_assert(std::is_same<InType, int8_t>());
240 + TORCH_CHECK(a.dtype() == torch::kInt8);
241 +
242 + using FallbackGemm =
243 + typename sm89_int8_fallback_gemm<InType, OutType,
244 + Epilogue>::Cutlass2xGemm;
245 +
246 + uint32_t const n = out.size(1);
247 + uint32_t const np2 = next_pow_2(n);
248 +
249 + if (np2 <= 8192) {
250 + using TileShape = cutlass::gemm::GemmShape<32, 64, 128>;
251 + using WarpShape = cutlass::gemm::GemmShape<16, 64, 64>;
252 +
253 + return vllm::fallback_cutlass_gemm_caller<
254 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
255 + InType, OutType, Epilogue, TileShape, WarpShape,
256 + InstructionShape, 5>,
257 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
258 + } else {
259 + using TileShape = cutlass::gemm::GemmShape<32, 128, 128>;
260 + using WarpShape = cutlass::gemm::GemmShape<32, 64, 64>;
261 +
262 + return vllm::fallback_cutlass_gemm_caller<
263 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
264 + InType, OutType, Epilogue, TileShape, WarpShape,
265 + InstructionShape, 4>,
266 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
267 + }
268 + }
269 +};
270 +
271 +struct sm89_int8_config_M16 {
272 + // M in [1, 16]
273 + using WarpShape = typename cutlass::gemm::GemmShape<16, 64, 64>;
274 + using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
275 +
276 + template <typename InType, typename OutType,
277 + template <typename, typename> typename Epilogue,
278 + typename... EpilogueArgs>
279 + static void dispatch(torch::Tensor& out, torch::Tensor const& a,
280 + torch::Tensor const& b, EpilogueArgs&&... args) {
281 + static_assert(std::is_same<InType, int8_t>());
282 + TORCH_CHECK(a.dtype() == torch::kInt8);
283 +
284 + using FallbackGemm =
285 + typename sm89_int8_fallback_gemm<InType, OutType,
286 + Epilogue>::Cutlass2xGemm;
287 +
288 + uint32_t const n = out.size(1);
289 + uint32_t const np2 = next_pow_2(n);
290 +
291 + if (np2 <= 8192) {
292 + using TileShape = cutlass::gemm::GemmShape<16, 64, 128>;
293 +
294 + return vllm::fallback_cutlass_gemm_caller<
295 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
296 + InType, OutType, Epilogue, TileShape, WarpShape,
297 + InstructionShape, 5>,
298 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
299 + } else {
300 + using TileShape = cutlass::gemm::GemmShape<16, 128, 128>;
301 +
302 + return vllm::fallback_cutlass_gemm_caller<
303 + vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
304 + InType, OutType, Epilogue, TileShape, WarpShape,
305 + InstructionShape, 4>,
306 + FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
307 + }
308 + }
309 +};
310 +
311 +template <typename InType, typename OutType,
312 + template <typename, typename> typename Epilogue,
313 + typename... EpilogueArgs>
314 +inline void cutlass_gemm_sm89_int8_dispatch(torch::Tensor& out,
315 + torch::Tensor const& a,
316 + torch::Tensor const& b,
317 + EpilogueArgs&&... args) {
318 + static_assert(std::is_same<InType, int8_t>());
319 + TORCH_CHECK(a.dtype() == torch::kInt8);
320 + TORCH_CHECK(b.dtype() == torch::kInt8);
321 +
322 + uint32_t const m = a.size(0);
323 + uint32_t const mp2 =
324 + std::max(static_cast<uint32_t>(32), next_pow_2(m)); // next power of 2
325 +
326 + if (mp2 <= 16) {
327 + // M in [1, 16]
328 + return sm89_int8_config_M16::dispatch<InType, OutType, Epilogue>(
329 + out, a, b, std::forward<EpilogueArgs>(args)...);
330 + } else if (mp2 <= 32) {
331 + // M in (16, 32]
332 + return sm89_int8_config_M32::dispatch<InType, OutType, Epilogue>(
333 + out, a, b, std::forward<EpilogueArgs>(args)...);
334 + } else if (mp2 <= 64) {
335 + // M in (32, 64]
336 + return sm89_int8_config_M64::dispatch<InType, OutType, Epilogue>(
337 + out, a, b, std::forward<EpilogueArgs>(args)...);
338 + } else if (mp2 <= 128) {
339 + // M in (64, 128]
340 + return sm89_int8_config_M128::dispatch<InType, OutType, Epilogue>(
341 + out, a, b, std::forward<EpilogueArgs>(args)...);
342 + } else if (mp2 <= 256) {
343 + // M in (128, 256]
344 + return sm89_int8_config_M256::dispatch<InType, OutType, Epilogue>(
345 + out, a, b, std::forward<EpilogueArgs>(args)...);
346 + } else {
347 + // M in (256, inf)
348 + return sm89_int8_config_default::dispatch<InType, OutType, Epilogue>(
349 + out, a, b, std::forward<EpilogueArgs>(args)...);
350 + }
351 +}
352 +
353 +} // namespace vllm
tests/kernels/test_cutlass.py
+2 -2
@@ -119,8 +119,8 @@ def test_cutlass_fp8_gemm(m: int, n: int, k: int, per_act_token: bool,
119 cutlass_fp8_gemm_helper(m, n, k, per_act_token, per_out_ch, use_bias)
120
121
122 -@pytest.mark.parametrize("m", [512, 222, 33, 1])
123 -@pytest.mark.parametrize("n", [2048, 256, 1024])
122 +@pytest.mark.parametrize("m", [1, 16, 32, 64, 128, 256, 512, 222, 33, 1])
123 +@pytest.mark.parametrize("n", [2048, 8192, 16384, 256, 1024])
124 @pytest.mark.parametrize("k", [128, 496, 1024])
125 @pytest.mark.parametrize("per_act_token", [True, False])
126 @pytest.mark.parametrize("per_out_ch", [True, False])