A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 01:15 UTC
fix cutlass_3x_gemm_fp8_blockwise on sm103a (#32224)
Signed-off-by: Siyuan Fu <siyuanf@nvidia.com> Co-authored-by: Pavani Majety <pmajety@nvidia.com>
Lain committed
Feb 2, 2026 at 11:47 UTC
089cd4f002484599aeed366c31629dccf491ce81
7 files changed
+129
-82
csrc/cutlass_extensions/common.hpp
+88
-6
@@ -3,7 +3,8 @@
3
#include "cutlass/cutlass.h"
4
#include <climits>
5
#include "cuda_runtime.h"
6
-#include <iostream>
6
+#include <cstdio>
7
+#include <cstdlib>
8
9
/**
10
* Helper function for checking CUTLASS errors
@@ -31,12 +32,63 @@ int32_t get_sm_version_num();
32
* __CUDA_ARCH__ is not defined in host code, so this lets us smuggle the ifdef
33
* into code that will be executed on the device where it is defined.
34
*/
35
+
36
+template <typename Kernel>
37
+struct enable_sm75_to_sm80 : Kernel {
38
+ template <typename... Args>
39
+ CUTLASS_DEVICE static void invoke(Args&&... args) {
40
+#if defined __CUDA_ARCH__
41
+ #if __CUDA_ARCH__ >= 750 && __CUDA_ARCH__ < 800
42
+ Kernel::invoke(std::forward<Args>(args)...);
43
+ #else
44
+ printf("This kernel only supports sm[75, 80).\n");
45
+ asm("trap;");
46
+ #endif
47
+#endif
48
+ }
49
+};
50
+
51
+template <typename Kernel>
52
+struct enable_sm80_to_sm89 : Kernel {
53
+ template <typename... Args>
54
+ CUTLASS_DEVICE static void invoke(Args&&... args) {
55
+#if defined __CUDA_ARCH__
56
+ #if __CUDA_ARCH__ >= 800 && __CUDA_ARCH__ < 890
57
+ Kernel::invoke(std::forward<Args>(args)...);
58
+ #else
59
+ printf("This kernel only supports sm[80, 89).\n");
60
+ asm("trap;");
61
+ #endif
62
+#endif
63
+ }
64
+};
65
+
66
+template <typename Kernel>
67
+struct enable_sm89_to_sm90 : Kernel {
68
+ template <typename... Args>
69
+ CUTLASS_DEVICE static void invoke(Args&&... args) {
70
+#if defined __CUDA_ARCH__
71
+ #if __CUDA_ARCH__ >= 890 && __CUDA_ARCH__ < 900
72
+ Kernel::invoke(std::forward<Args>(args)...);
73
+ #else
74
+ printf("This kernel only supports sm[89, 90).\n");
75
+ asm("trap;");
76
+ #endif
77
+#endif
78
+ }
79
+};
80
+
81
template <typename Kernel>
82
struct enable_sm90_or_later : Kernel {
83
template <typename... Args>
84
CUTLASS_DEVICE void operator()(Args&&... args) {
38
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 900
85
+#if defined __CUDA_ARCH__
86
+ #if __CUDA_ARCH__ >= 900
87
Kernel::operator()(std::forward<Args>(args)...);
88
+ #else
89
+ printf("This kernel only supports sm >= 90.\n");
90
+ asm("trap;");
91
+ #endif
92
#endif
93
}
94
};
@@ -45,18 +97,43 @@ template <typename Kernel>
97
struct enable_sm90_only : Kernel {
98
template <typename... Args>
99
CUTLASS_DEVICE void operator()(Args&&... args) {
48
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ == 900
100
+#if defined __CUDA_ARCH__
101
+ #if __CUDA_ARCH__ == 900
102
+ Kernel::operator()(std::forward<Args>(args)...);
103
+ #else
104
+ printf("This kernel only supports sm90.\n");
105
+ asm("trap;");
106
+ #endif
107
+#endif
108
+ }
109
+};
110
+
111
+template <typename Kernel>
112
+struct enable_sm100f_only : Kernel {
113
+ template <typename... Args>
114
+ CUTLASS_DEVICE void operator()(Args&&... args) {
115
+#if defined __CUDA_ARCH__
116
+ #if __CUDA_ARCH__ == 1000 || __CUDA_ARCH__ == 1030
117
Kernel::operator()(std::forward<Args>(args)...);
118
+ #else
119
+ printf("This kernel only supports sm100f.\n");
120
+ asm("trap;");
121
+ #endif
122
#endif
123
}
124
};
125
126
template <typename Kernel>
55
-struct enable_sm100_only : Kernel {
127
+struct enable_sm100a_only : Kernel {
128
template <typename... Args>
129
CUTLASS_DEVICE void operator()(Args&&... args) {
58
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ == 1000
130
+#if defined __CUDA_ARCH__
131
+ #if __CUDA_ARCH__ == 1000
132
Kernel::operator()(std::forward<Args>(args)...);
133
+ #else
134
+ printf("This kernel only supports sm100a.\n");
135
+ asm("trap;");
136
+ #endif
137
#endif
138
}
139
};
@@ -65,8 +142,13 @@ template <typename Kernel>
142
struct enable_sm120_only : Kernel {
143
template <typename... Args>
144
CUTLASS_DEVICE void operator()(Args&&... args) {
68
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ == 1200
145
+#if defined __CUDA_ARCH__
146
+ #if __CUDA_ARCH__ == 1200
147
Kernel::operator()(std::forward<Args>(args)...);
148
+ #else
149
+ printf("This kernel only supports sm120.\n");
150
+ asm("trap;");
151
+ #endif
152
#endif
153
}
154
};
csrc/quantization/w8a8/cutlass/c3x/scaled_mm.cuh
+4
-4
@@ -141,8 +141,8 @@ struct cutlass_3x_gemm_sm100 {
141
sizeof(typename CollectiveEpilogue::SharedStorage))>,
142
KernelSchedule>::CollectiveOp;
143
144
- using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
145
- Shape<int, int, int, int>, CollectiveMainloop, CollectiveEpilogue, void>;
144
+ using GemmKernel = enable_sm100f_only<cutlass::gemm::kernel::GemmUniversal<
145
+ Shape<int, int, int, int>, CollectiveMainloop, CollectiveEpilogue, void>>;
146
};
147
148
template <typename ElementAB_, typename ElementD_,
@@ -202,8 +202,8 @@ struct cutlass_3x_gemm_sm120 {
202
sizeof(typename CollectiveEpilogue::SharedStorage))>,
203
KernelSchedule>::CollectiveOp;
204
205
- using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
206
- Shape<int, int, int, int>, CollectiveMainloop, CollectiveEpilogue, void>;
205
+ using GemmKernel = enable_sm120_only<cutlass::gemm::kernel::GemmUniversal<
206
+ Shape<int, int, int, int>, CollectiveMainloop, CollectiveEpilogue, void>>;
207
};
208
209
} // namespace vllm
csrc/quantization/w8a8/cutlass/c3x/scaled_mm_blockwise_sm100_fp8_dispatch.cuh
+1
-1
@@ -123,7 +123,7 @@ struct cutlass_3x_gemm_fp8_blockwise {
123
MainloopScheduler
124
>::CollectiveOp>;
125
126
- using KernelType = enable_sm100_only<cutlass::gemm::kernel::GemmUniversal<
126
+ using KernelType = enable_sm100f_only<cutlass::gemm::kernel::GemmUniversal<
127
Shape<int, int, int, int>, CollectiveMainloop, CollectiveEpilogue>>;
128
129
struct GemmKernel : public KernelType {};
csrc/quantization/w8a8/cutlass/c3x/scaled_mm_sm100_fp8_dispatch.cuh
+2
-2
@@ -90,8 +90,8 @@ struct cutlass_3x_gemm_sm100_fp8 {
90
// -----------------------------------------------------------
91
// Kernel definition
92
// -----------------------------------------------------------
93
- using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
94
- Shape<int, int, int, int>, CollectiveMainloop, CollectiveEpilogue, void>;
93
+ using GemmKernel = enable_sm100f_only<cutlass::gemm::kernel::GemmUniversal<
94
+ Shape<int, int, int, int>, CollectiveMainloop, CollectiveEpilogue, void>>;
95
};
96
97
template <typename InType, typename OutType, bool EnableBias>
csrc/quantization/w8a8/cutlass/scaled_mm_c2x.cuh
-35
@@ -36,41 +36,6 @@ using namespace cute;
36
*/
37
38
namespace vllm {
39
-
40
-// Wrappers for the GEMM kernel that is used to guard against compilation on
41
-// architectures that will never use the kernel. The purpose of this is to
42
-// reduce the size of the compiled binary.
43
-// __CUDA_ARCH__ is not defined in host code, so this lets us smuggle the ifdef
44
-// into code that will be executed on the device where it is defined.
45
-template <typename Kernel>
46
-struct enable_sm75_to_sm80 : Kernel {
47
- template <typename... Args>
48
- CUTLASS_DEVICE static void invoke(Args&&... args) {
49
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 750 && __CUDA_ARCH__ < 800
50
- Kernel::invoke(std::forward<Args>(args)...);
51
-#endif
52
- }
53
-};
54
-
55
-template <typename Kernel>
56
-struct enable_sm80_to_sm89 : Kernel {
57
- template <typename... Args>
58
- CUTLASS_DEVICE static void invoke(Args&&... args) {
59
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 800 && __CUDA_ARCH__ < 890
60
- Kernel::invoke(std::forward<Args>(args)...);
61
-#endif
62
- }
63
-};
64
-
65
-template <typename Kernel>
66
-struct enable_sm89_to_sm90 : Kernel {
67
- template <typename... Args>
68
- CUTLASS_DEVICE static void invoke(Args&&... args) {
69
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 890 && __CUDA_ARCH__ < 900
70
- Kernel::invoke(std::forward<Args>(args)...);
71
-#endif
72
- }
73
-};
39
template <typename Arch, template <typename> typename ArchGuard,
40
typename ElementAB_, typename ElementD_,
41
template <typename, typename> typename Epilogue_, typename TileShape,
csrc/quantization/w8a8/cutlass/scaled_mm_c2x_sm89_fp8_dispatch.cuh
+17
-17
@@ -50,7 +50,7 @@ struct sm89_fp8_config_default {
50
using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
51
52
return vllm::fallback_cutlass_gemm_caller<
53
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
53
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
54
InType, OutType, Epilogue, TileShape, WarpShape,
55
InstructionShape, 5, FP8MathOperator>,
56
FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
@@ -58,7 +58,7 @@ struct sm89_fp8_config_default {
58
using TileShape = typename cutlass::gemm::GemmShape<256, 128, 64>;
59
60
return vllm::fallback_cutlass_gemm_caller<
61
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
61
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
62
InType, OutType, Epilogue, TileShape, WarpShape,
63
InstructionShape, 3, FP8MathOperator>,
64
FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
@@ -67,7 +67,7 @@ struct sm89_fp8_config_default {
67
using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
68
69
return vllm::fallback_cutlass_gemm_caller<
70
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
70
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
71
InType, OutType, Epilogue, TileShape, WarpShape,
72
InstructionShape, 5, FP8MathOperator>,
73
FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
@@ -100,7 +100,7 @@ struct sm89_fp8_config_M256 {
100
using TileShape = typename cutlass::gemm::GemmShape<64, 128, 128>;
101
102
return vllm::fallback_cutlass_gemm_caller<
103
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
103
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
104
InType, OutType, Epilogue, TileShape, WarpShape,
105
InstructionShape, 3, FP8MathOperator>,
106
FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
@@ -108,7 +108,7 @@ struct sm89_fp8_config_M256 {
108
using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
109
110
return vllm::fallback_cutlass_gemm_caller<
111
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
111
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
112
InType, OutType, Epilogue, TileShape, WarpShape,
113
InstructionShape, 5, FP8MathOperator>,
114
FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
@@ -141,7 +141,7 @@ struct sm89_fp8_config_M128 {
141
using TileShape = typename cutlass::gemm::GemmShape<64, 128, 128>;
142
143
return vllm::fallback_cutlass_gemm_caller<
144
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
144
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
145
InType, OutType, Epilogue, TileShape, WarpShape,
146
InstructionShape, 3, FP8MathOperator>,
147
FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
@@ -150,7 +150,7 @@ struct sm89_fp8_config_M128 {
150
using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
151
152
return vllm::fallback_cutlass_gemm_caller<
153
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
153
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
154
InType, OutType, Epilogue, TileShape, WarpShape,
155
InstructionShape, 5, FP8MathOperator>,
156
FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
@@ -158,7 +158,7 @@ struct sm89_fp8_config_M128 {
158
using TileShape = typename cutlass::gemm::GemmShape<128, 64, 128>;
159
160
return vllm::fallback_cutlass_gemm_caller<
161
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
161
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
162
InType, OutType, Epilogue, TileShape, WarpShape,
163
InstructionShape, 3, FP8MathOperator>,
164
FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
@@ -191,7 +191,7 @@ struct sm89_fp8_config_M64 {
191
using FP8MathOperator = typename cutlass::arch::OpMultiplyAdd;
192
193
return vllm::fallback_cutlass_gemm_caller<
194
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
194
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
195
InType, OutType, Epilogue, TileShape, WarpShape,
196
InstructionShape, 5, FP8MathOperator>,
197
FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
@@ -201,7 +201,7 @@ struct sm89_fp8_config_M64 {
201
using FP8MathOperator = typename cutlass::arch::OpMultiplyAddFastAccum;
202
203
return vllm::fallback_cutlass_gemm_caller<
204
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
204
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
205
InType, OutType, Epilogue, TileShape, WarpShape,
206
InstructionShape, 3, FP8MathOperator>,
207
FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
@@ -211,7 +211,7 @@ struct sm89_fp8_config_M64 {
211
using FP8MathOperator = typename cutlass::arch::OpMultiplyAdd;
212
213
return vllm::fallback_cutlass_gemm_caller<
214
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
214
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
215
InType, OutType, Epilogue, TileShape, WarpShape,
216
InstructionShape, 5, FP8MathOperator>,
217
FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
@@ -244,7 +244,7 @@ struct sm89_fp8_config_M32 {
244
using WarpShape = typename cutlass::gemm::GemmShape<16, 64, 64>;
245
246
return vllm::fallback_cutlass_gemm_caller<
247
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
247
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
248
InType, OutType, Epilogue, TileShape, WarpShape,
249
InstructionShape, 5, FP8MathOperator>,
250
FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
@@ -253,7 +253,7 @@ struct sm89_fp8_config_M32 {
253
using WarpShape = typename cutlass::gemm::GemmShape<32, 64, 64>;
254
255
return vllm::fallback_cutlass_gemm_caller<
256
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
256
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
257
InType, OutType, Epilogue, TileShape, WarpShape,
258
InstructionShape, 4, FP8MathOperator>,
259
FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
@@ -262,7 +262,7 @@ struct sm89_fp8_config_M32 {
262
using WarpShape = typename cutlass::gemm::GemmShape<16, 64, 64>;
263
264
return vllm::fallback_cutlass_gemm_caller<
265
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
265
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
266
InType, OutType, Epilogue, TileShape, WarpShape,
267
InstructionShape, 5, FP8MathOperator>,
268
FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
@@ -296,7 +296,7 @@ struct sm89_fp8_config_M16 {
296
using TileShape = typename cutlass::gemm::GemmShape<16, 64, 128>;
297
298
return vllm::fallback_cutlass_gemm_caller<
299
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
299
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
300
InType, OutType, Epilogue, TileShape, WarpShape,
301
InstructionShape, MainLoopStages,
302
FP8MathOperator>,
@@ -305,7 +305,7 @@ struct sm89_fp8_config_M16 {
305
using TileShape = typename cutlass::gemm::GemmShape<16, 128, 64>;
306
307
return vllm::fallback_cutlass_gemm_caller<
308
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
308
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
309
InType, OutType, Epilogue, TileShape, WarpShape,
310
InstructionShape, MainLoopStages,
311
FP8MathOperator>,
@@ -314,7 +314,7 @@ struct sm89_fp8_config_M16 {
314
using TileShape = typename cutlass::gemm::GemmShape<32, 64, 128>;
315
316
return vllm::fallback_cutlass_gemm_caller<
317
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
317
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
318
InType, OutType, Epilogue, TileShape, WarpShape,
319
InstructionShape, MainLoopStages,
320
FP8MathOperator>,
csrc/quantization/w8a8/cutlass/scaled_mm_c2x_sm89_int8_dispatch.cuh
+17
-17
@@ -48,7 +48,7 @@ struct sm89_int8_config_default {
48
using TileShape = cutlass::gemm::GemmShape<128, 128, 64>;
49
50
return vllm::fallback_cutlass_gemm_caller<
51
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
51
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
52
InType, OutType, Epilogue, TileShape, WarpShape,
53
InstructionShape, 5>,
54
FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
@@ -56,7 +56,7 @@ struct sm89_int8_config_default {
56
using TileShape = cutlass::gemm::GemmShape<256, 128, 64>;
57
58
return vllm::fallback_cutlass_gemm_caller<
59
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
59
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
60
InType, OutType, Epilogue, TileShape, WarpShape,
61
InstructionShape, 3>,
62
FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
@@ -64,7 +64,7 @@ struct sm89_int8_config_default {
64
using TileShape = cutlass::gemm::GemmShape<128, 128, 64>;
65
66
return vllm::fallback_cutlass_gemm_caller<
67
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
67
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
68
InType, OutType, Epilogue, TileShape, WarpShape,
69
InstructionShape, 5>,
70
FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
@@ -72,7 +72,7 @@ struct sm89_int8_config_default {
72
using TileShape = cutlass::gemm::GemmShape<256, 128, 64>;
73
74
return vllm::fallback_cutlass_gemm_caller<
75
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
75
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
76
InType, OutType, Epilogue, TileShape, WarpShape,
77
InstructionShape, 3>,
78
FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
@@ -104,7 +104,7 @@ struct sm89_int8_config_M256 {
104
using TileShape = cutlass::gemm::GemmShape<64, 128, 128>;
105
106
return vllm::fallback_cutlass_gemm_caller<
107
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
107
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
108
InType, OutType, Epilogue, TileShape, WarpShape,
109
InstructionShape, 3>,
110
FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
@@ -112,7 +112,7 @@ struct sm89_int8_config_M256 {
112
using TileShape = cutlass::gemm::GemmShape<128, 128, 64>;
113
114
return vllm::fallback_cutlass_gemm_caller<
115
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
115
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
116
InType, OutType, Epilogue, TileShape, WarpShape,
117
InstructionShape, 5>,
118
FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
@@ -120,7 +120,7 @@ struct sm89_int8_config_M256 {
120
using TileShape = cutlass::gemm::GemmShape<256, 128, 64>;
121
122
return vllm::fallback_cutlass_gemm_caller<
123
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
123
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
124
InType, OutType, Epilogue, TileShape, WarpShape,
125
InstructionShape, 3>,
126
FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
@@ -128,7 +128,7 @@ struct sm89_int8_config_M256 {
128
using TileShape = cutlass::gemm::GemmShape<128, 128, 64>;
129
130
return vllm::fallback_cutlass_gemm_caller<
131
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
131
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
132
InType, OutType, Epilogue, TileShape, WarpShape,
133
InstructionShape, 5>,
134
FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
@@ -160,7 +160,7 @@ struct sm89_int8_config_M128 {
160
using WarpShape = cutlass::gemm::GemmShape<64, 64, 64>;
161
162
return vllm::fallback_cutlass_gemm_caller<
163
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
163
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
164
InType, OutType, Epilogue, TileShape, WarpShape,
165
InstructionShape, 3>,
166
FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
@@ -169,7 +169,7 @@ struct sm89_int8_config_M128 {
169
using WarpShape = cutlass::gemm::GemmShape<64, 64, 64>;
170
171
return vllm::fallback_cutlass_gemm_caller<
172
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
172
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
173
InType, OutType, Epilogue, TileShape, WarpShape,
174
InstructionShape, 5>,
175
FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
@@ -178,7 +178,7 @@ struct sm89_int8_config_M128 {
178
using WarpShape = cutlass::gemm::GemmShape<32, 64, 64>;
179
180
return vllm::fallback_cutlass_gemm_caller<
181
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
181
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
182
InType, OutType, Epilogue, TileShape, WarpShape,
183
InstructionShape, 5>,
184
FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
@@ -210,7 +210,7 @@ struct sm89_int8_config_M64 {
210
using WarpShape = cutlass::gemm::GemmShape<32, 64, 64>;
211
212
return vllm::fallback_cutlass_gemm_caller<
213
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
213
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
214
InType, OutType, Epilogue, TileShape, WarpShape,
215
InstructionShape, 5>,
216
FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
@@ -219,7 +219,7 @@ struct sm89_int8_config_M64 {
219
using WarpShape = cutlass::gemm::GemmShape<64, 64, 64>;
220
221
return vllm::fallback_cutlass_gemm_caller<
222
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
222
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
223
InType, OutType, Epilogue, TileShape, WarpShape,
224
InstructionShape, 3>,
225
FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
@@ -251,7 +251,7 @@ struct sm89_int8_config_M32 {
251
using WarpShape = cutlass::gemm::GemmShape<16, 64, 64>;
252
253
return vllm::fallback_cutlass_gemm_caller<
254
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
254
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
255
InType, OutType, Epilogue, TileShape, WarpShape,
256
InstructionShape, 5>,
257
FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
@@ -260,7 +260,7 @@ struct sm89_int8_config_M32 {
260
using WarpShape = cutlass::gemm::GemmShape<32, 64, 64>;
261
262
return vllm::fallback_cutlass_gemm_caller<
263
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
263
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
264
InType, OutType, Epilogue, TileShape, WarpShape,
265
InstructionShape, 4>,
266
FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
@@ -292,7 +292,7 @@ struct sm89_int8_config_M16 {
292
using TileShape = cutlass::gemm::GemmShape<16, 64, 128>;
293
294
return vllm::fallback_cutlass_gemm_caller<
295
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
295
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
296
InType, OutType, Epilogue, TileShape, WarpShape,
297
InstructionShape, 5>,
298
FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);
@@ -300,7 +300,7 @@ struct sm89_int8_config_M16 {
300
using TileShape = cutlass::gemm::GemmShape<16, 128, 128>;
301
302
return vllm::fallback_cutlass_gemm_caller<
303
- vllm::cutlass_2x_gemm<cutlass::arch::Sm89, vllm::enable_sm89_to_sm90,
303
+ vllm::cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
304
InType, OutType, Epilogue, TileShape, WarpShape,
305
InstructionShape, 4>,
306
FallbackGemm>(out, a, b, std::forward<EpilogueArgs>(args)...);