A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Kernel] Add w8a8 CUTLASS kernels (#4749)
Tyler Michael Smith committed
May 16, 2024 at 18:32 UTC
2060e93659f1f63a3d2a76aee61559ccb1fe732e
10 files changed
+1197
-2
CMakeLists.txt
+26
-1
@@ -173,6 +173,16 @@ set(VLLM_EXT_SRC
173
"csrc/pybind.cpp")
174
175
if(VLLM_GPU_LANG STREQUAL "CUDA")
176
+ include(FetchContent)
177
+ SET(CUTLASS_ENABLE_HEADERS_ONLY=ON)
178
+ FetchContent_Declare(
179
+ cutlass
180
+ GIT_REPOSITORY https://github.com/nvidia/cutlass.git
181
+ # CUTLASS 3.5.0
182
+ GIT_TAG 7d49e6c7e2f8896c47f586706e67e1fb215529dc
183
+ )
184
+ FetchContent_MakeAvailable(cutlass)
185
+
186
list(APPEND VLLM_EXT_SRC
187
"csrc/quantization/aqlm/gemm_kernels.cu"
188
"csrc/quantization/awq/gemm_kernels.cu"
@@ -180,7 +190,21 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
190
"csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu"
191
"csrc/quantization/gptq_marlin/gptq_marlin.cu"
192
"csrc/quantization/gptq_marlin/gptq_marlin_repack.cu"
183
- "csrc/custom_all_reduce.cu")
193
+ "csrc/custom_all_reduce.cu"
194
+ "csrc/quantization/cutlass_w8a8/scaled_mm_dq_entry.cu"
195
+ "csrc/quantization/cutlass_w8a8/scaled_mm_dq_c2x.cu"
196
+ "csrc/quantization/cutlass_w8a8/scaled_mm_dq_c3x.cu")
197
+
198
+ #
199
+ # The CUTLASS kernels for Hopper require sm90a to be enabled.
200
+ # This is done via the below gencode option, BUT that creates kernels for both sm90 and sm90a.
201
+ # That adds an extra 17MB to compiled binary, so instead we selectively enable it.
202
+ set_source_files_properties(
203
+ "csrc/quantization/cutlass_w8a8/scaled_mm_dq_c3x.cu"
204
+ PROPERTIES
205
+ COMPILE_FLAGS
206
+ "-gencode arch=compute_90a,code=sm_90a")
207
+
208
endif()
209
210
define_gpu_extension_target(
@@ -190,6 +214,7 @@ define_gpu_extension_target(
214
SOURCES ${VLLM_EXT_SRC}
215
COMPILE_FLAGS ${VLLM_GPU_FLAGS}
216
ARCHITECTURES ${VLLM_GPU_ARCHES}
217
+ INCLUDE_DIRECTORIES ${CUTLASS_INCLUDE_DIR};${CUTLASS_TOOLS_UTIL_INCLUDE_DIR}
218
WITH_SOABI)
219
220
#
csrc/ops.h
+8
@@ -155,6 +155,14 @@ torch::Tensor gptq_marlin_repack(
155
int64_t size_k,
156
int64_t size_n,
157
int64_t num_bits);
158
+
159
+int cutlass_scaled_mm_dq(
160
+ torch::Tensor& out,
161
+ torch::Tensor const &a,
162
+ torch::Tensor const &b,
163
+ torch::Tensor const &a_scales,
164
+ torch::Tensor const &b_scales);
165
+
166
#endif
167
168
void squeezellm_gemm(
csrc/pybind.cpp
+1
@@ -71,6 +71,7 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
71
ops.def("gptq_marlin_gemm", &gptq_marlin_gemm, "gptq_marlin Optimized Quantized GEMM for GPTQ");
72
ops.def("gptq_marlin_repack", &gptq_marlin_repack, "gptq_marlin repack from GPTQ");
73
ops.def("awq_dequantize", &awq_dequantize, "Dequantization for AWQ");
74
+ ops.def("cutlass_scaled_mm_dq", &cutlass_scaled_mm_dq, "CUTLASS w8a8 GEMM, supporting symmetric per-tensor or per-row/column quantization.");
75
#endif
76
77
ops.def("gptq_gemm", &gptq_gemm, "Quantized GEMM for GPTQ");
csrc/quantization/cutlass_w8a8/common.hpp
new
+12
@@ -0,0 +1,12 @@
1
+#pragma once
2
+
3
+#include "cutlass/cutlass.h"
4
+
5
+/**
6
+ * Helper function for checking CUTLASS errors
7
+ */
8
+#define CUTLASS_CHECK(status) \
9
+ { \
10
+ TORCH_CHECK(status == cutlass::Status::kSuccess, \
11
+ cutlassGetStatusString(status)) \
12
+ }
csrc/quantization/cutlass_w8a8/cutlass_visitor_2x_broadcast_epilogue.hpp
new
+340
@@ -0,0 +1,340 @@
1
+/***************************************************************************************************
2
+ * Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights
3
+ *reserved. SPDX-License-Identifier: BSD-3-Clause
4
+ *
5
+ * Redistribution and use in source and binary forms, with or without
6
+ * modification, are permitted provided that the following conditions are met:
7
+ *
8
+ * 1. Redistributions of source code must retain the above copyright notice,
9
+ *this list of conditions and the following disclaimer.
10
+ *
11
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
12
+ * this list of conditions and the following disclaimer in the documentation
13
+ * and/or other materials provided with the distribution.
14
+ *
15
+ * 3. Neither the name of the copyright holder nor the names of its
16
+ * contributors may be used to endorse or promote products derived from
17
+ * this software without specific prior written permission.
18
+ *
19
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22
+ *ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
23
+ *LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24
+ *CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25
+ *SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26
+ *INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27
+ *CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28
+ *ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29
+ *POSSIBILITY OF SUCH DAMAGE.
30
+ *
31
+ **************************************************************************************************/
32
+
33
+//
34
+// This file is a modified excerpt of
35
+// include/cutlass/epilogue/fusion/visitor_load.hpp from
36
+// https://github.com/NVIDIA/cutlass It's beem modified to support either
37
+// row/column or scalar broadcasting, like is already supported in CUTLASS 3.x.
38
+// Important because this saves us a factor 4x on the number of kernels
39
+// compiled.
40
+//
41
+#pragma once
42
+
43
+// clang-format off
44
+
45
+#include "cutlass/epilogue/threadblock/fusion/visitor_2x.hpp"
46
+#include "cute/tensor.hpp"
47
+
48
+// clang-format on
49
+
50
+namespace cutlass::epilogue::threadblock {
51
+
52
+using namespace cute;
53
+using namespace detail;
54
+
55
+template<
56
+ class ThreadMap,
57
+ class Element,
58
+ class StrideMNL
59
+>
60
+struct VisitorRowOrScalarBroadcast {
61
+
62
+ struct Arguments {
63
+ Element const* ptr_row = nullptr;
64
+ Element null_default = Element(0);
65
+ StrideMNL dRow = {};
66
+ };
67
+
68
+ using Params = Arguments;
69
+
70
+ template <class ProblemShape>
71
+ static constexpr Params
72
+ to_underlying_arguments(ProblemShape const& problem_shape, Arguments const& args, void* workspace) {
73
+ return args;
74
+ }
75
+
76
+ template <class ProblemShape>
77
+ static size_t
78
+ get_workspace_size(ProblemShape const& problem_shape, Arguments const& args) {
79
+ return 0;
80
+ }
81
+
82
+ struct SharedStorage {};
83
+
84
+ // Global load type
85
+ static int constexpr vec_bits = ThreadMap::kElementsPerAccess * sizeof_bits<Element>::value;
86
+ using VecType = uint_bit_t<cute::min(128, vec_bits)>;
87
+ static int constexpr VecLength = sizeof(VecType) / sizeof(Element);
88
+
89
+ CUTLASS_HOST_DEVICE
90
+ VisitorRowOrScalarBroadcast() { }
91
+
92
+ CUTLASS_HOST_DEVICE
93
+ VisitorRowOrScalarBroadcast(Params const& params, SharedStorage const& shared_storage)
94
+ : params_ptr(¶ms) { }
95
+
96
+ Params const* params_ptr;
97
+
98
+ template <class GTensor, class RTensor, class CTensor, class ProblemShape>
99
+ struct Callbacks : EmptyCallbacks {
100
+ CUTLASS_DEVICE
101
+ Callbacks(
102
+ GTensor&& tC_gRow,
103
+ RTensor&& tC_rRow,
104
+ CTensor&& tC_cRow,
105
+ ProblemShape problem_shape,
106
+ Params const* params_ptr
107
+ ):
108
+ tC_gRow(cute::forward<GTensor>(tC_gRow)),
109
+ tC_rRow(cute::forward<RTensor>(tC_rRow)),
110
+ tC_cRow(cute::forward<CTensor>(tC_cRow)),
111
+ n(get<1>(problem_shape)),
112
+ params_ptr(params_ptr) { }
113
+
114
+ GTensor tC_gRow;
115
+ RTensor tC_rRow;
116
+ CTensor tC_cRow;
117
+ Params const* params_ptr;
118
+ int n;
119
+
120
+ // This function is modified from VisitorRowBroadcast
121
+ CUTLASS_DEVICE void
122
+ begin_epilogue() {
123
+ clear(tC_rRow);
124
+ auto src_v = filter(tC_gRow);
125
+ auto coord_v = filter(tC_cRow);
126
+ auto dst_v = filter(tC_rRow);
127
+
128
+ if (params_ptr->ptr_row) {
129
+ // In this case we are loading from a row vector and broadcasting
130
+ CUTLASS_PRAGMA_UNROLL
131
+ for (int i = 0; i < size(src_v); ++i) {
132
+ bool guard = get<1>(coord_v(i)) < n;
133
+ cutlass::arch::global_load<VecType, sizeof(VecType)>(dst_v(i), (void const*)&src_v(i), guard);
134
+ }
135
+ } else {
136
+ // In this case we are loading from a scalar and broadcasting
137
+ VecType filled_vec;
138
+ CUTLASS_PRAGMA_UNROLL
139
+ for (int i = 0; i < VecLength; i++) {
140
+ reinterpret_cast<Element*>(&filled_vec)[i] = params_ptr->null_default;
141
+ }
142
+
143
+ CUTLASS_PRAGMA_UNROLL
144
+ for (int i = 0; i < size(src_v); ++i) {
145
+ if(get<1>(coord_v(i)) < n)
146
+ {
147
+ dst_v(i) = filled_vec;
148
+ }
149
+ }
150
+ }
151
+ }
152
+
153
+ template <class ElementAccumulator, int FragmentSize>
154
+ CUTLASS_DEVICE auto // returns an Array
155
+ visit(int iter_idx, int row_idx, int column_idx, int frg_idx,
156
+ Array<ElementAccumulator, FragmentSize> const& frg_acc) {
157
+ Tensor rRow_frg = recast<Array<Element, FragmentSize>>(coalesce(tC_rRow));
158
+ return rRow_frg(column_idx);
159
+ }
160
+ };
161
+
162
+ template <class ProblemShape>
163
+ CUTLASS_DEVICE auto
164
+ get_callbacks(
165
+ gemm::GemmCoord threadblock_tile_offset,
166
+ int thread_idx,
167
+ ProblemShape problem_shape
168
+ ) {
169
+ Tensor mRow = make_tensor(
170
+ make_gmem_ptr(params_ptr->ptr_row),
171
+ problem_shape,
172
+ params_ptr->dRow);
173
+
174
+ // VECTOR, FRAGMENT_COLUMN
175
+ Tensor tC_gRow = recast<VecType>(
176
+ ThreadMap::partition(mRow, thread_idx, threadblock_tile_offset)
177
+ )(_,_,_0{},_0{},_0{},_0{});
178
+ Tensor tC_rRow = make_tensor_like(tC_gRow);
179
+
180
+ // Generate the pred tensor
181
+ Tensor cRow = make_identity_tensor(mRow.shape());
182
+ Tensor tC_cRow = outer_partition(
183
+ ThreadMap::partition(cRow, thread_idx, threadblock_tile_offset)(_,_,_0{},_0{},_0{},_0{}),
184
+ Shape<Int<VecLength>>{},
185
+ (_0{})
186
+ );
187
+
188
+ return Callbacks<
189
+ decltype(tC_gRow), decltype(tC_rRow),
190
+ decltype(tC_cRow), ProblemShape>(
191
+ cute::move(tC_gRow),
192
+ cute::move(tC_rRow),
193
+ cute::move(tC_cRow),
194
+ problem_shape,
195
+ params_ptr
196
+ );
197
+ }
198
+
199
+};
200
+
201
+/////////////////////////////////////////////////////////////////////////////////////////////////
202
+
203
+// Column vector broadcast
204
+template<
205
+ class ThreadMap,
206
+ class Element,
207
+ class StrideMNL = Stride<_1,_0,_0>
208
+>
209
+struct VisitorColOrScalarBroadcast {
210
+
211
+ struct Arguments {
212
+ Element const* ptr_col = nullptr;
213
+ Element null_default = Element(0);
214
+ StrideMNL dCol = {};
215
+ };
216
+
217
+ using Params = Arguments;
218
+
219
+ template <class ProblemShape>
220
+ static constexpr Params
221
+ to_underlying_arguments(ProblemShape const& problem_shape, Arguments const& args, void* workspace) {
222
+ return args;
223
+ }
224
+
225
+ template <class ProblemShape>
226
+ static size_t
227
+ get_workspace_size(ProblemShape const& problem_shape, Arguments const& args) {
228
+ return 0;
229
+ }
230
+
231
+ struct SharedStorage { };
232
+
233
+ // Global load type
234
+ static int constexpr vec_bits = ThreadMap::kElementsPerAccess * sizeof_bits<Element>::value;
235
+ using VecType = uint_bit_t<cute::min(128, vec_bits)>;
236
+ static int constexpr VecLength = sizeof(VecType) / sizeof(Element);
237
+
238
+ CUTLASS_HOST_DEVICE
239
+ VisitorColOrScalarBroadcast() { }
240
+
241
+ CUTLASS_HOST_DEVICE
242
+ VisitorColOrScalarBroadcast(Params const& params, SharedStorage const& shared_storage)
243
+ : params_ptr(¶ms) { }
244
+
245
+ Params const* params_ptr;
246
+
247
+ template <class GTensor, class RTensor, class CTensor, class ProblemShape>
248
+ struct Callbacks : EmptyCallbacks {
249
+ CUTLASS_DEVICE
250
+ Callbacks(
251
+ GTensor&& tC_gCol,
252
+ RTensor&& tC_rCol,
253
+ CTensor&& tC_cCol,
254
+ ProblemShape problem_shape,
255
+ Params const* params_ptr
256
+ ):
257
+ tC_gCol(cute::forward<GTensor>(tC_gCol)),
258
+ tC_rCol(cute::forward<RTensor>(tC_rCol)),
259
+ tC_cCol(cute::forward<CTensor>(tC_cCol)),
260
+ m(get<0>(problem_shape)),
261
+ params_ptr(params_ptr) { }
262
+
263
+ GTensor tC_gCol;
264
+ RTensor tC_rCol;
265
+ CTensor tC_cCol;
266
+ Params const* params_ptr;
267
+ int m;
268
+
269
+ // This function is modified from VisitorColBroadcast
270
+ CUTLASS_DEVICE void
271
+ begin_epilogue() {
272
+ clear(tC_rCol);
273
+
274
+ Tensor pred = make_tensor<bool>(shape(tC_gCol));
275
+ CUTLASS_PRAGMA_UNROLL
276
+ for (int i = 0; i < size(pred); ++i) {
277
+ pred(i) = get<0>(tC_cCol(i)) < m;
278
+ }
279
+
280
+ if (params_ptr->ptr_col) {
281
+ // In this case we are loading from a column vector and broadcasting
282
+ copy_if(pred, tC_gCol, tC_rCol);
283
+ } else {
284
+ // In this case we are loading from a scalar and broadcasting
285
+ auto dst_v = filter(tC_rCol);
286
+
287
+ CUTLASS_PRAGMA_UNROLL
288
+ for (int i = 0; i < size(dst_v); ++i) {
289
+ if(pred(i)){
290
+ dst_v(i) = params_ptr->null_default;
291
+ }
292
+ }
293
+ }
294
+ }
295
+
296
+ template <class ElementAccumulator, int FragmentSize>
297
+ CUTLASS_DEVICE auto // returns an Array
298
+ visit(int iter_idx, int row_idx, int column_idx, int frg_idx,
299
+ Array<ElementAccumulator, FragmentSize> const& frg_acc) {
300
+ Array<Element, FragmentSize> frg_col;
301
+ frg_col.fill(tC_rCol(row_idx,iter_idx));
302
+ return frg_col;
303
+ }
304
+ };
305
+
306
+ template <class ProblemShape>
307
+ CUTLASS_DEVICE auto
308
+ get_callbacks(
309
+ gemm::GemmCoord threadblock_tile_offset,
310
+ int thread_idx,
311
+ ProblemShape problem_shape
312
+ ) {
313
+ Tensor mCol = make_tensor(
314
+ make_gmem_ptr(params_ptr->ptr_col),
315
+ problem_shape,
316
+ params_ptr->dCol);
317
+
318
+ // VECTOR, FRAGMENT_COLUMN, FRAGMENT_ROW, ITERATION_ROW, ITERATION_GROUP, ITERATION_CLUSTER
319
+ Tensor tC_gCol = group_modes<1,4>(
320
+ ThreadMap::partition(mCol, thread_idx, threadblock_tile_offset)(_0{},_0{},_,_,_,_));
321
+ Tensor tC_rCol = make_tensor_like(tC_gCol);
322
+
323
+ // Generate the pred tensor
324
+ Tensor cCol = make_identity_tensor(mCol.shape());
325
+ Tensor tC_cCol = group_modes<1,4>(
326
+ ThreadMap::partition(cCol, thread_idx, threadblock_tile_offset)(_0{},_0{},_,_,_,_));
327
+
328
+ return Callbacks<
329
+ decltype(tC_gCol), decltype(tC_rCol),
330
+ decltype(tC_cCol), ProblemShape>(
331
+ cute::move(tC_gCol),
332
+ cute::move(tC_rCol),
333
+ cute::move(tC_cCol),
334
+ problem_shape,
335
+ params_ptr
336
+ );
337
+ }
338
+};
339
+
340
+}
csrc/quantization/cutlass_w8a8/scaled_mm_dq_c2x.cu
new
+296
@@ -0,0 +1,296 @@
1
+#include <stddef.h>
2
+#include <torch/extension.h>
3
+
4
+// clang-format will break include orders
5
+// clang-format off
6
+#include "cute/tensor.hpp"
7
+#include "cute/atom/mma_atom.hpp"
8
+#include "cutlass/numeric_types.h"
9
+
10
+#include "cutlass/util/device_memory.h"
11
+
12
+#include "cutlass/cutlass.h"
13
+#include "cutlass/gemm_coord.h"
14
+#include "cutlass/arch/mma_sm75.h"
15
+#include "cutlass/arch/arch.h"
16
+#include "cutlass/arch/mma.h"
17
+#include "cutlass/gemm/device/gemm.h"
18
+#include "cutlass/gemm/device/gemm_universal_adapter.h"
19
+
20
+#include "cutlass/epilogue/threadblock/fusion/visitors.hpp"
21
+#include "cutlass/gemm/kernel/default_gemm_universal_with_visitor.h"
22
+
23
+#include "cutlass_visitor_2x_broadcast_epilogue.hpp"
24
+#include "common.hpp"
25
+// clang-format on
26
+
27
+using namespace cute;
28
+
29
+/*
30
+ This defines a quantized GEMM operation with dequantized output, similar to
31
+ torch._scaled_mm. It is defined using the CUTLASS 2.x API, and is used for
32
+ NVIDIA GPUs with SM versions prior to sm90 (Hopper).
33
+
34
+ A and B may be both either int8 or fp8_e4m3. A can be quantized per-tensor or
35
+ per-row. B can be quantized per-tensor or per-column.
36
+ Any combination of per-tensor and per-row or column is supported.
37
+ A and B must have symmetric quantization (zero point == 0).
38
+
39
+ So the GEMM operation is D = (a_scales * A) (b_scales * B), where the
40
+ scales are applied elementwise with numpy-style broadcasting.
41
+
42
+ ScaleA and ScaleB define the epilogue functions that apply the scales for
43
+ the A and B operands respectively. These scales may be either per-tensor or
44
+ per row or column.
45
+*/
46
+
47
+namespace {
48
+
49
+template <typename Arch, typename ElementAB_, typename ElementD_,
50
+ typename TileShape, typename WarpShape, typename InstructionShape,
51
+ int32_t MainLoopStages>
52
+struct cutlass_2x_gemm {
53
+ using ElementAB = ElementAB_;
54
+ using ElementD = ElementD_;
55
+
56
+ using ElementAcc =
57
+ typename std::conditional<std::is_same_v<ElementAB, int8_t>, int32_t,
58
+ float>::type;
59
+
60
+ using Operator =
61
+ typename std::conditional<std::is_same_v<ElementAB, int8_t>,
62
+ cutlass::arch::OpMultiplyAddSaturate,
63
+ cutlass::arch::OpMultiplyAdd>::type;
64
+
65
+ using OutputTileThreadMap =
66
+ cutlass::epilogue::threadblock::OutputTileThreadLayout<
67
+ TileShape, WarpShape, float, 4, 1 /* epilogue stages */
68
+ >;
69
+
70
+ using Accum = cutlass::epilogue::threadblock::VisitorAccFetch;
71
+
72
+ using ScaleA = cutlass::epilogue::threadblock::VisitorColOrScalarBroadcast<
73
+ OutputTileThreadMap, float, Stride<Int<1>, Int<0>, Int<0>>>;
74
+
75
+ using ScaleB = cutlass::epilogue::threadblock::VisitorRowOrScalarBroadcast<
76
+ OutputTileThreadMap, float, Stride<Int<0>, Int<1>, Int<0>>>;
77
+
78
+ using Compute0 = cutlass::epilogue::threadblock::VisitorCompute<
79
+ cutlass::multiplies, float, float,
80
+ cutlass::FloatRoundStyle::round_to_nearest>;
81
+
82
+ using EVTCompute0 =
83
+ cutlass::epilogue::threadblock::Sm80EVT<Compute0, ScaleB, Accum>;
84
+
85
+ using Compute1 = cutlass::epilogue::threadblock::VisitorCompute<
86
+ cutlass::multiplies, ElementD, float,
87
+ cutlass::FloatRoundStyle::round_to_nearest>;
88
+
89
+ using EVTCompute1 =
90
+ cutlass::epilogue::threadblock::Sm80EVT<Compute1, ScaleA, EVTCompute0>;
91
+
92
+ using D = cutlass::epilogue::threadblock::VisitorAuxStore<
93
+ OutputTileThreadMap, ElementD, cutlass::FloatRoundStyle::round_to_nearest,
94
+ Stride<int64_t, Int<1>, Int<0>>>;
95
+
96
+ using EVTD = cutlass::epilogue::threadblock::Sm80EVT<D, EVTCompute1>;
97
+
98
+ // clang-format off
99
+ using RowMajor = typename cutlass::layout::RowMajor;
100
+ using ColumnMajor = typename cutlass::layout::ColumnMajor;
101
+ using KernelType =
102
+ typename cutlass::gemm::kernel::DefaultGemmWithVisitor<
103
+ ElementAB, RowMajor, cutlass::ComplexTransform::kNone, 16,
104
+ ElementAB, ColumnMajor, cutlass::ComplexTransform::kNone, 16,
105
+ float, cutlass::layout::RowMajor, 4,
106
+ ElementAcc, float, cutlass::arch::OpClassTensorOp,
107
+ Arch,
108
+ TileShape, WarpShape, InstructionShape,
109
+ EVTD,
110
+ cutlass::gemm::threadblock::ThreadblockSwizzleStreamK,
111
+ MainLoopStages, Operator,
112
+ 1 /* epilogue stages */
113
+ >::GemmKernel;
114
+ // clang-format on
115
+
116
+ using Op = cutlass::gemm::device::GemmUniversalAdapter<KernelType>;
117
+};
118
+
119
+template <typename Gemm>
120
+void cutlass_scaled_mm_dq_dispatcher(torch::Tensor &out, torch::Tensor const &a,
121
+ torch::Tensor const &b,
122
+ torch::Tensor const &a_scales,
123
+ torch::Tensor const &b_scales) {
124
+ using ElementAB = typename Gemm::ElementAB;
125
+ using ElementD = typename Gemm::ElementD;
126
+
127
+ int32_t m = a.size(0);
128
+ int32_t n = b.size(1);
129
+ int32_t k = a.size(1);
130
+ cutlass::gemm::GemmCoord problem_size{m, n, k};
131
+
132
+ int64_t lda = a.stride(0);
133
+ int64_t ldb = b.stride(1);
134
+ int64_t ldc = out.stride(0);
135
+
136
+ using StrideC = Stride<int64_t, Int<1>, Int<0>>;
137
+ StrideC c_stride{ldc, Int<1>{}, Int<0>{}};
138
+
139
+ auto a_ptr = static_cast<ElementAB const *>(a.data_ptr());
140
+ auto b_ptr = static_cast<ElementAB const *>(b.data_ptr());
141
+ auto c_ptr = static_cast<ElementD *>(out.data_ptr());
142
+
143
+ auto a_scales_ptr = a_scales.data_ptr<float>();
144
+ auto b_scales_ptr = b_scales.data_ptr<float>();
145
+
146
+ // If A and B are quantized per-tensor, then these scale tensors are scalars,
147
+ // and they are passed in via the second argument.
148
+ using ScaleAArgs = typename Gemm::ScaleA::Arguments;
149
+ ScaleAArgs a_args = a_scales.numel() == 1
150
+ ? ScaleAArgs{nullptr, a_scales.item<float>(), {}}
151
+ : ScaleAArgs{a_scales.data_ptr<float>(), {}, {}};
152
+
153
+ using ScaleBArgs = typename Gemm::ScaleB::Arguments;
154
+ ScaleBArgs b_args = b_scales.numel() == 1
155
+ ? ScaleBArgs{nullptr, b_scales.item<float>(), {}}
156
+ : ScaleBArgs{b_scales.data_ptr<float>(), {}, {}};
157
+
158
+ typename Gemm::EVTCompute0::Arguments evt0_compute_args{b_args};
159
+
160
+ typename Gemm::EVTCompute1::Arguments evt1_compute_args{a_args,
161
+ evt0_compute_args};
162
+ typename Gemm::D::Arguments d_args{c_ptr, c_stride};
163
+
164
+ typename Gemm::EVTD::Arguments epilogue_args{
165
+ evt1_compute_args,
166
+ d_args,
167
+ };
168
+
169
+ typename Gemm::Op::Arguments args{
170
+ cutlass::gemm::GemmUniversalMode::kGemmSplitKParallel, // universal mode
171
+ problem_size, // problem size
172
+ 1, // batch count
173
+ epilogue_args,
174
+ a_ptr,
175
+ b_ptr,
176
+ nullptr,
177
+ nullptr,
178
+ 0,
179
+ 0,
180
+ 0,
181
+ 0,
182
+ lda,
183
+ ldb,
184
+ ldc,
185
+ ldc};
186
+
187
+ // Launch the CUTLASS GEMM kernel.
188
+ typename Gemm::Op gemm_op;
189
+ size_t workspace_size = gemm_op.get_workspace_size(args);
190
+ cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
191
+
192
+ CUTLASS_CHECK(gemm_op.can_implement(args));
193
+ cutlass::Status status = gemm_op(args, workspace.get());
194
+ CUTLASS_CHECK(status);
195
+}
196
+
197
+} // namespace
198
+
199
+void cutlass_scaled_mm_dq_sm75(torch::Tensor &out, torch::Tensor const &a,
200
+ torch::Tensor const &b,
201
+ torch::Tensor const &a_scales,
202
+ torch::Tensor const &b_scales) {
203
+ TORCH_CHECK(a.dtype() == torch::kInt8);
204
+ TORCH_CHECK(b.dtype() == torch::kInt8);
205
+ TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
206
+ TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
207
+
208
+ using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
209
+ using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
210
+ using InstructionShape = typename cutlass::gemm::GemmShape<8, 8, 16>;
211
+
212
+ if (out.dtype() == torch::kBFloat16) {
213
+ return cutlass_scaled_mm_dq_dispatcher<
214
+ cutlass_2x_gemm<cutlass::arch::Sm75, int8_t, cutlass::bfloat16_t,
215
+ TileShape, WarpShape, InstructionShape, 2>>(
216
+ out, a, b, a_scales, b_scales);
217
+ } else {
218
+ TORCH_CHECK(out.dtype() == torch::kFloat16);
219
+ return cutlass_scaled_mm_dq_dispatcher<
220
+ cutlass_2x_gemm<cutlass::arch::Sm75, int8_t, cutlass::half_t, TileShape,
221
+ WarpShape, InstructionShape, 2>>(out, a, b, a_scales,
222
+ b_scales);
223
+ }
224
+}
225
+
226
+void cutlass_scaled_mm_dq_sm80(torch::Tensor &out, torch::Tensor const &a,
227
+ torch::Tensor const &b,
228
+ torch::Tensor const &a_scales,
229
+ torch::Tensor const &b_scales) {
230
+ TORCH_CHECK(a.dtype() == torch::kInt8);
231
+ TORCH_CHECK(b.dtype() == torch::kInt8);
232
+ TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
233
+ TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
234
+
235
+ using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
236
+ using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
237
+ using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
238
+
239
+ if (out.dtype() == torch::kBFloat16) {
240
+ return cutlass_scaled_mm_dq_dispatcher<
241
+ cutlass_2x_gemm<cutlass::arch::Sm80, int8_t, cutlass::bfloat16_t,
242
+ TileShape, WarpShape, InstructionShape, 5>>(
243
+ out, a, b, a_scales, b_scales);
244
+ } else {
245
+ TORCH_CHECK(out.dtype() == torch::kFloat16);
246
+ return cutlass_scaled_mm_dq_dispatcher<
247
+ cutlass_2x_gemm<cutlass::arch::Sm80, int8_t, cutlass::half_t, TileShape,
248
+ WarpShape, InstructionShape, 5>>(out, a, b, a_scales,
249
+ b_scales);
250
+ }
251
+}
252
+
253
+void cutlass_scaled_mm_dq_sm89(torch::Tensor &out, torch::Tensor const &a,
254
+ torch::Tensor const &b,
255
+ torch::Tensor const &a_scales,
256
+ torch::Tensor const &b_scales) {
257
+ using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
258
+ using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
259
+ using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
260
+
261
+ TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
262
+ TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
263
+
264
+ if (a.dtype() == torch::kInt8) {
265
+ TORCH_CHECK(b.dtype() == torch::kInt8);
266
+
267
+ if (out.dtype() == torch::kBFloat16) {
268
+ return cutlass_scaled_mm_dq_dispatcher<
269
+ cutlass_2x_gemm<cutlass::arch::Sm89, int8_t, cutlass::bfloat16_t,
270
+ TileShape, WarpShape, InstructionShape, 5>>(
271
+ out, a, b, a_scales, b_scales);
272
+ } else {
273
+ assert(out.dtype() == torch::kFloat16);
274
+ return cutlass_scaled_mm_dq_dispatcher<
275
+ cutlass_2x_gemm<cutlass::arch::Sm89, int8_t, cutlass::half_t,
276
+ TileShape, WarpShape, InstructionShape, 5>>(
277
+ out, a, b, a_scales, b_scales);
278
+ }
279
+ } else {
280
+ TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
281
+ TORCH_CHECK(b.dtype() == torch::kFloat8_e4m3fn);
282
+
283
+ if (out.dtype() == torch::kBFloat16) {
284
+ return cutlass_scaled_mm_dq_dispatcher<cutlass_2x_gemm<
285
+ cutlass::arch::Sm89, cutlass::float_e4m3_t, cutlass::bfloat16_t,
286
+ TileShape, WarpShape, InstructionShape, 5>>(out, a, b, a_scales,
287
+ b_scales);
288
+ } else {
289
+ TORCH_CHECK(out.dtype() == torch::kFloat16);
290
+ return cutlass_scaled_mm_dq_dispatcher<cutlass_2x_gemm<
291
+ cutlass::arch::Sm89, cutlass::float_e4m3_t, cutlass::half_t,
292
+ TileShape, WarpShape, InstructionShape, 5>>(out, a, b, a_scales,
293
+ b_scales);
294
+ }
295
+ }
296
+}
csrc/quantization/cutlass_w8a8/scaled_mm_dq_c3x.cu
new
+240
@@ -0,0 +1,240 @@
1
+#include <torch/extension.h>
2
+
3
+#include <iostream>
4
+#include <sstream>
5
+#include <vector>
6
+
7
+// clang-format will break include orders
8
+// clang-format off
9
+#include "cutlass/cutlass.h"
10
+
11
+#include "cute/tensor.hpp"
12
+#include "cute/atom/mma_atom.hpp"
13
+#include "cutlass/numeric_types.h"
14
+
15
+#include "cutlass/gemm/device/gemm_universal_adapter.h"
16
+#include "cutlass/gemm/kernel/gemm_universal.hpp"
17
+#include "cutlass/epilogue/collective/collective_builder.hpp"
18
+#include "cutlass/gemm/collective/collective_builder.hpp"
19
+
20
+#include "common.hpp"
21
+// clang-format on
22
+
23
+using namespace cute;
24
+
25
+/*
26
+ This defines a quantized GEMM operation with dequantized output, similar to
27
+ torch._scaled_mm. It is defined using the CUTLASS 3.x API, and is used for
28
+ NVIDIA GPUs with sm90a (Hopper) or later.
29
+
30
+ A and B may be both either int8 or fp8_e4m3. A can be quantized per-tensor or
31
+ per-row. B can be quantized per-tensor or per-column.
32
+ Any combination of per-tensor and per-row or column is supported.
33
+ A and B must have symmetric quantization (zero point == 0).
34
+
35
+ So the GEMM operation is D = (a_scales * A) (b_scales * B), where the
36
+ scales are applied elementwise with numpy-style broadcasting.
37
+
38
+ ScaleA and ScaleB define the epilogue functions that apply the scales for
39
+ the A and B operands respectively. These scales may be either per-tensor or
40
+ per row or column.
41
+*/
42
+
43
+namespace {
44
+
45
+template <typename ElementAB_, typename ElementD_, typename TileShape,
46
+ typename ClusterShape, typename KernelSchedule,
47
+ typename EpilogueSchedule>
48
+struct cutlass_3x_gemm {
49
+ using ElementAB = ElementAB_;
50
+ using ElementD = ElementD_;
51
+ using ElementAcc =
52
+ typename std::conditional<std::is_same_v<ElementAB, int8_t>, int32_t,
53
+ float>::type;
54
+
55
+ using EpilogueDescriptor =
56
+ cutlass::epilogue::collective::detail::EpilogueDescriptor<
57
+ TileShape, cutlass::epilogue::collective::EpilogueTileAuto, ElementD,
58
+ ElementD, EpilogueSchedule>;
59
+
60
+ using Accum = cutlass::epilogue::fusion::Sm90AccFetch;
61
+
62
+ using ScaleA = cutlass::epilogue::fusion::Sm90ColBroadcast<
63
+ 0 /*Stages*/, typename EpilogueDescriptor::TileShape, float,
64
+ Stride<Int<1>, Int<0>, Int<0>>>;
65
+
66
+ using ScaleBDescriptor =
67
+ cutlass::epilogue::collective::detail::RowBroadcastDescriptor<
68
+ EpilogueDescriptor, float>;
69
+
70
+ using ScaleB = cutlass::epilogue::fusion::Sm90RowBroadcast<
71
+ ScaleBDescriptor::Stages, typename EpilogueDescriptor::TileShape,
72
+ typename ScaleBDescriptor::Element, Stride<Int<0>, Int<1>, Int<0>>>;
73
+
74
+ using Compute0 = cutlass::epilogue::fusion::Sm90Compute<
75
+ cutlass::multiplies, float, float,
76
+ cutlass::FloatRoundStyle::round_to_nearest>;
77
+
78
+ using EVTCompute0 =
79
+ cutlass::epilogue::fusion::Sm90EVT<Compute0, ScaleB, Accum>;
80
+
81
+ using Compute1 = cutlass::epilogue::fusion::Sm90Compute<
82
+ cutlass::multiplies, ElementD, float,
83
+ cutlass::FloatRoundStyle::round_to_nearest>;
84
+
85
+ using EVTCompute1 =
86
+ cutlass::epilogue::fusion::Sm90EVT<Compute1, ScaleA, EVTCompute0>;
87
+
88
+ using StrideD = Stride<int64_t, Int<1>, Int<0>>;
89
+ using ElementC = void;
90
+ using StrideC = StrideD;
91
+
92
+ using CollectiveEpilogue =
93
+ typename cutlass::epilogue::collective::CollectiveBuilder<
94
+ cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp, TileShape,
95
+ ClusterShape, cutlass::epilogue::collective::EpilogueTileAuto,
96
+ ElementAcc, float, ElementC, StrideC, 4, ElementD, StrideD, 4,
97
+ EpilogueSchedule, EVTCompute1>::CollectiveOp;
98
+
99
+ static constexpr size_t CEStorageSize =
100
+ sizeof(typename CollectiveEpilogue::SharedStorage);
101
+ using Stages = typename cutlass::gemm::collective::StageCountAutoCarveout<
102
+ static_cast<int>(CEStorageSize)>;
103
+
104
+ // clang-format off
105
+ using CollectiveMainloop =
106
+ typename cutlass::gemm::collective::CollectiveBuilder<
107
+ cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
108
+ ElementAB, cutlass::layout::RowMajor, 16,
109
+ ElementAB, cutlass::layout::ColumnMajor, 16,
110
+ ElementAcc, TileShape, ClusterShape,
111
+ Stages,
112
+ KernelSchedule>::CollectiveOp;
113
+ // clang-format on
114
+
115
+ using KernelType = cutlass::gemm::kernel::GemmUniversal<
116
+ cute::Shape<int, int, int, int>, CollectiveMainloop, CollectiveEpilogue,
117
+ cutlass::gemm::PersistentScheduler>;
118
+
119
+ struct GemmKernel : public KernelType {};
120
+};
121
+
122
+template <typename Gemm>
123
+void cutlass_scaled_mm_dq_dispatcher(torch::Tensor &out, torch::Tensor const &a,
124
+ torch::Tensor const &b,
125
+ torch::Tensor const &a_scales,
126
+ torch::Tensor const &b_scales) {
127
+ using ElementAB = typename Gemm::ElementAB;
128
+ using ElementD = typename Gemm::ElementD;
129
+
130
+ int32_t m = a.size(0);
131
+ int32_t n = b.size(1);
132
+ int32_t k = a.size(1);
133
+
134
+ int64_t lda = a.stride(0);
135
+ int64_t ldb = b.stride(1);
136
+ int64_t ldc = out.stride(0);
137
+
138
+ using StrideA = Stride<int64_t, Int<1>, Int<0>>;
139
+ using StrideB = Stride<int64_t, Int<1>, Int<0>>;
140
+ using StrideC = typename Gemm::StrideC;
141
+
142
+ StrideA a_stride{lda, Int<1>{}, Int<0>{}};
143
+ StrideB b_stride{ldb, Int<1>{}, Int<0>{}};
144
+ StrideC c_stride{ldc, Int<1>{}, Int<0>{}};
145
+
146
+ using GemmKernel = typename Gemm::GemmKernel;
147
+ typename GemmKernel::ProblemShape prob_shape{m, n, k, 1};
148
+
149
+ auto a_ptr = static_cast<ElementAB *>(a.data_ptr());
150
+ auto b_ptr = static_cast<ElementAB *>(b.data_ptr());
151
+ typename GemmKernel::MainloopArguments mainloop_args{a_ptr, a_stride, b_ptr,
152
+ b_stride};
153
+
154
+ auto c_ptr = static_cast<ElementD *>(out.data_ptr());
155
+ typename GemmKernel::EpilogueArguments epilogue_args{
156
+ {}, c_ptr, c_stride, c_ptr, c_stride};
157
+
158
+ typename GemmKernel::Arguments args{cutlass::gemm::GemmUniversalMode::kGemm,
159
+ prob_shape, mainloop_args, epilogue_args};
160
+
161
+ using ScaleA_Args = typename Gemm::ScaleA::Arguments;
162
+ using ScaleB_Args = typename Gemm::ScaleB::Arguments;
163
+ ScaleA_Args a_args = a_scales.numel() == 1
164
+ ? ScaleA_Args{nullptr, a_scales.item<float>(), {}}
165
+ : ScaleA_Args{a_scales.data_ptr<float>(), {}, {}};
166
+
167
+ ScaleB_Args b_args = b_scales.numel() == 1
168
+ ? ScaleB_Args{nullptr, b_scales.item<float>(), {}}
169
+ : ScaleB_Args{b_scales.data_ptr<float>(), {}, {}};
170
+
171
+ args.epilogue.thread = {a_args, {b_args}};
172
+
173
+ // Launch the CUTLASS GEMM kernel.
174
+ using GemmOp = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
175
+ GemmOp gemm_op;
176
+ CUTLASS_CHECK(gemm_op.can_implement(args));
177
+
178
+ size_t workspace_size = gemm_op.get_workspace_size(args);
179
+ TORCH_CHECK(workspace_size == 0);
180
+
181
+ cutlass::Status status = gemm_op.run(args);
182
+ CUTLASS_CHECK(status);
183
+}
184
+} // namespace
185
+
186
+void cutlass_scaled_mm_dq_sm90(torch::Tensor &out, torch::Tensor const &a,
187
+ torch::Tensor const &b,
188
+ torch::Tensor const &a_scales,
189
+ torch::Tensor const &b_scales) {
190
+ TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
191
+ TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
192
+
193
+ if (a.dtype() == torch::kInt8) {
194
+ TORCH_CHECK(b.dtype() == torch::kInt8);
195
+
196
+ using TileShape = Shape<_128, _128, _128>;
197
+ using ClusterShape = Shape<_1, _2, _1>;
198
+ using KernelSchedule =
199
+ typename cutlass::gemm::KernelTmaWarpSpecializedPingpong;
200
+ using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
201
+
202
+ if (out.dtype() == torch::kBFloat16) {
203
+ return cutlass_scaled_mm_dq_dispatcher<
204
+ cutlass_3x_gemm<int8_t, cutlass::bfloat16_t, TileShape, ClusterShape,
205
+ KernelSchedule, EpilogueSchedule>>(
206
+ out, a, b, a_scales, b_scales);
207
+ } else {
208
+ TORCH_CHECK(out.dtype() == torch::kFloat16);
209
+
210
+ return cutlass_scaled_mm_dq_dispatcher<
211
+ cutlass_3x_gemm<int8_t, cutlass::half_t, TileShape, ClusterShape,
212
+ KernelSchedule, EpilogueSchedule>>(
213
+ out, a, b, a_scales, b_scales);
214
+ }
215
+ } else {
216
+ TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
217
+ TORCH_CHECK(b.dtype() == torch::kFloat8_e4m3fn);
218
+
219
+ using TileShape = Shape<_128, _128, _128>;
220
+ using ClusterShape = Shape<_1, _2, _1>;
221
+ using KernelSchedule =
222
+ typename cutlass::gemm::KernelCpAsyncWarpSpecializedCooperative;
223
+ using EpilogueSchedule =
224
+ typename cutlass::epilogue::TmaWarpSpecializedCooperative;
225
+
226
+ if (out.dtype() == torch::kBFloat16) {
227
+ return cutlass_scaled_mm_dq_dispatcher<
228
+ cutlass_3x_gemm<cutlass::float_e4m3_t, cutlass::bfloat16_t, TileShape,
229
+ ClusterShape, KernelSchedule, EpilogueSchedule>>(
230
+ out, a, b, a_scales, b_scales);
231
+ } else {
232
+ TORCH_CHECK(out.dtype() == torch::kFloat16);
233
+
234
+ return cutlass_scaled_mm_dq_dispatcher<
235
+ cutlass_3x_gemm<cutlass::float_e4m3_t, cutlass::half_t, TileShape,
236
+ ClusterShape, KernelSchedule, EpilogueSchedule>>(
237
+ out, a, b, a_scales, b_scales);
238
+ }
239
+ }
240
+}
csrc/quantization/cutlass_w8a8/scaled_mm_dq_entry.cu
new
+65
@@ -0,0 +1,65 @@
1
+#include <c10/cuda/CUDAGuard.h>
2
+#include <cuda_runtime.h>
3
+#include <torch/extension.h>
4
+
5
+void cutlass_scaled_mm_dq_sm75(torch::Tensor &c, torch::Tensor const &a,
6
+ torch::Tensor const &b,
7
+ torch::Tensor const &a_scales,
8
+ torch::Tensor const &b_scales);
9
+
10
+void cutlass_scaled_mm_dq_sm80(torch::Tensor &c, torch::Tensor const &a,
11
+ torch::Tensor const &b,
12
+ torch::Tensor const &a_scales,
13
+ torch::Tensor const &b_scales);
14
+
15
+void cutlass_scaled_mm_dq_sm89(torch::Tensor &c, torch::Tensor const &a,
16
+ torch::Tensor const &b,
17
+ torch::Tensor const &a_scales,
18
+ torch::Tensor const &b_scales);
19
+
20
+void cutlass_scaled_mm_dq_sm90(torch::Tensor &c, torch::Tensor const &a,
21
+ torch::Tensor const &b,
22
+ torch::Tensor const &a_scales,
23
+ torch::Tensor const &b_scales);
24
+
25
+void cutlass_scaled_mm_dq(torch::Tensor &c, torch::Tensor const &a,
26
+ torch::Tensor const &b, torch::Tensor const &a_scales,
27
+ torch::Tensor const &b_scales) {
28
+ int32_t major_capability;
29
+ int32_t minor_capability;
30
+ cudaDeviceGetAttribute(&major_capability, cudaDevAttrComputeCapabilityMajor,
31
+ 0);
32
+ cudaDeviceGetAttribute(&minor_capability, cudaDevAttrComputeCapabilityMinor,
33
+ 0);
34
+ int32_t version_num = major_capability * 10 + minor_capability;
35
+
36
+ // Checks for conformality
37
+ TORCH_CHECK(a.dim() == 2 && b.dim() == 2 && c.dim() == 2);
38
+ TORCH_CHECK(c.size(0) == a.size(0) && a.size(1) == b.size(0) &&
39
+ b.size(1) == c.size(1));
40
+ TORCH_CHECK(a_scales.numel() == 1 || a_scales.numel() == a.size(0));
41
+ TORCH_CHECK(b_scales.numel() == 1 || b_scales.numel() == b.size(1));
42
+
43
+ // Check for strides and alignment
44
+ TORCH_CHECK(a.stride(1) == 1 && c.stride(1) == 1); // Row-major
45
+ TORCH_CHECK(b.stride(0) == 1); // Column-major
46
+ TORCH_CHECK(c.stride(0) % 16 == 0 && b.stride(1) % 16 == 0); // 16 Byte Alignment
47
+ TORCH_CHECK(a_scales.is_contiguous() && b_scales.is_contiguous());
48
+
49
+ at::cuda::OptionalCUDAGuard const device_guard(device_of(a));
50
+
51
+ if (version_num >= 90) {
52
+ // Hopper
53
+ cutlass_scaled_mm_dq_sm90(c, a, b, a_scales, b_scales);
54
+ } else if (version_num == 89) {
55
+ // Ada Lovelace
56
+ cutlass_scaled_mm_dq_sm89(c, a, b, a_scales, b_scales);
57
+ } else if (version_num >= 80) {
58
+ // Ampere
59
+ cutlass_scaled_mm_dq_sm80(c, a, b, a_scales, b_scales);
60
+ } else {
61
+ // Turing
62
+ TORCH_CHECK(version_num >= 75);
63
+ cutlass_scaled_mm_dq_sm75(c, a, b, a_scales, b_scales);
64
+ }
65
+}
tests/kernels/test_cutlass.py
new
+192
@@ -0,0 +1,192 @@
1
+"""Tests for cutlass kernels
2
+
3
+Run `pytest tests/kernels/test_cutlass.py`.
4
+"""
5
+from typing import Type
6
+
7
+import pytest
8
+import torch
9
+
10
+from vllm import _custom_ops as ops
11
+
12
+CUDA_DEVICES = [
13
+ f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
14
+]
15
+
16
+capability = torch.cuda.get_device_capability()
17
+capability = capability[0] * 10 + capability[1]
18
+
19
+
20
+def to_fp8(tensor: torch.tensor):
21
+ finfo = torch.finfo(torch.float8_e4m3fn)
22
+ return torch.round(tensor.clamp(
23
+ min=finfo.min, max=finfo.max)).to(dtype=torch.float8_e4m3fn)
24
+
25
+
26
+def to_int8(tensor: torch.tensor):
27
+ return torch.round(tensor.clamp(min=-128, max=127)).to(dtype=torch.int8)
28
+
29
+
30
+def cutlass_fp8_gemm_helper(m: int,
31
+ n: int,
32
+ k: int,
33
+ per_token_act_quant: bool,
34
+ per_out_channel_weight_quant: bool,
35
+ out_dtype: Type[torch.dtype] = torch.bfloat16,
36
+ device: str = "cuda"):
37
+ # Test for a cutlass kernel with per-token activation quantization
38
+ # and per-output channel weight quantization.
39
+ a = to_fp8(torch.randn((m, k), device=device))
40
+ b = to_fp8(torch.randn((n, k), device=device).t())
41
+
42
+ m_a_scales = m if per_token_act_quant else 1
43
+ n_b_scales = n if per_out_channel_weight_quant else 1
44
+
45
+ scale_a = (torch.randn(
46
+ (m_a_scales, 1), device=device, dtype=torch.float32) / 10)
47
+ scale_b = (torch.randn(
48
+ (1, n_b_scales), device=device, dtype=torch.float32) / 10)
49
+
50
+ out = ops.cutlass_scaled_mm_dq(a, b, scale_a, scale_b, out_dtype)
51
+ baseline = torch.mm(scale_a * a.to(dtype=torch.float32),
52
+ scale_b * b.to(dtype=torch.float32)).to(out_dtype)
53
+
54
+ assert torch.allclose(out, baseline, rtol=1e-2, atol=1e-1)
55
+
56
+
57
+def cutlass_int8_gemm_helper(m: int,
58
+ n: int,
59
+ k: int,
60
+ per_token_act_quant: bool,
61
+ per_out_channel_weight_quant: bool,
62
+ out_dtype: Type[torch.dtype] = torch.bfloat16,
63
+ device: str = "cuda"):
64
+ # Test for a cutlass kernel with per-token activation quantization
65
+ # and per-output channel weight quantization.
66
+ a = to_int8(torch.randn((m, k), device=device) * 5)
67
+ b = to_int8(torch.randn((n, k), device=device).t() * 5)
68
+
69
+ m_a_scales = m if per_token_act_quant else 1
70
+ n_b_scales = n if per_out_channel_weight_quant else 1
71
+
72
+ scale_a = (torch.randn(
73
+ (m_a_scales, 1), device=device, dtype=torch.float32) / 10)
74
+ scale_b = (torch.randn(
75
+ (1, n_b_scales), device=device, dtype=torch.float32) / 10)
76
+
77
+ out = ops.cutlass_scaled_mm_dq(a, b, scale_a, scale_b, out_dtype)
78
+ baseline = torch.mm(scale_a * a.to(dtype=torch.float32),
79
+ scale_b *
80
+ b.to(dtype=torch.float32)).to(dtype=out_dtype)
81
+
82
+ assert torch.allclose(out, baseline, rtol=1e-1, atol=1e0)
83
+
84
+
85
+@pytest.mark.parametrize("m", [512, 222, 33, 1])
86
+@pytest.mark.parametrize("n", [2048, 256, 1024])
87
+@pytest.mark.parametrize("k", [128, 496, 1024])
88
+@pytest.mark.parametrize("per_act_token", [True, False])
89
+@pytest.mark.parametrize("per_out_ch", [True, False])
90
+@pytest.mark.skipif(capability < 89,
91
+ reason="FP8 is not supported on this GPU type.")
92
+def test_cutlass_fp8_gemm(m: int, n: int, k: int, per_act_token: bool,
93
+ per_out_ch: bool):
94
+ cutlass_fp8_gemm_helper(m, n, k, per_act_token, per_out_ch)
95
+
96
+
97
+@pytest.mark.parametrize("m", [512, 222, 33, 1])
98
+@pytest.mark.parametrize("n", [2048, 256, 1024])
99
+@pytest.mark.parametrize("k", [128, 496, 1024])
100
+@pytest.mark.parametrize("per_act_token", [True, False])
101
+@pytest.mark.parametrize("per_out_ch", [True, False])
102
+def test_cutlass_int8_gemm(m: int, n: int, k: int, per_act_token: bool,
103
+ per_out_ch: bool):
104
+ cutlass_int8_gemm_helper(m, n, k, per_act_token, per_out_ch)
105
+
106
+
107
+@pytest.mark.parametrize("per_act_token", [True, False])
108
+@pytest.mark.parametrize("per_out_ch", [True, False])
109
+@pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16])
110
+def test_cutlass_int8_gemm_output_dtype(per_act_token: bool, per_out_ch: bool,
111
+ out_dtype: Type[torch.dtype]):
112
+ cutlass_int8_gemm_helper(512, 512, 512, per_act_token, per_out_ch,
113
+ out_dtype)
114
+
115
+
116
+@pytest.mark.parametrize("per_act_token", [True, False])
117
+@pytest.mark.parametrize("per_out_ch", [True, False])
118
+@pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16])
119
+@pytest.mark.skipif(capability < 89,
120
+ reason="FP8 is not supported on this GPU type.")
121
+def test_cutlass_fp8_gemm_output_dtype(per_act_token: bool, per_out_ch: bool,
122
+ out_dtype: Type[torch.dtype]):
123
+ cutlass_fp8_gemm_helper(512, 512, 512, per_act_token, per_out_ch,
124
+ out_dtype)
125
+
126
+
127
+@pytest.mark.parametrize("per_act_token", [True, False])
128
+@pytest.mark.parametrize("per_out_ch", [True, False])
129
+@pytest.mark.parametrize("device", CUDA_DEVICES)
130
+@pytest.mark.skipif(capability < 89,
131
+ reason="FP8 is not supported on this GPU type.")
132
+def test_cutlass_fp8_gemm_devices(per_act_token: bool, per_out_ch: bool,
133
+ device: str):
134
+ cutlass_fp8_gemm_helper(512, 512, 512, per_act_token, per_out_ch,
135
+ torch.bfloat16, device)
136
+
137
+
138
+@pytest.mark.parametrize("per_act_token", [True, False])
139
+@pytest.mark.parametrize("per_out_ch", [True, False])
140
+@pytest.mark.parametrize("device", CUDA_DEVICES)
141
+def test_cutlass_int8_gemm_devices(per_act_token: bool, per_out_ch: bool,
142
+ device: str):
143
+ cutlass_int8_gemm_helper(512, 512, 512, per_act_token, per_out_ch,
144
+ torch.bfloat16, device)
145
+
146
+
147
+# For the following two tests:
148
+# N and K correspond to the size of the weight matrix and likely to be multiples
149
+# of a large power of two. In any case, the kernel will have a naive fallback
150
+# when N and K are not divisible by 16. But M is the number of tokens and the
151
+# kernel must handle any M thrown at it.
152
+@pytest.mark.parametrize("per_act_token", [True, False])
153
+@pytest.mark.parametrize("per_out_ch", [True, False])
154
+@pytest.mark.skipif(capability < 89,
155
+ reason="FP8 is not supported on this GPU type.")
156
+def test_cutlass_fp8_gemm_m_sweep(per_act_token: bool, per_out_ch: bool):
157
+ for nk in range(32, 128, 32):
158
+ for m in range(1, 128):
159
+ cutlass_fp8_gemm_helper(m, nk, nk, per_act_token, per_out_ch)
160
+
161
+
162
+@pytest.mark.parametrize("per_act_token", [True, False])
163
+@pytest.mark.parametrize("per_out_ch", [True, False])
164
+def test_cutlass_int8_gemm_m_sweep(per_act_token: bool, per_out_ch: bool):
165
+ for nk in range(32, 128, 32):
166
+ for m in range(1, 128):
167
+ cutlass_int8_gemm_helper(m, nk, nk, per_act_token, per_out_ch)
168
+
169
+
170
+# Test working with a subset of A and B
171
+def test_cutlass_subset():
172
+ big_m, big_n, big_k = 1024, 1024, 1024
173
+ m, n, k = 512, 512, 512
174
+
175
+ whole_a = to_int8(torch.randn((big_m, big_k), device="cuda") * 5)
176
+ whole_b = to_int8(torch.randn((big_n, big_k), device="cuda").t() * 5)
177
+ a = whole_a[0:m, 0:k]
178
+ b = whole_b[0:k, 0:n]
179
+
180
+ scale_a = torch.randn((1, 1), device="cuda", dtype=torch.float32) / 10
181
+ scale_b = torch.randn((1, 1), device="cuda", dtype=torch.float32) / 10
182
+
183
+ out = ops.cutlass_scaled_mm_dq(a,
184
+ b,
185
+ scale_a,
186
+ scale_b,
187
+ out_dtype=torch.bfloat16)
188
+ baseline = torch.mm(scale_a * a.to(dtype=torch.float32),
189
+ scale_b *
190
+ b.to(dtype=torch.float32)).to(dtype=torch.bfloat16)
191
+
192
+ assert torch.allclose(out, baseline, rtol=1e-1, atol=1e0)
vllm/_custom_ops.py
+17
-1
@@ -1,4 +1,4 @@
1
-from typing import Optional, Tuple
1
+from typing import Optional, Tuple, Type
2
3
import torch
4
@@ -163,6 +163,22 @@ def gptq_marlin_24_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
163
size_k)
164
165
166
+# cutlass
167
+def cutlass_scaled_mm_dq(a: torch.Tensor, b: torch.Tensor,
168
+ a_scales: torch.Tensor, b_scales: torch.Tensor,
169
+ out_dtype: Type[torch.dtype]) -> torch.Tensor:
170
+ assert (b.shape[0] % 16 == 0 and b.shape[1] % 16 == 0)
171
+ assert (out_dtype is torch.bfloat16 or out_dtype is torch.float16)
172
+
173
+ m = a.shape[0]
174
+ n = b.shape[1]
175
+ out = torch.empty((m, n), dtype=out_dtype, device=a.device)
176
+
177
+ vllm_ops.cutlass_scaled_mm_dq(out, a, b, a_scales, b_scales)
178
+
179
+ return out
180
+
181
+
182
# aqlm
183
def aqlm_gemm(input: torch.Tensor, codes: torch.Tensor,
184
codebooks: torch.Tensor, scales: torch.Tensor,