@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2060e93659
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Kernel] Add w8a8 CUTLASS kernels (#4749)

Tyler Michael Smith committed May 16, 2024 at 18:32 UTC 2060e93659f1f63a3d2a76aee61559ccb1fe732e
10 files changed +1197 -2
CMakeLists.txt
+26 -1
@@ -173,6 +173,16 @@ set(VLLM_EXT_SRC
173 "csrc/pybind.cpp")
174
175 if(VLLM_GPU_LANG STREQUAL "CUDA")
176 + include(FetchContent)
177 + SET(CUTLASS_ENABLE_HEADERS_ONLY=ON)
178 + FetchContent_Declare(
179 + cutlass
180 + GIT_REPOSITORY https://github.com/nvidia/cutlass.git
181 + # CUTLASS 3.5.0
182 + GIT_TAG 7d49e6c7e2f8896c47f586706e67e1fb215529dc
183 + )
184 + FetchContent_MakeAvailable(cutlass)
185 +
186 list(APPEND VLLM_EXT_SRC
187 "csrc/quantization/aqlm/gemm_kernels.cu"
188 "csrc/quantization/awq/gemm_kernels.cu"
@@ -180,7 +190,21 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
190 "csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu"
191 "csrc/quantization/gptq_marlin/gptq_marlin.cu"
192 "csrc/quantization/gptq_marlin/gptq_marlin_repack.cu"
183 - "csrc/custom_all_reduce.cu")
193 + "csrc/custom_all_reduce.cu"
194 + "csrc/quantization/cutlass_w8a8/scaled_mm_dq_entry.cu"
195 + "csrc/quantization/cutlass_w8a8/scaled_mm_dq_c2x.cu"
196 + "csrc/quantization/cutlass_w8a8/scaled_mm_dq_c3x.cu")
197 +
198 + #
199 + # The CUTLASS kernels for Hopper require sm90a to be enabled.
200 + # This is done via the below gencode option, BUT that creates kernels for both sm90 and sm90a.
201 + # That adds an extra 17MB to compiled binary, so instead we selectively enable it.
202 + set_source_files_properties(
203 + "csrc/quantization/cutlass_w8a8/scaled_mm_dq_c3x.cu"
204 + PROPERTIES
205 + COMPILE_FLAGS
206 + "-gencode arch=compute_90a,code=sm_90a")
207 +
208 endif()
209
210 define_gpu_extension_target(
@@ -190,6 +214,7 @@ define_gpu_extension_target(
214 SOURCES ${VLLM_EXT_SRC}
215 COMPILE_FLAGS ${VLLM_GPU_FLAGS}
216 ARCHITECTURES ${VLLM_GPU_ARCHES}
217 + INCLUDE_DIRECTORIES ${CUTLASS_INCLUDE_DIR};${CUTLASS_TOOLS_UTIL_INCLUDE_DIR}
218 WITH_SOABI)
219
220 #
csrc/ops.h
+8
@@ -155,6 +155,14 @@ torch::Tensor gptq_marlin_repack(
155 int64_t size_k,
156 int64_t size_n,
157 int64_t num_bits);
158 +
159 +int cutlass_scaled_mm_dq(
160 + torch::Tensor& out,
161 + torch::Tensor const &a,
162 + torch::Tensor const &b,
163 + torch::Tensor const &a_scales,
164 + torch::Tensor const &b_scales);
165 +
166 #endif
167
168 void squeezellm_gemm(
csrc/pybind.cpp
+1
@@ -71,6 +71,7 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
71 ops.def("gptq_marlin_gemm", &gptq_marlin_gemm, "gptq_marlin Optimized Quantized GEMM for GPTQ");
72 ops.def("gptq_marlin_repack", &gptq_marlin_repack, "gptq_marlin repack from GPTQ");
73 ops.def("awq_dequantize", &awq_dequantize, "Dequantization for AWQ");
74 + ops.def("cutlass_scaled_mm_dq", &cutlass_scaled_mm_dq, "CUTLASS w8a8 GEMM, supporting symmetric per-tensor or per-row/column quantization.");
75 #endif
76
77 ops.def("gptq_gemm", &gptq_gemm, "Quantized GEMM for GPTQ");
csrc/quantization/cutlass_w8a8/common.hpp new
+12
@@ -0,0 +1,12 @@
1 +#pragma once
2 +
3 +#include "cutlass/cutlass.h"
4 +
5 +/**
6 + * Helper function for checking CUTLASS errors
7 + */
8 +#define CUTLASS_CHECK(status) \
9 + { \
10 + TORCH_CHECK(status == cutlass::Status::kSuccess, \
11 + cutlassGetStatusString(status)) \
12 + }
csrc/quantization/cutlass_w8a8/cutlass_visitor_2x_broadcast_epilogue.hpp new
+340
@@ -0,0 +1,340 @@
1 +/***************************************************************************************************
2 + * Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights
3 + *reserved. SPDX-License-Identifier: BSD-3-Clause
4 + *
5 + * Redistribution and use in source and binary forms, with or without
6 + * modification, are permitted provided that the following conditions are met:
7 + *
8 + * 1. Redistributions of source code must retain the above copyright notice,
9 + *this list of conditions and the following disclaimer.
10 + *
11 + * 2. Redistributions in binary form must reproduce the above copyright notice,
12 + * this list of conditions and the following disclaimer in the documentation
13 + * and/or other materials provided with the distribution.
14 + *
15 + * 3. Neither the name of the copyright holder nor the names of its
16 + * contributors may be used to endorse or promote products derived from
17 + * this software without specific prior written permission.
18 + *
19 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 + *ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
23 + *LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 + *CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 + *SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 + *INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 + *CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 + *ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 + *POSSIBILITY OF SUCH DAMAGE.
30 + *
31 + **************************************************************************************************/
32 +
33 +//
34 +// This file is a modified excerpt of
35 +// include/cutlass/epilogue/fusion/visitor_load.hpp from
36 +// https://github.com/NVIDIA/cutlass It's beem modified to support either
37 +// row/column or scalar broadcasting, like is already supported in CUTLASS 3.x.
38 +// Important because this saves us a factor 4x on the number of kernels
39 +// compiled.
40 +//
41 +#pragma once
42 +
43 +// clang-format off
44 +
45 +#include "cutlass/epilogue/threadblock/fusion/visitor_2x.hpp"
46 +#include "cute/tensor.hpp"
47 +
48 +// clang-format on
49 +
50 +namespace cutlass::epilogue::threadblock {
51 +
52 +using namespace cute;
53 +using namespace detail;
54 +
55 +template<
56 + class ThreadMap,
57 + class Element,
58 + class StrideMNL
59 +>
60 +struct VisitorRowOrScalarBroadcast {
61 +
62 + struct Arguments {
63 + Element const* ptr_row = nullptr;
64 + Element null_default = Element(0);
65 + StrideMNL dRow = {};
66 + };
67 +
68 + using Params = Arguments;
69 +
70 + template <class ProblemShape>
71 + static constexpr Params
72 + to_underlying_arguments(ProblemShape const& problem_shape, Arguments const& args, void* workspace) {
73 + return args;
74 + }
75 +
76 + template <class ProblemShape>
77 + static size_t
78 + get_workspace_size(ProblemShape const& problem_shape, Arguments const& args) {
79 + return 0;
80 + }
81 +
82 + struct SharedStorage {};
83 +
84 + // Global load type
85 + static int constexpr vec_bits = ThreadMap::kElementsPerAccess * sizeof_bits<Element>::value;
86 + using VecType = uint_bit_t<cute::min(128, vec_bits)>;
87 + static int constexpr VecLength = sizeof(VecType) / sizeof(Element);
88 +
89 + CUTLASS_HOST_DEVICE
90 + VisitorRowOrScalarBroadcast() { }
91 +
92 + CUTLASS_HOST_DEVICE
93 + VisitorRowOrScalarBroadcast(Params const& params, SharedStorage const& shared_storage)
94 + : params_ptr(&params) { }
95 +
96 + Params const* params_ptr;
97 +
98 + template <class GTensor, class RTensor, class CTensor, class ProblemShape>
99 + struct Callbacks : EmptyCallbacks {
100 + CUTLASS_DEVICE
101 + Callbacks(
102 + GTensor&& tC_gRow,
103 + RTensor&& tC_rRow,
104 + CTensor&& tC_cRow,
105 + ProblemShape problem_shape,
106 + Params const* params_ptr
107 + ):
108 + tC_gRow(cute::forward<GTensor>(tC_gRow)),
109 + tC_rRow(cute::forward<RTensor>(tC_rRow)),
110 + tC_cRow(cute::forward<CTensor>(tC_cRow)),
111 + n(get<1>(problem_shape)),
112 + params_ptr(params_ptr) { }
113 +
114 + GTensor tC_gRow;
115 + RTensor tC_rRow;
116 + CTensor tC_cRow;
117 + Params const* params_ptr;
118 + int n;
119 +
120 + // This function is modified from VisitorRowBroadcast
121 + CUTLASS_DEVICE void
122 + begin_epilogue() {
123 + clear(tC_rRow);
124 + auto src_v = filter(tC_gRow);
125 + auto coord_v = filter(tC_cRow);
126 + auto dst_v = filter(tC_rRow);
127 +
128 + if (params_ptr->ptr_row) {
129 + // In this case we are loading from a row vector and broadcasting
130 + CUTLASS_PRAGMA_UNROLL
131 + for (int i = 0; i < size(src_v); ++i) {
132 + bool guard = get<1>(coord_v(i)) < n;
133 + cutlass::arch::global_load<VecType, sizeof(VecType)>(dst_v(i), (void const*)&src_v(i), guard);
134 + }
135 + } else {
136 + // In this case we are loading from a scalar and broadcasting
137 + VecType filled_vec;
138 + CUTLASS_PRAGMA_UNROLL
139 + for (int i = 0; i < VecLength; i++) {
140 + reinterpret_cast<Element*>(&filled_vec)[i] = params_ptr->null_default;
141 + }
142 +
143 + CUTLASS_PRAGMA_UNROLL
144 + for (int i = 0; i < size(src_v); ++i) {
145 + if(get<1>(coord_v(i)) < n)
146 + {
147 + dst_v(i) = filled_vec;
148 + }
149 + }
150 + }
151 + }
152 +
153 + template <class ElementAccumulator, int FragmentSize>
154 + CUTLASS_DEVICE auto // returns an Array
155 + visit(int iter_idx, int row_idx, int column_idx, int frg_idx,
156 + Array<ElementAccumulator, FragmentSize> const& frg_acc) {
157 + Tensor rRow_frg = recast<Array<Element, FragmentSize>>(coalesce(tC_rRow));
158 + return rRow_frg(column_idx);
159 + }
160 + };
161 +
162 + template <class ProblemShape>
163 + CUTLASS_DEVICE auto
164 + get_callbacks(
165 + gemm::GemmCoord threadblock_tile_offset,
166 + int thread_idx,
167 + ProblemShape problem_shape
168 + ) {
169 + Tensor mRow = make_tensor(
170 + make_gmem_ptr(params_ptr->ptr_row),
171 + problem_shape,
172 + params_ptr->dRow);
173 +
174 + // VECTOR, FRAGMENT_COLUMN
175 + Tensor tC_gRow = recast<VecType>(
176 + ThreadMap::partition(mRow, thread_idx, threadblock_tile_offset)
177 + )(_,_,_0{},_0{},_0{},_0{});
178 + Tensor tC_rRow = make_tensor_like(tC_gRow);
179 +
180 + // Generate the pred tensor
181 + Tensor cRow = make_identity_tensor(mRow.shape());
182 + Tensor tC_cRow = outer_partition(
183 + ThreadMap::partition(cRow, thread_idx, threadblock_tile_offset)(_,_,_0{},_0{},_0{},_0{}),
184 + Shape<Int<VecLength>>{},
185 + (_0{})
186 + );
187 +
188 + return Callbacks<
189 + decltype(tC_gRow), decltype(tC_rRow),
190 + decltype(tC_cRow), ProblemShape>(
191 + cute::move(tC_gRow),
192 + cute::move(tC_rRow),
193 + cute::move(tC_cRow),
194 + problem_shape,
195 + params_ptr
196 + );
197 + }
198 +
199 +};
200 +
201 +/////////////////////////////////////////////////////////////////////////////////////////////////
202 +
203 +// Column vector broadcast
204 +template<
205 + class ThreadMap,
206 + class Element,
207 + class StrideMNL = Stride<_1,_0,_0>
208 +>
209 +struct VisitorColOrScalarBroadcast {
210 +
211 + struct Arguments {
212 + Element const* ptr_col = nullptr;
213 + Element null_default = Element(0);
214 + StrideMNL dCol = {};
215 + };
216 +
217 + using Params = Arguments;
218 +
219 + template <class ProblemShape>
220 + static constexpr Params
221 + to_underlying_arguments(ProblemShape const& problem_shape, Arguments const& args, void* workspace) {
222 + return args;
223 + }
224 +
225 + template <class ProblemShape>
226 + static size_t
227 + get_workspace_size(ProblemShape const& problem_shape, Arguments const& args) {
228 + return 0;
229 + }
230 +
231 + struct SharedStorage { };
232 +
233 + // Global load type
234 + static int constexpr vec_bits = ThreadMap::kElementsPerAccess * sizeof_bits<Element>::value;
235 + using VecType = uint_bit_t<cute::min(128, vec_bits)>;
236 + static int constexpr VecLength = sizeof(VecType) / sizeof(Element);
237 +
238 + CUTLASS_HOST_DEVICE
239 + VisitorColOrScalarBroadcast() { }
240 +
241 + CUTLASS_HOST_DEVICE
242 + VisitorColOrScalarBroadcast(Params const& params, SharedStorage const& shared_storage)
243 + : params_ptr(&params) { }
244 +
245 + Params const* params_ptr;
246 +
247 + template <class GTensor, class RTensor, class CTensor, class ProblemShape>
248 + struct Callbacks : EmptyCallbacks {
249 + CUTLASS_DEVICE
250 + Callbacks(
251 + GTensor&& tC_gCol,
252 + RTensor&& tC_rCol,
253 + CTensor&& tC_cCol,
254 + ProblemShape problem_shape,
255 + Params const* params_ptr
256 + ):
257 + tC_gCol(cute::forward<GTensor>(tC_gCol)),
258 + tC_rCol(cute::forward<RTensor>(tC_rCol)),
259 + tC_cCol(cute::forward<CTensor>(tC_cCol)),
260 + m(get<0>(problem_shape)),
261 + params_ptr(params_ptr) { }
262 +
263 + GTensor tC_gCol;
264 + RTensor tC_rCol;
265 + CTensor tC_cCol;
266 + Params const* params_ptr;
267 + int m;
268 +
269 + // This function is modified from VisitorColBroadcast
270 + CUTLASS_DEVICE void
271 + begin_epilogue() {
272 + clear(tC_rCol);
273 +
274 + Tensor pred = make_tensor<bool>(shape(tC_gCol));
275 + CUTLASS_PRAGMA_UNROLL
276 + for (int i = 0; i < size(pred); ++i) {
277 + pred(i) = get<0>(tC_cCol(i)) < m;
278 + }
279 +
280 + if (params_ptr->ptr_col) {
281 + // In this case we are loading from a column vector and broadcasting
282 + copy_if(pred, tC_gCol, tC_rCol);
283 + } else {
284 + // In this case we are loading from a scalar and broadcasting
285 + auto dst_v = filter(tC_rCol);
286 +
287 + CUTLASS_PRAGMA_UNROLL
288 + for (int i = 0; i < size(dst_v); ++i) {
289 + if(pred(i)){
290 + dst_v(i) = params_ptr->null_default;
291 + }
292 + }
293 + }
294 + }
295 +
296 + template <class ElementAccumulator, int FragmentSize>
297 + CUTLASS_DEVICE auto // returns an Array
298 + visit(int iter_idx, int row_idx, int column_idx, int frg_idx,
299 + Array<ElementAccumulator, FragmentSize> const& frg_acc) {
300 + Array<Element, FragmentSize> frg_col;
301 + frg_col.fill(tC_rCol(row_idx,iter_idx));
302 + return frg_col;
303 + }
304 + };
305 +
306 + template <class ProblemShape>
307 + CUTLASS_DEVICE auto
308 + get_callbacks(
309 + gemm::GemmCoord threadblock_tile_offset,
310 + int thread_idx,
311 + ProblemShape problem_shape
312 + ) {
313 + Tensor mCol = make_tensor(
314 + make_gmem_ptr(params_ptr->ptr_col),
315 + problem_shape,
316 + params_ptr->dCol);
317 +
318 + // VECTOR, FRAGMENT_COLUMN, FRAGMENT_ROW, ITERATION_ROW, ITERATION_GROUP, ITERATION_CLUSTER
319 + Tensor tC_gCol = group_modes<1,4>(
320 + ThreadMap::partition(mCol, thread_idx, threadblock_tile_offset)(_0{},_0{},_,_,_,_));
321 + Tensor tC_rCol = make_tensor_like(tC_gCol);
322 +
323 + // Generate the pred tensor
324 + Tensor cCol = make_identity_tensor(mCol.shape());
325 + Tensor tC_cCol = group_modes<1,4>(
326 + ThreadMap::partition(cCol, thread_idx, threadblock_tile_offset)(_0{},_0{},_,_,_,_));
327 +
328 + return Callbacks<
329 + decltype(tC_gCol), decltype(tC_rCol),
330 + decltype(tC_cCol), ProblemShape>(
331 + cute::move(tC_gCol),
332 + cute::move(tC_rCol),
333 + cute::move(tC_cCol),
334 + problem_shape,
335 + params_ptr
336 + );
337 + }
338 +};
339 +
340 +}
csrc/quantization/cutlass_w8a8/scaled_mm_dq_c2x.cu new
+296
@@ -0,0 +1,296 @@
1 +#include <stddef.h>
2 +#include <torch/extension.h>
3 +
4 +// clang-format will break include orders
5 +// clang-format off
6 +#include "cute/tensor.hpp"
7 +#include "cute/atom/mma_atom.hpp"
8 +#include "cutlass/numeric_types.h"
9 +
10 +#include "cutlass/util/device_memory.h"
11 +
12 +#include "cutlass/cutlass.h"
13 +#include "cutlass/gemm_coord.h"
14 +#include "cutlass/arch/mma_sm75.h"
15 +#include "cutlass/arch/arch.h"
16 +#include "cutlass/arch/mma.h"
17 +#include "cutlass/gemm/device/gemm.h"
18 +#include "cutlass/gemm/device/gemm_universal_adapter.h"
19 +
20 +#include "cutlass/epilogue/threadblock/fusion/visitors.hpp"
21 +#include "cutlass/gemm/kernel/default_gemm_universal_with_visitor.h"
22 +
23 +#include "cutlass_visitor_2x_broadcast_epilogue.hpp"
24 +#include "common.hpp"
25 +// clang-format on
26 +
27 +using namespace cute;
28 +
29 +/*
30 + This defines a quantized GEMM operation with dequantized output, similar to
31 + torch._scaled_mm. It is defined using the CUTLASS 2.x API, and is used for
32 + NVIDIA GPUs with SM versions prior to sm90 (Hopper).
33 +
34 + A and B may be both either int8 or fp8_e4m3. A can be quantized per-tensor or
35 + per-row. B can be quantized per-tensor or per-column.
36 + Any combination of per-tensor and per-row or column is supported.
37 + A and B must have symmetric quantization (zero point == 0).
38 +
39 + So the GEMM operation is D = (a_scales * A) (b_scales * B), where the
40 + scales are applied elementwise with numpy-style broadcasting.
41 +
42 + ScaleA and ScaleB define the epilogue functions that apply the scales for
43 + the A and B operands respectively. These scales may be either per-tensor or
44 + per row or column.
45 +*/
46 +
47 +namespace {
48 +
49 +template <typename Arch, typename ElementAB_, typename ElementD_,
50 + typename TileShape, typename WarpShape, typename InstructionShape,
51 + int32_t MainLoopStages>
52 +struct cutlass_2x_gemm {
53 + using ElementAB = ElementAB_;
54 + using ElementD = ElementD_;
55 +
56 + using ElementAcc =
57 + typename std::conditional<std::is_same_v<ElementAB, int8_t>, int32_t,
58 + float>::type;
59 +
60 + using Operator =
61 + typename std::conditional<std::is_same_v<ElementAB, int8_t>,
62 + cutlass::arch::OpMultiplyAddSaturate,
63 + cutlass::arch::OpMultiplyAdd>::type;
64 +
65 + using OutputTileThreadMap =
66 + cutlass::epilogue::threadblock::OutputTileThreadLayout<
67 + TileShape, WarpShape, float, 4, 1 /* epilogue stages */
68 + >;
69 +
70 + using Accum = cutlass::epilogue::threadblock::VisitorAccFetch;
71 +
72 + using ScaleA = cutlass::epilogue::threadblock::VisitorColOrScalarBroadcast<
73 + OutputTileThreadMap, float, Stride<Int<1>, Int<0>, Int<0>>>;
74 +
75 + using ScaleB = cutlass::epilogue::threadblock::VisitorRowOrScalarBroadcast<
76 + OutputTileThreadMap, float, Stride<Int<0>, Int<1>, Int<0>>>;
77 +
78 + using Compute0 = cutlass::epilogue::threadblock::VisitorCompute<
79 + cutlass::multiplies, float, float,
80 + cutlass::FloatRoundStyle::round_to_nearest>;
81 +
82 + using EVTCompute0 =
83 + cutlass::epilogue::threadblock::Sm80EVT<Compute0, ScaleB, Accum>;
84 +
85 + using Compute1 = cutlass::epilogue::threadblock::VisitorCompute<
86 + cutlass::multiplies, ElementD, float,
87 + cutlass::FloatRoundStyle::round_to_nearest>;
88 +
89 + using EVTCompute1 =
90 + cutlass::epilogue::threadblock::Sm80EVT<Compute1, ScaleA, EVTCompute0>;
91 +
92 + using D = cutlass::epilogue::threadblock::VisitorAuxStore<
93 + OutputTileThreadMap, ElementD, cutlass::FloatRoundStyle::round_to_nearest,
94 + Stride<int64_t, Int<1>, Int<0>>>;
95 +
96 + using EVTD = cutlass::epilogue::threadblock::Sm80EVT<D, EVTCompute1>;
97 +
98 + // clang-format off
99 + using RowMajor = typename cutlass::layout::RowMajor;
100 + using ColumnMajor = typename cutlass::layout::ColumnMajor;
101 + using KernelType =
102 + typename cutlass::gemm::kernel::DefaultGemmWithVisitor<
103 + ElementAB, RowMajor, cutlass::ComplexTransform::kNone, 16,
104 + ElementAB, ColumnMajor, cutlass::ComplexTransform::kNone, 16,
105 + float, cutlass::layout::RowMajor, 4,
106 + ElementAcc, float, cutlass::arch::OpClassTensorOp,
107 + Arch,
108 + TileShape, WarpShape, InstructionShape,
109 + EVTD,
110 + cutlass::gemm::threadblock::ThreadblockSwizzleStreamK,
111 + MainLoopStages, Operator,
112 + 1 /* epilogue stages */
113 + >::GemmKernel;
114 + // clang-format on
115 +
116 + using Op = cutlass::gemm::device::GemmUniversalAdapter<KernelType>;
117 +};
118 +
119 +template <typename Gemm>
120 +void cutlass_scaled_mm_dq_dispatcher(torch::Tensor &out, torch::Tensor const &a,
121 + torch::Tensor const &b,
122 + torch::Tensor const &a_scales,
123 + torch::Tensor const &b_scales) {
124 + using ElementAB = typename Gemm::ElementAB;
125 + using ElementD = typename Gemm::ElementD;
126 +
127 + int32_t m = a.size(0);
128 + int32_t n = b.size(1);
129 + int32_t k = a.size(1);
130 + cutlass::gemm::GemmCoord problem_size{m, n, k};
131 +
132 + int64_t lda = a.stride(0);
133 + int64_t ldb = b.stride(1);
134 + int64_t ldc = out.stride(0);
135 +
136 + using StrideC = Stride<int64_t, Int<1>, Int<0>>;
137 + StrideC c_stride{ldc, Int<1>{}, Int<0>{}};
138 +
139 + auto a_ptr = static_cast<ElementAB const *>(a.data_ptr());
140 + auto b_ptr = static_cast<ElementAB const *>(b.data_ptr());
141 + auto c_ptr = static_cast<ElementD *>(out.data_ptr());
142 +
143 + auto a_scales_ptr = a_scales.data_ptr<float>();
144 + auto b_scales_ptr = b_scales.data_ptr<float>();
145 +
146 + // If A and B are quantized per-tensor, then these scale tensors are scalars,
147 + // and they are passed in via the second argument.
148 + using ScaleAArgs = typename Gemm::ScaleA::Arguments;
149 + ScaleAArgs a_args = a_scales.numel() == 1
150 + ? ScaleAArgs{nullptr, a_scales.item<float>(), {}}
151 + : ScaleAArgs{a_scales.data_ptr<float>(), {}, {}};
152 +
153 + using ScaleBArgs = typename Gemm::ScaleB::Arguments;
154 + ScaleBArgs b_args = b_scales.numel() == 1
155 + ? ScaleBArgs{nullptr, b_scales.item<float>(), {}}
156 + : ScaleBArgs{b_scales.data_ptr<float>(), {}, {}};
157 +
158 + typename Gemm::EVTCompute0::Arguments evt0_compute_args{b_args};
159 +
160 + typename Gemm::EVTCompute1::Arguments evt1_compute_args{a_args,
161 + evt0_compute_args};
162 + typename Gemm::D::Arguments d_args{c_ptr, c_stride};
163 +
164 + typename Gemm::EVTD::Arguments epilogue_args{
165 + evt1_compute_args,
166 + d_args,
167 + };
168 +
169 + typename Gemm::Op::Arguments args{
170 + cutlass::gemm::GemmUniversalMode::kGemmSplitKParallel, // universal mode
171 + problem_size, // problem size
172 + 1, // batch count
173 + epilogue_args,
174 + a_ptr,
175 + b_ptr,
176 + nullptr,
177 + nullptr,
178 + 0,
179 + 0,
180 + 0,
181 + 0,
182 + lda,
183 + ldb,
184 + ldc,
185 + ldc};
186 +
187 + // Launch the CUTLASS GEMM kernel.
188 + typename Gemm::Op gemm_op;
189 + size_t workspace_size = gemm_op.get_workspace_size(args);
190 + cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
191 +
192 + CUTLASS_CHECK(gemm_op.can_implement(args));
193 + cutlass::Status status = gemm_op(args, workspace.get());
194 + CUTLASS_CHECK(status);
195 +}
196 +
197 +} // namespace
198 +
199 +void cutlass_scaled_mm_dq_sm75(torch::Tensor &out, torch::Tensor const &a,
200 + torch::Tensor const &b,
201 + torch::Tensor const &a_scales,
202 + torch::Tensor const &b_scales) {
203 + TORCH_CHECK(a.dtype() == torch::kInt8);
204 + TORCH_CHECK(b.dtype() == torch::kInt8);
205 + TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
206 + TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
207 +
208 + using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
209 + using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
210 + using InstructionShape = typename cutlass::gemm::GemmShape<8, 8, 16>;
211 +
212 + if (out.dtype() == torch::kBFloat16) {
213 + return cutlass_scaled_mm_dq_dispatcher<
214 + cutlass_2x_gemm<cutlass::arch::Sm75, int8_t, cutlass::bfloat16_t,
215 + TileShape, WarpShape, InstructionShape, 2>>(
216 + out, a, b, a_scales, b_scales);
217 + } else {
218 + TORCH_CHECK(out.dtype() == torch::kFloat16);
219 + return cutlass_scaled_mm_dq_dispatcher<
220 + cutlass_2x_gemm<cutlass::arch::Sm75, int8_t, cutlass::half_t, TileShape,
221 + WarpShape, InstructionShape, 2>>(out, a, b, a_scales,
222 + b_scales);
223 + }
224 +}
225 +
226 +void cutlass_scaled_mm_dq_sm80(torch::Tensor &out, torch::Tensor const &a,
227 + torch::Tensor const &b,
228 + torch::Tensor const &a_scales,
229 + torch::Tensor const &b_scales) {
230 + TORCH_CHECK(a.dtype() == torch::kInt8);
231 + TORCH_CHECK(b.dtype() == torch::kInt8);
232 + TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
233 + TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
234 +
235 + using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
236 + using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
237 + using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
238 +
239 + if (out.dtype() == torch::kBFloat16) {
240 + return cutlass_scaled_mm_dq_dispatcher<
241 + cutlass_2x_gemm<cutlass::arch::Sm80, int8_t, cutlass::bfloat16_t,
242 + TileShape, WarpShape, InstructionShape, 5>>(
243 + out, a, b, a_scales, b_scales);
244 + } else {
245 + TORCH_CHECK(out.dtype() == torch::kFloat16);
246 + return cutlass_scaled_mm_dq_dispatcher<
247 + cutlass_2x_gemm<cutlass::arch::Sm80, int8_t, cutlass::half_t, TileShape,
248 + WarpShape, InstructionShape, 5>>(out, a, b, a_scales,
249 + b_scales);
250 + }
251 +}
252 +
253 +void cutlass_scaled_mm_dq_sm89(torch::Tensor &out, torch::Tensor const &a,
254 + torch::Tensor const &b,
255 + torch::Tensor const &a_scales,
256 + torch::Tensor const &b_scales) {
257 + using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
258 + using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
259 + using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
260 +
261 + TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
262 + TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
263 +
264 + if (a.dtype() == torch::kInt8) {
265 + TORCH_CHECK(b.dtype() == torch::kInt8);
266 +
267 + if (out.dtype() == torch::kBFloat16) {
268 + return cutlass_scaled_mm_dq_dispatcher<
269 + cutlass_2x_gemm<cutlass::arch::Sm89, int8_t, cutlass::bfloat16_t,
270 + TileShape, WarpShape, InstructionShape, 5>>(
271 + out, a, b, a_scales, b_scales);
272 + } else {
273 + assert(out.dtype() == torch::kFloat16);
274 + return cutlass_scaled_mm_dq_dispatcher<
275 + cutlass_2x_gemm<cutlass::arch::Sm89, int8_t, cutlass::half_t,
276 + TileShape, WarpShape, InstructionShape, 5>>(
277 + out, a, b, a_scales, b_scales);
278 + }
279 + } else {
280 + TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
281 + TORCH_CHECK(b.dtype() == torch::kFloat8_e4m3fn);
282 +
283 + if (out.dtype() == torch::kBFloat16) {
284 + return cutlass_scaled_mm_dq_dispatcher<cutlass_2x_gemm<
285 + cutlass::arch::Sm89, cutlass::float_e4m3_t, cutlass::bfloat16_t,
286 + TileShape, WarpShape, InstructionShape, 5>>(out, a, b, a_scales,
287 + b_scales);
288 + } else {
289 + TORCH_CHECK(out.dtype() == torch::kFloat16);
290 + return cutlass_scaled_mm_dq_dispatcher<cutlass_2x_gemm<
291 + cutlass::arch::Sm89, cutlass::float_e4m3_t, cutlass::half_t,
292 + TileShape, WarpShape, InstructionShape, 5>>(out, a, b, a_scales,
293 + b_scales);
294 + }
295 + }
296 +}
csrc/quantization/cutlass_w8a8/scaled_mm_dq_c3x.cu new
+240
@@ -0,0 +1,240 @@
1 +#include <torch/extension.h>
2 +
3 +#include <iostream>
4 +#include <sstream>
5 +#include <vector>
6 +
7 +// clang-format will break include orders
8 +// clang-format off
9 +#include "cutlass/cutlass.h"
10 +
11 +#include "cute/tensor.hpp"
12 +#include "cute/atom/mma_atom.hpp"
13 +#include "cutlass/numeric_types.h"
14 +
15 +#include "cutlass/gemm/device/gemm_universal_adapter.h"
16 +#include "cutlass/gemm/kernel/gemm_universal.hpp"
17 +#include "cutlass/epilogue/collective/collective_builder.hpp"
18 +#include "cutlass/gemm/collective/collective_builder.hpp"
19 +
20 +#include "common.hpp"
21 +// clang-format on
22 +
23 +using namespace cute;
24 +
25 +/*
26 + This defines a quantized GEMM operation with dequantized output, similar to
27 + torch._scaled_mm. It is defined using the CUTLASS 3.x API, and is used for
28 + NVIDIA GPUs with sm90a (Hopper) or later.
29 +
30 + A and B may be both either int8 or fp8_e4m3. A can be quantized per-tensor or
31 + per-row. B can be quantized per-tensor or per-column.
32 + Any combination of per-tensor and per-row or column is supported.
33 + A and B must have symmetric quantization (zero point == 0).
34 +
35 + So the GEMM operation is D = (a_scales * A) (b_scales * B), where the
36 + scales are applied elementwise with numpy-style broadcasting.
37 +
38 + ScaleA and ScaleB define the epilogue functions that apply the scales for
39 + the A and B operands respectively. These scales may be either per-tensor or
40 + per row or column.
41 +*/
42 +
43 +namespace {
44 +
45 +template <typename ElementAB_, typename ElementD_, typename TileShape,
46 + typename ClusterShape, typename KernelSchedule,
47 + typename EpilogueSchedule>
48 +struct cutlass_3x_gemm {
49 + using ElementAB = ElementAB_;
50 + using ElementD = ElementD_;
51 + using ElementAcc =
52 + typename std::conditional<std::is_same_v<ElementAB, int8_t>, int32_t,
53 + float>::type;
54 +
55 + using EpilogueDescriptor =
56 + cutlass::epilogue::collective::detail::EpilogueDescriptor<
57 + TileShape, cutlass::epilogue::collective::EpilogueTileAuto, ElementD,
58 + ElementD, EpilogueSchedule>;
59 +
60 + using Accum = cutlass::epilogue::fusion::Sm90AccFetch;
61 +
62 + using ScaleA = cutlass::epilogue::fusion::Sm90ColBroadcast<
63 + 0 /*Stages*/, typename EpilogueDescriptor::TileShape, float,
64 + Stride<Int<1>, Int<0>, Int<0>>>;
65 +
66 + using ScaleBDescriptor =
67 + cutlass::epilogue::collective::detail::RowBroadcastDescriptor<
68 + EpilogueDescriptor, float>;
69 +
70 + using ScaleB = cutlass::epilogue::fusion::Sm90RowBroadcast<
71 + ScaleBDescriptor::Stages, typename EpilogueDescriptor::TileShape,
72 + typename ScaleBDescriptor::Element, Stride<Int<0>, Int<1>, Int<0>>>;
73 +
74 + using Compute0 = cutlass::epilogue::fusion::Sm90Compute<
75 + cutlass::multiplies, float, float,
76 + cutlass::FloatRoundStyle::round_to_nearest>;
77 +
78 + using EVTCompute0 =
79 + cutlass::epilogue::fusion::Sm90EVT<Compute0, ScaleB, Accum>;
80 +
81 + using Compute1 = cutlass::epilogue::fusion::Sm90Compute<
82 + cutlass::multiplies, ElementD, float,
83 + cutlass::FloatRoundStyle::round_to_nearest>;
84 +
85 + using EVTCompute1 =
86 + cutlass::epilogue::fusion::Sm90EVT<Compute1, ScaleA, EVTCompute0>;
87 +
88 + using StrideD = Stride<int64_t, Int<1>, Int<0>>;
89 + using ElementC = void;
90 + using StrideC = StrideD;
91 +
92 + using CollectiveEpilogue =
93 + typename cutlass::epilogue::collective::CollectiveBuilder<
94 + cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp, TileShape,
95 + ClusterShape, cutlass::epilogue::collective::EpilogueTileAuto,
96 + ElementAcc, float, ElementC, StrideC, 4, ElementD, StrideD, 4,
97 + EpilogueSchedule, EVTCompute1>::CollectiveOp;
98 +
99 + static constexpr size_t CEStorageSize =
100 + sizeof(typename CollectiveEpilogue::SharedStorage);
101 + using Stages = typename cutlass::gemm::collective::StageCountAutoCarveout<
102 + static_cast<int>(CEStorageSize)>;
103 +
104 + // clang-format off
105 + using CollectiveMainloop =
106 + typename cutlass::gemm::collective::CollectiveBuilder<
107 + cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
108 + ElementAB, cutlass::layout::RowMajor, 16,
109 + ElementAB, cutlass::layout::ColumnMajor, 16,
110 + ElementAcc, TileShape, ClusterShape,
111 + Stages,
112 + KernelSchedule>::CollectiveOp;
113 + // clang-format on
114 +
115 + using KernelType = cutlass::gemm::kernel::GemmUniversal<
116 + cute::Shape<int, int, int, int>, CollectiveMainloop, CollectiveEpilogue,
117 + cutlass::gemm::PersistentScheduler>;
118 +
119 + struct GemmKernel : public KernelType {};
120 +};
121 +
122 +template <typename Gemm>
123 +void cutlass_scaled_mm_dq_dispatcher(torch::Tensor &out, torch::Tensor const &a,
124 + torch::Tensor const &b,
125 + torch::Tensor const &a_scales,
126 + torch::Tensor const &b_scales) {
127 + using ElementAB = typename Gemm::ElementAB;
128 + using ElementD = typename Gemm::ElementD;
129 +
130 + int32_t m = a.size(0);
131 + int32_t n = b.size(1);
132 + int32_t k = a.size(1);
133 +
134 + int64_t lda = a.stride(0);
135 + int64_t ldb = b.stride(1);
136 + int64_t ldc = out.stride(0);
137 +
138 + using StrideA = Stride<int64_t, Int<1>, Int<0>>;
139 + using StrideB = Stride<int64_t, Int<1>, Int<0>>;
140 + using StrideC = typename Gemm::StrideC;
141 +
142 + StrideA a_stride{lda, Int<1>{}, Int<0>{}};
143 + StrideB b_stride{ldb, Int<1>{}, Int<0>{}};
144 + StrideC c_stride{ldc, Int<1>{}, Int<0>{}};
145 +
146 + using GemmKernel = typename Gemm::GemmKernel;
147 + typename GemmKernel::ProblemShape prob_shape{m, n, k, 1};
148 +
149 + auto a_ptr = static_cast<ElementAB *>(a.data_ptr());
150 + auto b_ptr = static_cast<ElementAB *>(b.data_ptr());
151 + typename GemmKernel::MainloopArguments mainloop_args{a_ptr, a_stride, b_ptr,
152 + b_stride};
153 +
154 + auto c_ptr = static_cast<ElementD *>(out.data_ptr());
155 + typename GemmKernel::EpilogueArguments epilogue_args{
156 + {}, c_ptr, c_stride, c_ptr, c_stride};
157 +
158 + typename GemmKernel::Arguments args{cutlass::gemm::GemmUniversalMode::kGemm,
159 + prob_shape, mainloop_args, epilogue_args};
160 +
161 + using ScaleA_Args = typename Gemm::ScaleA::Arguments;
162 + using ScaleB_Args = typename Gemm::ScaleB::Arguments;
163 + ScaleA_Args a_args = a_scales.numel() == 1
164 + ? ScaleA_Args{nullptr, a_scales.item<float>(), {}}
165 + : ScaleA_Args{a_scales.data_ptr<float>(), {}, {}};
166 +
167 + ScaleB_Args b_args = b_scales.numel() == 1
168 + ? ScaleB_Args{nullptr, b_scales.item<float>(), {}}
169 + : ScaleB_Args{b_scales.data_ptr<float>(), {}, {}};
170 +
171 + args.epilogue.thread = {a_args, {b_args}};
172 +
173 + // Launch the CUTLASS GEMM kernel.
174 + using GemmOp = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
175 + GemmOp gemm_op;
176 + CUTLASS_CHECK(gemm_op.can_implement(args));
177 +
178 + size_t workspace_size = gemm_op.get_workspace_size(args);
179 + TORCH_CHECK(workspace_size == 0);
180 +
181 + cutlass::Status status = gemm_op.run(args);
182 + CUTLASS_CHECK(status);
183 +}
184 +} // namespace
185 +
186 +void cutlass_scaled_mm_dq_sm90(torch::Tensor &out, torch::Tensor const &a,
187 + torch::Tensor const &b,
188 + torch::Tensor const &a_scales,
189 + torch::Tensor const &b_scales) {
190 + TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
191 + TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
192 +
193 + if (a.dtype() == torch::kInt8) {
194 + TORCH_CHECK(b.dtype() == torch::kInt8);
195 +
196 + using TileShape = Shape<_128, _128, _128>;
197 + using ClusterShape = Shape<_1, _2, _1>;
198 + using KernelSchedule =
199 + typename cutlass::gemm::KernelTmaWarpSpecializedPingpong;
200 + using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
201 +
202 + if (out.dtype() == torch::kBFloat16) {
203 + return cutlass_scaled_mm_dq_dispatcher<
204 + cutlass_3x_gemm<int8_t, cutlass::bfloat16_t, TileShape, ClusterShape,
205 + KernelSchedule, EpilogueSchedule>>(
206 + out, a, b, a_scales, b_scales);
207 + } else {
208 + TORCH_CHECK(out.dtype() == torch::kFloat16);
209 +
210 + return cutlass_scaled_mm_dq_dispatcher<
211 + cutlass_3x_gemm<int8_t, cutlass::half_t, TileShape, ClusterShape,
212 + KernelSchedule, EpilogueSchedule>>(
213 + out, a, b, a_scales, b_scales);
214 + }
215 + } else {
216 + TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
217 + TORCH_CHECK(b.dtype() == torch::kFloat8_e4m3fn);
218 +
219 + using TileShape = Shape<_128, _128, _128>;
220 + using ClusterShape = Shape<_1, _2, _1>;
221 + using KernelSchedule =
222 + typename cutlass::gemm::KernelCpAsyncWarpSpecializedCooperative;
223 + using EpilogueSchedule =
224 + typename cutlass::epilogue::TmaWarpSpecializedCooperative;
225 +
226 + if (out.dtype() == torch::kBFloat16) {
227 + return cutlass_scaled_mm_dq_dispatcher<
228 + cutlass_3x_gemm<cutlass::float_e4m3_t, cutlass::bfloat16_t, TileShape,
229 + ClusterShape, KernelSchedule, EpilogueSchedule>>(
230 + out, a, b, a_scales, b_scales);
231 + } else {
232 + TORCH_CHECK(out.dtype() == torch::kFloat16);
233 +
234 + return cutlass_scaled_mm_dq_dispatcher<
235 + cutlass_3x_gemm<cutlass::float_e4m3_t, cutlass::half_t, TileShape,
236 + ClusterShape, KernelSchedule, EpilogueSchedule>>(
237 + out, a, b, a_scales, b_scales);
238 + }
239 + }
240 +}
csrc/quantization/cutlass_w8a8/scaled_mm_dq_entry.cu new
+65
@@ -0,0 +1,65 @@
1 +#include <c10/cuda/CUDAGuard.h>
2 +#include <cuda_runtime.h>
3 +#include <torch/extension.h>
4 +
5 +void cutlass_scaled_mm_dq_sm75(torch::Tensor &c, torch::Tensor const &a,
6 + torch::Tensor const &b,
7 + torch::Tensor const &a_scales,
8 + torch::Tensor const &b_scales);
9 +
10 +void cutlass_scaled_mm_dq_sm80(torch::Tensor &c, torch::Tensor const &a,
11 + torch::Tensor const &b,
12 + torch::Tensor const &a_scales,
13 + torch::Tensor const &b_scales);
14 +
15 +void cutlass_scaled_mm_dq_sm89(torch::Tensor &c, torch::Tensor const &a,
16 + torch::Tensor const &b,
17 + torch::Tensor const &a_scales,
18 + torch::Tensor const &b_scales);
19 +
20 +void cutlass_scaled_mm_dq_sm90(torch::Tensor &c, torch::Tensor const &a,
21 + torch::Tensor const &b,
22 + torch::Tensor const &a_scales,
23 + torch::Tensor const &b_scales);
24 +
25 +void cutlass_scaled_mm_dq(torch::Tensor &c, torch::Tensor const &a,
26 + torch::Tensor const &b, torch::Tensor const &a_scales,
27 + torch::Tensor const &b_scales) {
28 + int32_t major_capability;
29 + int32_t minor_capability;
30 + cudaDeviceGetAttribute(&major_capability, cudaDevAttrComputeCapabilityMajor,
31 + 0);
32 + cudaDeviceGetAttribute(&minor_capability, cudaDevAttrComputeCapabilityMinor,
33 + 0);
34 + int32_t version_num = major_capability * 10 + minor_capability;
35 +
36 + // Checks for conformality
37 + TORCH_CHECK(a.dim() == 2 && b.dim() == 2 && c.dim() == 2);
38 + TORCH_CHECK(c.size(0) == a.size(0) && a.size(1) == b.size(0) &&
39 + b.size(1) == c.size(1));
40 + TORCH_CHECK(a_scales.numel() == 1 || a_scales.numel() == a.size(0));
41 + TORCH_CHECK(b_scales.numel() == 1 || b_scales.numel() == b.size(1));
42 +
43 + // Check for strides and alignment
44 + TORCH_CHECK(a.stride(1) == 1 && c.stride(1) == 1); // Row-major
45 + TORCH_CHECK(b.stride(0) == 1); // Column-major
46 + TORCH_CHECK(c.stride(0) % 16 == 0 && b.stride(1) % 16 == 0); // 16 Byte Alignment
47 + TORCH_CHECK(a_scales.is_contiguous() && b_scales.is_contiguous());
48 +
49 + at::cuda::OptionalCUDAGuard const device_guard(device_of(a));
50 +
51 + if (version_num >= 90) {
52 + // Hopper
53 + cutlass_scaled_mm_dq_sm90(c, a, b, a_scales, b_scales);
54 + } else if (version_num == 89) {
55 + // Ada Lovelace
56 + cutlass_scaled_mm_dq_sm89(c, a, b, a_scales, b_scales);
57 + } else if (version_num >= 80) {
58 + // Ampere
59 + cutlass_scaled_mm_dq_sm80(c, a, b, a_scales, b_scales);
60 + } else {
61 + // Turing
62 + TORCH_CHECK(version_num >= 75);
63 + cutlass_scaled_mm_dq_sm75(c, a, b, a_scales, b_scales);
64 + }
65 +}
tests/kernels/test_cutlass.py new
+192
@@ -0,0 +1,192 @@
1 +"""Tests for cutlass kernels
2 +
3 +Run `pytest tests/kernels/test_cutlass.py`.
4 +"""
5 +from typing import Type
6 +
7 +import pytest
8 +import torch
9 +
10 +from vllm import _custom_ops as ops
11 +
12 +CUDA_DEVICES = [
13 + f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
14 +]
15 +
16 +capability = torch.cuda.get_device_capability()
17 +capability = capability[0] * 10 + capability[1]
18 +
19 +
20 +def to_fp8(tensor: torch.tensor):
21 + finfo = torch.finfo(torch.float8_e4m3fn)
22 + return torch.round(tensor.clamp(
23 + min=finfo.min, max=finfo.max)).to(dtype=torch.float8_e4m3fn)
24 +
25 +
26 +def to_int8(tensor: torch.tensor):
27 + return torch.round(tensor.clamp(min=-128, max=127)).to(dtype=torch.int8)
28 +
29 +
30 +def cutlass_fp8_gemm_helper(m: int,
31 + n: int,
32 + k: int,
33 + per_token_act_quant: bool,
34 + per_out_channel_weight_quant: bool,
35 + out_dtype: Type[torch.dtype] = torch.bfloat16,
36 + device: str = "cuda"):
37 + # Test for a cutlass kernel with per-token activation quantization
38 + # and per-output channel weight quantization.
39 + a = to_fp8(torch.randn((m, k), device=device))
40 + b = to_fp8(torch.randn((n, k), device=device).t())
41 +
42 + m_a_scales = m if per_token_act_quant else 1
43 + n_b_scales = n if per_out_channel_weight_quant else 1
44 +
45 + scale_a = (torch.randn(
46 + (m_a_scales, 1), device=device, dtype=torch.float32) / 10)
47 + scale_b = (torch.randn(
48 + (1, n_b_scales), device=device, dtype=torch.float32) / 10)
49 +
50 + out = ops.cutlass_scaled_mm_dq(a, b, scale_a, scale_b, out_dtype)
51 + baseline = torch.mm(scale_a * a.to(dtype=torch.float32),
52 + scale_b * b.to(dtype=torch.float32)).to(out_dtype)
53 +
54 + assert torch.allclose(out, baseline, rtol=1e-2, atol=1e-1)
55 +
56 +
57 +def cutlass_int8_gemm_helper(m: int,
58 + n: int,
59 + k: int,
60 + per_token_act_quant: bool,
61 + per_out_channel_weight_quant: bool,
62 + out_dtype: Type[torch.dtype] = torch.bfloat16,
63 + device: str = "cuda"):
64 + # Test for a cutlass kernel with per-token activation quantization
65 + # and per-output channel weight quantization.
66 + a = to_int8(torch.randn((m, k), device=device) * 5)
67 + b = to_int8(torch.randn((n, k), device=device).t() * 5)
68 +
69 + m_a_scales = m if per_token_act_quant else 1
70 + n_b_scales = n if per_out_channel_weight_quant else 1
71 +
72 + scale_a = (torch.randn(
73 + (m_a_scales, 1), device=device, dtype=torch.float32) / 10)
74 + scale_b = (torch.randn(
75 + (1, n_b_scales), device=device, dtype=torch.float32) / 10)
76 +
77 + out = ops.cutlass_scaled_mm_dq(a, b, scale_a, scale_b, out_dtype)
78 + baseline = torch.mm(scale_a * a.to(dtype=torch.float32),
79 + scale_b *
80 + b.to(dtype=torch.float32)).to(dtype=out_dtype)
81 +
82 + assert torch.allclose(out, baseline, rtol=1e-1, atol=1e0)
83 +
84 +
85 +@pytest.mark.parametrize("m", [512, 222, 33, 1])
86 +@pytest.mark.parametrize("n", [2048, 256, 1024])
87 +@pytest.mark.parametrize("k", [128, 496, 1024])
88 +@pytest.mark.parametrize("per_act_token", [True, False])
89 +@pytest.mark.parametrize("per_out_ch", [True, False])
90 +@pytest.mark.skipif(capability < 89,
91 + reason="FP8 is not supported on this GPU type.")
92 +def test_cutlass_fp8_gemm(m: int, n: int, k: int, per_act_token: bool,
93 + per_out_ch: bool):
94 + cutlass_fp8_gemm_helper(m, n, k, per_act_token, per_out_ch)
95 +
96 +
97 +@pytest.mark.parametrize("m", [512, 222, 33, 1])
98 +@pytest.mark.parametrize("n", [2048, 256, 1024])
99 +@pytest.mark.parametrize("k", [128, 496, 1024])
100 +@pytest.mark.parametrize("per_act_token", [True, False])
101 +@pytest.mark.parametrize("per_out_ch", [True, False])
102 +def test_cutlass_int8_gemm(m: int, n: int, k: int, per_act_token: bool,
103 + per_out_ch: bool):
104 + cutlass_int8_gemm_helper(m, n, k, per_act_token, per_out_ch)
105 +
106 +
107 +@pytest.mark.parametrize("per_act_token", [True, False])
108 +@pytest.mark.parametrize("per_out_ch", [True, False])
109 +@pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16])
110 +def test_cutlass_int8_gemm_output_dtype(per_act_token: bool, per_out_ch: bool,
111 + out_dtype: Type[torch.dtype]):
112 + cutlass_int8_gemm_helper(512, 512, 512, per_act_token, per_out_ch,
113 + out_dtype)
114 +
115 +
116 +@pytest.mark.parametrize("per_act_token", [True, False])
117 +@pytest.mark.parametrize("per_out_ch", [True, False])
118 +@pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16])
119 +@pytest.mark.skipif(capability < 89,
120 + reason="FP8 is not supported on this GPU type.")
121 +def test_cutlass_fp8_gemm_output_dtype(per_act_token: bool, per_out_ch: bool,
122 + out_dtype: Type[torch.dtype]):
123 + cutlass_fp8_gemm_helper(512, 512, 512, per_act_token, per_out_ch,
124 + out_dtype)
125 +
126 +
127 +@pytest.mark.parametrize("per_act_token", [True, False])
128 +@pytest.mark.parametrize("per_out_ch", [True, False])
129 +@pytest.mark.parametrize("device", CUDA_DEVICES)
130 +@pytest.mark.skipif(capability < 89,
131 + reason="FP8 is not supported on this GPU type.")
132 +def test_cutlass_fp8_gemm_devices(per_act_token: bool, per_out_ch: bool,
133 + device: str):
134 + cutlass_fp8_gemm_helper(512, 512, 512, per_act_token, per_out_ch,
135 + torch.bfloat16, device)
136 +
137 +
138 +@pytest.mark.parametrize("per_act_token", [True, False])
139 +@pytest.mark.parametrize("per_out_ch", [True, False])
140 +@pytest.mark.parametrize("device", CUDA_DEVICES)
141 +def test_cutlass_int8_gemm_devices(per_act_token: bool, per_out_ch: bool,
142 + device: str):
143 + cutlass_int8_gemm_helper(512, 512, 512, per_act_token, per_out_ch,
144 + torch.bfloat16, device)
145 +
146 +
147 +# For the following two tests:
148 +# N and K correspond to the size of the weight matrix and likely to be multiples
149 +# of a large power of two. In any case, the kernel will have a naive fallback
150 +# when N and K are not divisible by 16. But M is the number of tokens and the
151 +# kernel must handle any M thrown at it.
152 +@pytest.mark.parametrize("per_act_token", [True, False])
153 +@pytest.mark.parametrize("per_out_ch", [True, False])
154 +@pytest.mark.skipif(capability < 89,
155 + reason="FP8 is not supported on this GPU type.")
156 +def test_cutlass_fp8_gemm_m_sweep(per_act_token: bool, per_out_ch: bool):
157 + for nk in range(32, 128, 32):
158 + for m in range(1, 128):
159 + cutlass_fp8_gemm_helper(m, nk, nk, per_act_token, per_out_ch)
160 +
161 +
162 +@pytest.mark.parametrize("per_act_token", [True, False])
163 +@pytest.mark.parametrize("per_out_ch", [True, False])
164 +def test_cutlass_int8_gemm_m_sweep(per_act_token: bool, per_out_ch: bool):
165 + for nk in range(32, 128, 32):
166 + for m in range(1, 128):
167 + cutlass_int8_gemm_helper(m, nk, nk, per_act_token, per_out_ch)
168 +
169 +
170 +# Test working with a subset of A and B
171 +def test_cutlass_subset():
172 + big_m, big_n, big_k = 1024, 1024, 1024
173 + m, n, k = 512, 512, 512
174 +
175 + whole_a = to_int8(torch.randn((big_m, big_k), device="cuda") * 5)
176 + whole_b = to_int8(torch.randn((big_n, big_k), device="cuda").t() * 5)
177 + a = whole_a[0:m, 0:k]
178 + b = whole_b[0:k, 0:n]
179 +
180 + scale_a = torch.randn((1, 1), device="cuda", dtype=torch.float32) / 10
181 + scale_b = torch.randn((1, 1), device="cuda", dtype=torch.float32) / 10
182 +
183 + out = ops.cutlass_scaled_mm_dq(a,
184 + b,
185 + scale_a,
186 + scale_b,
187 + out_dtype=torch.bfloat16)
188 + baseline = torch.mm(scale_a * a.to(dtype=torch.float32),
189 + scale_b *
190 + b.to(dtype=torch.float32)).to(dtype=torch.bfloat16)
191 +
192 + assert torch.allclose(out, baseline, rtol=1e-1, atol=1e0)
vllm/_custom_ops.py
+17 -1
@@ -1,4 +1,4 @@
1 -from typing import Optional, Tuple
1 +from typing import Optional, Tuple, Type
2
3 import torch
4
@@ -163,6 +163,22 @@ def gptq_marlin_24_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
163 size_k)
164
165
166 +# cutlass
167 +def cutlass_scaled_mm_dq(a: torch.Tensor, b: torch.Tensor,
168 + a_scales: torch.Tensor, b_scales: torch.Tensor,
169 + out_dtype: Type[torch.dtype]) -> torch.Tensor:
170 + assert (b.shape[0] % 16 == 0 and b.shape[1] % 16 == 0)
171 + assert (out_dtype is torch.bfloat16 or out_dtype is torch.float16)
172 +
173 + m = a.shape[0]
174 + n = b.shape[1]
175 + out = torch.empty((m, n), dtype=out_dtype, device=a.device)
176 +
177 + vllm_ops.cutlass_scaled_mm_dq(out, a, b, a_scales, b_scales)
178 +
179 + return out
180 +
181 +
182 # aqlm
183 def aqlm_gemm(input: torch.Tensor, codes: torch.Tensor,
184 codebooks: torch.Tensor, scales: torch.Tensor,