@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 260d119e86
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Kernel] Refactor CUTLASS kernels to always take scales that reside on the GPU (#5137)

Tyler Michael Smith committed Jun 1, 2024 at 02:45 UTC 260d119e864edbf023b1be7fa446a08bbea11f80
7 files changed +445 -76
csrc/quantization/cutlass_w8a8/broadcast_load_epilogue_c2x.hpp renamed
+28 -22
@@ -33,20 +33,27 @@
33 //
34 // This file is a modified excerpt of
35 // include/cutlass/epilogue/fusion/visitor_load.hpp from
36 -// https://github.com/NVIDIA/cutlass It's beem modified to support either
37 -// row/column or scalar broadcasting, like is already supported in CUTLASS 3.x.
38 -// Important because this saves us a factor 4x on the number of kernels
39 -// compiled.
36 +// https://github.com/NVIDIA/cutlass v3.5.0
37 +// It has been modified to support either
38 +// row/column or scalar broadcasting where the tensor being loaded from is
39 +// always passed in via a device pointer. This lets one compiled kernel handle
40 +// all cases of per-tensor or per-channel/per-token quantization.
41 +//
42 +// This interface also allows the scales to be passed in as tensors that
43 +// consistently reside on the device, which avoids an issue with a previous
44 +// implementation where scalars needed to be on the CPU since they
45 +// were passed in via float values. This created a potential performance hazard
46 +// if scales were initially on the device, and caused torch.compile graph
47 +// breaks when moving scales to the CPU.
48 //
49 #pragma once
50
51 +// Turn off clang-format for the entire file to keep it close to upstream
52 // clang-format off
53
54 #include "cutlass/epilogue/threadblock/fusion/visitor_2x.hpp"
55 #include "cute/tensor.hpp"
56
48 -// clang-format on
49 -
57 namespace cutlass::epilogue::threadblock {
58
59 using namespace cute;
@@ -59,9 +66,11 @@ template<
66 >
67 struct VisitorRowOrScalarBroadcast {
68
69 + // This struct has been modified to have a bool indicating that ptr_row is a
70 + // scalar that must be broadcast.
71 struct Arguments {
72 Element const* ptr_row = nullptr;
64 - Element null_default = Element(0);
73 + bool row_broadcast = true;
74 StrideMNL dRow = {};
75 };
76
@@ -125,25 +134,25 @@ struct VisitorRowOrScalarBroadcast {
134 auto coord_v = filter(tC_cRow);
135 auto dst_v = filter(tC_rRow);
136
128 - if (params_ptr->ptr_row) {
137 + if (params_ptr->row_broadcast) {
138 // In this case we are loading from a row vector and broadcasting
139 CUTLASS_PRAGMA_UNROLL
140 for (int i = 0; i < size(src_v); ++i) {
141 bool guard = get<1>(coord_v(i)) < n;
133 - cutlass::arch::global_load<VecType, sizeof(VecType)>(dst_v(i), (void const*)&src_v(i), guard);
142 + cutlass::arch::global_load<VecType, sizeof(VecType)>(
143 + dst_v(i), (void const*)&src_v(i), guard);
144 }
145 } else {
146 // In this case we are loading from a scalar and broadcasting
147 VecType filled_vec;
148 CUTLASS_PRAGMA_UNROLL
149 for (int i = 0; i < VecLength; i++) {
140 - reinterpret_cast<Element*>(&filled_vec)[i] = params_ptr->null_default;
150 + reinterpret_cast<Element*>(&filled_vec)[i] = *(params_ptr->ptr_row);
151 }
152
153 CUTLASS_PRAGMA_UNROLL
154 for (int i = 0; i < size(src_v); ++i) {
145 - if(get<1>(coord_v(i)) < n)
146 - {
155 + if (get<1>(coord_v(i)) < n) {
156 dst_v(i) = filled_vec;
157 }
158 }
@@ -208,9 +217,11 @@ template<
217 >
218 struct VisitorColOrScalarBroadcast {
219
220 + // This struct has been modified to have a bool indicating that ptr_col is a
221 + // scalar that must be broadcast.
222 struct Arguments {
223 Element const* ptr_col = nullptr;
213 - Element null_default = Element(0);
224 + bool col_broadcast = true;
225 StrideMNL dCol = {};
226 };
227
@@ -230,11 +241,6 @@ struct VisitorColOrScalarBroadcast {
241
242 struct SharedStorage { };
243
233 - // Global load type
234 - static int constexpr vec_bits = ThreadMap::kElementsPerAccess * sizeof_bits<Element>::value;
235 - using VecType = uint_bit_t<cute::min(128, vec_bits)>;
236 - static int constexpr VecLength = sizeof(VecType) / sizeof(Element);
237 -
244 CUTLASS_HOST_DEVICE
245 VisitorColOrScalarBroadcast() { }
246
@@ -267,7 +273,7 @@ struct VisitorColOrScalarBroadcast {
273 int m;
274
275 // This function is modified from VisitorColBroadcast
270 - CUTLASS_DEVICE void
276 + CUTLASS_DEVICE void
277 begin_epilogue() {
278 clear(tC_rCol);
279
@@ -277,7 +283,7 @@ struct VisitorColOrScalarBroadcast {
283 pred(i) = get<0>(tC_cCol(i)) < m;
284 }
285
280 - if (params_ptr->ptr_col) {
286 + if (params_ptr->col_broadcast) {
287 // In this case we are loading from a column vector and broadcasting
288 copy_if(pred, tC_gCol, tC_rCol);
289 } else {
@@ -286,8 +292,8 @@ struct VisitorColOrScalarBroadcast {
292
293 CUTLASS_PRAGMA_UNROLL
294 for (int i = 0; i < size(dst_v); ++i) {
289 - if(pred(i)){
290 - dst_v(i) = params_ptr->null_default;
295 + if (pred(i)) {
296 + dst_v(i) = *(params_ptr->ptr_col);
297 }
298 }
299 }
csrc/quantization/cutlass_w8a8/broadcast_load_epilogue_c3x.hpp new
+389
@@ -0,0 +1,389 @@
1 +/***************************************************************************************************
2 + * Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights
3 + *reserved. SPDX-License-Identifier: BSD-3-Clause
4 + *
5 + * Redistribution and use in source and binary forms, with or without
6 + * modification, are permitted provided that the following conditions are met:
7 + *
8 + * 1. Redistributions of source code must retain the above copyright notice,
9 + *this list of conditions and the following disclaimer.
10 + *
11 + * 2. Redistributions in binary form must reproduce the above copyright notice,
12 + * this list of conditions and the following disclaimer in the documentation
13 + * and/or other materials provided with the distribution.
14 + *
15 + * 3. Neither the name of the copyright holder nor the names of its
16 + * contributors may be used to endorse or promote products derived from
17 + * this software without specific prior written permission.
18 + *
19 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 + *ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
23 + *LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 + *CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 + *SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 + *INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 + *CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 + *ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 + *POSSIBILITY OF SUCH DAMAGE.
30 + *
31 + **************************************************************************************************/
32 +
33 +//
34 +// This file is a modified excerpt of
35 +// include/cutlass/epilogue/fusion/sm90_visitor_load_tma_warpspecialized.hpp
36 +// from https://github.com/NVIDIA/cutlass v3.5.0
37 +// It has been modified to support either row/column or scalar broadcasting
38 +// where the tensor being loaded from is always passed in via a device pointer.
39 +// This lets one compiled kernel handle all cases of per-tensor or
40 +// per-channel/per-token quantization.
41 +//
42 +// This interface also allows the scales to be passed in as tensors that
43 +// consistently reside on the device, which avoids an issue with a previous
44 +// implementation where scalars needed to be on the CPU since they
45 +// were passed in via float values. This created a potential performance hazard
46 +// if scales were initially on the device, and caused torch.compile graphs
47 +// breaks when moving scales to the CPU.
48 +//
49 +#pragma once
50 +
51 +// Turn off clang-format for the entire file to keep it close to upstream
52 +// clang-format off
53 +
54 +#include "cutlass/cutlass.h"
55 +#include "cutlass/arch/barrier.h"
56 +
57 +#include "cute/tensor.hpp"
58 +#include "cutlass/epilogue/fusion/sm90_visitor_tma_warpspecialized.hpp"
59 +
60 +namespace cutlass::epilogue::fusion {
61 +
62 +using namespace cute;
63 +using namespace detail;
64 +
65 +// Row vector broadcast
66 +template<
67 + // Row bcast reuses the mbarriers from the epilogue subtile load pipeline, so this must be at least
68 + // ceil_div(StagesC, epi tiles per CTA tile) + 1 to ensure no data races
69 + int Stages,
70 + class CtaTileShapeMNK,
71 + class Element,
72 + class StrideMNL = Stride<_0,_1,_0>,
73 + int Alignment = 128 / sizeof_bits_v<Element>
74 +>
75 +struct Sm90RowOrScalarBroadcast {
76 + static_assert(Alignment * sizeof_bits_v<Element> % 128 == 0, "sub-16B alignment not supported yet");
77 + static_assert(
78 + (cute::is_same_v<StrideMNL, Stride<_0,_1, _0>>) || // row vector broadcast, e.g. per-col alpha/bias
79 + (cute::is_same_v<StrideMNL, Stride<_0,_1,int>>)); // batched row vector broadcast
80 +
81 + // Accumulator doesn't distribute row elements evenly amongst threads so we must buffer in smem
82 + struct SharedStorage {
83 + alignas(16) array_aligned<Element, size<1>(CtaTileShapeMNK{}) * Stages> smem_row;
84 + };
85 +
86 + // This struct has been modified to have a bool indicating that ptr_row is a
87 + // scalar that must be broadcast, instead of containing a scalar that is
88 + // valid if ptr_row is null.
89 + struct Arguments {
90 + Element const* ptr_row = nullptr;
91 + bool row_broadcast = true;
92 + StrideMNL dRow = {};
93 + };
94 +
95 + using Params = Arguments;
96 +
97 + template <class ProblemShape>
98 + static constexpr Params
99 + to_underlying_arguments(ProblemShape const& problem_shape, Arguments const& args, void* workspace) {
100 + return args;
101 + }
102 +
103 + template <class ProblemShape>
104 + static size_t
105 + get_workspace_size(ProblemShape const& problem_shape, Arguments const& args) {
106 + return 0;
107 + }
108 +
109 + template <class ProblemShape>
110 + static cutlass::Status
111 + initialize_workspace(ProblemShape const& problem_shape, Arguments const& args, void* workspace, cudaStream_t stream,
112 + CudaHostAdapter* cuda_adapter = nullptr) {
113 + return cutlass::Status::kSuccess;
114 + }
115 +
116 + CUTLASS_HOST_DEVICE
117 + Sm90RowOrScalarBroadcast() { }
118 +
119 + CUTLASS_HOST_DEVICE
120 + Sm90RowOrScalarBroadcast(Params const& params, SharedStorage const& shared_storage)
121 + : params(params),
122 + smem_row(const_cast<Element*>(shared_storage.smem_row.data())) { }
123 +
124 + Params params;
125 + Element* smem_row;
126 +
127 + CUTLASS_DEVICE bool
128 + is_producer_load_needed() const {
129 + return true;
130 + }
131 +
132 + CUTLASS_DEVICE bool
133 + is_C_load_needed() const {
134 + return false;
135 + }
136 +
137 + CUTLASS_DEVICE bool
138 + is_zero() const {
139 + return (!params.row_broadcast && *(params.ptr_row) == Element(0));
140 + }
141 +
142 + template <int EpiTiles, class GTensor, class STensor>
143 + struct ProducerLoadCallbacks : EmptyProducerLoadCallbacks {
144 + CUTLASS_DEVICE
145 + ProducerLoadCallbacks(GTensor&& gRow, STensor&& sRow, Params const& params)
146 + : gRow(cute::forward<GTensor>(gRow)),
147 + sRow(cute::forward<STensor>(sRow)),
148 + params(params) {}
149 +
150 + GTensor gRow; // (CTA_M,CTA_N)
151 + STensor sRow; // (CTA_M,CTA_N,PIPE)
152 + Params const& params;
153 +
154 + CUTLASS_DEVICE void
155 + begin(uint64_t* full_mbarrier_ptr, int load_iteration, bool issue_tma_load) {
156 + if (params.ptr_row == nullptr) {
157 + return;
158 + }
159 +
160 + if (issue_tma_load) {
161 + // Increment the expect-tx count of the first subtile's mbarrier by the row vector's byte-size
162 + constexpr uint32_t copy_bytes = size<1>(CtaTileShapeMNK{}) * sizeof_bits_v<Element> / 8;
163 + cutlass::arch::ClusterTransactionBarrier::expect_transaction(full_mbarrier_ptr, copy_bytes);
164 + // Issue the TMA bulk copy
165 + auto bulk_copy = Copy_Atom<SM90_BULK_COPY_AUTO, Element>{}.with(*full_mbarrier_ptr);
166 + // Filter so we don't issue redundant copies over stride-0 modes
167 + int bcast_pipe_index = (load_iteration / EpiTiles) % Stages;
168 + copy(bulk_copy, filter(gRow), filter(sRow(_,_,bcast_pipe_index)));
169 + }
170 + }
171 + };
172 +
173 + template <class... Args>
174 + CUTLASS_DEVICE auto
175 + get_producer_load_callbacks(ProducerLoadArgs<Args...> const& args) {
176 +
177 + auto [M, N, K, L] = args.problem_shape_mnkl;
178 + auto [m, n, k, l] = args.tile_coord_mnkl;
179 + Tensor mRow = make_tensor(make_gmem_ptr(params.ptr_row), make_shape(M,N,L), params.dRow);
180 + Tensor gRow = local_tile(mRow, take<0,2>(args.tile_shape_mnk), make_coord(m,n,l)); // (CTA_M,CTA_N)
181 + Tensor sRow = make_tensor(make_smem_ptr(smem_row), // (CTA_M,CTA_N,PIPE)
182 + make_shape(size<0>(CtaTileShapeMNK{}), size<1>(CtaTileShapeMNK{}), Stages),
183 + make_stride(_0{},_1{},size<1>(CtaTileShapeMNK{})));
184 +
185 + constexpr int EpiTiles = decltype(size<1>(zipped_divide(make_layout(take<0,2>(args.tile_shape_mnk)), args.epi_tile)))::value;
186 + return ProducerLoadCallbacks<EpiTiles, decltype(gRow), decltype(sRow)>(
187 + cute::move(gRow), cute::move(sRow), params);
188 + }
189 +
190 + template <int EpiTiles, class RTensor, class STensor>
191 + struct ConsumerStoreCallbacks : EmptyConsumerStoreCallbacks {
192 + CUTLASS_DEVICE
193 + ConsumerStoreCallbacks(RTensor&& tCrRow, STensor&& tCsRow, Params const& params)
194 + : tCrRow(cute::forward<RTensor>(tCrRow)),
195 + tCsRow(cute::forward<STensor>(tCsRow)),
196 + params(params) {}
197 +
198 + RTensor tCrRow; // (CPY,CPY_M,CPY_N)
199 + STensor tCsRow; // (CPY,CPY_M,CPY_N,EPI_M,EPI_N,PIPE)
200 + Params const& params;
201 +
202 + CUTLASS_DEVICE void
203 + previsit(int epi_m, int epi_n, int load_iteration, bool is_producer_load_needed) {
204 + if (!params.row_broadcast) {
205 + fill(tCrRow, *(params.ptr_row));
206 + return;
207 + }
208 +
209 + if (epi_m == 0) { // Assumes M-major subtile loop
210 + // Filter so we don't issue redundant copies over stride-0 modes
211 + // (only works if 0-strides are in same location, which is by construction)
212 + int bcast_pipe_index = (load_iteration / EpiTiles) % Stages;
213 + copy_aligned(filter(tCsRow(_,_,_,epi_m,epi_n,bcast_pipe_index)), filter(tCrRow));
214 + }
215 + }
216 +
217 + template <typename ElementAccumulator, int FragmentSize>
218 + CUTLASS_DEVICE Array<Element, FragmentSize>
219 + visit(Array<ElementAccumulator, FragmentSize> const& frg_acc, int epi_v, int epi_m, int epi_n) {
220 + Array<Element, FragmentSize> frg_row;
221 +
222 + CUTLASS_PRAGMA_UNROLL
223 + for (int i = 0; i < FragmentSize; ++i) {
224 + frg_row[i] = tCrRow(epi_v * FragmentSize + i);
225 + }
226 +
227 + return frg_row;
228 + }
229 + };
230 +
231 + template <
232 + bool ReferenceSrc, // do register tensors reference the src or dst layout of the tiled copy
233 + class... Args
234 + >
235 + CUTLASS_DEVICE auto
236 + get_consumer_store_callbacks(ConsumerStoreArgs<Args...> const& args) {
237 +
238 + Tensor sRow = make_tensor(make_smem_ptr(smem_row), // (CTA_M,CTA_N,PIPE)
239 + make_shape(size<0>(CtaTileShapeMNK{}), size<1>(CtaTileShapeMNK{}), Stages),
240 + make_stride(_0{},_1{},size<1>(CtaTileShapeMNK{})));
241 + Tensor tCsRow = sm90_partition_for_epilogue<ReferenceSrc>( // (CPY,CPY_M,CPY_N,EPI_M,EPI_N,PIPE)
242 + sRow, args.epi_tile, args.tiled_copy, args.thread_idx);
243 + Tensor tCrRow = make_tensor_like(take<0,3>(tCsRow)); // (CPY,CPY_M,CPY_N)
244 +
245 + constexpr int EpiTiles = decltype(size<1>(zipped_divide(make_layout(take<0,2>(args.tile_shape_mnk)), args.epi_tile)))::value;
246 + return ConsumerStoreCallbacks<EpiTiles, decltype(tCrRow), decltype(tCsRow)>(
247 + cute::move(tCrRow), cute::move(tCsRow), params);
248 + }
249 +};
250 +
251 +/////////////////////////////////////////////////////////////////////////////////////////////////
252 +
253 +// Column vector broadcast
254 +template<
255 + int Stages,
256 + class CtaTileShapeMNK,
257 + class Element,
258 + class StrideMNL = Stride<_1,_0,_0>,
259 + int Alignment = 128 / sizeof_bits_v<Element>
260 +>
261 +struct Sm90ColOrScalarBroadcast {
262 + static_assert(Stages == 0, "Column broadcast doesn't support smem usage yet");
263 + static_assert(Alignment * sizeof_bits_v<Element> % 128 == 0, "sub-16B alignment not supported yet");
264 + static_assert(
265 + (cute::is_same_v<StrideMNL, Stride<_1,_0, _0>>) || // col vector broadcast, e.g. per-row alpha/bias
266 + (cute::is_same_v<StrideMNL, Stride<_1,_0,int>>)); // batched col vector broadcast, e.g. batched per-row bias
267 +
268 + // Accumulator distributes col elements evenly amongst threads so we can just directly load from gmem
269 + struct SharedStorage { };
270 +
271 + // This struct has been modified to have a bool indicating that ptr_col is a
272 + // scalar that must be broadcast, instead of containing a scalar that is
273 + // valid if ptr_col is null.
274 + struct Arguments {
275 + Element const* ptr_col = nullptr;
276 + bool col_broadcast = true;
277 + StrideMNL dCol = {};
278 + };
279 +
280 + using Params = Arguments;
281 +
282 + template <class ProblemShape>
283 + static constexpr Params
284 + to_underlying_arguments(ProblemShape const& problem_shape, Arguments const& args, void* workspace) {
285 + return args;
286 + }
287 +
288 + template <class ProblemShape>
289 + static size_t
290 + get_workspace_size(ProblemShape const& problem_shape, Arguments const& args) {
291 + return 0;
292 + }
293 +
294 + template <class ProblemShape>
295 + static cutlass::Status
296 + initialize_workspace(ProblemShape const& problem_shape, Arguments const& args, void* workspace, cudaStream_t stream,
297 + CudaHostAdapter* cuda_adapter = nullptr) {
298 + return cutlass::Status::kSuccess;
299 + }
300 +
301 + CUTLASS_DEVICE bool
302 + is_producer_load_needed() const {
303 + return false;
304 + }
305 +
306 + CUTLASS_DEVICE bool
307 + is_C_load_needed() const {
308 + return false;
309 + }
310 +
311 + CUTLASS_DEVICE bool
312 + is_zero() const {
313 + return (!params.col_broadcast && *(params.ptr_col) == Element(0));
314 + }
315 +
316 + CUTLASS_HOST_DEVICE
317 + Sm90ColOrScalarBroadcast() { }
318 +
319 + CUTLASS_HOST_DEVICE
320 + Sm90ColOrScalarBroadcast(Params const& params, SharedStorage const& shared_storage)
321 + : params(params) { }
322 +
323 + Params params;
324 +
325 + template <class... Args>
326 + CUTLASS_DEVICE auto
327 + get_producer_load_callbacks(ProducerLoadArgs<Args...> const& args) {
328 + return EmptyProducerLoadCallbacks{};
329 + }
330 +
331 + template<class GTensor, class RTensor>
332 + struct ConsumerStoreCallbacks : EmptyConsumerStoreCallbacks {
333 + CUTLASS_DEVICE
334 + ConsumerStoreCallbacks(GTensor&& tCgCol, RTensor&& tCrCol, Params const& params)
335 + : tCgCol(cute::forward<GTensor>(tCgCol)),
336 + tCrCol(cute::forward<RTensor>(tCrCol)),
337 + params(params) {}
338 +
339 + GTensor tCgCol; // (CPY,CPY_M,CPY_N,EPI_M,EPI_N)
340 + RTensor tCrCol; // (CPY,CPY_M,CPY_N,EPI_M,EPI_N)
341 + Params const& params;
342 +
343 + CUTLASS_DEVICE void
344 + begin() {
345 + if (!params.col_broadcast) {
346 + fill(tCrCol, *(params.ptr_col));
347 + return;
348 + }
349 +
350 + // Filter so we don't issue redundant copies over stride-0 modes
351 + // (only works if 0-strides are in same location, which is by construction)
352 + copy_aligned(filter(tCgCol), filter(tCrCol));
353 + }
354 +
355 + template <typename ElementAccumulator, int FragmentSize>
356 + CUTLASS_DEVICE Array<Element, FragmentSize>
357 + visit(Array<ElementAccumulator, FragmentSize> const& frg_acc, int epi_v, int epi_m, int epi_n) {
358 + Array<Element, FragmentSize> frg_col;
359 + Tensor tCrCol_mn = tCrCol(_,_,_,epi_m,epi_n);
360 +
361 + CUTLASS_PRAGMA_UNROLL
362 + for (int i = 0; i < FragmentSize; ++i) {
363 + frg_col[i] = tCrCol_mn(epi_v * FragmentSize + i);
364 + }
365 +
366 + return frg_col;
367 + }
368 +
369 + };
370 +
371 + template <
372 + bool ReferenceSrc, // do register tensors reference the src or dst layout of the tiled copy
373 + class... Args
374 + >
375 + CUTLASS_DEVICE auto
376 + get_consumer_store_callbacks(ConsumerStoreArgs<Args...> const& args) {
377 +
378 + auto [M, N, K, L] = args.problem_shape_mnkl;
379 + Tensor mCol = make_tensor(make_gmem_ptr(params.ptr_col), make_shape(M,N,L), params.dCol);
380 + Tensor tCgCol = sm90_partition_for_epilogue<ReferenceSrc>( // (CPY,CPY_M,CPY_N,EPI_M,EPI_N)
381 + mCol, args.tile_shape_mnk, args.tile_coord_mnkl, args.epi_tile, args.tiled_copy, args.thread_idx);
382 + Tensor tCrCol = make_tensor_like(tCgCol); // (CPY,CPY_M,CPY_N,EPI_M,EPI_N)
383 +
384 + return ConsumerStoreCallbacks<decltype(tCgCol), decltype(tCrCol)>(
385 + cute::move(tCgCol), cute::move(tCrCol), params);
386 + }
387 +};
388 +
389 +}
csrc/quantization/cutlass_w8a8/scaled_mm_dq_c2x.cu
+4 -10
@@ -22,7 +22,7 @@
22 #include "cutlass/epilogue/threadblock/fusion/visitors.hpp"
23 #include "cutlass/gemm/kernel/default_gemm_universal_with_visitor.h"
24
25 -#include "cutlass_visitor_2x_broadcast_epilogue.hpp"
25 +#include "broadcast_load_epilogue_c2x.hpp"
26 #include "common.hpp"
27 // clang-format on
28
@@ -145,17 +145,11 @@ void cutlass_scaled_mm_dq_dispatcher(torch::Tensor& out, torch::Tensor const& a,
145 auto a_scales_ptr = a_scales.data_ptr<float>();
146 auto b_scales_ptr = b_scales.data_ptr<float>();
147
148 - // If A and B are quantized per-tensor, then these scale tensors are scalars,
149 - // and they are passed in via the second argument.
148 using ScaleAArgs = typename Gemm::ScaleA::Arguments;
151 - ScaleAArgs a_args = a_scales.numel() == 1
152 - ? ScaleAArgs{nullptr, a_scales.item<float>(), {}}
153 - : ScaleAArgs{a_scales.data_ptr<float>(), {}, {}};
154 -
149 using ScaleBArgs = typename Gemm::ScaleB::Arguments;
156 - ScaleBArgs b_args = b_scales.numel() == 1
157 - ? ScaleBArgs{nullptr, b_scales.item<float>(), {}}
158 - : ScaleBArgs{b_scales.data_ptr<float>(), {}, {}};
150 +
151 + ScaleBArgs b_args{b_scales.data_ptr<float>(), b_scales.numel() != 1, {}};
152 + ScaleAArgs a_args{a_scales.data_ptr<float>(), a_scales.numel() != 1, {}};
153
154 typename Gemm::EVTCompute0::Arguments evt0_compute_args{b_args};
155
csrc/quantization/cutlass_w8a8/scaled_mm_dq_c3x.cu
+10 -10
@@ -18,11 +18,14 @@
18 #include "cute/atom/mma_atom.hpp"
19 #include "cutlass/numeric_types.h"
20
21 +#include "cutlass/util/device_memory.h"
22 +
23 #include "cutlass/gemm/device/gemm_universal_adapter.h"
24 #include "cutlass/gemm/kernel/gemm_universal.hpp"
25 #include "cutlass/epilogue/collective/collective_builder.hpp"
26 #include "cutlass/gemm/collective/collective_builder.hpp"
27
28 +#include "broadcast_load_epilogue_c3x.hpp"
29 #include "common.hpp"
30 // clang-format on
31
@@ -65,7 +68,7 @@ struct cutlass_3x_gemm {
68
69 using Accum = cutlass::epilogue::fusion::Sm90AccFetch;
70
68 - using ScaleA = cutlass::epilogue::fusion::Sm90ColBroadcast<
71 + using ScaleA = cutlass::epilogue::fusion::Sm90ColOrScalarBroadcast<
72 0 /*Stages*/, typename EpilogueDescriptor::TileShape, float,
73 Stride<Int<1>, Int<0>, Int<0>>>;
74
@@ -73,7 +76,7 @@ struct cutlass_3x_gemm {
76 cutlass::epilogue::collective::detail::RowBroadcastDescriptor<
77 EpilogueDescriptor, float>;
78
76 - using ScaleB = cutlass::epilogue::fusion::Sm90RowBroadcast<
79 + using ScaleB = cutlass::epilogue::fusion::Sm90RowOrScalarBroadcast<
80 ScaleBDescriptor::Stages, typename EpilogueDescriptor::TileShape,
81 typename ScaleBDescriptor::Element, Stride<Int<0>, Int<1>, Int<0>>>;
82
@@ -166,13 +169,9 @@ void cutlass_scaled_mm_dq_dispatcher(torch::Tensor& out, torch::Tensor const& a,
169
170 using ScaleA_Args = typename Gemm::ScaleA::Arguments;
171 using ScaleB_Args = typename Gemm::ScaleB::Arguments;
169 - ScaleA_Args a_args = a_scales.numel() == 1
170 - ? ScaleA_Args{nullptr, a_scales.item<float>(), {}}
171 - : ScaleA_Args{a_scales.data_ptr<float>(), {}, {}};
172
173 - ScaleB_Args b_args = b_scales.numel() == 1
174 - ? ScaleB_Args{nullptr, b_scales.item<float>(), {}}
175 - : ScaleB_Args{b_scales.data_ptr<float>(), {}, {}};
173 + ScaleA_Args a_args{a_scales.data_ptr<float>(), a_scales.numel() != 1, {}};
174 + ScaleB_Args b_args{b_scales.data_ptr<float>(), b_scales.numel() != 1, {}};
175
176 args.epilogue.thread = {a_args, {b_args}};
177
@@ -182,10 +181,11 @@ void cutlass_scaled_mm_dq_dispatcher(torch::Tensor& out, torch::Tensor const& a,
181 CUTLASS_CHECK(gemm_op.can_implement(args));
182
183 size_t workspace_size = gemm_op.get_workspace_size(args);
185 - TORCH_CHECK(workspace_size == 0);
184 + cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
185
186 auto stream = at::cuda::getCurrentCUDAStream(a.get_device());
188 - cutlass::Status status = gemm_op.run(args, stream);
187 +
188 + cutlass::Status status = gemm_op.run(args, workspace.get(), stream);
189 CUTLASS_CHECK(status);
190 }
191 } // namespace
pyproject.toml
+1 -1
@@ -59,7 +59,7 @@ exclude = [
59 ]
60
61 [tool.codespell]
62 -ignore-words-list = "dout, te, indicies"
62 +ignore-words-list = "dout, te, indicies, subtile"
63 skip = "./tests/prompts,./benchmarks/sonnet.txt,./tests/lora/data,./build"
64
65 [tool.isort]
tests/kernels/test_cutlass.py
+10 -3
@@ -207,14 +207,21 @@ class CutlassLayer(torch.nn.Module):
207 self.out_dtype)
208
209
210 -def test_cutlass_cuda_graph():
210 +@pytest.mark.parametrize("per_act_token", [True, False])
211 +@pytest.mark.parametrize("per_out_ch", [True, False])
212 +def test_cutlass_cuda_graph(per_act_token: bool, per_out_ch: bool):
213 m, n, k = 512, 512, 512
214
215 a = to_int8(torch.randn((m, k), device="cuda"))
216 b = to_int8(torch.randn((n, k), device="cuda").t())
217
216 - scale_a = (torch.randn((m, 1), device="cuda", dtype=torch.float32) / 10)
217 - scale_b = (torch.randn((1, n), device="cuda", dtype=torch.float32) / 10)
218 + m_a_scales = m if per_act_token else 1
219 + n_b_scales = n if per_out_ch else 1
220 +
221 + scale_a = (torch.randn(
222 + (m_a_scales, 1), device="cuda", dtype=torch.float32) / 10)
223 + scale_b = (torch.randn(
224 + (1, n_b_scales), device="cuda", dtype=torch.float32) / 10)
225
226 # Construct a trivial model with a single layer that calls a CUTLASS kernel
227 model = CutlassLayer(b, scale_a, scale_b, torch.bfloat16)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_statictensor.py
+3 -30
@@ -41,46 +41,19 @@ class CompressedTensorsW8A8StaticTensor(CompressedTensorsScheme):
41
42 # TODO: remove zero_point parameters once the configs given remove them
43
44 - # Note on input/weight scales and zero_points
45 - #
46 - # When the scales have a single value, it is required that they be
47 - # on the CPU for 2 reasons,
48 - # 1. Performance:
49 - # When the scales (input_scale/weight_scales) have only a single
50 - # value, we perform a scalar broadcast of that value during the
51 - # quant/dequant operations. The "quant" and the "gemm+dequant"
52 - # kernels accept the Scalar by-value. These tensors are allocated
53 - # on the CPU in order to avoid the GPU-to-CPU copy when passing
54 - # by-value.
55 - #
56 - # 2. CUDA Graphs:
57 - # CUDA Graphs don't support GPU-to-CPU copy operations during
58 - # stream capture.
59 - #
60 - # TODO: zero-points are not supported yet. But we expect a similar
61 - # pattern.
62 -
44 is_tensor_partitioned = len(output_partition_sizes) != 1
45 weight_scale_dim = sum(
46 output_partition_sizes) if is_tensor_partitioned else 1
66 - weight_scale_device = "cpu" if weight_scale_dim == 1 else "cuda"
47
68 - input_scale = Parameter(torch.empty(1,
69 - device="cpu",
70 - dtype=torch.float32),
48 + input_scale = Parameter(torch.empty(1, dtype=torch.float32),
49 requires_grad=False)
72 - input_zero_point = Parameter(torch.empty(1,
73 - device="cpu",
74 - dtype=torch.int8),
50 + input_zero_point = Parameter(torch.empty(1, dtype=torch.int8),
51 requires_grad=False)
52
53 weight_scale = Parameter(torch.empty(weight_scale_dim,
78 - device=weight_scale_device,
54 dtype=torch.float32),
55 requires_grad=False)
81 - weight_zero_point = Parameter(torch.empty(1,
82 - device="cpu",
83 - dtype=torch.int8),
56 + weight_zero_point = Parameter(torch.empty(1, dtype=torch.int8),
57 requires_grad=False)
58
59 weight = Parameter(torch.empty(sum(output_partition_sizes),