A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Kernel] Refactor CUTLASS kernels to always take scales that reside on the GPU (#5137)
Tyler Michael Smith committed
Jun 1, 2024 at 02:45 UTC
260d119e864edbf023b1be7fa446a08bbea11f80
7 files changed
+445
-76
csrc/quantization/cutlass_w8a8/broadcast_load_epilogue_c2x.hpp
renamed
+28
-22
@@ -33,20 +33,27 @@
33
//
34
// This file is a modified excerpt of
35
// include/cutlass/epilogue/fusion/visitor_load.hpp from
36
-// https://github.com/NVIDIA/cutlass It's beem modified to support either
37
-// row/column or scalar broadcasting, like is already supported in CUTLASS 3.x.
38
-// Important because this saves us a factor 4x on the number of kernels
39
-// compiled.
36
+// https://github.com/NVIDIA/cutlass v3.5.0
37
+// It has been modified to support either
38
+// row/column or scalar broadcasting where the tensor being loaded from is
39
+// always passed in via a device pointer. This lets one compiled kernel handle
40
+// all cases of per-tensor or per-channel/per-token quantization.
41
+//
42
+// This interface also allows the scales to be passed in as tensors that
43
+// consistently reside on the device, which avoids an issue with a previous
44
+// implementation where scalars needed to be on the CPU since they
45
+// were passed in via float values. This created a potential performance hazard
46
+// if scales were initially on the device, and caused torch.compile graph
47
+// breaks when moving scales to the CPU.
48
//
49
#pragma once
50
51
+// Turn off clang-format for the entire file to keep it close to upstream
52
// clang-format off
53
54
#include "cutlass/epilogue/threadblock/fusion/visitor_2x.hpp"
55
#include "cute/tensor.hpp"
56
48
-// clang-format on
49
-
57
namespace cutlass::epilogue::threadblock {
58
59
using namespace cute;
@@ -59,9 +66,11 @@ template<
66
>
67
struct VisitorRowOrScalarBroadcast {
68
69
+ // This struct has been modified to have a bool indicating that ptr_row is a
70
+ // scalar that must be broadcast.
71
struct Arguments {
72
Element const* ptr_row = nullptr;
64
- Element null_default = Element(0);
73
+ bool row_broadcast = true;
74
StrideMNL dRow = {};
75
};
76
@@ -125,25 +134,25 @@ struct VisitorRowOrScalarBroadcast {
134
auto coord_v = filter(tC_cRow);
135
auto dst_v = filter(tC_rRow);
136
128
- if (params_ptr->ptr_row) {
137
+ if (params_ptr->row_broadcast) {
138
// In this case we are loading from a row vector and broadcasting
139
CUTLASS_PRAGMA_UNROLL
140
for (int i = 0; i < size(src_v); ++i) {
141
bool guard = get<1>(coord_v(i)) < n;
133
- cutlass::arch::global_load<VecType, sizeof(VecType)>(dst_v(i), (void const*)&src_v(i), guard);
142
+ cutlass::arch::global_load<VecType, sizeof(VecType)>(
143
+ dst_v(i), (void const*)&src_v(i), guard);
144
}
145
} else {
146
// In this case we are loading from a scalar and broadcasting
147
VecType filled_vec;
148
CUTLASS_PRAGMA_UNROLL
149
for (int i = 0; i < VecLength; i++) {
140
- reinterpret_cast<Element*>(&filled_vec)[i] = params_ptr->null_default;
150
+ reinterpret_cast<Element*>(&filled_vec)[i] = *(params_ptr->ptr_row);
151
}
152
153
CUTLASS_PRAGMA_UNROLL
154
for (int i = 0; i < size(src_v); ++i) {
145
- if(get<1>(coord_v(i)) < n)
146
- {
155
+ if (get<1>(coord_v(i)) < n) {
156
dst_v(i) = filled_vec;
157
}
158
}
@@ -208,9 +217,11 @@ template<
217
>
218
struct VisitorColOrScalarBroadcast {
219
220
+ // This struct has been modified to have a bool indicating that ptr_col is a
221
+ // scalar that must be broadcast.
222
struct Arguments {
223
Element const* ptr_col = nullptr;
213
- Element null_default = Element(0);
224
+ bool col_broadcast = true;
225
StrideMNL dCol = {};
226
};
227
@@ -230,11 +241,6 @@ struct VisitorColOrScalarBroadcast {
241
242
struct SharedStorage { };
243
233
- // Global load type
234
- static int constexpr vec_bits = ThreadMap::kElementsPerAccess * sizeof_bits<Element>::value;
235
- using VecType = uint_bit_t<cute::min(128, vec_bits)>;
236
- static int constexpr VecLength = sizeof(VecType) / sizeof(Element);
237
-
244
CUTLASS_HOST_DEVICE
245
VisitorColOrScalarBroadcast() { }
246
@@ -267,7 +273,7 @@ struct VisitorColOrScalarBroadcast {
273
int m;
274
275
// This function is modified from VisitorColBroadcast
270
- CUTLASS_DEVICE void
276
+ CUTLASS_DEVICE void
277
begin_epilogue() {
278
clear(tC_rCol);
279
@@ -277,7 +283,7 @@ struct VisitorColOrScalarBroadcast {
283
pred(i) = get<0>(tC_cCol(i)) < m;
284
}
285
280
- if (params_ptr->ptr_col) {
286
+ if (params_ptr->col_broadcast) {
287
// In this case we are loading from a column vector and broadcasting
288
copy_if(pred, tC_gCol, tC_rCol);
289
} else {
@@ -286,8 +292,8 @@ struct VisitorColOrScalarBroadcast {
292
293
CUTLASS_PRAGMA_UNROLL
294
for (int i = 0; i < size(dst_v); ++i) {
289
- if(pred(i)){
290
- dst_v(i) = params_ptr->null_default;
295
+ if (pred(i)) {
296
+ dst_v(i) = *(params_ptr->ptr_col);
297
}
298
}
299
}
csrc/quantization/cutlass_w8a8/broadcast_load_epilogue_c3x.hpp
new
+389
@@ -0,0 +1,389 @@
1
+/***************************************************************************************************
2
+ * Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights
3
+ *reserved. SPDX-License-Identifier: BSD-3-Clause
4
+ *
5
+ * Redistribution and use in source and binary forms, with or without
6
+ * modification, are permitted provided that the following conditions are met:
7
+ *
8
+ * 1. Redistributions of source code must retain the above copyright notice,
9
+ *this list of conditions and the following disclaimer.
10
+ *
11
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
12
+ * this list of conditions and the following disclaimer in the documentation
13
+ * and/or other materials provided with the distribution.
14
+ *
15
+ * 3. Neither the name of the copyright holder nor the names of its
16
+ * contributors may be used to endorse or promote products derived from
17
+ * this software without specific prior written permission.
18
+ *
19
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22
+ *ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
23
+ *LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24
+ *CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25
+ *SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26
+ *INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27
+ *CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28
+ *ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29
+ *POSSIBILITY OF SUCH DAMAGE.
30
+ *
31
+ **************************************************************************************************/
32
+
33
+//
34
+// This file is a modified excerpt of
35
+// include/cutlass/epilogue/fusion/sm90_visitor_load_tma_warpspecialized.hpp
36
+// from https://github.com/NVIDIA/cutlass v3.5.0
37
+// It has been modified to support either row/column or scalar broadcasting
38
+// where the tensor being loaded from is always passed in via a device pointer.
39
+// This lets one compiled kernel handle all cases of per-tensor or
40
+// per-channel/per-token quantization.
41
+//
42
+// This interface also allows the scales to be passed in as tensors that
43
+// consistently reside on the device, which avoids an issue with a previous
44
+// implementation where scalars needed to be on the CPU since they
45
+// were passed in via float values. This created a potential performance hazard
46
+// if scales were initially on the device, and caused torch.compile graphs
47
+// breaks when moving scales to the CPU.
48
+//
49
+#pragma once
50
+
51
+// Turn off clang-format for the entire file to keep it close to upstream
52
+// clang-format off
53
+
54
+#include "cutlass/cutlass.h"
55
+#include "cutlass/arch/barrier.h"
56
+
57
+#include "cute/tensor.hpp"
58
+#include "cutlass/epilogue/fusion/sm90_visitor_tma_warpspecialized.hpp"
59
+
60
+namespace cutlass::epilogue::fusion {
61
+
62
+using namespace cute;
63
+using namespace detail;
64
+
65
+// Row vector broadcast
66
+template<
67
+ // Row bcast reuses the mbarriers from the epilogue subtile load pipeline, so this must be at least
68
+ // ceil_div(StagesC, epi tiles per CTA tile) + 1 to ensure no data races
69
+ int Stages,
70
+ class CtaTileShapeMNK,
71
+ class Element,
72
+ class StrideMNL = Stride<_0,_1,_0>,
73
+ int Alignment = 128 / sizeof_bits_v<Element>
74
+>
75
+struct Sm90RowOrScalarBroadcast {
76
+ static_assert(Alignment * sizeof_bits_v<Element> % 128 == 0, "sub-16B alignment not supported yet");
77
+ static_assert(
78
+ (cute::is_same_v<StrideMNL, Stride<_0,_1, _0>>) || // row vector broadcast, e.g. per-col alpha/bias
79
+ (cute::is_same_v<StrideMNL, Stride<_0,_1,int>>)); // batched row vector broadcast
80
+
81
+ // Accumulator doesn't distribute row elements evenly amongst threads so we must buffer in smem
82
+ struct SharedStorage {
83
+ alignas(16) array_aligned<Element, size<1>(CtaTileShapeMNK{}) * Stages> smem_row;
84
+ };
85
+
86
+ // This struct has been modified to have a bool indicating that ptr_row is a
87
+ // scalar that must be broadcast, instead of containing a scalar that is
88
+ // valid if ptr_row is null.
89
+ struct Arguments {
90
+ Element const* ptr_row = nullptr;
91
+ bool row_broadcast = true;
92
+ StrideMNL dRow = {};
93
+ };
94
+
95
+ using Params = Arguments;
96
+
97
+ template <class ProblemShape>
98
+ static constexpr Params
99
+ to_underlying_arguments(ProblemShape const& problem_shape, Arguments const& args, void* workspace) {
100
+ return args;
101
+ }
102
+
103
+ template <class ProblemShape>
104
+ static size_t
105
+ get_workspace_size(ProblemShape const& problem_shape, Arguments const& args) {
106
+ return 0;
107
+ }
108
+
109
+ template <class ProblemShape>
110
+ static cutlass::Status
111
+ initialize_workspace(ProblemShape const& problem_shape, Arguments const& args, void* workspace, cudaStream_t stream,
112
+ CudaHostAdapter* cuda_adapter = nullptr) {
113
+ return cutlass::Status::kSuccess;
114
+ }
115
+
116
+ CUTLASS_HOST_DEVICE
117
+ Sm90RowOrScalarBroadcast() { }
118
+
119
+ CUTLASS_HOST_DEVICE
120
+ Sm90RowOrScalarBroadcast(Params const& params, SharedStorage const& shared_storage)
121
+ : params(params),
122
+ smem_row(const_cast<Element*>(shared_storage.smem_row.data())) { }
123
+
124
+ Params params;
125
+ Element* smem_row;
126
+
127
+ CUTLASS_DEVICE bool
128
+ is_producer_load_needed() const {
129
+ return true;
130
+ }
131
+
132
+ CUTLASS_DEVICE bool
133
+ is_C_load_needed() const {
134
+ return false;
135
+ }
136
+
137
+ CUTLASS_DEVICE bool
138
+ is_zero() const {
139
+ return (!params.row_broadcast && *(params.ptr_row) == Element(0));
140
+ }
141
+
142
+ template <int EpiTiles, class GTensor, class STensor>
143
+ struct ProducerLoadCallbacks : EmptyProducerLoadCallbacks {
144
+ CUTLASS_DEVICE
145
+ ProducerLoadCallbacks(GTensor&& gRow, STensor&& sRow, Params const& params)
146
+ : gRow(cute::forward<GTensor>(gRow)),
147
+ sRow(cute::forward<STensor>(sRow)),
148
+ params(params) {}
149
+
150
+ GTensor gRow; // (CTA_M,CTA_N)
151
+ STensor sRow; // (CTA_M,CTA_N,PIPE)
152
+ Params const& params;
153
+
154
+ CUTLASS_DEVICE void
155
+ begin(uint64_t* full_mbarrier_ptr, int load_iteration, bool issue_tma_load) {
156
+ if (params.ptr_row == nullptr) {
157
+ return;
158
+ }
159
+
160
+ if (issue_tma_load) {
161
+ // Increment the expect-tx count of the first subtile's mbarrier by the row vector's byte-size
162
+ constexpr uint32_t copy_bytes = size<1>(CtaTileShapeMNK{}) * sizeof_bits_v<Element> / 8;
163
+ cutlass::arch::ClusterTransactionBarrier::expect_transaction(full_mbarrier_ptr, copy_bytes);
164
+ // Issue the TMA bulk copy
165
+ auto bulk_copy = Copy_Atom<SM90_BULK_COPY_AUTO, Element>{}.with(*full_mbarrier_ptr);
166
+ // Filter so we don't issue redundant copies over stride-0 modes
167
+ int bcast_pipe_index = (load_iteration / EpiTiles) % Stages;
168
+ copy(bulk_copy, filter(gRow), filter(sRow(_,_,bcast_pipe_index)));
169
+ }
170
+ }
171
+ };
172
+
173
+ template <class... Args>
174
+ CUTLASS_DEVICE auto
175
+ get_producer_load_callbacks(ProducerLoadArgs<Args...> const& args) {
176
+
177
+ auto [M, N, K, L] = args.problem_shape_mnkl;
178
+ auto [m, n, k, l] = args.tile_coord_mnkl;
179
+ Tensor mRow = make_tensor(make_gmem_ptr(params.ptr_row), make_shape(M,N,L), params.dRow);
180
+ Tensor gRow = local_tile(mRow, take<0,2>(args.tile_shape_mnk), make_coord(m,n,l)); // (CTA_M,CTA_N)
181
+ Tensor sRow = make_tensor(make_smem_ptr(smem_row), // (CTA_M,CTA_N,PIPE)
182
+ make_shape(size<0>(CtaTileShapeMNK{}), size<1>(CtaTileShapeMNK{}), Stages),
183
+ make_stride(_0{},_1{},size<1>(CtaTileShapeMNK{})));
184
+
185
+ constexpr int EpiTiles = decltype(size<1>(zipped_divide(make_layout(take<0,2>(args.tile_shape_mnk)), args.epi_tile)))::value;
186
+ return ProducerLoadCallbacks<EpiTiles, decltype(gRow), decltype(sRow)>(
187
+ cute::move(gRow), cute::move(sRow), params);
188
+ }
189
+
190
+ template <int EpiTiles, class RTensor, class STensor>
191
+ struct ConsumerStoreCallbacks : EmptyConsumerStoreCallbacks {
192
+ CUTLASS_DEVICE
193
+ ConsumerStoreCallbacks(RTensor&& tCrRow, STensor&& tCsRow, Params const& params)
194
+ : tCrRow(cute::forward<RTensor>(tCrRow)),
195
+ tCsRow(cute::forward<STensor>(tCsRow)),
196
+ params(params) {}
197
+
198
+ RTensor tCrRow; // (CPY,CPY_M,CPY_N)
199
+ STensor tCsRow; // (CPY,CPY_M,CPY_N,EPI_M,EPI_N,PIPE)
200
+ Params const& params;
201
+
202
+ CUTLASS_DEVICE void
203
+ previsit(int epi_m, int epi_n, int load_iteration, bool is_producer_load_needed) {
204
+ if (!params.row_broadcast) {
205
+ fill(tCrRow, *(params.ptr_row));
206
+ return;
207
+ }
208
+
209
+ if (epi_m == 0) { // Assumes M-major subtile loop
210
+ // Filter so we don't issue redundant copies over stride-0 modes
211
+ // (only works if 0-strides are in same location, which is by construction)
212
+ int bcast_pipe_index = (load_iteration / EpiTiles) % Stages;
213
+ copy_aligned(filter(tCsRow(_,_,_,epi_m,epi_n,bcast_pipe_index)), filter(tCrRow));
214
+ }
215
+ }
216
+
217
+ template <typename ElementAccumulator, int FragmentSize>
218
+ CUTLASS_DEVICE Array<Element, FragmentSize>
219
+ visit(Array<ElementAccumulator, FragmentSize> const& frg_acc, int epi_v, int epi_m, int epi_n) {
220
+ Array<Element, FragmentSize> frg_row;
221
+
222
+ CUTLASS_PRAGMA_UNROLL
223
+ for (int i = 0; i < FragmentSize; ++i) {
224
+ frg_row[i] = tCrRow(epi_v * FragmentSize + i);
225
+ }
226
+
227
+ return frg_row;
228
+ }
229
+ };
230
+
231
+ template <
232
+ bool ReferenceSrc, // do register tensors reference the src or dst layout of the tiled copy
233
+ class... Args
234
+ >
235
+ CUTLASS_DEVICE auto
236
+ get_consumer_store_callbacks(ConsumerStoreArgs<Args...> const& args) {
237
+
238
+ Tensor sRow = make_tensor(make_smem_ptr(smem_row), // (CTA_M,CTA_N,PIPE)
239
+ make_shape(size<0>(CtaTileShapeMNK{}), size<1>(CtaTileShapeMNK{}), Stages),
240
+ make_stride(_0{},_1{},size<1>(CtaTileShapeMNK{})));
241
+ Tensor tCsRow = sm90_partition_for_epilogue<ReferenceSrc>( // (CPY,CPY_M,CPY_N,EPI_M,EPI_N,PIPE)
242
+ sRow, args.epi_tile, args.tiled_copy, args.thread_idx);
243
+ Tensor tCrRow = make_tensor_like(take<0,3>(tCsRow)); // (CPY,CPY_M,CPY_N)
244
+
245
+ constexpr int EpiTiles = decltype(size<1>(zipped_divide(make_layout(take<0,2>(args.tile_shape_mnk)), args.epi_tile)))::value;
246
+ return ConsumerStoreCallbacks<EpiTiles, decltype(tCrRow), decltype(tCsRow)>(
247
+ cute::move(tCrRow), cute::move(tCsRow), params);
248
+ }
249
+};
250
+
251
+/////////////////////////////////////////////////////////////////////////////////////////////////
252
+
253
+// Column vector broadcast
254
+template<
255
+ int Stages,
256
+ class CtaTileShapeMNK,
257
+ class Element,
258
+ class StrideMNL = Stride<_1,_0,_0>,
259
+ int Alignment = 128 / sizeof_bits_v<Element>
260
+>
261
+struct Sm90ColOrScalarBroadcast {
262
+ static_assert(Stages == 0, "Column broadcast doesn't support smem usage yet");
263
+ static_assert(Alignment * sizeof_bits_v<Element> % 128 == 0, "sub-16B alignment not supported yet");
264
+ static_assert(
265
+ (cute::is_same_v<StrideMNL, Stride<_1,_0, _0>>) || // col vector broadcast, e.g. per-row alpha/bias
266
+ (cute::is_same_v<StrideMNL, Stride<_1,_0,int>>)); // batched col vector broadcast, e.g. batched per-row bias
267
+
268
+ // Accumulator distributes col elements evenly amongst threads so we can just directly load from gmem
269
+ struct SharedStorage { };
270
+
271
+ // This struct has been modified to have a bool indicating that ptr_col is a
272
+ // scalar that must be broadcast, instead of containing a scalar that is
273
+ // valid if ptr_col is null.
274
+ struct Arguments {
275
+ Element const* ptr_col = nullptr;
276
+ bool col_broadcast = true;
277
+ StrideMNL dCol = {};
278
+ };
279
+
280
+ using Params = Arguments;
281
+
282
+ template <class ProblemShape>
283
+ static constexpr Params
284
+ to_underlying_arguments(ProblemShape const& problem_shape, Arguments const& args, void* workspace) {
285
+ return args;
286
+ }
287
+
288
+ template <class ProblemShape>
289
+ static size_t
290
+ get_workspace_size(ProblemShape const& problem_shape, Arguments const& args) {
291
+ return 0;
292
+ }
293
+
294
+ template <class ProblemShape>
295
+ static cutlass::Status
296
+ initialize_workspace(ProblemShape const& problem_shape, Arguments const& args, void* workspace, cudaStream_t stream,
297
+ CudaHostAdapter* cuda_adapter = nullptr) {
298
+ return cutlass::Status::kSuccess;
299
+ }
300
+
301
+ CUTLASS_DEVICE bool
302
+ is_producer_load_needed() const {
303
+ return false;
304
+ }
305
+
306
+ CUTLASS_DEVICE bool
307
+ is_C_load_needed() const {
308
+ return false;
309
+ }
310
+
311
+ CUTLASS_DEVICE bool
312
+ is_zero() const {
313
+ return (!params.col_broadcast && *(params.ptr_col) == Element(0));
314
+ }
315
+
316
+ CUTLASS_HOST_DEVICE
317
+ Sm90ColOrScalarBroadcast() { }
318
+
319
+ CUTLASS_HOST_DEVICE
320
+ Sm90ColOrScalarBroadcast(Params const& params, SharedStorage const& shared_storage)
321
+ : params(params) { }
322
+
323
+ Params params;
324
+
325
+ template <class... Args>
326
+ CUTLASS_DEVICE auto
327
+ get_producer_load_callbacks(ProducerLoadArgs<Args...> const& args) {
328
+ return EmptyProducerLoadCallbacks{};
329
+ }
330
+
331
+ template<class GTensor, class RTensor>
332
+ struct ConsumerStoreCallbacks : EmptyConsumerStoreCallbacks {
333
+ CUTLASS_DEVICE
334
+ ConsumerStoreCallbacks(GTensor&& tCgCol, RTensor&& tCrCol, Params const& params)
335
+ : tCgCol(cute::forward<GTensor>(tCgCol)),
336
+ tCrCol(cute::forward<RTensor>(tCrCol)),
337
+ params(params) {}
338
+
339
+ GTensor tCgCol; // (CPY,CPY_M,CPY_N,EPI_M,EPI_N)
340
+ RTensor tCrCol; // (CPY,CPY_M,CPY_N,EPI_M,EPI_N)
341
+ Params const& params;
342
+
343
+ CUTLASS_DEVICE void
344
+ begin() {
345
+ if (!params.col_broadcast) {
346
+ fill(tCrCol, *(params.ptr_col));
347
+ return;
348
+ }
349
+
350
+ // Filter so we don't issue redundant copies over stride-0 modes
351
+ // (only works if 0-strides are in same location, which is by construction)
352
+ copy_aligned(filter(tCgCol), filter(tCrCol));
353
+ }
354
+
355
+ template <typename ElementAccumulator, int FragmentSize>
356
+ CUTLASS_DEVICE Array<Element, FragmentSize>
357
+ visit(Array<ElementAccumulator, FragmentSize> const& frg_acc, int epi_v, int epi_m, int epi_n) {
358
+ Array<Element, FragmentSize> frg_col;
359
+ Tensor tCrCol_mn = tCrCol(_,_,_,epi_m,epi_n);
360
+
361
+ CUTLASS_PRAGMA_UNROLL
362
+ for (int i = 0; i < FragmentSize; ++i) {
363
+ frg_col[i] = tCrCol_mn(epi_v * FragmentSize + i);
364
+ }
365
+
366
+ return frg_col;
367
+ }
368
+
369
+ };
370
+
371
+ template <
372
+ bool ReferenceSrc, // do register tensors reference the src or dst layout of the tiled copy
373
+ class... Args
374
+ >
375
+ CUTLASS_DEVICE auto
376
+ get_consumer_store_callbacks(ConsumerStoreArgs<Args...> const& args) {
377
+
378
+ auto [M, N, K, L] = args.problem_shape_mnkl;
379
+ Tensor mCol = make_tensor(make_gmem_ptr(params.ptr_col), make_shape(M,N,L), params.dCol);
380
+ Tensor tCgCol = sm90_partition_for_epilogue<ReferenceSrc>( // (CPY,CPY_M,CPY_N,EPI_M,EPI_N)
381
+ mCol, args.tile_shape_mnk, args.tile_coord_mnkl, args.epi_tile, args.tiled_copy, args.thread_idx);
382
+ Tensor tCrCol = make_tensor_like(tCgCol); // (CPY,CPY_M,CPY_N,EPI_M,EPI_N)
383
+
384
+ return ConsumerStoreCallbacks<decltype(tCgCol), decltype(tCrCol)>(
385
+ cute::move(tCgCol), cute::move(tCrCol), params);
386
+ }
387
+};
388
+
389
+}
csrc/quantization/cutlass_w8a8/scaled_mm_dq_c2x.cu
+4
-10
@@ -22,7 +22,7 @@
22
#include "cutlass/epilogue/threadblock/fusion/visitors.hpp"
23
#include "cutlass/gemm/kernel/default_gemm_universal_with_visitor.h"
24
25
-#include "cutlass_visitor_2x_broadcast_epilogue.hpp"
25
+#include "broadcast_load_epilogue_c2x.hpp"
26
#include "common.hpp"
27
// clang-format on
28
@@ -145,17 +145,11 @@ void cutlass_scaled_mm_dq_dispatcher(torch::Tensor& out, torch::Tensor const& a,
145
auto a_scales_ptr = a_scales.data_ptr<float>();
146
auto b_scales_ptr = b_scales.data_ptr<float>();
147
148
- // If A and B are quantized per-tensor, then these scale tensors are scalars,
149
- // and they are passed in via the second argument.
148
using ScaleAArgs = typename Gemm::ScaleA::Arguments;
151
- ScaleAArgs a_args = a_scales.numel() == 1
152
- ? ScaleAArgs{nullptr, a_scales.item<float>(), {}}
153
- : ScaleAArgs{a_scales.data_ptr<float>(), {}, {}};
154
-
149
using ScaleBArgs = typename Gemm::ScaleB::Arguments;
156
- ScaleBArgs b_args = b_scales.numel() == 1
157
- ? ScaleBArgs{nullptr, b_scales.item<float>(), {}}
158
- : ScaleBArgs{b_scales.data_ptr<float>(), {}, {}};
150
+
151
+ ScaleBArgs b_args{b_scales.data_ptr<float>(), b_scales.numel() != 1, {}};
152
+ ScaleAArgs a_args{a_scales.data_ptr<float>(), a_scales.numel() != 1, {}};
153
154
typename Gemm::EVTCompute0::Arguments evt0_compute_args{b_args};
155
csrc/quantization/cutlass_w8a8/scaled_mm_dq_c3x.cu
+10
-10
@@ -18,11 +18,14 @@
18
#include "cute/atom/mma_atom.hpp"
19
#include "cutlass/numeric_types.h"
20
21
+#include "cutlass/util/device_memory.h"
22
+
23
#include "cutlass/gemm/device/gemm_universal_adapter.h"
24
#include "cutlass/gemm/kernel/gemm_universal.hpp"
25
#include "cutlass/epilogue/collective/collective_builder.hpp"
26
#include "cutlass/gemm/collective/collective_builder.hpp"
27
28
+#include "broadcast_load_epilogue_c3x.hpp"
29
#include "common.hpp"
30
// clang-format on
31
@@ -65,7 +68,7 @@ struct cutlass_3x_gemm {
68
69
using Accum = cutlass::epilogue::fusion::Sm90AccFetch;
70
68
- using ScaleA = cutlass::epilogue::fusion::Sm90ColBroadcast<
71
+ using ScaleA = cutlass::epilogue::fusion::Sm90ColOrScalarBroadcast<
72
0 /*Stages*/, typename EpilogueDescriptor::TileShape, float,
73
Stride<Int<1>, Int<0>, Int<0>>>;
74
@@ -73,7 +76,7 @@ struct cutlass_3x_gemm {
76
cutlass::epilogue::collective::detail::RowBroadcastDescriptor<
77
EpilogueDescriptor, float>;
78
76
- using ScaleB = cutlass::epilogue::fusion::Sm90RowBroadcast<
79
+ using ScaleB = cutlass::epilogue::fusion::Sm90RowOrScalarBroadcast<
80
ScaleBDescriptor::Stages, typename EpilogueDescriptor::TileShape,
81
typename ScaleBDescriptor::Element, Stride<Int<0>, Int<1>, Int<0>>>;
82
@@ -166,13 +169,9 @@ void cutlass_scaled_mm_dq_dispatcher(torch::Tensor& out, torch::Tensor const& a,
169
170
using ScaleA_Args = typename Gemm::ScaleA::Arguments;
171
using ScaleB_Args = typename Gemm::ScaleB::Arguments;
169
- ScaleA_Args a_args = a_scales.numel() == 1
170
- ? ScaleA_Args{nullptr, a_scales.item<float>(), {}}
171
- : ScaleA_Args{a_scales.data_ptr<float>(), {}, {}};
172
173
- ScaleB_Args b_args = b_scales.numel() == 1
174
- ? ScaleB_Args{nullptr, b_scales.item<float>(), {}}
175
- : ScaleB_Args{b_scales.data_ptr<float>(), {}, {}};
173
+ ScaleA_Args a_args{a_scales.data_ptr<float>(), a_scales.numel() != 1, {}};
174
+ ScaleB_Args b_args{b_scales.data_ptr<float>(), b_scales.numel() != 1, {}};
175
176
args.epilogue.thread = {a_args, {b_args}};
177
@@ -182,10 +181,11 @@ void cutlass_scaled_mm_dq_dispatcher(torch::Tensor& out, torch::Tensor const& a,
181
CUTLASS_CHECK(gemm_op.can_implement(args));
182
183
size_t workspace_size = gemm_op.get_workspace_size(args);
185
- TORCH_CHECK(workspace_size == 0);
184
+ cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
185
186
auto stream = at::cuda::getCurrentCUDAStream(a.get_device());
188
- cutlass::Status status = gemm_op.run(args, stream);
187
+
188
+ cutlass::Status status = gemm_op.run(args, workspace.get(), stream);
189
CUTLASS_CHECK(status);
190
}
191
} // namespace
pyproject.toml
+1
-1
@@ -59,7 +59,7 @@ exclude = [
59
]
60
61
[tool.codespell]
62
-ignore-words-list = "dout, te, indicies"
62
+ignore-words-list = "dout, te, indicies, subtile"
63
skip = "./tests/prompts,./benchmarks/sonnet.txt,./tests/lora/data,./build"
64
65
[tool.isort]
tests/kernels/test_cutlass.py
+10
-3
@@ -207,14 +207,21 @@ class CutlassLayer(torch.nn.Module):
207
self.out_dtype)
208
209
210
-def test_cutlass_cuda_graph():
210
+@pytest.mark.parametrize("per_act_token", [True, False])
211
+@pytest.mark.parametrize("per_out_ch", [True, False])
212
+def test_cutlass_cuda_graph(per_act_token: bool, per_out_ch: bool):
213
m, n, k = 512, 512, 512
214
215
a = to_int8(torch.randn((m, k), device="cuda"))
216
b = to_int8(torch.randn((n, k), device="cuda").t())
217
216
- scale_a = (torch.randn((m, 1), device="cuda", dtype=torch.float32) / 10)
217
- scale_b = (torch.randn((1, n), device="cuda", dtype=torch.float32) / 10)
218
+ m_a_scales = m if per_act_token else 1
219
+ n_b_scales = n if per_out_ch else 1
220
+
221
+ scale_a = (torch.randn(
222
+ (m_a_scales, 1), device="cuda", dtype=torch.float32) / 10)
223
+ scale_b = (torch.randn(
224
+ (1, n_b_scales), device="cuda", dtype=torch.float32) / 10)
225
226
# Construct a trivial model with a single layer that calls a CUTLASS kernel
227
model = CutlassLayer(b, scale_a, scale_b, torch.bfloat16)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_statictensor.py
+3
-30
@@ -41,46 +41,19 @@ class CompressedTensorsW8A8StaticTensor(CompressedTensorsScheme):
41
42
# TODO: remove zero_point parameters once the configs given remove them
43
44
- # Note on input/weight scales and zero_points
45
- #
46
- # When the scales have a single value, it is required that they be
47
- # on the CPU for 2 reasons,
48
- # 1. Performance:
49
- # When the scales (input_scale/weight_scales) have only a single
50
- # value, we perform a scalar broadcast of that value during the
51
- # quant/dequant operations. The "quant" and the "gemm+dequant"
52
- # kernels accept the Scalar by-value. These tensors are allocated
53
- # on the CPU in order to avoid the GPU-to-CPU copy when passing
54
- # by-value.
55
- #
56
- # 2. CUDA Graphs:
57
- # CUDA Graphs don't support GPU-to-CPU copy operations during
58
- # stream capture.
59
- #
60
- # TODO: zero-points are not supported yet. But we expect a similar
61
- # pattern.
62
-
44
is_tensor_partitioned = len(output_partition_sizes) != 1
45
weight_scale_dim = sum(
46
output_partition_sizes) if is_tensor_partitioned else 1
66
- weight_scale_device = "cpu" if weight_scale_dim == 1 else "cuda"
47
68
- input_scale = Parameter(torch.empty(1,
69
- device="cpu",
70
- dtype=torch.float32),
48
+ input_scale = Parameter(torch.empty(1, dtype=torch.float32),
49
requires_grad=False)
72
- input_zero_point = Parameter(torch.empty(1,
73
- device="cpu",
74
- dtype=torch.int8),
50
+ input_zero_point = Parameter(torch.empty(1, dtype=torch.int8),
51
requires_grad=False)
52
53
weight_scale = Parameter(torch.empty(weight_scale_dim,
78
- device=weight_scale_device,
54
dtype=torch.float32),
55
requires_grad=False)
81
- weight_zero_point = Parameter(torch.empty(1,
82
- device="cpu",
83
- dtype=torch.int8),
56
+ weight_zero_point = Parameter(torch.empty(1, dtype=torch.int8),
57
requires_grad=False)
58
59
weight = Parameter(torch.empty(sum(output_partition_sizes),