A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Kernel] Update CUTLASS to 3.5.1 (#7085)
Tyler Michael Smith committed
Aug 5, 2024 at 15:13 UTC
8571ac4672c8b599338cb95e23dfd624016aab36
4 files changed
+129
-107
CMakeLists.txt
+3
-3
@@ -193,8 +193,8 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
193
FetchContent_Declare(
194
cutlass
195
GIT_REPOSITORY https://github.com/nvidia/cutlass.git
196
- # CUTLASS 3.5.0
197
- GIT_TAG 7d49e6c7e2f8896c47f586706e67e1fb215529dc
196
+ # CUTLASS 3.5.1
197
+ GIT_TAG 06b21349bcf6ddf6a1686a47a137ad1446579db9
198
# Shallow clone with depth 1
199
GIT_SHALLOW TRUE
200
GIT_PROGRESS TRUE
@@ -237,7 +237,7 @@ define_gpu_extension_target(
237
SOURCES ${VLLM_EXT_SRC}
238
COMPILE_FLAGS ${VLLM_GPU_FLAGS}
239
ARCHITECTURES ${VLLM_GPU_ARCHES}
240
- INCLUDE_DIRECTORIES ${CUTLASS_INCLUDE_DIR};${CUTLASS_TOOLS_UTIL_INCLUDE_DIR}
240
+ INCLUDE_DIRECTORIES ${CUTLASS_INCLUDE_DIR}
241
USE_SABI 3
242
WITH_SOABI)
243
csrc/quantization/cutlass_w8a8/broadcast_load_epilogue_c3x.hpp
+111
-81
@@ -64,8 +64,6 @@ using namespace detail;
64
65
// Row vector broadcast
66
template<
67
- // Row bcast reuses the mbarriers from the epilogue subtile load pipeline, so this must be at least
68
- // ceil_div(StagesC, epi tiles per CTA tile) + 1 to ensure no data races
67
int Stages,
68
class CtaTileShapeMNK,
69
class Element,
@@ -73,14 +71,12 @@ template<
71
int Alignment = 128 / sizeof_bits_v<Element>
72
>
73
struct Sm90RowOrScalarBroadcast {
76
- static_assert(Alignment * sizeof_bits_v<Element> % 128 == 0, "sub-16B alignment not supported yet");
77
- static_assert(
78
- (cute::is_same_v<StrideMNL, Stride<_0,_1, _0>>) || // row vector broadcast, e.g. per-col alpha/bias
79
- (cute::is_same_v<StrideMNL, Stride<_0,_1,int>>)); // batched row vector broadcast
74
+ static_assert(Stages == 0, "Row broadcast doesn't support smem usage");
75
+ static_assert(is_static_v<decltype(take<0,2>(StrideMNL{}))>); // batch stride can be dynamic or static
76
+ static_assert(take<0,2>(StrideMNL{}) == Stride<_0,_1>{});
77
81
- // Accumulator doesn't distribute row elements evenly amongst threads so we must buffer in smem
82
- struct SharedStorage {
83
- alignas(16) array_aligned<Element, size<1>(CtaTileShapeMNK{}) * Stages> smem_row;
78
+ struct SharedStorage {
79
+ array_aligned<Element, size<1>(CtaTileShapeMNK{})> smem;
80
};
81
82
// This struct has been modified to have a bool indicating that ptr_row is a
@@ -100,6 +96,12 @@ struct Sm90RowOrScalarBroadcast {
96
return args;
97
}
98
99
+ template <class ProblemShape>
100
+ static bool
101
+ can_implement(ProblemShape const& problem_shape, Arguments const& args) {
102
+ return true;
103
+ }
104
+
105
template <class ProblemShape>
106
static size_t
107
get_workspace_size(ProblemShape const& problem_shape, Arguments const& args) {
@@ -118,15 +120,15 @@ struct Sm90RowOrScalarBroadcast {
120
121
CUTLASS_HOST_DEVICE
122
Sm90RowOrScalarBroadcast(Params const& params, SharedStorage const& shared_storage)
121
- : params(params),
122
- smem_row(const_cast<Element*>(shared_storage.smem_row.data())) { }
123
+ : params(params)
124
+ , smem(const_cast<Element*>(shared_storage.smem.data())) { }
125
126
Params params;
125
- Element* smem_row;
127
+ Element *smem = nullptr;
128
129
CUTLASS_DEVICE bool
130
is_producer_load_needed() const {
129
- return true;
131
+ return false;
132
}
133
134
CUTLASS_DEVICE bool
@@ -139,78 +141,76 @@ struct Sm90RowOrScalarBroadcast {
141
return (!params.row_broadcast && *(params.ptr_row) == Element(0));
142
}
143
142
- template <int EpiTiles, class GTensor, class STensor>
143
- struct ProducerLoadCallbacks : EmptyProducerLoadCallbacks {
144
- CUTLASS_DEVICE
145
- ProducerLoadCallbacks(GTensor&& gRow, STensor&& sRow, Params const& params)
146
- : gRow(cute::forward<GTensor>(gRow)),
147
- sRow(cute::forward<STensor>(sRow)),
148
- params(params) {}
149
-
150
- GTensor gRow; // (CTA_M,CTA_N)
151
- STensor sRow; // (CTA_M,CTA_N,PIPE)
152
- Params const& params;
153
-
154
- CUTLASS_DEVICE void
155
- begin(uint64_t* full_mbarrier_ptr, int load_iteration, bool issue_tma_load) {
156
- if (!params.row_broadcast) {
157
- return;
158
- }
159
-
160
- if (issue_tma_load) {
161
- // Increment the expect-tx count of the first subtile's mbarrier by the row vector's byte-size
162
- constexpr uint32_t copy_bytes = size<1>(CtaTileShapeMNK{}) * sizeof_bits_v<Element> / 8;
163
- cutlass::arch::ClusterTransactionBarrier::expect_transaction(full_mbarrier_ptr, copy_bytes);
164
- // Issue the TMA bulk copy
165
- auto bulk_copy = Copy_Atom<SM90_BULK_COPY_AUTO, Element>{}.with(*full_mbarrier_ptr);
166
- // Filter so we don't issue redundant copies over stride-0 modes
167
- int bcast_pipe_index = (load_iteration / EpiTiles) % Stages;
168
- copy(bulk_copy, filter(gRow), filter(sRow(_,_,bcast_pipe_index)));
169
- }
170
- }
171
- };
172
-
144
template <class... Args>
145
CUTLASS_DEVICE auto
146
get_producer_load_callbacks(ProducerLoadArgs<Args...> const& args) {
176
-
177
- auto [M, N, K, L] = args.problem_shape_mnkl;
178
- auto [m, n, k, l] = args.tile_coord_mnkl;
179
- Tensor mRow = make_tensor(make_gmem_ptr(params.ptr_row), make_shape(M,N,L), params.dRow);
180
- Tensor gRow = local_tile(mRow, take<0,2>(args.tile_shape_mnk), make_coord(m,n,l)); // (CTA_M,CTA_N)
181
- Tensor sRow = make_tensor(make_smem_ptr(smem_row), // (CTA_M,CTA_N,PIPE)
182
- make_shape(size<0>(CtaTileShapeMNK{}), size<1>(CtaTileShapeMNK{}), Stages),
183
- make_stride(_0{},_1{},size<1>(CtaTileShapeMNK{})));
184
-
185
- constexpr int EpiTiles = decltype(size<1>(zipped_divide(make_layout(take<0,2>(args.tile_shape_mnk)), args.epi_tile)))::value;
186
- return ProducerLoadCallbacks<EpiTiles, decltype(gRow), decltype(sRow)>(
187
- cute::move(gRow), cute::move(sRow), params);
147
+ return EmptyProducerLoadCallbacks{};
148
}
149
190
- template <int EpiTiles, class RTensor, class STensor>
150
+ template <class GS_GTensor, class GS_STensor, class GS_CTensor, class Tiled_G2S, class SR_STensor, class SR_RTensor, class CTensor, class ThrResidue, class ThrNum>
151
struct ConsumerStoreCallbacks : EmptyConsumerStoreCallbacks {
152
CUTLASS_DEVICE
193
- ConsumerStoreCallbacks(RTensor&& tCrRow, STensor&& tCsRow, Params const& params)
194
- : tCrRow(cute::forward<RTensor>(tCrRow)),
195
- tCsRow(cute::forward<STensor>(tCsRow)),
196
- params(params) {}
197
-
198
- RTensor tCrRow; // (CPY,CPY_M,CPY_N)
199
- STensor tCsRow; // (CPY,CPY_M,CPY_N,EPI_M,EPI_N,PIPE)
153
+ ConsumerStoreCallbacks(
154
+ GS_GTensor tGS_gRow_, GS_STensor tGS_sRow_,
155
+ GS_CTensor tGS_cRow_, Tiled_G2S tiled_g2s_,
156
+ SR_STensor tSR_sRow_, SR_RTensor tSR_rRow_,
157
+ CTensor tCcRow_, ThrResidue residue_tCcRow_, ThrNum thr_num_, Params const& params_)
158
+ : tGS_gRow(tGS_gRow_)
159
+ , tGS_sRow(tGS_sRow_)
160
+ , tGS_cRow(tGS_cRow_)
161
+ , tiled_G2S(tiled_g2s_)
162
+ , tSR_sRow(tSR_sRow_)
163
+ , tSR_rRow(tSR_rRow_)
164
+ , tCcRow(tCcRow_)
165
+ , residue_tCcRow(residue_tCcRow_)
166
+ , params(params_) {}
167
+
168
+ GS_GTensor tGS_gRow; // (CPY,CPY_M,CPY_N)
169
+ GS_STensor tGS_sRow; // (CPY,CPY_M,CPY_N)
170
+ GS_CTensor tGS_cRow; // (CPY,CPY_M,CPY_N)
171
+ Tiled_G2S tiled_G2S;
172
+
173
+ SR_STensor tSR_sRow; // (CPY,CPY_M,CPY_N,EPI_M,EPI_N)
174
+ SR_RTensor tSR_rRow; // (CPY,CPY_M,CPY_N,EPI_M,EPI_N)
175
+
176
+ CTensor tCcRow; // (CPY,CPY_M,CPY_N,EPI_M,EPI_N)
177
+ ThrResidue residue_tCcRow; // (m, n)
178
+ ThrNum thr_num;
179
Params const& params;
180
181
CUTLASS_DEVICE void
203
- previsit(int epi_m, int epi_n, int load_iteration, bool is_producer_load_needed) {
182
+ begin() {
183
if (!params.row_broadcast) {
205
- fill(tCrRow, *(params.ptr_row));
184
+ fill(tSR_rRow, *(params.ptr_row));
185
return;
186
}
187
188
+ auto synchronize = [&] () { cutlass::arch::NamedBarrier::sync(thr_num, cutlass::arch::ReservedNamedBarriers::EpilogueBarrier); };
189
+ Tensor tGS_gRow_flt = filter_zeros(tGS_gRow);
190
+ Tensor tGS_sRow_flt = filter_zeros(tGS_sRow);
191
+ Tensor tGS_cRow_flt = make_tensor(tGS_cRow.data(), make_layout(tGS_gRow_flt.shape(), tGS_cRow.stride()));
192
+
193
+ for (int i = 0; i < size(tGS_gRow_flt); ++i) {
194
+ if (get<1>(tGS_cRow_flt(i)) >= size<1>(CtaTileShapeMNK{})) {
195
+ continue; // OOB of SMEM,
196
+ }
197
+ if (elem_less(tGS_cRow_flt(i), make_coord(get<0>(residue_tCcRow), get<1>(residue_tCcRow)))) {
198
+ tGS_sRow_flt(i) = tGS_gRow_flt(i);
199
+ }
200
+ else {
201
+ tGS_sRow_flt(i) = Element(0); // Set to Zero when OOB so LDS could be issue without any preds.
202
+ }
203
+ }
204
+ synchronize();
205
+ }
206
+
207
+ CUTLASS_DEVICE void
208
+ begin_loop(int epi_m, int epi_n) {
209
if (epi_m == 0) { // Assumes M-major subtile loop
210
- // Filter so we don't issue redundant copies over stride-0 modes
211
- // (only works if 0-strides are in same location, which is by construction)
212
- int bcast_pipe_index = (load_iteration / EpiTiles) % Stages;
213
- copy_aligned(filter(tCsRow(_,_,_,epi_m,epi_n,bcast_pipe_index)), filter(tCrRow));
210
+ if (!params.row_broadcast) return; // Do not issue LDS when row is scalar
211
+ Tensor tSR_sRow_flt = filter_zeros(tSR_sRow(_,_,_,epi_m,epi_n));
212
+ Tensor tSR_rRow_flt = filter_zeros(tSR_rRow);
213
+ copy(tSR_sRow_flt, tSR_rRow_flt);
214
}
215
}
216
@@ -221,7 +221,7 @@ struct Sm90RowOrScalarBroadcast {
221
222
CUTLASS_PRAGMA_UNROLL
223
for (int i = 0; i < FragmentSize; ++i) {
224
- frg_row[i] = tCrRow(epi_v * FragmentSize + i);
224
+ frg_row[i] = tSR_rRow(epi_v * FragmentSize + i);
225
}
226
227
return frg_row;
@@ -234,17 +234,41 @@ struct Sm90RowOrScalarBroadcast {
234
>
235
CUTLASS_DEVICE auto
236
get_consumer_store_callbacks(ConsumerStoreArgs<Args...> const& args) {
237
+ auto [M, N, K, L] = args.problem_shape_mnkl;
238
+ auto [m, n, k, l] = args.tile_coord_mnkl;
239
+ using ThreadCount = decltype(size(args.tiled_copy));
240
238
- Tensor sRow = make_tensor(make_smem_ptr(smem_row), // (CTA_M,CTA_N,PIPE)
239
- make_shape(size<0>(CtaTileShapeMNK{}), size<1>(CtaTileShapeMNK{}), Stages),
240
- make_stride(_0{},_1{},size<1>(CtaTileShapeMNK{})));
241
- Tensor tCsRow = sm90_partition_for_epilogue<ReferenceSrc>( // (CPY,CPY_M,CPY_N,EPI_M,EPI_N,PIPE)
242
- sRow, args.epi_tile, args.tiled_copy, args.thread_idx);
243
- Tensor tCrRow = make_tensor_like(take<0,3>(tCsRow)); // (CPY,CPY_M,CPY_N)
244
-
245
- constexpr int EpiTiles = decltype(size<1>(zipped_divide(make_layout(take<0,2>(args.tile_shape_mnk)), args.epi_tile)))::value;
246
- return ConsumerStoreCallbacks<EpiTiles, decltype(tCrRow), decltype(tCsRow)>(
247
- cute::move(tCrRow), cute::move(tCsRow), params);
241
+ Tensor mRow = make_tensor(make_gmem_ptr(params.ptr_row), make_shape(M,N,L), params.dRow);
242
+ Tensor gRow = local_tile(mRow(_,_,l), take<0,2>(args.tile_shape_mnk), make_coord(m, n)); // (CTA_M, CTA_N)
243
+ Tensor sRow = make_tensor(make_smem_ptr(smem),
244
+ make_shape(size<0>(CtaTileShapeMNK{}), size<1>(CtaTileShapeMNK{})), make_shape(_0{}, _1{})); // (CTA_M, CTA_N)
245
+ //// G2S: Gmem to Smem
246
+ auto tiled_g2s = make_tiled_copy(Copy_Atom<DefaultCopy, Element>{},
247
+ Layout< Shape<_1, ThreadCount>,
248
+ Stride<_0, _1>>{},
249
+ Layout<_1>{});
250
+ auto thr_g2s = tiled_g2s.get_slice(args.thread_idx);
251
+ Tensor tGS_gRow = thr_g2s.partition_S(gRow);
252
+ Tensor tGS_sRow = thr_g2s.partition_D(sRow);
253
+
254
+ //// G2S: Coord
255
+ auto cRow = make_identity_tensor(make_shape(size<0>(CtaTileShapeMNK{}), size<1>(CtaTileShapeMNK{})));
256
+ Tensor tGS_cRow = thr_g2s.partition_S(cRow);
257
+
258
+ //// S2R: Smem to Reg
259
+ Tensor tSR_sRow = sm90_partition_for_epilogue<ReferenceSrc>(sRow, args.epi_tile, args.tiled_copy, args.thread_idx);
260
+ Tensor tSR_rRow = make_tensor_like(take<0,3>(tSR_sRow)); // (CPY,CPY_M,CPY_N)
261
+
262
+ return ConsumerStoreCallbacks<decltype(tGS_gRow), decltype(tGS_sRow), decltype(tGS_cRow), decltype(tiled_g2s), decltype(tSR_sRow), decltype(tSR_rRow), decltype(args.tCcD), decltype(args.residue_cD), ThreadCount>(
263
+ tGS_gRow,
264
+ tGS_sRow,
265
+ tGS_cRow, tiled_g2s,
266
+ tSR_sRow,
267
+ tSR_rRow,
268
+ args.tCcD,
269
+ args.residue_cD,
270
+ ThreadCount{},
271
+ params);
272
}
273
};
274
@@ -285,6 +309,12 @@ struct Sm90ColOrScalarBroadcast {
309
return args;
310
}
311
312
+ template <class ProblemShape>
313
+ static bool
314
+ can_implement(ProblemShape const& problem_shape, Arguments const& args) {
315
+ return true;
316
+ }
317
+
318
template <class ProblemShape>
319
static size_t
320
get_workspace_size(ProblemShape const& problem_shape, Arguments const& args) {
csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cuh
+4
-4
@@ -10,8 +10,6 @@
10
#include "cute/atom/mma_atom.hpp"
11
#include "cutlass/numeric_types.h"
12
13
-#include "cutlass/util/device_memory.h"
14
-
13
#include "cutlass/cutlass.h"
14
#include "cutlass/gemm_coord.h"
15
#include "cutlass/arch/mma_sm75.h"
@@ -301,12 +299,14 @@ inline void cutlass_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
299
// Launch the CUTLASS GEMM kernel.
300
typename Gemm::Op gemm_op;
301
size_t workspace_size = gemm_op.get_workspace_size(args);
304
- cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
302
+ auto const workspace_options =
303
+ torch::TensorOptions().dtype(torch::kUInt8).device(a.device());
304
+ auto workspace = torch::empty(workspace_size, workspace_options);
305
306
auto stream = at::cuda::getCurrentCUDAStream(a.get_device());
307
308
CUTLASS_CHECK(gemm_op.can_implement(args));
309
- cutlass::Status status = gemm_op(args, workspace.get(), stream);
309
+ cutlass::Status status = gemm_op(args, workspace.data_ptr(), stream);
310
CUTLASS_CHECK(status);
311
}
312
csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu
+11
-19
@@ -18,8 +18,6 @@
18
#include "cute/atom/mma_atom.hpp"
19
#include "cutlass/numeric_types.h"
20
21
-#include "cutlass/util/device_memory.h"
22
-
21
#include "cutlass/gemm/device/gemm_universal_adapter.h"
22
#include "cutlass/gemm/kernel/gemm_universal.hpp"
23
#include "cutlass/epilogue/collective/collective_builder.hpp"
@@ -72,13 +70,9 @@ struct ScaledEpilogueBase {
70
0 /*Stages*/, typename EpilogueDescriptor::TileShape, float,
71
Stride<Int<1>, Int<0>, Int<0>>>;
72
75
- using ScaleBDescriptor =
76
- cutlass::epilogue::collective::detail::RowBroadcastDescriptor<
77
- EpilogueDescriptor, float>;
78
-
73
using ScaleB = cutlass::epilogue::fusion::Sm90RowOrScalarBroadcast<
80
- ScaleBDescriptor::Stages, typename EpilogueDescriptor::TileShape,
81
- typename ScaleBDescriptor::Element, Stride<Int<0>, Int<1>, Int<0>>>;
74
+ 0 /*Stages*/, typename EpilogueDescriptor::TileShape, float,
75
+ Stride<Int<0>, Int<1>, Int<0>>>;
76
};
77
78
/*
@@ -154,12 +148,8 @@ struct ScaledEpilogueBias
148
cutlass::multiply_add, ElementD, float,
149
cutlass::FloatRoundStyle::round_to_nearest>;
150
157
- using BiasDescriptor =
158
- cutlass::epilogue::collective::detail::RowBroadcastDescriptor<
159
- EpilogueDescriptor, ElementD>;
160
-
151
using Bias = cutlass::epilogue::fusion::Sm90RowBroadcast<
162
- BiasDescriptor::Stages, typename EpilogueDescriptor::TileShape, ElementD,
152
+ 0 /*Stages*/, typename EpilogueDescriptor::TileShape, ElementD,
153
Stride<Int<0>, Int<1>, Int<0>>, 128 / sizeof_bits_v<ElementD>, false>;
154
155
public:
@@ -251,12 +241,12 @@ void cutlass_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
241
int64_t ldb = b.stride(1);
242
int64_t ldc = out.stride(0);
243
254
- using StrideA = Stride<int64_t, Int<1>, Int<0>>;
255
- using StrideB = Stride<int64_t, Int<1>, Int<0>>;
244
+ using StrideA = Stride<int64_t, Int<1>, int64_t>;
245
+ using StrideB = Stride<int64_t, Int<1>, int64_t>;
246
using StrideC = typename Gemm::StrideC;
247
258
- StrideA a_stride{lda, Int<1>{}, Int<0>{}};
259
- StrideB b_stride{ldb, Int<1>{}, Int<0>{}};
248
+ StrideA a_stride{lda, Int<1>{}, 0};
249
+ StrideB b_stride{ldb, Int<1>{}, 0};
250
StrideC c_stride{ldc, Int<1>{}, Int<0>{}};
251
252
using GemmKernel = typename Gemm::GemmKernel;
@@ -282,11 +272,13 @@ void cutlass_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
272
CUTLASS_CHECK(gemm_op.can_implement(args));
273
274
size_t workspace_size = gemm_op.get_workspace_size(args);
285
- cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
275
+ auto const workspace_options =
276
+ torch::TensorOptions().dtype(torch::kUInt8).device(a.device());
277
+ auto workspace = torch::empty(workspace_size, workspace_options);
278
279
auto stream = at::cuda::getCurrentCUDAStream(a.get_device());
280
289
- cutlass::Status status = gemm_op.run(args, workspace.get(), stream);
281
+ cutlass::Status status = gemm_op.run(args, workspace.data_ptr(), stream);
282
CUTLASS_CHECK(status);
283
}
284