A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Kernel][Bugfix] Refactor and Fix CUTLASS 2:4 Sparse Kernels (#13198)
Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com>
Tyler Michael Smith committed
Feb 13, 2025 at 19:01 UTC
c1e37bf71b99f3c8cb4923c0b620918a93170206
16 files changed
+576
-473
CMakeLists.txt
+5
-5
@@ -228,7 +228,8 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
228
SET(CUTLASS_ENABLE_HEADERS_ONLY ON CACHE BOOL "Enable only the header library")
229
230
# Set CUTLASS_REVISION manually -- its revision detection doesn't work in this case.
231
- set(CUTLASS_REVISION "v3.6.0" CACHE STRING "CUTLASS revision to use")
231
+ # Please keep this in sync with FetchContent_Declare line below.
232
+ set(CUTLASS_REVISION "v3.7.0" CACHE STRING "CUTLASS revision to use")
233
234
# Use the specified CUTLASS source directory for compilation if VLLM_CUTLASS_SRC_DIR is provided
235
if (DEFINED ENV{VLLM_CUTLASS_SRC_DIR})
@@ -245,6 +246,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
246
FetchContent_Declare(
247
cutlass
248
GIT_REPOSITORY https://github.com/nvidia/cutlass.git
249
+ # Please keep this in sync with CUTLASS_REVISION line above.
250
GIT_TAG v3.7.0
251
GIT_PROGRESS TRUE
252
@@ -266,7 +268,6 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
268
"csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu"
269
"csrc/quantization/fp4/nvfp4_quant_entry.cu"
270
"csrc/sparse/cutlass/sparse_scaled_mm_entry.cu"
269
- "csrc/sparse/cutlass/sparse_compressor_entry.cu"
271
"csrc/cutlass_extensions/common.cpp")
272
273
set_gencode_flags_for_srcs(
@@ -359,8 +360,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
360
# The 2:4 sparse kernels cutlass_scaled_sparse_mm and cutlass_compressor
361
# require CUDA 12.2 or later (and only work on Hopper, 9.0a for now).
362
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.2 AND SCALED_MM_3X_ARCHS)
362
- set(SRCS "csrc/sparse/cutlass/sparse_compressor_c3x.cu"
363
- "csrc/sparse/cutlass/sparse_scaled_mm_c3x.cu")
363
+ set(SRCS "csrc/sparse/cutlass/sparse_scaled_mm_c3x.cu")
364
set_gencode_flags_for_srcs(
365
SRCS "${SRCS}"
366
CUDA_ARCHS "${SCALED_MM_3X_ARCHS}")
@@ -476,7 +476,7 @@ define_gpu_extension_target(
476
SOURCES ${VLLM_EXT_SRC}
477
COMPILE_FLAGS ${VLLM_GPU_FLAGS}
478
ARCHITECTURES ${VLLM_GPU_ARCHES}
479
- INCLUDE_DIRECTORIES ${CUTLASS_INCLUDE_DIR};${CUTLASS_TOOLS_UTIL_INCLUDE_DIR}
479
+ INCLUDE_DIRECTORIES ${CUTLASS_INCLUDE_DIR}
480
USE_SABI 3
481
WITH_SOABI)
482
csrc/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp
+68
-1
@@ -16,6 +16,30 @@ namespace vllm::c3x {
16
17
using namespace cute;
18
19
+template <typename T>
20
+struct identity {
21
+ CUTLASS_HOST_DEVICE
22
+ T operator()(T lhs) const { return lhs; }
23
+};
24
+
25
+template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
26
+struct TrivialEpilogue {
27
+ private:
28
+ using Accum = cutlass::epilogue::fusion::Sm90AccFetch;
29
+ using Compute = cutlass::epilogue::fusion::Sm90Compute<
30
+ cutlass::epilogue::thread::Identity, ElementD, ElementAcc,
31
+ cutlass::FloatRoundStyle::round_to_nearest>;
32
+
33
+ public:
34
+ using EVTCompute = cutlass::epilogue::fusion::Sm90EVT<Compute, Accum>;
35
+ using ArgumentType = typename EVTCompute::Arguments;
36
+
37
+ template <typename... Args>
38
+ static ArgumentType prepare_args(Args... args) {
39
+ return {};
40
+ }
41
+};
42
+
43
/*
44
* This class provides the common load descriptors for the
45
* ScaledEpilogue[...] classes
@@ -174,6 +198,49 @@ struct ScaledEpilogueBias
198
}
199
};
200
201
+/*
202
+ * This epilogue performs the same operation as ScaledEpilogueBias, but the
203
+ * bias is a column vector instead of a row vector. Useful e.g. if we are
204
+ * computing a GEMM via C^T += B^T A^T. This happens in the 2:4 sparse kernels.
205
+ */
206
+template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
207
+struct ScaledEpilogueColumnBias
208
+ : private ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor> {
209
+ private:
210
+ using SUPER = ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor>;
211
+ using Accum = typename SUPER::Accum;
212
+ using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
213
+ using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
214
+ using Bias = typename SUPER::template ColLoad<ElementD>;
215
+
216
+ using Compute0 = cutlass::epilogue::fusion::Sm90Compute<
217
+ cutlass::multiplies, float, float,
218
+ cutlass::FloatRoundStyle::round_to_nearest>;
219
+
220
+ using EVTCompute0 =
221
+ cutlass::epilogue::fusion::Sm90EVT<Compute0, ScaleB, Accum>;
222
+
223
+ using Compute1 = cutlass::epilogue::fusion::Sm90Compute<
224
+ cutlass::multiply_add, ElementD, float,
225
+ cutlass::FloatRoundStyle::round_to_nearest>;
226
+
227
+ public:
228
+ using EVTCompute =
229
+ cutlass::epilogue::fusion::Sm90EVT<Compute1, ScaleA, EVTCompute0, Bias>;
230
+
231
+ using ArgumentType = typename EVTCompute::Arguments;
232
+ static ArgumentType prepare_args(torch::Tensor const& a_scales,
233
+ torch::Tensor const& b_scales,
234
+ torch::Tensor const& bias) {
235
+ auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
236
+ auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
237
+ auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
238
+
239
+ typename EVTCompute0::Arguments evt0_args{b_args};
240
+ return ArgumentType{a_args, evt0_args, bias_args};
241
+ }
242
+};
243
+
244
/*
245
* This epilogue directly supports per-tensor azp in int32 form.
246
* As opposed to the per-token epilogue below, this epilogue only has an azp_adj
@@ -314,4 +381,4 @@ struct ScaledEpilogueBiasAzpToken
381
}
382
};
383
317
-}; // namespace vllm::c3x
\ No newline at end of file
384
+}; // namespace vllm::c3x
csrc/ops.h
+1
-2
@@ -176,8 +176,7 @@ void cutlass_scaled_sparse_mm(torch::Tensor& out, torch::Tensor const& a,
176
torch::Tensor const& b_scales,
177
std::optional<torch::Tensor> const& bias);
178
179
-bool cutlass_sparse_compress_entry(torch::Tensor& a_compressed,
180
- torch::Tensor& e, torch::Tensor const& a);
179
+std::vector<torch::Tensor> cutlass_sparse_compress(torch::Tensor const& a);
180
#endif
181
182
void static_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
csrc/quantization/cutlass_w8a8/c3x/scaled_mm.cuh
+9
-4
@@ -53,12 +53,17 @@ struct cutlass_3x_gemm {
53
54
using EVTCompute = typename Epilogue::EVTCompute;
55
56
+ // These are the minimum alignments needed for the kernels to compile
57
+ static constexpr int AlignmentAB =
58
+ 128 / cutlass::sizeof_bits<ElementAB>::value;
59
+ static constexpr int AlignmentCD = 4;
60
+
61
using CollectiveEpilogue =
62
typename cutlass::epilogue::collective::CollectiveBuilder<
63
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp, TileShape,
64
ClusterShape, cutlass::epilogue::collective::EpilogueTileAuto,
60
- ElementAcc, float, ElementC, StrideC, 4, ElementD, StrideD, 4,
61
- EpilogueSchedule, EVTCompute>::CollectiveOp;
65
+ ElementAcc, float, ElementC, StrideC, AlignmentCD, ElementD, StrideD,
66
+ AlignmentCD, EpilogueSchedule, EVTCompute>::CollectiveOp;
67
68
static constexpr size_t CEStorageSize =
69
sizeof(typename CollectiveEpilogue::SharedStorage);
@@ -69,8 +74,8 @@ struct cutlass_3x_gemm {
74
using CollectiveMainloop =
75
typename cutlass::gemm::collective::CollectiveBuilder<
76
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
72
- ElementAB, cutlass::layout::RowMajor, 16,
73
- ElementAB, cutlass::layout::ColumnMajor, 16,
77
+ ElementAB, cutlass::layout::RowMajor, AlignmentAB,
78
+ ElementAB, cutlass::layout::ColumnMajor, AlignmentAB,
79
ElementAcc, TileShape, ClusterShape,
80
Stages,
81
KernelSchedule>::CollectiveOp;
csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cuh
+8
-3
@@ -103,14 +103,19 @@ struct cutlass_2x_gemm {
103
104
using EVTD = cutlass::epilogue::threadblock::Sm80EVT<D, EVTCompute>;
105
106
+ // These are the minimum alignments needed for the kernels to compile
107
+ static constexpr int AlignmentAB =
108
+ 128 / cutlass::sizeof_bits<ElementAB>::value;
109
+ static constexpr int AlignmentCD = 4;
110
+
111
// clang-format off
112
using RowMajor = typename cutlass::layout::RowMajor;
113
using ColumnMajor = typename cutlass::layout::ColumnMajor;
114
using KernelType =
115
ArchGuard<typename cutlass::gemm::kernel::DefaultGemmWithVisitor<
111
- ElementAB, RowMajor, cutlass::ComplexTransform::kNone, 16,
112
- ElementAB, ColumnMajor, cutlass::ComplexTransform::kNone, 16,
113
- float, cutlass::layout::RowMajor, 4,
116
+ ElementAB, RowMajor, cutlass::ComplexTransform::kNone, AlignmentAB,
117
+ ElementAB, ColumnMajor, cutlass::ComplexTransform::kNone, AlignmentAB,
118
+ float, cutlass::layout::RowMajor, AlignmentCD,
119
ElementAcc, float, cutlass::arch::OpClassTensorOp,
120
Arch,
121
TileShape, WarpShape, InstructionShape,
csrc/sparse/cutlass/sparse_compressor_c3x.cu
deleted
-165
@@ -1,165 +0,0 @@
1
-// clang-format will break include orders
2
-// clang-format off
3
-#include <cudaTypedefs.h>
4
-
5
-#if defined CUDA_VERSION && CUDA_VERSION >= 12020
6
-#include "sparse_scaled_mm_c3x.cuh"
7
-
8
-#include "cutlass/numeric_conversion.h"
9
-#include "cutlass/transform/device/transform_universal_adapter.hpp"
10
-#include "cutlass/transform/kernel/sparse_gemm_compressor.hpp"
11
-#include "cutlass/epilogue/collective/default_epilogue.hpp"
12
-
13
-#include "cutlass/util/host_tensor.h"
14
-#include "cutlass/util/packed_stride.hpp"
15
-// clang-format on
16
-
17
-using namespace cute;
18
-using namespace vllm;
19
-
20
-/// Make A structured sparse by replacing elements with 0 and compress it
21
-template <typename ElementA_, typename ElementAcc_>
22
-bool cutlass_sparse_compress(torch::Tensor& a_nzs, torch::Tensor& a_meta,
23
- torch::Tensor const& a) {
24
- // Checks for conformality
25
- TORCH_CHECK(a.dtype() == torch::kInt8 || a.dtype() == torch::kFloat8_e4m3fn ||
26
- a.dtype() == torch::kFloat16 || a.dtype() == torch::kBFloat16);
27
- TORCH_CHECK(a.dim() == 2)
28
- // Check for strides and alignment
29
- TORCH_CHECK(a.stride(0) % 4 == 0) // Required for semi-structured sparsity
30
- TORCH_CHECK(a.stride(1) == 1)
31
-
32
- int m = a.size(0);
33
- int k = a.size(1);
34
-
35
- // Sparse kernel setup; this kernel is not used for matmul,
36
- // but just for setting up the compressor utility
37
- // A matrix configuration
38
- using ElementA = ElementA_;
39
- using LayoutTagA = cutlass::layout::RowMajor;
40
- constexpr int AlignmentA = 128 / cutlass::sizeof_bits<ElementA>::value;
41
- // B matrix configuration
42
- using ElementB = ElementA;
43
- using LayoutTagB = cutlass::layout::ColumnMajor;
44
- constexpr int AlignmentB = 128 / cutlass::sizeof_bits<ElementB>::value;
45
- // C/D matrix configuration
46
- using ElementC = float;
47
- using LayoutTagC = cutlass::layout::ColumnMajor;
48
- constexpr int AlignmentC = 128 / cutlass::sizeof_bits<ElementC>::value;
49
- // Core kernel configurations
50
- using ElementAccumulator = ElementAcc_;
51
- using TileShape = Shape<_128, _128, _128>;
52
- using TileShapeRef = Shape<_128, _128, _64>;
53
- using ClusterShape = Shape<_1, _2, _1>;
54
- using KernelSchedule = typename std::conditional<
55
- std::is_same_v<ElementA, cutlass::float_e4m3_t>,
56
- cutlass::gemm::KernelTmaWarpSpecializedFP8FastAccum,
57
- cutlass::gemm::KernelTmaWarpSpecialized>::type;
58
-
59
- using EpilogueSchedule = cutlass::epilogue::TmaWarpSpecialized;
60
- using ProblemShape = Shape<int, int, int, int>;
61
-
62
- using CollectiveEpilogue =
63
- typename cutlass::epilogue::collective::CollectiveBuilder<
64
- cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp, TileShape,
65
- ClusterShape, cutlass::epilogue::collective::EpilogueTileAuto,
66
- ElementAccumulator, ElementAccumulator, ElementC, LayoutTagC,
67
- AlignmentC, ElementC, LayoutTagC, AlignmentC,
68
- EpilogueSchedule>::CollectiveOp;
69
-
70
- using CollectiveMainloop =
71
- typename cutlass::gemm::collective::CollectiveBuilder<
72
- cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp, ElementA,
73
- LayoutTagA, AlignmentA, ElementB, LayoutTagB, AlignmentB,
74
- ElementAccumulator, TileShape, ClusterShape,
75
- cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(
76
- sizeof(typename CollectiveEpilogue::SharedStorage))>,
77
- KernelSchedule>::CollectiveOp;
78
-
79
- using GemmKernel =
80
- cutlass::gemm::kernel::GemmUniversal<ProblemShape, CollectiveMainloop,
81
- CollectiveEpilogue>;
82
-
83
- using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
84
-
85
- using StrideA = cutlass::gemm::TagToStrideA_t<LayoutTagA>;
86
- using StrideE = StrideA;
87
-
88
- using StrideA = Stride<int64_t, Int<1>, int64_t>;
89
-
90
- // The n (=1) dimension does not matter for the compressor
91
- typename GemmKernel::ProblemShape prob_shape{m, 1, k, 1};
92
-
93
- using LayoutA = typename GemmKernel::CollectiveMainloop::LayoutA;
94
- using LayoutE = typename GemmKernel::CollectiveMainloop::LayoutE;
95
-
96
- using ElementE = typename GemmKernel::CollectiveMainloop::ElementE;
97
- using SparseConfig = typename GemmKernel::CollectiveMainloop::SparseConfig;
98
-
99
- // Offline compressor kernel
100
- using CompressorUtility =
101
- cutlass::transform::kernel::StructuredSparseCompressorUtility<
102
- ProblemShape, ElementA, LayoutTagA, SparseConfig>;
103
-
104
- using CompressorKernel =
105
- cutlass::transform::kernel::StructuredSparseCompressor<
106
- ProblemShape, ElementA, LayoutTagA, SparseConfig,
107
- cutlass::arch::Sm90>;
108
-
109
- using Compressor =
110
- cutlass::transform::device::TransformUniversalAdapter<CompressorKernel>;
111
-
112
- auto [M, N, K, L] = prob_shape;
113
-
114
- StrideA stride_A;
115
- stride_A =
116
- cutlass::make_cute_packed_stride(StrideA{}, cute::make_shape(M, K, L));
117
-
118
- CompressorUtility compressor_utility(prob_shape, stride_A);
119
-
120
- int ME = compressor_utility.get_metadata_m_physical();
121
- int KE = compressor_utility.get_metadata_k_physical();
122
- int KC = compressor_utility.get_tensorA_k_physical();
123
-
124
- auto a_ptr = static_cast<ElementA*>(a.data_ptr());
125
-
126
- auto a_nzs_ptr = static_cast<ElementA*>(a_nzs.data_ptr());
127
- auto a_meta_ptr = static_cast<typename Gemm::CollectiveMainloop::ElementE*>(
128
- a_meta.data_ptr());
129
-
130
- cutlass::KernelHardwareInfo hw_info;
131
- hw_info.device_id = 0;
132
- hw_info.sm_count =
133
- cutlass::KernelHardwareInfo::query_device_multiprocessor_count(
134
- hw_info.device_id);
135
- typename Compressor::Arguments arguments{
136
- prob_shape, {a_ptr, stride_A, a_nzs_ptr, a_meta_ptr}, {hw_info}};
137
-
138
- Compressor compressor_op;
139
- size_t workspace_size = Compressor::get_workspace_size(arguments);
140
- cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
141
-
142
- CUTLASS_CHECK(compressor_op.can_implement(arguments));
143
- CUTLASS_CHECK(compressor_op.initialize(arguments, workspace.get()));
144
- CUTLASS_CHECK(compressor_op.run());
145
- CUDA_CHECK(cudaDeviceSynchronize());
146
-
147
- return true;
148
-}
149
-
150
-bool cutlass_sparse_compress_sm90(torch::Tensor& a_nzs, torch::Tensor& a_meta,
151
- torch::Tensor const& a) {
152
- if (a.dtype() == torch::kBFloat16) {
153
- return cutlass_sparse_compress<cutlass::bfloat16_t, float>(a_nzs, a_meta,
154
- a);
155
- } else if (a.dtype() == torch::kFloat16) {
156
- return cutlass_sparse_compress<cutlass::half_t, float>(a_nzs, a_meta, a);
157
- } else if (a.dtype() == torch::kFloat8_e4m3fn) {
158
- return cutlass_sparse_compress<cutlass::float_e4m3_t, float>(a_nzs, a_meta,
159
- a);
160
- } else if (a.dtype() == torch::kInt8) {
161
- return cutlass_sparse_compress<int8_t, int32_t>(a_nzs, a_meta, a);
162
- }
163
- return false;
164
-}
165
-#endif
csrc/sparse/cutlass/sparse_compressor_c3x.cuh
new
+90
@@ -0,0 +1,90 @@
1
+#pragma once
2
+
3
+// clang-format will break include orders
4
+// clang-format off
5
+#include <cudaTypedefs.h>
6
+
7
+#if defined CUDA_VERSION && CUDA_VERSION >= 12020
8
+#include "sparse_scaled_mm_c3x.cuh"
9
+
10
+#include "cutlass/numeric_conversion.h"
11
+#include "cutlass/transform/device/transform_universal_adapter.hpp"
12
+#include "cutlass/transform/kernel/sparse_gemm_compressor.hpp"
13
+#include "cutlass/epilogue/collective/default_epilogue.hpp"
14
+
15
+// clang-format on
16
+
17
+using namespace cute;
18
+using namespace vllm;
19
+
20
+using CompressorResult = std::tuple<torch::Tensor, torch::Tensor>;
21
+/// Make A structured sparse by replacing elements with 0 and compress it
22
+template <typename Gemm>
23
+CompressorResult cutlass_sparse_compress(torch::Tensor const& a) {
24
+ // Checks for conformality
25
+ TORCH_CHECK(a.dtype() == torch::kInt8 || a.dtype() == torch::kFloat8_e4m3fn ||
26
+ a.dtype() == torch::kFloat16 || a.dtype() == torch::kBFloat16);
27
+ TORCH_CHECK(a.dim() == 2)
28
+ // Check for strides and alignment
29
+ TORCH_CHECK(a.stride(0) % 4 == 0) // Required for semi-structured sparsity
30
+ TORCH_CHECK(a.stride(1) == 1)
31
+
32
+ using GemmKernel = typename Gemm::KernelType;
33
+ using ElementA = typename Gemm::ElementAB;
34
+ using ElementE = typename GemmKernel::CollectiveMainloop::ElementE;
35
+
36
+ int m = a.size(0);
37
+ int k = a.size(1);
38
+ using ProblemShape = typename GemmKernel::ProblemShape;
39
+ ProblemShape prob_shape{m, 1, k, 1};
40
+
41
+ int64_t lda = a.stride(0);
42
+ using StrideA = Stride<int64_t, Int<1>, int64_t>;
43
+ StrideA a_stride{lda, Int<1>{}, 0};
44
+
45
+ using CompressorUtility = typename Gemm::CompressorUtility;
46
+ CompressorUtility compressor_utility(prob_shape, a_stride);
47
+
48
+ // Allocate buffers for the metadata E and the compressed matrix A
49
+ int ME = compressor_utility.get_metadata_m_physical();
50
+ int KE = compressor_utility.get_metadata_k_physical();
51
+ int MC = compressor_utility.get_tensorA_m_physical();
52
+ int KC = compressor_utility.get_tensorA_k_physical();
53
+
54
+ auto const a_meta_options =
55
+ torch::TensorOptions().dtype(torch::kUInt8).device(a.device());
56
+ auto const a_nzs_options =
57
+ torch::TensorOptions().dtype(a.dtype()).device(a.device());
58
+
59
+ auto a_meta = torch::zeros({ME, KE}, a_meta_options);
60
+ auto a_nzs = torch::zeros({MC, KC}, a_nzs_options);
61
+
62
+ auto a_ptr = static_cast<ElementA*>(a.data_ptr());
63
+ auto a_nzs_ptr = static_cast<ElementA*>(a_nzs.data_ptr());
64
+ auto a_meta_ptr = static_cast<ElementE*>(a_meta.data_ptr());
65
+
66
+ cutlass::KernelHardwareInfo hw_info;
67
+ hw_info.device_id = a.device().index();
68
+ hw_info.sm_count =
69
+ cutlass::KernelHardwareInfo::query_device_multiprocessor_count(
70
+ hw_info.device_id);
71
+
72
+ using Compressor = typename Gemm::Compressor;
73
+ typename Compressor::Arguments arguments{
74
+ prob_shape, {a_ptr, a_stride, a_nzs_ptr, a_meta_ptr}, {hw_info}};
75
+
76
+ Compressor compressor_op;
77
+ size_t workspace_size = Compressor::get_workspace_size(arguments);
78
+ auto const workspace_options =
79
+ torch::TensorOptions().dtype(torch::kUInt8).device(a.device());
80
+ auto workspace = torch::empty(workspace_size, workspace_options);
81
+
82
+ CUTLASS_CHECK(compressor_op.can_implement(arguments));
83
+ CUTLASS_CHECK(compressor_op.initialize(arguments, workspace.data_ptr()));
84
+ CUTLASS_CHECK(compressor_op.run());
85
+ CUDA_CHECK(cudaDeviceSynchronize());
86
+
87
+ return {a_meta, a_nzs};
88
+}
89
+
90
+#endif
csrc/sparse/cutlass/sparse_compressor_entry.cu
deleted
-42
@@ -1,42 +0,0 @@
1
-#include <cudaTypedefs.h>
2
-
3
-#include <c10/cuda/CUDAGuard.h>
4
-#include <torch/all.h>
5
-
6
-#include "cutlass_extensions/common.hpp"
7
-
8
-#if defined ENABLE_SPARSE_SCALED_MM_C3X && ENABLE_SPARSE_SCALED_MM_C3X
9
-bool cutlass_sparse_compress_sm90(torch::Tensor& a_nzs, torch::Tensor& a_meta,
10
- torch::Tensor const& a);
11
-#endif
12
-
13
-bool cutlass_sparse_compress_entry(torch::Tensor& a_nzs, torch::Tensor& a_meta,
14
- torch::Tensor const& a) {
15
- // Checks for conformality
16
- TORCH_CHECK(a.dim() == 2 && a_meta.dim() == 2 && a_nzs.dim() == 2);
17
- TORCH_CHECK(a.size(0) == a_nzs.size(0) && a.size(0) == a_meta.size(0) &&
18
- a_nzs.size(1) * 2 == a.size(1) &&
19
- a_meta.size(1) * 2 * 4 == a.size(1));
20
- // Considering elemsPerMetaElem = 8b / 2b_per_nz = 4
21
-
22
- // Check for strides and alignment
23
- TORCH_CHECK(a.stride(1) == 1 && a_nzs.stride(1) == 1 &&
24
- a_meta.stride(1) == 1); // Row-major
25
- TORCH_CHECK(a.stride(0) % 8 == 0); // 8 Byte Alignment for Compression
26
-
27
- at::cuda::OptionalCUDAGuard const device_guard(device_of(a));
28
- int32_t version_num = get_sm_version_num();
29
-
30
- // Guard against compilation issues for sm90 kernels
31
-#if defined ENABLE_SPARSE_SCALED_MM_C3X && ENABLE_SPARSE_SCALED_MM_C3X
32
- if (version_num >= 90) {
33
- return cutlass_sparse_compress_sm90(a_nzs, a_meta, a);
34
- }
35
-#endif
36
-
37
- TORCH_CHECK_NOT_IMPLEMENTED(
38
- false,
39
- "No compiled cutlass_scaled_sparse_mm for a compute capability less than "
40
- "CUDA device capability: ",
41
- version_num);
42
-}
csrc/sparse/cutlass/sparse_scaled_mm_c3x.cu
+134
-130
@@ -9,17 +9,30 @@
9
using namespace cute;
10
using namespace vllm;
11
12
+struct GemmCallerTraits {
13
+ using return_type = void;
14
+
15
+ template <typename GemmConfig, typename... Args>
16
+ static return_type invoke(Args&&... args) {
17
+ return cutlass_sparse_gemm_caller<GemmConfig>(std::forward<Args>(args)...);
18
+ }
19
+};
20
+
21
+struct GemmCompressorTraits {
22
+ using return_type = CompressorResult;
23
+
24
+ template <typename GemmConfig, typename... Args>
25
+ static return_type invoke(Args&&... args) {
26
+ return cutlass_sparse_compress<GemmConfig>(std::forward<Args>(args)...);
27
+ }
28
+};
29
+
30
template <typename InType, typename OutType,
31
template <typename, typename, typename> typename Epilogue,
14
- typename... EpilogueArgs>
15
-void cutlass_gemm_sm90_fp8_dispatch(torch::Tensor& out, torch::Tensor const& a,
16
- torch::Tensor const& bt_nzs,
17
- torch::Tensor const& bt_meta,
18
- EpilogueArgs&&... args) {
19
- static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
20
- TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
21
- TORCH_CHECK(bt_meta.dtype() == torch::kUInt8);
22
- TORCH_CHECK(bt_nzs.dtype() == torch::kFloat8_e4m3fn);
32
+ typename DispatchFunc, typename... Args>
33
+typename DispatchFunc::return_type cutlass_gemm_sm90_fp8_dispatch(
34
+ uint32_t m, uint32_t n, Args&&... args) {
35
+ static_assert(std::is_same_v<InType, cutlass::float_e4m3_t>);
36
37
using Cutlass3xGemmDefault =
38
typename sm90_config_default<InType, OutType, Epilogue>::Cutlass3xGemm;
@@ -49,122 +62,87 @@ void cutlass_gemm_sm90_fp8_dispatch(torch::Tensor& out, torch::Tensor const& a,
62
using Cutlass3xGemm8 =
63
typename sm90_fp8_config_8<InType, OutType, Epilogue>::Cutlass3xGemm;
64
52
- uint32_t const n = bt_nzs.size(0);
53
- uint32_t const m = a.size(0); // Batch size
65
uint32_t const mp2 =
66
std::max(static_cast<uint32_t>(64), next_pow_2(m)); // next power of 2
67
68
if (mp2 <= 64) {
69
if (n == 28672) {
59
- return cutlass_sparse_gemm_caller<Cutlass3xGemm2>(
60
- out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
70
+ return DispatchFunc::template invoke<Cutlass3xGemm2>(
71
+ std::forward<Args>(args)...);
72
} else if (n == 4096 || n == 6144) {
62
- return cutlass_sparse_gemm_caller<Cutlass3xGemm1>(
63
- out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
73
+ return DispatchFunc::template invoke<Cutlass3xGemm1>(
74
+ std::forward<Args>(args)...);
75
}
76
} else if (mp2 <= 128) {
77
if (n == 4096) {
67
- return cutlass_sparse_gemm_caller<Cutlass3xGemm3>(
68
- out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
78
+ return DispatchFunc::template invoke<Cutlass3xGemm3>(
79
+ std::forward<Args>(args)...);
80
} else if (n == 28672) {
70
- return cutlass_sparse_gemm_caller<Cutlass3xGemm5>(
71
- out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
81
+ return DispatchFunc::template invoke<Cutlass3xGemm5>(
82
+ std::forward<Args>(args)...);
83
} else if (n == 6144) {
73
- return cutlass_sparse_gemm_caller<Cutlass3xGemm4>(
74
- out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
84
+ return DispatchFunc::template invoke<Cutlass3xGemm4>(
85
+ std::forward<Args>(args)...);
86
}
87
} else if (mp2 <= 256) {
88
if (n == 4096) {
78
- return cutlass_sparse_gemm_caller<Cutlass3xGemm6>(
79
- out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
89
+ return DispatchFunc::template invoke<Cutlass3xGemm6>(
90
+ std::forward<Args>(args)...);
91
} else if (n == 28672) {
81
- return cutlass_sparse_gemm_caller<Cutlass3xGemm8>(
82
- out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
92
+ return DispatchFunc::template invoke<Cutlass3xGemm8>(
93
+ std::forward<Args>(args)...);
94
} else if (n == 6144) {
84
- return cutlass_sparse_gemm_caller<Cutlass3xGemm7>(
85
- out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
95
+ return DispatchFunc::template invoke<Cutlass3xGemm7>(
96
+ std::forward<Args>(args)...);
97
}
98
} else {
99
if (n == 6144 || n == 28672) {
89
- return cutlass_sparse_gemm_caller<Cutlass3xGemm8>(
90
- out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
100
+ return DispatchFunc::template invoke<Cutlass3xGemm8>(
101
+ std::forward<Args>(args)...);
102
} else if (n == 4096) {
92
- return cutlass_sparse_gemm_caller<Cutlass3xGemm7>(
93
- out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
103
+ return DispatchFunc::template invoke<Cutlass3xGemm7>(
104
+ std::forward<Args>(args)...);
105
}
106
}
107
108
// Otherwise the default heuristic
109
if (mp2 <= 64) {
110
// n in [1, 64]
100
- return cutlass_sparse_gemm_caller<Cutlass3xGemmM64>(
101
- out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
111
+ return DispatchFunc::template invoke<Cutlass3xGemmM64>(
112
+ std::forward<Args>(args)...);
113
} else if (mp2 <= 128) {
114
// n in (64, 128]
104
- return cutlass_sparse_gemm_caller<Cutlass3xGemmM128>(
105
- out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
115
+ return DispatchFunc::template invoke<Cutlass3xGemmM128>(
116
+ std::forward<Args>(args)...);
117
} else if (mp2 <= 256) {
118
// n in (128, 256]
108
- return cutlass_sparse_gemm_caller<Cutlass3xGemmM256>(
109
- out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
119
+ return DispatchFunc::template invoke<Cutlass3xGemmM256>(
120
+ std::forward<Args>(args)...);
121
} else {
122
// n in (256, inf)
112
- return cutlass_sparse_gemm_caller<Cutlass3xGemmM512>(
113
- out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
123
+ return DispatchFunc::template invoke<Cutlass3xGemmM512>(
124
+ std::forward<Args>(args)...);
125
}
126
}
127
128
template <typename InType, typename OutType,
129
template <typename, typename, typename> typename Epilogue,
119
- typename... EpilogueArgs>
120
-void cutlass_gemm_sm90_fp16_dispatch(torch::Tensor& out, torch::Tensor const& a,
121
- torch::Tensor const& bt_nzs,
122
- torch::Tensor const& bt_meta,
123
- EpilogueArgs&&... args) {
124
- static_assert(std::is_same<InType, cutlass::half_t>());
125
- TORCH_CHECK(a.dtype() == torch::kFloat16);
126
- TORCH_CHECK(bt_meta.dtype() == torch::kUInt8);
127
- TORCH_CHECK(bt_nzs.dtype() == torch::kFloat16);
128
-
129
- using Cutlass3xGemmDefault =
130
- typename sm90_config_default<InType, OutType, Epilogue>::Cutlass3xGemm;
131
-
132
- // m in (128, inf)
133
- return cutlass_sparse_gemm_caller<Cutlass3xGemmDefault>(
134
- out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
135
-}
136
-
137
-template <typename InType, typename OutType,
138
- template <typename, typename, typename> typename Epilogue,
139
- typename... EpilogueArgs>
140
-void cutlass_gemm_sm90_bf16_dispatch(torch::Tensor& out, torch::Tensor const& a,
141
- torch::Tensor const& bt_nzs,
142
- torch::Tensor const& bt_meta,
143
- EpilogueArgs&&... args) {
144
- static_assert(std::is_same<InType, cutlass::bfloat16_t>());
145
- TORCH_CHECK(a.dtype() == torch::kBFloat16);
146
- TORCH_CHECK(bt_meta.dtype() == torch::kUInt8);
147
- TORCH_CHECK(bt_nzs.dtype() == torch::kBFloat16);
148
-
130
+ typename DispatchFunc, typename... Args>
131
+typename DispatchFunc::return_type cutlass_gemm_sm90_16bit_dispatch(
132
+ uint32_t m, uint32_t n, Args&&... args) {
133
using Cutlass3xGemmDefault =
134
typename sm90_config_default<InType, OutType, Epilogue>::Cutlass3xGemm;
135
152
- // m in (128, inf)
153
- return cutlass_sparse_gemm_caller<Cutlass3xGemmDefault>(
154
- out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
136
+ return DispatchFunc::template invoke<Cutlass3xGemmDefault>(
137
+ std::forward<Args>(args)...);
138
}
139
140
template <typename InType, typename OutType,
141
template <typename, typename, typename> typename Epilogue,
159
- typename... EpilogueArgs>
160
-void cutlass_gemm_sm90_int8_dispatch(torch::Tensor& out, torch::Tensor const& a,
161
- torch::Tensor const& bt_nzs,
162
- torch::Tensor const& bt_meta,
163
- EpilogueArgs&&... args) {
164
- static_assert(std::is_same<InType, int8_t>());
165
- TORCH_CHECK(a.dtype() == torch::kInt8);
166
- TORCH_CHECK(bt_meta.dtype() == torch::kUInt8);
167
- TORCH_CHECK(bt_nzs.dtype() == torch::kInt8);
142
+ typename DispatchFunc, typename... Args>
143
+typename DispatchFunc::return_type cutlass_gemm_sm90_int8_dispatch(
144
+ uint32_t m, uint32_t n, Args&&... args) {
145
+ static_assert(std::is_same_v<InType, int8_t>);
146
147
using Cutlass3xGemmDefault =
148
typename sm90_config_default<InType, OutType, Epilogue>::Cutlass3xGemm;
@@ -179,37 +157,35 @@ void cutlass_gemm_sm90_int8_dispatch(torch::Tensor& out, torch::Tensor const& a,
157
typename sm90_int8_config_M32_NSmall<InType, OutType,
158
Epilogue>::Cutlass3xGemm;
159
182
- uint32_t const n = out.size(1);
160
bool const is_small_n = n < 8192;
184
-
185
- uint32_t const m = a.size(0);
161
uint32_t const mp2 =
162
std::max(static_cast<uint32_t>(32), next_pow_2(m)); // next power of 2
163
164
if (mp2 <= 32) {
165
// m in [1, 32]
166
if (is_small_n) {
192
- return cutlass_sparse_gemm_caller<Cutlass3xGemmM32NSmall>(
193
- out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
167
+ return DispatchFunc::template invoke<Cutlass3xGemmM32NSmall>(
168
+ std::forward<Args>(args)...);
169
} else {
195
- return cutlass_sparse_gemm_caller<Cutlass3xGemmM32NBig>(
196
- out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
170
+ return DispatchFunc::template invoke<Cutlass3xGemmM32NBig>(
171
+ std::forward<Args>(args)...);
172
}
173
} else if (mp2 <= 64) {
174
// m in (32, 64]
200
- return cutlass_sparse_gemm_caller<Cutlass3xGemmM64>(
201
- out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
175
+ return DispatchFunc::template invoke<Cutlass3xGemmM64>(
176
+ std::forward<Args>(args)...);
177
} else if (mp2 <= 128) {
178
// m in (64, 128]
204
- return cutlass_sparse_gemm_caller<Cutlass3xGemmM128>(
205
- out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
179
+ return DispatchFunc::template invoke<Cutlass3xGemmM128>(
180
+ std::forward<Args>(args)...);
181
} else {
182
// m in (128, inf)
208
- return cutlass_sparse_gemm_caller<Cutlass3xGemmDefault>(
209
- out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
183
+ return DispatchFunc::template invoke<Cutlass3xGemmDefault>(
184
+ std::forward<Args>(args)...);
185
}
186
}
187
188
+// Dispatch to GEMM implementations based on element types
189
template <template <typename, typename, typename> typename Epilogue,
190
typename... EpilogueArgs>
191
void cutlass_scaled_sparse_mm_sm90_epilogue(torch::Tensor& out,
@@ -217,19 +193,24 @@ void cutlass_scaled_sparse_mm_sm90_epilogue(torch::Tensor& out,
193
torch::Tensor const& bt_nzs,
194
torch::Tensor const& bt_meta,
195
EpilogueArgs&&... epilogue_args) {
196
+ uint32_t const m = out.size(0);
197
+ uint32_t const n = out.size(1);
198
+
199
+ // TODO: add dispatch functions to all of these
200
TORCH_CHECK(bt_meta.dtype() == torch::kUInt8);
201
if (a.dtype() == torch::kInt8) {
202
TORCH_CHECK(bt_nzs.dtype() == torch::kInt8);
203
204
if (out.dtype() == torch::kBFloat16) {
205
return cutlass_gemm_sm90_int8_dispatch<int8_t, cutlass::bfloat16_t,
226
- Epilogue>(
227
- out, a, bt_nzs, bt_meta,
206
+ Epilogue, GemmCallerTraits>(
207
+ m, n, out, a, bt_nzs, bt_meta,
208
std::forward<EpilogueArgs>(epilogue_args)...);
209
} else {
210
TORCH_CHECK(out.dtype() == torch::kFloat16);
231
- return cutlass_gemm_sm90_int8_dispatch<int8_t, cutlass::half_t, Epilogue>(
232
- out, a, bt_nzs, bt_meta,
211
+ return cutlass_gemm_sm90_int8_dispatch<int8_t, cutlass::half_t, Epilogue,
212
+ GemmCallerTraits>(
213
+ m, n, out, a, bt_nzs, bt_meta,
214
std::forward<EpilogueArgs>(epilogue_args)...);
215
}
216
} else if (a.dtype() == torch::kFloat8_e4m3fn) {
@@ -237,47 +218,34 @@ void cutlass_scaled_sparse_mm_sm90_epilogue(torch::Tensor& out,
218
219
if (out.dtype() == torch::kBFloat16) {
220
return cutlass_gemm_sm90_fp8_dispatch<cutlass::float_e4m3_t,
240
- cutlass::bfloat16_t, Epilogue>(
241
- out, a, bt_nzs, bt_meta,
221
+ cutlass::bfloat16_t, Epilogue,
222
+ GemmCallerTraits>(
223
+ m, n, out, a, bt_nzs, bt_meta,
224
std::forward<EpilogueArgs>(epilogue_args)...);
225
} else {
226
TORCH_CHECK(out.dtype() == torch::kFloat16);
245
- return cutlass_gemm_sm90_fp8_dispatch<cutlass::float_e4m3_t,
246
- cutlass::half_t, Epilogue>(
247
- out, a, bt_nzs, bt_meta,
227
+ return cutlass_gemm_sm90_fp8_dispatch<
228
+ cutlass::float_e4m3_t, cutlass::half_t, Epilogue, GemmCallerTraits>(
229
+ m, n, out, a, bt_nzs, bt_meta,
230
std::forward<EpilogueArgs>(epilogue_args)...);
231
}
232
} else if (a.dtype() == torch::kFloat16) {
233
TORCH_CHECK(bt_nzs.dtype() == torch::kFloat16);
234
+ TORCH_CHECK(out.dtype() == torch::kFloat16);
235
253
- if (out.dtype() == torch::kBFloat16) {
254
- return cutlass_gemm_sm90_fp16_dispatch<cutlass::half_t,
255
- cutlass::bfloat16_t, Epilogue>(
256
- out, a, bt_nzs, bt_meta,
257
- std::forward<EpilogueArgs>(epilogue_args)...);
258
- } else {
259
- TORCH_CHECK(out.dtype() == torch::kFloat16);
260
- return cutlass_gemm_sm90_fp16_dispatch<cutlass::half_t, cutlass::half_t,
261
- Epilogue>(
262
- out, a, bt_nzs, bt_meta,
263
- std::forward<EpilogueArgs>(epilogue_args)...);
264
- }
236
+ return cutlass_gemm_sm90_16bit_dispatch<cutlass::half_t, cutlass::half_t,
237
+ Epilogue, GemmCallerTraits>(
238
+ m, n, out, a, bt_nzs, bt_meta,
239
+ std::forward<EpilogueArgs>(epilogue_args)...);
240
} else { // a.dtype() == torch::kBFloat16
241
TORCH_CHECK(a.dtype() == torch::kBFloat16);
242
TORCH_CHECK(bt_nzs.dtype() == torch::kBFloat16);
243
+ TORCH_CHECK(out.dtype() == torch::kBFloat16);
244
269
- if (out.dtype() == torch::kBFloat16) {
270
- return cutlass_gemm_sm90_bf16_dispatch<cutlass::bfloat16_t,
271
- cutlass::bfloat16_t, Epilogue>(
272
- out, a, bt_nzs, bt_meta,
273
- std::forward<EpilogueArgs>(epilogue_args)...);
274
- } else {
275
- TORCH_CHECK(out.dtype() == torch::kFloat16);
276
- return cutlass_gemm_sm90_bf16_dispatch<cutlass::bfloat16_t,
277
- cutlass::half_t, Epilogue>(
278
- out, a, bt_nzs, bt_meta,
279
- std::forward<EpilogueArgs>(epilogue_args)...);
280
- }
245
+ return cutlass_gemm_sm90_16bit_dispatch<
246
+ cutlass::bfloat16_t, cutlass::bfloat16_t, Epilogue, GemmCallerTraits>(
247
+ m, n, out, a, bt_nzs, bt_meta,
248
+ std::forward<EpilogueArgs>(epilogue_args)...);
249
}
250
}
251
@@ -287,17 +255,53 @@ void cutlass_scaled_sparse_mm_sm90(torch::Tensor& out, torch::Tensor const& a,
255
torch::Tensor const& a_scales,
256
torch::Tensor const& b_scales,
257
std::optional<torch::Tensor> const& bias) {
258
+ TORCH_CHECK(bt_meta.dtype() == torch::kUInt8);
259
TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
260
TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
261
+
262
if (bias) {
263
TORCH_CHECK(bias->dtype() == out.dtype(),
294
- "currently bias dtype must match output dtype ", out.dtype());
295
- return cutlass_scaled_sparse_mm_sm90_epilogue<c3x::ScaledEpilogueBias>(
296
- out, a, bt_nzs, bt_meta, b_scales, a_scales, *bias);
264
+ "CUTLASS scaled_mm bias dtype must match output dtype ",
265
+ out.dtype());
266
+ return cutlass_scaled_sparse_mm_sm90_epilogue<
267
+ c3x::ScaledEpilogueColumnBias>(out, a, bt_nzs, bt_meta, b_scales,
268
+ a_scales, *bias);
269
} else {
270
return cutlass_scaled_sparse_mm_sm90_epilogue<c3x::ScaledEpilogue>(
271
out, a, bt_nzs, bt_meta, b_scales, a_scales);
272
}
273
}
274
275
+CompressorResult cutlass_sparse_compress_sm90(torch::Tensor const& a) {
276
+ // These m and n variables are fordispatching to different GEMM algorithms.
277
+ uint32_t const m = 1; // Set M to 1 for compression
278
+ uint32_t const n = a.size(1);
279
+
280
+ // Note: For correctess, the compressed format must be invariant in:
281
+ // - M, the flattened number of tokens
282
+ // - Whether output dtype is fp16 or bf16
283
+ // - CUTLASS epilogues
284
+
285
+ if (a.dtype() == torch::kInt8) {
286
+ return cutlass_gemm_sm90_int8_dispatch<int8_t, cutlass::bfloat16_t,
287
+ c3x::TrivialEpilogue,
288
+ GemmCompressorTraits>(m, n, a);
289
+ } else if (a.dtype() == torch::kFloat8_e4m3fn) {
290
+ return cutlass_gemm_sm90_fp8_dispatch<
291
+ cutlass::float_e4m3_t, cutlass::bfloat16_t, c3x::TrivialEpilogue,
292
+ GemmCompressorTraits>(m, n, a);
293
+ } else if (a.dtype() == torch::kFloat16) {
294
+ return cutlass_gemm_sm90_16bit_dispatch<
295
+ cutlass::bfloat16_t, cutlass::bfloat16_t, c3x::TrivialEpilogue,
296
+ GemmCompressorTraits>(m, n, a);
297
+ } else {
298
+ TORCH_CHECK(a.dtype() == torch::kBFloat16,
299
+ "cutlass_sparse_compress only supports int8, fp8_e4m3, fp16, "
300
+ "and bf16 datatypes");
301
+ return cutlass_gemm_sm90_16bit_dispatch<cutlass::half_t, cutlass::half_t,
302
+ c3x::TrivialEpilogue,
303
+ GemmCompressorTraits>(m, n, a);
304
+ }
305
+}
306
+
307
#endif
csrc/sparse/cutlass/sparse_scaled_mm_c3x.cuh
+154
-78
@@ -1,3 +1,5 @@
1
+#pragma once
2
+
3
// clang-format will break include orders
4
// clang-format off
5
#include <cudaTypedefs.h>
@@ -12,6 +14,9 @@
14
#include "cutlass/epilogue/collective/collective_builder.hpp"
15
#include "cutlass/gemm/collective/collective_builder.hpp"
16
17
+#include "cutlass/transform/device/transform_universal_adapter.hpp"
18
+#include "cutlass/transform/kernel/sparse_gemm_compressor.hpp"
19
+
20
#include "core/math.hpp"
21
#include "cutlass_extensions/cute_utils.cuh"
22
#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
@@ -22,7 +27,7 @@
27
using namespace cute;
28
29
/*
25
- This file defines sparse quantized GEMM operations using the CUTLASS 3.x API,
30
+ This file defines 2:4 sparse GEMM operations using the CUTLASS 3.x API,
31
for NVIDIA GPUs with sm90a (Hopper) or later.
32
*/
33
@@ -45,17 +50,20 @@ struct enable_sm90_or_later : Kernel {
50
51
using GemmUniversalMode = cutlass::gemm::GemmUniversalMode;
52
53
+/*
54
+ * cutlass_sparse_3x_gemm defines a 2:4 sparse GEMM kernel via CUTLASS
55
+ * for SM90 Hopper systems.
56
+ */
57
template <typename ElementAB_, typename ElementD_,
58
template <typename, typename, typename> typename Epilogue_,
59
typename TileShape, typename ClusterShape, typename KernelSchedule,
51
- typename EpilogueSchedule, typename AccType,
52
- typename TileSchedule = cutlass::gemm::PersistentScheduler,
53
- GemmUniversalMode Mode_ = GemmUniversalMode::kGemm>
60
+ typename EpilogueSchedule>
61
struct cutlass_sparse_3x_gemm {
55
- static const GemmUniversalMode Mode = Mode_;
62
using ElementAB = ElementAB_;
63
using ElementD = ElementD_;
58
- using ElementAcc = AccType;
64
+ using ElementAcc =
65
+ typename std::conditional<std::is_same_v<ElementAB, int8_t>, int32_t,
66
+ float>::type;
67
68
using EpilogueDescriptor =
69
cutlass::epilogue::collective::detail::EpilogueDescriptor<
@@ -66,30 +74,22 @@ struct cutlass_sparse_3x_gemm {
74
75
using ElementC = void;
76
using LayoutC = cutlass::layout::RowMajor;
69
- using LayoutD = LayoutC;
70
- using StrideC = cutlass::detail::TagToStrideA_t<LayoutC>;
71
- using StrideD = cutlass::detail::TagToStrideA_t<LayoutD>;
72
-
77
using LayoutC_Transpose =
78
typename cutlass::layout::LayoutTranspose<LayoutC>::type;
75
- using LayoutD_Transpose =
76
- typename cutlass::layout::LayoutTranspose<LayoutD>::type;
79
80
using EVTCompute = typename Epilogue::EVTCompute;
81
80
- static constexpr int AlignmentA =
81
- 128 / cutlass::sizeof_bits<ElementAB>::value;
82
- static constexpr int AlignmentB =
82
+ // These are the minimum alignments needed for the kernels to compile
83
+ static constexpr int AlignmentAB =
84
128 / cutlass::sizeof_bits<ElementAB>::value;
84
- static constexpr int AlignmentCD =
85
- 128 / cutlass::sizeof_bits<ElementD>::value;
85
+ static constexpr int AlignmentCD = 4;
86
87
using CollectiveEpilogue =
88
typename cutlass::epilogue::collective::CollectiveBuilder<
89
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp, TileShape,
90
ClusterShape, cutlass::epilogue::collective::EpilogueTileAuto,
91
- ElementAcc, ElementAcc, ElementC, LayoutC_Transpose, AlignmentCD,
92
- ElementD, LayoutD_Transpose, AlignmentCD, EpilogueSchedule,
91
+ ElementAcc, float, ElementC, LayoutC_Transpose, AlignmentCD, ElementD,
92
+ LayoutC_Transpose, AlignmentCD, EpilogueSchedule,
93
EVTCompute>::CollectiveOp;
94
95
static constexpr size_t CEStorageSize =
@@ -101,8 +101,8 @@ struct cutlass_sparse_3x_gemm {
101
using CollectiveMainloop =
102
typename cutlass::gemm::collective::CollectiveBuilder<
103
cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
104
- ElementAB, cutlass::layout::RowMajor, AlignmentA,
105
- ElementAB, cutlass::layout::ColumnMajor, AlignmentB,
104
+ ElementAB, cutlass::layout::RowMajor, AlignmentAB,
105
+ ElementAB, cutlass::layout::ColumnMajor, AlignmentAB,
106
ElementAcc, TileShape, ClusterShape,
107
Stages,
108
KernelSchedule>::CollectiveOp;
@@ -110,11 +110,100 @@ struct cutlass_sparse_3x_gemm {
110
111
using KernelType = enable_sm90_or_later<cutlass::gemm::kernel::GemmUniversal<
112
cute::Shape<int, int, int, int>, CollectiveMainloop, CollectiveEpilogue,
113
- TileSchedule>>;
113
+ cutlass::gemm::PersistentScheduler>>;
114
115
struct GemmKernel : public KernelType {};
116
+
117
+ // Sparse compressor definitions
118
+ using SparseConfig = typename GemmKernel::CollectiveMainloop::SparseConfig;
119
+ using LayoutTagA = cutlass::layout::RowMajor;
120
+ using CompressorUtility =
121
+ cutlass::transform::kernel::StructuredSparseCompressorUtility<
122
+ typename GemmKernel::ProblemShape, ElementAB, LayoutTagA,
123
+ SparseConfig>;
124
+ using CompressorKernel =
125
+ cutlass::transform::kernel::StructuredSparseCompressor<
126
+ typename GemmKernel::ProblemShape, ElementAB, LayoutTagA,
127
+ SparseConfig, cutlass::arch::Sm90>;
128
+ using Compressor =
129
+ cutlass::transform::device::TransformUniversalAdapter<CompressorKernel>;
130
};
131
132
+/*
133
+ * This class defines kernel to compress a 2:4 sparse matrix.
134
+ * The particular format is defined by the Gemm template parameter,
135
+ * which is a cutlass_sparse_3x_gemm.
136
+ */
137
+using CompressorResult = std::tuple<torch::Tensor, torch::Tensor>;
138
+/// Make A structured sparse by replacing elements with 0 and compress it
139
+template <typename Gemm>
140
+CompressorResult cutlass_sparse_compress(torch::Tensor const& a) {
141
+ // Checks for conformality
142
+ TORCH_CHECK(a.dtype() == torch::kInt8 || a.dtype() == torch::kFloat8_e4m3fn ||
143
+ a.dtype() == torch::kFloat16 || a.dtype() == torch::kBFloat16);
144
+ TORCH_CHECK(a.dim() == 2)
145
+ // Check for strides and alignment
146
+ TORCH_CHECK(a.stride(0) % 4 == 0) // Required for semi-structured sparsity
147
+ TORCH_CHECK(a.stride(1) == 1)
148
+
149
+ using GemmKernel = typename Gemm::KernelType;
150
+ using ElementA = typename Gemm::ElementAB;
151
+ using ElementE = typename GemmKernel::CollectiveMainloop::ElementE;
152
+
153
+ int m = a.size(0);
154
+ int k = a.size(1);
155
+ using ProblemShape = typename GemmKernel::ProblemShape;
156
+ ProblemShape prob_shape{m, 1, k, 1};
157
+
158
+ int64_t lda = a.stride(0);
159
+ using StrideA = Stride<int64_t, Int<1>, int64_t>;
160
+ StrideA a_stride{lda, Int<1>{}, 0};
161
+
162
+ using CompressorUtility = typename Gemm::CompressorUtility;
163
+ CompressorUtility compressor_utility(prob_shape, a_stride);
164
+
165
+ // Allocate buffers for the metadata E and the compressed matrix A
166
+ int ME = compressor_utility.get_metadata_m_physical();
167
+ int KE = compressor_utility.get_metadata_k_physical();
168
+ int MC = compressor_utility.get_tensorA_m_physical();
169
+ int KC = compressor_utility.get_tensorA_k_physical();
170
+
171
+ auto const a_meta_options =
172
+ torch::TensorOptions().dtype(torch::kUInt8).device(a.device());
173
+ auto const a_nzs_options =
174
+ torch::TensorOptions().dtype(a.dtype()).device(a.device());
175
+
176
+ auto a_meta = torch::zeros({ME, KE}, a_meta_options);
177
+ auto a_nzs = torch::zeros({MC, KC}, a_nzs_options);
178
+
179
+ auto a_ptr = static_cast<ElementA*>(a.data_ptr());
180
+ auto a_nzs_ptr = static_cast<ElementA*>(a_nzs.data_ptr());
181
+ auto a_meta_ptr = static_cast<ElementE*>(a_meta.data_ptr());
182
+
183
+ cutlass::KernelHardwareInfo hw_info;
184
+ hw_info.device_id = a.device().index();
185
+ hw_info.sm_count =
186
+ cutlass::KernelHardwareInfo::query_device_multiprocessor_count(
187
+ hw_info.device_id);
188
+
189
+ using Compressor = typename Gemm::Compressor;
190
+ typename Compressor::Arguments arguments{
191
+ prob_shape, {a_ptr, a_stride, a_nzs_ptr, a_meta_ptr}, {hw_info}};
192
+
193
+ Compressor compressor_op;
194
+ size_t workspace_size = Compressor::get_workspace_size(arguments);
195
+ auto const workspace_options =
196
+ torch::TensorOptions().dtype(torch::kUInt8).device(a.device());
197
+ auto workspace = torch::empty(workspace_size, workspace_options);
198
+
199
+ CUTLASS_CHECK(compressor_op.can_implement(arguments));
200
+ CUTLASS_CHECK(compressor_op.initialize(arguments, workspace.data_ptr()));
201
+ CUTLASS_CHECK(compressor_op.run());
202
+ CUDA_CHECK(cudaDeviceSynchronize());
203
+
204
+ return {a_meta, a_nzs};
205
+}
206
+
207
template <typename Gemm, typename... EpilogueArgs>
208
void cutlass_sparse_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
209
torch::Tensor const& bt_nzs,
@@ -126,27 +215,25 @@ void cutlass_sparse_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
215
// Interface stride expected from the argument a (will get transposed)
216
// We compute C^T = B^T * A^T, but we assume B is transposed before
217
// compression and hence the bt_* naming
129
- using LayoutA = cutlass::layout::RowMajor;
218
using LayoutB = typename Gemm::GemmKernel::CollectiveMainloop::LayoutA;
219
using LayoutE = typename Gemm::GemmKernel::CollectiveMainloop::LayoutE;
132
- using LayoutD = cutlass::layout::RowMajor;
220
134
- using StrideA = cutlass::detail::TagToStrideA_t<LayoutA>;
135
- using StrideD = cutlass::detail::TagToStrideA_t<LayoutD>;
221
+ // M, N, K after transposition
222
+ int32_t m = out.size(1);
223
+ int32_t n = out.size(0);
224
+ int32_t k = a.size(1);
225
+
226
+ int64_t lda = a.stride(0);
227
+ int64_t ldc = out.stride(0);
228
137
- auto layout_A = make_cute_layout<StrideA>(a, "A");
138
- auto layout_D = make_cute_layout<StrideD>(out, "D");
229
+ using StrideA = Stride<int64_t, Int<1>, int64_t>;
230
+ using StrideC = Stride<Int<1>, int64_t, int64_t>;
231
140
- // Transpose A and D
141
- // A doesn't need to be transposed since cutlass expects a NxK matrix
142
- // for B (which is At)
143
- auto stride_At = layout_A.stride();
144
- auto stride_Dt = permute_layout<1, 0, 2>(layout_D).stride();
232
+ StrideA a_stride{lda, Int<1>{}, Int<0>{}};
233
+ StrideC c_stride{Int<1>{}, ldc, Int<0>{}};
234
235
using GemmKernel = typename Gemm::GemmKernel;
147
- typename GemmKernel::ProblemShape prob_shape{
148
- static_cast<int>(bt_nzs.size(0)), static_cast<int>(size<0>(layout_A)),
149
- static_cast<int>(size<1>(layout_A)), 1};
236
+ typename GemmKernel::ProblemShape prob_shape{m, n, k, 1};
237
238
using ElementE = typename GemmKernel::CollectiveMainloop::ElementE;
239
using SparseConfig = typename GemmKernel::CollectiveMainloop::SparseConfig;
@@ -158,13 +245,13 @@ void cutlass_sparse_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
245
auto b_ptr = static_cast<ElementAB*>(bt_nzs.data_ptr());
246
auto e_ptr = static_cast<ElementE*>(bt_meta.data_ptr());
247
typename GemmKernel::MainloopArguments mainloop_args{
161
- b_ptr, b_layout, a_ptr, stride_At, e_ptr, e_layout};
248
+ b_ptr, b_layout, a_ptr, a_stride, e_ptr, e_layout};
249
250
auto c_ptr = static_cast<ElementD*>(out.data_ptr());
251
typename GemmKernel::EpilogueArguments epilogue_args{
252
Gemm::Epilogue::prepare_args(
253
std::forward<EpilogueArgs>(epilogue_params)...),
167
- c_ptr, stride_Dt, c_ptr, stride_Dt};
254
+ c_ptr, c_stride, c_ptr, c_stride};
255
256
typename GemmKernel::Arguments args{cutlass::gemm::GemmUniversalMode::kGemm,
257
prob_shape, mainloop_args, epilogue_args};
@@ -185,6 +272,10 @@ void cutlass_sparse_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
272
CUTLASS_CHECK(status);
273
}
274
275
+//////////////////////////////////////////////////
276
+// Gemm Configs are defined below
277
+//////////////////////////////////////////////////
278
+
279
template <typename InType, typename OutType,
280
template <typename, typename, typename> typename Epilogue>
281
struct sm90_config_default {};
@@ -192,28 +283,25 @@ struct sm90_config_default {};
283
template <typename OutType,
284
template <typename, typename, typename> typename Epilogue>
285
struct sm90_config_default<half_t, OutType, Epilogue> {
195
- // M in (128, inf)
196
- using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecializedPingpong;
286
+ using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecialized;
287
using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
288
using TileShape = Shape<_128, _128, _128>;
199
- using ClusterShape = Shape<_2, _1, _1>;
289
+ using ClusterShape = Shape<_1, _1, _1>;
290
using Cutlass3xGemm =
291
cutlass_sparse_3x_gemm<half_t, OutType, Epilogue, TileShape, ClusterShape,
202
- KernelSchedule, EpilogueSchedule, float>;
292
+ KernelSchedule, EpilogueSchedule>;
293
};
294
295
template <typename OutType,
296
template <typename, typename, typename> typename Epilogue>
297
struct sm90_config_default<cutlass::bfloat16_t, OutType, Epilogue> {
208
- // M in (128, inf)
209
- using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecializedPingpong;
298
+ using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecialized;
299
using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
300
using TileShape = Shape<_128, _128, _128>;
212
- using ClusterShape = Shape<_2, _1, _1>;
301
+ using ClusterShape = Shape<_1, _1, _1>;
302
using Cutlass3xGemm =
303
cutlass_sparse_3x_gemm<cutlass::bfloat16_t, OutType, Epilogue, TileShape,
215
- ClusterShape, KernelSchedule, EpilogueSchedule,
216
- float>;
304
+ ClusterShape, KernelSchedule, EpilogueSchedule>;
305
};
306
307
//////////////////////// Cherry-Picking Kernels ////////////////////////
@@ -227,7 +315,7 @@ struct sm90_fp8_config_1 {
315
using ClusterShape = Shape<_8, _1, _1>;
316
using Cutlass3xGemm =
317
cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
230
- KernelSchedule, EpilogueSchedule, float>;
318
+ KernelSchedule, EpilogueSchedule>;
319
};
320
321
template <typename InType, typename OutType,
@@ -242,7 +330,7 @@ struct sm90_fp8_config_2 {
330
using ClusterShape = Shape<_8, _1, _1>;
331
using Cutlass3xGemm =
332
cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
245
- KernelSchedule, EpilogueSchedule, float>;
333
+ KernelSchedule, EpilogueSchedule>;
334
};
335
336
template <typename InType, typename OutType,
@@ -255,7 +343,7 @@ struct sm90_fp8_config_3 {
343
using ClusterShape = Shape<_1, _2, _1>;
344
using Cutlass3xGemm =
345
cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
258
- KernelSchedule, EpilogueSchedule, float>;
346
+ KernelSchedule, EpilogueSchedule>;
347
};
348
349
template <typename InType, typename OutType,
@@ -269,7 +357,7 @@ struct sm90_fp8_config_4 {
357
using ClusterShape = Shape<_8, _1, _1>;
358
using Cutlass3xGemm =
359
cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
272
- KernelSchedule, EpilogueSchedule, float>;
360
+ KernelSchedule, EpilogueSchedule>;
361
};
362
363
template <typename InType, typename OutType,
@@ -283,7 +371,7 @@ struct sm90_fp8_config_5 {
371
using ClusterShape = Shape<_8, _1, _1>;
372
using Cutlass3xGemm =
373
cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
286
- KernelSchedule, EpilogueSchedule, float>;
374
+ KernelSchedule, EpilogueSchedule>;
375
};
376
377
template <typename InType, typename OutType,
@@ -296,7 +384,7 @@ struct sm90_fp8_config_6 {
384
using ClusterShape = Shape<_1, _2, _1>;
385
using Cutlass3xGemm =
386
cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
299
- KernelSchedule, EpilogueSchedule, float>;
387
+ KernelSchedule, EpilogueSchedule>;
388
};
389
390
template <typename InType, typename OutType,
@@ -311,7 +399,7 @@ struct sm90_fp8_config_7 {
399
using ClusterShape = Shape<_1, _1, _1>;
400
using Cutlass3xGemm =
401
cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
314
- KernelSchedule, EpilogueSchedule, float>;
402
+ KernelSchedule, EpilogueSchedule>;
403
};
404
405
template <typename InType, typename OutType,
@@ -326,7 +414,7 @@ struct sm90_fp8_config_8 {
414
using ClusterShape = Shape<_8, _1, _1>;
415
using Cutlass3xGemm =
416
cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
329
- KernelSchedule, EpilogueSchedule, float>;
417
+ KernelSchedule, EpilogueSchedule>;
418
};
419
////////////////////////////////////////////////////////////////////////
420
@@ -341,7 +429,7 @@ struct sm90_config_default<cutlass::float_e4m3_t, OutType, Epilogue> {
429
using Cutlass3xGemm =
430
cutlass_sparse_3x_gemm<cutlass::float_e4m3_t, OutType, Epilogue,
431
TileShape, ClusterShape, KernelSchedule,
344
- EpilogueSchedule, float>;
432
+ EpilogueSchedule>;
433
};
434
435
template <typename InType, typename OutType,
@@ -355,12 +443,9 @@ struct sm90_fp8_config_M64 {
443
using TileShape = Shape<_64, _64, _256>;
444
using ClusterShape = Shape<_1, _1, _1>;
445
358
- using TileSchedule = cutlass::gemm::PersistentScheduler;
359
-
446
using Cutlass3xGemm =
447
cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
362
- KernelSchedule, EpilogueSchedule, float,
363
- TileSchedule>;
448
+ KernelSchedule, EpilogueSchedule>;
449
};
450
451
template <typename InType, typename OutType,
@@ -374,12 +459,9 @@ struct sm90_fp8_config_M128 {
459
using TileShape = Shape<_64, _128, _256>;
460
using ClusterShape = Shape<_1, _1, _1>;
461
377
- using TileSchedule = cutlass::gemm::PersistentScheduler;
378
-
462
using Cutlass3xGemm =
463
cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
381
- KernelSchedule, EpilogueSchedule, float,
382
- TileSchedule>;
464
+ KernelSchedule, EpilogueSchedule>;
465
};
466
467
template <typename InType, typename OutType,
@@ -394,12 +476,9 @@ struct sm90_fp8_config_M256 {
476
using TileShape = Shape<_128, _128, _256>;
477
using ClusterShape = Shape<_1, _1, _1>;
478
397
- using TileSchedule = cutlass::gemm::PersistentScheduler;
398
-
479
using Cutlass3xGemm =
480
cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
401
- KernelSchedule, EpilogueSchedule, float,
402
- TileSchedule>;
481
+ KernelSchedule, EpilogueSchedule>;
482
};
483
484
template <typename InType, typename OutType,
@@ -414,12 +493,9 @@ struct sm90_fp8_config_M512 {
493
using TileShape = Shape<_128, _128, _256>;
494
using ClusterShape = Shape<_1, _1, _1>;
495
417
- using TileSchedule = cutlass::gemm::PersistentScheduler;
418
-
496
using Cutlass3xGemm =
497
cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
421
- KernelSchedule, EpilogueSchedule, float,
422
- TileSchedule>;
498
+ KernelSchedule, EpilogueSchedule>;
499
};
500
501
template <typename OutType,
@@ -433,7 +509,7 @@ struct sm90_config_default<int8_t, OutType, Epilogue> {
509
using ClusterShape = Shape<_2, _1, _1>;
510
using Cutlass3xGemm =
511
cutlass_sparse_3x_gemm<int8_t, OutType, Epilogue, TileShape, ClusterShape,
436
- KernelSchedule, EpilogueSchedule, int32_t>;
512
+ KernelSchedule, EpilogueSchedule>;
513
};
514
515
template <typename InType, typename OutType,
@@ -448,7 +524,7 @@ struct sm90_int8_config_M128 {
524
using ClusterShape = Shape<_2, _1, _1>;
525
using Cutlass3xGemm =
526
cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
451
- KernelSchedule, EpilogueSchedule, int32_t>;
527
+ KernelSchedule, EpilogueSchedule>;
528
};
529
530
template <typename InType, typename OutType,
@@ -462,7 +538,7 @@ struct sm90_int8_config_M64 {
538
using ClusterShape = Shape<_1, _1, _1>;
539
using Cutlass3xGemm =
540
cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
465
- KernelSchedule, EpilogueSchedule, int32_t>;
541
+ KernelSchedule, EpilogueSchedule>;
542
};
543
544
template <typename InType, typename OutType,
@@ -476,7 +552,7 @@ struct sm90_int8_config_M32_NBig {
552
using ClusterShape = Shape<_1, _4, _1>;
553
using Cutlass3xGemm =
554
cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
479
- KernelSchedule, EpilogueSchedule, int32_t>;
555
+ KernelSchedule, EpilogueSchedule>;
556
};
557
558
template <typename InType, typename OutType,
@@ -490,7 +566,7 @@ struct sm90_int8_config_M32_NSmall {
566
using ClusterShape = Shape<_1, _8, _1>;
567
using Cutlass3xGemm =
568
cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
493
- KernelSchedule, EpilogueSchedule, int32_t>;
569
+ KernelSchedule, EpilogueSchedule>;
570
};
571
496
-} // namespace
\ No newline at end of file
572
+} // namespace
csrc/sparse/cutlass/sparse_scaled_mm_entry.cu
+30
@@ -23,6 +23,9 @@ void cutlass_scaled_sparse_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
23
torch::Tensor const& a_scales,
24
torch::Tensor const& b_scales,
25
std::optional<torch::Tensor> const& bias);
26
+
27
+using CompressorResult = std::tuple<torch::Tensor, torch::Tensor>;
28
+CompressorResult cutlass_sparse_compress_sm90(torch::Tensor const& a);
29
#endif
30
31
void cutlass_scaled_sparse_mm(torch::Tensor& c, torch::Tensor const& a,
@@ -68,3 +71,30 @@ void cutlass_scaled_sparse_mm(torch::Tensor& c, torch::Tensor const& a,
71
"CUDA device capability: ",
72
version_num);
73
}
74
+
75
+std::vector<torch::Tensor> cutlass_sparse_compress(torch::Tensor const& a) {
76
+ // Check for strides and alignment
77
+ TORCH_CHECK(a.stride(1) == 1); // Row-major
78
+ TORCH_CHECK(a.stride(0) % 8 == 0); // 8 Byte Alignment for Compression
79
+
80
+ at::cuda::OptionalCUDAGuard const device_guard(device_of(a));
81
+ int32_t version_num = get_sm_version_num();
82
+
83
+ // Guard against compilation issues for sm90 kernels
84
+#if defined ENABLE_SPARSE_SCALED_MM_C3X && ENABLE_SPARSE_SCALED_MM_C3X
85
+ if (version_num >= 90) {
86
+ std::vector<torch::Tensor> result_tensors;
87
+
88
+ auto [a_meta, a_nzs] = cutlass_sparse_compress_sm90(a);
89
+ result_tensors.push_back(std::move(a_nzs));
90
+ result_tensors.push_back(std::move(a_meta));
91
+ return result_tensors;
92
+ }
93
+#endif
94
+
95
+ TORCH_CHECK_NOT_IMPLEMENTED(
96
+ false,
97
+ "No compiled cutlass_sparse_compress for a compute capability less than "
98
+ "CUDA device capability: ",
99
+ version_num);
100
+}
csrc/torch_bindings.cpp
+2
-4
@@ -348,10 +348,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
348
ops.impl("cutlass_scaled_sparse_mm", torch::kCUDA, &cutlass_scaled_sparse_mm);
349
350
// CUTLASS sparse matrix compressor
351
- ops.def(
352
- "cutlass_sparse_compress_entry(Tensor! a_nzs, Tensor! a_meta,"
353
- " Tensor a) -> bool");
354
- ops.impl("cutlass_sparse_compress_entry", &cutlass_sparse_compress_entry);
351
+ ops.def("cutlass_sparse_compress(Tensor a) -> Tensor[]");
352
+ ops.impl("cutlass_sparse_compress", &cutlass_sparse_compress);
353
354
// Mamba selective scan kernel
355
ops.def(
tests/kernels/test_cutlass_2of4_sparse.py
+66
-15
@@ -7,7 +7,6 @@ from typing import Tuple, Type
7
8
import pytest
9
import torch
10
-import torch.nn.functional as F
10
11
from vllm import _custom_ops as ops
12
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
@@ -55,11 +54,39 @@ def prune_to_2_4(tensor):
54
return pruned.reshape(original_shape)
55
56
57
+# This function checks that applying an identity matrix multiplication
58
+# to the compressed weights yields the original uncompressed weights.
59
+def check_compress_decompress_invariance(dtype: torch.dtype, b: torch.Tensor,
60
+ b_compressed: torch.Tensor,
61
+ b_metadata: torch.Tensor):
62
+
63
+ # For float16 and bfloat16, cutlass_scaled_sparse_mm's output must be the
64
+ # same dtype as its inputs. This line addresses that constraint while
65
+ # arbitrarily using bfloat16 for the int8/fp8 cases.
66
+ out_dtype = torch.float16 if dtype is torch.float16 else torch.bfloat16
67
+
68
+ eye = torch.eye(b.shape[0], device='cuda', dtype=dtype)
69
+ eye_scale = torch.ones(1, device='cuda', dtype=torch.float32)
70
+ b_decomp = ops.cutlass_scaled_sparse_mm(eye,
71
+ b_compressed,
72
+ b_metadata,
73
+ eye_scale,
74
+ eye_scale,
75
+ out_dtype=out_dtype)
76
+
77
+ torch.testing.assert_close(b.to(dtype=out_dtype), b_decomp)
78
+
79
+
80
def make_rand_sparse_tensors(
81
dtype: torch.dtype, m: int, n: int, k: int
82
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
61
- a = torch.randn((m, k), device='cuda') * 5
62
- b = torch.randn((n, k), device='cuda').t() * 5
83
+ a = torch.randn((m, k), device='cuda')
84
+ b = torch.randn((n, k), device='cuda').t()
85
+
86
+ if dtype == torch.int8:
87
+ # ensure A and B aren't all zeros after rounding
88
+ a = a * 5.0
89
+ b = b * 5.0
90
91
b = prune_to_2_4(b.t()).t()
92
@@ -75,6 +102,7 @@ def make_rand_sparse_tensors(
102
raise ValueError("unsupported dtype")
103
104
b_compressed, e = ops.cutlass_sparse_compress(b.t())
105
+ check_compress_decompress_invariance(dtype, b, b_compressed, e)
106
107
# Compressed B, Metadata, Original A, B
108
return b_compressed, e, a, b
@@ -134,27 +162,37 @@ MNK_FACTORS = [
162
163
164
# Test working with a subset of A and B for sparse matmul
137
-@pytest.mark.skip(reason="2of4 sparse w16a16 CUTLASS produces bad output.")
165
@pytest.mark.skipif(not sparse_cutlass_supported(),
166
reason="Sparse CUTLASS is not supported on this GPU type.")
140
-@pytest.mark.parametrize("m, k, n", MNK_FACTORS)
167
+@pytest.mark.parametrize("m, n, k", MNK_FACTORS)
168
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
142
-def test_cutlass_sparse_gemm(m: int, k: int, n: int, dtype: Type[torch.dtype]):
169
+@pytest.mark.parametrize("use_bias", [True, False])
170
+def test_cutlass_sparse_gemm(m: int, k: int, n: int, dtype: Type[torch.dtype],
171
+ use_bias: bool):
172
173
# Create tensors
174
b_comp, e, a, b = make_rand_sparse_tensors(dtype, m, n, k)
175
scale_a = torch.ones((1, 1), device="cuda", dtype=torch.float32)
176
scale_b = torch.ones((1, 1), device="cuda", dtype=torch.float32)
177
178
+ bias = torch.rand((n, ), device="cuda", dtype=dtype) if use_bias else None
179
+
180
out = ops.cutlass_scaled_sparse_mm(a,
181
b_comp,
182
e,
183
scale_a,
184
scale_b,
154
- out_dtype=dtype)
155
- baseline = F.linear(a, b.T)
185
+ out_dtype=dtype,
186
+ bias=bias)
187
157
- torch.testing.assert_close(out, baseline, rtol=1e-2, atol=1e-2)
188
+ baseline = baseline_scaled_mm(a,
189
+ b,
190
+ scale_a,
191
+ scale_b,
192
+ out_dtype=dtype,
193
+ bias=bias)
194
+
195
+ torch.testing.assert_close(out, baseline, rtol=1e-2, atol=3e-1)
196
197
198
@pytest.mark.skipif(not sparse_cutlass_supported(),
@@ -162,27 +200,34 @@ def test_cutlass_sparse_gemm(m: int, k: int, n: int, dtype: Type[torch.dtype]):
200
@pytest.mark.parametrize("m, k, n", MNK_FACTORS)
201
@pytest.mark.skipif(not current_platform.has_device_capability(89),
202
reason="FP8 is not supported on this GPU type.")
165
-def test_cutlass_sparse_fp8_gemm(m: int, n: int, k: int):
203
+@pytest.mark.parametrize("use_bias", [True, False])
204
+def test_cutlass_sparse_fp8_gemm(m: int, n: int, k: int, use_bias: bool):
205
206
# Create tensors
207
b_comp, e, a, b = make_rand_sparse_tensors(torch.float8_e4m3fn, m, n, k)
208
scale_a = (torch.randn((1, 1), device="cuda", dtype=torch.float32))
209
scale_b = (torch.randn((1, 1), device="cuda", dtype=torch.float32))
210
+ out_dtype = torch.bfloat16
211
+
212
+ bias = torch.rand(
213
+ (n, ), device="cuda", dtype=out_dtype) * 10 if use_bias else None
214
215
out = ops.cutlass_scaled_sparse_mm(a,
216
b_comp,
217
e,
218
scale_a,
219
scale_b,
177
- out_dtype=torch.bfloat16)
220
+ out_dtype=out_dtype,
221
+ bias=bias)
222
223
baseline = baseline_scaled_mm(a,
224
b,
225
scale_a,
226
scale_b,
183
- out_dtype=torch.bfloat16)
227
+ out_dtype=out_dtype,
228
+ bias=bias)
229
185
- torch.testing.assert_close(out, baseline, rtol=1e0, atol=2e0)
230
+ torch.testing.assert_close(out, baseline, rtol=1e-2, atol=3e-1)
231
232
233
@pytest.mark.skipif(not sparse_cutlass_supported(),
@@ -198,18 +243,24 @@ def test_cutlass_sparse_int8_gemm(m: int, n: int, k: int, per_act_token: bool,
243
b_comp, e, a, b = make_rand_sparse_tensors(torch.int8, m, n, k)
244
scale_a = (torch.randn((1, 1), device="cuda", dtype=torch.float32))
245
scale_b = (torch.randn((1, 1), device="cuda", dtype=torch.float32))
246
+ out_dtype = torch.bfloat16
247
+
248
+ bias = torch.rand(
249
+ (n, ), device="cuda", dtype=out_dtype) * 10 if use_bias else None
250
251
out = ops.cutlass_scaled_sparse_mm(a,
252
b_comp,
253
e,
254
scale_a,
255
scale_b,
207
- out_dtype=torch.bfloat16)
256
+ out_dtype=out_dtype,
257
+ bias=bias)
258
259
baseline = baseline_scaled_mm(a,
260
b,
261
scale_a,
262
scale_b,
213
- out_dtype=torch.bfloat16)
263
+ out_dtype=out_dtype,
264
+ bias=bias)
265
266
torch.testing.assert_close(out, baseline, rtol=1e0, atol=2e0)
vllm/_custom_ops.py
+2
-15
@@ -564,22 +564,9 @@ def cutlass_sparse_compress(a: torch.Tensor) \
564
565
# a_meta.dtype: torch.uint8 so elemsPerMetaElem = 8b / 2b_per_nz = 4
566
elemsPerMetaElem = 4
567
+ assert (a.shape[1] % (2 * elemsPerMetaElem) == 0)
568
568
- m = a.shape[0]
569
- k = a.shape[1]
570
- assert (k % 2 == 0)
571
- a_nzs = torch.empty((m, k // 2), dtype=a.dtype, device=a.device)
572
- a_meta = torch.empty((m, k // 2 // elemsPerMetaElem),
573
- dtype=torch.uint8,
574
- device=a.device)
575
-
576
- if not (torch.ops._C.cutlass_sparse_compress_entry(a_nzs, a_meta, a)):
577
- raise ValueError
578
-
579
- assert (a_nzs.is_contiguous())
580
- assert (a_meta.is_contiguous())
581
-
582
- return a_nzs, a_meta
569
+ return torch.ops._C.cutlass_sparse_compress(a)
570
571
572
def cutlass_scaled_sparse_mm(
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
-7
@@ -408,13 +408,6 @@ class CompressedTensorsConfig(QuantizationConfig):
408
if self.supports_cutlass_24(weight_quant=weight_quant,
409
input_quant=input_quant,
410
sparsity_scheme=sparsity_scheme):
411
- # FIXME(tlrmchlsmth): layers using W16A16 CUTLASS 2:4 sparse kernels
412
- # currently produce bad output in some cases
413
- if weight_quant is None:
414
- logger.warning_once(
415
- "CompressedTensors24 scheme is disabled for the w16a16 "
416
- "case. Falling back to UnquantizedLinearMethod")
417
- return None
411
# Have a valid sparsity scheme
412
# Validate layer is supported by Cutlass 2:4 Kernel
413
model_compression_config = (None if sparsity_scheme is None
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_24.py
+7
-2
@@ -64,7 +64,6 @@ class CompressedTensors24(CompressedTensorsScheme):
64
"Sparse CUTLASS not supported. vLLM must be built with "
65
"CUDA 12.2 or later to use this feature")
66
67
- self.output_dtype = params_dtype
67
layer.logical_widths = output_partition_sizes
68
layer.input_size = input_size
69
layer.input_size_per_partition = input_size_per_partition
@@ -205,6 +204,11 @@ class CompressedTensors24(CompressedTensorsScheme):
204
layer.weight_scale = torch.nn.Parameter(
205
layer.weight_scale.data, requires_grad=False)
206
207
+ # Set all negative zero values to 0 prior to compression
208
+ if (layer.weight.dtype.is_floating_point
209
+ and layer.weight.dtype.itemsize >= 2):
210
+ layer.weight.data[layer.weight.data == -0.0] = 0.0
211
+
212
w_compressed, meta = ops.cutlass_sparse_compress(layer.weight.data)
213
layer.weight = torch.nn.Parameter(w_compressed, requires_grad=False)
214
layer.meta = torch.nn.Parameter(meta, requires_grad=False)
@@ -254,9 +258,10 @@ class CompressedTensors24(CompressedTensorsScheme):
258
bt_meta=layer.meta,
259
scale_a=input_scale,
260
scale_b=layer.weight_scale,
257
- out_dtype=self.output_dtype,
261
+ out_dtype=x.dtype,
262
bias=bias,
263
)
264
+
265
assert out.is_contiguous()
266
return out
267