@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c1e37bf71b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[Kernel][Bugfix] Refactor and Fix CUTLASS 2:4 Sparse Kernels (#13198)

Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com>

Tyler Michael Smith committed Feb 13, 2025 at 19:01 UTC c1e37bf71b99f3c8cb4923c0b620918a93170206
16 files changed +576 -473
CMakeLists.txt
+5 -5
@@ -228,7 +228,8 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
228 SET(CUTLASS_ENABLE_HEADERS_ONLY ON CACHE BOOL "Enable only the header library")
229
230 # Set CUTLASS_REVISION manually -- its revision detection doesn't work in this case.
231 - set(CUTLASS_REVISION "v3.6.0" CACHE STRING "CUTLASS revision to use")
231 + # Please keep this in sync with FetchContent_Declare line below.
232 + set(CUTLASS_REVISION "v3.7.0" CACHE STRING "CUTLASS revision to use")
233
234 # Use the specified CUTLASS source directory for compilation if VLLM_CUTLASS_SRC_DIR is provided
235 if (DEFINED ENV{VLLM_CUTLASS_SRC_DIR})
@@ -245,6 +246,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
246 FetchContent_Declare(
247 cutlass
248 GIT_REPOSITORY https://github.com/nvidia/cutlass.git
249 + # Please keep this in sync with CUTLASS_REVISION line above.
250 GIT_TAG v3.7.0
251 GIT_PROGRESS TRUE
252
@@ -266,7 +268,6 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
268 "csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu"
269 "csrc/quantization/fp4/nvfp4_quant_entry.cu"
270 "csrc/sparse/cutlass/sparse_scaled_mm_entry.cu"
269 - "csrc/sparse/cutlass/sparse_compressor_entry.cu"
271 "csrc/cutlass_extensions/common.cpp")
272
273 set_gencode_flags_for_srcs(
@@ -359,8 +360,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
360 # The 2:4 sparse kernels cutlass_scaled_sparse_mm and cutlass_compressor
361 # require CUDA 12.2 or later (and only work on Hopper, 9.0a for now).
362 if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.2 AND SCALED_MM_3X_ARCHS)
362 - set(SRCS "csrc/sparse/cutlass/sparse_compressor_c3x.cu"
363 - "csrc/sparse/cutlass/sparse_scaled_mm_c3x.cu")
363 + set(SRCS "csrc/sparse/cutlass/sparse_scaled_mm_c3x.cu")
364 set_gencode_flags_for_srcs(
365 SRCS "${SRCS}"
366 CUDA_ARCHS "${SCALED_MM_3X_ARCHS}")
@@ -476,7 +476,7 @@ define_gpu_extension_target(
476 SOURCES ${VLLM_EXT_SRC}
477 COMPILE_FLAGS ${VLLM_GPU_FLAGS}
478 ARCHITECTURES ${VLLM_GPU_ARCHES}
479 - INCLUDE_DIRECTORIES ${CUTLASS_INCLUDE_DIR};${CUTLASS_TOOLS_UTIL_INCLUDE_DIR}
479 + INCLUDE_DIRECTORIES ${CUTLASS_INCLUDE_DIR}
480 USE_SABI 3
481 WITH_SOABI)
482
csrc/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp
+68 -1
@@ -16,6 +16,30 @@ namespace vllm::c3x {
16
17 using namespace cute;
18
19 +template <typename T>
20 +struct identity {
21 + CUTLASS_HOST_DEVICE
22 + T operator()(T lhs) const { return lhs; }
23 +};
24 +
25 +template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
26 +struct TrivialEpilogue {
27 + private:
28 + using Accum = cutlass::epilogue::fusion::Sm90AccFetch;
29 + using Compute = cutlass::epilogue::fusion::Sm90Compute<
30 + cutlass::epilogue::thread::Identity, ElementD, ElementAcc,
31 + cutlass::FloatRoundStyle::round_to_nearest>;
32 +
33 + public:
34 + using EVTCompute = cutlass::epilogue::fusion::Sm90EVT<Compute, Accum>;
35 + using ArgumentType = typename EVTCompute::Arguments;
36 +
37 + template <typename... Args>
38 + static ArgumentType prepare_args(Args... args) {
39 + return {};
40 + }
41 +};
42 +
43 /*
44 * This class provides the common load descriptors for the
45 * ScaledEpilogue[...] classes
@@ -174,6 +198,49 @@ struct ScaledEpilogueBias
198 }
199 };
200
201 +/*
202 + * This epilogue performs the same operation as ScaledEpilogueBias, but the
203 + * bias is a column vector instead of a row vector. Useful e.g. if we are
204 + * computing a GEMM via C^T += B^T A^T. This happens in the 2:4 sparse kernels.
205 + */
206 +template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
207 +struct ScaledEpilogueColumnBias
208 + : private ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor> {
209 + private:
210 + using SUPER = ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor>;
211 + using Accum = typename SUPER::Accum;
212 + using ScaleA = typename SUPER::template ColOrScalarLoad<float>;
213 + using ScaleB = typename SUPER::template RowOrScalarLoad<float>;
214 + using Bias = typename SUPER::template ColLoad<ElementD>;
215 +
216 + using Compute0 = cutlass::epilogue::fusion::Sm90Compute<
217 + cutlass::multiplies, float, float,
218 + cutlass::FloatRoundStyle::round_to_nearest>;
219 +
220 + using EVTCompute0 =
221 + cutlass::epilogue::fusion::Sm90EVT<Compute0, ScaleB, Accum>;
222 +
223 + using Compute1 = cutlass::epilogue::fusion::Sm90Compute<
224 + cutlass::multiply_add, ElementD, float,
225 + cutlass::FloatRoundStyle::round_to_nearest>;
226 +
227 + public:
228 + using EVTCompute =
229 + cutlass::epilogue::fusion::Sm90EVT<Compute1, ScaleA, EVTCompute0, Bias>;
230 +
231 + using ArgumentType = typename EVTCompute::Arguments;
232 + static ArgumentType prepare_args(torch::Tensor const& a_scales,
233 + torch::Tensor const& b_scales,
234 + torch::Tensor const& bias) {
235 + auto a_args = SUPER::template args_from_tensor<ScaleA, float>(a_scales);
236 + auto b_args = SUPER::template args_from_tensor<ScaleB, float>(b_scales);
237 + auto bias_args = SUPER::template args_from_tensor<Bias, ElementD>(bias);
238 +
239 + typename EVTCompute0::Arguments evt0_args{b_args};
240 + return ArgumentType{a_args, evt0_args, bias_args};
241 + }
242 +};
243 +
244 /*
245 * This epilogue directly supports per-tensor azp in int32 form.
246 * As opposed to the per-token epilogue below, this epilogue only has an azp_adj
@@ -314,4 +381,4 @@ struct ScaledEpilogueBiasAzpToken
381 }
382 };
383
317 -}; // namespace vllm::c3x
\ No newline at end of file
384 +}; // namespace vllm::c3x
csrc/ops.h
+1 -2
@@ -176,8 +176,7 @@ void cutlass_scaled_sparse_mm(torch::Tensor& out, torch::Tensor const& a,
176 torch::Tensor const& b_scales,
177 std::optional<torch::Tensor> const& bias);
178
179 -bool cutlass_sparse_compress_entry(torch::Tensor& a_compressed,
180 - torch::Tensor& e, torch::Tensor const& a);
179 +std::vector<torch::Tensor> cutlass_sparse_compress(torch::Tensor const& a);
180 #endif
181
182 void static_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
csrc/quantization/cutlass_w8a8/c3x/scaled_mm.cuh
+9 -4
@@ -53,12 +53,17 @@ struct cutlass_3x_gemm {
53
54 using EVTCompute = typename Epilogue::EVTCompute;
55
56 + // These are the minimum alignments needed for the kernels to compile
57 + static constexpr int AlignmentAB =
58 + 128 / cutlass::sizeof_bits<ElementAB>::value;
59 + static constexpr int AlignmentCD = 4;
60 +
61 using CollectiveEpilogue =
62 typename cutlass::epilogue::collective::CollectiveBuilder<
63 cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp, TileShape,
64 ClusterShape, cutlass::epilogue::collective::EpilogueTileAuto,
60 - ElementAcc, float, ElementC, StrideC, 4, ElementD, StrideD, 4,
61 - EpilogueSchedule, EVTCompute>::CollectiveOp;
65 + ElementAcc, float, ElementC, StrideC, AlignmentCD, ElementD, StrideD,
66 + AlignmentCD, EpilogueSchedule, EVTCompute>::CollectiveOp;
67
68 static constexpr size_t CEStorageSize =
69 sizeof(typename CollectiveEpilogue::SharedStorage);
@@ -69,8 +74,8 @@ struct cutlass_3x_gemm {
74 using CollectiveMainloop =
75 typename cutlass::gemm::collective::CollectiveBuilder<
76 cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
72 - ElementAB, cutlass::layout::RowMajor, 16,
73 - ElementAB, cutlass::layout::ColumnMajor, 16,
77 + ElementAB, cutlass::layout::RowMajor, AlignmentAB,
78 + ElementAB, cutlass::layout::ColumnMajor, AlignmentAB,
79 ElementAcc, TileShape, ClusterShape,
80 Stages,
81 KernelSchedule>::CollectiveOp;
csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cuh
+8 -3
@@ -103,14 +103,19 @@ struct cutlass_2x_gemm {
103
104 using EVTD = cutlass::epilogue::threadblock::Sm80EVT<D, EVTCompute>;
105
106 + // These are the minimum alignments needed for the kernels to compile
107 + static constexpr int AlignmentAB =
108 + 128 / cutlass::sizeof_bits<ElementAB>::value;
109 + static constexpr int AlignmentCD = 4;
110 +
111 // clang-format off
112 using RowMajor = typename cutlass::layout::RowMajor;
113 using ColumnMajor = typename cutlass::layout::ColumnMajor;
114 using KernelType =
115 ArchGuard<typename cutlass::gemm::kernel::DefaultGemmWithVisitor<
111 - ElementAB, RowMajor, cutlass::ComplexTransform::kNone, 16,
112 - ElementAB, ColumnMajor, cutlass::ComplexTransform::kNone, 16,
113 - float, cutlass::layout::RowMajor, 4,
116 + ElementAB, RowMajor, cutlass::ComplexTransform::kNone, AlignmentAB,
117 + ElementAB, ColumnMajor, cutlass::ComplexTransform::kNone, AlignmentAB,
118 + float, cutlass::layout::RowMajor, AlignmentCD,
119 ElementAcc, float, cutlass::arch::OpClassTensorOp,
120 Arch,
121 TileShape, WarpShape, InstructionShape,
csrc/sparse/cutlass/sparse_compressor_c3x.cu deleted
-165
@@ -1,165 +0,0 @@
1 -// clang-format will break include orders
2 -// clang-format off
3 -#include <cudaTypedefs.h>
4 -
5 -#if defined CUDA_VERSION && CUDA_VERSION >= 12020
6 -#include "sparse_scaled_mm_c3x.cuh"
7 -
8 -#include "cutlass/numeric_conversion.h"
9 -#include "cutlass/transform/device/transform_universal_adapter.hpp"
10 -#include "cutlass/transform/kernel/sparse_gemm_compressor.hpp"
11 -#include "cutlass/epilogue/collective/default_epilogue.hpp"
12 -
13 -#include "cutlass/util/host_tensor.h"
14 -#include "cutlass/util/packed_stride.hpp"
15 -// clang-format on
16 -
17 -using namespace cute;
18 -using namespace vllm;
19 -
20 -/// Make A structured sparse by replacing elements with 0 and compress it
21 -template <typename ElementA_, typename ElementAcc_>
22 -bool cutlass_sparse_compress(torch::Tensor& a_nzs, torch::Tensor& a_meta,
23 - torch::Tensor const& a) {
24 - // Checks for conformality
25 - TORCH_CHECK(a.dtype() == torch::kInt8 || a.dtype() == torch::kFloat8_e4m3fn ||
26 - a.dtype() == torch::kFloat16 || a.dtype() == torch::kBFloat16);
27 - TORCH_CHECK(a.dim() == 2)
28 - // Check for strides and alignment
29 - TORCH_CHECK(a.stride(0) % 4 == 0) // Required for semi-structured sparsity
30 - TORCH_CHECK(a.stride(1) == 1)
31 -
32 - int m = a.size(0);
33 - int k = a.size(1);
34 -
35 - // Sparse kernel setup; this kernel is not used for matmul,
36 - // but just for setting up the compressor utility
37 - // A matrix configuration
38 - using ElementA = ElementA_;
39 - using LayoutTagA = cutlass::layout::RowMajor;
40 - constexpr int AlignmentA = 128 / cutlass::sizeof_bits<ElementA>::value;
41 - // B matrix configuration
42 - using ElementB = ElementA;
43 - using LayoutTagB = cutlass::layout::ColumnMajor;
44 - constexpr int AlignmentB = 128 / cutlass::sizeof_bits<ElementB>::value;
45 - // C/D matrix configuration
46 - using ElementC = float;
47 - using LayoutTagC = cutlass::layout::ColumnMajor;
48 - constexpr int AlignmentC = 128 / cutlass::sizeof_bits<ElementC>::value;
49 - // Core kernel configurations
50 - using ElementAccumulator = ElementAcc_;
51 - using TileShape = Shape<_128, _128, _128>;
52 - using TileShapeRef = Shape<_128, _128, _64>;
53 - using ClusterShape = Shape<_1, _2, _1>;
54 - using KernelSchedule = typename std::conditional<
55 - std::is_same_v<ElementA, cutlass::float_e4m3_t>,
56 - cutlass::gemm::KernelTmaWarpSpecializedFP8FastAccum,
57 - cutlass::gemm::KernelTmaWarpSpecialized>::type;
58 -
59 - using EpilogueSchedule = cutlass::epilogue::TmaWarpSpecialized;
60 - using ProblemShape = Shape<int, int, int, int>;
61 -
62 - using CollectiveEpilogue =
63 - typename cutlass::epilogue::collective::CollectiveBuilder<
64 - cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp, TileShape,
65 - ClusterShape, cutlass::epilogue::collective::EpilogueTileAuto,
66 - ElementAccumulator, ElementAccumulator, ElementC, LayoutTagC,
67 - AlignmentC, ElementC, LayoutTagC, AlignmentC,
68 - EpilogueSchedule>::CollectiveOp;
69 -
70 - using CollectiveMainloop =
71 - typename cutlass::gemm::collective::CollectiveBuilder<
72 - cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp, ElementA,
73 - LayoutTagA, AlignmentA, ElementB, LayoutTagB, AlignmentB,
74 - ElementAccumulator, TileShape, ClusterShape,
75 - cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(
76 - sizeof(typename CollectiveEpilogue::SharedStorage))>,
77 - KernelSchedule>::CollectiveOp;
78 -
79 - using GemmKernel =
80 - cutlass::gemm::kernel::GemmUniversal<ProblemShape, CollectiveMainloop,
81 - CollectiveEpilogue>;
82 -
83 - using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
84 -
85 - using StrideA = cutlass::gemm::TagToStrideA_t<LayoutTagA>;
86 - using StrideE = StrideA;
87 -
88 - using StrideA = Stride<int64_t, Int<1>, int64_t>;
89 -
90 - // The n (=1) dimension does not matter for the compressor
91 - typename GemmKernel::ProblemShape prob_shape{m, 1, k, 1};
92 -
93 - using LayoutA = typename GemmKernel::CollectiveMainloop::LayoutA;
94 - using LayoutE = typename GemmKernel::CollectiveMainloop::LayoutE;
95 -
96 - using ElementE = typename GemmKernel::CollectiveMainloop::ElementE;
97 - using SparseConfig = typename GemmKernel::CollectiveMainloop::SparseConfig;
98 -
99 - // Offline compressor kernel
100 - using CompressorUtility =
101 - cutlass::transform::kernel::StructuredSparseCompressorUtility<
102 - ProblemShape, ElementA, LayoutTagA, SparseConfig>;
103 -
104 - using CompressorKernel =
105 - cutlass::transform::kernel::StructuredSparseCompressor<
106 - ProblemShape, ElementA, LayoutTagA, SparseConfig,
107 - cutlass::arch::Sm90>;
108 -
109 - using Compressor =
110 - cutlass::transform::device::TransformUniversalAdapter<CompressorKernel>;
111 -
112 - auto [M, N, K, L] = prob_shape;
113 -
114 - StrideA stride_A;
115 - stride_A =
116 - cutlass::make_cute_packed_stride(StrideA{}, cute::make_shape(M, K, L));
117 -
118 - CompressorUtility compressor_utility(prob_shape, stride_A);
119 -
120 - int ME = compressor_utility.get_metadata_m_physical();
121 - int KE = compressor_utility.get_metadata_k_physical();
122 - int KC = compressor_utility.get_tensorA_k_physical();
123 -
124 - auto a_ptr = static_cast<ElementA*>(a.data_ptr());
125 -
126 - auto a_nzs_ptr = static_cast<ElementA*>(a_nzs.data_ptr());
127 - auto a_meta_ptr = static_cast<typename Gemm::CollectiveMainloop::ElementE*>(
128 - a_meta.data_ptr());
129 -
130 - cutlass::KernelHardwareInfo hw_info;
131 - hw_info.device_id = 0;
132 - hw_info.sm_count =
133 - cutlass::KernelHardwareInfo::query_device_multiprocessor_count(
134 - hw_info.device_id);
135 - typename Compressor::Arguments arguments{
136 - prob_shape, {a_ptr, stride_A, a_nzs_ptr, a_meta_ptr}, {hw_info}};
137 -
138 - Compressor compressor_op;
139 - size_t workspace_size = Compressor::get_workspace_size(arguments);
140 - cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
141 -
142 - CUTLASS_CHECK(compressor_op.can_implement(arguments));
143 - CUTLASS_CHECK(compressor_op.initialize(arguments, workspace.get()));
144 - CUTLASS_CHECK(compressor_op.run());
145 - CUDA_CHECK(cudaDeviceSynchronize());
146 -
147 - return true;
148 -}
149 -
150 -bool cutlass_sparse_compress_sm90(torch::Tensor& a_nzs, torch::Tensor& a_meta,
151 - torch::Tensor const& a) {
152 - if (a.dtype() == torch::kBFloat16) {
153 - return cutlass_sparse_compress<cutlass::bfloat16_t, float>(a_nzs, a_meta,
154 - a);
155 - } else if (a.dtype() == torch::kFloat16) {
156 - return cutlass_sparse_compress<cutlass::half_t, float>(a_nzs, a_meta, a);
157 - } else if (a.dtype() == torch::kFloat8_e4m3fn) {
158 - return cutlass_sparse_compress<cutlass::float_e4m3_t, float>(a_nzs, a_meta,
159 - a);
160 - } else if (a.dtype() == torch::kInt8) {
161 - return cutlass_sparse_compress<int8_t, int32_t>(a_nzs, a_meta, a);
162 - }
163 - return false;
164 -}
165 -#endif
csrc/sparse/cutlass/sparse_compressor_c3x.cuh new
+90
@@ -0,0 +1,90 @@
1 +#pragma once
2 +
3 +// clang-format will break include orders
4 +// clang-format off
5 +#include <cudaTypedefs.h>
6 +
7 +#if defined CUDA_VERSION && CUDA_VERSION >= 12020
8 +#include "sparse_scaled_mm_c3x.cuh"
9 +
10 +#include "cutlass/numeric_conversion.h"
11 +#include "cutlass/transform/device/transform_universal_adapter.hpp"
12 +#include "cutlass/transform/kernel/sparse_gemm_compressor.hpp"
13 +#include "cutlass/epilogue/collective/default_epilogue.hpp"
14 +
15 +// clang-format on
16 +
17 +using namespace cute;
18 +using namespace vllm;
19 +
20 +using CompressorResult = std::tuple<torch::Tensor, torch::Tensor>;
21 +/// Make A structured sparse by replacing elements with 0 and compress it
22 +template <typename Gemm>
23 +CompressorResult cutlass_sparse_compress(torch::Tensor const& a) {
24 + // Checks for conformality
25 + TORCH_CHECK(a.dtype() == torch::kInt8 || a.dtype() == torch::kFloat8_e4m3fn ||
26 + a.dtype() == torch::kFloat16 || a.dtype() == torch::kBFloat16);
27 + TORCH_CHECK(a.dim() == 2)
28 + // Check for strides and alignment
29 + TORCH_CHECK(a.stride(0) % 4 == 0) // Required for semi-structured sparsity
30 + TORCH_CHECK(a.stride(1) == 1)
31 +
32 + using GemmKernel = typename Gemm::KernelType;
33 + using ElementA = typename Gemm::ElementAB;
34 + using ElementE = typename GemmKernel::CollectiveMainloop::ElementE;
35 +
36 + int m = a.size(0);
37 + int k = a.size(1);
38 + using ProblemShape = typename GemmKernel::ProblemShape;
39 + ProblemShape prob_shape{m, 1, k, 1};
40 +
41 + int64_t lda = a.stride(0);
42 + using StrideA = Stride<int64_t, Int<1>, int64_t>;
43 + StrideA a_stride{lda, Int<1>{}, 0};
44 +
45 + using CompressorUtility = typename Gemm::CompressorUtility;
46 + CompressorUtility compressor_utility(prob_shape, a_stride);
47 +
48 + // Allocate buffers for the metadata E and the compressed matrix A
49 + int ME = compressor_utility.get_metadata_m_physical();
50 + int KE = compressor_utility.get_metadata_k_physical();
51 + int MC = compressor_utility.get_tensorA_m_physical();
52 + int KC = compressor_utility.get_tensorA_k_physical();
53 +
54 + auto const a_meta_options =
55 + torch::TensorOptions().dtype(torch::kUInt8).device(a.device());
56 + auto const a_nzs_options =
57 + torch::TensorOptions().dtype(a.dtype()).device(a.device());
58 +
59 + auto a_meta = torch::zeros({ME, KE}, a_meta_options);
60 + auto a_nzs = torch::zeros({MC, KC}, a_nzs_options);
61 +
62 + auto a_ptr = static_cast<ElementA*>(a.data_ptr());
63 + auto a_nzs_ptr = static_cast<ElementA*>(a_nzs.data_ptr());
64 + auto a_meta_ptr = static_cast<ElementE*>(a_meta.data_ptr());
65 +
66 + cutlass::KernelHardwareInfo hw_info;
67 + hw_info.device_id = a.device().index();
68 + hw_info.sm_count =
69 + cutlass::KernelHardwareInfo::query_device_multiprocessor_count(
70 + hw_info.device_id);
71 +
72 + using Compressor = typename Gemm::Compressor;
73 + typename Compressor::Arguments arguments{
74 + prob_shape, {a_ptr, a_stride, a_nzs_ptr, a_meta_ptr}, {hw_info}};
75 +
76 + Compressor compressor_op;
77 + size_t workspace_size = Compressor::get_workspace_size(arguments);
78 + auto const workspace_options =
79 + torch::TensorOptions().dtype(torch::kUInt8).device(a.device());
80 + auto workspace = torch::empty(workspace_size, workspace_options);
81 +
82 + CUTLASS_CHECK(compressor_op.can_implement(arguments));
83 + CUTLASS_CHECK(compressor_op.initialize(arguments, workspace.data_ptr()));
84 + CUTLASS_CHECK(compressor_op.run());
85 + CUDA_CHECK(cudaDeviceSynchronize());
86 +
87 + return {a_meta, a_nzs};
88 +}
89 +
90 +#endif
csrc/sparse/cutlass/sparse_compressor_entry.cu deleted
-42
@@ -1,42 +0,0 @@
1 -#include <cudaTypedefs.h>
2 -
3 -#include <c10/cuda/CUDAGuard.h>
4 -#include <torch/all.h>
5 -
6 -#include "cutlass_extensions/common.hpp"
7 -
8 -#if defined ENABLE_SPARSE_SCALED_MM_C3X && ENABLE_SPARSE_SCALED_MM_C3X
9 -bool cutlass_sparse_compress_sm90(torch::Tensor& a_nzs, torch::Tensor& a_meta,
10 - torch::Tensor const& a);
11 -#endif
12 -
13 -bool cutlass_sparse_compress_entry(torch::Tensor& a_nzs, torch::Tensor& a_meta,
14 - torch::Tensor const& a) {
15 - // Checks for conformality
16 - TORCH_CHECK(a.dim() == 2 && a_meta.dim() == 2 && a_nzs.dim() == 2);
17 - TORCH_CHECK(a.size(0) == a_nzs.size(0) && a.size(0) == a_meta.size(0) &&
18 - a_nzs.size(1) * 2 == a.size(1) &&
19 - a_meta.size(1) * 2 * 4 == a.size(1));
20 - // Considering elemsPerMetaElem = 8b / 2b_per_nz = 4
21 -
22 - // Check for strides and alignment
23 - TORCH_CHECK(a.stride(1) == 1 && a_nzs.stride(1) == 1 &&
24 - a_meta.stride(1) == 1); // Row-major
25 - TORCH_CHECK(a.stride(0) % 8 == 0); // 8 Byte Alignment for Compression
26 -
27 - at::cuda::OptionalCUDAGuard const device_guard(device_of(a));
28 - int32_t version_num = get_sm_version_num();
29 -
30 - // Guard against compilation issues for sm90 kernels
31 -#if defined ENABLE_SPARSE_SCALED_MM_C3X && ENABLE_SPARSE_SCALED_MM_C3X
32 - if (version_num >= 90) {
33 - return cutlass_sparse_compress_sm90(a_nzs, a_meta, a);
34 - }
35 -#endif
36 -
37 - TORCH_CHECK_NOT_IMPLEMENTED(
38 - false,
39 - "No compiled cutlass_scaled_sparse_mm for a compute capability less than "
40 - "CUDA device capability: ",
41 - version_num);
42 -}
csrc/sparse/cutlass/sparse_scaled_mm_c3x.cu
+134 -130
@@ -9,17 +9,30 @@
9 using namespace cute;
10 using namespace vllm;
11
12 +struct GemmCallerTraits {
13 + using return_type = void;
14 +
15 + template <typename GemmConfig, typename... Args>
16 + static return_type invoke(Args&&... args) {
17 + return cutlass_sparse_gemm_caller<GemmConfig>(std::forward<Args>(args)...);
18 + }
19 +};
20 +
21 +struct GemmCompressorTraits {
22 + using return_type = CompressorResult;
23 +
24 + template <typename GemmConfig, typename... Args>
25 + static return_type invoke(Args&&... args) {
26 + return cutlass_sparse_compress<GemmConfig>(std::forward<Args>(args)...);
27 + }
28 +};
29 +
30 template <typename InType, typename OutType,
31 template <typename, typename, typename> typename Epilogue,
14 - typename... EpilogueArgs>
15 -void cutlass_gemm_sm90_fp8_dispatch(torch::Tensor& out, torch::Tensor const& a,
16 - torch::Tensor const& bt_nzs,
17 - torch::Tensor const& bt_meta,
18 - EpilogueArgs&&... args) {
19 - static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
20 - TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
21 - TORCH_CHECK(bt_meta.dtype() == torch::kUInt8);
22 - TORCH_CHECK(bt_nzs.dtype() == torch::kFloat8_e4m3fn);
32 + typename DispatchFunc, typename... Args>
33 +typename DispatchFunc::return_type cutlass_gemm_sm90_fp8_dispatch(
34 + uint32_t m, uint32_t n, Args&&... args) {
35 + static_assert(std::is_same_v<InType, cutlass::float_e4m3_t>);
36
37 using Cutlass3xGemmDefault =
38 typename sm90_config_default<InType, OutType, Epilogue>::Cutlass3xGemm;
@@ -49,122 +62,87 @@ void cutlass_gemm_sm90_fp8_dispatch(torch::Tensor& out, torch::Tensor const& a,
62 using Cutlass3xGemm8 =
63 typename sm90_fp8_config_8<InType, OutType, Epilogue>::Cutlass3xGemm;
64
52 - uint32_t const n = bt_nzs.size(0);
53 - uint32_t const m = a.size(0); // Batch size
65 uint32_t const mp2 =
66 std::max(static_cast<uint32_t>(64), next_pow_2(m)); // next power of 2
67
68 if (mp2 <= 64) {
69 if (n == 28672) {
59 - return cutlass_sparse_gemm_caller<Cutlass3xGemm2>(
60 - out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
70 + return DispatchFunc::template invoke<Cutlass3xGemm2>(
71 + std::forward<Args>(args)...);
72 } else if (n == 4096 || n == 6144) {
62 - return cutlass_sparse_gemm_caller<Cutlass3xGemm1>(
63 - out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
73 + return DispatchFunc::template invoke<Cutlass3xGemm1>(
74 + std::forward<Args>(args)...);
75 }
76 } else if (mp2 <= 128) {
77 if (n == 4096) {
67 - return cutlass_sparse_gemm_caller<Cutlass3xGemm3>(
68 - out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
78 + return DispatchFunc::template invoke<Cutlass3xGemm3>(
79 + std::forward<Args>(args)...);
80 } else if (n == 28672) {
70 - return cutlass_sparse_gemm_caller<Cutlass3xGemm5>(
71 - out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
81 + return DispatchFunc::template invoke<Cutlass3xGemm5>(
82 + std::forward<Args>(args)...);
83 } else if (n == 6144) {
73 - return cutlass_sparse_gemm_caller<Cutlass3xGemm4>(
74 - out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
84 + return DispatchFunc::template invoke<Cutlass3xGemm4>(
85 + std::forward<Args>(args)...);
86 }
87 } else if (mp2 <= 256) {
88 if (n == 4096) {
78 - return cutlass_sparse_gemm_caller<Cutlass3xGemm6>(
79 - out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
89 + return DispatchFunc::template invoke<Cutlass3xGemm6>(
90 + std::forward<Args>(args)...);
91 } else if (n == 28672) {
81 - return cutlass_sparse_gemm_caller<Cutlass3xGemm8>(
82 - out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
92 + return DispatchFunc::template invoke<Cutlass3xGemm8>(
93 + std::forward<Args>(args)...);
94 } else if (n == 6144) {
84 - return cutlass_sparse_gemm_caller<Cutlass3xGemm7>(
85 - out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
95 + return DispatchFunc::template invoke<Cutlass3xGemm7>(
96 + std::forward<Args>(args)...);
97 }
98 } else {
99 if (n == 6144 || n == 28672) {
89 - return cutlass_sparse_gemm_caller<Cutlass3xGemm8>(
90 - out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
100 + return DispatchFunc::template invoke<Cutlass3xGemm8>(
101 + std::forward<Args>(args)...);
102 } else if (n == 4096) {
92 - return cutlass_sparse_gemm_caller<Cutlass3xGemm7>(
93 - out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
103 + return DispatchFunc::template invoke<Cutlass3xGemm7>(
104 + std::forward<Args>(args)...);
105 }
106 }
107
108 // Otherwise the default heuristic
109 if (mp2 <= 64) {
110 // n in [1, 64]
100 - return cutlass_sparse_gemm_caller<Cutlass3xGemmM64>(
101 - out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
111 + return DispatchFunc::template invoke<Cutlass3xGemmM64>(
112 + std::forward<Args>(args)...);
113 } else if (mp2 <= 128) {
114 // n in (64, 128]
104 - return cutlass_sparse_gemm_caller<Cutlass3xGemmM128>(
105 - out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
115 + return DispatchFunc::template invoke<Cutlass3xGemmM128>(
116 + std::forward<Args>(args)...);
117 } else if (mp2 <= 256) {
118 // n in (128, 256]
108 - return cutlass_sparse_gemm_caller<Cutlass3xGemmM256>(
109 - out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
119 + return DispatchFunc::template invoke<Cutlass3xGemmM256>(
120 + std::forward<Args>(args)...);
121 } else {
122 // n in (256, inf)
112 - return cutlass_sparse_gemm_caller<Cutlass3xGemmM512>(
113 - out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
123 + return DispatchFunc::template invoke<Cutlass3xGemmM512>(
124 + std::forward<Args>(args)...);
125 }
126 }
127
128 template <typename InType, typename OutType,
129 template <typename, typename, typename> typename Epilogue,
119 - typename... EpilogueArgs>
120 -void cutlass_gemm_sm90_fp16_dispatch(torch::Tensor& out, torch::Tensor const& a,
121 - torch::Tensor const& bt_nzs,
122 - torch::Tensor const& bt_meta,
123 - EpilogueArgs&&... args) {
124 - static_assert(std::is_same<InType, cutlass::half_t>());
125 - TORCH_CHECK(a.dtype() == torch::kFloat16);
126 - TORCH_CHECK(bt_meta.dtype() == torch::kUInt8);
127 - TORCH_CHECK(bt_nzs.dtype() == torch::kFloat16);
128 -
129 - using Cutlass3xGemmDefault =
130 - typename sm90_config_default<InType, OutType, Epilogue>::Cutlass3xGemm;
131 -
132 - // m in (128, inf)
133 - return cutlass_sparse_gemm_caller<Cutlass3xGemmDefault>(
134 - out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
135 -}
136 -
137 -template <typename InType, typename OutType,
138 - template <typename, typename, typename> typename Epilogue,
139 - typename... EpilogueArgs>
140 -void cutlass_gemm_sm90_bf16_dispatch(torch::Tensor& out, torch::Tensor const& a,
141 - torch::Tensor const& bt_nzs,
142 - torch::Tensor const& bt_meta,
143 - EpilogueArgs&&... args) {
144 - static_assert(std::is_same<InType, cutlass::bfloat16_t>());
145 - TORCH_CHECK(a.dtype() == torch::kBFloat16);
146 - TORCH_CHECK(bt_meta.dtype() == torch::kUInt8);
147 - TORCH_CHECK(bt_nzs.dtype() == torch::kBFloat16);
148 -
130 + typename DispatchFunc, typename... Args>
131 +typename DispatchFunc::return_type cutlass_gemm_sm90_16bit_dispatch(
132 + uint32_t m, uint32_t n, Args&&... args) {
133 using Cutlass3xGemmDefault =
134 typename sm90_config_default<InType, OutType, Epilogue>::Cutlass3xGemm;
135
152 - // m in (128, inf)
153 - return cutlass_sparse_gemm_caller<Cutlass3xGemmDefault>(
154 - out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
136 + return DispatchFunc::template invoke<Cutlass3xGemmDefault>(
137 + std::forward<Args>(args)...);
138 }
139
140 template <typename InType, typename OutType,
141 template <typename, typename, typename> typename Epilogue,
159 - typename... EpilogueArgs>
160 -void cutlass_gemm_sm90_int8_dispatch(torch::Tensor& out, torch::Tensor const& a,
161 - torch::Tensor const& bt_nzs,
162 - torch::Tensor const& bt_meta,
163 - EpilogueArgs&&... args) {
164 - static_assert(std::is_same<InType, int8_t>());
165 - TORCH_CHECK(a.dtype() == torch::kInt8);
166 - TORCH_CHECK(bt_meta.dtype() == torch::kUInt8);
167 - TORCH_CHECK(bt_nzs.dtype() == torch::kInt8);
142 + typename DispatchFunc, typename... Args>
143 +typename DispatchFunc::return_type cutlass_gemm_sm90_int8_dispatch(
144 + uint32_t m, uint32_t n, Args&&... args) {
145 + static_assert(std::is_same_v<InType, int8_t>);
146
147 using Cutlass3xGemmDefault =
148 typename sm90_config_default<InType, OutType, Epilogue>::Cutlass3xGemm;
@@ -179,37 +157,35 @@ void cutlass_gemm_sm90_int8_dispatch(torch::Tensor& out, torch::Tensor const& a,
157 typename sm90_int8_config_M32_NSmall<InType, OutType,
158 Epilogue>::Cutlass3xGemm;
159
182 - uint32_t const n = out.size(1);
160 bool const is_small_n = n < 8192;
184 -
185 - uint32_t const m = a.size(0);
161 uint32_t const mp2 =
162 std::max(static_cast<uint32_t>(32), next_pow_2(m)); // next power of 2
163
164 if (mp2 <= 32) {
165 // m in [1, 32]
166 if (is_small_n) {
192 - return cutlass_sparse_gemm_caller<Cutlass3xGemmM32NSmall>(
193 - out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
167 + return DispatchFunc::template invoke<Cutlass3xGemmM32NSmall>(
168 + std::forward<Args>(args)...);
169 } else {
195 - return cutlass_sparse_gemm_caller<Cutlass3xGemmM32NBig>(
196 - out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
170 + return DispatchFunc::template invoke<Cutlass3xGemmM32NBig>(
171 + std::forward<Args>(args)...);
172 }
173 } else if (mp2 <= 64) {
174 // m in (32, 64]
200 - return cutlass_sparse_gemm_caller<Cutlass3xGemmM64>(
201 - out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
175 + return DispatchFunc::template invoke<Cutlass3xGemmM64>(
176 + std::forward<Args>(args)...);
177 } else if (mp2 <= 128) {
178 // m in (64, 128]
204 - return cutlass_sparse_gemm_caller<Cutlass3xGemmM128>(
205 - out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
179 + return DispatchFunc::template invoke<Cutlass3xGemmM128>(
180 + std::forward<Args>(args)...);
181 } else {
182 // m in (128, inf)
208 - return cutlass_sparse_gemm_caller<Cutlass3xGemmDefault>(
209 - out, a, bt_nzs, bt_meta, std::forward<EpilogueArgs>(args)...);
183 + return DispatchFunc::template invoke<Cutlass3xGemmDefault>(
184 + std::forward<Args>(args)...);
185 }
186 }
187
188 +// Dispatch to GEMM implementations based on element types
189 template <template <typename, typename, typename> typename Epilogue,
190 typename... EpilogueArgs>
191 void cutlass_scaled_sparse_mm_sm90_epilogue(torch::Tensor& out,
@@ -217,19 +193,24 @@ void cutlass_scaled_sparse_mm_sm90_epilogue(torch::Tensor& out,
193 torch::Tensor const& bt_nzs,
194 torch::Tensor const& bt_meta,
195 EpilogueArgs&&... epilogue_args) {
196 + uint32_t const m = out.size(0);
197 + uint32_t const n = out.size(1);
198 +
199 + // TODO: add dispatch functions to all of these
200 TORCH_CHECK(bt_meta.dtype() == torch::kUInt8);
201 if (a.dtype() == torch::kInt8) {
202 TORCH_CHECK(bt_nzs.dtype() == torch::kInt8);
203
204 if (out.dtype() == torch::kBFloat16) {
205 return cutlass_gemm_sm90_int8_dispatch<int8_t, cutlass::bfloat16_t,
226 - Epilogue>(
227 - out, a, bt_nzs, bt_meta,
206 + Epilogue, GemmCallerTraits>(
207 + m, n, out, a, bt_nzs, bt_meta,
208 std::forward<EpilogueArgs>(epilogue_args)...);
209 } else {
210 TORCH_CHECK(out.dtype() == torch::kFloat16);
231 - return cutlass_gemm_sm90_int8_dispatch<int8_t, cutlass::half_t, Epilogue>(
232 - out, a, bt_nzs, bt_meta,
211 + return cutlass_gemm_sm90_int8_dispatch<int8_t, cutlass::half_t, Epilogue,
212 + GemmCallerTraits>(
213 + m, n, out, a, bt_nzs, bt_meta,
214 std::forward<EpilogueArgs>(epilogue_args)...);
215 }
216 } else if (a.dtype() == torch::kFloat8_e4m3fn) {
@@ -237,47 +218,34 @@ void cutlass_scaled_sparse_mm_sm90_epilogue(torch::Tensor& out,
218
219 if (out.dtype() == torch::kBFloat16) {
220 return cutlass_gemm_sm90_fp8_dispatch<cutlass::float_e4m3_t,
240 - cutlass::bfloat16_t, Epilogue>(
241 - out, a, bt_nzs, bt_meta,
221 + cutlass::bfloat16_t, Epilogue,
222 + GemmCallerTraits>(
223 + m, n, out, a, bt_nzs, bt_meta,
224 std::forward<EpilogueArgs>(epilogue_args)...);
225 } else {
226 TORCH_CHECK(out.dtype() == torch::kFloat16);
245 - return cutlass_gemm_sm90_fp8_dispatch<cutlass::float_e4m3_t,
246 - cutlass::half_t, Epilogue>(
247 - out, a, bt_nzs, bt_meta,
227 + return cutlass_gemm_sm90_fp8_dispatch<
228 + cutlass::float_e4m3_t, cutlass::half_t, Epilogue, GemmCallerTraits>(
229 + m, n, out, a, bt_nzs, bt_meta,
230 std::forward<EpilogueArgs>(epilogue_args)...);
231 }
232 } else if (a.dtype() == torch::kFloat16) {
233 TORCH_CHECK(bt_nzs.dtype() == torch::kFloat16);
234 + TORCH_CHECK(out.dtype() == torch::kFloat16);
235
253 - if (out.dtype() == torch::kBFloat16) {
254 - return cutlass_gemm_sm90_fp16_dispatch<cutlass::half_t,
255 - cutlass::bfloat16_t, Epilogue>(
256 - out, a, bt_nzs, bt_meta,
257 - std::forward<EpilogueArgs>(epilogue_args)...);
258 - } else {
259 - TORCH_CHECK(out.dtype() == torch::kFloat16);
260 - return cutlass_gemm_sm90_fp16_dispatch<cutlass::half_t, cutlass::half_t,
261 - Epilogue>(
262 - out, a, bt_nzs, bt_meta,
263 - std::forward<EpilogueArgs>(epilogue_args)...);
264 - }
236 + return cutlass_gemm_sm90_16bit_dispatch<cutlass::half_t, cutlass::half_t,
237 + Epilogue, GemmCallerTraits>(
238 + m, n, out, a, bt_nzs, bt_meta,
239 + std::forward<EpilogueArgs>(epilogue_args)...);
240 } else { // a.dtype() == torch::kBFloat16
241 TORCH_CHECK(a.dtype() == torch::kBFloat16);
242 TORCH_CHECK(bt_nzs.dtype() == torch::kBFloat16);
243 + TORCH_CHECK(out.dtype() == torch::kBFloat16);
244
269 - if (out.dtype() == torch::kBFloat16) {
270 - return cutlass_gemm_sm90_bf16_dispatch<cutlass::bfloat16_t,
271 - cutlass::bfloat16_t, Epilogue>(
272 - out, a, bt_nzs, bt_meta,
273 - std::forward<EpilogueArgs>(epilogue_args)...);
274 - } else {
275 - TORCH_CHECK(out.dtype() == torch::kFloat16);
276 - return cutlass_gemm_sm90_bf16_dispatch<cutlass::bfloat16_t,
277 - cutlass::half_t, Epilogue>(
278 - out, a, bt_nzs, bt_meta,
279 - std::forward<EpilogueArgs>(epilogue_args)...);
280 - }
245 + return cutlass_gemm_sm90_16bit_dispatch<
246 + cutlass::bfloat16_t, cutlass::bfloat16_t, Epilogue, GemmCallerTraits>(
247 + m, n, out, a, bt_nzs, bt_meta,
248 + std::forward<EpilogueArgs>(epilogue_args)...);
249 }
250 }
251
@@ -287,17 +255,53 @@ void cutlass_scaled_sparse_mm_sm90(torch::Tensor& out, torch::Tensor const& a,
255 torch::Tensor const& a_scales,
256 torch::Tensor const& b_scales,
257 std::optional<torch::Tensor> const& bias) {
258 + TORCH_CHECK(bt_meta.dtype() == torch::kUInt8);
259 TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
260 TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
261 +
262 if (bias) {
263 TORCH_CHECK(bias->dtype() == out.dtype(),
294 - "currently bias dtype must match output dtype ", out.dtype());
295 - return cutlass_scaled_sparse_mm_sm90_epilogue<c3x::ScaledEpilogueBias>(
296 - out, a, bt_nzs, bt_meta, b_scales, a_scales, *bias);
264 + "CUTLASS scaled_mm bias dtype must match output dtype ",
265 + out.dtype());
266 + return cutlass_scaled_sparse_mm_sm90_epilogue<
267 + c3x::ScaledEpilogueColumnBias>(out, a, bt_nzs, bt_meta, b_scales,
268 + a_scales, *bias);
269 } else {
270 return cutlass_scaled_sparse_mm_sm90_epilogue<c3x::ScaledEpilogue>(
271 out, a, bt_nzs, bt_meta, b_scales, a_scales);
272 }
273 }
274
275 +CompressorResult cutlass_sparse_compress_sm90(torch::Tensor const& a) {
276 + // These m and n variables are fordispatching to different GEMM algorithms.
277 + uint32_t const m = 1; // Set M to 1 for compression
278 + uint32_t const n = a.size(1);
279 +
280 + // Note: For correctess, the compressed format must be invariant in:
281 + // - M, the flattened number of tokens
282 + // - Whether output dtype is fp16 or bf16
283 + // - CUTLASS epilogues
284 +
285 + if (a.dtype() == torch::kInt8) {
286 + return cutlass_gemm_sm90_int8_dispatch<int8_t, cutlass::bfloat16_t,
287 + c3x::TrivialEpilogue,
288 + GemmCompressorTraits>(m, n, a);
289 + } else if (a.dtype() == torch::kFloat8_e4m3fn) {
290 + return cutlass_gemm_sm90_fp8_dispatch<
291 + cutlass::float_e4m3_t, cutlass::bfloat16_t, c3x::TrivialEpilogue,
292 + GemmCompressorTraits>(m, n, a);
293 + } else if (a.dtype() == torch::kFloat16) {
294 + return cutlass_gemm_sm90_16bit_dispatch<
295 + cutlass::bfloat16_t, cutlass::bfloat16_t, c3x::TrivialEpilogue,
296 + GemmCompressorTraits>(m, n, a);
297 + } else {
298 + TORCH_CHECK(a.dtype() == torch::kBFloat16,
299 + "cutlass_sparse_compress only supports int8, fp8_e4m3, fp16, "
300 + "and bf16 datatypes");
301 + return cutlass_gemm_sm90_16bit_dispatch<cutlass::half_t, cutlass::half_t,
302 + c3x::TrivialEpilogue,
303 + GemmCompressorTraits>(m, n, a);
304 + }
305 +}
306 +
307 #endif
csrc/sparse/cutlass/sparse_scaled_mm_c3x.cuh
+154 -78
@@ -1,3 +1,5 @@
1 +#pragma once
2 +
3 // clang-format will break include orders
4 // clang-format off
5 #include <cudaTypedefs.h>
@@ -12,6 +14,9 @@
14 #include "cutlass/epilogue/collective/collective_builder.hpp"
15 #include "cutlass/gemm/collective/collective_builder.hpp"
16
17 +#include "cutlass/transform/device/transform_universal_adapter.hpp"
18 +#include "cutlass/transform/kernel/sparse_gemm_compressor.hpp"
19 +
20 #include "core/math.hpp"
21 #include "cutlass_extensions/cute_utils.cuh"
22 #include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
@@ -22,7 +27,7 @@
27 using namespace cute;
28
29 /*
25 - This file defines sparse quantized GEMM operations using the CUTLASS 3.x API,
30 + This file defines 2:4 sparse GEMM operations using the CUTLASS 3.x API,
31 for NVIDIA GPUs with sm90a (Hopper) or later.
32 */
33
@@ -45,17 +50,20 @@ struct enable_sm90_or_later : Kernel {
50
51 using GemmUniversalMode = cutlass::gemm::GemmUniversalMode;
52
53 +/*
54 + * cutlass_sparse_3x_gemm defines a 2:4 sparse GEMM kernel via CUTLASS
55 + * for SM90 Hopper systems.
56 + */
57 template <typename ElementAB_, typename ElementD_,
58 template <typename, typename, typename> typename Epilogue_,
59 typename TileShape, typename ClusterShape, typename KernelSchedule,
51 - typename EpilogueSchedule, typename AccType,
52 - typename TileSchedule = cutlass::gemm::PersistentScheduler,
53 - GemmUniversalMode Mode_ = GemmUniversalMode::kGemm>
60 + typename EpilogueSchedule>
61 struct cutlass_sparse_3x_gemm {
55 - static const GemmUniversalMode Mode = Mode_;
62 using ElementAB = ElementAB_;
63 using ElementD = ElementD_;
58 - using ElementAcc = AccType;
64 + using ElementAcc =
65 + typename std::conditional<std::is_same_v<ElementAB, int8_t>, int32_t,
66 + float>::type;
67
68 using EpilogueDescriptor =
69 cutlass::epilogue::collective::detail::EpilogueDescriptor<
@@ -66,30 +74,22 @@ struct cutlass_sparse_3x_gemm {
74
75 using ElementC = void;
76 using LayoutC = cutlass::layout::RowMajor;
69 - using LayoutD = LayoutC;
70 - using StrideC = cutlass::detail::TagToStrideA_t<LayoutC>;
71 - using StrideD = cutlass::detail::TagToStrideA_t<LayoutD>;
72 -
77 using LayoutC_Transpose =
78 typename cutlass::layout::LayoutTranspose<LayoutC>::type;
75 - using LayoutD_Transpose =
76 - typename cutlass::layout::LayoutTranspose<LayoutD>::type;
79
80 using EVTCompute = typename Epilogue::EVTCompute;
81
80 - static constexpr int AlignmentA =
81 - 128 / cutlass::sizeof_bits<ElementAB>::value;
82 - static constexpr int AlignmentB =
82 + // These are the minimum alignments needed for the kernels to compile
83 + static constexpr int AlignmentAB =
84 128 / cutlass::sizeof_bits<ElementAB>::value;
84 - static constexpr int AlignmentCD =
85 - 128 / cutlass::sizeof_bits<ElementD>::value;
85 + static constexpr int AlignmentCD = 4;
86
87 using CollectiveEpilogue =
88 typename cutlass::epilogue::collective::CollectiveBuilder<
89 cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp, TileShape,
90 ClusterShape, cutlass::epilogue::collective::EpilogueTileAuto,
91 - ElementAcc, ElementAcc, ElementC, LayoutC_Transpose, AlignmentCD,
92 - ElementD, LayoutD_Transpose, AlignmentCD, EpilogueSchedule,
91 + ElementAcc, float, ElementC, LayoutC_Transpose, AlignmentCD, ElementD,
92 + LayoutC_Transpose, AlignmentCD, EpilogueSchedule,
93 EVTCompute>::CollectiveOp;
94
95 static constexpr size_t CEStorageSize =
@@ -101,8 +101,8 @@ struct cutlass_sparse_3x_gemm {
101 using CollectiveMainloop =
102 typename cutlass::gemm::collective::CollectiveBuilder<
103 cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
104 - ElementAB, cutlass::layout::RowMajor, AlignmentA,
105 - ElementAB, cutlass::layout::ColumnMajor, AlignmentB,
104 + ElementAB, cutlass::layout::RowMajor, AlignmentAB,
105 + ElementAB, cutlass::layout::ColumnMajor, AlignmentAB,
106 ElementAcc, TileShape, ClusterShape,
107 Stages,
108 KernelSchedule>::CollectiveOp;
@@ -110,11 +110,100 @@ struct cutlass_sparse_3x_gemm {
110
111 using KernelType = enable_sm90_or_later<cutlass::gemm::kernel::GemmUniversal<
112 cute::Shape<int, int, int, int>, CollectiveMainloop, CollectiveEpilogue,
113 - TileSchedule>>;
113 + cutlass::gemm::PersistentScheduler>>;
114
115 struct GemmKernel : public KernelType {};
116 +
117 + // Sparse compressor definitions
118 + using SparseConfig = typename GemmKernel::CollectiveMainloop::SparseConfig;
119 + using LayoutTagA = cutlass::layout::RowMajor;
120 + using CompressorUtility =
121 + cutlass::transform::kernel::StructuredSparseCompressorUtility<
122 + typename GemmKernel::ProblemShape, ElementAB, LayoutTagA,
123 + SparseConfig>;
124 + using CompressorKernel =
125 + cutlass::transform::kernel::StructuredSparseCompressor<
126 + typename GemmKernel::ProblemShape, ElementAB, LayoutTagA,
127 + SparseConfig, cutlass::arch::Sm90>;
128 + using Compressor =
129 + cutlass::transform::device::TransformUniversalAdapter<CompressorKernel>;
130 };
131
132 +/*
133 + * This class defines kernel to compress a 2:4 sparse matrix.
134 + * The particular format is defined by the Gemm template parameter,
135 + * which is a cutlass_sparse_3x_gemm.
136 + */
137 +using CompressorResult = std::tuple<torch::Tensor, torch::Tensor>;
138 +/// Make A structured sparse by replacing elements with 0 and compress it
139 +template <typename Gemm>
140 +CompressorResult cutlass_sparse_compress(torch::Tensor const& a) {
141 + // Checks for conformality
142 + TORCH_CHECK(a.dtype() == torch::kInt8 || a.dtype() == torch::kFloat8_e4m3fn ||
143 + a.dtype() == torch::kFloat16 || a.dtype() == torch::kBFloat16);
144 + TORCH_CHECK(a.dim() == 2)
145 + // Check for strides and alignment
146 + TORCH_CHECK(a.stride(0) % 4 == 0) // Required for semi-structured sparsity
147 + TORCH_CHECK(a.stride(1) == 1)
148 +
149 + using GemmKernel = typename Gemm::KernelType;
150 + using ElementA = typename Gemm::ElementAB;
151 + using ElementE = typename GemmKernel::CollectiveMainloop::ElementE;
152 +
153 + int m = a.size(0);
154 + int k = a.size(1);
155 + using ProblemShape = typename GemmKernel::ProblemShape;
156 + ProblemShape prob_shape{m, 1, k, 1};
157 +
158 + int64_t lda = a.stride(0);
159 + using StrideA = Stride<int64_t, Int<1>, int64_t>;
160 + StrideA a_stride{lda, Int<1>{}, 0};
161 +
162 + using CompressorUtility = typename Gemm::CompressorUtility;
163 + CompressorUtility compressor_utility(prob_shape, a_stride);
164 +
165 + // Allocate buffers for the metadata E and the compressed matrix A
166 + int ME = compressor_utility.get_metadata_m_physical();
167 + int KE = compressor_utility.get_metadata_k_physical();
168 + int MC = compressor_utility.get_tensorA_m_physical();
169 + int KC = compressor_utility.get_tensorA_k_physical();
170 +
171 + auto const a_meta_options =
172 + torch::TensorOptions().dtype(torch::kUInt8).device(a.device());
173 + auto const a_nzs_options =
174 + torch::TensorOptions().dtype(a.dtype()).device(a.device());
175 +
176 + auto a_meta = torch::zeros({ME, KE}, a_meta_options);
177 + auto a_nzs = torch::zeros({MC, KC}, a_nzs_options);
178 +
179 + auto a_ptr = static_cast<ElementA*>(a.data_ptr());
180 + auto a_nzs_ptr = static_cast<ElementA*>(a_nzs.data_ptr());
181 + auto a_meta_ptr = static_cast<ElementE*>(a_meta.data_ptr());
182 +
183 + cutlass::KernelHardwareInfo hw_info;
184 + hw_info.device_id = a.device().index();
185 + hw_info.sm_count =
186 + cutlass::KernelHardwareInfo::query_device_multiprocessor_count(
187 + hw_info.device_id);
188 +
189 + using Compressor = typename Gemm::Compressor;
190 + typename Compressor::Arguments arguments{
191 + prob_shape, {a_ptr, a_stride, a_nzs_ptr, a_meta_ptr}, {hw_info}};
192 +
193 + Compressor compressor_op;
194 + size_t workspace_size = Compressor::get_workspace_size(arguments);
195 + auto const workspace_options =
196 + torch::TensorOptions().dtype(torch::kUInt8).device(a.device());
197 + auto workspace = torch::empty(workspace_size, workspace_options);
198 +
199 + CUTLASS_CHECK(compressor_op.can_implement(arguments));
200 + CUTLASS_CHECK(compressor_op.initialize(arguments, workspace.data_ptr()));
201 + CUTLASS_CHECK(compressor_op.run());
202 + CUDA_CHECK(cudaDeviceSynchronize());
203 +
204 + return {a_meta, a_nzs};
205 +}
206 +
207 template <typename Gemm, typename... EpilogueArgs>
208 void cutlass_sparse_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
209 torch::Tensor const& bt_nzs,
@@ -126,27 +215,25 @@ void cutlass_sparse_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
215 // Interface stride expected from the argument a (will get transposed)
216 // We compute C^T = B^T * A^T, but we assume B is transposed before
217 // compression and hence the bt_* naming
129 - using LayoutA = cutlass::layout::RowMajor;
218 using LayoutB = typename Gemm::GemmKernel::CollectiveMainloop::LayoutA;
219 using LayoutE = typename Gemm::GemmKernel::CollectiveMainloop::LayoutE;
132 - using LayoutD = cutlass::layout::RowMajor;
220
134 - using StrideA = cutlass::detail::TagToStrideA_t<LayoutA>;
135 - using StrideD = cutlass::detail::TagToStrideA_t<LayoutD>;
221 + // M, N, K after transposition
222 + int32_t m = out.size(1);
223 + int32_t n = out.size(0);
224 + int32_t k = a.size(1);
225 +
226 + int64_t lda = a.stride(0);
227 + int64_t ldc = out.stride(0);
228
137 - auto layout_A = make_cute_layout<StrideA>(a, "A");
138 - auto layout_D = make_cute_layout<StrideD>(out, "D");
229 + using StrideA = Stride<int64_t, Int<1>, int64_t>;
230 + using StrideC = Stride<Int<1>, int64_t, int64_t>;
231
140 - // Transpose A and D
141 - // A doesn't need to be transposed since cutlass expects a NxK matrix
142 - // for B (which is At)
143 - auto stride_At = layout_A.stride();
144 - auto stride_Dt = permute_layout<1, 0, 2>(layout_D).stride();
232 + StrideA a_stride{lda, Int<1>{}, Int<0>{}};
233 + StrideC c_stride{Int<1>{}, ldc, Int<0>{}};
234
235 using GemmKernel = typename Gemm::GemmKernel;
147 - typename GemmKernel::ProblemShape prob_shape{
148 - static_cast<int>(bt_nzs.size(0)), static_cast<int>(size<0>(layout_A)),
149 - static_cast<int>(size<1>(layout_A)), 1};
236 + typename GemmKernel::ProblemShape prob_shape{m, n, k, 1};
237
238 using ElementE = typename GemmKernel::CollectiveMainloop::ElementE;
239 using SparseConfig = typename GemmKernel::CollectiveMainloop::SparseConfig;
@@ -158,13 +245,13 @@ void cutlass_sparse_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
245 auto b_ptr = static_cast<ElementAB*>(bt_nzs.data_ptr());
246 auto e_ptr = static_cast<ElementE*>(bt_meta.data_ptr());
247 typename GemmKernel::MainloopArguments mainloop_args{
161 - b_ptr, b_layout, a_ptr, stride_At, e_ptr, e_layout};
248 + b_ptr, b_layout, a_ptr, a_stride, e_ptr, e_layout};
249
250 auto c_ptr = static_cast<ElementD*>(out.data_ptr());
251 typename GemmKernel::EpilogueArguments epilogue_args{
252 Gemm::Epilogue::prepare_args(
253 std::forward<EpilogueArgs>(epilogue_params)...),
167 - c_ptr, stride_Dt, c_ptr, stride_Dt};
254 + c_ptr, c_stride, c_ptr, c_stride};
255
256 typename GemmKernel::Arguments args{cutlass::gemm::GemmUniversalMode::kGemm,
257 prob_shape, mainloop_args, epilogue_args};
@@ -185,6 +272,10 @@ void cutlass_sparse_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
272 CUTLASS_CHECK(status);
273 }
274
275 +//////////////////////////////////////////////////
276 +// Gemm Configs are defined below
277 +//////////////////////////////////////////////////
278 +
279 template <typename InType, typename OutType,
280 template <typename, typename, typename> typename Epilogue>
281 struct sm90_config_default {};
@@ -192,28 +283,25 @@ struct sm90_config_default {};
283 template <typename OutType,
284 template <typename, typename, typename> typename Epilogue>
285 struct sm90_config_default<half_t, OutType, Epilogue> {
195 - // M in (128, inf)
196 - using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecializedPingpong;
286 + using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecialized;
287 using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
288 using TileShape = Shape<_128, _128, _128>;
199 - using ClusterShape = Shape<_2, _1, _1>;
289 + using ClusterShape = Shape<_1, _1, _1>;
290 using Cutlass3xGemm =
291 cutlass_sparse_3x_gemm<half_t, OutType, Epilogue, TileShape, ClusterShape,
202 - KernelSchedule, EpilogueSchedule, float>;
292 + KernelSchedule, EpilogueSchedule>;
293 };
294
295 template <typename OutType,
296 template <typename, typename, typename> typename Epilogue>
297 struct sm90_config_default<cutlass::bfloat16_t, OutType, Epilogue> {
208 - // M in (128, inf)
209 - using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecializedPingpong;
298 + using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecialized;
299 using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
300 using TileShape = Shape<_128, _128, _128>;
212 - using ClusterShape = Shape<_2, _1, _1>;
301 + using ClusterShape = Shape<_1, _1, _1>;
302 using Cutlass3xGemm =
303 cutlass_sparse_3x_gemm<cutlass::bfloat16_t, OutType, Epilogue, TileShape,
215 - ClusterShape, KernelSchedule, EpilogueSchedule,
216 - float>;
304 + ClusterShape, KernelSchedule, EpilogueSchedule>;
305 };
306
307 //////////////////////// Cherry-Picking Kernels ////////////////////////
@@ -227,7 +315,7 @@ struct sm90_fp8_config_1 {
315 using ClusterShape = Shape<_8, _1, _1>;
316 using Cutlass3xGemm =
317 cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
230 - KernelSchedule, EpilogueSchedule, float>;
318 + KernelSchedule, EpilogueSchedule>;
319 };
320
321 template <typename InType, typename OutType,
@@ -242,7 +330,7 @@ struct sm90_fp8_config_2 {
330 using ClusterShape = Shape<_8, _1, _1>;
331 using Cutlass3xGemm =
332 cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
245 - KernelSchedule, EpilogueSchedule, float>;
333 + KernelSchedule, EpilogueSchedule>;
334 };
335
336 template <typename InType, typename OutType,
@@ -255,7 +343,7 @@ struct sm90_fp8_config_3 {
343 using ClusterShape = Shape<_1, _2, _1>;
344 using Cutlass3xGemm =
345 cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
258 - KernelSchedule, EpilogueSchedule, float>;
346 + KernelSchedule, EpilogueSchedule>;
347 };
348
349 template <typename InType, typename OutType,
@@ -269,7 +357,7 @@ struct sm90_fp8_config_4 {
357 using ClusterShape = Shape<_8, _1, _1>;
358 using Cutlass3xGemm =
359 cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
272 - KernelSchedule, EpilogueSchedule, float>;
360 + KernelSchedule, EpilogueSchedule>;
361 };
362
363 template <typename InType, typename OutType,
@@ -283,7 +371,7 @@ struct sm90_fp8_config_5 {
371 using ClusterShape = Shape<_8, _1, _1>;
372 using Cutlass3xGemm =
373 cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
286 - KernelSchedule, EpilogueSchedule, float>;
374 + KernelSchedule, EpilogueSchedule>;
375 };
376
377 template <typename InType, typename OutType,
@@ -296,7 +384,7 @@ struct sm90_fp8_config_6 {
384 using ClusterShape = Shape<_1, _2, _1>;
385 using Cutlass3xGemm =
386 cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
299 - KernelSchedule, EpilogueSchedule, float>;
387 + KernelSchedule, EpilogueSchedule>;
388 };
389
390 template <typename InType, typename OutType,
@@ -311,7 +399,7 @@ struct sm90_fp8_config_7 {
399 using ClusterShape = Shape<_1, _1, _1>;
400 using Cutlass3xGemm =
401 cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
314 - KernelSchedule, EpilogueSchedule, float>;
402 + KernelSchedule, EpilogueSchedule>;
403 };
404
405 template <typename InType, typename OutType,
@@ -326,7 +414,7 @@ struct sm90_fp8_config_8 {
414 using ClusterShape = Shape<_8, _1, _1>;
415 using Cutlass3xGemm =
416 cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
329 - KernelSchedule, EpilogueSchedule, float>;
417 + KernelSchedule, EpilogueSchedule>;
418 };
419 ////////////////////////////////////////////////////////////////////////
420
@@ -341,7 +429,7 @@ struct sm90_config_default<cutlass::float_e4m3_t, OutType, Epilogue> {
429 using Cutlass3xGemm =
430 cutlass_sparse_3x_gemm<cutlass::float_e4m3_t, OutType, Epilogue,
431 TileShape, ClusterShape, KernelSchedule,
344 - EpilogueSchedule, float>;
432 + EpilogueSchedule>;
433 };
434
435 template <typename InType, typename OutType,
@@ -355,12 +443,9 @@ struct sm90_fp8_config_M64 {
443 using TileShape = Shape<_64, _64, _256>;
444 using ClusterShape = Shape<_1, _1, _1>;
445
358 - using TileSchedule = cutlass::gemm::PersistentScheduler;
359 -
446 using Cutlass3xGemm =
447 cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
362 - KernelSchedule, EpilogueSchedule, float,
363 - TileSchedule>;
448 + KernelSchedule, EpilogueSchedule>;
449 };
450
451 template <typename InType, typename OutType,
@@ -374,12 +459,9 @@ struct sm90_fp8_config_M128 {
459 using TileShape = Shape<_64, _128, _256>;
460 using ClusterShape = Shape<_1, _1, _1>;
461
377 - using TileSchedule = cutlass::gemm::PersistentScheduler;
378 -
462 using Cutlass3xGemm =
463 cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
381 - KernelSchedule, EpilogueSchedule, float,
382 - TileSchedule>;
464 + KernelSchedule, EpilogueSchedule>;
465 };
466
467 template <typename InType, typename OutType,
@@ -394,12 +476,9 @@ struct sm90_fp8_config_M256 {
476 using TileShape = Shape<_128, _128, _256>;
477 using ClusterShape = Shape<_1, _1, _1>;
478
397 - using TileSchedule = cutlass::gemm::PersistentScheduler;
398 -
479 using Cutlass3xGemm =
480 cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
401 - KernelSchedule, EpilogueSchedule, float,
402 - TileSchedule>;
481 + KernelSchedule, EpilogueSchedule>;
482 };
483
484 template <typename InType, typename OutType,
@@ -414,12 +493,9 @@ struct sm90_fp8_config_M512 {
493 using TileShape = Shape<_128, _128, _256>;
494 using ClusterShape = Shape<_1, _1, _1>;
495
417 - using TileSchedule = cutlass::gemm::PersistentScheduler;
418 -
496 using Cutlass3xGemm =
497 cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
421 - KernelSchedule, EpilogueSchedule, float,
422 - TileSchedule>;
498 + KernelSchedule, EpilogueSchedule>;
499 };
500
501 template <typename OutType,
@@ -433,7 +509,7 @@ struct sm90_config_default<int8_t, OutType, Epilogue> {
509 using ClusterShape = Shape<_2, _1, _1>;
510 using Cutlass3xGemm =
511 cutlass_sparse_3x_gemm<int8_t, OutType, Epilogue, TileShape, ClusterShape,
436 - KernelSchedule, EpilogueSchedule, int32_t>;
512 + KernelSchedule, EpilogueSchedule>;
513 };
514
515 template <typename InType, typename OutType,
@@ -448,7 +524,7 @@ struct sm90_int8_config_M128 {
524 using ClusterShape = Shape<_2, _1, _1>;
525 using Cutlass3xGemm =
526 cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
451 - KernelSchedule, EpilogueSchedule, int32_t>;
527 + KernelSchedule, EpilogueSchedule>;
528 };
529
530 template <typename InType, typename OutType,
@@ -462,7 +538,7 @@ struct sm90_int8_config_M64 {
538 using ClusterShape = Shape<_1, _1, _1>;
539 using Cutlass3xGemm =
540 cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
465 - KernelSchedule, EpilogueSchedule, int32_t>;
541 + KernelSchedule, EpilogueSchedule>;
542 };
543
544 template <typename InType, typename OutType,
@@ -476,7 +552,7 @@ struct sm90_int8_config_M32_NBig {
552 using ClusterShape = Shape<_1, _4, _1>;
553 using Cutlass3xGemm =
554 cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
479 - KernelSchedule, EpilogueSchedule, int32_t>;
555 + KernelSchedule, EpilogueSchedule>;
556 };
557
558 template <typename InType, typename OutType,
@@ -490,7 +566,7 @@ struct sm90_int8_config_M32_NSmall {
566 using ClusterShape = Shape<_1, _8, _1>;
567 using Cutlass3xGemm =
568 cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
493 - KernelSchedule, EpilogueSchedule, int32_t>;
569 + KernelSchedule, EpilogueSchedule>;
570 };
571
496 -} // namespace
\ No newline at end of file
572 +} // namespace
csrc/sparse/cutlass/sparse_scaled_mm_entry.cu
+30
@@ -23,6 +23,9 @@ void cutlass_scaled_sparse_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
23 torch::Tensor const& a_scales,
24 torch::Tensor const& b_scales,
25 std::optional<torch::Tensor> const& bias);
26 +
27 +using CompressorResult = std::tuple<torch::Tensor, torch::Tensor>;
28 +CompressorResult cutlass_sparse_compress_sm90(torch::Tensor const& a);
29 #endif
30
31 void cutlass_scaled_sparse_mm(torch::Tensor& c, torch::Tensor const& a,
@@ -68,3 +71,30 @@ void cutlass_scaled_sparse_mm(torch::Tensor& c, torch::Tensor const& a,
71 "CUDA device capability: ",
72 version_num);
73 }
74 +
75 +std::vector<torch::Tensor> cutlass_sparse_compress(torch::Tensor const& a) {
76 + // Check for strides and alignment
77 + TORCH_CHECK(a.stride(1) == 1); // Row-major
78 + TORCH_CHECK(a.stride(0) % 8 == 0); // 8 Byte Alignment for Compression
79 +
80 + at::cuda::OptionalCUDAGuard const device_guard(device_of(a));
81 + int32_t version_num = get_sm_version_num();
82 +
83 + // Guard against compilation issues for sm90 kernels
84 +#if defined ENABLE_SPARSE_SCALED_MM_C3X && ENABLE_SPARSE_SCALED_MM_C3X
85 + if (version_num >= 90) {
86 + std::vector<torch::Tensor> result_tensors;
87 +
88 + auto [a_meta, a_nzs] = cutlass_sparse_compress_sm90(a);
89 + result_tensors.push_back(std::move(a_nzs));
90 + result_tensors.push_back(std::move(a_meta));
91 + return result_tensors;
92 + }
93 +#endif
94 +
95 + TORCH_CHECK_NOT_IMPLEMENTED(
96 + false,
97 + "No compiled cutlass_sparse_compress for a compute capability less than "
98 + "CUDA device capability: ",
99 + version_num);
100 +}
csrc/torch_bindings.cpp
+2 -4
@@ -348,10 +348,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
348 ops.impl("cutlass_scaled_sparse_mm", torch::kCUDA, &cutlass_scaled_sparse_mm);
349
350 // CUTLASS sparse matrix compressor
351 - ops.def(
352 - "cutlass_sparse_compress_entry(Tensor! a_nzs, Tensor! a_meta,"
353 - " Tensor a) -> bool");
354 - ops.impl("cutlass_sparse_compress_entry", &cutlass_sparse_compress_entry);
351 + ops.def("cutlass_sparse_compress(Tensor a) -> Tensor[]");
352 + ops.impl("cutlass_sparse_compress", &cutlass_sparse_compress);
353
354 // Mamba selective scan kernel
355 ops.def(
tests/kernels/test_cutlass_2of4_sparse.py
+66 -15
@@ -7,7 +7,6 @@ from typing import Tuple, Type
7
8 import pytest
9 import torch
10 -import torch.nn.functional as F
10
11 from vllm import _custom_ops as ops
12 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
@@ -55,11 +54,39 @@ def prune_to_2_4(tensor):
54 return pruned.reshape(original_shape)
55
56
57 +# This function checks that applying an identity matrix multiplication
58 +# to the compressed weights yields the original uncompressed weights.
59 +def check_compress_decompress_invariance(dtype: torch.dtype, b: torch.Tensor,
60 + b_compressed: torch.Tensor,
61 + b_metadata: torch.Tensor):
62 +
63 + # For float16 and bfloat16, cutlass_scaled_sparse_mm's output must be the
64 + # same dtype as its inputs. This line addresses that constraint while
65 + # arbitrarily using bfloat16 for the int8/fp8 cases.
66 + out_dtype = torch.float16 if dtype is torch.float16 else torch.bfloat16
67 +
68 + eye = torch.eye(b.shape[0], device='cuda', dtype=dtype)
69 + eye_scale = torch.ones(1, device='cuda', dtype=torch.float32)
70 + b_decomp = ops.cutlass_scaled_sparse_mm(eye,
71 + b_compressed,
72 + b_metadata,
73 + eye_scale,
74 + eye_scale,
75 + out_dtype=out_dtype)
76 +
77 + torch.testing.assert_close(b.to(dtype=out_dtype), b_decomp)
78 +
79 +
80 def make_rand_sparse_tensors(
81 dtype: torch.dtype, m: int, n: int, k: int
82 ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
61 - a = torch.randn((m, k), device='cuda') * 5
62 - b = torch.randn((n, k), device='cuda').t() * 5
83 + a = torch.randn((m, k), device='cuda')
84 + b = torch.randn((n, k), device='cuda').t()
85 +
86 + if dtype == torch.int8:
87 + # ensure A and B aren't all zeros after rounding
88 + a = a * 5.0
89 + b = b * 5.0
90
91 b = prune_to_2_4(b.t()).t()
92
@@ -75,6 +102,7 @@ def make_rand_sparse_tensors(
102 raise ValueError("unsupported dtype")
103
104 b_compressed, e = ops.cutlass_sparse_compress(b.t())
105 + check_compress_decompress_invariance(dtype, b, b_compressed, e)
106
107 # Compressed B, Metadata, Original A, B
108 return b_compressed, e, a, b
@@ -134,27 +162,37 @@ MNK_FACTORS = [
162
163
164 # Test working with a subset of A and B for sparse matmul
137 -@pytest.mark.skip(reason="2of4 sparse w16a16 CUTLASS produces bad output.")
165 @pytest.mark.skipif(not sparse_cutlass_supported(),
166 reason="Sparse CUTLASS is not supported on this GPU type.")
140 -@pytest.mark.parametrize("m, k, n", MNK_FACTORS)
167 +@pytest.mark.parametrize("m, n, k", MNK_FACTORS)
168 @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
142 -def test_cutlass_sparse_gemm(m: int, k: int, n: int, dtype: Type[torch.dtype]):
169 +@pytest.mark.parametrize("use_bias", [True, False])
170 +def test_cutlass_sparse_gemm(m: int, k: int, n: int, dtype: Type[torch.dtype],
171 + use_bias: bool):
172
173 # Create tensors
174 b_comp, e, a, b = make_rand_sparse_tensors(dtype, m, n, k)
175 scale_a = torch.ones((1, 1), device="cuda", dtype=torch.float32)
176 scale_b = torch.ones((1, 1), device="cuda", dtype=torch.float32)
177
178 + bias = torch.rand((n, ), device="cuda", dtype=dtype) if use_bias else None
179 +
180 out = ops.cutlass_scaled_sparse_mm(a,
181 b_comp,
182 e,
183 scale_a,
184 scale_b,
154 - out_dtype=dtype)
155 - baseline = F.linear(a, b.T)
185 + out_dtype=dtype,
186 + bias=bias)
187
157 - torch.testing.assert_close(out, baseline, rtol=1e-2, atol=1e-2)
188 + baseline = baseline_scaled_mm(a,
189 + b,
190 + scale_a,
191 + scale_b,
192 + out_dtype=dtype,
193 + bias=bias)
194 +
195 + torch.testing.assert_close(out, baseline, rtol=1e-2, atol=3e-1)
196
197
198 @pytest.mark.skipif(not sparse_cutlass_supported(),
@@ -162,27 +200,34 @@ def test_cutlass_sparse_gemm(m: int, k: int, n: int, dtype: Type[torch.dtype]):
200 @pytest.mark.parametrize("m, k, n", MNK_FACTORS)
201 @pytest.mark.skipif(not current_platform.has_device_capability(89),
202 reason="FP8 is not supported on this GPU type.")
165 -def test_cutlass_sparse_fp8_gemm(m: int, n: int, k: int):
203 +@pytest.mark.parametrize("use_bias", [True, False])
204 +def test_cutlass_sparse_fp8_gemm(m: int, n: int, k: int, use_bias: bool):
205
206 # Create tensors
207 b_comp, e, a, b = make_rand_sparse_tensors(torch.float8_e4m3fn, m, n, k)
208 scale_a = (torch.randn((1, 1), device="cuda", dtype=torch.float32))
209 scale_b = (torch.randn((1, 1), device="cuda", dtype=torch.float32))
210 + out_dtype = torch.bfloat16
211 +
212 + bias = torch.rand(
213 + (n, ), device="cuda", dtype=out_dtype) * 10 if use_bias else None
214
215 out = ops.cutlass_scaled_sparse_mm(a,
216 b_comp,
217 e,
218 scale_a,
219 scale_b,
177 - out_dtype=torch.bfloat16)
220 + out_dtype=out_dtype,
221 + bias=bias)
222
223 baseline = baseline_scaled_mm(a,
224 b,
225 scale_a,
226 scale_b,
183 - out_dtype=torch.bfloat16)
227 + out_dtype=out_dtype,
228 + bias=bias)
229
185 - torch.testing.assert_close(out, baseline, rtol=1e0, atol=2e0)
230 + torch.testing.assert_close(out, baseline, rtol=1e-2, atol=3e-1)
231
232
233 @pytest.mark.skipif(not sparse_cutlass_supported(),
@@ -198,18 +243,24 @@ def test_cutlass_sparse_int8_gemm(m: int, n: int, k: int, per_act_token: bool,
243 b_comp, e, a, b = make_rand_sparse_tensors(torch.int8, m, n, k)
244 scale_a = (torch.randn((1, 1), device="cuda", dtype=torch.float32))
245 scale_b = (torch.randn((1, 1), device="cuda", dtype=torch.float32))
246 + out_dtype = torch.bfloat16
247 +
248 + bias = torch.rand(
249 + (n, ), device="cuda", dtype=out_dtype) * 10 if use_bias else None
250
251 out = ops.cutlass_scaled_sparse_mm(a,
252 b_comp,
253 e,
254 scale_a,
255 scale_b,
207 - out_dtype=torch.bfloat16)
256 + out_dtype=out_dtype,
257 + bias=bias)
258
259 baseline = baseline_scaled_mm(a,
260 b,
261 scale_a,
262 scale_b,
213 - out_dtype=torch.bfloat16)
263 + out_dtype=out_dtype,
264 + bias=bias)
265
266 torch.testing.assert_close(out, baseline, rtol=1e0, atol=2e0)
vllm/_custom_ops.py
+2 -15
@@ -564,22 +564,9 @@ def cutlass_sparse_compress(a: torch.Tensor) \
564
565 # a_meta.dtype: torch.uint8 so elemsPerMetaElem = 8b / 2b_per_nz = 4
566 elemsPerMetaElem = 4
567 + assert (a.shape[1] % (2 * elemsPerMetaElem) == 0)
568
568 - m = a.shape[0]
569 - k = a.shape[1]
570 - assert (k % 2 == 0)
571 - a_nzs = torch.empty((m, k // 2), dtype=a.dtype, device=a.device)
572 - a_meta = torch.empty((m, k // 2 // elemsPerMetaElem),
573 - dtype=torch.uint8,
574 - device=a.device)
575 -
576 - if not (torch.ops._C.cutlass_sparse_compress_entry(a_nzs, a_meta, a)):
577 - raise ValueError
578 -
579 - assert (a_nzs.is_contiguous())
580 - assert (a_meta.is_contiguous())
581 -
582 - return a_nzs, a_meta
569 + return torch.ops._C.cutlass_sparse_compress(a)
570
571
572 def cutlass_scaled_sparse_mm(
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
-7
@@ -408,13 +408,6 @@ class CompressedTensorsConfig(QuantizationConfig):
408 if self.supports_cutlass_24(weight_quant=weight_quant,
409 input_quant=input_quant,
410 sparsity_scheme=sparsity_scheme):
411 - # FIXME(tlrmchlsmth): layers using W16A16 CUTLASS 2:4 sparse kernels
412 - # currently produce bad output in some cases
413 - if weight_quant is None:
414 - logger.warning_once(
415 - "CompressedTensors24 scheme is disabled for the w16a16 "
416 - "case. Falling back to UnquantizedLinearMethod")
417 - return None
411 # Have a valid sparsity scheme
412 # Validate layer is supported by Cutlass 2:4 Kernel
413 model_compression_config = (None if sparsity_scheme is None
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_24.py
+7 -2
@@ -64,7 +64,6 @@ class CompressedTensors24(CompressedTensorsScheme):
64 "Sparse CUTLASS not supported. vLLM must be built with "
65 "CUDA 12.2 or later to use this feature")
66
67 - self.output_dtype = params_dtype
67 layer.logical_widths = output_partition_sizes
68 layer.input_size = input_size
69 layer.input_size_per_partition = input_size_per_partition
@@ -205,6 +204,11 @@ class CompressedTensors24(CompressedTensorsScheme):
204 layer.weight_scale = torch.nn.Parameter(
205 layer.weight_scale.data, requires_grad=False)
206
207 + # Set all negative zero values to 0 prior to compression
208 + if (layer.weight.dtype.is_floating_point
209 + and layer.weight.dtype.itemsize >= 2):
210 + layer.weight.data[layer.weight.data == -0.0] = 0.0
211 +
212 w_compressed, meta = ops.cutlass_sparse_compress(layer.weight.data)
213 layer.weight = torch.nn.Parameter(w_compressed, requires_grad=False)
214 layer.meta = torch.nn.Parameter(meta, requires_grad=False)
@@ -254,9 +258,10 @@ class CompressedTensors24(CompressedTensorsScheme):
258 bt_meta=layer.meta,
259 scale_a=input_scale,
260 scale_b=layer.weight_scale,
257 - out_dtype=self.output_dtype,
261 + out_dtype=x.dtype,
262 bias=bias,
263 )
264 +
265 assert out.is_contiguous()
266 return out
267