@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 85657b5607
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Kernel] Factor out epilogues from cutlass kernels (#5391)

Co-authored-by: Michael Goin <michael@neuralmagic.com> Co-authored-by: youkaichao <youkaichao@gmail.com> Co-authored-by: zifeitong <zifei.tong@parasail.io> Co-authored-by: Robert Shaw <114415538+robertgshaw2-neuralmagic@users.noreply.github.com>

Tyler Michael Smith committed Jun 13, 2024 at 14:22 UTC 85657b56071b7c21586d88389c6e817f11c69e04
12 files changed +274 -232
CMakeLists.txt
+4 -4
@@ -179,9 +179,9 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
179 "csrc/quantization/gptq_marlin/gptq_marlin.cu"
180 "csrc/quantization/gptq_marlin/gptq_marlin_repack.cu"
181 "csrc/custom_all_reduce.cu"
182 - "csrc/quantization/cutlass_w8a8/scaled_mm_dq_entry.cu"
183 - "csrc/quantization/cutlass_w8a8/scaled_mm_dq_c2x.cu"
184 - "csrc/quantization/cutlass_w8a8/scaled_mm_dq_c3x.cu")
182 + "csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu"
183 + "csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cu"
184 + "csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu")
185
186 #
187 # The CUTLASS kernels for Hopper require sm90a to be enabled.
@@ -189,7 +189,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
189 # That adds an extra 17MB to compiled binary, so instead we selectively enable it.
190 if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0)
191 set_source_files_properties(
192 - "csrc/quantization/cutlass_w8a8/scaled_mm_dq_c3x.cu"
192 + "csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu"
193 PROPERTIES
194 COMPILE_FLAGS
195 "-gencode arch=compute_90a,code=sm_90a")
benchmarks/cutlass_benchmarks/w8a8_benchmarks.py
+1 -5
@@ -76,11 +76,7 @@ def pytorch_fp8_impl_fast_accum(a: torch.tensor, b: torch.tensor,
76 def cutlass_impl(a: torch.tensor, b: torch.tensor, scale_a: torch.tensor,
77 scale_b: torch.tensor,
78 out_dtype: torch.dtype) -> torch.tensor:
79 - return ops.cutlass_scaled_mm_dq(a,
80 - b,
81 - scale_a,
82 - scale_b,
83 - out_dtype=out_dtype)
79 + return ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype=out_dtype)
80
81
82 # bench
csrc/ops.h
+3 -3
@@ -90,9 +90,9 @@ torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
90 int64_t size_k, int64_t size_n,
91 int64_t num_bits);
92
93 -void cutlass_scaled_mm_dq(torch::Tensor& out, torch::Tensor const& a,
94 - torch::Tensor const& b, torch::Tensor const& a_scales,
95 - torch::Tensor const& b_scales);
93 +void cutlass_scaled_mm(torch::Tensor& out, torch::Tensor const& a,
94 + torch::Tensor const& b, torch::Tensor const& a_scales,
95 + torch::Tensor const& b_scales);
96
97 #endif
98
csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cu renamed
+111 -88
@@ -29,21 +29,14 @@
29 using namespace cute;
30
31 /*
32 - This defines a quantized GEMM operation with dequantized output, similar to
33 - torch._scaled_mm. It is defined using the CUTLASS 2.x API, and is used for
32 + This file defines quantized GEMM operations using the CUTLASS 2.x API, for
33 NVIDIA GPUs with SM versions prior to sm90 (Hopper).
34
36 - A and B may be both either int8 or fp8_e4m3. A can be quantized per-tensor or
37 - per-row. B can be quantized per-tensor or per-column.
38 - Any combination of per-tensor and per-row or column is supported.
39 - A and B must have symmetric quantization (zero point == 0).
40 -
41 - So the GEMM operation is D = (a_scales * A) (b_scales * B), where the
42 - scales are applied elementwise with numpy-style broadcasting.
43 -
44 - ScaleA and ScaleB define the epilogue functions that apply the scales for
45 - the A and B operands respectively. These scales may be either per-tensor or
46 - per row or column.
35 + Epilogue functions can be defined to post-process the output before it is
36 + written to GPU memory.
37 + Epilogues must contain a public type named EVTCompute of type Sm80EVT,
38 + as well as a static prepare_args function that constructs an
39 + EVTCompute::Arguments struct.
40 */
41
42 namespace {
@@ -83,27 +76,25 @@ struct enable_sm89_to_sm90 : Kernel {
76 }
77 };
78
86 -template <typename Arch, template <typename> typename ArchGuard,
87 - typename ElementAB_, typename ElementD_, typename TileShape,
88 - typename WarpShape, typename InstructionShape, int32_t MainLoopStages>
89 -struct cutlass_2x_gemm {
90 - using ElementAB = ElementAB_;
91 - using ElementD = ElementD_;
92 -
93 - using ElementAcc =
94 - typename std::conditional<std::is_same_v<ElementAB, int8_t>, int32_t,
95 - float>::type;
79 +/*
80 + This epilogue function defines a quantized GEMM operation similar to
81 + torch._scaled_mm.
82
97 - using Operator =
98 - typename std::conditional<std::is_same_v<ElementAB, int8_t>,
99 - cutlass::arch::OpMultiplyAddSaturate,
100 - cutlass::arch::OpMultiplyAdd>::type;
83 + A and B may be both either int8 or fp8_e4m3. A can be quantized per-tensor or
84 + per-row. B can be quantized per-tensor or per-column.
85 + Any combination of per-tensor and per-row or column is supported.
86 + A and B must have symmetric quantization (zero point == 0).
87
102 - using OutputTileThreadMap =
103 - cutlass::epilogue::threadblock::OutputTileThreadLayout<
104 - TileShape, WarpShape, float, 4, 1 /* epilogue stages */
105 - >;
88 + So the GEMM operation is D = (a_scales * A) (b_scales * B), where the
89 + scales are applied elementwise with numpy-style broadcasting.
90
91 + ScaleA and ScaleB define the epilogue functions that apply the scales for
92 + the A and B operands respectively. These scales may be either per-tensor or
93 + per row or column.
94 +*/
95 +template <typename ElementD, typename OutputTileThreadMap>
96 +struct ScaledEpilogue {
97 + private:
98 using Accum = cutlass::epilogue::threadblock::VisitorAccFetch;
99
100 using ScaleA = cutlass::epilogue::threadblock::VisitorColOrScalarBroadcast<
@@ -123,14 +114,56 @@ struct cutlass_2x_gemm {
114 cutlass::multiplies, ElementD, float,
115 cutlass::FloatRoundStyle::round_to_nearest>;
116
126 - using EVTCompute1 =
117 + public:
118 + using EVTCompute =
119 cutlass::epilogue::threadblock::Sm80EVT<Compute1, ScaleA, EVTCompute0>;
120 + using ArgumentType = typename EVTCompute::Arguments;
121 +
122 + static ArgumentType prepare_args(torch::Tensor const& a_scales,
123 + torch::Tensor const& b_scales) {
124 + using ScaleAArgs = typename ScaleA::Arguments;
125 + using ScaleBArgs = typename ScaleB::Arguments;
126 +
127 + ScaleBArgs b_args{b_scales.data_ptr<float>(), b_scales.numel() != 1, {}};
128 + ScaleAArgs a_args{a_scales.data_ptr<float>(), a_scales.numel() != 1, {}};
129 +
130 + typename EVTCompute0::Arguments evt0_compute_args{b_args};
131 +
132 + typename EVTCompute::Arguments evt_compute_args{a_args, evt0_compute_args};
133 + return evt_compute_args;
134 + }
135 +};
136 +
137 +template <typename Arch, template <typename> typename ArchGuard,
138 + typename ElementAB_, typename ElementD_,
139 + template <typename, typename> typename Epilogue_, typename TileShape,
140 + typename WarpShape, typename InstructionShape, int32_t MainLoopStages>
141 +struct cutlass_2x_gemm {
142 + using ElementAB = ElementAB_;
143 + using ElementD = ElementD_;
144 +
145 + using ElementAcc =
146 + typename std::conditional<std::is_same_v<ElementAB, int8_t>, int32_t,
147 + float>::type;
148 +
149 + using Operator =
150 + typename std::conditional<std::is_same_v<ElementAB, int8_t>,
151 + cutlass::arch::OpMultiplyAddSaturate,
152 + cutlass::arch::OpMultiplyAdd>::type;
153 +
154 + using OutputTileThreadMap =
155 + cutlass::epilogue::threadblock::OutputTileThreadLayout<
156 + TileShape, WarpShape, float, 4, 1 /* epilogue stages */
157 + >;
158 +
159 + using Epilogue = Epilogue_<ElementD, OutputTileThreadMap>;
160 + using EVTCompute = typename Epilogue::EVTCompute;
161
162 using D = cutlass::epilogue::threadblock::VisitorAuxStore<
163 OutputTileThreadMap, ElementD, cutlass::FloatRoundStyle::round_to_nearest,
164 Stride<int64_t, Int<1>, Int<0>>>;
165
133 - using EVTD = cutlass::epilogue::threadblock::Sm80EVT<D, EVTCompute1>;
166 + using EVTD = cutlass::epilogue::threadblock::Sm80EVT<D, EVTCompute>;
167
168 // clang-format off
169 using RowMajor = typename cutlass::layout::RowMajor;
@@ -153,11 +186,10 @@ struct cutlass_2x_gemm {
186 using Op = cutlass::gemm::device::GemmUniversalAdapter<KernelType>;
187 };
188
156 -template <typename Gemm>
157 -void cutlass_scaled_mm_dq_dispatcher(torch::Tensor& out, torch::Tensor const& a,
158 - torch::Tensor const& b,
159 - torch::Tensor const& a_scales,
160 - torch::Tensor const& b_scales) {
189 +template <typename Gemm, typename... EpilogueArgs>
190 +void cutlass_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
191 + torch::Tensor const& b,
192 + EpilogueArgs&&... epilogue_params) {
193 using ElementAB = typename Gemm::ElementAB;
194 using ElementD = typename Gemm::ElementD;
195
@@ -177,23 +209,14 @@ void cutlass_scaled_mm_dq_dispatcher(torch::Tensor& out, torch::Tensor const& a,
209 auto b_ptr = static_cast<ElementAB const*>(b.data_ptr());
210 auto c_ptr = static_cast<ElementD*>(out.data_ptr());
211
180 - auto a_scales_ptr = a_scales.data_ptr<float>();
181 - auto b_scales_ptr = b_scales.data_ptr<float>();
182 -
183 - using ScaleAArgs = typename Gemm::ScaleA::Arguments;
184 - using ScaleBArgs = typename Gemm::ScaleB::Arguments;
185 -
186 - ScaleBArgs b_args{b_scales.data_ptr<float>(), b_scales.numel() != 1, {}};
187 - ScaleAArgs a_args{a_scales.data_ptr<float>(), a_scales.numel() != 1, {}};
188 -
189 - typename Gemm::EVTCompute0::Arguments evt0_compute_args{b_args};
190 -
191 - typename Gemm::EVTCompute1::Arguments evt1_compute_args{a_args,
192 - evt0_compute_args};
212 typename Gemm::D::Arguments d_args{c_ptr, c_stride};
213
214 + using Epilogue = typename Gemm::Epilogue;
215 + auto evt_args =
216 + Epilogue::prepare_args(std::forward<EpilogueArgs>(epilogue_params)...);
217 +
218 typename Gemm::EVTD::Arguments epilogue_args{
196 - evt1_compute_args,
219 + evt_args,
220 d_args,
221 };
222
@@ -229,10 +252,10 @@ void cutlass_scaled_mm_dq_dispatcher(torch::Tensor& out, torch::Tensor const& a,
252
253 } // namespace
254
232 -void cutlass_scaled_mm_dq_sm75(torch::Tensor& out, torch::Tensor const& a,
233 - torch::Tensor const& b,
234 - torch::Tensor const& a_scales,
235 - torch::Tensor const& b_scales) {
255 +void cutlass_scaled_mm_sm75(torch::Tensor& out, torch::Tensor const& a,
256 + torch::Tensor const& b,
257 + torch::Tensor const& a_scales,
258 + torch::Tensor const& b_scales) {
259 TORCH_CHECK(a.dtype() == torch::kInt8);
260 TORCH_CHECK(b.dtype() == torch::kInt8);
261 TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
@@ -243,23 +266,23 @@ void cutlass_scaled_mm_dq_sm75(torch::Tensor& out, torch::Tensor const& a,
266 using InstructionShape = typename cutlass::gemm::GemmShape<8, 8, 16>;
267
268 if (out.dtype() == torch::kBFloat16) {
246 - return cutlass_scaled_mm_dq_dispatcher<cutlass_2x_gemm<
269 + return cutlass_gemm_caller<cutlass_2x_gemm<
270 cutlass::arch::Sm75, enable_sm75_to_sm80, int8_t, cutlass::bfloat16_t,
248 - TileShape, WarpShape, InstructionShape, 2>>(out, a, b, a_scales,
249 - b_scales);
271 + ScaledEpilogue, TileShape, WarpShape, InstructionShape, 2>>(
272 + out, a, b, a_scales, b_scales);
273 } else {
274 TORCH_CHECK(out.dtype() == torch::kFloat16);
252 - return cutlass_scaled_mm_dq_dispatcher<cutlass_2x_gemm<
275 + return cutlass_gemm_caller<cutlass_2x_gemm<
276 cutlass::arch::Sm75, enable_sm75_to_sm80, int8_t, cutlass::half_t,
254 - TileShape, WarpShape, InstructionShape, 2>>(out, a, b, a_scales,
255 - b_scales);
277 + ScaledEpilogue, TileShape, WarpShape, InstructionShape, 2>>(
278 + out, a, b, a_scales, b_scales);
279 }
280 }
281
259 -void cutlass_scaled_mm_dq_sm80(torch::Tensor& out, torch::Tensor const& a,
260 - torch::Tensor const& b,
261 - torch::Tensor const& a_scales,
262 - torch::Tensor const& b_scales) {
282 +void cutlass_scaled_mm_sm80(torch::Tensor& out, torch::Tensor const& a,
283 + torch::Tensor const& b,
284 + torch::Tensor const& a_scales,
285 + torch::Tensor const& b_scales) {
286 TORCH_CHECK(a.dtype() == torch::kInt8);
287 TORCH_CHECK(b.dtype() == torch::kInt8);
288 TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
@@ -270,23 +293,23 @@ void cutlass_scaled_mm_dq_sm80(torch::Tensor& out, torch::Tensor const& a,
293 using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
294
295 if (out.dtype() == torch::kBFloat16) {
273 - return cutlass_scaled_mm_dq_dispatcher<cutlass_2x_gemm<
296 + return cutlass_gemm_caller<cutlass_2x_gemm<
297 cutlass::arch::Sm80, enable_sm80_to_sm89, int8_t, cutlass::bfloat16_t,
275 - TileShape, WarpShape, InstructionShape, 5>>(out, a, b, a_scales,
276 - b_scales);
298 + ScaledEpilogue, TileShape, WarpShape, InstructionShape, 5>>(
299 + out, a, b, a_scales, b_scales);
300 } else {
301 TORCH_CHECK(out.dtype() == torch::kFloat16);
279 - return cutlass_scaled_mm_dq_dispatcher<cutlass_2x_gemm<
302 + return cutlass_gemm_caller<cutlass_2x_gemm<
303 cutlass::arch::Sm80, enable_sm80_to_sm89, int8_t, cutlass::half_t,
281 - TileShape, WarpShape, InstructionShape, 5>>(out, a, b, a_scales,
282 - b_scales);
304 + ScaledEpilogue, TileShape, WarpShape, InstructionShape, 5>>(
305 + out, a, b, a_scales, b_scales);
306 }
307 }
308
286 -void cutlass_scaled_mm_dq_sm89(torch::Tensor& out, torch::Tensor const& a,
287 - torch::Tensor const& b,
288 - torch::Tensor const& a_scales,
289 - torch::Tensor const& b_scales) {
309 +void cutlass_scaled_mm_sm89(torch::Tensor& out, torch::Tensor const& a,
310 + torch::Tensor const& b,
311 + torch::Tensor const& a_scales,
312 + torch::Tensor const& b_scales) {
313 using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
314 using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
315 using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
@@ -298,32 +321,32 @@ void cutlass_scaled_mm_dq_sm89(torch::Tensor& out, torch::Tensor const& a,
321 TORCH_CHECK(b.dtype() == torch::kInt8);
322
323 if (out.dtype() == torch::kBFloat16) {
301 - return cutlass_scaled_mm_dq_dispatcher<cutlass_2x_gemm<
324 + return cutlass_gemm_caller<cutlass_2x_gemm<
325 cutlass::arch::Sm89, enable_sm89_to_sm90, int8_t, cutlass::bfloat16_t,
303 - TileShape, WarpShape, InstructionShape, 5>>(out, a, b, a_scales,
304 - b_scales);
326 + ScaledEpilogue, TileShape, WarpShape, InstructionShape, 5>>(
327 + out, a, b, a_scales, b_scales);
328 } else {
329 assert(out.dtype() == torch::kFloat16);
307 - return cutlass_scaled_mm_dq_dispatcher<cutlass_2x_gemm<
330 + return cutlass_gemm_caller<cutlass_2x_gemm<
331 cutlass::arch::Sm89, enable_sm89_to_sm90, int8_t, cutlass::half_t,
309 - TileShape, WarpShape, InstructionShape, 5>>(out, a, b, a_scales,
310 - b_scales);
332 + ScaledEpilogue, TileShape, WarpShape, InstructionShape, 5>>(
333 + out, a, b, a_scales, b_scales);
334 }
335 } else {
336 TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
337 TORCH_CHECK(b.dtype() == torch::kFloat8_e4m3fn);
338
339 if (out.dtype() == torch::kBFloat16) {
317 - return cutlass_scaled_mm_dq_dispatcher<cutlass_2x_gemm<
340 + return cutlass_gemm_caller<cutlass_2x_gemm<
341 cutlass::arch::Sm89, enable_sm89_to_sm90, cutlass::float_e4m3_t,
319 - cutlass::bfloat16_t, TileShape, WarpShape, InstructionShape, 5>>(
320 - out, a, b, a_scales, b_scales);
342 + cutlass::bfloat16_t, ScaledEpilogue, TileShape, WarpShape,
343 + InstructionShape, 5>>(out, a, b, a_scales, b_scales);
344 } else {
345 TORCH_CHECK(out.dtype() == torch::kFloat16);
323 - return cutlass_scaled_mm_dq_dispatcher<cutlass_2x_gemm<
346 + return cutlass_gemm_caller<cutlass_2x_gemm<
347 cutlass::arch::Sm89, enable_sm89_to_sm90, cutlass::float_e4m3_t,
325 - cutlass::half_t, TileShape, WarpShape, InstructionShape, 5>>(
326 - out, a, b, a_scales, b_scales);
348 + cutlass::half_t, ScaledEpilogue, TileShape, WarpShape,
349 + InstructionShape, 5>>(out, a, b, a_scales, b_scales);
350 }
351 }
352 }
csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu renamed
+109 -85
@@ -32,21 +32,14 @@
32 using namespace cute;
33
34 /*
35 - This defines a quantized GEMM operation with dequantized output, similar to
36 - torch._scaled_mm. It is defined using the CUTLASS 3.x API, and is used for
35 + This file defines quantized GEMM operations using the CUTLASS 3.x API, for
36 NVIDIA GPUs with sm90a (Hopper) or later.
37
39 - A and B may be both either int8 or fp8_e4m3. A can be quantized per-tensor or
40 - per-row. B can be quantized per-tensor or per-column.
41 - Any combination of per-tensor and per-row or column is supported.
42 - A and B must have symmetric quantization (zero point == 0).
43 -
44 - So the GEMM operation is D = (a_scales * A) (b_scales * B), where the
45 - scales are applied elementwise with numpy-style broadcasting.
46 -
47 - ScaleA and ScaleB define the epilogue functions that apply the scales for
48 - the A and B operands respectively. These scales may be either per-tensor or
49 - per row or column.
38 + Epilogue functions can be defined to post-process the output before it is
39 + written to GPU memory.
40 + Epilogues must contain a public type named EVTCompute of type Sm90EVT,
41 + as well as a static prepare_args function that constructs an
42 + EVTCompute::Arguments struct.
43 */
44
45 namespace {
@@ -71,21 +64,25 @@ struct enable_sm90_or_later : Kernel {
64 }
65 };
66
74 -template <typename ElementAB_, typename ElementD_, typename TileShape,
75 - typename ClusterShape, typename KernelSchedule,
76 - typename EpilogueSchedule>
77 -struct cutlass_3x_gemm {
78 - using ElementAB = ElementAB_;
79 - using ElementD = ElementD_;
80 - using ElementAcc =
81 - typename std::conditional<std::is_same_v<ElementAB, int8_t>, int32_t,
82 - float>::type;
67 +/*
68 + This epilogue function defines a quantized GEMM operation similar to
69 + torch.scaled_mm_.
70
84 - using EpilogueDescriptor =
85 - cutlass::epilogue::collective::detail::EpilogueDescriptor<
86 - TileShape, cutlass::epilogue::collective::EpilogueTileAuto, ElementD,
87 - ElementD, EpilogueSchedule>;
71 + A and B may be both either int8 or fp8_e4m3. A can be
72 + quantized per-tensor or per-row. B can be quantized per-tensor or per-column.
73 + Any combination of per-tensor and per-row or column is supported.
74 + A and B must have symmetric quantization (zero point == 0).
75
76 + So the GEMM operation is D = (a_scales * A) (b_scales * B), where the
77 + scales are applied elementwise with numpy-style broadcasting.
78 +
79 + ScaleA and ScaleB define the epilogue functions that apply the scales for
80 + the A and B operands respectively. These scales may be either per-tensor or
81 + per row or column.
82 +*/
83 +template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
84 +struct ScaledEpilogue {
85 + private:
86 using Accum = cutlass::epilogue::fusion::Sm90AccFetch;
87
88 using ScaleA = cutlass::epilogue::fusion::Sm90ColOrScalarBroadcast<
@@ -111,19 +108,53 @@ struct cutlass_3x_gemm {
108 cutlass::multiplies, ElementD, float,
109 cutlass::FloatRoundStyle::round_to_nearest>;
110
114 - using EVTCompute1 =
111 + public:
112 + using EVTCompute =
113 cutlass::epilogue::fusion::Sm90EVT<Compute1, ScaleA, EVTCompute0>;
114 + using ArgumentType = typename EVTCompute::Arguments;
115 +
116 + static ArgumentType prepare_args(torch::Tensor const& a_scales,
117 + torch::Tensor const& b_scales) {
118 + using ScaleA_Args = typename ScaleA::Arguments;
119 + using ScaleB_Args = typename ScaleB::Arguments;
120 +
121 + ScaleA_Args a_args{a_scales.data_ptr<float>(), a_scales.numel() != 1, {}};
122 + ScaleB_Args b_args{b_scales.data_ptr<float>(), b_scales.numel() != 1, {}};
123 +
124 + return ArgumentType{a_args, {b_args}};
125 + }
126 +};
127 +
128 +template <typename ElementAB_, typename ElementD_,
129 + template <typename, typename, typename> typename Epilogue_,
130 + typename TileShape, typename ClusterShape, typename KernelSchedule,
131 + typename EpilogueSchedule>
132 +struct cutlass_3x_gemm {
133 + using ElementAB = ElementAB_;
134 + using ElementD = ElementD_;
135 + using ElementAcc =
136 + typename std::conditional<std::is_same_v<ElementAB, int8_t>, int32_t,
137 + float>::type;
138 +
139 + using EpilogueDescriptor =
140 + cutlass::epilogue::collective::detail::EpilogueDescriptor<
141 + TileShape, cutlass::epilogue::collective::EpilogueTileAuto, ElementD,
142 + ElementD, EpilogueSchedule>;
143 +
144 + using Epilogue = Epilogue_<ElementAcc, ElementD, EpilogueDescriptor>;
145
146 using StrideD = Stride<int64_t, Int<1>, Int<0>>;
147 using ElementC = void;
148 using StrideC = StrideD;
149
150 + using EVTCompute = typename Epilogue::EVTCompute;
151 +
152 using CollectiveEpilogue =
153 typename cutlass::epilogue::collective::CollectiveBuilder<
154 cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp, TileShape,
155 ClusterShape, cutlass::epilogue::collective::EpilogueTileAuto,
156 ElementAcc, float, ElementC, StrideC, 4, ElementD, StrideD, 4,
126 - EpilogueSchedule, EVTCompute1>::CollectiveOp;
157 + EpilogueSchedule, EVTCompute>::CollectiveOp;
158
159 static constexpr size_t CEStorageSize =
160 sizeof(typename CollectiveEpilogue::SharedStorage);
@@ -148,11 +179,10 @@ struct cutlass_3x_gemm {
179 struct GemmKernel : public KernelType {};
180 };
181
151 -template <typename Gemm>
152 -void cutlass_scaled_mm_dq_dispatcher(torch::Tensor& out, torch::Tensor const& a,
153 - torch::Tensor const& b,
154 - torch::Tensor const& a_scales,
155 - torch::Tensor const& b_scales) {
182 +template <typename Gemm, typename... EpilogueArgs>
183 +void cutlass_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
184 + torch::Tensor const& b,
185 + EpilogueArgs&&... epilogue_params) {
186 using ElementAB = typename Gemm::ElementAB;
187 using ElementD = typename Gemm::ElementD;
188
@@ -182,19 +212,13 @@ void cutlass_scaled_mm_dq_dispatcher(torch::Tensor& out, torch::Tensor const& a,
212
213 auto c_ptr = static_cast<ElementD*>(out.data_ptr());
214 typename GemmKernel::EpilogueArguments epilogue_args{
185 - {}, c_ptr, c_stride, c_ptr, c_stride};
215 + Gemm::Epilogue::prepare_args(
216 + std::forward<EpilogueArgs>(epilogue_params)...),
217 + c_ptr, c_stride, c_ptr, c_stride};
218
219 typename GemmKernel::Arguments args{cutlass::gemm::GemmUniversalMode::kGemm,
220 prob_shape, mainloop_args, epilogue_args};
221
190 - using ScaleA_Args = typename Gemm::ScaleA::Arguments;
191 - using ScaleB_Args = typename Gemm::ScaleB::Arguments;
192 -
193 - ScaleA_Args a_args{a_scales.data_ptr<float>(), a_scales.numel() != 1, {}};
194 - ScaleB_Args b_args{b_scales.data_ptr<float>(), b_scales.numel() != 1, {}};
195 -
196 - args.epilogue.thread = {a_args, {b_args}};
197 -
222 // Launch the CUTLASS GEMM kernel.
223 using GemmOp = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
224 GemmOp gemm_op;
@@ -209,7 +233,8 @@ void cutlass_scaled_mm_dq_dispatcher(torch::Tensor& out, torch::Tensor const& a,
233 CUTLASS_CHECK(status);
234 }
235
212 -template <typename InType, typename OutType, int32_t M>
236 +template <typename InType, typename OutType,
237 + template <typename, typename, typename> typename Epilogue, int32_t M>
238 struct sm90_fp8_config {
239 static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
240 using KernelSchedule =
@@ -219,12 +244,13 @@ struct sm90_fp8_config {
244 using ClusterShape = Shape<_2, _1, _1>;
245
246 using Cutlass3xGemm =
222 - cutlass_3x_gemm<InType, OutType, TileShape, ClusterShape, KernelSchedule,
223 - EpilogueSchedule>;
247 + cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
248 + KernelSchedule, EpilogueSchedule>;
249 };
250
226 -template <typename InType, typename OutType>
227 -struct sm90_fp8_config<InType, OutType, 128> {
251 +template <typename InType, typename OutType,
252 + template <typename, typename, typename> typename Epilogue>
253 +struct sm90_fp8_config<InType, OutType, Epilogue, 128> {
254 static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
255 using KernelSchedule =
256 cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum;
@@ -233,12 +259,13 @@ struct sm90_fp8_config<InType, OutType, 128> {
259 using ClusterShape = Shape<_2, _1, _1>;
260
261 using Cutlass3xGemm =
236 - cutlass_3x_gemm<InType, OutType, TileShape, ClusterShape, KernelSchedule,
237 - EpilogueSchedule>;
262 + cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
263 + KernelSchedule, EpilogueSchedule>;
264 };
265
240 -template <typename InType, typename OutType>
241 -struct sm90_fp8_config<InType, OutType, 64> {
266 +template <typename InType, typename OutType,
267 + template <typename, typename, typename> typename Epilogue>
268 +struct sm90_fp8_config<InType, OutType, Epilogue, 64> {
269 static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
270 using KernelSchedule =
271 cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum;
@@ -247,30 +274,28 @@ struct sm90_fp8_config<InType, OutType, 64> {
274 using ClusterShape = Shape<_1, _8, _1>;
275
276 using Cutlass3xGemm =
250 - cutlass_3x_gemm<InType, OutType, TileShape, ClusterShape, KernelSchedule,
251 - EpilogueSchedule>;
277 + cutlass_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
278 + KernelSchedule, EpilogueSchedule>;
279 };
280
281 } // namespace
282
256 -template <typename InType, typename OutType>
257 -void cutlass_scaled_mm_dq_sm90_fp8_dispatch(torch::Tensor& out,
258 - torch::Tensor const& a,
259 - torch::Tensor const& b,
260 - torch::Tensor const& a_scales,
261 - torch::Tensor const& b_scales) {
283 +template <typename InType, typename OutType,
284 + template <typename, typename, typename> typename Epilogue,
285 + typename... EpilogueArgs>
286 +void cutlass_gemm_sm90_fp8_dispatch(torch::Tensor& out, torch::Tensor const& a,
287 + torch::Tensor const& b,
288 + EpilogueArgs&&... args) {
289 static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
290 TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
291 TORCH_CHECK(b.dtype() == torch::kFloat8_e4m3fn);
265 - TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
266 - TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
292
293 using Cutlass3xGemmDefault =
269 - typename sm90_fp8_config<InType, OutType, 0>::Cutlass3xGemm;
294 + typename sm90_fp8_config<InType, OutType, Epilogue, 0>::Cutlass3xGemm;
295 using Cutlass3xGemmM64 =
271 - typename sm90_fp8_config<InType, OutType, 64>::Cutlass3xGemm;
296 + typename sm90_fp8_config<InType, OutType, Epilogue, 64>::Cutlass3xGemm;
297 using Cutlass3xGemmM128 =
273 - typename sm90_fp8_config<InType, OutType, 128>::Cutlass3xGemm;
298 + typename sm90_fp8_config<InType, OutType, Epilogue, 128>::Cutlass3xGemm;
299
300 uint32_t const m = a.size(0);
301 uint32_t const mp2 =
@@ -278,23 +303,23 @@ void cutlass_scaled_mm_dq_sm90_fp8_dispatch(torch::Tensor& out,
303
304 if (mp2 <= 64) {
305 // m in [1, 64]
281 - return cutlass_scaled_mm_dq_dispatcher<Cutlass3xGemmM64>(
282 - out, a, b, a_scales, b_scales);
306 + return cutlass_gemm_caller<Cutlass3xGemmM64>(
307 + out, a, b, std::forward<EpilogueArgs>(args)...);
308 } else if (mp2 <= 128) {
309 // m in (64, 128]
285 - return cutlass_scaled_mm_dq_dispatcher<Cutlass3xGemmM128>(
286 - out, a, b, a_scales, b_scales);
310 + return cutlass_gemm_caller<Cutlass3xGemmM128>(
311 + out, a, b, std::forward<EpilogueArgs>(args)...);
312 } else {
313 // m in (128, inf)
289 - return cutlass_scaled_mm_dq_dispatcher<Cutlass3xGemmDefault>(
290 - out, a, b, a_scales, b_scales);
314 + return cutlass_gemm_caller<Cutlass3xGemmDefault>(
315 + out, a, b, std::forward<EpilogueArgs>(args)...);
316 }
317 }
318
294 -void cutlass_scaled_mm_dq_sm90(torch::Tensor& out, torch::Tensor const& a,
295 - torch::Tensor const& b,
296 - torch::Tensor const& a_scales,
297 - torch::Tensor const& b_scales) {
319 +void cutlass_scaled_mm_sm90(torch::Tensor& out, torch::Tensor const& a,
320 + torch::Tensor const& b,
321 + torch::Tensor const& a_scales,
322 + torch::Tensor const& b_scales) {
323 TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
324 TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
325
@@ -308,16 +333,15 @@ void cutlass_scaled_mm_dq_sm90(torch::Tensor& out, torch::Tensor const& a,
333 using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
334
335 if (out.dtype() == torch::kBFloat16) {
311 - return cutlass_scaled_mm_dq_dispatcher<
312 - cutlass_3x_gemm<int8_t, cutlass::bfloat16_t, TileShape, ClusterShape,
313 - KernelSchedule, EpilogueSchedule>>(
314 - out, a, b, a_scales, b_scales);
336 + return cutlass_gemm_caller<cutlass_3x_gemm<
337 + int8_t, cutlass::bfloat16_t, ScaledEpilogue, TileShape, ClusterShape,
338 + KernelSchedule, EpilogueSchedule>>(out, a, b, a_scales, b_scales);
339 } else {
340 TORCH_CHECK(out.dtype() == torch::kFloat16);
341
318 - return cutlass_scaled_mm_dq_dispatcher<
319 - cutlass_3x_gemm<int8_t, cutlass::half_t, TileShape, ClusterShape,
320 - KernelSchedule, EpilogueSchedule>>(
342 + return cutlass_gemm_caller<
343 + cutlass_3x_gemm<int8_t, cutlass::half_t, ScaledEpilogue, TileShape,
344 + ClusterShape, KernelSchedule, EpilogueSchedule>>(
345 out, a, b, a_scales, b_scales);
346 }
347 } else {
@@ -325,13 +349,13 @@ void cutlass_scaled_mm_dq_sm90(torch::Tensor& out, torch::Tensor const& a,
349 TORCH_CHECK(b.dtype() == torch::kFloat8_e4m3fn);
350
351 if (out.dtype() == torch::kBFloat16) {
328 - return cutlass_scaled_mm_dq_sm90_fp8_dispatch<cutlass::float_e4m3_t,
329 - cutlass::bfloat16_t>(
352 + return cutlass_gemm_sm90_fp8_dispatch<
353 + cutlass::float_e4m3_t, cutlass::bfloat16_t, ScaledEpilogue>(
354 out, a, b, a_scales, b_scales);
355 } else {
356 TORCH_CHECK(out.dtype() == torch::kFloat16);
333 - return cutlass_scaled_mm_dq_sm90_fp8_dispatch<cutlass::float_e4m3_t,
334 - cutlass::half_t>(
357 + return cutlass_gemm_sm90_fp8_dispatch<cutlass::float_e4m3_t,
358 + cutlass::half_t, ScaledEpilogue>(
359 out, a, b, a_scales, b_scales);
360 }
361 }
csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu renamed
+24 -24
@@ -3,31 +3,31 @@
3 #include <c10/cuda/CUDAGuard.h>
4 #include <torch/all.h>
5
6 -void cutlass_scaled_mm_dq_sm75(torch::Tensor& c, torch::Tensor const& a,
7 - torch::Tensor const& b,
8 - torch::Tensor const& a_scales,
9 - torch::Tensor const& b_scales);
6 +void cutlass_scaled_mm_sm75(torch::Tensor& c, torch::Tensor const& a,
7 + torch::Tensor const& b,
8 + torch::Tensor const& a_scales,
9 + torch::Tensor const& b_scales);
10
11 -void cutlass_scaled_mm_dq_sm80(torch::Tensor& c, torch::Tensor const& a,
12 - torch::Tensor const& b,
13 - torch::Tensor const& a_scales,
14 - torch::Tensor const& b_scales);
11 +void cutlass_scaled_mm_sm80(torch::Tensor& c, torch::Tensor const& a,
12 + torch::Tensor const& b,
13 + torch::Tensor const& a_scales,
14 + torch::Tensor const& b_scales);
15
16 -void cutlass_scaled_mm_dq_sm89(torch::Tensor& c, torch::Tensor const& a,
17 - torch::Tensor const& b,
18 - torch::Tensor const& a_scales,
19 - torch::Tensor const& b_scales);
16 +void cutlass_scaled_mm_sm89(torch::Tensor& c, torch::Tensor const& a,
17 + torch::Tensor const& b,
18 + torch::Tensor const& a_scales,
19 + torch::Tensor const& b_scales);
20
21 #if defined CUDA_VERSION && CUDA_VERSION >= 12000
22 -void cutlass_scaled_mm_dq_sm90(torch::Tensor& c, torch::Tensor const& a,
23 - torch::Tensor const& b,
24 - torch::Tensor const& a_scales,
25 - torch::Tensor const& b_scales);
22 +void cutlass_scaled_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
23 + torch::Tensor const& b,
24 + torch::Tensor const& a_scales,
25 + torch::Tensor const& b_scales);
26 #endif
27
28 -void cutlass_scaled_mm_dq(torch::Tensor& c, torch::Tensor const& a,
29 - torch::Tensor const& b, torch::Tensor const& a_scales,
30 - torch::Tensor const& b_scales) {
28 +void cutlass_scaled_mm(torch::Tensor& c, torch::Tensor const& a,
29 + torch::Tensor const& b, torch::Tensor const& a_scales,
30 + torch::Tensor const& b_scales) {
31 int32_t major_capability;
32 int32_t minor_capability;
33 cudaDeviceGetAttribute(&major_capability, cudaDevAttrComputeCapabilityMajor,
@@ -57,19 +57,19 @@ void cutlass_scaled_mm_dq(torch::Tensor& c, torch::Tensor const& a,
57
58 // Guard against compilation issues for sm90 kernels
59 #if defined CUDA_VERSION && CUDA_VERSION >= 12000
60 - cutlass_scaled_mm_dq_sm90(c, a, b, a_scales, b_scales);
60 + cutlass_scaled_mm_sm90(c, a, b, a_scales, b_scales);
61 #else
62 - cutlass_scaled_mm_dq_sm80(c, a, b, a_scales, b_scales);
62 + cutlass_scaled_mm_sm80(c, a, b, a_scales, b_scales);
63 #endif
64 } else if (version_num == 89) {
65 // Ada Lovelace
66 - cutlass_scaled_mm_dq_sm89(c, a, b, a_scales, b_scales);
66 + cutlass_scaled_mm_sm89(c, a, b, a_scales, b_scales);
67 } else if (version_num >= 80) {
68 // Ampere
69 - cutlass_scaled_mm_dq_sm80(c, a, b, a_scales, b_scales);
69 + cutlass_scaled_mm_sm80(c, a, b, a_scales, b_scales);
70 } else {
71 // Turing
72 TORCH_CHECK(version_num >= 75);
73 - cutlass_scaled_mm_dq_sm75(c, a, b, a_scales, b_scales);
73 + cutlass_scaled_mm_sm75(c, a, b, a_scales, b_scales);
74 }
75 }
csrc/torch_bindings.cpp
+4 -4
@@ -136,10 +136,10 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
136 // CUTLASS w8a8 GEMM, supporting symmetric per-tensor or per-row/column
137 // quantization.
138 ops.def(
139 - "cutlass_scaled_mm_dq(Tensor! out, Tensor a,"
140 - " Tensor b, Tensor a_scales,"
141 - " Tensor b_scales) -> ()");
142 - ops.impl("cutlass_scaled_mm_dq", torch::kCUDA, &cutlass_scaled_mm_dq);
139 + "cutlass_scaled_mm(Tensor! out, Tensor a,"
140 + " Tensor b, Tensor a_scales,"
141 + " Tensor b_scales) -> ()");
142 + ops.impl("cutlass_scaled_mm", torch::kCUDA, &cutlass_scaled_mm);
143 #endif
144
145 // Quantized GEMM for GPTQ.
tests/kernels/test_cutlass.py
+9 -9
@@ -47,7 +47,7 @@ def cutlass_fp8_gemm_helper(m: int,
47 scale_b = (torch.randn(
48 (1, n_b_scales), device=device, dtype=torch.float32) / 10)
49
50 - out = ops.cutlass_scaled_mm_dq(a, b, scale_a, scale_b, out_dtype)
50 + out = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype)
51 baseline = torch.mm(scale_a * a.to(dtype=torch.float32),
52 scale_b * b.to(dtype=torch.float32)).to(out_dtype)
53
@@ -74,7 +74,7 @@ def cutlass_int8_gemm_helper(m: int,
74 scale_b = (torch.randn(
75 (1, n_b_scales), device=device, dtype=torch.float32) / 10)
76
77 - out = ops.cutlass_scaled_mm_dq(a, b, scale_a, scale_b, out_dtype)
77 + out = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype)
78 baseline = torch.mm(scale_a * a.to(dtype=torch.float32),
79 scale_b *
80 b.to(dtype=torch.float32)).to(dtype=out_dtype)
@@ -180,11 +180,11 @@ def test_cutlass_subset():
180 scale_a = torch.randn((1, 1), device="cuda", dtype=torch.float32) / 10
181 scale_b = torch.randn((1, 1), device="cuda", dtype=torch.float32) / 10
182
183 - out = ops.cutlass_scaled_mm_dq(a,
184 - b,
185 - scale_a,
186 - scale_b,
187 - out_dtype=torch.bfloat16)
183 + out = ops.cutlass_scaled_mm(a,
184 + b,
185 + scale_a,
186 + scale_b,
187 + out_dtype=torch.bfloat16)
188 baseline = torch.mm(scale_a * a.to(dtype=torch.float32),
189 scale_b *
190 b.to(dtype=torch.float32)).to(dtype=torch.bfloat16)
@@ -203,8 +203,8 @@ class CutlassLayer(torch.nn.Module):
203 self.out_dtype = out_dtype
204
205 def forward(self, a):
206 - return ops.cutlass_scaled_mm_dq(a, self.b, self.scale_a, self.scale_b,
207 - self.out_dtype)
206 + return ops.cutlass_scaled_mm(a, self.b, self.scale_a, self.scale_b,
207 + self.out_dtype)
208
209
210 @pytest.mark.parametrize("per_act_token", [True, False])
vllm/_custom_ops.py
+4 -5
@@ -212,9 +212,9 @@ def gptq_marlin_24_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
212
213
214 # cutlass
215 -def cutlass_scaled_mm_dq(a: torch.Tensor, b: torch.Tensor,
216 - scale_a: torch.Tensor, scale_b: torch.Tensor,
217 - out_dtype: Type[torch.dtype]) -> torch.Tensor:
215 +def cutlass_scaled_mm(a: torch.Tensor, b: torch.Tensor, scale_a: torch.Tensor,
216 + scale_b: torch.Tensor,
217 + out_dtype: Type[torch.dtype]) -> torch.Tensor:
218 assert (b.shape[0] % 16 == 0 and b.shape[1] % 16 == 0)
219 assert (out_dtype is torch.bfloat16 or out_dtype is torch.float16)
220
@@ -222,8 +222,7 @@ def cutlass_scaled_mm_dq(a: torch.Tensor, b: torch.Tensor,
222 n = b.shape[1]
223 out = torch.empty((m, n), dtype=out_dtype, device=a.device)
224
225 - torch.ops._C.cutlass_scaled_mm_dq(out, a, b, scale_a, scale_b)
226 -
225 + torch.ops._C.cutlass_scaled_mm(out, a, b, scale_a, scale_b)
226 return out
227
228
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_dynamictoken.py
+2 -2
@@ -81,5 +81,5 @@ class CompressedTensorsW8A8DynamicToken(CompressedTensorsScheme):
81 weight_scale = layer.weight_scale
82
83 x_q, input_scales = custom_ops.scaled_int8_quant(x)
84 - return custom_ops.cutlass_scaled_mm_dq(x_q, weight.t(), input_scales,
85 - weight_scale, x.dtype)
84 + return custom_ops.cutlass_scaled_mm(x_q, weight.t(), input_scales,
85 + weight_scale, x.dtype)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_statictensor.py
+2 -2
@@ -99,5 +99,5 @@ class CompressedTensorsW8A8StaticTensor(CompressedTensorsScheme):
99 # Input quantize
100 x_q, _ = custom_ops.scaled_int8_quant(x, act_scale)
101
102 - return custom_ops.cutlass_scaled_mm_dq(x_q, weight.t(), act_scale,
103 - weight_scale, x.dtype)
102 + return custom_ops.cutlass_scaled_mm(x_q, weight.t(), act_scale,
103 + weight_scale, x.dtype)
vllm/model_executor/layers/quantization/fp8.py
+1 -1
@@ -261,7 +261,7 @@ class Fp8LinearMethod(LinearMethodBase):
261 qinput, x_scale = ops.scaled_fp8_quant(x, layer.input_scale)
262
263 # Fused GEMM_DQ
264 - output = ops.cutlass_scaled_mm_dq(
264 + output = ops.cutlass_scaled_mm(
265 qinput,
266 layer.weight,
267 out_dtype=x.dtype,