A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Kernel] Adding bias epilogue support for `cutlass_scaled_mm` (#5560)
Co-authored-by: Chih-Chieh-Yang <7364402+cyang49@users.noreply.github.com> Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Luka Govedič committed
Jun 26, 2024 at 11:16 UTC
5bfd1bbc9831fed39632f071f16bb62373ec1249
8 files changed
+383
-134
CMakeLists.txt
+2
-1
@@ -2,7 +2,8 @@ cmake_minimum_required(VERSION 3.21)
2
3
project(vllm_extensions LANGUAGES CXX)
4
5
-option(VLLM_TARGET_DEVICE "Target device backend for vLLM" "cuda")
5
+# CUDA by default, can be overridden by using -DVLLM_TARGET_DEVICE=... (used by setup.py)
6
+set(VLLM_TARGET_DEVICE "cuda" CACHE STRING "Target device backend for vLLM")
7
8
message(STATUS "Build type: ${CMAKE_BUILD_TYPE}")
9
message(STATUS "Target device: ${VLLM_TARGET_DEVICE}")
csrc/ops.h
+2
-1
@@ -96,7 +96,8 @@ bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability);
96
97
void cutlass_scaled_mm(torch::Tensor& out, torch::Tensor const& a,
98
torch::Tensor const& b, torch::Tensor const& a_scales,
99
- torch::Tensor const& b_scales);
99
+ torch::Tensor const& b_scales,
100
+ c10::optional<torch::Tensor> const& bias);
101
102
#endif
103
csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cu
+170
-58
@@ -77,24 +77,12 @@ struct enable_sm89_to_sm90 : Kernel {
77
};
78
79
/*
80
- This epilogue function defines a quantized GEMM operation similar to
81
- torch._scaled_mm.
82
-
83
- A and B may be both either int8 or fp8_e4m3. A can be quantized per-tensor or
84
- per-row. B can be quantized per-tensor or per-column.
85
- Any combination of per-tensor and per-row or column is supported.
86
- A and B must have symmetric quantization (zero point == 0).
87
-
88
- So the GEMM operation is D = (a_scales * A) (b_scales * B), where the
89
- scales are applied elementwise with numpy-style broadcasting.
90
-
91
- ScaleA and ScaleB define the epilogue functions that apply the scales for
92
- the A and B operands respectively. These scales may be either per-tensor or
93
- per row or column.
94
-*/
80
+ * This class provides the common ScaleA and ScaleB descriptors for the
81
+ * ScaledEpilogue and ScaledEpilogueBias classes.
82
+ */
83
template <typename ElementD, typename OutputTileThreadMap>
96
-struct ScaledEpilogue {
97
- private:
84
+struct ScaledEpilogueBase {
85
+ protected:
86
using Accum = cutlass::epilogue::threadblock::VisitorAccFetch;
87
88
using ScaleA = cutlass::epilogue::threadblock::VisitorColOrScalarBroadcast<
@@ -102,6 +90,32 @@ struct ScaledEpilogue {
90
91
using ScaleB = cutlass::epilogue::threadblock::VisitorRowOrScalarBroadcast<
92
OutputTileThreadMap, float, Stride<Int<0>, Int<1>, Int<0>>>;
93
+};
94
+
95
+/*
96
+ This epilogue function defines a quantized GEMM operation similar to
97
+ torch._scaled_mm.
98
+
99
+ A and B may be both either int8 or fp8_e4m3. A can be quantized per-tensor or
100
+ per-row. B can be quantized per-tensor or per-column.
101
+ Any combination of per-tensor and per-row or column is supported.
102
+ A and B must have symmetric quantization (zero point == 0).
103
+
104
+ So the GEMM operation is D = (a_scales * A) (b_scales * B), where the
105
+ scales are applied elementwise with numpy-style broadcasting.
106
+
107
+ ScaleA and ScaleB define the epilogue functions that apply the scales for
108
+ the A and B operands respectively. These scales may be either per-tensor or
109
+ per row or column.
110
+*/
111
+template <typename ElementD, typename OutputTileThreadMap>
112
+struct ScaledEpilogue
113
+ : private ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
114
+ private:
115
+ using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
116
+ using Accum = typename SUPER::Accum;
117
+ using ScaleA = typename SUPER::ScaleA;
118
+ using ScaleB = typename SUPER::ScaleB;
119
120
using Compute0 = cutlass::epilogue::threadblock::VisitorCompute<
121
cutlass::multiplies, float, float,
@@ -134,6 +148,53 @@ struct ScaledEpilogue {
148
}
149
};
150
151
+template <typename ElementD, typename OutputTileThreadMap>
152
+struct ScaledEpilogueBias
153
+ : private ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
154
+ private:
155
+ using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
156
+ using Accum = typename SUPER::Accum;
157
+ using ScaleA = typename SUPER::ScaleA;
158
+ using ScaleB = typename SUPER::ScaleB;
159
+
160
+ using Compute0 = cutlass::epilogue::threadblock::VisitorCompute<
161
+ cutlass::multiplies, float, float,
162
+ cutlass::FloatRoundStyle::round_to_nearest>;
163
+
164
+ using EVTCompute0 =
165
+ cutlass::epilogue::threadblock::Sm80EVT<Compute0, ScaleB, Accum>;
166
+
167
+ using Compute1 = cutlass::epilogue::threadblock::VisitorCompute<
168
+ cutlass::multiply_add, ElementD, float,
169
+ cutlass::FloatRoundStyle::round_to_nearest>;
170
+
171
+ using Bias = cutlass::epilogue::threadblock::VisitorRowBroadcast<
172
+ OutputTileThreadMap, ElementD, Stride<Int<0>, Int<1>, Int<0>>>;
173
+
174
+ public:
175
+ using EVTCompute = cutlass::epilogue::threadblock::Sm80EVT<Compute1, ScaleA,
176
+ EVTCompute0, Bias>;
177
+ using ArgumentType = typename EVTCompute::Arguments;
178
+
179
+ static ArgumentType prepare_args(torch::Tensor const& a_scales,
180
+ torch::Tensor const& b_scales,
181
+ torch::Tensor const& bias) {
182
+ using ScaleAArgs = typename ScaleA::Arguments;
183
+ using ScaleBArgs = typename ScaleB::Arguments;
184
+ using BiasArgs = typename Bias::Arguments;
185
+
186
+ ScaleBArgs b_args{b_scales.data_ptr<float>(), b_scales.numel() != 1, {}};
187
+ ScaleAArgs a_args{a_scales.data_ptr<float>(), a_scales.numel() != 1, {}};
188
+ BiasArgs bias_args{static_cast<ElementD*>(bias.data_ptr()), {}};
189
+
190
+ typename EVTCompute0::Arguments evt0_compute_args{b_args};
191
+
192
+ typename EVTCompute::Arguments evt_compute_args{a_args, evt0_compute_args,
193
+ bias_args};
194
+ return evt_compute_args;
195
+ }
196
+};
197
+
198
template <typename Arch, template <typename> typename ArchGuard,
199
typename ElementAB_, typename ElementD_,
200
template <typename, typename> typename Epilogue_, typename TileShape,
@@ -168,13 +229,13 @@ struct cutlass_2x_gemm {
229
// clang-format off
230
using RowMajor = typename cutlass::layout::RowMajor;
231
using ColumnMajor = typename cutlass::layout::ColumnMajor;
171
- using KernelType =
232
+ using KernelType =
233
ArchGuard<typename cutlass::gemm::kernel::DefaultGemmWithVisitor<
173
- ElementAB, RowMajor, cutlass::ComplexTransform::kNone, 16,
174
- ElementAB, ColumnMajor, cutlass::ComplexTransform::kNone, 16,
234
+ ElementAB, RowMajor, cutlass::ComplexTransform::kNone, 16,
235
+ ElementAB, ColumnMajor, cutlass::ComplexTransform::kNone, 16,
236
float, cutlass::layout::RowMajor, 4,
176
- ElementAcc, float, cutlass::arch::OpClassTensorOp,
177
- Arch,
237
+ ElementAcc, float, cutlass::arch::OpClassTensorOp,
238
+ Arch,
239
TileShape, WarpShape, InstructionShape,
240
EVTD,
241
cutlass::gemm::threadblock::ThreadblockSwizzleStreamK,
@@ -404,14 +465,13 @@ void cutlass_gemm_sm80_dispatch(torch::Tensor& out, torch::Tensor const& a,
465
}
466
}
467
407
-void cutlass_scaled_mm_sm75(torch::Tensor& out, torch::Tensor const& a,
408
- torch::Tensor const& b,
409
- torch::Tensor const& a_scales,
410
- torch::Tensor const& b_scales) {
468
+template <template <typename, typename> typename Epilogue,
469
+ typename... EpilogueArgs>
470
+void cutlass_scaled_mm_sm75_epilogue(torch::Tensor& out, torch::Tensor const& a,
471
+ torch::Tensor const& b,
472
+ EpilogueArgs&&... epilogue_args) {
473
TORCH_CHECK(a.dtype() == torch::kInt8);
474
TORCH_CHECK(b.dtype() == torch::kInt8);
413
- TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
414
- TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
475
476
using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
477
using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
@@ -420,78 +480,130 @@ void cutlass_scaled_mm_sm75(torch::Tensor& out, torch::Tensor const& a,
480
if (out.dtype() == torch::kBFloat16) {
481
return cutlass_gemm_caller<cutlass_2x_gemm<
482
cutlass::arch::Sm75, enable_sm75_to_sm80, int8_t, cutlass::bfloat16_t,
423
- ScaledEpilogue, TileShape, WarpShape, InstructionShape, 2>>(
424
- out, a, b, a_scales, b_scales);
483
+ Epilogue, TileShape, WarpShape, InstructionShape, 2>>(
484
+ out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
485
} else {
486
TORCH_CHECK(out.dtype() == torch::kFloat16);
487
return cutlass_gemm_caller<cutlass_2x_gemm<
488
cutlass::arch::Sm75, enable_sm75_to_sm80, int8_t, cutlass::half_t,
429
- ScaledEpilogue, TileShape, WarpShape, InstructionShape, 2>>(
430
- out, a, b, a_scales, b_scales);
489
+ Epilogue, TileShape, WarpShape, InstructionShape, 2>>(
490
+ out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
491
}
492
}
493
434
-void cutlass_scaled_mm_sm80(torch::Tensor& out, torch::Tensor const& a,
494
+void cutlass_scaled_mm_sm75(torch::Tensor& out, torch::Tensor const& a,
495
torch::Tensor const& b,
496
torch::Tensor const& a_scales,
437
- torch::Tensor const& b_scales) {
438
- TORCH_CHECK(a.dtype() == torch::kInt8);
439
- TORCH_CHECK(b.dtype() == torch::kInt8);
497
+ torch::Tensor const& b_scales,
498
+ c10::optional<torch::Tensor> const& bias) {
499
TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
500
TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
501
+ if (bias) {
502
+ TORCH_CHECK(bias->dtype() == out.dtype(),
503
+ "currently bias dtype must match output dtype ", out.dtype());
504
+ return cutlass_scaled_mm_sm75_epilogue<ScaledEpilogueBias>(
505
+ out, a, b, a_scales, b_scales, *bias);
506
+ } else {
507
+ return cutlass_scaled_mm_sm75_epilogue<ScaledEpilogue>(out, a, b, a_scales,
508
+ b_scales);
509
+ }
510
+}
511
+
512
+template <template <typename, typename> typename Epilogue,
513
+ typename... EpilogueArgs>
514
+void cutlass_scaled_mm_sm80_epilogue(torch::Tensor& out, torch::Tensor const& a,
515
+ torch::Tensor const& b,
516
+ EpilogueArgs&&... epilogue_args) {
517
+ TORCH_CHECK(a.dtype() == torch::kInt8);
518
+ TORCH_CHECK(b.dtype() == torch::kInt8);
519
520
if (out.dtype() == torch::kBFloat16) {
444
- return cutlass_gemm_sm80_dispatch<int8_t, cutlass::bfloat16_t,
445
- ScaledEpilogue>(out, a, b, a_scales,
446
- b_scales);
521
+ return cutlass_gemm_sm80_dispatch<int8_t, cutlass::bfloat16_t, Epilogue>(
522
+ out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
523
} else {
524
TORCH_CHECK(out.dtype() == torch::kFloat16);
449
- return cutlass_gemm_sm80_dispatch<int8_t, cutlass::half_t, ScaledEpilogue>(
450
- out, a, b, a_scales, b_scales);
525
+ return cutlass_gemm_sm80_dispatch<int8_t, cutlass::half_t, Epilogue>(
526
+ out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
527
}
528
}
529
454
-void cutlass_scaled_mm_sm89(torch::Tensor& out, torch::Tensor const& a,
530
+void cutlass_scaled_mm_sm80(torch::Tensor& out, torch::Tensor const& a,
531
torch::Tensor const& b,
532
torch::Tensor const& a_scales,
457
- torch::Tensor const& b_scales) {
533
+ torch::Tensor const& b_scales,
534
+ c10::optional<torch::Tensor> const& bias) {
535
+ TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
536
+ TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
537
+ if (bias) {
538
+ TORCH_CHECK(bias->dtype() == out.dtype(),
539
+ "currently bias dtype must match output dtype ", out.dtype());
540
+ return cutlass_scaled_mm_sm80_epilogue<ScaledEpilogueBias>(
541
+ out, a, b, a_scales, b_scales, *bias);
542
+ } else {
543
+ return cutlass_scaled_mm_sm80_epilogue<ScaledEpilogue>(out, a, b, a_scales,
544
+ b_scales);
545
+ }
546
+}
547
+
548
+template <template <typename, typename> typename Epilogue,
549
+ typename... EpilogueArgs>
550
+void cutlass_scaled_mm_sm89_epilogue(torch::Tensor& out, torch::Tensor const& a,
551
+ torch::Tensor const& b,
552
+ EpilogueArgs&&... epilogue_args) {
553
using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
554
using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
555
using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
556
462
- TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
463
- TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
464
-
557
if (a.dtype() == torch::kInt8) {
558
TORCH_CHECK(b.dtype() == torch::kInt8);
559
560
if (out.dtype() == torch::kBFloat16) {
561
return cutlass_gemm_caller<cutlass_2x_gemm<
562
cutlass::arch::Sm89, enable_sm89_to_sm90, int8_t, cutlass::bfloat16_t,
471
- ScaledEpilogue, TileShape, WarpShape, InstructionShape, 5>>(
472
- out, a, b, a_scales, b_scales);
563
+ Epilogue, TileShape, WarpShape, InstructionShape, 5>>(
564
+ out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
565
} else {
566
assert(out.dtype() == torch::kFloat16);
567
return cutlass_gemm_caller<cutlass_2x_gemm<
568
cutlass::arch::Sm89, enable_sm89_to_sm90, int8_t, cutlass::half_t,
477
- ScaledEpilogue, TileShape, WarpShape, InstructionShape, 5>>(
478
- out, a, b, a_scales, b_scales);
569
+ Epilogue, TileShape, WarpShape, InstructionShape, 5>>(
570
+ out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
571
}
572
} else {
573
TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
574
TORCH_CHECK(b.dtype() == torch::kFloat8_e4m3fn);
575
576
if (out.dtype() == torch::kBFloat16) {
485
- return cutlass_gemm_caller<cutlass_2x_gemm<
486
- cutlass::arch::Sm89, enable_sm89_to_sm90, cutlass::float_e4m3_t,
487
- cutlass::bfloat16_t, ScaledEpilogue, TileShape, WarpShape,
488
- InstructionShape, 5>>(out, a, b, a_scales, b_scales);
577
+ return cutlass_gemm_caller<
578
+ cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
579
+ cutlass::float_e4m3_t, cutlass::bfloat16_t, Epilogue,
580
+ TileShape, WarpShape, InstructionShape, 5>>(
581
+ out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
582
} else {
583
TORCH_CHECK(out.dtype() == torch::kFloat16);
491
- return cutlass_gemm_caller<cutlass_2x_gemm<
492
- cutlass::arch::Sm89, enable_sm89_to_sm90, cutlass::float_e4m3_t,
493
- cutlass::half_t, ScaledEpilogue, TileShape, WarpShape,
494
- InstructionShape, 5>>(out, a, b, a_scales, b_scales);
584
+ return cutlass_gemm_caller<
585
+ cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
586
+ cutlass::float_e4m3_t, cutlass::half_t, Epilogue,
587
+ TileShape, WarpShape, InstructionShape, 5>>(
588
+ out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
589
}
590
}
591
}
592
+
593
+void cutlass_scaled_mm_sm89(torch::Tensor& out, torch::Tensor const& a,
594
+ torch::Tensor const& b,
595
+ torch::Tensor const& a_scales,
596
+ torch::Tensor const& b_scales,
597
+ c10::optional<torch::Tensor> const& bias) {
598
+ TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
599
+ TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
600
+ if (bias) {
601
+ TORCH_CHECK(bias->dtype() == out.dtype(),
602
+ "currently bias dtype must match output dtype ", out.dtype());
603
+ return cutlass_scaled_mm_sm89_epilogue<ScaledEpilogueBias>(
604
+ out, a, b, a_scales, b_scales, *bias);
605
+ } else {
606
+ return cutlass_scaled_mm_sm89_epilogue<ScaledEpilogue>(out, a, b, a_scales,
607
+ b_scales);
608
+ }
609
+}
csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu
+108
-31
@@ -59,6 +59,28 @@ struct enable_sm90_or_later : Kernel {
59
}
60
};
61
62
+/*
63
+ * This class provides the common ScaleA and ScaleB descriptors for the
64
+ * ScaledEpilogue and ScaledEpilogueBias classes.
65
+ */
66
+template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
67
+struct ScaledEpilogueBase {
68
+ protected:
69
+ using Accum = cutlass::epilogue::fusion::Sm90AccFetch;
70
+
71
+ using ScaleA = cutlass::epilogue::fusion::Sm90ColOrScalarBroadcast<
72
+ 0 /*Stages*/, typename EpilogueDescriptor::TileShape, float,
73
+ Stride<Int<1>, Int<0>, Int<0>>>;
74
+
75
+ using ScaleBDescriptor =
76
+ cutlass::epilogue::collective::detail::RowBroadcastDescriptor<
77
+ EpilogueDescriptor, float>;
78
+
79
+ using ScaleB = cutlass::epilogue::fusion::Sm90RowOrScalarBroadcast<
80
+ ScaleBDescriptor::Stages, typename EpilogueDescriptor::TileShape,
81
+ typename ScaleBDescriptor::Element, Stride<Int<0>, Int<1>, Int<0>>>;
82
+};
83
+
84
/*
85
This epilogue function defines a quantized GEMM operation similar to
86
torch.scaled_mm_.
@@ -76,21 +98,13 @@ struct enable_sm90_or_later : Kernel {
98
per row or column.
99
*/
100
template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
79
-struct ScaledEpilogue {
101
+struct ScaledEpilogue
102
+ : private ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor> {
103
private:
81
- using Accum = cutlass::epilogue::fusion::Sm90AccFetch;
82
-
83
- using ScaleA = cutlass::epilogue::fusion::Sm90ColOrScalarBroadcast<
84
- 0 /*Stages*/, typename EpilogueDescriptor::TileShape, float,
85
- Stride<Int<1>, Int<0>, Int<0>>>;
86
-
87
- using ScaleBDescriptor =
88
- cutlass::epilogue::collective::detail::RowBroadcastDescriptor<
89
- EpilogueDescriptor, float>;
90
-
91
- using ScaleB = cutlass::epilogue::fusion::Sm90RowOrScalarBroadcast<
92
- ScaleBDescriptor::Stages, typename EpilogueDescriptor::TileShape,
93
- typename ScaleBDescriptor::Element, Stride<Int<0>, Int<1>, Int<0>>>;
104
+ using SUPER = ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor>;
105
+ using Accum = typename SUPER::Accum;
106
+ using ScaleA = typename SUPER::ScaleA;
107
+ using ScaleB = typename SUPER::ScaleB;
108
109
using Compute0 = cutlass::epilogue::fusion::Sm90Compute<
110
cutlass::multiplies, float, float,
@@ -120,6 +134,54 @@ struct ScaledEpilogue {
134
}
135
};
136
137
+template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
138
+struct ScaledEpilogueBias
139
+ : private ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor> {
140
+ private:
141
+ using SUPER = ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor>;
142
+ using Accum = typename SUPER::Accum;
143
+ using ScaleA = typename SUPER::ScaleA;
144
+ using ScaleB = typename SUPER::ScaleB;
145
+
146
+ using Compute0 = cutlass::epilogue::fusion::Sm90Compute<
147
+ cutlass::multiplies, ElementD, ElementD,
148
+ cutlass::FloatRoundStyle::round_to_nearest>;
149
+
150
+ using EVTCompute0 =
151
+ cutlass::epilogue::fusion::Sm90EVT<Compute0, ScaleB, Accum>;
152
+
153
+ using Compute1 = cutlass::epilogue::fusion::Sm90Compute<
154
+ cutlass::multiply_add, ElementD, ElementD,
155
+ cutlass::FloatRoundStyle::round_to_nearest>;
156
+
157
+ using BiasDescriptor =
158
+ cutlass::epilogue::collective::detail::RowBroadcastDescriptor<
159
+ EpilogueDescriptor, ElementD>;
160
+
161
+ using Bias = cutlass::epilogue::fusion::Sm90RowBroadcast<
162
+ BiasDescriptor::Stages, typename EpilogueDescriptor::TileShape, ElementD,
163
+ Stride<Int<0>, Int<1>, Int<0>>, 128 / sizeof_bits_v<ElementD>, false>;
164
+
165
+ public:
166
+ using EVTCompute =
167
+ cutlass::epilogue::fusion::Sm90EVT<Compute1, ScaleA, EVTCompute0, Bias>;
168
+ using ArgumentType = typename EVTCompute::Arguments;
169
+
170
+ static ArgumentType prepare_args(torch::Tensor const& a_scales,
171
+ torch::Tensor const& b_scales,
172
+ torch::Tensor const& bias) {
173
+ using ScaleA_Args = typename ScaleA::Arguments;
174
+ using ScaleB_Args = typename ScaleB::Arguments;
175
+ using Bias_Args = typename Bias::Arguments;
176
+
177
+ ScaleA_Args a_args{a_scales.data_ptr<float>(), a_scales.numel() != 1, {}};
178
+ ScaleB_Args b_args{b_scales.data_ptr<float>(), b_scales.numel() != 1, {}};
179
+ Bias_Args bias_args{static_cast<ElementD*>(bias.data_ptr())};
180
+
181
+ return ArgumentType{a_args, {b_args}, bias_args};
182
+ }
183
+};
184
+
185
template <typename ElementAB_, typename ElementD_,
186
template <typename, typename, typename> typename Epilogue_,
187
typename TileShape, typename ClusterShape, typename KernelSchedule,
@@ -440,41 +502,56 @@ void cutlass_gemm_sm90_int8_dispatch(torch::Tensor& out, torch::Tensor const& a,
502
}
503
}
504
443
-void cutlass_scaled_mm_sm90(torch::Tensor& out, torch::Tensor const& a,
444
- torch::Tensor const& b,
445
- torch::Tensor const& a_scales,
446
- torch::Tensor const& b_scales) {
447
- TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
448
- TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
449
-
505
+template <template <typename, typename, typename> typename Epilogue,
506
+ typename... EpilogueArgs>
507
+void cutlass_scaled_mm_sm90_epilogue(torch::Tensor& out, torch::Tensor const& a,
508
+ torch::Tensor const& b,
509
+ EpilogueArgs&&... epilogue_args) {
510
if (a.dtype() == torch::kInt8) {
511
TORCH_CHECK(b.dtype() == torch::kInt8);
512
513
if (out.dtype() == torch::kBFloat16) {
514
return cutlass_gemm_sm90_int8_dispatch<int8_t, cutlass::bfloat16_t,
455
- ScaledEpilogue>(
456
- out, a, b, a_scales, b_scales);
515
+ Epilogue>(
516
+ out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
517
} else {
518
TORCH_CHECK(out.dtype() == torch::kFloat16);
459
- return cutlass_gemm_sm90_int8_dispatch<int8_t, cutlass::half_t,
460
- ScaledEpilogue>(
461
- out, a, b, a_scales, b_scales);
519
+ return cutlass_gemm_sm90_int8_dispatch<int8_t, cutlass::half_t, Epilogue>(
520
+ out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
521
}
522
} else {
523
TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
524
TORCH_CHECK(b.dtype() == torch::kFloat8_e4m3fn);
525
526
if (out.dtype() == torch::kBFloat16) {
468
- return cutlass_gemm_sm90_fp8_dispatch<
469
- cutlass::float_e4m3_t, cutlass::bfloat16_t, ScaledEpilogue>(
470
- out, a, b, a_scales, b_scales);
527
+ return cutlass_gemm_sm90_fp8_dispatch<cutlass::float_e4m3_t,
528
+ cutlass::bfloat16_t, Epilogue>(
529
+ out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
530
} else {
531
TORCH_CHECK(out.dtype() == torch::kFloat16);
532
return cutlass_gemm_sm90_fp8_dispatch<cutlass::float_e4m3_t,
474
- cutlass::half_t, ScaledEpilogue>(
475
- out, a, b, a_scales, b_scales);
533
+ cutlass::half_t, Epilogue>(
534
+ out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
535
}
536
}
537
}
538
539
+void cutlass_scaled_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
540
+ torch::Tensor const& b,
541
+ torch::Tensor const& a_scales,
542
+ torch::Tensor const& b_scales,
543
+ c10::optional<torch::Tensor> const& bias) {
544
+ TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
545
+ TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
546
+ if (bias) {
547
+ TORCH_CHECK(bias->dtype() == c.dtype(),
548
+ "currently bias dtype must match output dtype ", c.dtype());
549
+ return cutlass_scaled_mm_sm90_epilogue<ScaledEpilogueBias>(
550
+ c, a, b, a_scales, b_scales, *bias);
551
+ } else {
552
+ return cutlass_scaled_mm_sm90_epilogue<ScaledEpilogue>(c, a, b, a_scales,
553
+ b_scales);
554
+ }
555
+}
556
+
557
#endif
csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu
+21
-11
@@ -6,23 +6,27 @@
6
void cutlass_scaled_mm_sm75(torch::Tensor& c, torch::Tensor const& a,
7
torch::Tensor const& b,
8
torch::Tensor const& a_scales,
9
- torch::Tensor const& b_scales);
9
+ torch::Tensor const& b_scales,
10
+ c10::optional<torch::Tensor> const& bias);
11
12
void cutlass_scaled_mm_sm80(torch::Tensor& c, torch::Tensor const& a,
13
torch::Tensor const& b,
14
torch::Tensor const& a_scales,
14
- torch::Tensor const& b_scales);
15
+ torch::Tensor const& b_scales,
16
+ c10::optional<torch::Tensor> const& bias);
17
18
void cutlass_scaled_mm_sm89(torch::Tensor& c, torch::Tensor const& a,
19
torch::Tensor const& b,
20
torch::Tensor const& a_scales,
19
- torch::Tensor const& b_scales);
21
+ torch::Tensor const& b_scales,
22
+ c10::optional<torch::Tensor> const& bias);
23
24
#if defined CUDA_VERSION && CUDA_VERSION >= 12000
25
void cutlass_scaled_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
26
torch::Tensor const& b,
27
torch::Tensor const& a_scales,
25
- torch::Tensor const& b_scales);
28
+ torch::Tensor const& b_scales,
29
+ c10::optional<torch::Tensor> const& bias);
30
#endif
31
32
bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability) {
@@ -43,7 +47,8 @@ bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability) {
47
48
void cutlass_scaled_mm(torch::Tensor& c, torch::Tensor const& a,
49
torch::Tensor const& b, torch::Tensor const& a_scales,
46
- torch::Tensor const& b_scales) {
50
+ torch::Tensor const& b_scales,
51
+ c10::optional<torch::Tensor> const& bias) {
52
int32_t major_capability;
53
int32_t minor_capability;
54
cudaDeviceGetAttribute(&major_capability, cudaDevAttrComputeCapabilityMajor,
@@ -66,6 +71,11 @@ void cutlass_scaled_mm(torch::Tensor& c, torch::Tensor const& a,
71
b.stride(1) % 16 == 0); // 16 Byte Alignment
72
TORCH_CHECK(a_scales.is_contiguous() && b_scales.is_contiguous());
73
74
+ if (bias) {
75
+ TORCH_CHECK(bias->numel() == b.size(1) && bias->is_contiguous() &&
76
+ bias->dim() == 1);
77
+ }
78
+
79
at::cuda::OptionalCUDAGuard const device_guard(device_of(a));
80
81
if (version_num >= 90) {
@@ -73,19 +83,19 @@ void cutlass_scaled_mm(torch::Tensor& c, torch::Tensor const& a,
83
84
// Guard against compilation issues for sm90 kernels
85
#if defined CUDA_VERSION && CUDA_VERSION >= 12000
76
- cutlass_scaled_mm_sm90(c, a, b, a_scales, b_scales);
86
+ cutlass_scaled_mm_sm90(c, a, b, a_scales, b_scales, bias);
87
#else
78
- cutlass_scaled_mm_sm80(c, a, b, a_scales, b_scales);
88
+ cutlass_scaled_mm_sm80(c, a, b, a_scales, b_scales, bias);
89
#endif
90
} else if (version_num == 89) {
91
// Ada Lovelace
82
- cutlass_scaled_mm_sm89(c, a, b, a_scales, b_scales);
92
+ cutlass_scaled_mm_sm89(c, a, b, a_scales, b_scales, bias);
93
} else if (version_num >= 80) {
94
// Ampere
85
- cutlass_scaled_mm_sm80(c, a, b, a_scales, b_scales);
95
+ cutlass_scaled_mm_sm80(c, a, b, a_scales, b_scales, bias);
96
} else {
97
// Turing
98
TORCH_CHECK(version_num >= 75);
89
- cutlass_scaled_mm_sm75(c, a, b, a_scales, b_scales);
99
+ cutlass_scaled_mm_sm75(c, a, b, a_scales, b_scales, bias);
100
}
91
-}
101
+}
\ No newline at end of file
csrc/torch_bindings.cpp
+1
-1
@@ -142,7 +142,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
142
ops.def(
143
"cutlass_scaled_mm(Tensor! out, Tensor a,"
144
" Tensor b, Tensor a_scales,"
145
- " Tensor b_scales) -> ()");
145
+ " Tensor b_scales, Tensor? bias) -> ()");
146
ops.impl("cutlass_scaled_mm", torch::kCUDA, &cutlass_scaled_mm);
147
148
// Check if cutlass scaled_mm is supported for CUDA devices of the given
tests/kernels/test_cutlass.py
+72
-28
@@ -32,6 +32,7 @@ def cutlass_fp8_gemm_helper(m: int,
32
k: int,
33
per_token_act_quant: bool,
34
per_out_channel_weight_quant: bool,
35
+ bias: bool,
36
out_dtype: Type[torch.dtype] = torch.bfloat16,
37
device: str = "cuda"):
38
# Test for a cutlass kernel with per-token activation quantization
@@ -46,10 +47,17 @@ def cutlass_fp8_gemm_helper(m: int,
47
(m_a_scales, 1), device=device, dtype=torch.float32) / 10)
48
scale_b = (torch.randn(
49
(1, n_b_scales), device=device, dtype=torch.float32) / 10)
49
-
50
- out = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype)
51
- baseline = torch.mm(scale_a * a.to(dtype=torch.float32),
52
- scale_b * b.to(dtype=torch.float32)).to(out_dtype)
50
+ if bias:
51
+ # bias term should be > 1 so that the absolute tolerance can catch it
52
+ bias_t = torch.rand((n, ), device=device, dtype=out_dtype) + 1.0
53
+ out = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias_t)
54
+ else:
55
+ out = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype)
56
+ bias_t = 0
57
+
58
+ baseline = (torch.mm(scale_a * a.to(dtype=torch.float32),
59
+ scale_b * b.to(dtype=torch.float32)) +
60
+ bias_t).to(out_dtype)
61
62
assert torch.allclose(out, baseline, rtol=1e-2, atol=1e-1)
63
@@ -59,6 +67,7 @@ def cutlass_int8_gemm_helper(m: int,
67
k: int,
68
per_token_act_quant: bool,
69
per_out_channel_weight_quant: bool,
70
+ bias: bool,
71
out_dtype: Type[torch.dtype] = torch.bfloat16,
72
device: str = "cuda"):
73
# Test for a cutlass kernel with per-token activation quantization
@@ -74,11 +83,17 @@ def cutlass_int8_gemm_helper(m: int,
83
scale_b = (torch.randn(
84
(1, n_b_scales), device=device, dtype=torch.float32) / 10)
85
77
- out = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype)
78
- baseline = torch.mm(scale_a * a.to(dtype=torch.float32),
79
- scale_b *
80
- b.to(dtype=torch.float32)).to(dtype=out_dtype)
81
-
86
+ if bias:
87
+ # bias term should be > 1 so that the absolute tolerance can catch it
88
+ bias_t = torch.rand((n, ), device=device, dtype=out_dtype) + 1.0
89
+ out = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias_t)
90
+ else:
91
+ out = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype)
92
+ bias_t = 0
93
+
94
+ baseline = (torch.mm(scale_a * a.to(dtype=torch.float32),
95
+ scale_b * b.to(dtype=torch.float32)) +
96
+ bias_t).to(dtype=out_dtype)
97
assert torch.allclose(out, baseline, rtol=1e-1, atol=1e0)
98
99
@@ -87,11 +102,12 @@ def cutlass_int8_gemm_helper(m: int,
102
@pytest.mark.parametrize("k", [128, 496, 1024])
103
@pytest.mark.parametrize("per_act_token", [True, False])
104
@pytest.mark.parametrize("per_out_ch", [True, False])
105
+@pytest.mark.parametrize("bias", [True, False])
106
@pytest.mark.skipif(capability < 89,
107
reason="FP8 is not supported on this GPU type.")
108
def test_cutlass_fp8_gemm(m: int, n: int, k: int, per_act_token: bool,
93
- per_out_ch: bool):
94
- cutlass_fp8_gemm_helper(m, n, k, per_act_token, per_out_ch)
109
+ per_out_ch: bool, bias: bool):
110
+ cutlass_fp8_gemm_helper(m, n, k, per_act_token, per_out_ch, bias)
111
112
113
@pytest.mark.parametrize("m", [512, 222, 33, 1])
@@ -99,49 +115,72 @@ def test_cutlass_fp8_gemm(m: int, n: int, k: int, per_act_token: bool,
115
@pytest.mark.parametrize("k", [128, 496, 1024])
116
@pytest.mark.parametrize("per_act_token", [True, False])
117
@pytest.mark.parametrize("per_out_ch", [True, False])
118
+@pytest.mark.parametrize("bias", [True, False])
119
def test_cutlass_int8_gemm(m: int, n: int, k: int, per_act_token: bool,
103
- per_out_ch: bool):
104
- cutlass_int8_gemm_helper(m, n, k, per_act_token, per_out_ch)
120
+ per_out_ch: bool, bias: bool):
121
+ cutlass_int8_gemm_helper(m, n, k, per_act_token, per_out_ch, bias)
122
123
124
@pytest.mark.parametrize("per_act_token", [True, False])
125
@pytest.mark.parametrize("per_out_ch", [True, False])
126
@pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16])
127
+@pytest.mark.parametrize("bias", [True, False])
128
def test_cutlass_int8_gemm_output_dtype(per_act_token: bool, per_out_ch: bool,
111
- out_dtype: Type[torch.dtype]):
112
- cutlass_int8_gemm_helper(512, 512, 512, per_act_token, per_out_ch,
113
- out_dtype)
129
+ out_dtype: Type[torch.dtype],
130
+ bias: bool):
131
+ cutlass_int8_gemm_helper(512,
132
+ 512,
133
+ 512,
134
+ per_act_token,
135
+ per_out_ch,
136
+ bias,
137
+ out_dtype=out_dtype)
138
139
140
@pytest.mark.parametrize("per_act_token", [True, False])
141
@pytest.mark.parametrize("per_out_ch", [True, False])
142
@pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16])
143
+@pytest.mark.parametrize("bias", [True, False])
144
@pytest.mark.skipif(capability < 89,
145
reason="FP8 is not supported on this GPU type.")
146
def test_cutlass_fp8_gemm_output_dtype(per_act_token: bool, per_out_ch: bool,
122
- out_dtype: Type[torch.dtype]):
123
- cutlass_fp8_gemm_helper(512, 512, 512, per_act_token, per_out_ch,
124
- out_dtype)
147
+ out_dtype: Type[torch.dtype],
148
+ bias: bool):
149
+ cutlass_fp8_gemm_helper(512,
150
+ 512,
151
+ 512,
152
+ per_act_token,
153
+ per_out_ch,
154
+ bias,
155
+ out_dtype=out_dtype)
156
157
158
@pytest.mark.parametrize("per_act_token", [True, False])
159
@pytest.mark.parametrize("per_out_ch", [True, False])
160
+@pytest.mark.parametrize("bias", [True, False])
161
@pytest.mark.parametrize("device", CUDA_DEVICES)
162
@pytest.mark.skipif(capability < 89,
163
reason="FP8 is not supported on this GPU type.")
164
def test_cutlass_fp8_gemm_devices(per_act_token: bool, per_out_ch: bool,
133
- device: str):
134
- cutlass_fp8_gemm_helper(512, 512, 512, per_act_token, per_out_ch,
165
+ bias: bool, device: str):
166
+ cutlass_fp8_gemm_helper(512, 512, 512, per_act_token, per_out_ch, bias,
167
torch.bfloat16, device)
168
169
170
@pytest.mark.parametrize("per_act_token", [True, False])
171
@pytest.mark.parametrize("per_out_ch", [True, False])
172
+@pytest.mark.parametrize("bias", [True, False])
173
@pytest.mark.parametrize("device", CUDA_DEVICES)
174
def test_cutlass_int8_gemm_devices(per_act_token: bool, per_out_ch: bool,
142
- device: str):
143
- cutlass_int8_gemm_helper(512, 512, 512, per_act_token, per_out_ch,
144
- torch.bfloat16, device)
175
+ bias: bool, device: str):
176
+ cutlass_int8_gemm_helper(512,
177
+ 512,
178
+ 512,
179
+ per_act_token,
180
+ per_out_ch,
181
+ bias,
182
+ out_dtype=torch.bfloat16,
183
+ device=device)
184
185
186
# For the following two tests:
@@ -151,20 +190,25 @@ def test_cutlass_int8_gemm_devices(per_act_token: bool, per_out_ch: bool,
190
# kernel must handle any M thrown at it.
191
@pytest.mark.parametrize("per_act_token", [True, False])
192
@pytest.mark.parametrize("per_out_ch", [True, False])
193
+@pytest.mark.parametrize("bias", [True, False])
194
@pytest.mark.skipif(capability < 89,
195
reason="FP8 is not supported on this GPU type.")
156
-def test_cutlass_fp8_gemm_m_sweep(per_act_token: bool, per_out_ch: bool):
196
+def test_cutlass_fp8_gemm_m_sweep(per_act_token: bool, per_out_ch: bool,
197
+ bias: bool):
198
for nk in range(32, 128, 32):
199
for m in range(1, 128):
159
- cutlass_fp8_gemm_helper(m, nk, nk, per_act_token, per_out_ch)
200
+ cutlass_fp8_gemm_helper(m, nk, nk, per_act_token, per_out_ch, bias)
201
202
203
@pytest.mark.parametrize("per_act_token", [True, False])
204
@pytest.mark.parametrize("per_out_ch", [True, False])
164
-def test_cutlass_int8_gemm_m_sweep(per_act_token: bool, per_out_ch: bool):
205
+@pytest.mark.parametrize("bias", [True, False])
206
+def test_cutlass_int8_gemm_m_sweep(per_act_token: bool, per_out_ch: bool,
207
+ bias: bool):
208
for nk in range(32, 128, 32):
209
for m in range(1, 128):
167
- cutlass_int8_gemm_helper(m, nk, nk, per_act_token, per_out_ch)
210
+ cutlass_int8_gemm_helper(m, nk, nk, per_act_token, per_out_ch,
211
+ bias)
212
213
214
# Test working with a subset of A and B
vllm/_custom_ops.py
+7
-3
@@ -220,9 +220,12 @@ def cutlass_scaled_mm_supports_fp8(cuda_device_capability: int) -> bool:
220
return torch.ops._C.cutlass_scaled_mm_supports_fp8(cuda_device_capability)
221
222
223
-def cutlass_scaled_mm(a: torch.Tensor, b: torch.Tensor, scale_a: torch.Tensor,
223
+def cutlass_scaled_mm(a: torch.Tensor,
224
+ b: torch.Tensor,
225
+ scale_a: torch.Tensor,
226
scale_b: torch.Tensor,
225
- out_dtype: Type[torch.dtype]) -> torch.Tensor:
227
+ out_dtype: Type[torch.dtype],
228
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
229
assert (b.shape[0] % 16 == 0 and b.shape[1] % 16 == 0)
230
assert (out_dtype is torch.bfloat16 or out_dtype is torch.float16)
231
@@ -230,7 +233,8 @@ def cutlass_scaled_mm(a: torch.Tensor, b: torch.Tensor, scale_a: torch.Tensor,
233
n = b.shape[1]
234
out = torch.empty((m, n), dtype=out_dtype, device=a.device)
235
233
- torch.ops._C.cutlass_scaled_mm(out, a, b, scale_a, scale_b)
236
+ torch.ops._C.cutlass_scaled_mm(out, a, b, scale_a, scale_b, bias)
237
+
238
return out
239
240