@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5bfd1bbc98
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Kernel] Adding bias epilogue support for `cutlass_scaled_mm` (#5560)

Co-authored-by: Chih-Chieh-Yang <7364402+cyang49@users.noreply.github.com> Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>

Luka Govedič committed Jun 26, 2024 at 11:16 UTC 5bfd1bbc9831fed39632f071f16bb62373ec1249
8 files changed +383 -134
CMakeLists.txt
+2 -1
@@ -2,7 +2,8 @@ cmake_minimum_required(VERSION 3.21)
2
3 project(vllm_extensions LANGUAGES CXX)
4
5 -option(VLLM_TARGET_DEVICE "Target device backend for vLLM" "cuda")
5 +# CUDA by default, can be overridden by using -DVLLM_TARGET_DEVICE=... (used by setup.py)
6 +set(VLLM_TARGET_DEVICE "cuda" CACHE STRING "Target device backend for vLLM")
7
8 message(STATUS "Build type: ${CMAKE_BUILD_TYPE}")
9 message(STATUS "Target device: ${VLLM_TARGET_DEVICE}")
csrc/ops.h
+2 -1
@@ -96,7 +96,8 @@ bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability);
96
97 void cutlass_scaled_mm(torch::Tensor& out, torch::Tensor const& a,
98 torch::Tensor const& b, torch::Tensor const& a_scales,
99 - torch::Tensor const& b_scales);
99 + torch::Tensor const& b_scales,
100 + c10::optional<torch::Tensor> const& bias);
101
102 #endif
103
csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cu
+170 -58
@@ -77,24 +77,12 @@ struct enable_sm89_to_sm90 : Kernel {
77 };
78
79 /*
80 - This epilogue function defines a quantized GEMM operation similar to
81 - torch._scaled_mm.
82 -
83 - A and B may be both either int8 or fp8_e4m3. A can be quantized per-tensor or
84 - per-row. B can be quantized per-tensor or per-column.
85 - Any combination of per-tensor and per-row or column is supported.
86 - A and B must have symmetric quantization (zero point == 0).
87 -
88 - So the GEMM operation is D = (a_scales * A) (b_scales * B), where the
89 - scales are applied elementwise with numpy-style broadcasting.
90 -
91 - ScaleA and ScaleB define the epilogue functions that apply the scales for
92 - the A and B operands respectively. These scales may be either per-tensor or
93 - per row or column.
94 -*/
80 + * This class provides the common ScaleA and ScaleB descriptors for the
81 + * ScaledEpilogue and ScaledEpilogueBias classes.
82 + */
83 template <typename ElementD, typename OutputTileThreadMap>
96 -struct ScaledEpilogue {
97 - private:
84 +struct ScaledEpilogueBase {
85 + protected:
86 using Accum = cutlass::epilogue::threadblock::VisitorAccFetch;
87
88 using ScaleA = cutlass::epilogue::threadblock::VisitorColOrScalarBroadcast<
@@ -102,6 +90,32 @@ struct ScaledEpilogue {
90
91 using ScaleB = cutlass::epilogue::threadblock::VisitorRowOrScalarBroadcast<
92 OutputTileThreadMap, float, Stride<Int<0>, Int<1>, Int<0>>>;
93 +};
94 +
95 +/*
96 + This epilogue function defines a quantized GEMM operation similar to
97 + torch._scaled_mm.
98 +
99 + A and B may be both either int8 or fp8_e4m3. A can be quantized per-tensor or
100 + per-row. B can be quantized per-tensor or per-column.
101 + Any combination of per-tensor and per-row or column is supported.
102 + A and B must have symmetric quantization (zero point == 0).
103 +
104 + So the GEMM operation is D = (a_scales * A) (b_scales * B), where the
105 + scales are applied elementwise with numpy-style broadcasting.
106 +
107 + ScaleA and ScaleB define the epilogue functions that apply the scales for
108 + the A and B operands respectively. These scales may be either per-tensor or
109 + per row or column.
110 +*/
111 +template <typename ElementD, typename OutputTileThreadMap>
112 +struct ScaledEpilogue
113 + : private ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
114 + private:
115 + using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
116 + using Accum = typename SUPER::Accum;
117 + using ScaleA = typename SUPER::ScaleA;
118 + using ScaleB = typename SUPER::ScaleB;
119
120 using Compute0 = cutlass::epilogue::threadblock::VisitorCompute<
121 cutlass::multiplies, float, float,
@@ -134,6 +148,53 @@ struct ScaledEpilogue {
148 }
149 };
150
151 +template <typename ElementD, typename OutputTileThreadMap>
152 +struct ScaledEpilogueBias
153 + : private ScaledEpilogueBase<ElementD, OutputTileThreadMap> {
154 + private:
155 + using SUPER = ScaledEpilogueBase<ElementD, OutputTileThreadMap>;
156 + using Accum = typename SUPER::Accum;
157 + using ScaleA = typename SUPER::ScaleA;
158 + using ScaleB = typename SUPER::ScaleB;
159 +
160 + using Compute0 = cutlass::epilogue::threadblock::VisitorCompute<
161 + cutlass::multiplies, float, float,
162 + cutlass::FloatRoundStyle::round_to_nearest>;
163 +
164 + using EVTCompute0 =
165 + cutlass::epilogue::threadblock::Sm80EVT<Compute0, ScaleB, Accum>;
166 +
167 + using Compute1 = cutlass::epilogue::threadblock::VisitorCompute<
168 + cutlass::multiply_add, ElementD, float,
169 + cutlass::FloatRoundStyle::round_to_nearest>;
170 +
171 + using Bias = cutlass::epilogue::threadblock::VisitorRowBroadcast<
172 + OutputTileThreadMap, ElementD, Stride<Int<0>, Int<1>, Int<0>>>;
173 +
174 + public:
175 + using EVTCompute = cutlass::epilogue::threadblock::Sm80EVT<Compute1, ScaleA,
176 + EVTCompute0, Bias>;
177 + using ArgumentType = typename EVTCompute::Arguments;
178 +
179 + static ArgumentType prepare_args(torch::Tensor const& a_scales,
180 + torch::Tensor const& b_scales,
181 + torch::Tensor const& bias) {
182 + using ScaleAArgs = typename ScaleA::Arguments;
183 + using ScaleBArgs = typename ScaleB::Arguments;
184 + using BiasArgs = typename Bias::Arguments;
185 +
186 + ScaleBArgs b_args{b_scales.data_ptr<float>(), b_scales.numel() != 1, {}};
187 + ScaleAArgs a_args{a_scales.data_ptr<float>(), a_scales.numel() != 1, {}};
188 + BiasArgs bias_args{static_cast<ElementD*>(bias.data_ptr()), {}};
189 +
190 + typename EVTCompute0::Arguments evt0_compute_args{b_args};
191 +
192 + typename EVTCompute::Arguments evt_compute_args{a_args, evt0_compute_args,
193 + bias_args};
194 + return evt_compute_args;
195 + }
196 +};
197 +
198 template <typename Arch, template <typename> typename ArchGuard,
199 typename ElementAB_, typename ElementD_,
200 template <typename, typename> typename Epilogue_, typename TileShape,
@@ -168,13 +229,13 @@ struct cutlass_2x_gemm {
229 // clang-format off
230 using RowMajor = typename cutlass::layout::RowMajor;
231 using ColumnMajor = typename cutlass::layout::ColumnMajor;
171 - using KernelType =
232 + using KernelType =
233 ArchGuard<typename cutlass::gemm::kernel::DefaultGemmWithVisitor<
173 - ElementAB, RowMajor, cutlass::ComplexTransform::kNone, 16,
174 - ElementAB, ColumnMajor, cutlass::ComplexTransform::kNone, 16,
234 + ElementAB, RowMajor, cutlass::ComplexTransform::kNone, 16,
235 + ElementAB, ColumnMajor, cutlass::ComplexTransform::kNone, 16,
236 float, cutlass::layout::RowMajor, 4,
176 - ElementAcc, float, cutlass::arch::OpClassTensorOp,
177 - Arch,
237 + ElementAcc, float, cutlass::arch::OpClassTensorOp,
238 + Arch,
239 TileShape, WarpShape, InstructionShape,
240 EVTD,
241 cutlass::gemm::threadblock::ThreadblockSwizzleStreamK,
@@ -404,14 +465,13 @@ void cutlass_gemm_sm80_dispatch(torch::Tensor& out, torch::Tensor const& a,
465 }
466 }
467
407 -void cutlass_scaled_mm_sm75(torch::Tensor& out, torch::Tensor const& a,
408 - torch::Tensor const& b,
409 - torch::Tensor const& a_scales,
410 - torch::Tensor const& b_scales) {
468 +template <template <typename, typename> typename Epilogue,
469 + typename... EpilogueArgs>
470 +void cutlass_scaled_mm_sm75_epilogue(torch::Tensor& out, torch::Tensor const& a,
471 + torch::Tensor const& b,
472 + EpilogueArgs&&... epilogue_args) {
473 TORCH_CHECK(a.dtype() == torch::kInt8);
474 TORCH_CHECK(b.dtype() == torch::kInt8);
413 - TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
414 - TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
475
476 using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
477 using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
@@ -420,78 +480,130 @@ void cutlass_scaled_mm_sm75(torch::Tensor& out, torch::Tensor const& a,
480 if (out.dtype() == torch::kBFloat16) {
481 return cutlass_gemm_caller<cutlass_2x_gemm<
482 cutlass::arch::Sm75, enable_sm75_to_sm80, int8_t, cutlass::bfloat16_t,
423 - ScaledEpilogue, TileShape, WarpShape, InstructionShape, 2>>(
424 - out, a, b, a_scales, b_scales);
483 + Epilogue, TileShape, WarpShape, InstructionShape, 2>>(
484 + out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
485 } else {
486 TORCH_CHECK(out.dtype() == torch::kFloat16);
487 return cutlass_gemm_caller<cutlass_2x_gemm<
488 cutlass::arch::Sm75, enable_sm75_to_sm80, int8_t, cutlass::half_t,
429 - ScaledEpilogue, TileShape, WarpShape, InstructionShape, 2>>(
430 - out, a, b, a_scales, b_scales);
489 + Epilogue, TileShape, WarpShape, InstructionShape, 2>>(
490 + out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
491 }
492 }
493
434 -void cutlass_scaled_mm_sm80(torch::Tensor& out, torch::Tensor const& a,
494 +void cutlass_scaled_mm_sm75(torch::Tensor& out, torch::Tensor const& a,
495 torch::Tensor const& b,
496 torch::Tensor const& a_scales,
437 - torch::Tensor const& b_scales) {
438 - TORCH_CHECK(a.dtype() == torch::kInt8);
439 - TORCH_CHECK(b.dtype() == torch::kInt8);
497 + torch::Tensor const& b_scales,
498 + c10::optional<torch::Tensor> const& bias) {
499 TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
500 TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
501 + if (bias) {
502 + TORCH_CHECK(bias->dtype() == out.dtype(),
503 + "currently bias dtype must match output dtype ", out.dtype());
504 + return cutlass_scaled_mm_sm75_epilogue<ScaledEpilogueBias>(
505 + out, a, b, a_scales, b_scales, *bias);
506 + } else {
507 + return cutlass_scaled_mm_sm75_epilogue<ScaledEpilogue>(out, a, b, a_scales,
508 + b_scales);
509 + }
510 +}
511 +
512 +template <template <typename, typename> typename Epilogue,
513 + typename... EpilogueArgs>
514 +void cutlass_scaled_mm_sm80_epilogue(torch::Tensor& out, torch::Tensor const& a,
515 + torch::Tensor const& b,
516 + EpilogueArgs&&... epilogue_args) {
517 + TORCH_CHECK(a.dtype() == torch::kInt8);
518 + TORCH_CHECK(b.dtype() == torch::kInt8);
519
520 if (out.dtype() == torch::kBFloat16) {
444 - return cutlass_gemm_sm80_dispatch<int8_t, cutlass::bfloat16_t,
445 - ScaledEpilogue>(out, a, b, a_scales,
446 - b_scales);
521 + return cutlass_gemm_sm80_dispatch<int8_t, cutlass::bfloat16_t, Epilogue>(
522 + out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
523 } else {
524 TORCH_CHECK(out.dtype() == torch::kFloat16);
449 - return cutlass_gemm_sm80_dispatch<int8_t, cutlass::half_t, ScaledEpilogue>(
450 - out, a, b, a_scales, b_scales);
525 + return cutlass_gemm_sm80_dispatch<int8_t, cutlass::half_t, Epilogue>(
526 + out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
527 }
528 }
529
454 -void cutlass_scaled_mm_sm89(torch::Tensor& out, torch::Tensor const& a,
530 +void cutlass_scaled_mm_sm80(torch::Tensor& out, torch::Tensor const& a,
531 torch::Tensor const& b,
532 torch::Tensor const& a_scales,
457 - torch::Tensor const& b_scales) {
533 + torch::Tensor const& b_scales,
534 + c10::optional<torch::Tensor> const& bias) {
535 + TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
536 + TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
537 + if (bias) {
538 + TORCH_CHECK(bias->dtype() == out.dtype(),
539 + "currently bias dtype must match output dtype ", out.dtype());
540 + return cutlass_scaled_mm_sm80_epilogue<ScaledEpilogueBias>(
541 + out, a, b, a_scales, b_scales, *bias);
542 + } else {
543 + return cutlass_scaled_mm_sm80_epilogue<ScaledEpilogue>(out, a, b, a_scales,
544 + b_scales);
545 + }
546 +}
547 +
548 +template <template <typename, typename> typename Epilogue,
549 + typename... EpilogueArgs>
550 +void cutlass_scaled_mm_sm89_epilogue(torch::Tensor& out, torch::Tensor const& a,
551 + torch::Tensor const& b,
552 + EpilogueArgs&&... epilogue_args) {
553 using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
554 using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
555 using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
556
462 - TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
463 - TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
464 -
557 if (a.dtype() == torch::kInt8) {
558 TORCH_CHECK(b.dtype() == torch::kInt8);
559
560 if (out.dtype() == torch::kBFloat16) {
561 return cutlass_gemm_caller<cutlass_2x_gemm<
562 cutlass::arch::Sm89, enable_sm89_to_sm90, int8_t, cutlass::bfloat16_t,
471 - ScaledEpilogue, TileShape, WarpShape, InstructionShape, 5>>(
472 - out, a, b, a_scales, b_scales);
563 + Epilogue, TileShape, WarpShape, InstructionShape, 5>>(
564 + out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
565 } else {
566 assert(out.dtype() == torch::kFloat16);
567 return cutlass_gemm_caller<cutlass_2x_gemm<
568 cutlass::arch::Sm89, enable_sm89_to_sm90, int8_t, cutlass::half_t,
477 - ScaledEpilogue, TileShape, WarpShape, InstructionShape, 5>>(
478 - out, a, b, a_scales, b_scales);
569 + Epilogue, TileShape, WarpShape, InstructionShape, 5>>(
570 + out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
571 }
572 } else {
573 TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
574 TORCH_CHECK(b.dtype() == torch::kFloat8_e4m3fn);
575
576 if (out.dtype() == torch::kBFloat16) {
485 - return cutlass_gemm_caller<cutlass_2x_gemm<
486 - cutlass::arch::Sm89, enable_sm89_to_sm90, cutlass::float_e4m3_t,
487 - cutlass::bfloat16_t, ScaledEpilogue, TileShape, WarpShape,
488 - InstructionShape, 5>>(out, a, b, a_scales, b_scales);
577 + return cutlass_gemm_caller<
578 + cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
579 + cutlass::float_e4m3_t, cutlass::bfloat16_t, Epilogue,
580 + TileShape, WarpShape, InstructionShape, 5>>(
581 + out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
582 } else {
583 TORCH_CHECK(out.dtype() == torch::kFloat16);
491 - return cutlass_gemm_caller<cutlass_2x_gemm<
492 - cutlass::arch::Sm89, enable_sm89_to_sm90, cutlass::float_e4m3_t,
493 - cutlass::half_t, ScaledEpilogue, TileShape, WarpShape,
494 - InstructionShape, 5>>(out, a, b, a_scales, b_scales);
584 + return cutlass_gemm_caller<
585 + cutlass_2x_gemm<cutlass::arch::Sm89, enable_sm89_to_sm90,
586 + cutlass::float_e4m3_t, cutlass::half_t, Epilogue,
587 + TileShape, WarpShape, InstructionShape, 5>>(
588 + out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
589 }
590 }
591 }
592 +
593 +void cutlass_scaled_mm_sm89(torch::Tensor& out, torch::Tensor const& a,
594 + torch::Tensor const& b,
595 + torch::Tensor const& a_scales,
596 + torch::Tensor const& b_scales,
597 + c10::optional<torch::Tensor> const& bias) {
598 + TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
599 + TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
600 + if (bias) {
601 + TORCH_CHECK(bias->dtype() == out.dtype(),
602 + "currently bias dtype must match output dtype ", out.dtype());
603 + return cutlass_scaled_mm_sm89_epilogue<ScaledEpilogueBias>(
604 + out, a, b, a_scales, b_scales, *bias);
605 + } else {
606 + return cutlass_scaled_mm_sm89_epilogue<ScaledEpilogue>(out, a, b, a_scales,
607 + b_scales);
608 + }
609 +}
csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu
+108 -31
@@ -59,6 +59,28 @@ struct enable_sm90_or_later : Kernel {
59 }
60 };
61
62 +/*
63 + * This class provides the common ScaleA and ScaleB descriptors for the
64 + * ScaledEpilogue and ScaledEpilogueBias classes.
65 + */
66 +template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
67 +struct ScaledEpilogueBase {
68 + protected:
69 + using Accum = cutlass::epilogue::fusion::Sm90AccFetch;
70 +
71 + using ScaleA = cutlass::epilogue::fusion::Sm90ColOrScalarBroadcast<
72 + 0 /*Stages*/, typename EpilogueDescriptor::TileShape, float,
73 + Stride<Int<1>, Int<0>, Int<0>>>;
74 +
75 + using ScaleBDescriptor =
76 + cutlass::epilogue::collective::detail::RowBroadcastDescriptor<
77 + EpilogueDescriptor, float>;
78 +
79 + using ScaleB = cutlass::epilogue::fusion::Sm90RowOrScalarBroadcast<
80 + ScaleBDescriptor::Stages, typename EpilogueDescriptor::TileShape,
81 + typename ScaleBDescriptor::Element, Stride<Int<0>, Int<1>, Int<0>>>;
82 +};
83 +
84 /*
85 This epilogue function defines a quantized GEMM operation similar to
86 torch.scaled_mm_.
@@ -76,21 +98,13 @@ struct enable_sm90_or_later : Kernel {
98 per row or column.
99 */
100 template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
79 -struct ScaledEpilogue {
101 +struct ScaledEpilogue
102 + : private ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor> {
103 private:
81 - using Accum = cutlass::epilogue::fusion::Sm90AccFetch;
82 -
83 - using ScaleA = cutlass::epilogue::fusion::Sm90ColOrScalarBroadcast<
84 - 0 /*Stages*/, typename EpilogueDescriptor::TileShape, float,
85 - Stride<Int<1>, Int<0>, Int<0>>>;
86 -
87 - using ScaleBDescriptor =
88 - cutlass::epilogue::collective::detail::RowBroadcastDescriptor<
89 - EpilogueDescriptor, float>;
90 -
91 - using ScaleB = cutlass::epilogue::fusion::Sm90RowOrScalarBroadcast<
92 - ScaleBDescriptor::Stages, typename EpilogueDescriptor::TileShape,
93 - typename ScaleBDescriptor::Element, Stride<Int<0>, Int<1>, Int<0>>>;
104 + using SUPER = ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor>;
105 + using Accum = typename SUPER::Accum;
106 + using ScaleA = typename SUPER::ScaleA;
107 + using ScaleB = typename SUPER::ScaleB;
108
109 using Compute0 = cutlass::epilogue::fusion::Sm90Compute<
110 cutlass::multiplies, float, float,
@@ -120,6 +134,54 @@ struct ScaledEpilogue {
134 }
135 };
136
137 +template <typename ElementAcc, typename ElementD, typename EpilogueDescriptor>
138 +struct ScaledEpilogueBias
139 + : private ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor> {
140 + private:
141 + using SUPER = ScaledEpilogueBase<ElementAcc, ElementD, EpilogueDescriptor>;
142 + using Accum = typename SUPER::Accum;
143 + using ScaleA = typename SUPER::ScaleA;
144 + using ScaleB = typename SUPER::ScaleB;
145 +
146 + using Compute0 = cutlass::epilogue::fusion::Sm90Compute<
147 + cutlass::multiplies, ElementD, ElementD,
148 + cutlass::FloatRoundStyle::round_to_nearest>;
149 +
150 + using EVTCompute0 =
151 + cutlass::epilogue::fusion::Sm90EVT<Compute0, ScaleB, Accum>;
152 +
153 + using Compute1 = cutlass::epilogue::fusion::Sm90Compute<
154 + cutlass::multiply_add, ElementD, ElementD,
155 + cutlass::FloatRoundStyle::round_to_nearest>;
156 +
157 + using BiasDescriptor =
158 + cutlass::epilogue::collective::detail::RowBroadcastDescriptor<
159 + EpilogueDescriptor, ElementD>;
160 +
161 + using Bias = cutlass::epilogue::fusion::Sm90RowBroadcast<
162 + BiasDescriptor::Stages, typename EpilogueDescriptor::TileShape, ElementD,
163 + Stride<Int<0>, Int<1>, Int<0>>, 128 / sizeof_bits_v<ElementD>, false>;
164 +
165 + public:
166 + using EVTCompute =
167 + cutlass::epilogue::fusion::Sm90EVT<Compute1, ScaleA, EVTCompute0, Bias>;
168 + using ArgumentType = typename EVTCompute::Arguments;
169 +
170 + static ArgumentType prepare_args(torch::Tensor const& a_scales,
171 + torch::Tensor const& b_scales,
172 + torch::Tensor const& bias) {
173 + using ScaleA_Args = typename ScaleA::Arguments;
174 + using ScaleB_Args = typename ScaleB::Arguments;
175 + using Bias_Args = typename Bias::Arguments;
176 +
177 + ScaleA_Args a_args{a_scales.data_ptr<float>(), a_scales.numel() != 1, {}};
178 + ScaleB_Args b_args{b_scales.data_ptr<float>(), b_scales.numel() != 1, {}};
179 + Bias_Args bias_args{static_cast<ElementD*>(bias.data_ptr())};
180 +
181 + return ArgumentType{a_args, {b_args}, bias_args};
182 + }
183 +};
184 +
185 template <typename ElementAB_, typename ElementD_,
186 template <typename, typename, typename> typename Epilogue_,
187 typename TileShape, typename ClusterShape, typename KernelSchedule,
@@ -440,41 +502,56 @@ void cutlass_gemm_sm90_int8_dispatch(torch::Tensor& out, torch::Tensor const& a,
502 }
503 }
504
443 -void cutlass_scaled_mm_sm90(torch::Tensor& out, torch::Tensor const& a,
444 - torch::Tensor const& b,
445 - torch::Tensor const& a_scales,
446 - torch::Tensor const& b_scales) {
447 - TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
448 - TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
449 -
505 +template <template <typename, typename, typename> typename Epilogue,
506 + typename... EpilogueArgs>
507 +void cutlass_scaled_mm_sm90_epilogue(torch::Tensor& out, torch::Tensor const& a,
508 + torch::Tensor const& b,
509 + EpilogueArgs&&... epilogue_args) {
510 if (a.dtype() == torch::kInt8) {
511 TORCH_CHECK(b.dtype() == torch::kInt8);
512
513 if (out.dtype() == torch::kBFloat16) {
514 return cutlass_gemm_sm90_int8_dispatch<int8_t, cutlass::bfloat16_t,
455 - ScaledEpilogue>(
456 - out, a, b, a_scales, b_scales);
515 + Epilogue>(
516 + out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
517 } else {
518 TORCH_CHECK(out.dtype() == torch::kFloat16);
459 - return cutlass_gemm_sm90_int8_dispatch<int8_t, cutlass::half_t,
460 - ScaledEpilogue>(
461 - out, a, b, a_scales, b_scales);
519 + return cutlass_gemm_sm90_int8_dispatch<int8_t, cutlass::half_t, Epilogue>(
520 + out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
521 }
522 } else {
523 TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
524 TORCH_CHECK(b.dtype() == torch::kFloat8_e4m3fn);
525
526 if (out.dtype() == torch::kBFloat16) {
468 - return cutlass_gemm_sm90_fp8_dispatch<
469 - cutlass::float_e4m3_t, cutlass::bfloat16_t, ScaledEpilogue>(
470 - out, a, b, a_scales, b_scales);
527 + return cutlass_gemm_sm90_fp8_dispatch<cutlass::float_e4m3_t,
528 + cutlass::bfloat16_t, Epilogue>(
529 + out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
530 } else {
531 TORCH_CHECK(out.dtype() == torch::kFloat16);
532 return cutlass_gemm_sm90_fp8_dispatch<cutlass::float_e4m3_t,
474 - cutlass::half_t, ScaledEpilogue>(
475 - out, a, b, a_scales, b_scales);
533 + cutlass::half_t, Epilogue>(
534 + out, a, b, std::forward<EpilogueArgs>(epilogue_args)...);
535 }
536 }
537 }
538
539 +void cutlass_scaled_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
540 + torch::Tensor const& b,
541 + torch::Tensor const& a_scales,
542 + torch::Tensor const& b_scales,
543 + c10::optional<torch::Tensor> const& bias) {
544 + TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
545 + TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
546 + if (bias) {
547 + TORCH_CHECK(bias->dtype() == c.dtype(),
548 + "currently bias dtype must match output dtype ", c.dtype());
549 + return cutlass_scaled_mm_sm90_epilogue<ScaledEpilogueBias>(
550 + c, a, b, a_scales, b_scales, *bias);
551 + } else {
552 + return cutlass_scaled_mm_sm90_epilogue<ScaledEpilogue>(c, a, b, a_scales,
553 + b_scales);
554 + }
555 +}
556 +
557 #endif
csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu
+21 -11
@@ -6,23 +6,27 @@
6 void cutlass_scaled_mm_sm75(torch::Tensor& c, torch::Tensor const& a,
7 torch::Tensor const& b,
8 torch::Tensor const& a_scales,
9 - torch::Tensor const& b_scales);
9 + torch::Tensor const& b_scales,
10 + c10::optional<torch::Tensor> const& bias);
11
12 void cutlass_scaled_mm_sm80(torch::Tensor& c, torch::Tensor const& a,
13 torch::Tensor const& b,
14 torch::Tensor const& a_scales,
14 - torch::Tensor const& b_scales);
15 + torch::Tensor const& b_scales,
16 + c10::optional<torch::Tensor> const& bias);
17
18 void cutlass_scaled_mm_sm89(torch::Tensor& c, torch::Tensor const& a,
19 torch::Tensor const& b,
20 torch::Tensor const& a_scales,
19 - torch::Tensor const& b_scales);
21 + torch::Tensor const& b_scales,
22 + c10::optional<torch::Tensor> const& bias);
23
24 #if defined CUDA_VERSION && CUDA_VERSION >= 12000
25 void cutlass_scaled_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
26 torch::Tensor const& b,
27 torch::Tensor const& a_scales,
25 - torch::Tensor const& b_scales);
28 + torch::Tensor const& b_scales,
29 + c10::optional<torch::Tensor> const& bias);
30 #endif
31
32 bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability) {
@@ -43,7 +47,8 @@ bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability) {
47
48 void cutlass_scaled_mm(torch::Tensor& c, torch::Tensor const& a,
49 torch::Tensor const& b, torch::Tensor const& a_scales,
46 - torch::Tensor const& b_scales) {
50 + torch::Tensor const& b_scales,
51 + c10::optional<torch::Tensor> const& bias) {
52 int32_t major_capability;
53 int32_t minor_capability;
54 cudaDeviceGetAttribute(&major_capability, cudaDevAttrComputeCapabilityMajor,
@@ -66,6 +71,11 @@ void cutlass_scaled_mm(torch::Tensor& c, torch::Tensor const& a,
71 b.stride(1) % 16 == 0); // 16 Byte Alignment
72 TORCH_CHECK(a_scales.is_contiguous() && b_scales.is_contiguous());
73
74 + if (bias) {
75 + TORCH_CHECK(bias->numel() == b.size(1) && bias->is_contiguous() &&
76 + bias->dim() == 1);
77 + }
78 +
79 at::cuda::OptionalCUDAGuard const device_guard(device_of(a));
80
81 if (version_num >= 90) {
@@ -73,19 +83,19 @@ void cutlass_scaled_mm(torch::Tensor& c, torch::Tensor const& a,
83
84 // Guard against compilation issues for sm90 kernels
85 #if defined CUDA_VERSION && CUDA_VERSION >= 12000
76 - cutlass_scaled_mm_sm90(c, a, b, a_scales, b_scales);
86 + cutlass_scaled_mm_sm90(c, a, b, a_scales, b_scales, bias);
87 #else
78 - cutlass_scaled_mm_sm80(c, a, b, a_scales, b_scales);
88 + cutlass_scaled_mm_sm80(c, a, b, a_scales, b_scales, bias);
89 #endif
90 } else if (version_num == 89) {
91 // Ada Lovelace
82 - cutlass_scaled_mm_sm89(c, a, b, a_scales, b_scales);
92 + cutlass_scaled_mm_sm89(c, a, b, a_scales, b_scales, bias);
93 } else if (version_num >= 80) {
94 // Ampere
85 - cutlass_scaled_mm_sm80(c, a, b, a_scales, b_scales);
95 + cutlass_scaled_mm_sm80(c, a, b, a_scales, b_scales, bias);
96 } else {
97 // Turing
98 TORCH_CHECK(version_num >= 75);
89 - cutlass_scaled_mm_sm75(c, a, b, a_scales, b_scales);
99 + cutlass_scaled_mm_sm75(c, a, b, a_scales, b_scales, bias);
100 }
91 -}
101 +}
\ No newline at end of file
csrc/torch_bindings.cpp
+1 -1
@@ -142,7 +142,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
142 ops.def(
143 "cutlass_scaled_mm(Tensor! out, Tensor a,"
144 " Tensor b, Tensor a_scales,"
145 - " Tensor b_scales) -> ()");
145 + " Tensor b_scales, Tensor? bias) -> ()");
146 ops.impl("cutlass_scaled_mm", torch::kCUDA, &cutlass_scaled_mm);
147
148 // Check if cutlass scaled_mm is supported for CUDA devices of the given
tests/kernels/test_cutlass.py
+72 -28
@@ -32,6 +32,7 @@ def cutlass_fp8_gemm_helper(m: int,
32 k: int,
33 per_token_act_quant: bool,
34 per_out_channel_weight_quant: bool,
35 + bias: bool,
36 out_dtype: Type[torch.dtype] = torch.bfloat16,
37 device: str = "cuda"):
38 # Test for a cutlass kernel with per-token activation quantization
@@ -46,10 +47,17 @@ def cutlass_fp8_gemm_helper(m: int,
47 (m_a_scales, 1), device=device, dtype=torch.float32) / 10)
48 scale_b = (torch.randn(
49 (1, n_b_scales), device=device, dtype=torch.float32) / 10)
49 -
50 - out = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype)
51 - baseline = torch.mm(scale_a * a.to(dtype=torch.float32),
52 - scale_b * b.to(dtype=torch.float32)).to(out_dtype)
50 + if bias:
51 + # bias term should be > 1 so that the absolute tolerance can catch it
52 + bias_t = torch.rand((n, ), device=device, dtype=out_dtype) + 1.0
53 + out = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias_t)
54 + else:
55 + out = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype)
56 + bias_t = 0
57 +
58 + baseline = (torch.mm(scale_a * a.to(dtype=torch.float32),
59 + scale_b * b.to(dtype=torch.float32)) +
60 + bias_t).to(out_dtype)
61
62 assert torch.allclose(out, baseline, rtol=1e-2, atol=1e-1)
63
@@ -59,6 +67,7 @@ def cutlass_int8_gemm_helper(m: int,
67 k: int,
68 per_token_act_quant: bool,
69 per_out_channel_weight_quant: bool,
70 + bias: bool,
71 out_dtype: Type[torch.dtype] = torch.bfloat16,
72 device: str = "cuda"):
73 # Test for a cutlass kernel with per-token activation quantization
@@ -74,11 +83,17 @@ def cutlass_int8_gemm_helper(m: int,
83 scale_b = (torch.randn(
84 (1, n_b_scales), device=device, dtype=torch.float32) / 10)
85
77 - out = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype)
78 - baseline = torch.mm(scale_a * a.to(dtype=torch.float32),
79 - scale_b *
80 - b.to(dtype=torch.float32)).to(dtype=out_dtype)
81 -
86 + if bias:
87 + # bias term should be > 1 so that the absolute tolerance can catch it
88 + bias_t = torch.rand((n, ), device=device, dtype=out_dtype) + 1.0
89 + out = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias_t)
90 + else:
91 + out = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype)
92 + bias_t = 0
93 +
94 + baseline = (torch.mm(scale_a * a.to(dtype=torch.float32),
95 + scale_b * b.to(dtype=torch.float32)) +
96 + bias_t).to(dtype=out_dtype)
97 assert torch.allclose(out, baseline, rtol=1e-1, atol=1e0)
98
99
@@ -87,11 +102,12 @@ def cutlass_int8_gemm_helper(m: int,
102 @pytest.mark.parametrize("k", [128, 496, 1024])
103 @pytest.mark.parametrize("per_act_token", [True, False])
104 @pytest.mark.parametrize("per_out_ch", [True, False])
105 +@pytest.mark.parametrize("bias", [True, False])
106 @pytest.mark.skipif(capability < 89,
107 reason="FP8 is not supported on this GPU type.")
108 def test_cutlass_fp8_gemm(m: int, n: int, k: int, per_act_token: bool,
93 - per_out_ch: bool):
94 - cutlass_fp8_gemm_helper(m, n, k, per_act_token, per_out_ch)
109 + per_out_ch: bool, bias: bool):
110 + cutlass_fp8_gemm_helper(m, n, k, per_act_token, per_out_ch, bias)
111
112
113 @pytest.mark.parametrize("m", [512, 222, 33, 1])
@@ -99,49 +115,72 @@ def test_cutlass_fp8_gemm(m: int, n: int, k: int, per_act_token: bool,
115 @pytest.mark.parametrize("k", [128, 496, 1024])
116 @pytest.mark.parametrize("per_act_token", [True, False])
117 @pytest.mark.parametrize("per_out_ch", [True, False])
118 +@pytest.mark.parametrize("bias", [True, False])
119 def test_cutlass_int8_gemm(m: int, n: int, k: int, per_act_token: bool,
103 - per_out_ch: bool):
104 - cutlass_int8_gemm_helper(m, n, k, per_act_token, per_out_ch)
120 + per_out_ch: bool, bias: bool):
121 + cutlass_int8_gemm_helper(m, n, k, per_act_token, per_out_ch, bias)
122
123
124 @pytest.mark.parametrize("per_act_token", [True, False])
125 @pytest.mark.parametrize("per_out_ch", [True, False])
126 @pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16])
127 +@pytest.mark.parametrize("bias", [True, False])
128 def test_cutlass_int8_gemm_output_dtype(per_act_token: bool, per_out_ch: bool,
111 - out_dtype: Type[torch.dtype]):
112 - cutlass_int8_gemm_helper(512, 512, 512, per_act_token, per_out_ch,
113 - out_dtype)
129 + out_dtype: Type[torch.dtype],
130 + bias: bool):
131 + cutlass_int8_gemm_helper(512,
132 + 512,
133 + 512,
134 + per_act_token,
135 + per_out_ch,
136 + bias,
137 + out_dtype=out_dtype)
138
139
140 @pytest.mark.parametrize("per_act_token", [True, False])
141 @pytest.mark.parametrize("per_out_ch", [True, False])
142 @pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16])
143 +@pytest.mark.parametrize("bias", [True, False])
144 @pytest.mark.skipif(capability < 89,
145 reason="FP8 is not supported on this GPU type.")
146 def test_cutlass_fp8_gemm_output_dtype(per_act_token: bool, per_out_ch: bool,
122 - out_dtype: Type[torch.dtype]):
123 - cutlass_fp8_gemm_helper(512, 512, 512, per_act_token, per_out_ch,
124 - out_dtype)
147 + out_dtype: Type[torch.dtype],
148 + bias: bool):
149 + cutlass_fp8_gemm_helper(512,
150 + 512,
151 + 512,
152 + per_act_token,
153 + per_out_ch,
154 + bias,
155 + out_dtype=out_dtype)
156
157
158 @pytest.mark.parametrize("per_act_token", [True, False])
159 @pytest.mark.parametrize("per_out_ch", [True, False])
160 +@pytest.mark.parametrize("bias", [True, False])
161 @pytest.mark.parametrize("device", CUDA_DEVICES)
162 @pytest.mark.skipif(capability < 89,
163 reason="FP8 is not supported on this GPU type.")
164 def test_cutlass_fp8_gemm_devices(per_act_token: bool, per_out_ch: bool,
133 - device: str):
134 - cutlass_fp8_gemm_helper(512, 512, 512, per_act_token, per_out_ch,
165 + bias: bool, device: str):
166 + cutlass_fp8_gemm_helper(512, 512, 512, per_act_token, per_out_ch, bias,
167 torch.bfloat16, device)
168
169
170 @pytest.mark.parametrize("per_act_token", [True, False])
171 @pytest.mark.parametrize("per_out_ch", [True, False])
172 +@pytest.mark.parametrize("bias", [True, False])
173 @pytest.mark.parametrize("device", CUDA_DEVICES)
174 def test_cutlass_int8_gemm_devices(per_act_token: bool, per_out_ch: bool,
142 - device: str):
143 - cutlass_int8_gemm_helper(512, 512, 512, per_act_token, per_out_ch,
144 - torch.bfloat16, device)
175 + bias: bool, device: str):
176 + cutlass_int8_gemm_helper(512,
177 + 512,
178 + 512,
179 + per_act_token,
180 + per_out_ch,
181 + bias,
182 + out_dtype=torch.bfloat16,
183 + device=device)
184
185
186 # For the following two tests:
@@ -151,20 +190,25 @@ def test_cutlass_int8_gemm_devices(per_act_token: bool, per_out_ch: bool,
190 # kernel must handle any M thrown at it.
191 @pytest.mark.parametrize("per_act_token", [True, False])
192 @pytest.mark.parametrize("per_out_ch", [True, False])
193 +@pytest.mark.parametrize("bias", [True, False])
194 @pytest.mark.skipif(capability < 89,
195 reason="FP8 is not supported on this GPU type.")
156 -def test_cutlass_fp8_gemm_m_sweep(per_act_token: bool, per_out_ch: bool):
196 +def test_cutlass_fp8_gemm_m_sweep(per_act_token: bool, per_out_ch: bool,
197 + bias: bool):
198 for nk in range(32, 128, 32):
199 for m in range(1, 128):
159 - cutlass_fp8_gemm_helper(m, nk, nk, per_act_token, per_out_ch)
200 + cutlass_fp8_gemm_helper(m, nk, nk, per_act_token, per_out_ch, bias)
201
202
203 @pytest.mark.parametrize("per_act_token", [True, False])
204 @pytest.mark.parametrize("per_out_ch", [True, False])
164 -def test_cutlass_int8_gemm_m_sweep(per_act_token: bool, per_out_ch: bool):
205 +@pytest.mark.parametrize("bias", [True, False])
206 +def test_cutlass_int8_gemm_m_sweep(per_act_token: bool, per_out_ch: bool,
207 + bias: bool):
208 for nk in range(32, 128, 32):
209 for m in range(1, 128):
167 - cutlass_int8_gemm_helper(m, nk, nk, per_act_token, per_out_ch)
210 + cutlass_int8_gemm_helper(m, nk, nk, per_act_token, per_out_ch,
211 + bias)
212
213
214 # Test working with a subset of A and B
vllm/_custom_ops.py
+7 -3
@@ -220,9 +220,12 @@ def cutlass_scaled_mm_supports_fp8(cuda_device_capability: int) -> bool:
220 return torch.ops._C.cutlass_scaled_mm_supports_fp8(cuda_device_capability)
221
222
223 -def cutlass_scaled_mm(a: torch.Tensor, b: torch.Tensor, scale_a: torch.Tensor,
223 +def cutlass_scaled_mm(a: torch.Tensor,
224 + b: torch.Tensor,
225 + scale_a: torch.Tensor,
226 scale_b: torch.Tensor,
225 - out_dtype: Type[torch.dtype]) -> torch.Tensor:
227 + out_dtype: Type[torch.dtype],
228 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
229 assert (b.shape[0] % 16 == 0 and b.shape[1] % 16 == 0)
230 assert (out_dtype is torch.bfloat16 or out_dtype is torch.float16)
231
@@ -230,7 +233,8 @@ def cutlass_scaled_mm(a: torch.Tensor, b: torch.Tensor, scale_a: torch.Tensor,
233 n = b.shape[1]
234 out = torch.empty((m, n), dtype=out_dtype, device=a.device)
235
233 - torch.ops._C.cutlass_scaled_mm(out, a, b, scale_a, scale_b)
236 + torch.ops._C.cutlass_scaled_mm(out, a, b, scale_a, scale_b, bias)
237 +
238 return out
239
240