@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ccd4f129e8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Kernel] Add GPU architecture guards to the CUTLASS w8a8 kernels to reduce binary size (#5157)

Co-authored-by: Cody Yu <hao.yu.cody@gmail.com>

Tyler Michael Smith committed Jun 5, 2024 at 13:44 UTC ccd4f129e8ad95191b3c8d6d0e935382b10c5164
2 files changed +87 -37
csrc/quantization/cutlass_w8a8/scaled_mm_dq_c2x.cu
+70 -35
@@ -48,9 +48,44 @@ using namespace cute;
48
49 namespace {
50
51 -template <typename Arch, typename ElementAB_, typename ElementD_,
52 - typename TileShape, typename WarpShape, typename InstructionShape,
53 - int32_t MainLoopStages>
51 +// Wrappers for the GEMM kernel that is used to guard against compilation on
52 +// architectures that will never use the kernel. The purpose of this is to
53 +// reduce the size of the compiled binary.
54 +// __CUDA_ARCH__ is not defined in host code, so this lets us smuggle the ifdef
55 +// into code that will be executed on the device where it is defined.
56 +template <typename Kernel>
57 +struct enable_sm75_to_sm80 : Kernel {
58 + template <typename... Args>
59 + CUTLASS_DEVICE static void invoke(Args&&... args) {
60 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 750 && __CUDA_ARCH__ < 800
61 + Kernel::invoke(std::forward<Args>(args)...);
62 +#endif
63 + }
64 +};
65 +
66 +template <typename Kernel>
67 +struct enable_sm80_to_sm89 : Kernel {
68 + template <typename... Args>
69 + CUTLASS_DEVICE static void invoke(Args&&... args) {
70 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 800 && __CUDA_ARCH__ < 890
71 + Kernel::invoke(std::forward<Args>(args)...);
72 +#endif
73 + }
74 +};
75 +
76 +template <typename Kernel>
77 +struct enable_sm89_to_sm90 : Kernel {
78 + template <typename... Args>
79 + CUTLASS_DEVICE static void invoke(Args&&... args) {
80 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 890 && __CUDA_ARCH__ < 900
81 + Kernel::invoke(std::forward<Args>(args)...);
82 +#endif
83 + }
84 +};
85 +
86 +template <typename Arch, template <typename> typename ArchGuard,
87 + typename ElementAB_, typename ElementD_, typename TileShape,
88 + typename WarpShape, typename InstructionShape, int32_t MainLoopStages>
89 struct cutlass_2x_gemm {
90 using ElementAB = ElementAB_;
91 using ElementD = ElementD_;
@@ -101,7 +136,7 @@ struct cutlass_2x_gemm {
136 using RowMajor = typename cutlass::layout::RowMajor;
137 using ColumnMajor = typename cutlass::layout::ColumnMajor;
138 using KernelType =
104 - typename cutlass::gemm::kernel::DefaultGemmWithVisitor<
139 + ArchGuard<typename cutlass::gemm::kernel::DefaultGemmWithVisitor<
140 ElementAB, RowMajor, cutlass::ComplexTransform::kNone, 16,
141 ElementAB, ColumnMajor, cutlass::ComplexTransform::kNone, 16,
142 float, cutlass::layout::RowMajor, 4,
@@ -112,7 +147,7 @@ struct cutlass_2x_gemm {
147 cutlass::gemm::threadblock::ThreadblockSwizzleStreamK,
148 MainLoopStages, Operator,
149 1 /* epilogue stages */
115 - >::GemmKernel;
150 + >::GemmKernel>;
151 // clang-format on
152
153 using Op = cutlass::gemm::device::GemmUniversalAdapter<KernelType>;
@@ -208,16 +243,16 @@ void cutlass_scaled_mm_dq_sm75(torch::Tensor& out, torch::Tensor const& a,
243 using InstructionShape = typename cutlass::gemm::GemmShape<8, 8, 16>;
244
245 if (out.dtype() == torch::kBFloat16) {
211 - return cutlass_scaled_mm_dq_dispatcher<
212 - cutlass_2x_gemm<cutlass::arch::Sm75, int8_t, cutlass::bfloat16_t,
213 - TileShape, WarpShape, InstructionShape, 2>>(
214 - out, a, b, a_scales, b_scales);
246 + return cutlass_scaled_mm_dq_dispatcher<cutlass_2x_gemm<
247 + cutlass::arch::Sm75, enable_sm75_to_sm80, int8_t, cutlass::bfloat16_t,
248 + TileShape, WarpShape, InstructionShape, 2>>(out, a, b, a_scales,
249 + b_scales);
250 } else {
251 TORCH_CHECK(out.dtype() == torch::kFloat16);
217 - return cutlass_scaled_mm_dq_dispatcher<
218 - cutlass_2x_gemm<cutlass::arch::Sm75, int8_t, cutlass::half_t, TileShape,
219 - WarpShape, InstructionShape, 2>>(out, a, b, a_scales,
220 - b_scales);
252 + return cutlass_scaled_mm_dq_dispatcher<cutlass_2x_gemm<
253 + cutlass::arch::Sm75, enable_sm75_to_sm80, int8_t, cutlass::half_t,
254 + TileShape, WarpShape, InstructionShape, 2>>(out, a, b, a_scales,
255 + b_scales);
256 }
257 }
258
@@ -235,16 +270,16 @@ void cutlass_scaled_mm_dq_sm80(torch::Tensor& out, torch::Tensor const& a,
270 using InstructionShape = typename cutlass::gemm::GemmShape<16, 8, 32>;
271
272 if (out.dtype() == torch::kBFloat16) {
238 - return cutlass_scaled_mm_dq_dispatcher<
239 - cutlass_2x_gemm<cutlass::arch::Sm80, int8_t, cutlass::bfloat16_t,
240 - TileShape, WarpShape, InstructionShape, 5>>(
241 - out, a, b, a_scales, b_scales);
273 + return cutlass_scaled_mm_dq_dispatcher<cutlass_2x_gemm<
274 + cutlass::arch::Sm80, enable_sm80_to_sm89, int8_t, cutlass::bfloat16_t,
275 + TileShape, WarpShape, InstructionShape, 5>>(out, a, b, a_scales,
276 + b_scales);
277 } else {
278 TORCH_CHECK(out.dtype() == torch::kFloat16);
244 - return cutlass_scaled_mm_dq_dispatcher<
245 - cutlass_2x_gemm<cutlass::arch::Sm80, int8_t, cutlass::half_t, TileShape,
246 - WarpShape, InstructionShape, 5>>(out, a, b, a_scales,
247 - b_scales);
279 + return cutlass_scaled_mm_dq_dispatcher<cutlass_2x_gemm<
280 + cutlass::arch::Sm80, enable_sm80_to_sm89, int8_t, cutlass::half_t,
281 + TileShape, WarpShape, InstructionShape, 5>>(out, a, b, a_scales,
282 + b_scales);
283 }
284 }
285
@@ -263,16 +298,16 @@ void cutlass_scaled_mm_dq_sm89(torch::Tensor& out, torch::Tensor const& a,
298 TORCH_CHECK(b.dtype() == torch::kInt8);
299
300 if (out.dtype() == torch::kBFloat16) {
266 - return cutlass_scaled_mm_dq_dispatcher<
267 - cutlass_2x_gemm<cutlass::arch::Sm89, int8_t, cutlass::bfloat16_t,
268 - TileShape, WarpShape, InstructionShape, 5>>(
269 - out, a, b, a_scales, b_scales);
301 + return cutlass_scaled_mm_dq_dispatcher<cutlass_2x_gemm<
302 + cutlass::arch::Sm89, enable_sm89_to_sm90, int8_t, cutlass::bfloat16_t,
303 + TileShape, WarpShape, InstructionShape, 5>>(out, a, b, a_scales,
304 + b_scales);
305 } else {
306 assert(out.dtype() == torch::kFloat16);
272 - return cutlass_scaled_mm_dq_dispatcher<
273 - cutlass_2x_gemm<cutlass::arch::Sm89, int8_t, cutlass::half_t,
274 - TileShape, WarpShape, InstructionShape, 5>>(
275 - out, a, b, a_scales, b_scales);
307 + return cutlass_scaled_mm_dq_dispatcher<cutlass_2x_gemm<
308 + cutlass::arch::Sm89, enable_sm89_to_sm90, int8_t, cutlass::half_t,
309 + TileShape, WarpShape, InstructionShape, 5>>(out, a, b, a_scales,
310 + b_scales);
311 }
312 } else {
313 TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn);
@@ -280,15 +315,15 @@ void cutlass_scaled_mm_dq_sm89(torch::Tensor& out, torch::Tensor const& a,
315
316 if (out.dtype() == torch::kBFloat16) {
317 return cutlass_scaled_mm_dq_dispatcher<cutlass_2x_gemm<
283 - cutlass::arch::Sm89, cutlass::float_e4m3_t, cutlass::bfloat16_t,
284 - TileShape, WarpShape, InstructionShape, 5>>(out, a, b, a_scales,
285 - b_scales);
318 + cutlass::arch::Sm89, enable_sm89_to_sm90, cutlass::float_e4m3_t,
319 + cutlass::bfloat16_t, TileShape, WarpShape, InstructionShape, 5>>(
320 + out, a, b, a_scales, b_scales);
321 } else {
322 TORCH_CHECK(out.dtype() == torch::kFloat16);
323 return cutlass_scaled_mm_dq_dispatcher<cutlass_2x_gemm<
289 - cutlass::arch::Sm89, cutlass::float_e4m3_t, cutlass::half_t,
290 - TileShape, WarpShape, InstructionShape, 5>>(out, a, b, a_scales,
291 - b_scales);
324 + cutlass::arch::Sm89, enable_sm89_to_sm90, cutlass::float_e4m3_t,
325 + cutlass::half_t, TileShape, WarpShape, InstructionShape, 5>>(
326 + out, a, b, a_scales, b_scales);
327 }
328 }
329 }
csrc/quantization/cutlass_w8a8/scaled_mm_dq_c3x.cu
+17 -2
@@ -56,6 +56,21 @@ uint32_t next_pow_2(uint32_t const num) {
56 return 1 << (CHAR_BIT * sizeof(num) - __builtin_clz(num - 1));
57 }
58
59 +// A wrapper for the GEMM kernel that is used to guard against compilation on
60 +// architectures that will never use the kernel. The purpose of this is to
61 +// reduce the size of the compiled binary.
62 +// __CUDA_ARCH__ is not defined in host code, so this lets us smuggle the ifdef
63 +// into code that will be executed on the device where it is defined.
64 +template <typename Kernel>
65 +struct enable_sm90_or_later : Kernel {
66 + template <typename... Args>
67 + CUTLASS_DEVICE void operator()(Args&&... args) {
68 + #if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 900
69 + Kernel::operator()(std::forward<Args>(args)...);
70 + #endif
71 + }
72 +};
73 +
74 template <typename ElementAB_, typename ElementD_, typename TileShape,
75 typename ClusterShape, typename KernelSchedule,
76 typename EpilogueSchedule>
@@ -126,9 +141,9 @@ struct cutlass_3x_gemm {
141 KernelSchedule>::CollectiveOp;
142 // clang-format on
143
129 - using KernelType = cutlass::gemm::kernel::GemmUniversal<
144 + using KernelType = enable_sm90_or_later<cutlass::gemm::kernel::GemmUniversal<
145 cute::Shape<int, int, int, int>, CollectiveMainloop, CollectiveEpilogue,
131 - cutlass::gemm::PersistentScheduler>;
146 + cutlass::gemm::PersistentScheduler>>;
147
148 struct GemmKernel : public KernelType {};
149 };