@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6c916ac8a8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[BugFix] [Kernel] Add Cutlass2x fallback kernels (#5744)

Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>

Varun Sundar Rabindranath committed Jun 24, 2024 at 02:37 UTC 6c916ac8a80d1b2f4e0d0113a67767dc254a3598
2 files changed +54 -6
csrc/quantization/cutlass_w8a8/common.hpp
+8
@@ -17,3 +17,11 @@ inline uint32_t next_pow_2(uint32_t const num) {
17 return 1 << (CHAR_BIT * sizeof(num) - __builtin_clz(num - 1));
18 }
19
20 +inline int get_cuda_max_shared_memory_per_block_opt_in(int const device) {
21 + int max_shared_mem_per_block_opt_in = 0;
22 + cudaDeviceGetAttribute(&max_shared_mem_per_block_opt_in,
23 + cudaDevAttrMaxSharedMemoryPerBlockOptin,
24 + device);
25 + return max_shared_mem_per_block_opt_in;
26 +}
27 +
csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cu
+46 -6
@@ -250,12 +250,39 @@ void cutlass_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
250 CUTLASS_CHECK(status);
251 }
252
253 +template <typename Gemm, typename FallbackGemm, typename... EpilogueArgs>
254 +void fallback_cutlass_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
255 + torch::Tensor const& b,
256 + EpilogueArgs&&... args) {
257 + // In some cases, the GPU isn't able to accommodate the
258 + // shared memory requirements of the Gemm. In such cases, use
259 + // the FallbackGemm instead.
260 + static const int max_shared_mem_per_block_opt_in =
261 + get_cuda_max_shared_memory_per_block_opt_in(0);
262 +
263 + size_t const gemm_shared_mem_size =
264 + sizeof(typename Gemm::KernelType::SharedStorage);
265 + size_t const fallback_gemm_shared_mem_size =
266 + sizeof(typename FallbackGemm::KernelType::SharedStorage);
267 +
268 + if (gemm_shared_mem_size <= max_shared_mem_per_block_opt_in) {
269 + return cutlass_gemm_caller<Gemm>(out, a, b,
270 + std::forward<EpilogueArgs>(args)...);
271 + } else {
272 + TORCH_CHECK(fallback_gemm_shared_mem_size <=
273 + max_shared_mem_per_block_opt_in);
274 + return cutlass_gemm_caller<FallbackGemm>(
275 + out, a, b, std::forward<EpilogueArgs>(args)...);
276 + }
277 +}
278 +
279 template <typename InType, typename OutType,
280 template <typename, typename> typename Epilogue>
281 struct sm80_config_default {
282 // This config is used in 2 cases,
283 // - M in (128, inf)
284 // - M in (64, 128] and N >= 8192
285 + // Shared Memory required by this Gemm - 81920 bytes
286 static_assert(std::is_same<InType, int8_t>());
287 using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
288 using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
@@ -271,6 +298,7 @@ struct sm80_config_M64 {
298 // This config is used in 2 cases,
299 // - M in (32, 64]
300 // - M in (64, 128] and N < 8192
301 + // Shared Memory required by this Gemm - 122880 bytes
302 static_assert(std::is_same<InType, int8_t>());
303 using TileShape = typename cutlass::gemm::GemmShape<64, 128, 128>;
304 using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
@@ -284,6 +312,7 @@ template <typename InType, typename OutType,
312 template <typename, typename> typename Epilogue>
313 struct sm80_config_M32 {
314 // M in (16, 32]
315 + // Shared Memory required by this Gemm - 61440 bytes
316 static_assert(std::is_same<InType, int8_t>());
317 using TileShape = typename cutlass::gemm::GemmShape<32, 64, 128>;
318 using WarpShape = typename cutlass::gemm::GemmShape<32, 64, 64>;
@@ -297,6 +326,7 @@ template <typename InType, typename OutType,
326 template <typename, typename> typename Epilogue>
327 struct sm80_config_M16 {
328 // M in [1, 16]
329 + // Shared Memory required by this Gemm - 51200 bytes
330 static_assert(std::is_same<InType, int8_t>());
331 using TileShape = typename cutlass::gemm::GemmShape<16, 64, 128>;
332 using WarpShape = typename cutlass::gemm::GemmShape<16, 64, 64>;
@@ -331,35 +361,45 @@ void cutlass_gemm_sm80_dispatch(torch::Tensor& out, torch::Tensor const& a,
361 using Cutlass2xGemmM16 =
362 typename sm80_config_M16<InType, OutType, Epilogue>::Cutlass2xGemm;
363
364 + // Due to shared memory requirements, some Gemms may fail to run on some
365 + // GPUs. As the name indicates, the Fallback Gemm is used as an alternative
366 + // in such cases.
367 + // sm80_config_M16 has the least shared-memory requirement. However,
368 + // based on some profiling, we select sm80_config_M32 as a better alternative
369 + // performance wise.
370 + using FallbackGemm =
371 + typename sm80_config_M32<InType, OutType, Epilogue>::Cutlass2xGemm;
372 +
373 uint32_t const m = a.size(0);
374 uint32_t const mp2 =
375 std::max(static_cast<uint32_t>(16), next_pow_2(m)); // next power of 2
376 if (mp2 <= 16) {
377 // M in [1, 16]
339 - return cutlass_gemm_caller<Cutlass2xGemmM16>(
378 + return fallback_cutlass_gemm_caller<Cutlass2xGemmM16, FallbackGemm>(
379 out, a, b, std::forward<EpilogueArgs>(args)...);
380 } else if (mp2 <= 32) {
381 // M in (16, 32]
343 - return cutlass_gemm_caller<Cutlass2xGemmM32>(
382 + return fallback_cutlass_gemm_caller<Cutlass2xGemmM32, FallbackGemm>(
383 out, a, b, std::forward<EpilogueArgs>(args)...);
384 } else if (mp2 <= 64) {
385 // M in (32, 64]
347 - return cutlass_gemm_caller<Cutlass2xGemmM64>(
386 + return fallback_cutlass_gemm_caller<Cutlass2xGemmM64, FallbackGemm>(
387 out, a, b, std::forward<EpilogueArgs>(args)...);
388 } else if (mp2 <= 128) {
389 // M in (64, 128]
390 uint32_t const n = out.size(1);
391 bool const small_n = n < 8192;
392 if (small_n) {
354 - return cutlass_gemm_caller<Cutlass2xGemmM128SmallN>(
393 + return fallback_cutlass_gemm_caller<Cutlass2xGemmM128SmallN,
394 + FallbackGemm>(
395 out, a, b, std::forward<EpilogueArgs>(args)...);
396 } else {
357 - return cutlass_gemm_caller<Cutlass2xGemmM128BigN>(
397 + return fallback_cutlass_gemm_caller<Cutlass2xGemmM128BigN, FallbackGemm>(
398 out, a, b, std::forward<EpilogueArgs>(args)...);
399 }
400 } else {
401 // M in (128, inf)
362 - return cutlass_gemm_caller<Cutlass2xGemmDefault>(
402 + return fallback_cutlass_gemm_caller<Cutlass2xGemmDefault, FallbackGemm>(
403 out, a, b, std::forward<EpilogueArgs>(args)...);
404 }
405 }