A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[BugFix] [Kernel] Add Cutlass2x fallback kernels (#5744)
Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
Varun Sundar Rabindranath committed
Jun 24, 2024 at 02:37 UTC
6c916ac8a80d1b2f4e0d0113a67767dc254a3598
2 files changed
+54
-6
csrc/quantization/cutlass_w8a8/common.hpp
+8
@@ -17,3 +17,11 @@ inline uint32_t next_pow_2(uint32_t const num) {
17
return 1 << (CHAR_BIT * sizeof(num) - __builtin_clz(num - 1));
18
}
19
20
+inline int get_cuda_max_shared_memory_per_block_opt_in(int const device) {
21
+ int max_shared_mem_per_block_opt_in = 0;
22
+ cudaDeviceGetAttribute(&max_shared_mem_per_block_opt_in,
23
+ cudaDevAttrMaxSharedMemoryPerBlockOptin,
24
+ device);
25
+ return max_shared_mem_per_block_opt_in;
26
+}
27
+
csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cu
+46
-6
@@ -250,12 +250,39 @@ void cutlass_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
250
CUTLASS_CHECK(status);
251
}
252
253
+template <typename Gemm, typename FallbackGemm, typename... EpilogueArgs>
254
+void fallback_cutlass_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
255
+ torch::Tensor const& b,
256
+ EpilogueArgs&&... args) {
257
+ // In some cases, the GPU isn't able to accommodate the
258
+ // shared memory requirements of the Gemm. In such cases, use
259
+ // the FallbackGemm instead.
260
+ static const int max_shared_mem_per_block_opt_in =
261
+ get_cuda_max_shared_memory_per_block_opt_in(0);
262
+
263
+ size_t const gemm_shared_mem_size =
264
+ sizeof(typename Gemm::KernelType::SharedStorage);
265
+ size_t const fallback_gemm_shared_mem_size =
266
+ sizeof(typename FallbackGemm::KernelType::SharedStorage);
267
+
268
+ if (gemm_shared_mem_size <= max_shared_mem_per_block_opt_in) {
269
+ return cutlass_gemm_caller<Gemm>(out, a, b,
270
+ std::forward<EpilogueArgs>(args)...);
271
+ } else {
272
+ TORCH_CHECK(fallback_gemm_shared_mem_size <=
273
+ max_shared_mem_per_block_opt_in);
274
+ return cutlass_gemm_caller<FallbackGemm>(
275
+ out, a, b, std::forward<EpilogueArgs>(args)...);
276
+ }
277
+}
278
+
279
template <typename InType, typename OutType,
280
template <typename, typename> typename Epilogue>
281
struct sm80_config_default {
282
// This config is used in 2 cases,
283
// - M in (128, inf)
284
// - M in (64, 128] and N >= 8192
285
+ // Shared Memory required by this Gemm - 81920 bytes
286
static_assert(std::is_same<InType, int8_t>());
287
using TileShape = typename cutlass::gemm::GemmShape<128, 128, 64>;
288
using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
@@ -271,6 +298,7 @@ struct sm80_config_M64 {
298
// This config is used in 2 cases,
299
// - M in (32, 64]
300
// - M in (64, 128] and N < 8192
301
+ // Shared Memory required by this Gemm - 122880 bytes
302
static_assert(std::is_same<InType, int8_t>());
303
using TileShape = typename cutlass::gemm::GemmShape<64, 128, 128>;
304
using WarpShape = typename cutlass::gemm::GemmShape<64, 64, 64>;
@@ -284,6 +312,7 @@ template <typename InType, typename OutType,
312
template <typename, typename> typename Epilogue>
313
struct sm80_config_M32 {
314
// M in (16, 32]
315
+ // Shared Memory required by this Gemm - 61440 bytes
316
static_assert(std::is_same<InType, int8_t>());
317
using TileShape = typename cutlass::gemm::GemmShape<32, 64, 128>;
318
using WarpShape = typename cutlass::gemm::GemmShape<32, 64, 64>;
@@ -297,6 +326,7 @@ template <typename InType, typename OutType,
326
template <typename, typename> typename Epilogue>
327
struct sm80_config_M16 {
328
// M in [1, 16]
329
+ // Shared Memory required by this Gemm - 51200 bytes
330
static_assert(std::is_same<InType, int8_t>());
331
using TileShape = typename cutlass::gemm::GemmShape<16, 64, 128>;
332
using WarpShape = typename cutlass::gemm::GemmShape<16, 64, 64>;
@@ -331,35 +361,45 @@ void cutlass_gemm_sm80_dispatch(torch::Tensor& out, torch::Tensor const& a,
361
using Cutlass2xGemmM16 =
362
typename sm80_config_M16<InType, OutType, Epilogue>::Cutlass2xGemm;
363
364
+ // Due to shared memory requirements, some Gemms may fail to run on some
365
+ // GPUs. As the name indicates, the Fallback Gemm is used as an alternative
366
+ // in such cases.
367
+ // sm80_config_M16 has the least shared-memory requirement. However,
368
+ // based on some profiling, we select sm80_config_M32 as a better alternative
369
+ // performance wise.
370
+ using FallbackGemm =
371
+ typename sm80_config_M32<InType, OutType, Epilogue>::Cutlass2xGemm;
372
+
373
uint32_t const m = a.size(0);
374
uint32_t const mp2 =
375
std::max(static_cast<uint32_t>(16), next_pow_2(m)); // next power of 2
376
if (mp2 <= 16) {
377
// M in [1, 16]
339
- return cutlass_gemm_caller<Cutlass2xGemmM16>(
378
+ return fallback_cutlass_gemm_caller<Cutlass2xGemmM16, FallbackGemm>(
379
out, a, b, std::forward<EpilogueArgs>(args)...);
380
} else if (mp2 <= 32) {
381
// M in (16, 32]
343
- return cutlass_gemm_caller<Cutlass2xGemmM32>(
382
+ return fallback_cutlass_gemm_caller<Cutlass2xGemmM32, FallbackGemm>(
383
out, a, b, std::forward<EpilogueArgs>(args)...);
384
} else if (mp2 <= 64) {
385
// M in (32, 64]
347
- return cutlass_gemm_caller<Cutlass2xGemmM64>(
386
+ return fallback_cutlass_gemm_caller<Cutlass2xGemmM64, FallbackGemm>(
387
out, a, b, std::forward<EpilogueArgs>(args)...);
388
} else if (mp2 <= 128) {
389
// M in (64, 128]
390
uint32_t const n = out.size(1);
391
bool const small_n = n < 8192;
392
if (small_n) {
354
- return cutlass_gemm_caller<Cutlass2xGemmM128SmallN>(
393
+ return fallback_cutlass_gemm_caller<Cutlass2xGemmM128SmallN,
394
+ FallbackGemm>(
395
out, a, b, std::forward<EpilogueArgs>(args)...);
396
} else {
357
- return cutlass_gemm_caller<Cutlass2xGemmM128BigN>(
397
+ return fallback_cutlass_gemm_caller<Cutlass2xGemmM128BigN, FallbackGemm>(
398
out, a, b, std::forward<EpilogueArgs>(args)...);
399
}
400
} else {
401
// M in (128, inf)
362
- return cutlass_gemm_caller<Cutlass2xGemmDefault>(
402
+ return fallback_cutlass_gemm_caller<Cutlass2xGemmDefault, FallbackGemm>(
403
out, a, b, std::forward<EpilogueArgs>(args)...);
404
}
405
}