A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
[Bugfix] Fix cutlass dispatch for fp8/int8 to properly invoke M<=16 c… (#16751)
Signed-off-by: Ther-LF <2639852836@qq.com>
TherLF committed
Apr 28, 2025 at 10:38 UTC
c12df53b60d246f5cffd5f011b3141b9e7b5307b
2 files changed
+2
-2
csrc/quantization/cutlass_w8a8/scaled_mm_c2x_sm89_fp8_dispatch.cuh
+1
-1
@@ -336,7 +336,7 @@ inline void cutlass_gemm_sm89_fp8_dispatch(torch::Tensor& out,
336
337
uint32_t const m = a.size(0);
338
uint32_t const mp2 =
339
- std::max(static_cast<uint32_t>(32), next_pow_2(m)); // next power of 2
339
+ std::max(static_cast<uint32_t>(16), next_pow_2(m)); // next power of 2
340
341
if (mp2 <= 16) {
342
// M in [1, 16]
csrc/quantization/cutlass_w8a8/scaled_mm_c2x_sm89_int8_dispatch.cuh
+1
-1
@@ -321,7 +321,7 @@ inline void cutlass_gemm_sm89_int8_dispatch(torch::Tensor& out,
321
322
uint32_t const m = a.size(0);
323
uint32_t const mp2 =
324
- std::max(static_cast<uint32_t>(32), next_pow_2(m)); // next power of 2
324
+ std::max(static_cast<uint32_t>(16), next_pow_2(m)); // next power of 2
325
326
if (mp2 <= 16) {
327
// M in [1, 16]