@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d1e8240875
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Bugfix] Fix spurious "No compiled cutlass_scaled_mm ..." for W8A8 on Turing (#9487)

Lucas Wilkinson committed Oct 22, 2024 at 18:41 UTC d1e82408759067eca0ae55e548f6243a9e0aa12d
2 files changed +7 -5
CMakeLists.txt
+2 -2
@@ -252,7 +252,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
252 message(STATUS "Building Marlin kernels for archs: ${MARLIN_ARCHS}")
253 else()
254 message(STATUS "Not building Marlin kernels as no compatible archs found"
255 - "in CUDA target architectures")
255 + " in CUDA target architectures")
256 endif()
257
258 #
@@ -432,7 +432,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
432 message(STATUS "Building Marlin MOE kernels for archs: ${MARLIN_MOE_ARCHS}")
433 else()
434 message(STATUS "Not building Marlin MOE kernels as no compatible archs found"
435 - "in CUDA target architectures")
435 + " in CUDA target architectures")
436 endif()
437 endif()
438
csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu
+5 -3
@@ -137,9 +137,11 @@ void cutlass_scaled_mm(torch::Tensor& c, torch::Tensor const& a,
137 return;
138 }
139
140 - // Turing
141 - TORCH_CHECK(version_num >= 75);
142 - cutlass_scaled_mm_sm75(c, a, b, a_scales, b_scales, bias);
140 + if (version_num >= 75) {
141 + // Turing
142 + cutlass_scaled_mm_sm75(c, a, b, a_scales, b_scales, bias);
143 + return;
144 + }
145 #endif
146
147 TORCH_CHECK_NOT_IMPLEMENTED(