A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Bugfix] Fix spurious "No compiled cutlass_scaled_mm ..." for W8A8 on Turing (#9487)
Lucas Wilkinson committed
Oct 22, 2024 at 18:41 UTC
d1e82408759067eca0ae55e548f6243a9e0aa12d
2 files changed
+7
-5
CMakeLists.txt
+2
-2
@@ -252,7 +252,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
252
message(STATUS "Building Marlin kernels for archs: ${MARLIN_ARCHS}")
253
else()
254
message(STATUS "Not building Marlin kernels as no compatible archs found"
255
- "in CUDA target architectures")
255
+ " in CUDA target architectures")
256
endif()
257
258
#
@@ -432,7 +432,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
432
message(STATUS "Building Marlin MOE kernels for archs: ${MARLIN_MOE_ARCHS}")
433
else()
434
message(STATUS "Not building Marlin MOE kernels as no compatible archs found"
435
- "in CUDA target architectures")
435
+ " in CUDA target architectures")
436
endif()
437
endif()
438
csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu
+5
-3
@@ -137,9 +137,11 @@ void cutlass_scaled_mm(torch::Tensor& c, torch::Tensor const& a,
137
return;
138
}
139
140
- // Turing
141
- TORCH_CHECK(version_num >= 75);
142
- cutlass_scaled_mm_sm75(c, a, b, a_scales, b_scales, bias);
140
+ if (version_num >= 75) {
141
+ // Turing
142
+ cutlass_scaled_mm_sm75(c, a, b, a_scales, b_scales, bias);
143
+ return;
144
+ }
145
#endif
146
147
TORCH_CHECK_NOT_IMPLEMENTED(