A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 19:45 UTC
[BugFix/Build] Fix sparse kernels not getting built on hopper (#14572)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Lucas Wilkinson committed
Mar 11, 2025 at 13:09 UTC
07b4b7a37fe6f10294fc2c0c5505208097e44bb9
2 files changed
+8
-3
CMakeLists.txt
+2
-1
@@ -422,7 +422,8 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
422
# 2:4 Sparse Kernels
423
424
# The 2:4 sparse kernels cutlass_scaled_sparse_mm and cutlass_compressor
425
- # require CUDA 12.2 or later (and only work on Hopper and Blackwell).
425
+ # require CUDA 12.2 or later (and only work on Hopper).
426
+ cuda_archs_loose_intersection(SCALED_MM_ARCHS "9.0a;" "${CUDA_ARCHS}")
427
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.2 AND SCALED_MM_ARCHS)
428
set(SRCS "csrc/sparse/cutlass/sparse_scaled_mm_c3x.cu")
429
set_gencode_flags_for_srcs(
csrc/sparse/cutlass/sparse_scaled_mm_entry.cu
+6
-2
@@ -58,7 +58,9 @@ void cutlass_scaled_sparse_mm(torch::Tensor& c, torch::Tensor const& a,
58
59
// Guard against compilation issues for sm90 kernels
60
#if defined ENABLE_SPARSE_SCALED_MM_C3X && ENABLE_SPARSE_SCALED_MM_C3X
61
- if (version_num >= 90) {
61
+ // We build for 9.0a which is not forward compatible, so restrict this to
62
+ // Hopper only
63
+ if (version_num == 90) {
64
cutlass_scaled_sparse_mm_sm90(c, a, bt_nzs, bt_meta, a_scales, b_scales,
65
bias);
66
return;
@@ -82,7 +84,9 @@ std::vector<torch::Tensor> cutlass_sparse_compress(torch::Tensor const& a) {
84
85
// Guard against compilation issues for sm90 kernels
86
#if defined ENABLE_SPARSE_SCALED_MM_C3X && ENABLE_SPARSE_SCALED_MM_C3X
85
- if (version_num >= 90) {
87
+ // We build for 9.0a which is not forward compatible, so restrict this to
88
+ // Hopper only
89
+ if (version_num == 90) {
90
std::vector<torch::Tensor> result_tensors;
91
92
auto [a_meta, a_nzs] = cutlass_sparse_compress_sm90(a);