A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 19:45 UTC
[Build/BugFix] Fix hopper 12.8 build (#14354)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com> Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com>
Lucas Wilkinson committed
Mar 8, 2025 at 03:11 UTC
7caff01a7b1ae454f6c74e7da9523bb89e08ff4c
4 files changed
+96
-73
CMakeLists.txt
+45
-33
@@ -333,36 +333,64 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
333
" in CUDA target architectures, or CUDA not >= 12.0")
334
endif()
335
336
+
337
+ set(SCALED_MM_3X_ARCHS)
338
# The cutlass_scaled_mm kernels for Hopper (c3x, i.e. CUTLASS 3.x) require
337
- # CUDA 12.0 or later (and only work on Hopper, 9.0a for now).
338
- cuda_archs_loose_intersection(SCALED_MM_3X_ARCHS "9.0a;10.0a;10.1a;12.0a" "${CUDA_ARCHS}")
339
- if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0 AND SCALED_MM_3X_ARCHS)
339
+ # CUDA 12.0 or later
340
+ cuda_archs_loose_intersection(SCALED_MM_ARCHS "9.0a;" "${CUDA_ARCHS}")
341
+ if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0 AND SCALED_MM_ARCHS)
342
set(SRCS
341
- "csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu"
343
+ "csrc/quantization/cutlass_w8a8/scaled_mm_c3x_sm90.cu"
344
"csrc/quantization/cutlass_w8a8/c3x/scaled_mm_sm90_fp8.cu"
345
"csrc/quantization/cutlass_w8a8/c3x/scaled_mm_sm90_int8.cu"
346
"csrc/quantization/cutlass_w8a8/c3x/scaled_mm_azp_sm90_int8.cu"
347
"csrc/quantization/cutlass_w8a8/c3x/scaled_mm_blockwise_sm90_fp8.cu")
348
set_gencode_flags_for_srcs(
349
SRCS "${SRCS}"
348
- CUDA_ARCHS "${SCALED_MM_3X_ARCHS}")
350
+ CUDA_ARCHS "${SCALED_MM_ARCHS}")
351
list(APPEND VLLM_EXT_SRC "${SRCS}")
350
- list(APPEND VLLM_GPU_FLAGS "-DENABLE_SCALED_MM_C3X=1")
351
- message(STATUS "Building scaled_mm_c3x for archs: ${SCALED_MM_3X_ARCHS}")
352
+ list(APPEND VLLM_GPU_FLAGS "-DENABLE_SCALED_MM_SM90=1")
353
+ # Let scaled_mm_c2x know it doesn't need to build these arches
354
+ list(APPEND SCALED_MM_3X_ARCHS "${SCALED_MM_ARCHS}")
355
+ message(STATUS "Building scaled_mm_c3x_sm90 for archs: ${SCALED_MM_ARCHS}")
356
else()
353
- if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0 AND SCALED_MM_3X_ARCHS)
354
- message(STATUS "Not building scaled_mm_c3x as CUDA Compiler version is "
357
+ if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0 AND SCALED_MM_ARCHS)
358
+ message(STATUS "Not building scaled_mm_c3x_sm90 as CUDA Compiler version is "
359
"not >= 12.0, we recommend upgrading to CUDA 12.0 or "
360
"later if you intend on running FP8 quantized models on "
361
"Hopper.")
362
else()
359
- message(STATUS "Not building scaled_mm_c3x as no compatible archs found "
363
+ message(STATUS "Not building scaled_mm_c3x_sm90 as no compatible archs found "
364
"in CUDA target architectures")
365
endif()
366
+ endif()
367
363
- # clear SCALED_MM_3X_ARCHS so the scaled_mm_c2x kernels know we didn't
364
- # build any 3x kernels
365
- set(SCALED_MM_3X_ARCHS)
368
+ # The cutlass_scaled_mm kernels for Blackwell (c3x, i.e. CUTLASS 3.x) require
369
+ # CUDA 12.8 or later
370
+ cuda_archs_loose_intersection(SCALED_MM_ARCHS "10.0a;10.1a;12.0a" "${CUDA_ARCHS}")
371
+ if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.8 AND SCALED_MM_ARCHS)
372
+ set(SRCS
373
+ "csrc/quantization/cutlass_w8a8/scaled_mm_c3x_sm100.cu"
374
+ "csrc/quantization/cutlass_w8a8/c3x/scaled_mm_sm100_fp8.cu"
375
+ )
376
+ set_gencode_flags_for_srcs(
377
+ SRCS "${SRCS}"
378
+ CUDA_ARCHS "${SCALED_MM_ARCHS}")
379
+ list(APPEND VLLM_EXT_SRC "${SRCS}")
380
+ list(APPEND VLLM_GPU_FLAGS "-DENABLE_SCALED_MM_SM100=1")
381
+ # Let scaled_mm_c2x know it doesn't need to build these arches
382
+ list(APPEND SCALED_MM_3X_ARCHS "${SCALED_MM_ARCHS}")
383
+ message(STATUS "Building scaled_mm_c3x_sm100 for archs: ${SCALED_MM_ARCHS}")
384
+ else()
385
+ if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.8 AND SCALED_MM_ARCHS)
386
+ message(STATUS "Not building scaled_mm_c3x_sm100 as CUDA Compiler version is "
387
+ "not >= 12.8, we recommend upgrading to CUDA 12.8 or "
388
+ "later if you intend on running FP8 quantized models on "
389
+ "Blackwell.")
390
+ else()
391
+ message(STATUS "Not building scaled_mm_c3x_100 as no compatible archs found "
392
+ "in CUDA target architectures")
393
+ endif()
394
endif()
395
396
#
@@ -395,16 +423,16 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
423
424
# The 2:4 sparse kernels cutlass_scaled_sparse_mm and cutlass_compressor
425
# require CUDA 12.2 or later (and only work on Hopper and Blackwell).
398
- if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.2 AND SCALED_MM_3X_ARCHS)
426
+ if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.2 AND SCALED_MM_ARCHS)
427
set(SRCS "csrc/sparse/cutlass/sparse_scaled_mm_c3x.cu")
428
set_gencode_flags_for_srcs(
429
SRCS "${SRCS}"
402
- CUDA_ARCHS "${SCALED_MM_3X_ARCHS}")
430
+ CUDA_ARCHS "${SCALED_MM_ARCHS}")
431
list(APPEND VLLM_EXT_SRC "${SRCS}")
432
list(APPEND VLLM_GPU_FLAGS "-DENABLE_SPARSE_SCALED_MM_C3X=1")
405
- message(STATUS "Building sparse_scaled_mm_c3x for archs: ${SCALED_MM_3X_ARCHS}")
433
+ message(STATUS "Building sparse_scaled_mm_c3x for archs: ${SCALED_MM_ARCHS}")
434
else()
407
- if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.2 AND SCALED_MM_3X_ARCHS)
435
+ if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.2 AND SCALED_MM_ARCHS)
436
message(STATUS "Not building sparse_scaled_mm_c3x kernels as CUDA Compiler version is "
437
"not >= 12.2, we recommend upgrading to CUDA 12.2 or later "
438
"if you intend on running FP8 sparse quantized models on Hopper.")
@@ -432,22 +460,6 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
460
set(FP4_ARCHS)
461
endif()
462
435
- # FP8 Blackwell Archs
436
- cuda_archs_loose_intersection(BLACKWELL_ARCHS "10.0;10.1;12.0" "${CUDA_ARCHS}")
437
- if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.8 AND BLACKWELL_ARCHS)
438
- set(SRCS
439
- "csrc/quantization/cutlass_w8a8/c3x/scaled_mm_sm100_fp8.cu"
440
- )
441
- set_gencode_flags_for_srcs(
442
- SRCS "${SRCS}"
443
- CUDA_ARCHS "${BLACKWELL_ARCHS}")
444
- list(APPEND VLLM_EXT_SRC "${SRCS}")
445
- message(STATUS "Building FP8 for archs: ${BLACKWELL_ARCHS}")
446
- else()
447
- # clear BLACKWELL_ARCHS
448
- set(BLACKWELL_ARCHS)
449
- endif()
450
-
463
#
464
# Machete kernels
465
csrc/quantization/cutlass_w8a8/scaled_mm_c3x_sm100.cu
new
+34
@@ -0,0 +1,34 @@
1
+#include <cudaTypedefs.h>
2
+#include "c3x/scaled_mm_kernels.hpp"
3
+
4
+#include "cuda_utils.h"
5
+
6
+/*
7
+ This file defines quantized GEMM operations using the CUTLASS 3.x API, for
8
+ NVIDIA GPUs with sm100 (Blackwell).
9
+*/
10
+
11
+#if defined ENABLE_SCALED_MM_SM100 && ENABLE_SCALED_MM_SM100
12
+
13
+void cutlass_scaled_mm_sm100(torch::Tensor& c, torch::Tensor const& a,
14
+ torch::Tensor const& b,
15
+ torch::Tensor const& a_scales,
16
+ torch::Tensor const& b_scales,
17
+ std::optional<torch::Tensor> const& bias) {
18
+ TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
19
+ TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
20
+
21
+ int M = a.size(0), N = b.size(1), K = a.size(1);
22
+ TORCH_CHECK(
23
+ (a_scales.numel() == 1 || a_scales.numel() == a.size(0)) &&
24
+ (b_scales.numel() == 1 || b_scales.numel() == b.size(1)),
25
+ "Currently, block scaled fp8 gemm is not implemented for Blackwell");
26
+
27
+ // Standard per-tensor/per-token/per-channel scaling
28
+ TORCH_CHECK(a_scales.is_contiguous() && b_scales.is_contiguous());
29
+ TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn,
30
+ "Currently, only fp8 gemm is implemented for Blackwell");
31
+ vllm::cutlass_scaled_mm_sm100_fp8(c, a, b, a_scales, b_scales, bias);
32
+}
33
+
34
+#endif
csrc/quantization/cutlass_w8a8/scaled_mm_c3x_sm90.cu
renamed
+3
-24
@@ -5,9 +5,11 @@
5
6
/*
7
This file defines quantized GEMM operations using the CUTLASS 3.x API, for
8
- NVIDIA GPUs with sm90a (Hopper) or later.
8
+ NVIDIA GPUs with sm90a (Hopper).
9
*/
10
11
+#if defined ENABLE_SCALED_MM_SM90 && ENABLE_SCALED_MM_SM90
12
+
13
void cutlass_scaled_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
14
torch::Tensor const& b,
15
torch::Tensor const& a_scales,
@@ -72,27 +74,4 @@ void cutlass_scaled_mm_azp_sm90(torch::Tensor& out, torch::Tensor const& a,
74
azp, bias);
75
}
76
75
-#if defined CUDA_VERSION && CUDA_VERSION >= 12080
76
-
77
-void cutlass_scaled_mm_sm100(torch::Tensor& c, torch::Tensor const& a,
78
- torch::Tensor const& b,
79
- torch::Tensor const& a_scales,
80
- torch::Tensor const& b_scales,
81
- std::optional<torch::Tensor> const& bias) {
82
- TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
83
- TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
84
-
85
- int M = a.size(0), N = b.size(1), K = a.size(1);
86
- TORCH_CHECK(
87
- (a_scales.numel() == 1 || a_scales.numel() == a.size(0)) &&
88
- (b_scales.numel() == 1 || b_scales.numel() == b.size(1)),
89
- "Currently, block scaled fp8 gemm is not implemented for Blackwell");
90
-
91
- // Standard per-tensor/per-token/per-channel scaling
92
- TORCH_CHECK(a_scales.is_contiguous() && b_scales.is_contiguous());
93
- TORCH_CHECK(a.dtype() == torch::kFloat8_e4m3fn,
94
- "Currently, only fp8 gemm is implemented for Blackwell");
95
- vllm::cutlass_scaled_mm_sm100_fp8(c, a, b, a_scales, b_scales, bias);
96
-}
97
-
77
#endif
csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu
+14
-16
@@ -23,12 +23,15 @@ void cutlass_scaled_mm_sm89(torch::Tensor& c, torch::Tensor const& a,
23
torch::Tensor const& b_scales,
24
std::optional<torch::Tensor> const& bias);
25
26
-#if defined ENABLE_SCALED_MM_C3X && ENABLE_SCALED_MM_C3X
26
+#if defined ENABLE_SCALED_MM_SM90 && ENABLE_SCALED_MM_SM90
27
void cutlass_scaled_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
28
torch::Tensor const& b,
29
torch::Tensor const& a_scales,
30
torch::Tensor const& b_scales,
31
std::optional<torch::Tensor> const& bias);
32
+#endif
33
+
34
+#if defined ENABLE_SCALED_MM_SM100 && ENABLE_SCALED_MM_SM100
35
void cutlass_scaled_mm_sm100(torch::Tensor& c, torch::Tensor const& a,
36
torch::Tensor const& b,
37
torch::Tensor const& a_scales,
@@ -60,7 +63,7 @@ void cutlass_scaled_mm_azp_sm89(torch::Tensor& c, torch::Tensor const& a,
63
std::optional<torch::Tensor> const& azp,
64
std::optional<torch::Tensor> const& bias);
65
63
-#if defined CUDA_VERSION && CUDA_VERSION >= 12000
66
+#if defined ENABLE_SCALED_MM_SM90 && ENABLE_SCALED_MM_SM90
67
void cutlass_scaled_mm_azp_sm90(torch::Tensor& c, torch::Tensor const& a,
68
torch::Tensor const& b,
69
torch::Tensor const& a_scales,
@@ -121,26 +124,21 @@ void cutlass_scaled_mm(torch::Tensor& c, torch::Tensor const& a,
124
125
at::cuda::OptionalCUDAGuard const device_guard(device_of(a));
126
int32_t version_num = get_sm_version_num();
124
- // Hopper
125
-
126
- // Guard against compilation issues for sm90 kernels
127
-#if defined ENABLE_SCALED_MM_C3X && ENABLE_SCALED_MM_C3X
127
129
- #if defined CUDA_VERSION && CUDA_VERSION < 12080
130
- if (version_num >= 90 && version_num < 100) {
131
- cutlass_scaled_mm_sm90(c, a, b, a_scales, b_scales, bias);
128
+#if defined ENABLE_SCALED_MM_SM100 && ENABLE_SCALED_MM_SM100
129
+ if (version_num >= 100) {
130
+ cutlass_scaled_mm_sm100(c, a, b, a_scales, b_scales, bias);
131
return;
132
}
134
- #else
133
+#endif
134
+
135
+ // Guard against compilation issues for sm90 kernels
136
+#if defined ENABLE_SCALED_MM_SM90 && ENABLE_SCALED_MM_SM90
137
if (version_num >= 90 && version_num < 100) {
138
+ // Hopper
139
cutlass_scaled_mm_sm90(c, a, b, a_scales, b_scales, bias);
140
return;
138
- } else if (version_num >= 100) {
139
- cutlass_scaled_mm_sm100(c, a, b, a_scales, b_scales, bias);
140
- return;
141
}
142
- #endif
143
-
142
#endif
143
144
#if defined ENABLE_SCALED_MM_C2X && ENABLE_SCALED_MM_C2X
@@ -211,7 +209,7 @@ void cutlass_scaled_mm_azp(torch::Tensor& c, torch::Tensor const& a,
209
210
int32_t version_num = get_sm_version_num();
211
214
-#if defined ENABLE_SCALED_MM_C3X && ENABLE_SCALED_MM_C3X
212
+#if defined ENABLE_SCALED_MM_SM90 && ENABLE_SCALED_MM_SM90
213
if (version_num >= 90) {
214
cutlass_scaled_mm_azp_sm90(c, a, b, a_scales, b_scales, azp_adj, azp, bias);
215
return;