A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[NVIDIA] Blackwell Family (#24673)
Signed-off-by: Johnny <johnnynuca14@gmail.com> Signed-off-by: johnnynunez <johnnynuca14@gmail.com> Signed-off-by: Johnny <johnnync13@gmail.com> Signed-off-by: Salvatore Cena <cena@cenas.it> Co-authored-by: Aidyn-A <31858918+Aidyn-A@users.noreply.github.com> Co-authored-by: Salvatore Cena <cena@cenas.it>
Johnny committed
Oct 1, 2025 at 19:50 UTC
5234dc74514a6b3d0740b39f56a4a4208ec86ecc
5 files changed
+66
-22
CMakeLists.txt
+53
-10
@@ -86,6 +86,9 @@ find_package(Torch REQUIRED)
86
# Supported NVIDIA architectures.
87
# This check must happen after find_package(Torch) because that's when CMAKE_CUDA_COMPILER_VERSION gets defined
88
if(DEFINED CMAKE_CUDA_COMPILER_VERSION AND
89
+ CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 13.0)
90
+ set(CUDA_SUPPORTED_ARCHS "7.5;8.0;8.6;8.7;8.9;9.0;10.0;11.0;12.0")
91
+elseif(DEFINED CMAKE_CUDA_COMPILER_VERSION AND
92
CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 12.8)
93
set(CUDA_SUPPORTED_ARCHS "7.0;7.2;7.5;8.0;8.6;8.7;8.9;9.0;10.0;10.1;12.0")
94
else()
@@ -175,6 +178,15 @@ if(NVCC_THREADS AND VLLM_GPU_LANG STREQUAL "CUDA")
178
list(APPEND VLLM_GPU_FLAGS "--threads=${NVCC_THREADS}")
179
endif()
180
181
+#
182
+# Set compression mode for CUDA >=13.x.
183
+#
184
+if(VLLM_GPU_LANG STREQUAL "CUDA" AND
185
+ DEFINED CMAKE_CUDA_COMPILER_VERSION AND
186
+ CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 13.0)
187
+ list(APPEND VLLM_GPU_FLAGS "--compress-mode=size")
188
+endif()
189
+
190
#
191
# Set CUDA include flags for CXX compiler.
192
#
@@ -270,7 +282,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
282
SET(CUTLASS_ENABLE_HEADERS_ONLY ON CACHE BOOL "Enable only the header library")
283
284
# Set CUTLASS_REVISION. Used for FetchContent. Also fixes some bogus messages when building.
273
- set(CUTLASS_REVISION "v4.0.0" CACHE STRING "CUTLASS revision to use")
285
+ set(CUTLASS_REVISION "v4.2.1" CACHE STRING "CUTLASS revision to use")
286
287
# Use the specified CUTLASS source directory for compilation if VLLM_CUTLASS_SRC_DIR is provided
288
if (DEFINED ENV{VLLM_CUTLASS_SRC_DIR})
@@ -305,7 +317,6 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
317
"csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu"
318
"csrc/quantization/fp4/nvfp4_quant_entry.cu"
319
"csrc/quantization/fp4/nvfp4_scaled_mm_entry.cu"
308
- "csrc/quantization/fp4/nvfp4_blockwise_moe_kernel.cu"
320
"csrc/sparse/cutlass/sparse_scaled_mm_entry.cu"
321
"csrc/cutlass_extensions/common.cpp"
322
"csrc/quantization/fp8/per_token_group_quant.cu")
@@ -440,7 +451,11 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
451
452
# The cutlass_scaled_mm kernels for Geforce Blackwell SM120 (c3x, i.e. CUTLASS 3.x) require
453
# CUDA 12.8 or later
443
- cuda_archs_loose_intersection(SCALED_MM_ARCHS "12.0;12.0a" "${CUDA_ARCHS}")
454
+ if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0)
455
+ cuda_archs_loose_intersection(SCALED_MM_ARCHS "12.0f" "${CUDA_ARCHS}")
456
+ else()
457
+ cuda_archs_loose_intersection(SCALED_MM_ARCHS "12.0a" "${CUDA_ARCHS}")
458
+ endif()
459
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8 AND SCALED_MM_ARCHS)
460
set(SRCS
461
"csrc/quantization/cutlass_w8a8/scaled_mm_c3x_sm120.cu"
@@ -470,7 +485,11 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
485
486
# The cutlass_scaled_mm kernels for Blackwell SM100 (c3x, i.e. CUTLASS 3.x)
487
# require CUDA 12.8 or later
473
- cuda_archs_loose_intersection(SCALED_MM_ARCHS "10.0a;10.1a" "${CUDA_ARCHS}")
488
+ if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0)
489
+ cuda_archs_loose_intersection(SCALED_MM_ARCHS "10.0f;11.0f;12.0f" "${CUDA_ARCHS}")
490
+ else()
491
+ cuda_archs_loose_intersection(SCALED_MM_ARCHS "10.0a;10.1a;10.3a;12.0a;12.1a" "${CUDA_ARCHS}")
492
+ endif()
493
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8 AND SCALED_MM_ARCHS)
494
set(SRCS
495
"csrc/quantization/cutlass_w8a8/scaled_mm_c3x_sm100.cu"
@@ -550,7 +569,11 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
569
570
# The nvfp4_scaled_mm_sm120 kernels for Geforce Blackwell SM120 require
571
# CUDA 12.8 or later
553
- cuda_archs_loose_intersection(FP4_ARCHS "12.0;12.0a" "${CUDA_ARCHS}")
572
+ if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0)
573
+ cuda_archs_loose_intersection(FP4_ARCHS "12.0f" "${CUDA_ARCHS}")
574
+ else()
575
+ cuda_archs_loose_intersection(FP4_ARCHS "12.0a" "${CUDA_ARCHS}")
576
+ endif()
577
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8 AND FP4_ARCHS)
578
set(SRCS
579
"csrc/quantization/fp4/nvfp4_quant_kernels.cu"
@@ -569,7 +592,11 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
592
endif()
593
594
# FP4 Archs and flags
572
- cuda_archs_loose_intersection(FP4_ARCHS "10.0a" "${CUDA_ARCHS}")
595
+ if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0)
596
+ cuda_archs_loose_intersection(FP4_ARCHS "10.0f;11.0f;12.0f" "${CUDA_ARCHS}")
597
+ else()
598
+ cuda_archs_loose_intersection(FP4_ARCHS "10.0a;10.1a;12.0a;12.1a" "${CUDA_ARCHS}")
599
+ endif()
600
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8 AND FP4_ARCHS)
601
set(SRCS
602
"csrc/quantization/fp4/nvfp4_quant_kernels.cu"
@@ -591,7 +618,11 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
618
endif()
619
620
# CUTLASS MLA Archs and flags
594
- cuda_archs_loose_intersection(MLA_ARCHS "10.0a" "${CUDA_ARCHS}")
621
+ if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0)
622
+ cuda_archs_loose_intersection(MLA_ARCHS "10.0f;11.0f;12.0f" "${CUDA_ARCHS}")
623
+ else()
624
+ cuda_archs_loose_intersection(MLA_ARCHS "10.0a;10.1a;10.3a;12.0a;12.1a" "${CUDA_ARCHS}")
625
+ endif()
626
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8 AND MLA_ARCHS)
627
set(SRCS
628
"csrc/attention/mla/sm100_cutlass_mla_kernel.cu")
@@ -635,7 +666,11 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
666
endif()
667
endif()
668
638
- cuda_archs_loose_intersection(SCALED_MM_ARCHS "10.0a" "${CUDA_ARCHS}")
669
+ if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0)
670
+ cuda_archs_loose_intersection(SCALED_MM_ARCHS "10.0f" "${CUDA_ARCHS}")
671
+ else()
672
+ cuda_archs_loose_intersection(SCALED_MM_ARCHS "10.0a" "${CUDA_ARCHS}")
673
+ endif()
674
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8 AND SCALED_MM_ARCHS)
675
set(SRCS "csrc/quantization/cutlass_w8a8/moe/grouped_mm_c3x_sm100.cu")
676
set_gencode_flags_for_srcs(
@@ -656,7 +691,11 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
691
endif()
692
693
# moe_data.cu is used by all CUTLASS MoE kernels.
659
- cuda_archs_loose_intersection(CUTLASS_MOE_DATA_ARCHS "9.0a;10.0a" "${CUDA_ARCHS}")
694
+ if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0)
695
+ cuda_archs_loose_intersection(CUTLASS_MOE_DATA_ARCHS "9.0a;10.0f;11.0f;12.0f" "${CUDA_ARCHS}")
696
+ else()
697
+ cuda_archs_loose_intersection(CUTLASS_MOE_DATA_ARCHS "9.0a;10.0a;10.1a;10.3a;12.0a;12.1a" "${CUDA_ARCHS}")
698
+ endif()
699
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.3 AND CUTLASS_MOE_DATA_ARCHS)
700
set(SRCS "csrc/quantization/cutlass_w8a8/moe/moe_data.cu")
701
set_gencode_flags_for_srcs(
@@ -675,7 +714,11 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
714
endif()
715
endif()
716
678
- cuda_archs_loose_intersection(SCALED_MM_ARCHS "10.0a" "${CUDA_ARCHS}")
717
+ if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0)
718
+ cuda_archs_loose_intersection(SCALED_MM_ARCHS "10.0f;11.0f;12.0f" "${CUDA_ARCHS}")
719
+ else()
720
+ cuda_archs_loose_intersection(SCALED_MM_ARCHS "10.0a;10.1a;10.3a;12.0a;12.1a" "${CUDA_ARCHS}")
721
+ endif()
722
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8 AND SCALED_MM_ARCHS)
723
set(SRCS "csrc/quantization/cutlass_w8a8/moe/blockwise_scaled_group_mm_sm100.cu")
724
set_gencode_flags_for_srcs(
cmake/utils.cmake
+4
-4
@@ -310,13 +310,13 @@ function(cuda_archs_loose_intersection OUT_CUDA_ARCHS SRC_CUDA_ARCHS TGT_CUDA_AR
310
list(REMOVE_DUPLICATES _PTX_ARCHS)
311
list(REMOVE_DUPLICATES _SRC_CUDA_ARCHS)
312
313
- # if x.0a is in SRC_CUDA_ARCHS and x.0 is in CUDA_ARCHS then we should
314
- # remove x.0a from SRC_CUDA_ARCHS and add x.0a to _CUDA_ARCHS
313
+ # If x.0a or x.0f is in SRC_CUDA_ARCHS and x.0 is in CUDA_ARCHS then we should
314
+ # remove x.0a or x.0f from SRC_CUDA_ARCHS and add x.0a or x.0f to _CUDA_ARCHS
315
set(_CUDA_ARCHS)
316
foreach(_arch ${_SRC_CUDA_ARCHS})
317
- if(_arch MATCHES "\\a$")
317
+ if(_arch MATCHES "[af]$")
318
list(REMOVE_ITEM _SRC_CUDA_ARCHS "${_arch}")
319
- string(REPLACE "a" "" _base "${_arch}")
319
+ string(REGEX REPLACE "[af]$" "" _base "${_arch}")
320
if ("${_base}" IN_LIST TGT_CUDA_ARCHS)
321
list(REMOVE_ITEM _TGT_CUDA_ARCHS "${_base}")
322
list(APPEND _CUDA_ARCHS "${_arch}")
csrc/quantization/cutlass_w8a8/c3x/scaled_mm_blockwise_sm100_fp8_dispatch.cuh
+5
-5
@@ -231,7 +231,7 @@ void cutlass_gemm_blockwise_sm100_fp8_dispatch(torch::Tensor& out,
231
} else {
232
cutlass_gemm_caller_blockwise<cutlass_3x_gemm_fp8_blockwise<
233
OutType, 1, TILE_N, TILE_K, Shape<_64, Int<TILE_N>, Int<TILE_K>>,
234
- Shape<_1, _1, _1>, cutlass::epilogue::NoSmemWarpSpecialized1Sm,
234
+ Shape<_1, _1, _1>, cutlass::epilogue::BlockwiseNoSmemWarpSpecialized1Sm,
235
cutlass::gemm::KernelTmaWarpSpecializedBlockwise1SmSm100>>(
236
out, a, b, a_scales, b_scales);
237
}
@@ -245,7 +245,7 @@ void cutlass_gemm_blockwise_sm100_fp8_dispatch(torch::Tensor& out,
245
} else {
246
cutlass_gemm_caller_blockwise<cutlass_3x_gemm_fp8_blockwise<
247
OutType, 1, TILE_N, TILE_K, Shape<_128, Int<TILE_N>, Int<TILE_K>>,
248
- Shape<_1, _1, _1>, cutlass::epilogue::NoSmemWarpSpecialized1Sm,
248
+ Shape<_1, _1, _1>, cutlass::epilogue::BlockwiseNoSmemWarpSpecialized1Sm,
249
cutlass::gemm::KernelTmaWarpSpecializedBlockwise1SmSm100>>(
250
out, a, b, a_scales, b_scales);
251
}
@@ -259,7 +259,7 @@ void cutlass_gemm_blockwise_sm100_fp8_dispatch(torch::Tensor& out,
259
} else {
260
cutlass_gemm_caller_blockwise<cutlass_3x_gemm_fp8_blockwise<
261
OutType, 1, TILE_N, TILE_K, Shape<_256, Int<TILE_N>, Int<TILE_K>>,
262
- Shape<_2, _1, _1>, cutlass::epilogue::NoSmemWarpSpecialized2Sm,
262
+ Shape<_2, _1, _1>, cutlass::epilogue::BlockwiseNoSmemWarpSpecialized2Sm,
263
cutlass::gemm::KernelTmaWarpSpecializedBlockwise2SmSm100>>(
264
out, a, b, a_scales, b_scales);
265
}
@@ -271,10 +271,10 @@ void cutlass_gemm_blockwise_sm100_fp8_dispatch(torch::Tensor& out,
271
// TMA epilogue isn't compatible with Swap A/B
272
cutlass_gemm_caller_blockwise<cutlass_3x_gemm_fp8_blockwise<
273
OutType, TILE_M, 1, TILE_K, Shape<Int<TILE_M>, Int<TILE_N>, Int<TILE_K>>,
274
- Shape<_1, _1, _1>, cutlass::epilogue::NoSmemWarpSpecialized1Sm,
274
+ Shape<_1, _1, _1>, cutlass::epilogue::BlockwiseNoSmemWarpSpecialized1Sm,
275
cutlass::gemm::KernelTmaWarpSpecializedBlockwise1SmSm100, true>>(
276
out, a, b, a_scales, b_scales);
277
}
278
}
279
280
-} // namespace vllm
280
+} // namespace vllm
\ No newline at end of file
csrc/quantization/cutlass_w8a8/c3x/scaled_mm_sm100_fp8_dispatch.cuh
+1
-1
@@ -133,4 +133,4 @@ void cutlass_scaled_mm_sm100_fp8_epilogue(torch::Tensor& out,
133
}
134
}
135
136
-} // namespace vllm
\ No newline at end of file
136
+} // namespace vllm
csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu
+3
-2
@@ -67,8 +67,9 @@ void cutlass_scaled_mm_sm100(torch::Tensor& c, torch::Tensor const& a,
67
std::optional<torch::Tensor> const& bias);
68
#endif
69
70
-#if defined(ENABLE_SCALED_MM_SM90) && ENABLE_SCALED_MM_SM90 || \
71
- defined(ENABLE_SCALED_MM_SM100) && ENABLE_SCALED_MM_SM100
70
+#if defined(ENABLE_SCALED_MM_SM90) && ENABLE_SCALED_MM_SM90 || \
71
+ defined(ENABLE_SCALED_MM_SM100) && ENABLE_SCALED_MM_SM100 || \
72
+ defined(ENABLE_SCALED_MM_SM120) && ENABLE_SCALED_MM_SM120
73
void get_cutlass_moe_mm_data_caller(
74
const torch::Tensor& topk_ids, torch::Tensor& expert_offsets,
75
torch::Tensor& problem_sizes1, torch::Tensor& problem_sizes2,