A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 02:15 UTC
Use NVCC `--compress-mode` to reduce binary size by 30% (#20694)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Jul 10, 2025 at 10:26 UTC
49e8c7ea256bd48a36391b5bc72212af39278b67
1 file changed
+19
-12
CMakeLists.txt
+19
-12
@@ -171,6 +171,13 @@ if(NVCC_THREADS AND VLLM_GPU_LANG STREQUAL "CUDA")
171
list(APPEND VLLM_GPU_FLAGS "--threads=${NVCC_THREADS}")
172
endif()
173
174
+#
175
+# Set nvcc fatbin compression.
176
+#
177
+if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8 AND VLLM_GPU_LANG STREQUAL "CUDA")
178
+ list(APPEND VLLM_GPU_FLAGS "-Xfatbin" "-compress-all" "-compress-mode=size")
179
+endif()
180
+
181
182
#
183
# Use FetchContent for C++ dependencies that are compiled as part of vLLM's build process.
@@ -392,7 +399,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
399
# The cutlass_scaled_mm kernels for Hopper (c3x, i.e. CUTLASS 3.x) require
400
# CUDA 12.0 or later
401
cuda_archs_loose_intersection(SCALED_MM_ARCHS "9.0a;" "${CUDA_ARCHS}")
395
- if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0 AND SCALED_MM_ARCHS)
402
+ if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.0 AND SCALED_MM_ARCHS)
403
set(SRCS
404
"csrc/quantization/cutlass_w8a8/scaled_mm_c3x_sm90.cu"
405
"csrc/quantization/cutlass_w8a8/c3x/scaled_mm_sm90_fp8.cu"
@@ -408,7 +415,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
415
list(APPEND SCALED_MM_3X_ARCHS "${SCALED_MM_ARCHS}")
416
message(STATUS "Building scaled_mm_c3x_sm90 for archs: ${SCALED_MM_ARCHS}")
417
else()
411
- if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0 AND SCALED_MM_ARCHS)
418
+ if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.0 AND SCALED_MM_ARCHS)
419
message(STATUS "Not building scaled_mm_c3x_sm90 as CUDA Compiler version is "
420
"not >= 12.0, we recommend upgrading to CUDA 12.0 or "
421
"later if you intend on running FP8 quantized models on "
@@ -423,7 +430,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
430
# The cutlass_scaled_mm kernels for Geforce Blackwell SM120 (c3x, i.e. CUTLASS 3.x) require
431
# CUDA 12.8 or later
432
cuda_archs_loose_intersection(SCALED_MM_ARCHS "12.0;12.0a" "${CUDA_ARCHS}")
426
- if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.8 AND SCALED_MM_ARCHS)
433
+ if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8 AND SCALED_MM_ARCHS)
434
set(SRCS
435
"csrc/quantization/cutlass_w8a8/scaled_mm_c3x_sm120.cu"
436
"csrc/quantization/cutlass_w8a8/c3x/scaled_mm_sm120_fp8.cu"
@@ -437,7 +444,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
444
list(APPEND SCALED_MM_3X_ARCHS "${SCALED_MM_ARCHS}")
445
message(STATUS "Building scaled_mm_c3x_sm120 for archs: ${SCALED_MM_ARCHS}")
446
else()
440
- if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.8 AND SCALED_MM_ARCHS)
447
+ if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8 AND SCALED_MM_ARCHS)
448
message(STATUS "Not building scaled_mm_c3x_sm120 as CUDA Compiler version is "
449
"not >= 12.8, we recommend upgrading to CUDA 12.8 or "
450
"later if you intend on running FP8 quantized models on "
@@ -452,7 +459,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
459
# The cutlass_scaled_mm kernels for Blackwell SM100 (c3x, i.e. CUTLASS 3.x)
460
# require CUDA 12.8 or later
461
cuda_archs_loose_intersection(SCALED_MM_ARCHS "10.0a;10.1a" "${CUDA_ARCHS}")
455
- if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.8 AND SCALED_MM_ARCHS)
462
+ if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8 AND SCALED_MM_ARCHS)
463
set(SRCS
464
"csrc/quantization/cutlass_w8a8/scaled_mm_c3x_sm100.cu"
465
"csrc/quantization/cutlass_w8a8/c3x/scaled_mm_sm100_fp8.cu"
@@ -467,7 +474,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
474
list(APPEND SCALED_MM_3X_ARCHS "${SCALED_MM_ARCHS}")
475
message(STATUS "Building scaled_mm_c3x_sm100 for archs: ${SCALED_MM_ARCHS}")
476
else()
470
- if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.8 AND SCALED_MM_ARCHS)
477
+ if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8 AND SCALED_MM_ARCHS)
478
message(STATUS "Not building scaled_mm_c3x_sm100 as CUDA Compiler version is "
479
"not >= 12.8, we recommend upgrading to CUDA 12.8 or "
480
"later if you intend on running FP8 quantized models on "
@@ -510,7 +517,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
517
# The 2:4 sparse kernels cutlass_scaled_sparse_mm and cutlass_compressor
518
# require CUDA 12.2 or later (and only work on Hopper).
519
cuda_archs_loose_intersection(SCALED_MM_ARCHS "9.0a;" "${CUDA_ARCHS}")
513
- if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.2 AND SCALED_MM_ARCHS)
520
+ if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.2 AND SCALED_MM_ARCHS)
521
set(SRCS "csrc/sparse/cutlass/sparse_scaled_mm_c3x.cu")
522
set_gencode_flags_for_srcs(
523
SRCS "${SRCS}"
@@ -519,7 +526,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
526
list(APPEND VLLM_GPU_FLAGS "-DENABLE_SPARSE_SCALED_MM_C3X=1")
527
message(STATUS "Building sparse_scaled_mm_c3x for archs: ${SCALED_MM_ARCHS}")
528
else()
522
- if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.2 AND SCALED_MM_ARCHS)
529
+ if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.2 AND SCALED_MM_ARCHS)
530
message(STATUS "Not building sparse_scaled_mm_c3x kernels as CUDA Compiler version is "
531
"not >= 12.2, we recommend upgrading to CUDA 12.2 or later "
532
"if you intend on running FP8 sparse quantized models on Hopper.")
@@ -531,7 +538,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
538
539
# FP4 Archs and flags
540
cuda_archs_loose_intersection(FP4_ARCHS "10.0a" "${CUDA_ARCHS}")
534
- if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.8 AND FP4_ARCHS)
541
+ if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8 AND FP4_ARCHS)
542
set(SRCS
543
"csrc/quantization/fp4/nvfp4_quant_kernels.cu"
544
"csrc/quantization/fp4/nvfp4_experts_quant.cu"
@@ -552,7 +559,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
559
560
# CUTLASS MLA Archs and flags
561
cuda_archs_loose_intersection(MLA_ARCHS "10.0a" "${CUDA_ARCHS}")
555
- if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.8 AND MLA_ARCHS)
562
+ if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8 AND MLA_ARCHS)
563
set(SRCS
564
"csrc/attention/mla/cutlass_mla_kernels.cu")
565
set_gencode_flags_for_srcs(
@@ -641,7 +648,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
648
# The machete kernels only work on hopper and require CUDA 12.0 or later.
649
# Only build Machete kernels if we are building for something compatible with sm90a
650
cuda_archs_loose_intersection(MACHETE_ARCHS "9.0a" "${CUDA_ARCHS}")
644
- if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0 AND MACHETE_ARCHS)
651
+ if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.0 AND MACHETE_ARCHS)
652
#
653
# For the Machete kernels we automatically generate sources for various
654
# preselected input type pairs and schedules.
@@ -693,7 +700,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
700
701
message(STATUS "Building Machete kernels for archs: ${MACHETE_ARCHS}")
702
else()
696
- if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0
703
+ if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.0
704
AND MACHETE_ARCHS)
705
message(STATUS "Not building Machete kernels as CUDA Compiler version is "
706
"not >= 12.0, we recommend upgrading to CUDA 12.0 or "