A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 04:45 UTC
Fix cuda_archs_loose_intersection when handling sm_*a (#20207)
Signed-off-by: Huy Do <huydhn@gmail.com>
Huy Do committed
Jun 29, 2025 at 16:52 UTC
6c9837a761409aecf47e94ae4272879ce0c81590
2 files changed
+26
-21
CMakeLists.txt
+12
-2
@@ -562,7 +562,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
562
"if you intend on running FP8 quantized MoE models on Hopper.")
563
else()
564
message(STATUS "Not building grouped_mm_c3x as no compatible archs found "
565
- "in CUDA target architectures")
565
+ "in CUDA target architectures.")
566
endif()
567
endif()
568
@@ -574,7 +574,17 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
574
SRCS "${SRCS}"
575
CUDA_ARCHS "${CUTLASS_MOE_DATA_ARCHS}")
576
list(APPEND VLLM_EXT_SRC "${SRCS}")
577
- endif()
577
+ message(STATUS "Building moe_data for archs: ${CUTLASS_MOE_DATA_ARCHS}")
578
+ else()
579
+ if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.3 AND CUTLASS_MOE_DATA_ARCHS)
580
+ message(STATUS "Not building moe_data as CUDA Compiler version is "
581
+ "not >= 12.3, we recommend upgrading to CUDA 12.3 or later "
582
+ "if you intend on running FP8 quantized MoE models on Hopper or Blackwell.")
583
+ else()
584
+ message(STATUS "Not building moe_data as no compatible archs found "
585
+ "in CUDA target architectures.")
586
+ endif()
587
+ endif()
588
589
#
590
# Machete kernels
cmake/utils.cmake
+14
-19
@@ -265,8 +265,8 @@ macro(set_gencode_flags_for_srcs)
265
endmacro()
266
267
#
268
-# For the given `SRC_CUDA_ARCHS` list of gencode versions in the form
269
-# `<major>.<minor>[letter]` compute the "loose intersection" with the
268
+# For the given `SRC_CUDA_ARCHS` list of gencode versions in the form
269
+# `<major>.<minor>[letter]` compute the "loose intersection" with the
270
# `TGT_CUDA_ARCHS` list of gencodes. We also support the `+PTX` suffix in
271
# `SRC_CUDA_ARCHS` which indicates that the PTX code should be built when there
272
# is a CUDA_ARCH in `TGT_CUDA_ARCHS` that is equal to or larger than the
@@ -278,7 +278,7 @@ endmacro()
278
# in `SRC_CUDA_ARCHS` that is less or equal to the version in `TGT_CUDA_ARCHS`.
279
# We have special handling for x.0a, if x.0a is in `SRC_CUDA_ARCHS` and x.0 is
280
# in `TGT_CUDA_ARCHS` then we should remove x.0a from `SRC_CUDA_ARCHS` and add
281
-# x.0a to the result (and remove x.0 from TGT_CUDA_ARCHS).
281
+# x.0a to the result (and remove x.0 from TGT_CUDA_ARCHS).
282
# The result is stored in `OUT_CUDA_ARCHS`.
283
#
284
# Example:
@@ -313,21 +313,16 @@ function(cuda_archs_loose_intersection OUT_CUDA_ARCHS SRC_CUDA_ARCHS TGT_CUDA_AR
313
# if x.0a is in SRC_CUDA_ARCHS and x.0 is in CUDA_ARCHS then we should
314
# remove x.0a from SRC_CUDA_ARCHS and add x.0a to _CUDA_ARCHS
315
set(_CUDA_ARCHS)
316
- if ("9.0a" IN_LIST _SRC_CUDA_ARCHS)
317
- list(REMOVE_ITEM _SRC_CUDA_ARCHS "9.0a")
318
- if ("9.0" IN_LIST TGT_CUDA_ARCHS)
319
- list(REMOVE_ITEM _TGT_CUDA_ARCHS "9.0")
320
- set(_CUDA_ARCHS "9.0a")
321
- endif()
322
- endif()
323
-
324
- if ("10.0a" IN_LIST _SRC_CUDA_ARCHS)
325
- list(REMOVE_ITEM _SRC_CUDA_ARCHS "10.0a")
326
- if ("10.0" IN_LIST TGT_CUDA_ARCHS)
327
- list(REMOVE_ITEM _TGT_CUDA_ARCHS "10.0")
328
- set(_CUDA_ARCHS "10.0a")
316
+ foreach(_arch ${_SRC_CUDA_ARCHS})
317
+ if(_arch MATCHES "\\a$")
318
+ list(REMOVE_ITEM _SRC_CUDA_ARCHS "${_arch}")
319
+ string(REPLACE "a" "" _base "${_arch}")
320
+ if ("${_base}" IN_LIST TGT_CUDA_ARCHS)
321
+ list(REMOVE_ITEM _TGT_CUDA_ARCHS "${_base}")
322
+ list(APPEND _CUDA_ARCHS "${_arch}")
323
+ endif()
324
endif()
330
- endif()
325
+ endforeach()
326
327
list(SORT _SRC_CUDA_ARCHS COMPARE NATURAL ORDER ASCENDING)
328
@@ -359,7 +354,7 @@ function(cuda_archs_loose_intersection OUT_CUDA_ARCHS SRC_CUDA_ARCHS TGT_CUDA_AR
354
endforeach()
355
356
list(REMOVE_DUPLICATES _CUDA_ARCHS)
362
-
357
+
358
# reapply +PTX suffix to architectures that requested PTX
359
set(_FINAL_ARCHS)
360
foreach(_arch ${_CUDA_ARCHS})
@@ -370,7 +365,7 @@ function(cuda_archs_loose_intersection OUT_CUDA_ARCHS SRC_CUDA_ARCHS TGT_CUDA_AR
365
endif()
366
endforeach()
367
set(_CUDA_ARCHS ${_FINAL_ARCHS})
373
-
368
+
369
set(${OUT_CUDA_ARCHS} ${_CUDA_ARCHS} PARENT_SCOPE)
370
endfunction()
371