A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 21:00 UTC
[NVIDIA] Fix DGX Spark logic (#38126)
Signed-off-by: johnnynunez <johnnynuca14@gmail.com> Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: Nick Hill <nickhill123@gmail.com> Signed-off-by: Mark McLoughlin <markmc@redhat.com> Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com> Signed-off-by: Sathish Sanjeevi <sathish.krishnan.p.s@gmail.com> Signed-off-by: guillaume_guy <guillaume.guy@airbnb.com> Signed-off-by: Guillaume Guy <guillaume.c.guy@gmail.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Mark McLoughlin <markmc@redhat.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu> Co-authored-by: Matthew Wong <Matthew.Wong2@amd.com> Co-authored-by: Sathish Sanjeevi <SKPsanjeevi@users.noreply.github.com> Co-authored-by: Guillaume Guy <guillaume.c.guy@gmail.com> Co-authored-by: guillaume_guy <guillaume.guy@airbnb.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
Johnny committed
Mar 27, 2026 at 23:26 UTC
97d19197bcd05cd018dbf3a8f612b8e30d1b2c5b
3 files changed
+47
-12
CMakeLists.txt
+6
-6
@@ -363,7 +363,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
363
# - sm80 doesn't support fp8 computation
364
# - sm90 and sm100 don't support QMMA.16832.F32.E4M3.E4M3 SAAS instruction
365
# so we only enable fp8 computation for SM89 (e.g. RTX 40x0) and 12.0 (e.g. RTX 50x0)
366
- cuda_archs_loose_intersection(MARLIN_FP8_ARCHS "8.9;12.0" "${CUDA_ARCHS}")
366
+ cuda_archs_loose_intersection(MARLIN_FP8_ARCHS "8.9;12.0;12.1" "${CUDA_ARCHS}")
367
# marlin arches for other files
368
cuda_archs_loose_intersection(MARLIN_OTHER_ARCHS "7.5;8.0+PTX" "${CUDA_ARCHS}")
369
@@ -523,12 +523,12 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
523
endif()
524
525
526
- # The cutlass_scaled_mm kernels for Geforce Blackwell SM120 (c3x, i.e. CUTLASS 3.x) require
526
+ # The cutlass_scaled_mm kernels for Blackwell SM12x (c3x, i.e. CUTLASS 3.x) require
527
# CUDA 12.8 or later
528
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0)
529
cuda_archs_loose_intersection(SCALED_MM_ARCHS "12.0f" "${CUDA_ARCHS}")
530
else()
531
- cuda_archs_loose_intersection(SCALED_MM_ARCHS "12.0a" "${CUDA_ARCHS}")
531
+ cuda_archs_loose_intersection(SCALED_MM_ARCHS "12.0a;12.1a" "${CUDA_ARCHS}")
532
endif()
533
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8 AND SCALED_MM_ARCHS)
534
set(SRCS
@@ -616,12 +616,12 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
616
endif()
617
endif()
618
619
- # The nvfp4_scaled_mm_sm120 kernels for Geforce Blackwell SM120 require
619
+ # The nvfp4_scaled_mm_sm120 kernels for Blackwell SM12x require
620
# CUDA 12.8 or later
621
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0)
622
cuda_archs_loose_intersection(FP4_ARCHS "12.0f" "${CUDA_ARCHS}")
623
else()
624
- cuda_archs_loose_intersection(FP4_ARCHS "12.0a" "${CUDA_ARCHS}")
624
+ cuda_archs_loose_intersection(FP4_ARCHS "12.0a;12.1a" "${CUDA_ARCHS}")
625
endif()
626
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8 AND FP4_ARCHS)
627
set(SRCS
@@ -1050,7 +1050,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
1050
# - sm80 doesn't support fp8 computation
1051
# - sm90 and sm100 don't support QMMA.16832.F32.E4M3.E4M3 SAAS instruction
1052
# so we only enable fp8 computation for SM89 (e.g. RTX 40x0) and 12.0 (e.g. RTX 50x0)
1053
- cuda_archs_loose_intersection(MARLIN_MOE_FP8_ARCHS "8.9;12.0" "${CUDA_ARCHS}")
1053
+ cuda_archs_loose_intersection(MARLIN_MOE_FP8_ARCHS "8.9;12.0;12.1" "${CUDA_ARCHS}")
1054
# moe marlin arches for other files
1055
cuda_archs_loose_intersection(MARLIN_MOE_OTHER_ARCHS "7.5;8.0+PTX" "${CUDA_ARCHS}")
1056
if (MARLIN_MOE_OTHER_ARCHS)
cmake/external_projects/qutlass.cmake
+4
-4
@@ -32,16 +32,16 @@ endif()
32
message(STATUS "[QUTLASS] QuTLASS is available at ${qutlass_SOURCE_DIR}")
33
34
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0)
35
- cuda_archs_loose_intersection(QUTLASS_ARCHS "12.0a;10.0f" "${CUDA_ARCHS}")
35
+ cuda_archs_loose_intersection(QUTLASS_ARCHS "10.0f;12.0f" "${CUDA_ARCHS}")
36
else()
37
- cuda_archs_loose_intersection(QUTLASS_ARCHS "12.0a;10.0a;10.3a" "${CUDA_ARCHS}")
37
+ cuda_archs_loose_intersection(QUTLASS_ARCHS "12.0a;12.1a;10.0a;10.3a" "${CUDA_ARCHS}")
38
endif()
39
40
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8 AND QUTLASS_ARCHS)
41
42
if(QUTLASS_ARCHS MATCHES "10\\.(0a|3a|0f)")
43
set(QUTLASS_TARGET_CC 100)
44
- elseif(QUTLASS_ARCHS MATCHES "12\\.0a")
44
+ elseif(QUTLASS_ARCHS MATCHES "12\\.[01][af]?")
45
set(QUTLASS_TARGET_CC 120)
46
else()
47
message(FATAL_ERROR "[QUTLASS] internal error parsing CUDA_ARCHS='${QUTLASS_ARCHS}'.")
@@ -96,7 +96,7 @@ else()
96
"[QUTLASS] Skipping build: CUDA 12.8 or newer is required (found ${CMAKE_CUDA_COMPILER_VERSION}).")
97
else()
98
message(STATUS
99
- "[QUTLASS] Skipping build: no supported arch (12.0a / 10.0a) found in "
99
+ "[QUTLASS] Skipping build: no supported arch (12.0f / 10.0f) found in "
100
"CUDA_ARCHS='${CUDA_ARCHS}'.")
101
endif()
102
endif()
cmake/utils.cmake
+37
-2
@@ -355,8 +355,11 @@ function(cuda_archs_loose_intersection OUT_CUDA_ARCHS SRC_CUDA_ARCHS TGT_CUDA_AR
355
list(REMOVE_DUPLICATES _PTX_ARCHS)
356
list(REMOVE_DUPLICATES _SRC_CUDA_ARCHS)
357
358
- # If x.0a or x.0f is in SRC_CUDA_ARCHS and x.0 is in CUDA_ARCHS then we should
359
- # remove x.0a or x.0f from SRC_CUDA_ARCHS and add x.0a or x.0f to _CUDA_ARCHS
358
+ # Handle architecture-specific suffixes (a/f) for SRC entries.
359
+ # First try exact base match (x.y), then cross-suffix match (x.ya / x.yf).
360
+ # For 'f' (family) suffix: if no exact/cross match, fall back to major-version
361
+ # match — e.g. SRC="12.0f" matches TGT="12.1a" since SM121 is in the SM12x
362
+ # family. The output uses TGT's value to preserve the user's compilation flags.
363
set(_CUDA_ARCHS)
364
foreach(_arch ${_SRC_CUDA_ARCHS})
365
if(_arch MATCHES "[af]$")
@@ -365,6 +368,38 @@ function(cuda_archs_loose_intersection OUT_CUDA_ARCHS SRC_CUDA_ARCHS TGT_CUDA_AR
368
if ("${_base}" IN_LIST TGT_CUDA_ARCHS)
369
list(REMOVE_ITEM _TGT_CUDA_ARCHS "${_base}")
370
list(APPEND _CUDA_ARCHS "${_arch}")
371
+ elseif("${_base}a" IN_LIST _TGT_CUDA_ARCHS)
372
+ list(REMOVE_ITEM _TGT_CUDA_ARCHS "${_base}a")
373
+ list(APPEND _CUDA_ARCHS "${_base}a")
374
+ elseif("${_base}f" IN_LIST _TGT_CUDA_ARCHS)
375
+ list(REMOVE_ITEM _TGT_CUDA_ARCHS "${_base}f")
376
+ list(APPEND _CUDA_ARCHS "${_base}f")
377
+ elseif(_arch MATCHES "f$")
378
+ # Family suffix: match any TGT entry in the same major version family.
379
+ string(REGEX REPLACE "^([0-9]+)\\..*$" "\\1" _src_major "${_base}")
380
+ foreach(_tgt ${_TGT_CUDA_ARCHS})
381
+ string(REGEX REPLACE "[af]$" "" _tgt_base "${_tgt}")
382
+ string(REGEX REPLACE "^([0-9]+)\\..*$" "\\1" _tgt_major "${_tgt_base}")
383
+ if(_tgt_major STREQUAL _src_major)
384
+ list(REMOVE_ITEM _TGT_CUDA_ARCHS "${_tgt}")
385
+ list(APPEND _CUDA_ARCHS "${_tgt}")
386
+ break()
387
+ endif()
388
+ endforeach()
389
+ endif()
390
+ endif()
391
+ endforeach()
392
+
393
+ # Symmetric handling: if TGT has x.ya/f and SRC has x.y (without suffix),
394
+ # preserve TGT's suffix in the output.
395
+ set(_tgt_copy ${_TGT_CUDA_ARCHS})
396
+ foreach(_arch ${_tgt_copy})
397
+ if(_arch MATCHES "[af]$")
398
+ string(REGEX REPLACE "[af]$" "" _base "${_arch}")
399
+ if ("${_base}" IN_LIST _SRC_CUDA_ARCHS)
400
+ list(REMOVE_ITEM _TGT_CUDA_ARCHS "${_arch}")
401
+ list(REMOVE_ITEM _SRC_CUDA_ARCHS "${_base}")
402
+ list(APPEND _CUDA_ARCHS "${_arch}")
403
endif()
404
endif()
405
endforeach()