A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 14:45 UTC
[Kernel] Marlin MoE: include SM 12.x in default arch list (#40923)
Signed-off-by: Tony Liu <tonyliu0512@gmail.com> Co-authored-by: Tony Liu <tonyliu0512@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Shengqi Chen <harry-chen@outlook.com>
tonyliu312 committed
May 28, 2026 at 15:30 UTC
6cc8577421f2c02b1a908a551e15d4e8fdd2ba49
2 files changed
+26
-6
CMakeLists.txt
+22
-4
@@ -365,16 +365,30 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
365
# are not supported by Machete yet.
366
367
# marlin arches for fp16 output
368
- cuda_archs_loose_intersection(MARLIN_ARCHS "8.0+PTX" "${CUDA_ARCHS}")
368
+ # Family-conditional 12.0f (one cubin for SM12x family) requires CUDA >= 13.0;
369
+ # fall back to architecture-specific 12.0a;12.1a on CUDA < 13.0 (e.g. 12.8).
370
+ if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0)
371
+ cuda_archs_loose_intersection(MARLIN_ARCHS "8.0+PTX;12.0f" "${CUDA_ARCHS}")
372
+ else()
373
+ cuda_archs_loose_intersection(MARLIN_ARCHS "8.0+PTX;12.0a;12.1a" "${CUDA_ARCHS}")
374
+ endif()
375
# marlin has limited support for turing
376
cuda_archs_loose_intersection(MARLIN_SM75_ARCHS "7.5" "${CUDA_ARCHS}")
377
# marlin arches for bf16 output (we need 9.0 for bf16 atomicAdd PTX)
372
- cuda_archs_loose_intersection(MARLIN_BF16_ARCHS "8.0+PTX;9.0+PTX" "${CUDA_ARCHS}")
378
+ if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0)
379
+ cuda_archs_loose_intersection(MARLIN_BF16_ARCHS "8.0+PTX;9.0+PTX;12.0f" "${CUDA_ARCHS}")
380
+ else()
381
+ cuda_archs_loose_intersection(MARLIN_BF16_ARCHS "8.0+PTX;9.0+PTX;12.0a;12.1a" "${CUDA_ARCHS}")
382
+ endif()
383
# marlin arches for fp8 input
384
# - sm80 doesn't support fp8 computation
385
# - sm90 and sm100 don't support QMMA.16832.F32.E4M3.E4M3 SAAS instruction
386
# so we only enable fp8 computation for SM89 (e.g. RTX 40x0) and 12.0 (e.g. RTX 50x0)
377
- cuda_archs_loose_intersection(MARLIN_FP8_ARCHS "8.9;12.0;12.1" "${CUDA_ARCHS}")
387
+ if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0)
388
+ cuda_archs_loose_intersection(MARLIN_FP8_ARCHS "8.9;12.0f" "${CUDA_ARCHS}")
389
+ else()
390
+ cuda_archs_loose_intersection(MARLIN_FP8_ARCHS "8.9;12.0a;12.1a" "${CUDA_ARCHS}")
391
+ endif()
392
# marlin arches for other files
393
cuda_archs_loose_intersection(MARLIN_OTHER_ARCHS "7.5;8.0+PTX" "${CUDA_ARCHS}")
394
@@ -1121,7 +1135,11 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
1135
# moe marlin arches
1136
# note that we always set `use_atomic_add=False` for moe marlin now,
1137
# so we don't need 9.0 for bf16 atomicAdd PTX
1124
- cuda_archs_loose_intersection(MARLIN_MOE_ARCHS "8.0+PTX" "${CUDA_ARCHS}")
1138
+ if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0)
1139
+ cuda_archs_loose_intersection(MARLIN_MOE_ARCHS "8.0+PTX;12.0f" "${CUDA_ARCHS}")
1140
+ else()
1141
+ cuda_archs_loose_intersection(MARLIN_MOE_ARCHS "8.0+PTX;12.0a;12.1a" "${CUDA_ARCHS}")
1142
+ endif()
1143
# moe marlin has limited support for turing
1144
cuda_archs_loose_intersection(MARLIN_MOE_SM75_ARCHS "7.5" "${CUDA_ARCHS}")
1145
# moe marlin arches for fp8 input
csrc/quantization/marlin/marlin.cu
+4
-2
@@ -400,10 +400,12 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
400
"Turing only support FP16 or INT8 activation.");
401
}
402
if (a_type == vllm::kFE4M3fn) {
403
+ TORCH_CHECK(major_capability * 10 + minor_capability >= 89,
404
+ "FP8 only support Ada Lovelace or newer GPUs.");
405
TORCH_CHECK(
406
major_capability * 10 + minor_capability == 89 ||
405
- major_capability * 10 + minor_capability == 120,
406
- "Marlin W4A8-FP8 only support SM89 or SM120 device (It is slower than "
407
+ major_capability == 12,
408
+ "Marlin W4A8-FP8 only support SM89 or SM12x device (It is slower than "
409
"Marlin W4A16 on other devices).");
410
}
411