@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 103bd17ac5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Build] Only build 9.0a for scaled_mm and sparse kernels (#12339)

Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>

Lucas Wilkinson committed Jan 27, 2025 at 10:40 UTC 103bd17ac585b44372a47f365d80f13446cf362d
2 files changed +32 -19
CMakeLists.txt
+4 -4
@@ -275,7 +275,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
275 # Only build Marlin kernels if we are building for at least some compatible archs.
276 # Keep building Marlin for 9.0 as there are some group sizes and shapes that
277 # are not supported by Machete yet.
278 - cuda_archs_loose_intersection(MARLIN_ARCHS "8.0;8.6;8.7;8.9;9.0" ${CUDA_ARCHS})
278 + cuda_archs_loose_intersection(MARLIN_ARCHS "8.0;8.6;8.7;8.9;9.0" "${CUDA_ARCHS}")
279 if (MARLIN_ARCHS)
280 set(MARLIN_SRCS
281 "csrc/quantization/fp8/fp8_marlin.cu"
@@ -296,8 +296,8 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
296 endif()
297
298 # The cutlass_scaled_mm kernels for Hopper (c3x, i.e. CUTLASS 3.x) require
299 - # CUDA 12.0 or later (and only work on Hopper, 9.0/9.0a for now).
300 - cuda_archs_loose_intersection(SCALED_MM_3X_ARCHS "9.0;9.0a" "${CUDA_ARCHS}")
299 + # CUDA 12.0 or later (and only work on Hopper, 9.0a for now).
300 + cuda_archs_loose_intersection(SCALED_MM_3X_ARCHS "9.0a" "${CUDA_ARCHS}")
301 if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0 AND SCALED_MM_3X_ARCHS)
302 set(SRCS "csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu")
303 set_gencode_flags_for_srcs(
@@ -351,7 +351,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
351 # 2:4 Sparse Kernels
352
353 # The 2:4 sparse kernels cutlass_scaled_sparse_mm and cutlass_compressor
354 - # require CUDA 12.2 or later (and only work on Hopper, 9.0/9.0a for now).
354 + # require CUDA 12.2 or later (and only work on Hopper, 9.0a for now).
355 if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.2 AND SCALED_MM_3X_ARCHS)
356 set(SRCS "csrc/sparse/cutlass/sparse_compressor_c3x.cu"
357 "csrc/sparse/cutlass/sparse_scaled_mm_c3x.cu")
cmake/utils.cmake
+28 -15
@@ -259,7 +259,7 @@ endmacro()
259 # in `SRC_CUDA_ARCHS` that is less or equal to the version in `TGT_CUDA_ARCHS`.
260 # We have special handling for 9.0a, if 9.0a is in `SRC_CUDA_ARCHS` and 9.0 is
261 # in `TGT_CUDA_ARCHS` then we should remove 9.0a from `SRC_CUDA_ARCHS` and add
262 -# 9.0a to the result.
262 +# 9.0a to the result (and remove 9.0 from TGT_CUDA_ARCHS).
263 # The result is stored in `OUT_CUDA_ARCHS`.
264 #
265 # Example:
@@ -270,34 +270,47 @@ endmacro()
270 #
271 function(cuda_archs_loose_intersection OUT_CUDA_ARCHS SRC_CUDA_ARCHS TGT_CUDA_ARCHS)
272 list(REMOVE_DUPLICATES SRC_CUDA_ARCHS)
273 + set(TGT_CUDA_ARCHS_ ${TGT_CUDA_ARCHS})
274
275 # if 9.0a is in SRC_CUDA_ARCHS and 9.0 is in CUDA_ARCHS then we should
276 # remove 9.0a from SRC_CUDA_ARCHS and add 9.0a to _CUDA_ARCHS
277 set(_CUDA_ARCHS)
278 if ("9.0a" IN_LIST SRC_CUDA_ARCHS)
279 list(REMOVE_ITEM SRC_CUDA_ARCHS "9.0a")
279 - if ("9.0" IN_LIST TGT_CUDA_ARCHS)
280 + if ("9.0" IN_LIST TGT_CUDA_ARCHS_)
281 + list(REMOVE_ITEM TGT_CUDA_ARCHS_ "9.0")
282 set(_CUDA_ARCHS "9.0a")
283 endif()
284 endif()
285
286 list(SORT SRC_CUDA_ARCHS COMPARE NATURAL ORDER ASCENDING)
287
286 - # for each ARCH in CUDA_ARCHS find the highest arch in SRC_CUDA_ARCHS that is
287 - # less or eqault to ARCH
288 - foreach(_ARCH ${CUDA_ARCHS})
289 - set(_TMP_ARCH)
290 - foreach(_SRC_ARCH ${SRC_CUDA_ARCHS})
291 - if (_SRC_ARCH VERSION_LESS_EQUAL _ARCH)
292 - set(_TMP_ARCH ${_SRC_ARCH})
293 - else()
294 - break()
288 + # for each ARCH in TGT_CUDA_ARCHS find the highest arch in SRC_CUDA_ARCHS that
289 + # is less or equal to ARCH (but has the same major version since SASS binary
290 + # compatibility is only forward compatible within the same major version).
291 + foreach(_ARCH ${TGT_CUDA_ARCHS_})
292 + set(_TMP_ARCH)
293 + # Extract the major version of the target arch
294 + string(REGEX REPLACE "^([0-9]+)\\..*$" "\\1" TGT_ARCH_MAJOR "${_ARCH}")
295 + foreach(_SRC_ARCH ${SRC_CUDA_ARCHS})
296 + # Extract the major version of the source arch
297 + string(REGEX REPLACE "^([0-9]+)\\..*$" "\\1" SRC_ARCH_MAJOR "${_SRC_ARCH}")
298 + # Check major-version match AND version-less-or-equal
299 + if (_SRC_ARCH VERSION_LESS_EQUAL _ARCH)
300 + if (SRC_ARCH_MAJOR STREQUAL TGT_ARCH_MAJOR)
301 + set(_TMP_ARCH "${_SRC_ARCH}")
302 + endif()
303 + else()
304 + # If we hit a version greater than the target, we can break
305 + break()
306 + endif()
307 + endforeach()
308 +
309 + # If we found a matching _TMP_ARCH, append it to _CUDA_ARCHS
310 + if (_TMP_ARCH)
311 + list(APPEND _CUDA_ARCHS "${_TMP_ARCH}")
312 endif()
313 endforeach()
297 - if (_TMP_ARCH)
298 - list(APPEND _CUDA_ARCHS ${_TMP_ARCH})
299 - endif()
300 - endforeach()
314
315 list(REMOVE_DUPLICATES _CUDA_ARCHS)
316 set(${OUT_CUDA_ARCHS} ${_CUDA_ARCHS} PARENT_SCOPE)