@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 22482e495e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Bugfix] Flash attention arches not getting set properly (#9062)

Lucas Wilkinson committed Oct 4, 2024 at 11:43 UTC 22482e495e00d409c9b5c78dade6e672ddf7fbc2
1 file changed +11
CMakeLists.txt
+11
@@ -482,6 +482,17 @@ if (NOT VLLM_TARGET_DEVICE STREQUAL "cuda")
482 return()
483 endif ()
484
485 +# vLLM flash attention requires VLLM_GPU_ARCHES to contain the set of target
486 +# arches in the CMake syntax (75-real, 89-virtual, etc), since we clear the
487 +# arches in the CUDA case (and instead set the gencodes on a per file basis)
488 +# we need to manually set VLLM_GPU_ARCHES here.
489 +if(VLLM_GPU_LANG STREQUAL "CUDA")
490 + foreach(_ARCH ${CUDA_ARCHS})
491 + string(REPLACE "." "" _ARCH "${_ARCH}")
492 + list(APPEND VLLM_GPU_ARCHES "${_ARCH}-real")
493 + endforeach()
494 +endif()
495 +
496 #
497 # Build vLLM flash attention from source
498 #