A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Bugfix] Flash attention arches not getting set properly (#9062)
Lucas Wilkinson committed
Oct 4, 2024 at 11:43 UTC
22482e495e00d409c9b5c78dade6e672ddf7fbc2
1 file changed
+11
CMakeLists.txt
+11
@@ -482,6 +482,17 @@ if (NOT VLLM_TARGET_DEVICE STREQUAL "cuda")
482
return()
483
endif ()
484
485
+# vLLM flash attention requires VLLM_GPU_ARCHES to contain the set of target
486
+# arches in the CMake syntax (75-real, 89-virtual, etc), since we clear the
487
+# arches in the CUDA case (and instead set the gencodes on a per file basis)
488
+# we need to manually set VLLM_GPU_ARCHES here.
489
+if(VLLM_GPU_LANG STREQUAL "CUDA")
490
+ foreach(_ARCH ${CUDA_ARCHS})
491
+ string(REPLACE "." "" _ARCH "${_ARCH}")
492
+ list(APPEND VLLM_GPU_ARCHES "${_ARCH}-real")
493
+ endforeach()
494
+endif()
495
+
496
#
497
# Build vLLM flash attention from source
498
#