@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 71c60491f2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Kernel] Build flash-attn from source (#8245)

Luka Govedič committed Sep 21, 2024 at 02:27 UTC 71c60491f287d8a23bed1743513b4b3e7927c69e
9 files changed +124 -41
.github/workflows/scripts/build.sh
+1
@@ -15,5 +15,6 @@ $python_executable -m pip install -r requirements-cuda.txt
15 export MAX_JOBS=1
16 # Make sure release wheels are built for the following architectures
17 export TORCH_CUDA_ARCH_LIST="7.0 7.5 8.0 8.6 8.9 9.0+PTX"
18 +export VLLM_FA_CMAKE_GPU_ARCHES="80-real;90-real"
19 # Build
20 $python_executable setup.py bdist_wheel --dist-dir=dist
.gitignore
+5
@@ -1,6 +1,9 @@
1 # vllm commit id, generated by setup.py
2 vllm/commit_id.py
3
4 +# vllm-flash-attn built from source
5 +vllm/vllm_flash_attn/
6 +
7 # Byte-compiled / optimized / DLL files
8 __pycache__/
9 *.py[cod]
@@ -12,6 +15,8 @@ __pycache__/
15 # Distribution / packaging
16 .Python
17 build/
18 +cmake-build-*/
19 +CMakeUserPresets.json
20 develop-eggs/
21 dist/
22 downloads/
CMakeLists.txt
+73 -25
@@ -1,5 +1,16 @@
1 cmake_minimum_required(VERSION 3.26)
2
3 +# When building directly using CMake, make sure you run the install step
4 +# (it places the .so files in the correct location).
5 +#
6 +# Example:
7 +# mkdir build && cd build
8 +# cmake -G Ninja -DVLLM_PYTHON_EXECUTABLE=`which python3` -DCMAKE_INSTALL_PREFIX=.. ..
9 +# cmake --build . --target install
10 +#
11 +# If you want to only build one target, make sure to install it manually:
12 +# cmake --build . --target _C
13 +# cmake --install . --component _C
14 project(vllm_extensions LANGUAGES CXX)
15
16 # CUDA by default, can be overridden by using -DVLLM_TARGET_DEVICE=... (used by setup.py)
@@ -13,6 +24,9 @@ include(${CMAKE_CURRENT_LIST_DIR}/cmake/utils.cmake)
24 # Suppress potential warnings about unused manually-specified variables
25 set(ignoreMe "${VLLM_PYTHON_PATH}")
26
27 +# Prevent installation of dependencies (cutlass) by default.
28 +install(CODE "set(CMAKE_INSTALL_LOCAL_ONLY TRUE)" ALL_COMPONENTS)
29 +
30 #
31 # Supported python versions. These versions will be searched in order, the
32 # first match will be selected. These should be kept in sync with setup.py.
@@ -70,19 +84,6 @@ endif()
84 find_package(Torch REQUIRED)
85
86 #
73 -# Add the `default` target which detects which extensions should be
74 -# built based on platform/architecture. This is the same logic that
75 -# setup.py uses to select which extensions should be built and should
76 -# be kept in sync.
77 -#
78 -# The `default` target makes direct use of cmake easier since knowledge
79 -# of which extensions are supported has been factored in, e.g.
80 -#
81 -# mkdir build && cd build
82 -# cmake -G Ninja -DVLLM_PYTHON_EXECUTABLE=`which python3` -DCMAKE_LIBRARY_OUTPUT_DIRECTORY=../vllm ..
83 -# cmake --build . --target default
84 -#
85 -add_custom_target(default)
87 message(STATUS "Enabling core extension.")
88
89 # Define _core_C extension
@@ -100,8 +101,6 @@ define_gpu_extension_target(
101 USE_SABI 3
102 WITH_SOABI)
103
103 -add_dependencies(default _core_C)
104 -
104 #
105 # Forward the non-CUDA device extensions to external CMake scripts.
106 #
@@ -167,6 +166,8 @@ if(NVCC_THREADS AND VLLM_GPU_LANG STREQUAL "CUDA")
166 list(APPEND VLLM_GPU_FLAGS "--threads=${NVCC_THREADS}")
167 endif()
168
169 +include(FetchContent)
170 +
171 #
172 # Define other extension targets
173 #
@@ -190,7 +191,6 @@ set(VLLM_EXT_SRC
191 "csrc/torch_bindings.cpp")
192
193 if(VLLM_GPU_LANG STREQUAL "CUDA")
193 - include(FetchContent)
194 SET(CUTLASS_ENABLE_HEADERS_ONLY ON CACHE BOOL "Enable only the header library")
195 FetchContent_Declare(
196 cutlass
@@ -283,6 +283,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
283 csrc/quantization/machete/machete_pytorch.cu)
284 endif()
285
286 +message(STATUS "Enabling C extension.")
287 define_gpu_extension_target(
288 _C
289 DESTINATION vllm
@@ -313,6 +314,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
314 "csrc/moe/marlin_moe_ops.cu")
315 endif()
316
317 +message(STATUS "Enabling moe extension.")
318 define_gpu_extension_target(
319 _moe_C
320 DESTINATION vllm
@@ -323,7 +325,6 @@ define_gpu_extension_target(
325 USE_SABI 3
326 WITH_SOABI)
327
326 -
328 if(VLLM_GPU_LANG STREQUAL "HIP")
329 #
330 # _rocm_C extension
@@ -343,16 +344,63 @@ if(VLLM_GPU_LANG STREQUAL "HIP")
344 WITH_SOABI)
345 endif()
346
347 +# vllm-flash-attn currently only supported on CUDA
348 +if (NOT VLLM_TARGET_DEVICE STREQUAL "cuda")
349 + return()
350 +endif ()
351
347 -if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
348 - message(STATUS "Enabling C extension.")
349 - add_dependencies(default _C)
352 +#
353 +# Build vLLM flash attention from source
354 +#
355 +# IMPORTANT: This has to be the last thing we do, because vllm-flash-attn uses the same macros/functions as vLLM.
356 +# Because functions all belong to the global scope, vllm-flash-attn's functions overwrite vLLMs.
357 +# They should be identical but if they aren't, this is a massive footgun.
358 +#
359 +# The vllm-flash-attn install rules are nested under vllm to make sure the library gets installed in the correct place.
360 +# To only install vllm-flash-attn, use --component vllm_flash_attn_c.
361 +# If no component is specified, vllm-flash-attn is still installed.
362
351 - message(STATUS "Enabling moe extension.")
352 - add_dependencies(default _moe_C)
363 +# If VLLM_FLASH_ATTN_SRC_DIR is set, vllm-flash-attn is installed from that directory instead of downloading.
364 +# This is to enable local development of vllm-flash-attn within vLLM.
365 +# It can be set as an environment variable or passed as a cmake argument.
366 +# The environment variable takes precedence.
367 +if (DEFINED ENV{VLLM_FLASH_ATTN_SRC_DIR})
368 + set(VLLM_FLASH_ATTN_SRC_DIR $ENV{VLLM_FLASH_ATTN_SRC_DIR})
369 endif()
370
355 -if(VLLM_GPU_LANG STREQUAL "HIP")
356 - message(STATUS "Enabling rocm extension.")
357 - add_dependencies(default _rocm_C)
371 +if(VLLM_FLASH_ATTN_SRC_DIR)
372 + FetchContent_Declare(vllm-flash-attn SOURCE_DIR ${VLLM_FLASH_ATTN_SRC_DIR})
373 +else()
374 + FetchContent_Declare(
375 + vllm-flash-attn
376 + GIT_REPOSITORY https://github.com/vllm-project/flash-attention.git
377 + GIT_TAG 013f0c4fc47e6574060879d9734c1df8c5c273bd
378 + GIT_PROGRESS TRUE
379 + )
380 endif()
381 +
382 +# Set the parent build flag so that the vllm-flash-attn library does not redo compile flag and arch initialization.
383 +set(VLLM_PARENT_BUILD ON)
384 +
385 +# Make sure vllm-flash-attn install rules are nested under vllm/
386 +install(CODE "set(CMAKE_INSTALL_LOCAL_ONLY FALSE)" COMPONENT vllm_flash_attn_c)
387 +install(CODE "set(OLD_CMAKE_INSTALL_PREFIX \"\${CMAKE_INSTALL_PREFIX}\")" COMPONENT vllm_flash_attn_c)
388 +install(CODE "set(CMAKE_INSTALL_PREFIX \"\${CMAKE_INSTALL_PREFIX}/vllm/\")" COMPONENT vllm_flash_attn_c)
389 +
390 +# Fetch the vllm-flash-attn library
391 +FetchContent_MakeAvailable(vllm-flash-attn)
392 +message(STATUS "vllm-flash-attn is available at ${vllm-flash-attn_SOURCE_DIR}")
393 +
394 +# Restore the install prefix
395 +install(CODE "set(CMAKE_INSTALL_PREFIX \"\${OLD_CMAKE_INSTALL_PREFIX}\")" COMPONENT vllm_flash_attn_c)
396 +install(CODE "set(CMAKE_INSTALL_LOCAL_ONLY TRUE)" COMPONENT vllm_flash_attn_c)
397 +
398 +# Copy over the vllm-flash-attn python files
399 +install(
400 + DIRECTORY ${vllm-flash-attn_SOURCE_DIR}/vllm_flash_attn/
401 + DESTINATION vllm/vllm_flash_attn
402 + COMPONENT vllm_flash_attn_c
403 + FILES_MATCHING PATTERN "*.py"
404 +)
405 +
406 +# Nothing after vllm-flash-attn, see comment about macros above
Dockerfile
+3
@@ -48,6 +48,9 @@ RUN --mount=type=cache,target=/root/.cache/pip \
48 # see https://github.com/pytorch/pytorch/pull/123243
49 ARG torch_cuda_arch_list='7.0 7.5 8.0 8.6 8.9 9.0+PTX'
50 ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list}
51 +# Override the arch list for flash-attn to reduce the binary size
52 +ARG vllm_fa_cmake_gpu_arches='80-real;90-real'
53 +ENV VLLM_FA_CMAKE_GPU_ARCHES=${vllm_fa_cmake_gpu_arches}
54 #################### BASE BUILD IMAGE ####################
55
56 #################### WHEEL BUILD IMAGE ####################
cmake/utils.cmake
+1 -1
@@ -364,5 +364,5 @@ function (define_gpu_extension_target GPU_MOD_NAME)
364 target_link_libraries(${GPU_MOD_NAME} PRIVATE ${TORCH_LIBRARIES})
365 endif()
366
367 - install(TARGETS ${GPU_MOD_NAME} LIBRARY DESTINATION ${GPU_DESTINATION})
367 + install(TARGETS ${GPU_MOD_NAME} LIBRARY DESTINATION ${GPU_DESTINATION} COMPONENT ${GPU_MOD_NAME})
368 endfunction()
requirements-cuda.txt
-1
@@ -8,4 +8,3 @@ torch == 2.4.0
8 # These must be updated alongside torch
9 torchvision == 0.19 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
10 xformers == 0.0.27.post2; platform_system == 'Linux' and platform_machine == 'x86_64' # Requires PyTorch 2.4.0
11 -vllm-flash-attn == 2.6.1; platform_system == 'Linux' and platform_machine == 'x86_64' # Requires PyTorch 2.4.0
setup.py
+30 -8
@@ -6,6 +6,7 @@ import re
6 import subprocess
7 import sys
8 import warnings
9 +from pathlib import Path
10 from shutil import which
11 from typing import Dict, List
12
@@ -152,15 +153,8 @@ class cmake_build_ext(build_ext):
153 default_cfg = "Debug" if self.debug else "RelWithDebInfo"
154 cfg = envs.CMAKE_BUILD_TYPE or default_cfg
155
155 - # where .so files will be written, should be the same for all extensions
156 - # that use the same CMakeLists.txt.
157 - outdir = os.path.abspath(
158 - os.path.dirname(self.get_ext_fullpath(ext.name)))
159 -
156 cmake_args = [
157 '-DCMAKE_BUILD_TYPE={}'.format(cfg),
162 - '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={}'.format(outdir),
163 - '-DCMAKE_ARCHIVE_OUTPUT_DIRECTORY={}'.format(self.build_temp),
158 '-DVLLM_TARGET_DEVICE={}'.format(VLLM_TARGET_DEVICE),
159 ]
160
@@ -224,10 +218,12 @@ class cmake_build_ext(build_ext):
218 os.makedirs(self.build_temp)
219
220 targets = []
221 + target_name = lambda s: remove_prefix(remove_prefix(s, "vllm."),
222 + "vllm_flash_attn.")
223 # Build all the extensions
224 for ext in self.extensions:
225 self.configure(ext)
230 - targets.append(remove_prefix(ext.name, "vllm."))
226 + targets.append(target_name(ext.name))
227
228 num_jobs, _ = self.compute_num_jobs()
229
@@ -240,6 +236,28 @@ class cmake_build_ext(build_ext):
236
237 subprocess.check_call(["cmake", *build_args], cwd=self.build_temp)
238
239 + # Install the libraries
240 + for ext in self.extensions:
241 + # Install the extension into the proper location
242 + outdir = Path(self.get_ext_fullpath(ext.name)).parent.absolute()
243 +
244 + # Skip if the install directory is the same as the build directory
245 + if outdir == self.build_temp:
246 + continue
247 +
248 + # CMake appends the extension prefix to the install path,
249 + # and outdir already contains that prefix, so we need to remove it.
250 + prefix = outdir
251 + for i in range(ext.name.count('.')):
252 + prefix = prefix.parent
253 +
254 + # prefix here should actually be the same for all components
255 + install_args = [
256 + "cmake", "--install", ".", "--prefix", prefix, "--component",
257 + target_name(ext.name)
258 + ]
259 + subprocess.check_call(install_args, cwd=self.build_temp)
260 +
261
262 def _no_device() -> bool:
263 return VLLM_TARGET_DEVICE == "empty"
@@ -467,6 +485,10 @@ if _is_cuda() or _is_hip():
485 if _is_hip():
486 ext_modules.append(CMakeExtension(name="vllm._rocm_C"))
487
488 +if _is_cuda():
489 + ext_modules.append(
490 + CMakeExtension(name="vllm.vllm_flash_attn.vllm_flash_attn_c"))
491 +
492 if _build_custom_ops():
493 ext_modules.append(CMakeExtension(name="vllm._C"))
494
vllm/attention/backends/flash_attn.py
+7 -2
@@ -19,8 +19,13 @@ if TYPE_CHECKING:
19 from vllm.worker.model_runner import (ModelInputForGPUBuilder,
20 ModelInputForGPUWithSamplingMetadata)
21
22 -from vllm_flash_attn import flash_attn_varlen_func as _flash_attn_varlen_func
23 -from vllm_flash_attn import flash_attn_with_kvcache as _flash_attn_with_kvcache
22 +# yapf: disable
23 +from vllm.vllm_flash_attn import (
24 + flash_attn_varlen_func as _flash_attn_varlen_func)
25 +from vllm.vllm_flash_attn import (
26 + flash_attn_with_kvcache as _flash_attn_with_kvcache)
27 +
28 +# yapf: enable
29
30
31 @torch.library.custom_op("vllm::flash_attn_varlen_func", mutates_args=[])
vllm/attention/selector.py
+4 -4
@@ -244,8 +244,7 @@ def which_attn_to_use(
244 # FlashAttn is valid for the model, checking if the package is installed.
245 if selected_backend == _Backend.FLASH_ATTN:
246 try:
247 - import vllm_flash_attn # noqa: F401
248 -
247 + import vllm.vllm_flash_attn # noqa: F401
248 from vllm.attention.backends.flash_attn import ( # noqa: F401
249 FlashAttentionBackend)
250
@@ -258,8 +257,9 @@ def which_attn_to_use(
257 except ImportError:
258 logger.info(
259 "Cannot use FlashAttention-2 backend because the "
261 - "vllm_flash_attn package is not found. "
262 - "`pip install vllm-flash-attn` for better performance.")
260 + "vllm.vllm_flash_attn package is not found. "
261 + "Make sure that vllm_flash_attn was built and installed "
262 + "(on by default).")
263 selected_backend = _Backend.XFORMERS
264
265 return selected_backend