A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Kernel] Build flash-attn from source (#8245)
Luka Govedič committed
Sep 21, 2024 at 02:27 UTC
71c60491f287d8a23bed1743513b4b3e7927c69e
9 files changed
+124
-41
.github/workflows/scripts/build.sh
+1
@@ -15,5 +15,6 @@ $python_executable -m pip install -r requirements-cuda.txt
15
export MAX_JOBS=1
16
# Make sure release wheels are built for the following architectures
17
export TORCH_CUDA_ARCH_LIST="7.0 7.5 8.0 8.6 8.9 9.0+PTX"
18
+export VLLM_FA_CMAKE_GPU_ARCHES="80-real;90-real"
19
# Build
20
$python_executable setup.py bdist_wheel --dist-dir=dist
.gitignore
+5
@@ -1,6 +1,9 @@
1
# vllm commit id, generated by setup.py
2
vllm/commit_id.py
3
4
+# vllm-flash-attn built from source
5
+vllm/vllm_flash_attn/
6
+
7
# Byte-compiled / optimized / DLL files
8
__pycache__/
9
*.py[cod]
@@ -12,6 +15,8 @@ __pycache__/
15
# Distribution / packaging
16
.Python
17
build/
18
+cmake-build-*/
19
+CMakeUserPresets.json
20
develop-eggs/
21
dist/
22
downloads/
CMakeLists.txt
+73
-25
@@ -1,5 +1,16 @@
1
cmake_minimum_required(VERSION 3.26)
2
3
+# When building directly using CMake, make sure you run the install step
4
+# (it places the .so files in the correct location).
5
+#
6
+# Example:
7
+# mkdir build && cd build
8
+# cmake -G Ninja -DVLLM_PYTHON_EXECUTABLE=`which python3` -DCMAKE_INSTALL_PREFIX=.. ..
9
+# cmake --build . --target install
10
+#
11
+# If you want to only build one target, make sure to install it manually:
12
+# cmake --build . --target _C
13
+# cmake --install . --component _C
14
project(vllm_extensions LANGUAGES CXX)
15
16
# CUDA by default, can be overridden by using -DVLLM_TARGET_DEVICE=... (used by setup.py)
@@ -13,6 +24,9 @@ include(${CMAKE_CURRENT_LIST_DIR}/cmake/utils.cmake)
24
# Suppress potential warnings about unused manually-specified variables
25
set(ignoreMe "${VLLM_PYTHON_PATH}")
26
27
+# Prevent installation of dependencies (cutlass) by default.
28
+install(CODE "set(CMAKE_INSTALL_LOCAL_ONLY TRUE)" ALL_COMPONENTS)
29
+
30
#
31
# Supported python versions. These versions will be searched in order, the
32
# first match will be selected. These should be kept in sync with setup.py.
@@ -70,19 +84,6 @@ endif()
84
find_package(Torch REQUIRED)
85
86
#
73
-# Add the `default` target which detects which extensions should be
74
-# built based on platform/architecture. This is the same logic that
75
-# setup.py uses to select which extensions should be built and should
76
-# be kept in sync.
77
-#
78
-# The `default` target makes direct use of cmake easier since knowledge
79
-# of which extensions are supported has been factored in, e.g.
80
-#
81
-# mkdir build && cd build
82
-# cmake -G Ninja -DVLLM_PYTHON_EXECUTABLE=`which python3` -DCMAKE_LIBRARY_OUTPUT_DIRECTORY=../vllm ..
83
-# cmake --build . --target default
84
-#
85
-add_custom_target(default)
87
message(STATUS "Enabling core extension.")
88
89
# Define _core_C extension
@@ -100,8 +101,6 @@ define_gpu_extension_target(
101
USE_SABI 3
102
WITH_SOABI)
103
103
-add_dependencies(default _core_C)
104
-
104
#
105
# Forward the non-CUDA device extensions to external CMake scripts.
106
#
@@ -167,6 +166,8 @@ if(NVCC_THREADS AND VLLM_GPU_LANG STREQUAL "CUDA")
166
list(APPEND VLLM_GPU_FLAGS "--threads=${NVCC_THREADS}")
167
endif()
168
169
+include(FetchContent)
170
+
171
#
172
# Define other extension targets
173
#
@@ -190,7 +191,6 @@ set(VLLM_EXT_SRC
191
"csrc/torch_bindings.cpp")
192
193
if(VLLM_GPU_LANG STREQUAL "CUDA")
193
- include(FetchContent)
194
SET(CUTLASS_ENABLE_HEADERS_ONLY ON CACHE BOOL "Enable only the header library")
195
FetchContent_Declare(
196
cutlass
@@ -283,6 +283,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
283
csrc/quantization/machete/machete_pytorch.cu)
284
endif()
285
286
+message(STATUS "Enabling C extension.")
287
define_gpu_extension_target(
288
_C
289
DESTINATION vllm
@@ -313,6 +314,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
314
"csrc/moe/marlin_moe_ops.cu")
315
endif()
316
317
+message(STATUS "Enabling moe extension.")
318
define_gpu_extension_target(
319
_moe_C
320
DESTINATION vllm
@@ -323,7 +325,6 @@ define_gpu_extension_target(
325
USE_SABI 3
326
WITH_SOABI)
327
326
-
328
if(VLLM_GPU_LANG STREQUAL "HIP")
329
#
330
# _rocm_C extension
@@ -343,16 +344,63 @@ if(VLLM_GPU_LANG STREQUAL "HIP")
344
WITH_SOABI)
345
endif()
346
347
+# vllm-flash-attn currently only supported on CUDA
348
+if (NOT VLLM_TARGET_DEVICE STREQUAL "cuda")
349
+ return()
350
+endif ()
351
347
-if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
348
- message(STATUS "Enabling C extension.")
349
- add_dependencies(default _C)
352
+#
353
+# Build vLLM flash attention from source
354
+#
355
+# IMPORTANT: This has to be the last thing we do, because vllm-flash-attn uses the same macros/functions as vLLM.
356
+# Because functions all belong to the global scope, vllm-flash-attn's functions overwrite vLLMs.
357
+# They should be identical but if they aren't, this is a massive footgun.
358
+#
359
+# The vllm-flash-attn install rules are nested under vllm to make sure the library gets installed in the correct place.
360
+# To only install vllm-flash-attn, use --component vllm_flash_attn_c.
361
+# If no component is specified, vllm-flash-attn is still installed.
362
351
- message(STATUS "Enabling moe extension.")
352
- add_dependencies(default _moe_C)
363
+# If VLLM_FLASH_ATTN_SRC_DIR is set, vllm-flash-attn is installed from that directory instead of downloading.
364
+# This is to enable local development of vllm-flash-attn within vLLM.
365
+# It can be set as an environment variable or passed as a cmake argument.
366
+# The environment variable takes precedence.
367
+if (DEFINED ENV{VLLM_FLASH_ATTN_SRC_DIR})
368
+ set(VLLM_FLASH_ATTN_SRC_DIR $ENV{VLLM_FLASH_ATTN_SRC_DIR})
369
endif()
370
355
-if(VLLM_GPU_LANG STREQUAL "HIP")
356
- message(STATUS "Enabling rocm extension.")
357
- add_dependencies(default _rocm_C)
371
+if(VLLM_FLASH_ATTN_SRC_DIR)
372
+ FetchContent_Declare(vllm-flash-attn SOURCE_DIR ${VLLM_FLASH_ATTN_SRC_DIR})
373
+else()
374
+ FetchContent_Declare(
375
+ vllm-flash-attn
376
+ GIT_REPOSITORY https://github.com/vllm-project/flash-attention.git
377
+ GIT_TAG 013f0c4fc47e6574060879d9734c1df8c5c273bd
378
+ GIT_PROGRESS TRUE
379
+ )
380
endif()
381
+
382
+# Set the parent build flag so that the vllm-flash-attn library does not redo compile flag and arch initialization.
383
+set(VLLM_PARENT_BUILD ON)
384
+
385
+# Make sure vllm-flash-attn install rules are nested under vllm/
386
+install(CODE "set(CMAKE_INSTALL_LOCAL_ONLY FALSE)" COMPONENT vllm_flash_attn_c)
387
+install(CODE "set(OLD_CMAKE_INSTALL_PREFIX \"\${CMAKE_INSTALL_PREFIX}\")" COMPONENT vllm_flash_attn_c)
388
+install(CODE "set(CMAKE_INSTALL_PREFIX \"\${CMAKE_INSTALL_PREFIX}/vllm/\")" COMPONENT vllm_flash_attn_c)
389
+
390
+# Fetch the vllm-flash-attn library
391
+FetchContent_MakeAvailable(vllm-flash-attn)
392
+message(STATUS "vllm-flash-attn is available at ${vllm-flash-attn_SOURCE_DIR}")
393
+
394
+# Restore the install prefix
395
+install(CODE "set(CMAKE_INSTALL_PREFIX \"\${OLD_CMAKE_INSTALL_PREFIX}\")" COMPONENT vllm_flash_attn_c)
396
+install(CODE "set(CMAKE_INSTALL_LOCAL_ONLY TRUE)" COMPONENT vllm_flash_attn_c)
397
+
398
+# Copy over the vllm-flash-attn python files
399
+install(
400
+ DIRECTORY ${vllm-flash-attn_SOURCE_DIR}/vllm_flash_attn/
401
+ DESTINATION vllm/vllm_flash_attn
402
+ COMPONENT vllm_flash_attn_c
403
+ FILES_MATCHING PATTERN "*.py"
404
+)
405
+
406
+# Nothing after vllm-flash-attn, see comment about macros above
Dockerfile
+3
@@ -48,6 +48,9 @@ RUN --mount=type=cache,target=/root/.cache/pip \
48
# see https://github.com/pytorch/pytorch/pull/123243
49
ARG torch_cuda_arch_list='7.0 7.5 8.0 8.6 8.9 9.0+PTX'
50
ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list}
51
+# Override the arch list for flash-attn to reduce the binary size
52
+ARG vllm_fa_cmake_gpu_arches='80-real;90-real'
53
+ENV VLLM_FA_CMAKE_GPU_ARCHES=${vllm_fa_cmake_gpu_arches}
54
#################### BASE BUILD IMAGE ####################
55
56
#################### WHEEL BUILD IMAGE ####################
cmake/utils.cmake
+1
-1
@@ -364,5 +364,5 @@ function (define_gpu_extension_target GPU_MOD_NAME)
364
target_link_libraries(${GPU_MOD_NAME} PRIVATE ${TORCH_LIBRARIES})
365
endif()
366
367
- install(TARGETS ${GPU_MOD_NAME} LIBRARY DESTINATION ${GPU_DESTINATION})
367
+ install(TARGETS ${GPU_MOD_NAME} LIBRARY DESTINATION ${GPU_DESTINATION} COMPONENT ${GPU_MOD_NAME})
368
endfunction()
requirements-cuda.txt
-1
@@ -8,4 +8,3 @@ torch == 2.4.0
8
# These must be updated alongside torch
9
torchvision == 0.19 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
10
xformers == 0.0.27.post2; platform_system == 'Linux' and platform_machine == 'x86_64' # Requires PyTorch 2.4.0
11
-vllm-flash-attn == 2.6.1; platform_system == 'Linux' and platform_machine == 'x86_64' # Requires PyTorch 2.4.0
setup.py
+30
-8
@@ -6,6 +6,7 @@ import re
6
import subprocess
7
import sys
8
import warnings
9
+from pathlib import Path
10
from shutil import which
11
from typing import Dict, List
12
@@ -152,15 +153,8 @@ class cmake_build_ext(build_ext):
153
default_cfg = "Debug" if self.debug else "RelWithDebInfo"
154
cfg = envs.CMAKE_BUILD_TYPE or default_cfg
155
155
- # where .so files will be written, should be the same for all extensions
156
- # that use the same CMakeLists.txt.
157
- outdir = os.path.abspath(
158
- os.path.dirname(self.get_ext_fullpath(ext.name)))
159
-
156
cmake_args = [
157
'-DCMAKE_BUILD_TYPE={}'.format(cfg),
162
- '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={}'.format(outdir),
163
- '-DCMAKE_ARCHIVE_OUTPUT_DIRECTORY={}'.format(self.build_temp),
158
'-DVLLM_TARGET_DEVICE={}'.format(VLLM_TARGET_DEVICE),
159
]
160
@@ -224,10 +218,12 @@ class cmake_build_ext(build_ext):
218
os.makedirs(self.build_temp)
219
220
targets = []
221
+ target_name = lambda s: remove_prefix(remove_prefix(s, "vllm."),
222
+ "vllm_flash_attn.")
223
# Build all the extensions
224
for ext in self.extensions:
225
self.configure(ext)
230
- targets.append(remove_prefix(ext.name, "vllm."))
226
+ targets.append(target_name(ext.name))
227
228
num_jobs, _ = self.compute_num_jobs()
229
@@ -240,6 +236,28 @@ class cmake_build_ext(build_ext):
236
237
subprocess.check_call(["cmake", *build_args], cwd=self.build_temp)
238
239
+ # Install the libraries
240
+ for ext in self.extensions:
241
+ # Install the extension into the proper location
242
+ outdir = Path(self.get_ext_fullpath(ext.name)).parent.absolute()
243
+
244
+ # Skip if the install directory is the same as the build directory
245
+ if outdir == self.build_temp:
246
+ continue
247
+
248
+ # CMake appends the extension prefix to the install path,
249
+ # and outdir already contains that prefix, so we need to remove it.
250
+ prefix = outdir
251
+ for i in range(ext.name.count('.')):
252
+ prefix = prefix.parent
253
+
254
+ # prefix here should actually be the same for all components
255
+ install_args = [
256
+ "cmake", "--install", ".", "--prefix", prefix, "--component",
257
+ target_name(ext.name)
258
+ ]
259
+ subprocess.check_call(install_args, cwd=self.build_temp)
260
+
261
262
def _no_device() -> bool:
263
return VLLM_TARGET_DEVICE == "empty"
@@ -467,6 +485,10 @@ if _is_cuda() or _is_hip():
485
if _is_hip():
486
ext_modules.append(CMakeExtension(name="vllm._rocm_C"))
487
488
+if _is_cuda():
489
+ ext_modules.append(
490
+ CMakeExtension(name="vllm.vllm_flash_attn.vllm_flash_attn_c"))
491
+
492
if _build_custom_ops():
493
ext_modules.append(CMakeExtension(name="vllm._C"))
494
vllm/attention/backends/flash_attn.py
+7
-2
@@ -19,8 +19,13 @@ if TYPE_CHECKING:
19
from vllm.worker.model_runner import (ModelInputForGPUBuilder,
20
ModelInputForGPUWithSamplingMetadata)
21
22
-from vllm_flash_attn import flash_attn_varlen_func as _flash_attn_varlen_func
23
-from vllm_flash_attn import flash_attn_with_kvcache as _flash_attn_with_kvcache
22
+# yapf: disable
23
+from vllm.vllm_flash_attn import (
24
+ flash_attn_varlen_func as _flash_attn_varlen_func)
25
+from vllm.vllm_flash_attn import (
26
+ flash_attn_with_kvcache as _flash_attn_with_kvcache)
27
+
28
+# yapf: enable
29
30
31
@torch.library.custom_op("vllm::flash_attn_varlen_func", mutates_args=[])
vllm/attention/selector.py
+4
-4
@@ -244,8 +244,7 @@ def which_attn_to_use(
244
# FlashAttn is valid for the model, checking if the package is installed.
245
if selected_backend == _Backend.FLASH_ATTN:
246
try:
247
- import vllm_flash_attn # noqa: F401
248
-
247
+ import vllm.vllm_flash_attn # noqa: F401
248
from vllm.attention.backends.flash_attn import ( # noqa: F401
249
FlashAttentionBackend)
250
@@ -258,8 +257,9 @@ def which_attn_to_use(
257
except ImportError:
258
logger.info(
259
"Cannot use FlashAttention-2 backend because the "
261
- "vllm_flash_attn package is not found. "
262
- "`pip install vllm-flash-attn` for better performance.")
260
+ "vllm.vllm_flash_attn package is not found. "
261
+ "Make sure that vllm_flash_attn was built and installed "
262
+ "(on by default).")
263
selected_backend = _Backend.XFORMERS
264
265
return selected_backend