A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Misc] Use vllm-flash-attn instead of flash-attn (#4686)
Woosuk Kwon committed
May 8, 2024 at 13:15 UTC
89579a201f2c84b512f0e1006ac2ea0d979803ab
6 files changed
+16
-31
Dockerfile
-21
@@ -87,23 +87,6 @@ RUN --mount=type=cache,target=/root/.cache/pip \
87
pip cache remove vllm_nccl*
88
#################### EXTENSION Build IMAGE ####################
89
90
-#################### FLASH_ATTENTION Build IMAGE ####################
91
-FROM dev as flash-attn-builder
92
-# max jobs used for build
93
-ARG max_jobs=2
94
-ENV MAX_JOBS=${max_jobs}
95
-# flash attention version
96
-ARG flash_attn_version=v2.5.8
97
-ENV FLASH_ATTN_VERSION=${flash_attn_version}
98
-
99
-WORKDIR /usr/src/flash-attention-v2
100
-
101
-# Download the wheel or build it if a pre-compiled release doesn't exist
102
-RUN pip --verbose wheel flash-attn==${FLASH_ATTN_VERSION} \
103
- --no-build-isolation --no-deps --no-cache-dir
104
-
105
-#################### FLASH_ATTENTION Build IMAGE ####################
106
-
90
#################### vLLM installation IMAGE ####################
91
# image with vLLM installed
92
FROM nvidia/cuda:12.4.1-base-ubuntu22.04 AS vllm-base
@@ -122,10 +105,6 @@ RUN ldconfig /usr/local/cuda-12.4/compat/
105
RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist \
106
--mount=type=cache,target=/root/.cache/pip \
107
pip install dist/*.whl --verbose
125
-
126
-RUN --mount=type=bind,from=flash-attn-builder,src=/usr/src/flash-attention-v2,target=/usr/src/flash-attention-v2 \
127
- --mount=type=cache,target=/root/.cache/pip \
128
- pip install /usr/src/flash-attention-v2/*.whl --no-cache-dir
108
#################### vLLM installation IMAGE ####################
109
110
requirements-cuda.txt
+1
@@ -7,3 +7,4 @@ nvidia-ml-py # for pynvml package
7
vllm-nccl-cu12>=2.18,<2.19 # for downloading nccl library
8
torch == 2.3.0
9
xformers == 0.0.26.post1 # Requires PyTorch 2.3.0
10
+vllm-flash-attn == 2.5.8.post1 # Requires PyTorch 2.3.0
setup.py
+9
-5
@@ -355,14 +355,18 @@ def get_requirements() -> List[str]:
355
356
if _is_cuda():
357
requirements = _read_requirements("requirements-cuda.txt")
358
- cuda_major = torch.version.cuda.split(".")[0]
358
+ cuda_major, cuda_minor = torch.version.cuda.split(".")
359
modified_requirements = []
360
for req in requirements:
361
if "vllm-nccl-cu12" in req:
362
- modified_requirements.append(
363
- req.replace("vllm-nccl-cu12", f"vllm-nccl-cu{cuda_major}"))
364
- else:
365
- modified_requirements.append(req)
362
+ req = req.replace("vllm-nccl-cu12",
363
+ f"vllm-nccl-cu{cuda_major}")
364
+ elif ("vllm-flash-attn" in req
365
+ and not (cuda_major == "12" and cuda_minor == "1")):
366
+ # vllm-flash-attn is built only for CUDA 12.1.
367
+ # Skip for other versions.
368
+ continue
369
+ modified_requirements.append(req)
370
requirements = modified_requirements
371
elif _is_hip():
372
requirements = _read_requirements("requirements-rocm.txt")
vllm/attention/backends/flash_attn.py
+1
-1
@@ -8,7 +8,7 @@ from dataclasses import dataclass
8
from typing import List, Optional, Tuple, Type
9
10
import torch
11
-from flash_attn import flash_attn_varlen_func
11
+from vllm_flash_attn import flash_attn_varlen_func
12
13
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
14
AttentionMetadata,
vllm/attention/backends/flashinfer.py
+1
-1
@@ -3,8 +3,8 @@ from typing import Any, Dict, List, Optional, Set, Tuple, Type
3
4
import flashinfer
5
import torch
6
-from flash_attn import flash_attn_varlen_func
6
from flashinfer import BatchDecodeWithPagedKVCacheWrapper
7
+from vllm_flash_attn import flash_attn_varlen_func
8
9
from vllm import _custom_ops as ops
10
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
vllm/attention/selector.py
+4
-3
@@ -76,11 +76,12 @@ def _which_attn_to_use(dtype: torch.dtype) -> _Backend:
76
return _Backend.XFORMERS
77
78
try:
79
- import flash_attn # noqa: F401
79
+ import vllm_flash_attn # noqa: F401
80
except ImportError:
81
logger.info(
82
- "Cannot use FlashAttention-2 backend because the flash_attn "
83
- "package is not found. Please install it for better performance.")
82
+ "Cannot use FlashAttention-2 backend because the vllm_flash_attn "
83
+ "package is not found. `pip install vllm-flash-attn` for better "
84
+ "performance.")
85
return _Backend.XFORMERS
86
87
backend_by_env_var = envs.VLLM_ATTENTION_BACKEND