@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 89579a201f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Misc] Use vllm-flash-attn instead of flash-attn (#4686)

Woosuk Kwon committed May 8, 2024 at 13:15 UTC 89579a201f2c84b512f0e1006ac2ea0d979803ab
6 files changed +16 -31
Dockerfile
-21
@@ -87,23 +87,6 @@ RUN --mount=type=cache,target=/root/.cache/pip \
87 pip cache remove vllm_nccl*
88 #################### EXTENSION Build IMAGE ####################
89
90 -#################### FLASH_ATTENTION Build IMAGE ####################
91 -FROM dev as flash-attn-builder
92 -# max jobs used for build
93 -ARG max_jobs=2
94 -ENV MAX_JOBS=${max_jobs}
95 -# flash attention version
96 -ARG flash_attn_version=v2.5.8
97 -ENV FLASH_ATTN_VERSION=${flash_attn_version}
98 -
99 -WORKDIR /usr/src/flash-attention-v2
100 -
101 -# Download the wheel or build it if a pre-compiled release doesn't exist
102 -RUN pip --verbose wheel flash-attn==${FLASH_ATTN_VERSION} \
103 - --no-build-isolation --no-deps --no-cache-dir
104 -
105 -#################### FLASH_ATTENTION Build IMAGE ####################
106 -
90 #################### vLLM installation IMAGE ####################
91 # image with vLLM installed
92 FROM nvidia/cuda:12.4.1-base-ubuntu22.04 AS vllm-base
@@ -122,10 +105,6 @@ RUN ldconfig /usr/local/cuda-12.4/compat/
105 RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist \
106 --mount=type=cache,target=/root/.cache/pip \
107 pip install dist/*.whl --verbose
125 -
126 -RUN --mount=type=bind,from=flash-attn-builder,src=/usr/src/flash-attention-v2,target=/usr/src/flash-attention-v2 \
127 - --mount=type=cache,target=/root/.cache/pip \
128 - pip install /usr/src/flash-attention-v2/*.whl --no-cache-dir
108 #################### vLLM installation IMAGE ####################
109
110
requirements-cuda.txt
+1
@@ -7,3 +7,4 @@ nvidia-ml-py # for pynvml package
7 vllm-nccl-cu12>=2.18,<2.19 # for downloading nccl library
8 torch == 2.3.0
9 xformers == 0.0.26.post1 # Requires PyTorch 2.3.0
10 +vllm-flash-attn == 2.5.8.post1 # Requires PyTorch 2.3.0
setup.py
+9 -5
@@ -355,14 +355,18 @@ def get_requirements() -> List[str]:
355
356 if _is_cuda():
357 requirements = _read_requirements("requirements-cuda.txt")
358 - cuda_major = torch.version.cuda.split(".")[0]
358 + cuda_major, cuda_minor = torch.version.cuda.split(".")
359 modified_requirements = []
360 for req in requirements:
361 if "vllm-nccl-cu12" in req:
362 - modified_requirements.append(
363 - req.replace("vllm-nccl-cu12", f"vllm-nccl-cu{cuda_major}"))
364 - else:
365 - modified_requirements.append(req)
362 + req = req.replace("vllm-nccl-cu12",
363 + f"vllm-nccl-cu{cuda_major}")
364 + elif ("vllm-flash-attn" in req
365 + and not (cuda_major == "12" and cuda_minor == "1")):
366 + # vllm-flash-attn is built only for CUDA 12.1.
367 + # Skip for other versions.
368 + continue
369 + modified_requirements.append(req)
370 requirements = modified_requirements
371 elif _is_hip():
372 requirements = _read_requirements("requirements-rocm.txt")
vllm/attention/backends/flash_attn.py
+1 -1
@@ -8,7 +8,7 @@ from dataclasses import dataclass
8 from typing import List, Optional, Tuple, Type
9
10 import torch
11 -from flash_attn import flash_attn_varlen_func
11 +from vllm_flash_attn import flash_attn_varlen_func
12
13 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
14 AttentionMetadata,
vllm/attention/backends/flashinfer.py
+1 -1
@@ -3,8 +3,8 @@ from typing import Any, Dict, List, Optional, Set, Tuple, Type
3
4 import flashinfer
5 import torch
6 -from flash_attn import flash_attn_varlen_func
6 from flashinfer import BatchDecodeWithPagedKVCacheWrapper
7 +from vllm_flash_attn import flash_attn_varlen_func
8
9 from vllm import _custom_ops as ops
10 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
vllm/attention/selector.py
+4 -3
@@ -76,11 +76,12 @@ def _which_attn_to_use(dtype: torch.dtype) -> _Backend:
76 return _Backend.XFORMERS
77
78 try:
79 - import flash_attn # noqa: F401
79 + import vllm_flash_attn # noqa: F401
80 except ImportError:
81 logger.info(
82 - "Cannot use FlashAttention-2 backend because the flash_attn "
83 - "package is not found. Please install it for better performance.")
82 + "Cannot use FlashAttention-2 backend because the vllm_flash_attn "
83 + "package is not found. `pip install vllm-flash-attn` for better "
84 + "performance.")
85 return _Backend.XFORMERS
86
87 backend_by_env_var = envs.VLLM_ATTENTION_BACKEND