A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
[ROCm] support Radeon™ 7900 series (gfx1100) without using flash-attention (#2768)
Hongxia Yang committed
Feb 11, 2024 at 02:14 UTC
0580aab02ffe60fee50bddc80b787828eb233c44
4 files changed
+60
-5
Dockerfile.rocm
+12
-3
@@ -17,6 +17,12 @@ RUN echo "FA_GFX_ARCHS is $FA_GFX_ARCHS"
17
ARG FA_BRANCH="3d2b6f5"
18
RUN echo "FA_BRANCH is $FA_BRANCH"
19
20
+# whether to build flash-attention
21
+# if 0, will not build flash attention
22
+# this is useful for gfx target where flash-attention is not supported
23
+# In that case, we need to use the python reference attention implementation in vllm
24
+ARG BUILD_FA="1"
25
+
26
# Install some basic utilities
27
RUN apt-get update && apt-get install python3 python3-pip -y
28
@@ -50,7 +56,8 @@ ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/rocm/lib/:/libtorch/lib:
56
ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:/libtorch/include:/libtorch/include/torch/csrc/api/include/:/opt/rocm/include/:
57
58
# Install ROCm flash-attention
53
-RUN mkdir libs \
59
+RUN if [ "$BUILD_FA" == "1" ]; then \
60
+ mkdir libs \
61
&& cd libs \
62
&& git clone https://github.com/ROCmSoftwarePlatform/flash-attention.git \
63
&& cd flash-attention \
@@ -60,7 +67,8 @@ RUN mkdir libs \
67
&& if [ "$BASE_IMAGE" = "rocm/pytorch:rocm5.7_ubuntu22.04_py3.10_pytorch_2.0.1" ]; then \
68
patch /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/utils/hipify/hipify_python.py hipify_patch.patch; fi \
69
&& python3 setup.py install \
63
- && cd ..
70
+ && cd ..; \
71
+ fi
72
73
COPY ./ /app/vllm
74
@@ -75,7 +83,8 @@ RUN if [ "$BASE_IMAGE" = "rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1"
83
RUN cd /app \
84
&& cd vllm \
85
&& pip install -U -r requirements-rocm.txt \
78
- && bash patch_xformers.rocm.sh \
86
+ && if [ "$BUILD_FA" == "1" ]; then \
87
+ bash patch_xformers.rocm.sh; fi \
88
&& patch /opt/rocm/include/hip/amd_detail/amd_hip_bf16.h /app/vllm/rocm_patch/rocm_bf16.patch \
89
&& python3 setup.py install \
90
&& cd ..
docs/source/getting_started/amd-installation.rst
+2
-1
@@ -12,7 +12,7 @@ Requirements
12
13
* OS: Linux
14
* Python: 3.8 -- 3.11
15
-* GPU: MI200s (gfx90a), MI300 (gfx942)
15
+* GPU: MI200s (gfx90a), MI300 (gfx942), Radeon RX 7900 series (gfx1100)
16
* Pytorch 2.0.1/2.1.1/2.2
17
* ROCm 5.7 (Verified on python 3.10) or ROCm 6.0 (Verified on python 3.9)
18
@@ -105,6 +105,7 @@ The `Dokerfile.rocm` is designed to support both ROCm 5.7 and ROCm 6.0 and later
105
* `BASE_IMAGE`: specifies the base image used when running ``docker build``, specifically the PyTorch on ROCm base image. We have tested ROCm 5.7 and ROCm 6.0. The default is `rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1`
106
* `FX_GFX_ARCHS`: specifies the GFX architecture that is used to build flash-attention, for example, `gfx90a;gfx942` for MI200 and MI300. The default is `gfx90a;gfx942`
107
* `FA_BRANCH`: specifies the branch used to build the flash-attention in `ROCmSoftwarePlatform's flash-attention repo <https://github.com/ROCmSoftwarePlatform/flash-attention>`_. The default is `3d2b6f5`
108
+* `BUILD_FA`: specifies whether to build flash-attention. For `Radeon RX 7900 series (gfx1100) <https://rocm.docs.amd.com/projects/radeon/en/latest/index.html>`_, this should be set to 0 before flash-attention supports this target.
109
110
Their values can be passed in when running ``docker build`` with ``--build-arg`` options.
111
setup.py
+1
-1
@@ -24,7 +24,7 @@ MAIN_CUDA_VERSION = "12.1"
24
25
# Supported NVIDIA GPU architectures.
26
NVIDIA_SUPPORTED_ARCHS = {"7.0", "7.5", "8.0", "8.6", "8.9", "9.0"}
27
-ROCM_SUPPORTED_ARCHS = {"gfx90a", "gfx942"}
27
+ROCM_SUPPORTED_ARCHS = {"gfx90a", "gfx942", "gfx1100"}
28
# SUPPORTED_ARCHS = NVIDIA_SUPPORTED_ARCHS.union(ROCM_SUPPORTED_ARCHS)
29
30
vllm/model_executor/layers/attention.py
+45
@@ -1,6 +1,7 @@
1
"""Multi-head attention."""
2
from typing import List, Optional
3
4
+import importlib
5
import torch
6
import torch.nn as nn
7
from xformers import ops as xops
@@ -58,6 +59,40 @@ class PagedAttention(nn.Module):
59
raise ValueError(f"head_size ({self.head_size}) is not supported. "
60
f"Supported head sizes: {_SUPPORTED_HEAD_SIZES}.")
61
62
+ self.use_ref_attention = self.check_use_ref_attention()
63
+
64
+ def check_use_ref_attention(self) -> bool:
65
+ if not is_hip():
66
+ return False
67
+ # For ROCm, check whether flash attention is installed or not.
68
+ # if not, use_ref_attention needs to be True
69
+ return importlib.util.find_spec("flash_attn") is None
70
+
71
+ def ref_masked_attention(
72
+ self,
73
+ query: torch.Tensor,
74
+ key: torch.Tensor,
75
+ value: torch.Tensor,
76
+ ) -> torch.Tensor:
77
+ query = query.view(-1, self.num_heads, self.head_size)
78
+ key = key.view(-1, self.num_kv_heads, self.head_size)
79
+ value = value.view(-1, self.num_kv_heads, self.head_size)
80
+
81
+ seq_len, _, _ = query.shape
82
+ attn_mask = torch.triu(torch.ones(seq_len,
83
+ seq_len,
84
+ dtype=query.dtype,
85
+ device=query.device),
86
+ diagonal=1)
87
+ attn_mask = attn_mask * torch.finfo(query.dtype).min
88
+
89
+ attn_weights = self.scale * torch.einsum("qhd,khd->hqk", query,
90
+ key).float()
91
+ attn_weights = attn_weights + attn_mask.float()
92
+ attn_weights = torch.softmax(attn_weights, dim=-1).to(value.dtype)
93
+ out = torch.einsum("hqk,khd->qhd", attn_weights, value)
94
+ return out
95
+
96
def forward(
97
self,
98
query: torch.Tensor,
@@ -137,6 +172,16 @@ class PagedAttention(nn.Module):
172
self.alibi_slopes, self.num_kv_heads, batch_size,
173
seq_len, query.dtype)
174
175
+ if self.use_ref_attention:
176
+ output = self.ref_masked_attention(
177
+ query,
178
+ key,
179
+ value,
180
+ )
181
+ # Using view got RuntimeError: view size is not compatible with input tensor's size and stride
182
+ # (at least one dimension spans across two contiguous subspaces). Use reshape instead
183
+ return output.reshape(batch_size, seq_len, hidden_size)
184
+
185
# TODO(woosuk): Too many view operations. Let's try to reduce
186
# them in the future for code readability.
187
if self.alibi_slopes is None: