@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0580aab02f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[ROCm] support Radeon™ 7900 series (gfx1100) without using flash-attention (#2768)

Hongxia Yang committed Feb 11, 2024 at 02:14 UTC 0580aab02ffe60fee50bddc80b787828eb233c44
4 files changed +60 -5
Dockerfile.rocm
+12 -3
@@ -17,6 +17,12 @@ RUN echo "FA_GFX_ARCHS is $FA_GFX_ARCHS"
17 ARG FA_BRANCH="3d2b6f5"
18 RUN echo "FA_BRANCH is $FA_BRANCH"
19
20 +# whether to build flash-attention
21 +# if 0, will not build flash attention
22 +# this is useful for gfx target where flash-attention is not supported
23 +# In that case, we need to use the python reference attention implementation in vllm
24 +ARG BUILD_FA="1"
25 +
26 # Install some basic utilities
27 RUN apt-get update && apt-get install python3 python3-pip -y
28
@@ -50,7 +56,8 @@ ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/rocm/lib/:/libtorch/lib:
56 ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:/libtorch/include:/libtorch/include/torch/csrc/api/include/:/opt/rocm/include/:
57
58 # Install ROCm flash-attention
53 -RUN mkdir libs \
59 +RUN if [ "$BUILD_FA" == "1" ]; then \
60 + mkdir libs \
61 && cd libs \
62 && git clone https://github.com/ROCmSoftwarePlatform/flash-attention.git \
63 && cd flash-attention \
@@ -60,7 +67,8 @@ RUN mkdir libs \
67 && if [ "$BASE_IMAGE" = "rocm/pytorch:rocm5.7_ubuntu22.04_py3.10_pytorch_2.0.1" ]; then \
68 patch /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/utils/hipify/hipify_python.py hipify_patch.patch; fi \
69 && python3 setup.py install \
63 - && cd ..
70 + && cd ..; \
71 + fi
72
73 COPY ./ /app/vllm
74
@@ -75,7 +83,8 @@ RUN if [ "$BASE_IMAGE" = "rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1"
83 RUN cd /app \
84 && cd vllm \
85 && pip install -U -r requirements-rocm.txt \
78 - && bash patch_xformers.rocm.sh \
86 + && if [ "$BUILD_FA" == "1" ]; then \
87 + bash patch_xformers.rocm.sh; fi \
88 && patch /opt/rocm/include/hip/amd_detail/amd_hip_bf16.h /app/vllm/rocm_patch/rocm_bf16.patch \
89 && python3 setup.py install \
90 && cd ..
docs/source/getting_started/amd-installation.rst
+2 -1
@@ -12,7 +12,7 @@ Requirements
12
13 * OS: Linux
14 * Python: 3.8 -- 3.11
15 -* GPU: MI200s (gfx90a), MI300 (gfx942)
15 +* GPU: MI200s (gfx90a), MI300 (gfx942), Radeon RX 7900 series (gfx1100)
16 * Pytorch 2.0.1/2.1.1/2.2
17 * ROCm 5.7 (Verified on python 3.10) or ROCm 6.0 (Verified on python 3.9)
18
@@ -105,6 +105,7 @@ The `Dokerfile.rocm` is designed to support both ROCm 5.7 and ROCm 6.0 and later
105 * `BASE_IMAGE`: specifies the base image used when running ``docker build``, specifically the PyTorch on ROCm base image. We have tested ROCm 5.7 and ROCm 6.0. The default is `rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1`
106 * `FX_GFX_ARCHS`: specifies the GFX architecture that is used to build flash-attention, for example, `gfx90a;gfx942` for MI200 and MI300. The default is `gfx90a;gfx942`
107 * `FA_BRANCH`: specifies the branch used to build the flash-attention in `ROCmSoftwarePlatform's flash-attention repo <https://github.com/ROCmSoftwarePlatform/flash-attention>`_. The default is `3d2b6f5`
108 +* `BUILD_FA`: specifies whether to build flash-attention. For `Radeon RX 7900 series (gfx1100) <https://rocm.docs.amd.com/projects/radeon/en/latest/index.html>`_, this should be set to 0 before flash-attention supports this target.
109
110 Their values can be passed in when running ``docker build`` with ``--build-arg`` options.
111
setup.py
+1 -1
@@ -24,7 +24,7 @@ MAIN_CUDA_VERSION = "12.1"
24
25 # Supported NVIDIA GPU architectures.
26 NVIDIA_SUPPORTED_ARCHS = {"7.0", "7.5", "8.0", "8.6", "8.9", "9.0"}
27 -ROCM_SUPPORTED_ARCHS = {"gfx90a", "gfx942"}
27 +ROCM_SUPPORTED_ARCHS = {"gfx90a", "gfx942", "gfx1100"}
28 # SUPPORTED_ARCHS = NVIDIA_SUPPORTED_ARCHS.union(ROCM_SUPPORTED_ARCHS)
29
30
vllm/model_executor/layers/attention.py
+45
@@ -1,6 +1,7 @@
1 """Multi-head attention."""
2 from typing import List, Optional
3
4 +import importlib
5 import torch
6 import torch.nn as nn
7 from xformers import ops as xops
@@ -58,6 +59,40 @@ class PagedAttention(nn.Module):
59 raise ValueError(f"head_size ({self.head_size}) is not supported. "
60 f"Supported head sizes: {_SUPPORTED_HEAD_SIZES}.")
61
62 + self.use_ref_attention = self.check_use_ref_attention()
63 +
64 + def check_use_ref_attention(self) -> bool:
65 + if not is_hip():
66 + return False
67 + # For ROCm, check whether flash attention is installed or not.
68 + # if not, use_ref_attention needs to be True
69 + return importlib.util.find_spec("flash_attn") is None
70 +
71 + def ref_masked_attention(
72 + self,
73 + query: torch.Tensor,
74 + key: torch.Tensor,
75 + value: torch.Tensor,
76 + ) -> torch.Tensor:
77 + query = query.view(-1, self.num_heads, self.head_size)
78 + key = key.view(-1, self.num_kv_heads, self.head_size)
79 + value = value.view(-1, self.num_kv_heads, self.head_size)
80 +
81 + seq_len, _, _ = query.shape
82 + attn_mask = torch.triu(torch.ones(seq_len,
83 + seq_len,
84 + dtype=query.dtype,
85 + device=query.device),
86 + diagonal=1)
87 + attn_mask = attn_mask * torch.finfo(query.dtype).min
88 +
89 + attn_weights = self.scale * torch.einsum("qhd,khd->hqk", query,
90 + key).float()
91 + attn_weights = attn_weights + attn_mask.float()
92 + attn_weights = torch.softmax(attn_weights, dim=-1).to(value.dtype)
93 + out = torch.einsum("hqk,khd->qhd", attn_weights, value)
94 + return out
95 +
96 def forward(
97 self,
98 query: torch.Tensor,
@@ -137,6 +172,16 @@ class PagedAttention(nn.Module):
172 self.alibi_slopes, self.num_kv_heads, batch_size,
173 seq_len, query.dtype)
174
175 + if self.use_ref_attention:
176 + output = self.ref_masked_attention(
177 + query,
178 + key,
179 + value,
180 + )
181 + # Using view got RuntimeError: view size is not compatible with input tensor's size and stride
182 + # (at least one dimension spans across two contiguous subspaces). Use reshape instead
183 + return output.reshape(batch_size, seq_len, hidden_size)
184 +
185 # TODO(woosuk): Too many view operations. Let's try to reduce
186 # them in the future for code readability.
187 if self.alibi_slopes is None: