A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
[ROCm] Upgrade xformers version for ROCm & update doc (#2079)
Co-authored-by: miloice <jeffaw99@hotmail.com>
TJian committed
Dec 13, 2023 at 16:56 UTC
f375ec844036104cbae36477ce3541588d026829
6 files changed
+84
-56
Dockerfile.rocm
+2
-2
@@ -47,12 +47,12 @@ RUN mkdir libs \
47
COPY ./ /app/vllm
48
49
RUN python3 -m pip install --upgrade pip
50
-RUN pip install xformers==0.0.22.post7 --no-deps
50
+RUN pip install xformers==0.0.23 --no-deps
51
52
RUN cd /app \
53
&& cd vllm \
54
&& pip install -U -r requirements-rocm.txt \
55
- && bash patch_xformers-0.0.22.post7.rocm.sh \
55
+ && bash patch_xformers-0.0.23.rocm.sh \
56
&& python3 setup.py install \
57
&& cd ..
58
docs/source/getting_started/amd-installation.rst
+8
-8
@@ -3,7 +3,7 @@
3
Installation with ROCm
4
======================
5
6
-vLLM 0.2.x onwards supports model inferencing and serving on AMD GPUs with ROCm.
6
+vLLM 0.2.4 onwards supports model inferencing and serving on AMD GPUs with ROCm.
7
At the moment AWQ quantization is not supported in ROCm, but SqueezeLLM quantization has been ported.
8
Data types currently supported in ROCm are FP16 and BF16.
9
@@ -29,7 +29,7 @@ Installation options:
29
30
.. code-block:: console
31
32
- $ docker pull embeddedllminfo/vllm-rocm:vllm-v0.2.3
32
+ $ docker pull embeddedllminfo/vllm-rocm:vllm-v0.2.4
33
$ docker run -it \
34
--network=host \
35
--group-add=video \
@@ -70,12 +70,12 @@ You can build and install vLLM from source:
70
- ROCm's Flash-attention-2 (v2.0.4) does not support sliding windows attention.
71
- You might need to downgrade the "ninja" version to 1.10 it is not used when compiling flash-attention-2 (e.g. `pip install ninja==1.10.2.4`)
72
73
-2. Setup `xformers==0.0.22.post7` without dependencies, and apply patches to adapt for ROCm flash attention
73
+2. Setup `xformers==0.0.23` without dependencies, and apply patches to adapt for ROCm flash attention
74
75
.. code-block:: console
76
77
- $ pip install xformers==0.0.22.post7 --no-deps
78
- $ bash patch_xformers-0.0.22.post7.rocm.sh
77
+ $ pip install xformers==0.0.23 --no-deps
78
+ $ bash patch_xformers.rocm.sh
79
80
3. Build vLLM.
81
@@ -127,12 +127,12 @@ Alternatively, if you plan to install vLLM-ROCm on a local machine or start from
127
- ROCm's Flash-attention-2 (v2.0.4) does not support sliding windows attention.
128
- You might need to downgrade the "ninja" version to 1.10 it is not used when compiling flash-attention-2 (e.g. `pip install ninja==1.10.2.4`)
129
130
-2. Setup `xformers==0.0.22.post7` without dependencies, and apply patches to adapt for ROCm flash attention
130
+2. Setup `xformers==0.0.23` without dependencies, and apply patches to adapt for ROCm flash attention
131
132
.. code-block:: console
133
134
- $ pip install xformers==0.0.22.post7 --no-deps
135
- $ bash patch_xformers-0.0.22.post7.rocm.sh
134
+ $ pip install xformers==0.0.23 --no-deps
135
+ $ bash patch_xformers.rocm.sh
136
137
3. Build vLLM.
138
patch_xformers.rocm.sh
renamed
+17
-6
@@ -1,21 +1,32 @@
1
#!/bin/bash
2
+set -e
3
+
4
+XFORMERS_VERSION="0.0.23"
5
+
6
+export XFORMERS_INSTALLED_VERSION=$(python -c 'import xformers; print(xformers.__version__)')
7
+
8
+if [ "$XFORMERS_INSTALLED_VERSION" != "$XFORMERS_VERSION" ]; then
9
+ echo "ERROR: xformers version must be ${XFORMERS_VERSION}. ${XFORMERS_INSTALLED_VERSION} is installed"
10
+ exit 1
11
+fi
12
+
13
export XFORMERS_FMHA_FLASH_PATH=$(python -c 'from xformers import ops as xops; print(xops.fmha.flash.__file__)')
14
export XFORMERS_FMHA_COMMON_PATH=$(python -c 'from xformers import ops as xops; print(xops.fmha.common.__file__)')
15
5
-echo $XFORMERS_FMHA_FLASH_PATH
6
-echo $XFORMERS_FMHA_COMMON_PATH
16
+echo "XFORMERS_FMHA_FLASH_PATH = ${XFORMERS_FMHA_FLASH_PATH}"
17
+echo "XFORMERS_FMHA_COMMON_PATH = ${XFORMERS_FMHA_COMMON_PATH}"
18
8
-if ! patch -R -p0 -s -f --dry-run $XFORMERS_FMHA_FLASH_PATH "./rocm_patch/flashpy_xformers-0.0.22.post7.rocm.patch"; then
19
+if ! patch -R -p0 -s -f --dry-run $XFORMERS_FMHA_FLASH_PATH "./rocm_patch/flashpy_xformers-${XFORMERS_VERSION}.rocm.patch"; then
20
echo "Applying patch to ${XFORMERS_FMHA_FLASH_PATH}"
10
- patch -p0 $XFORMERS_FMHA_FLASH_PATH "./rocm_patch/flashpy_xformers-0.0.22.post7.rocm.patch"
21
+ patch -p0 $XFORMERS_FMHA_FLASH_PATH "./rocm_patch/flashpy_xformers-${XFORMERS_VERSION}.rocm.patch"
22
echo "Successfully patch ${XFORMERS_FMHA_FLASH_PATH}"
23
else
24
echo "${XFORMERS_FMHA_FLASH_PATH} was patched before"
25
fi
26
16
-if ! patch -R -p0 -s -f --dry-run $XFORMERS_FMHA_COMMON_PATH "./rocm_patch/commonpy_xformers-0.0.22.post7.rocm.patch"; then
27
+if ! patch -R -p0 -s -f --dry-run $XFORMERS_FMHA_COMMON_PATH "./rocm_patch/commonpy_xformers-${XFORMERS_VERSION}.rocm.patch"; then
28
echo "Applying patch to ${XFORMERS_FMHA_COMMON_PATH}"
18
- patch -p0 $XFORMERS_FMHA_COMMON_PATH "./rocm_patch/commonpy_xformers-0.0.22.post7.rocm.patch"
29
+ patch -p0 $XFORMERS_FMHA_COMMON_PATH "./rocm_patch/commonpy_xformers-${XFORMERS_VERSION}.rocm.patch"
30
echo "Successfully patch ${XFORMERS_FMHA_COMMON_PATH}"
31
else
32
echo "${XFORMERS_FMHA_COMMON_PATH} was patched before"
requirements-rocm.txt
-1
@@ -8,7 +8,6 @@ pyarrow # Required for Ray data.
8
sentencepiece # Required for LLaMA tokenizer.
9
numpy
10
tokenizers>=0.15.0
11
-huggingface_hub<0.18,>=0.16.4
11
transformers >= 4.36.0 # Required for Mixtral.
12
fastapi
13
uvicorn[standard]
rocm_patch/commonpy_xformers-0.0.23.rocm.patch
renamed
rocm_patch/flashpy_xformers-0.0.23.rocm.patch
renamed
+57
-39
@@ -1,6 +1,6 @@
1
---- /opt/conda/envs/py_3.10/lib/python3.10/site-packages/xformers/ops/fmha/flash.py 2023-11-29 03:17:03.930103539 +0000
2
-+++ flash.py 2023-11-28 16:14:25.206128903 +0000
3
-@@ -31,39 +31,39 @@
1
+--- flash_ori.py 2023-12-13 05:43:31.530752623 +0000
2
++++ flash_patch.py 2023-12-13 06:00:45.962403104 +0000
3
+@@ -36,44 +36,44 @@
4
5
FLASH_VERSION = "0.0.0"
6
try:
@@ -15,9 +15,12 @@
15
- from flash_attn.flash_attn_interface import flash_attn_cuda as _C_flashattention
16
-
17
- FLASH_VERSION = flash_attn.__version__
18
-- flash_ver_parsed = tuple(int(s) for s in FLASH_VERSION.split(".")[:2])
19
-- if flash_ver_parsed < (2, 3):
20
-- raise ImportError("Requires 2.3 for sliding window support")
18
+- flash_ver_parsed = tuple(int(s) for s in FLASH_VERSION.split(".")[:3])
19
+- if (
20
+- flash_ver_parsed != (2, 3, 6)
21
+- and os.environ.get("XFORMERS_IGNORE_FLASH_VERSION_CHECK", "0") != "1"
22
+- ):
23
+- raise ImportError("Requires Flash attention 2.3.6 for varlen_fwd api")
24
+ #try:
25
+ # from ... import _C_flashattention # type: ignore[attr-defined]
26
+ # from ..._cpp_lib import _build_metadata
@@ -29,35 +32,41 @@
32
+ from flash_attn.flash_attn_interface import flash_attn_cuda as _C_flashattention
33
+
34
+ FLASH_VERSION = flash_attn.__version__
32
-+ # flash_ver_parsed = tuple(int(s) for s in FLASH_VERSION.split(".")[:2])
33
-+ # if flash_ver_parsed < (2, 3):
34
-+ # raise ImportError("Requires 2.3 for sliding window support")
35
++ # flash_ver_parsed = tuple(int(s) for s in FLASH_VERSION.split(".")[:3])
36
++ # if (
37
++ # flash_ver_parsed != (2, 3, 6)
38
++ # and os.environ.get("XFORMERS_IGNORE_FLASH_VERSION_CHECK", "0") != "1"
39
++ # ):
40
++ # raise ImportError("Requires Flash attention 2.3.6 for varlen_fwd api")
41
42
# create library so that flash-attn goes through the PyTorch Dispatcher
43
- _flash_lib = torch.library.Library("xformers_flash", "DEF")
38
-+ #_flash_lib = torch.library.Library("xformers_flash", "DEF")
39
-
44
+-
45
- _flash_lib.define(
46
- "flash_fwd(Tensor query, Tensor key, Tensor value, "
42
-- "Tensor? cu_seqlens_q, Tensor? cu_seqlens_k, "
47
+- "Tensor? cu_seqlens_q, Tensor? cu_seqlens_k, Tensor? seqused_k, "
48
- "int max_seqlen_q, int max_seqlen_k, "
49
- "float p, float softmax_scale, "
45
-- "bool is_causal, int window_size, bool return_softmax) -> (Tensor, Tensor, Tensor)"
50
+- "bool is_causal, int window_left, "
51
+- "int window_right, bool return_softmax) -> (Tensor, Tensor, Tensor)"
52
- )
47
--
53
++ #_flash_lib = torch.library.Library("xformers_flash", "DEF")
54
+
55
- _flash_lib.define(
56
- "flash_bwd(Tensor dout, Tensor query, Tensor key, Tensor value, "
57
- "Tensor out, Tensor softmax_lse_, Tensor dq, Tensor dk, Tensor dv, "
58
- "Tensor cu_seqlens_q, Tensor cu_seqlens_k, "
59
- "int max_seqlen_q, int max_seqlen_k, "
53
-- "float p, float softmax_scale, bool is_causal, int window_size, Tensor rng_state) -> (Tensor, Tensor, Tensor)"
60
+- "float p, float softmax_scale, bool is_causal, "
61
+- "int window_left, int window_right, Tensor rng_state) -> (Tensor, Tensor, Tensor)"
62
- )
63
+ #_flash_lib.define(
64
+ # "flash_fwd(Tensor query, Tensor key, Tensor value, "
57
-+ # "Tensor? cu_seqlens_q, Tensor? cu_seqlens_k, "
65
++ # "Tensor? cu_seqlens_q, Tensor? cu_seqlens_k, Tensor? seqused_k, "
66
+ # "int max_seqlen_q, int max_seqlen_k, "
67
+ # "float p, float softmax_scale, "
60
-+ # "bool is_causal, int window_size, bool return_softmax) -> (Tensor, Tensor, Tensor)"
68
++ # "bool is_causal, int window_left, "
69
++ # "int window_right, bool return_softmax) -> (Tensor, Tensor, Tensor)"
70
+ #)
71
+
72
+ #_flash_lib.define(
@@ -65,52 +74,61 @@
74
+ # "Tensor out, Tensor softmax_lse_, Tensor dq, Tensor dk, Tensor dv, "
75
+ # "Tensor cu_seqlens_q, Tensor cu_seqlens_k, "
76
+ # "int max_seqlen_q, int max_seqlen_k, "
68
-+ # "float p, float softmax_scale, bool is_causal, int window_size, Tensor rng_state) -> (Tensor, Tensor, Tensor)"
77
++ # "float p, float softmax_scale, bool is_causal, "
78
++ # "int window_left, int window_right, Tensor rng_state) -> (Tensor, Tensor, Tensor)"
79
+ #)
80
81
def _flash_fwd(
82
query,
73
-@@ -98,8 +98,8 @@
83
+@@ -111,8 +111,8 @@
84
p,
85
softmax_scale,
86
is_causal,
77
-- window_size - 1, # window_size_left
78
-- -1, # window_size_right
79
-+ # window_size - 1, # window_size_left
80
-+ # -1, # window_size_right
87
+- window_left, # window_size_left
88
+- window_right, # window_size_right
89
++ # window_left, # window_size_left
90
++ # window_right, # window_size_right
91
return_softmax,
92
None, # rng
93
)
84
-@@ -127,8 +127,8 @@
94
+@@ -134,15 +134,15 @@
95
+ out,
96
+ cu_seq_lens_q,
97
+ cu_seq_lens_k,
98
+- seqused_k,
99
++ # seqused_k,
100
+ max_seq_len_q,
101
+ max_seq_len_k,
102
+ p,
103
softmax_scale,
104
False,
105
is_causal,
88
-- window_size - 1, # window_size_left
89
-- -1, # window_size_right
90
-+ # window_size - 1, # window_size_left
91
-+ # -1, # window_size_right
106
+- window_left,
107
+- window_right,
108
++ # window_left,
109
++ # window_right,
110
return_softmax,
111
None,
112
)
95
-@@ -169,8 +169,8 @@
113
+@@ -184,8 +184,8 @@
114
p,
115
softmax_scale,
116
is_causal,
99
-- window_size - 1, # window_size_left
100
-- -1, # window_size_right
101
-+ # window_size - 1, # window_size_left
102
-+ # -1, # window_size_right
117
+- window_left,
118
+- window_right,
119
++ # window_left,
120
++ # window_right,
121
None,
122
rng_state,
123
)
106
-@@ -193,15 +193,15 @@
124
+@@ -208,15 +208,15 @@
125
softmax_scale,
126
False, # zero_tensors
127
is_causal,
110
-- window_size - 1, # window_size_left
111
-- -1, # window_size_right
112
-+ # window_size - 1, # window_size_left
113
-+ # -1, # window_size_right
128
+- window_left,
129
+- window_right,
130
++ # window_left,
131
++ # window_right,
132
None,
133
rng_state,
134
)
@@ -123,7 +141,7 @@
141
except ImportError:
142
pass
143
126
-@@ -348,7 +348,7 @@
144
+@@ -400,7 +400,7 @@
145
implementation.
146
"""
147