@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f375ec8440
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[ROCm] Upgrade xformers version for ROCm & update doc (#2079)

Co-authored-by: miloice <jeffaw99@hotmail.com>

TJian committed Dec 13, 2023 at 16:56 UTC f375ec844036104cbae36477ce3541588d026829
6 files changed +84 -56
Dockerfile.rocm
+2 -2
@@ -47,12 +47,12 @@ RUN mkdir libs \
47 COPY ./ /app/vllm
48
49 RUN python3 -m pip install --upgrade pip
50 -RUN pip install xformers==0.0.22.post7 --no-deps
50 +RUN pip install xformers==0.0.23 --no-deps
51
52 RUN cd /app \
53 && cd vllm \
54 && pip install -U -r requirements-rocm.txt \
55 - && bash patch_xformers-0.0.22.post7.rocm.sh \
55 + && bash patch_xformers-0.0.23.rocm.sh \
56 && python3 setup.py install \
57 && cd ..
58
docs/source/getting_started/amd-installation.rst
+8 -8
@@ -3,7 +3,7 @@
3 Installation with ROCm
4 ======================
5
6 -vLLM 0.2.x onwards supports model inferencing and serving on AMD GPUs with ROCm.
6 +vLLM 0.2.4 onwards supports model inferencing and serving on AMD GPUs with ROCm.
7 At the moment AWQ quantization is not supported in ROCm, but SqueezeLLM quantization has been ported.
8 Data types currently supported in ROCm are FP16 and BF16.
9
@@ -29,7 +29,7 @@ Installation options:
29
30 .. code-block:: console
31
32 - $ docker pull embeddedllminfo/vllm-rocm:vllm-v0.2.3
32 + $ docker pull embeddedllminfo/vllm-rocm:vllm-v0.2.4
33 $ docker run -it \
34 --network=host \
35 --group-add=video \
@@ -70,12 +70,12 @@ You can build and install vLLM from source:
70 - ROCm's Flash-attention-2 (v2.0.4) does not support sliding windows attention.
71 - You might need to downgrade the "ninja" version to 1.10 it is not used when compiling flash-attention-2 (e.g. `pip install ninja==1.10.2.4`)
72
73 -2. Setup `xformers==0.0.22.post7` without dependencies, and apply patches to adapt for ROCm flash attention
73 +2. Setup `xformers==0.0.23` without dependencies, and apply patches to adapt for ROCm flash attention
74
75 .. code-block:: console
76
77 - $ pip install xformers==0.0.22.post7 --no-deps
78 - $ bash patch_xformers-0.0.22.post7.rocm.sh
77 + $ pip install xformers==0.0.23 --no-deps
78 + $ bash patch_xformers.rocm.sh
79
80 3. Build vLLM.
81
@@ -127,12 +127,12 @@ Alternatively, if you plan to install vLLM-ROCm on a local machine or start from
127 - ROCm's Flash-attention-2 (v2.0.4) does not support sliding windows attention.
128 - You might need to downgrade the "ninja" version to 1.10 it is not used when compiling flash-attention-2 (e.g. `pip install ninja==1.10.2.4`)
129
130 -2. Setup `xformers==0.0.22.post7` without dependencies, and apply patches to adapt for ROCm flash attention
130 +2. Setup `xformers==0.0.23` without dependencies, and apply patches to adapt for ROCm flash attention
131
132 .. code-block:: console
133
134 - $ pip install xformers==0.0.22.post7 --no-deps
135 - $ bash patch_xformers-0.0.22.post7.rocm.sh
134 + $ pip install xformers==0.0.23 --no-deps
135 + $ bash patch_xformers.rocm.sh
136
137 3. Build vLLM.
138
patch_xformers.rocm.sh renamed
+17 -6
@@ -1,21 +1,32 @@
1 #!/bin/bash
2 +set -e
3 +
4 +XFORMERS_VERSION="0.0.23"
5 +
6 +export XFORMERS_INSTALLED_VERSION=$(python -c 'import xformers; print(xformers.__version__)')
7 +
8 +if [ "$XFORMERS_INSTALLED_VERSION" != "$XFORMERS_VERSION" ]; then
9 + echo "ERROR: xformers version must be ${XFORMERS_VERSION}. ${XFORMERS_INSTALLED_VERSION} is installed"
10 + exit 1
11 +fi
12 +
13 export XFORMERS_FMHA_FLASH_PATH=$(python -c 'from xformers import ops as xops; print(xops.fmha.flash.__file__)')
14 export XFORMERS_FMHA_COMMON_PATH=$(python -c 'from xformers import ops as xops; print(xops.fmha.common.__file__)')
15
5 -echo $XFORMERS_FMHA_FLASH_PATH
6 -echo $XFORMERS_FMHA_COMMON_PATH
16 +echo "XFORMERS_FMHA_FLASH_PATH = ${XFORMERS_FMHA_FLASH_PATH}"
17 +echo "XFORMERS_FMHA_COMMON_PATH = ${XFORMERS_FMHA_COMMON_PATH}"
18
8 -if ! patch -R -p0 -s -f --dry-run $XFORMERS_FMHA_FLASH_PATH "./rocm_patch/flashpy_xformers-0.0.22.post7.rocm.patch"; then
19 +if ! patch -R -p0 -s -f --dry-run $XFORMERS_FMHA_FLASH_PATH "./rocm_patch/flashpy_xformers-${XFORMERS_VERSION}.rocm.patch"; then
20 echo "Applying patch to ${XFORMERS_FMHA_FLASH_PATH}"
10 - patch -p0 $XFORMERS_FMHA_FLASH_PATH "./rocm_patch/flashpy_xformers-0.0.22.post7.rocm.patch"
21 + patch -p0 $XFORMERS_FMHA_FLASH_PATH "./rocm_patch/flashpy_xformers-${XFORMERS_VERSION}.rocm.patch"
22 echo "Successfully patch ${XFORMERS_FMHA_FLASH_PATH}"
23 else
24 echo "${XFORMERS_FMHA_FLASH_PATH} was patched before"
25 fi
26
16 -if ! patch -R -p0 -s -f --dry-run $XFORMERS_FMHA_COMMON_PATH "./rocm_patch/commonpy_xformers-0.0.22.post7.rocm.patch"; then
27 +if ! patch -R -p0 -s -f --dry-run $XFORMERS_FMHA_COMMON_PATH "./rocm_patch/commonpy_xformers-${XFORMERS_VERSION}.rocm.patch"; then
28 echo "Applying patch to ${XFORMERS_FMHA_COMMON_PATH}"
18 - patch -p0 $XFORMERS_FMHA_COMMON_PATH "./rocm_patch/commonpy_xformers-0.0.22.post7.rocm.patch"
29 + patch -p0 $XFORMERS_FMHA_COMMON_PATH "./rocm_patch/commonpy_xformers-${XFORMERS_VERSION}.rocm.patch"
30 echo "Successfully patch ${XFORMERS_FMHA_COMMON_PATH}"
31 else
32 echo "${XFORMERS_FMHA_COMMON_PATH} was patched before"
requirements-rocm.txt
-1
@@ -8,7 +8,6 @@ pyarrow # Required for Ray data.
8 sentencepiece # Required for LLaMA tokenizer.
9 numpy
10 tokenizers>=0.15.0
11 -huggingface_hub<0.18,>=0.16.4
11 transformers >= 4.36.0 # Required for Mixtral.
12 fastapi
13 uvicorn[standard]
rocm_patch/commonpy_xformers-0.0.23.rocm.patch renamed
rocm_patch/flashpy_xformers-0.0.23.rocm.patch renamed
+57 -39
@@ -1,6 +1,6 @@
1 ---- /opt/conda/envs/py_3.10/lib/python3.10/site-packages/xformers/ops/fmha/flash.py 2023-11-29 03:17:03.930103539 +0000
2 -+++ flash.py 2023-11-28 16:14:25.206128903 +0000
3 -@@ -31,39 +31,39 @@
1 +--- flash_ori.py 2023-12-13 05:43:31.530752623 +0000
2 ++++ flash_patch.py 2023-12-13 06:00:45.962403104 +0000
3 +@@ -36,44 +36,44 @@
4
5 FLASH_VERSION = "0.0.0"
6 try:
@@ -15,9 +15,12 @@
15 - from flash_attn.flash_attn_interface import flash_attn_cuda as _C_flashattention
16 -
17 - FLASH_VERSION = flash_attn.__version__
18 -- flash_ver_parsed = tuple(int(s) for s in FLASH_VERSION.split(".")[:2])
19 -- if flash_ver_parsed < (2, 3):
20 -- raise ImportError("Requires 2.3 for sliding window support")
18 +- flash_ver_parsed = tuple(int(s) for s in FLASH_VERSION.split(".")[:3])
19 +- if (
20 +- flash_ver_parsed != (2, 3, 6)
21 +- and os.environ.get("XFORMERS_IGNORE_FLASH_VERSION_CHECK", "0") != "1"
22 +- ):
23 +- raise ImportError("Requires Flash attention 2.3.6 for varlen_fwd api")
24 + #try:
25 + # from ... import _C_flashattention # type: ignore[attr-defined]
26 + # from ..._cpp_lib import _build_metadata
@@ -29,35 +32,41 @@
32 + from flash_attn.flash_attn_interface import flash_attn_cuda as _C_flashattention
33 +
34 + FLASH_VERSION = flash_attn.__version__
32 -+ # flash_ver_parsed = tuple(int(s) for s in FLASH_VERSION.split(".")[:2])
33 -+ # if flash_ver_parsed < (2, 3):
34 -+ # raise ImportError("Requires 2.3 for sliding window support")
35 ++ # flash_ver_parsed = tuple(int(s) for s in FLASH_VERSION.split(".")[:3])
36 ++ # if (
37 ++ # flash_ver_parsed != (2, 3, 6)
38 ++ # and os.environ.get("XFORMERS_IGNORE_FLASH_VERSION_CHECK", "0") != "1"
39 ++ # ):
40 ++ # raise ImportError("Requires Flash attention 2.3.6 for varlen_fwd api")
41
42 # create library so that flash-attn goes through the PyTorch Dispatcher
43 - _flash_lib = torch.library.Library("xformers_flash", "DEF")
38 -+ #_flash_lib = torch.library.Library("xformers_flash", "DEF")
39 -
44 +-
45 - _flash_lib.define(
46 - "flash_fwd(Tensor query, Tensor key, Tensor value, "
42 -- "Tensor? cu_seqlens_q, Tensor? cu_seqlens_k, "
47 +- "Tensor? cu_seqlens_q, Tensor? cu_seqlens_k, Tensor? seqused_k, "
48 - "int max_seqlen_q, int max_seqlen_k, "
49 - "float p, float softmax_scale, "
45 -- "bool is_causal, int window_size, bool return_softmax) -> (Tensor, Tensor, Tensor)"
50 +- "bool is_causal, int window_left, "
51 +- "int window_right, bool return_softmax) -> (Tensor, Tensor, Tensor)"
52 - )
47 --
53 ++ #_flash_lib = torch.library.Library("xformers_flash", "DEF")
54 +
55 - _flash_lib.define(
56 - "flash_bwd(Tensor dout, Tensor query, Tensor key, Tensor value, "
57 - "Tensor out, Tensor softmax_lse_, Tensor dq, Tensor dk, Tensor dv, "
58 - "Tensor cu_seqlens_q, Tensor cu_seqlens_k, "
59 - "int max_seqlen_q, int max_seqlen_k, "
53 -- "float p, float softmax_scale, bool is_causal, int window_size, Tensor rng_state) -> (Tensor, Tensor, Tensor)"
60 +- "float p, float softmax_scale, bool is_causal, "
61 +- "int window_left, int window_right, Tensor rng_state) -> (Tensor, Tensor, Tensor)"
62 - )
63 + #_flash_lib.define(
64 + # "flash_fwd(Tensor query, Tensor key, Tensor value, "
57 -+ # "Tensor? cu_seqlens_q, Tensor? cu_seqlens_k, "
65 ++ # "Tensor? cu_seqlens_q, Tensor? cu_seqlens_k, Tensor? seqused_k, "
66 + # "int max_seqlen_q, int max_seqlen_k, "
67 + # "float p, float softmax_scale, "
60 -+ # "bool is_causal, int window_size, bool return_softmax) -> (Tensor, Tensor, Tensor)"
68 ++ # "bool is_causal, int window_left, "
69 ++ # "int window_right, bool return_softmax) -> (Tensor, Tensor, Tensor)"
70 + #)
71 +
72 + #_flash_lib.define(
@@ -65,52 +74,61 @@
74 + # "Tensor out, Tensor softmax_lse_, Tensor dq, Tensor dk, Tensor dv, "
75 + # "Tensor cu_seqlens_q, Tensor cu_seqlens_k, "
76 + # "int max_seqlen_q, int max_seqlen_k, "
68 -+ # "float p, float softmax_scale, bool is_causal, int window_size, Tensor rng_state) -> (Tensor, Tensor, Tensor)"
77 ++ # "float p, float softmax_scale, bool is_causal, "
78 ++ # "int window_left, int window_right, Tensor rng_state) -> (Tensor, Tensor, Tensor)"
79 + #)
80
81 def _flash_fwd(
82 query,
73 -@@ -98,8 +98,8 @@
83 +@@ -111,8 +111,8 @@
84 p,
85 softmax_scale,
86 is_causal,
77 -- window_size - 1, # window_size_left
78 -- -1, # window_size_right
79 -+ # window_size - 1, # window_size_left
80 -+ # -1, # window_size_right
87 +- window_left, # window_size_left
88 +- window_right, # window_size_right
89 ++ # window_left, # window_size_left
90 ++ # window_right, # window_size_right
91 return_softmax,
92 None, # rng
93 )
84 -@@ -127,8 +127,8 @@
94 +@@ -134,15 +134,15 @@
95 + out,
96 + cu_seq_lens_q,
97 + cu_seq_lens_k,
98 +- seqused_k,
99 ++ # seqused_k,
100 + max_seq_len_q,
101 + max_seq_len_k,
102 + p,
103 softmax_scale,
104 False,
105 is_causal,
88 -- window_size - 1, # window_size_left
89 -- -1, # window_size_right
90 -+ # window_size - 1, # window_size_left
91 -+ # -1, # window_size_right
106 +- window_left,
107 +- window_right,
108 ++ # window_left,
109 ++ # window_right,
110 return_softmax,
111 None,
112 )
95 -@@ -169,8 +169,8 @@
113 +@@ -184,8 +184,8 @@
114 p,
115 softmax_scale,
116 is_causal,
99 -- window_size - 1, # window_size_left
100 -- -1, # window_size_right
101 -+ # window_size - 1, # window_size_left
102 -+ # -1, # window_size_right
117 +- window_left,
118 +- window_right,
119 ++ # window_left,
120 ++ # window_right,
121 None,
122 rng_state,
123 )
106 -@@ -193,15 +193,15 @@
124 +@@ -208,15 +208,15 @@
125 softmax_scale,
126 False, # zero_tensors
127 is_causal,
110 -- window_size - 1, # window_size_left
111 -- -1, # window_size_right
112 -+ # window_size - 1, # window_size_left
113 -+ # -1, # window_size_right
128 +- window_left,
129 +- window_right,
130 ++ # window_left,
131 ++ # window_right,
132 None,
133 rng_state,
134 )
@@ -123,7 +141,7 @@
141 except ImportError:
142 pass
143
126 -@@ -348,7 +348,7 @@
144 +@@ -400,7 +400,7 @@
145 implementation.
146 """
147