A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[AMD][Hardware][Misc][Bugfix] xformer cleanup and light navi logic and CI fixes and refactoring (#4129)
Hongxia Yang committed
Apr 22, 2024 at 00:57 UTC
95e5b087cfed33bf20891852b2d7a0cac2341519
6 files changed
+19
-217
.buildkite/test-pipeline.yaml
-2
@@ -15,10 +15,8 @@ steps:
15
commands:
16
- VLLM_ATTENTION_BACKEND=XFORMERS pytest -v -s basic_correctness/test_basic_correctness.py
17
- VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s basic_correctness/test_basic_correctness.py
18
- - VLLM_ATTENTION_BACKEND=ROCM_FLASH pytest -v -s basic_correctness/test_basic_correctness.py
18
- VLLM_ATTENTION_BACKEND=XFORMERS pytest -v -s basic_correctness/test_chunked_prefill.py
19
- VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s basic_correctness/test_chunked_prefill.py
21
- - VLLM_ATTENTION_BACKEND=ROCM_FLASH pytest -v -s basic_correctness/test_chunked_prefill.py
20
21
- label: Core Test
22
command: pytest -v -s core
Dockerfile.rocm
+1
-4
@@ -14,7 +14,7 @@ RUN echo "Base image is $BASE_IMAGE"
14
ARG FA_GFX_ARCHS="gfx90a;gfx942"
15
RUN echo "FA_GFX_ARCHS is $FA_GFX_ARCHS"
16
17
-ARG FA_BRANCH="3d2b6f5"
17
+ARG FA_BRANCH="ae7928c"
18
RUN echo "FA_BRANCH is $FA_BRANCH"
19
20
# whether to build flash-attention
@@ -92,13 +92,10 @@ RUN if [ "$BUILD_TRITON" = "1" ]; then \
92
COPY ./ /app/vllm
93
94
RUN python3 -m pip install --upgrade pip numba
95
-RUN python3 -m pip install xformers==0.0.23 --no-deps
95
96
RUN cd /app \
97
&& cd vllm \
98
&& pip install -U -r requirements-rocm.txt \
100
- && if [ "$BUILD_FA" = "1" ]; then \
101
- bash patch_xformers.rocm.sh; fi \
99
&& patch /opt/rocm/include/hip/amd_detail/amd_hip_bf16.h /app/vllm/rocm_patch/rocm_bf16.patch \
100
&& python3 setup.py install \
101
&& cd ..
patch_xformers.rocm.sh
deleted
-33
@@ -1,33 +0,0 @@
1
-#!/bin/bash
2
-set -e
3
-
4
-XFORMERS_VERSION="0.0.23"
5
-
6
-export XFORMERS_INSTALLED_VERSION=$(python -c 'import xformers; print(xformers.__version__)')
7
-
8
-if [ "$XFORMERS_INSTALLED_VERSION" != "$XFORMERS_VERSION" ]; then
9
- echo "ERROR: xformers version must be ${XFORMERS_VERSION}. ${XFORMERS_INSTALLED_VERSION} is installed"
10
- exit 1
11
-fi
12
-
13
-export XFORMERS_FMHA_FLASH_PATH=$(python -c 'from xformers import ops as xops; print(xops.fmha.flash.__file__)')
14
-export XFORMERS_FMHA_COMMON_PATH=$(python -c 'from xformers import ops as xops; print(xops.fmha.common.__file__)')
15
-
16
-echo "XFORMERS_FMHA_FLASH_PATH = ${XFORMERS_FMHA_FLASH_PATH}"
17
-echo "XFORMERS_FMHA_COMMON_PATH = ${XFORMERS_FMHA_COMMON_PATH}"
18
-
19
-if ! patch -R -p0 -s -f --dry-run $XFORMERS_FMHA_FLASH_PATH "./rocm_patch/flashpy_xformers-${XFORMERS_VERSION}.rocm.patch"; then
20
- echo "Applying patch to ${XFORMERS_FMHA_FLASH_PATH}"
21
- patch -p0 $XFORMERS_FMHA_FLASH_PATH "./rocm_patch/flashpy_xformers-${XFORMERS_VERSION}.rocm.patch"
22
- echo "Successfully patch ${XFORMERS_FMHA_FLASH_PATH}"
23
-else
24
- echo "${XFORMERS_FMHA_FLASH_PATH} was patched before"
25
-fi
26
-
27
-if ! patch -R -p0 -s -f --dry-run $XFORMERS_FMHA_COMMON_PATH "./rocm_patch/commonpy_xformers-${XFORMERS_VERSION}.rocm.patch"; then
28
- echo "Applying patch to ${XFORMERS_FMHA_COMMON_PATH}"
29
- patch -p0 $XFORMERS_FMHA_COMMON_PATH "./rocm_patch/commonpy_xformers-${XFORMERS_VERSION}.rocm.patch"
30
- echo "Successfully patch ${XFORMERS_FMHA_COMMON_PATH}"
31
-else
32
- echo "${XFORMERS_FMHA_COMMON_PATH} was patched before"
33
-fi
rocm_patch/commonpy_xformers-0.0.23.rocm.patch
deleted
-13
@@ -1,13 +0,0 @@
1
---- /opt/conda/envs/py_3.10/lib/python3.10/site-packages/xformers/ops/fmha/common.py 2023-11-29 03:17:03.930103539 +0000
2
-+++ common.py 2023-11-28 16:14:19.846233146 +0000
3
-@@ -298,8 +298,8 @@
4
- dtype = d.query.dtype
5
- if device_type not in cls.SUPPORTED_DEVICES:
6
- reasons.append(f"device={device_type} (supported: {cls.SUPPORTED_DEVICES})")
7
-- if device_type == "cuda" and not _built_with_cuda:
8
-- reasons.append("xFormers wasn't build with CUDA support")
9
-+ #if device_type == "cuda" and not _built_with_cuda:
10
-+ # reasons.append("xFormers wasn't build with CUDA support")
11
- if device_type == "cuda":
12
- device_capability = torch.cuda.get_device_capability(d.device)
13
- if device_capability < cls.CUDA_MINIMUM_COMPUTE_CAPABILITY:
rocm_patch/flashpy_xformers-0.0.23.rocm.patch
deleted
-152
@@ -1,152 +0,0 @@
1
---- flash_ori.py 2023-12-13 05:43:31.530752623 +0000
2
-+++ flash_patch.py 2023-12-13 06:00:45.962403104 +0000
3
-@@ -36,44 +36,44 @@
4
-
5
- FLASH_VERSION = "0.0.0"
6
- try:
7
-- try:
8
-- from ... import _C_flashattention # type: ignore[attr-defined]
9
-- from ..._cpp_lib import _build_metadata
10
--
11
-- if _build_metadata is not None:
12
-- FLASH_VERSION = _build_metadata.flash_version
13
-- except ImportError:
14
-- import flash_attn
15
-- from flash_attn.flash_attn_interface import flash_attn_cuda as _C_flashattention
16
--
17
-- FLASH_VERSION = flash_attn.__version__
18
-- flash_ver_parsed = tuple(int(s) for s in FLASH_VERSION.split(".")[:3])
19
-- if (
20
-- flash_ver_parsed != (2, 3, 6)
21
-- and os.environ.get("XFORMERS_IGNORE_FLASH_VERSION_CHECK", "0") != "1"
22
-- ):
23
-- raise ImportError("Requires Flash attention 2.3.6 for varlen_fwd api")
24
-+ #try:
25
-+ # from ... import _C_flashattention # type: ignore[attr-defined]
26
-+ # from ..._cpp_lib import _build_metadata
27
-+
28
-+ # if _build_metadata is not None:
29
-+ # FLASH_VERSION = _build_metadata.flash_version
30
-+ #except ImportError:
31
-+ import flash_attn
32
-+ from flash_attn.flash_attn_interface import flash_attn_cuda as _C_flashattention
33
-+
34
-+ FLASH_VERSION = flash_attn.__version__
35
-+ # flash_ver_parsed = tuple(int(s) for s in FLASH_VERSION.split(".")[:3])
36
-+ # if (
37
-+ # flash_ver_parsed != (2, 3, 6)
38
-+ # and os.environ.get("XFORMERS_IGNORE_FLASH_VERSION_CHECK", "0") != "1"
39
-+ # ):
40
-+ # raise ImportError("Requires Flash attention 2.3.6 for varlen_fwd api")
41
-
42
- # create library so that flash-attn goes through the PyTorch Dispatcher
43
-- _flash_lib = torch.library.Library("xformers_flash", "DEF")
44
--
45
-- _flash_lib.define(
46
-- "flash_fwd(Tensor query, Tensor key, Tensor value, "
47
-- "Tensor? cu_seqlens_q, Tensor? cu_seqlens_k, Tensor? seqused_k, "
48
-- "int max_seqlen_q, int max_seqlen_k, "
49
-- "float p, float softmax_scale, "
50
-- "bool is_causal, int window_left, "
51
-- "int window_right, bool return_softmax) -> (Tensor, Tensor, Tensor)"
52
-- )
53
-+ #_flash_lib = torch.library.Library("xformers_flash", "DEF")
54
-
55
-- _flash_lib.define(
56
-- "flash_bwd(Tensor dout, Tensor query, Tensor key, Tensor value, "
57
-- "Tensor out, Tensor softmax_lse_, Tensor dq, Tensor dk, Tensor dv, "
58
-- "Tensor cu_seqlens_q, Tensor cu_seqlens_k, "
59
-- "int max_seqlen_q, int max_seqlen_k, "
60
-- "float p, float softmax_scale, bool is_causal, "
61
-- "int window_left, int window_right, Tensor rng_state) -> (Tensor, Tensor, Tensor)"
62
-- )
63
-+ #_flash_lib.define(
64
-+ # "flash_fwd(Tensor query, Tensor key, Tensor value, "
65
-+ # "Tensor? cu_seqlens_q, Tensor? cu_seqlens_k, Tensor? seqused_k, "
66
-+ # "int max_seqlen_q, int max_seqlen_k, "
67
-+ # "float p, float softmax_scale, "
68
-+ # "bool is_causal, int window_left, "
69
-+ # "int window_right, bool return_softmax) -> (Tensor, Tensor, Tensor)"
70
-+ #)
71
-+
72
-+ #_flash_lib.define(
73
-+ # "flash_bwd(Tensor dout, Tensor query, Tensor key, Tensor value, "
74
-+ # "Tensor out, Tensor softmax_lse_, Tensor dq, Tensor dk, Tensor dv, "
75
-+ # "Tensor cu_seqlens_q, Tensor cu_seqlens_k, "
76
-+ # "int max_seqlen_q, int max_seqlen_k, "
77
-+ # "float p, float softmax_scale, bool is_causal, "
78
-+ # "int window_left, int window_right, Tensor rng_state) -> (Tensor, Tensor, Tensor)"
79
-+ #)
80
-
81
- def _flash_fwd(
82
- query,
83
-@@ -111,8 +111,8 @@
84
- p,
85
- softmax_scale,
86
- is_causal,
87
-- window_left, # window_size_left
88
-- window_right, # window_size_right
89
-+ # window_left, # window_size_left
90
-+ # window_right, # window_size_right
91
- return_softmax,
92
- None, # rng
93
- )
94
-@@ -134,15 +134,15 @@
95
- out,
96
- cu_seq_lens_q,
97
- cu_seq_lens_k,
98
-- seqused_k,
99
-+ # seqused_k,
100
- max_seq_len_q,
101
- max_seq_len_k,
102
- p,
103
- softmax_scale,
104
- False,
105
- is_causal,
106
-- window_left,
107
-- window_right,
108
-+ # window_left,
109
-+ # window_right,
110
- return_softmax,
111
- None,
112
- )
113
-@@ -184,8 +184,8 @@
114
- p,
115
- softmax_scale,
116
- is_causal,
117
-- window_left,
118
-- window_right,
119
-+ # window_left,
120
-+ # window_right,
121
- None,
122
- rng_state,
123
- )
124
-@@ -208,15 +208,15 @@
125
- softmax_scale,
126
- False, # zero_tensors
127
- is_causal,
128
-- window_left,
129
-- window_right,
130
-+ # window_left,
131
-+ # window_right,
132
- None,
133
- rng_state,
134
- )
135
- return dq, dk, dv
136
-
137
-- _flash_lib.impl("flash_fwd", _flash_fwd, "CUDA")
138
-- _flash_lib.impl("flash_bwd", _flash_bwd, "CUDA")
139
-+ #_flash_lib.impl("flash_fwd", _flash_fwd, "CUDA")
140
-+ #_flash_lib.impl("flash_bwd", _flash_bwd, "CUDA")
141
- except ImportError:
142
- pass
143
-
144
-@@ -400,7 +400,7 @@
145
- implementation.
146
- """
147
-
148
-- OPERATOR = get_operator("xformers_flash", "flash_fwd")
149
-+ OPERATOR = _flash_fwd # get_operator("xformers_flash", "flash_fwd")
150
- SUPPORTED_DEVICES: Set[str] = {"cuda"}
151
- CUDA_MINIMUM_COMPUTE_CAPABILITY = (8, 0)
152
- SUPPORTED_DTYPES: Set[torch.dtype] = {torch.half, torch.bfloat16}
vllm/attention/backends/rocm_flash_attn.py
+18
-13
@@ -154,25 +154,30 @@ class ROCmFlashAttentionImpl(AttentionImpl):
154
f"Head size {head_size} is not supported by PagedAttention. "
155
f"Supported head sizes are: {suppored_head_sizes}.")
156
157
- self.use_naive_attn = torch.cuda.get_device_capability()[0] != 9
157
+ self.use_naive_attn = False
158
# NOTE: Allow for switching between Triton and CK. Defaulting to triton.
159
self.use_triton_flash_attn = (os.environ.get(
160
"VLLM_USE_TRITON_FLASH_ATTN", "True").lower() in ("true", "1"))
161
- if self.use_naive_attn:
162
- # AMD Radeon 7900 series (gfx1100) currently does not support
163
- # xFormers nor FlashAttention. As a temporary workaround, we use
164
- # naive PyTorch implementation of attention.
165
- self.attn_fuc = _naive_attention
166
- logger.debug("Using naive attention in ROCmBackend")
167
- elif self.use_triton_flash_attn:
161
+ if self.use_triton_flash_attn:
162
from vllm.attention.ops.triton_flash_attention import ( # noqa: F401
163
triton_attention)
164
self.attn_func = triton_attention
165
logger.debug("Using Triton FA in ROCmBackend")
166
else:
173
- from flash_attn import flash_attn_varlen_func # noqa: F401
174
- self.attn_func = flash_attn_varlen_func
175
- logger.debug("Using CK FA in ROCmBackend")
167
+ # if not using triton, navi3x not use flash-attn either
168
+ if torch.cuda.get_device_capability()[0] == 11:
169
+ self.use_naive_attn = True
170
+ else:
171
+ try:
172
+ from flash_attn import flash_attn_varlen_func # noqa: F401
173
+ self.attn_func = flash_attn_varlen_func
174
+ logger.debug("Using CK FA in ROCmBackend")
175
+ except ModuleNotFoundError:
176
+ self.use_naive_attn = True
177
+
178
+ if self.use_naive_attn:
179
+ self.attn_func = _naive_attention
180
+ logger.debug("Using naive attention in ROCmBackend")
181
182
def repeat_kv(self, x: torch.Tensor, n_rep: int) -> torch.Tensor:
183
"""torch.repeat_interleave(x, dim=1, repeats=n_rep)"""
@@ -247,13 +252,13 @@ class ROCmFlashAttentionImpl(AttentionImpl):
252
# triton attention
253
# When block_tables are not filled, it means q and k are the
254
# prompt, and they have the same length.
250
- if self.use_naive_attn or self.use_triton_flash_attn:
255
+ if self.use_triton_flash_attn or self.use_naive_attn:
256
if self.num_kv_heads != self.num_heads:
257
# Interleave for MQA workaround.
258
key = self.repeat_kv(key, self.num_queries_per_kv)
259
value = self.repeat_kv(value, self.num_queries_per_kv)
260
if self.use_naive_attn:
256
- out = self.attn_fuc(
261
+ out = self.attn_func(
262
query,
263
key,
264
value,