@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 95e5b087cf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[AMD][Hardware][Misc][Bugfix] xformer cleanup and light navi logic and CI fixes and refactoring (#4129)

Hongxia Yang committed Apr 22, 2024 at 00:57 UTC 95e5b087cfed33bf20891852b2d7a0cac2341519
6 files changed +19 -217
.buildkite/test-pipeline.yaml
-2
@@ -15,10 +15,8 @@ steps:
15 commands:
16 - VLLM_ATTENTION_BACKEND=XFORMERS pytest -v -s basic_correctness/test_basic_correctness.py
17 - VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s basic_correctness/test_basic_correctness.py
18 - - VLLM_ATTENTION_BACKEND=ROCM_FLASH pytest -v -s basic_correctness/test_basic_correctness.py
18 - VLLM_ATTENTION_BACKEND=XFORMERS pytest -v -s basic_correctness/test_chunked_prefill.py
19 - VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s basic_correctness/test_chunked_prefill.py
21 - - VLLM_ATTENTION_BACKEND=ROCM_FLASH pytest -v -s basic_correctness/test_chunked_prefill.py
20
21 - label: Core Test
22 command: pytest -v -s core
Dockerfile.rocm
+1 -4
@@ -14,7 +14,7 @@ RUN echo "Base image is $BASE_IMAGE"
14 ARG FA_GFX_ARCHS="gfx90a;gfx942"
15 RUN echo "FA_GFX_ARCHS is $FA_GFX_ARCHS"
16
17 -ARG FA_BRANCH="3d2b6f5"
17 +ARG FA_BRANCH="ae7928c"
18 RUN echo "FA_BRANCH is $FA_BRANCH"
19
20 # whether to build flash-attention
@@ -92,13 +92,10 @@ RUN if [ "$BUILD_TRITON" = "1" ]; then \
92 COPY ./ /app/vllm
93
94 RUN python3 -m pip install --upgrade pip numba
95 -RUN python3 -m pip install xformers==0.0.23 --no-deps
95
96 RUN cd /app \
97 && cd vllm \
98 && pip install -U -r requirements-rocm.txt \
100 - && if [ "$BUILD_FA" = "1" ]; then \
101 - bash patch_xformers.rocm.sh; fi \
99 && patch /opt/rocm/include/hip/amd_detail/amd_hip_bf16.h /app/vllm/rocm_patch/rocm_bf16.patch \
100 && python3 setup.py install \
101 && cd ..
patch_xformers.rocm.sh deleted
-33
@@ -1,33 +0,0 @@
1 -#!/bin/bash
2 -set -e
3 -
4 -XFORMERS_VERSION="0.0.23"
5 -
6 -export XFORMERS_INSTALLED_VERSION=$(python -c 'import xformers; print(xformers.__version__)')
7 -
8 -if [ "$XFORMERS_INSTALLED_VERSION" != "$XFORMERS_VERSION" ]; then
9 - echo "ERROR: xformers version must be ${XFORMERS_VERSION}. ${XFORMERS_INSTALLED_VERSION} is installed"
10 - exit 1
11 -fi
12 -
13 -export XFORMERS_FMHA_FLASH_PATH=$(python -c 'from xformers import ops as xops; print(xops.fmha.flash.__file__)')
14 -export XFORMERS_FMHA_COMMON_PATH=$(python -c 'from xformers import ops as xops; print(xops.fmha.common.__file__)')
15 -
16 -echo "XFORMERS_FMHA_FLASH_PATH = ${XFORMERS_FMHA_FLASH_PATH}"
17 -echo "XFORMERS_FMHA_COMMON_PATH = ${XFORMERS_FMHA_COMMON_PATH}"
18 -
19 -if ! patch -R -p0 -s -f --dry-run $XFORMERS_FMHA_FLASH_PATH "./rocm_patch/flashpy_xformers-${XFORMERS_VERSION}.rocm.patch"; then
20 - echo "Applying patch to ${XFORMERS_FMHA_FLASH_PATH}"
21 - patch -p0 $XFORMERS_FMHA_FLASH_PATH "./rocm_patch/flashpy_xformers-${XFORMERS_VERSION}.rocm.patch"
22 - echo "Successfully patch ${XFORMERS_FMHA_FLASH_PATH}"
23 -else
24 - echo "${XFORMERS_FMHA_FLASH_PATH} was patched before"
25 -fi
26 -
27 -if ! patch -R -p0 -s -f --dry-run $XFORMERS_FMHA_COMMON_PATH "./rocm_patch/commonpy_xformers-${XFORMERS_VERSION}.rocm.patch"; then
28 - echo "Applying patch to ${XFORMERS_FMHA_COMMON_PATH}"
29 - patch -p0 $XFORMERS_FMHA_COMMON_PATH "./rocm_patch/commonpy_xformers-${XFORMERS_VERSION}.rocm.patch"
30 - echo "Successfully patch ${XFORMERS_FMHA_COMMON_PATH}"
31 -else
32 - echo "${XFORMERS_FMHA_COMMON_PATH} was patched before"
33 -fi
rocm_patch/commonpy_xformers-0.0.23.rocm.patch deleted
-13
@@ -1,13 +0,0 @@
1 ---- /opt/conda/envs/py_3.10/lib/python3.10/site-packages/xformers/ops/fmha/common.py 2023-11-29 03:17:03.930103539 +0000
2 -+++ common.py 2023-11-28 16:14:19.846233146 +0000
3 -@@ -298,8 +298,8 @@
4 - dtype = d.query.dtype
5 - if device_type not in cls.SUPPORTED_DEVICES:
6 - reasons.append(f"device={device_type} (supported: {cls.SUPPORTED_DEVICES})")
7 -- if device_type == "cuda" and not _built_with_cuda:
8 -- reasons.append("xFormers wasn't build with CUDA support")
9 -+ #if device_type == "cuda" and not _built_with_cuda:
10 -+ # reasons.append("xFormers wasn't build with CUDA support")
11 - if device_type == "cuda":
12 - device_capability = torch.cuda.get_device_capability(d.device)
13 - if device_capability < cls.CUDA_MINIMUM_COMPUTE_CAPABILITY:
rocm_patch/flashpy_xformers-0.0.23.rocm.patch deleted
-152
@@ -1,152 +0,0 @@
1 ---- flash_ori.py 2023-12-13 05:43:31.530752623 +0000
2 -+++ flash_patch.py 2023-12-13 06:00:45.962403104 +0000
3 -@@ -36,44 +36,44 @@
4 -
5 - FLASH_VERSION = "0.0.0"
6 - try:
7 -- try:
8 -- from ... import _C_flashattention # type: ignore[attr-defined]
9 -- from ..._cpp_lib import _build_metadata
10 --
11 -- if _build_metadata is not None:
12 -- FLASH_VERSION = _build_metadata.flash_version
13 -- except ImportError:
14 -- import flash_attn
15 -- from flash_attn.flash_attn_interface import flash_attn_cuda as _C_flashattention
16 --
17 -- FLASH_VERSION = flash_attn.__version__
18 -- flash_ver_parsed = tuple(int(s) for s in FLASH_VERSION.split(".")[:3])
19 -- if (
20 -- flash_ver_parsed != (2, 3, 6)
21 -- and os.environ.get("XFORMERS_IGNORE_FLASH_VERSION_CHECK", "0") != "1"
22 -- ):
23 -- raise ImportError("Requires Flash attention 2.3.6 for varlen_fwd api")
24 -+ #try:
25 -+ # from ... import _C_flashattention # type: ignore[attr-defined]
26 -+ # from ..._cpp_lib import _build_metadata
27 -+
28 -+ # if _build_metadata is not None:
29 -+ # FLASH_VERSION = _build_metadata.flash_version
30 -+ #except ImportError:
31 -+ import flash_attn
32 -+ from flash_attn.flash_attn_interface import flash_attn_cuda as _C_flashattention
33 -+
34 -+ FLASH_VERSION = flash_attn.__version__
35 -+ # flash_ver_parsed = tuple(int(s) for s in FLASH_VERSION.split(".")[:3])
36 -+ # if (
37 -+ # flash_ver_parsed != (2, 3, 6)
38 -+ # and os.environ.get("XFORMERS_IGNORE_FLASH_VERSION_CHECK", "0") != "1"
39 -+ # ):
40 -+ # raise ImportError("Requires Flash attention 2.3.6 for varlen_fwd api")
41 -
42 - # create library so that flash-attn goes through the PyTorch Dispatcher
43 -- _flash_lib = torch.library.Library("xformers_flash", "DEF")
44 --
45 -- _flash_lib.define(
46 -- "flash_fwd(Tensor query, Tensor key, Tensor value, "
47 -- "Tensor? cu_seqlens_q, Tensor? cu_seqlens_k, Tensor? seqused_k, "
48 -- "int max_seqlen_q, int max_seqlen_k, "
49 -- "float p, float softmax_scale, "
50 -- "bool is_causal, int window_left, "
51 -- "int window_right, bool return_softmax) -> (Tensor, Tensor, Tensor)"
52 -- )
53 -+ #_flash_lib = torch.library.Library("xformers_flash", "DEF")
54 -
55 -- _flash_lib.define(
56 -- "flash_bwd(Tensor dout, Tensor query, Tensor key, Tensor value, "
57 -- "Tensor out, Tensor softmax_lse_, Tensor dq, Tensor dk, Tensor dv, "
58 -- "Tensor cu_seqlens_q, Tensor cu_seqlens_k, "
59 -- "int max_seqlen_q, int max_seqlen_k, "
60 -- "float p, float softmax_scale, bool is_causal, "
61 -- "int window_left, int window_right, Tensor rng_state) -> (Tensor, Tensor, Tensor)"
62 -- )
63 -+ #_flash_lib.define(
64 -+ # "flash_fwd(Tensor query, Tensor key, Tensor value, "
65 -+ # "Tensor? cu_seqlens_q, Tensor? cu_seqlens_k, Tensor? seqused_k, "
66 -+ # "int max_seqlen_q, int max_seqlen_k, "
67 -+ # "float p, float softmax_scale, "
68 -+ # "bool is_causal, int window_left, "
69 -+ # "int window_right, bool return_softmax) -> (Tensor, Tensor, Tensor)"
70 -+ #)
71 -+
72 -+ #_flash_lib.define(
73 -+ # "flash_bwd(Tensor dout, Tensor query, Tensor key, Tensor value, "
74 -+ # "Tensor out, Tensor softmax_lse_, Tensor dq, Tensor dk, Tensor dv, "
75 -+ # "Tensor cu_seqlens_q, Tensor cu_seqlens_k, "
76 -+ # "int max_seqlen_q, int max_seqlen_k, "
77 -+ # "float p, float softmax_scale, bool is_causal, "
78 -+ # "int window_left, int window_right, Tensor rng_state) -> (Tensor, Tensor, Tensor)"
79 -+ #)
80 -
81 - def _flash_fwd(
82 - query,
83 -@@ -111,8 +111,8 @@
84 - p,
85 - softmax_scale,
86 - is_causal,
87 -- window_left, # window_size_left
88 -- window_right, # window_size_right
89 -+ # window_left, # window_size_left
90 -+ # window_right, # window_size_right
91 - return_softmax,
92 - None, # rng
93 - )
94 -@@ -134,15 +134,15 @@
95 - out,
96 - cu_seq_lens_q,
97 - cu_seq_lens_k,
98 -- seqused_k,
99 -+ # seqused_k,
100 - max_seq_len_q,
101 - max_seq_len_k,
102 - p,
103 - softmax_scale,
104 - False,
105 - is_causal,
106 -- window_left,
107 -- window_right,
108 -+ # window_left,
109 -+ # window_right,
110 - return_softmax,
111 - None,
112 - )
113 -@@ -184,8 +184,8 @@
114 - p,
115 - softmax_scale,
116 - is_causal,
117 -- window_left,
118 -- window_right,
119 -+ # window_left,
120 -+ # window_right,
121 - None,
122 - rng_state,
123 - )
124 -@@ -208,15 +208,15 @@
125 - softmax_scale,
126 - False, # zero_tensors
127 - is_causal,
128 -- window_left,
129 -- window_right,
130 -+ # window_left,
131 -+ # window_right,
132 - None,
133 - rng_state,
134 - )
135 - return dq, dk, dv
136 -
137 -- _flash_lib.impl("flash_fwd", _flash_fwd, "CUDA")
138 -- _flash_lib.impl("flash_bwd", _flash_bwd, "CUDA")
139 -+ #_flash_lib.impl("flash_fwd", _flash_fwd, "CUDA")
140 -+ #_flash_lib.impl("flash_bwd", _flash_bwd, "CUDA")
141 - except ImportError:
142 - pass
143 -
144 -@@ -400,7 +400,7 @@
145 - implementation.
146 - """
147 -
148 -- OPERATOR = get_operator("xformers_flash", "flash_fwd")
149 -+ OPERATOR = _flash_fwd # get_operator("xformers_flash", "flash_fwd")
150 - SUPPORTED_DEVICES: Set[str] = {"cuda"}
151 - CUDA_MINIMUM_COMPUTE_CAPABILITY = (8, 0)
152 - SUPPORTED_DTYPES: Set[torch.dtype] = {torch.half, torch.bfloat16}
vllm/attention/backends/rocm_flash_attn.py
+18 -13
@@ -154,25 +154,30 @@ class ROCmFlashAttentionImpl(AttentionImpl):
154 f"Head size {head_size} is not supported by PagedAttention. "
155 f"Supported head sizes are: {suppored_head_sizes}.")
156
157 - self.use_naive_attn = torch.cuda.get_device_capability()[0] != 9
157 + self.use_naive_attn = False
158 # NOTE: Allow for switching between Triton and CK. Defaulting to triton.
159 self.use_triton_flash_attn = (os.environ.get(
160 "VLLM_USE_TRITON_FLASH_ATTN", "True").lower() in ("true", "1"))
161 - if self.use_naive_attn:
162 - # AMD Radeon 7900 series (gfx1100) currently does not support
163 - # xFormers nor FlashAttention. As a temporary workaround, we use
164 - # naive PyTorch implementation of attention.
165 - self.attn_fuc = _naive_attention
166 - logger.debug("Using naive attention in ROCmBackend")
167 - elif self.use_triton_flash_attn:
161 + if self.use_triton_flash_attn:
162 from vllm.attention.ops.triton_flash_attention import ( # noqa: F401
163 triton_attention)
164 self.attn_func = triton_attention
165 logger.debug("Using Triton FA in ROCmBackend")
166 else:
173 - from flash_attn import flash_attn_varlen_func # noqa: F401
174 - self.attn_func = flash_attn_varlen_func
175 - logger.debug("Using CK FA in ROCmBackend")
167 + # if not using triton, navi3x not use flash-attn either
168 + if torch.cuda.get_device_capability()[0] == 11:
169 + self.use_naive_attn = True
170 + else:
171 + try:
172 + from flash_attn import flash_attn_varlen_func # noqa: F401
173 + self.attn_func = flash_attn_varlen_func
174 + logger.debug("Using CK FA in ROCmBackend")
175 + except ModuleNotFoundError:
176 + self.use_naive_attn = True
177 +
178 + if self.use_naive_attn:
179 + self.attn_func = _naive_attention
180 + logger.debug("Using naive attention in ROCmBackend")
181
182 def repeat_kv(self, x: torch.Tensor, n_rep: int) -> torch.Tensor:
183 """torch.repeat_interleave(x, dim=1, repeats=n_rep)"""
@@ -247,13 +252,13 @@ class ROCmFlashAttentionImpl(AttentionImpl):
252 # triton attention
253 # When block_tables are not filled, it means q and k are the
254 # prompt, and they have the same length.
250 - if self.use_naive_attn or self.use_triton_flash_attn:
255 + if self.use_triton_flash_attn or self.use_naive_attn:
256 if self.num_kv_heads != self.num_heads:
257 # Interleave for MQA workaround.
258 key = self.repeat_kv(key, self.num_queries_per_kv)
259 value = self.repeat_kv(value, self.num_queries_per_kv)
260 if self.use_naive_attn:
256 - out = self.attn_fuc(
261 + out = self.attn_func(
262 query,
263 key,
264 value,