A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Bugfix][CI/Build][Hardware][AMD] Fix AMD tests, add HF cache, update CK FA, add partially supported model notes (#6543)
Matt Wong committed
Jul 20, 2024 at 11:39 UTC
06d6c5fe9fad0b798afe058c9e5e872193390456
12 files changed
+116
-39
.buildkite/run-amd-test.sh
+7
@@ -66,11 +66,18 @@ trap remove_docker_container EXIT
66
67
echo "--- Running container"
68
69
+HF_CACHE="$(realpath ~)/huggingface"
70
+mkdir -p ${HF_CACHE}
71
+HF_MOUNT="/root/.cache/huggingface"
72
+
73
docker run \
74
--device /dev/kfd --device /dev/dri \
75
--network host \
76
+ --shm-size=16gb \
77
--rm \
78
-e HF_TOKEN \
79
+ -v ${HF_CACHE}:${HF_MOUNT} \
80
+ -e HF_HOME=${HF_MOUNT} \
81
--name ${container_name} \
82
${image_name} \
83
/bin/bash -c "${@}"
.buildkite/test-pipeline.yaml
+2
-1
@@ -44,7 +44,8 @@ steps:
44
mirror_hardwares: [amd]
45
fast_check: true
46
commands:
47
- - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.0.8/flashinfer-0.0.8+cu121torch2.3-cp310-cp310-linux_x86_64.whl
47
+ # This flashinfer installation will fail on AMD ROCm, so it is set as optional.
48
+ - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.0.8/flashinfer-0.0.8+cu121torch2.3-cp310-cp310-linux_x86_64.whl || true
49
- pytest -v -s basic_correctness/test_basic_correctness.py
50
- pytest -v -s basic_correctness/test_cpu_offload.py
51
- VLLM_ATTENTION_BACKEND=XFORMERS pytest -v -s basic_correctness/test_chunked_prefill.py
CMakeLists.txt
+2
-2
@@ -33,7 +33,7 @@ set(HIP_SUPPORTED_ARCHS "gfx906;gfx908;gfx90a;gfx940;gfx941;gfx942;gfx1030;gfx11
33
# versions are derived from Dockerfile.rocm
34
#
35
set(TORCH_SUPPORTED_VERSION_CUDA "2.3.1")
36
-set(TORCH_SUPPORTED_VERSION_ROCM "2.4.0")
36
+set(TORCH_SUPPORTED_VERSION_ROCM "2.5.0")
37
38
#
39
# Try to find python package with an executable that exactly matches
@@ -101,7 +101,7 @@ elseif(HIP_FOUND)
101
# ROCm 5.X and 6.X
102
if (ROCM_VERSION_DEV_MAJOR GREATER_EQUAL 5 AND
103
NOT Torch_VERSION VERSION_EQUAL ${TORCH_SUPPORTED_VERSION_ROCM})
104
- message(WARNING "Pytorch version ${TORCH_SUPPORTED_VERSION_ROCM} "
104
+ message(WARNING "Pytorch version >= ${TORCH_SUPPORTED_VERSION_ROCM} "
105
"expected for ROCm build, saw ${Torch_VERSION} instead.")
106
endif()
107
else()
Dockerfile.rocm
+35
-25
@@ -4,18 +4,21 @@ ARG BASE_IMAGE="rocm/pytorch:rocm6.1.2_ubuntu20.04_py3.9_pytorch_staging"
4
# Default ROCm ARCHes to build vLLM for.
5
ARG PYTORCH_ROCM_ARCH="gfx908;gfx90a;gfx942;gfx1100"
6
7
-# Whether to build CK-based flash-attention
8
-# If 0, will not build flash attention
9
-# This is useful for gfx target where flash-attention is not supported
10
-# (i.e. those that do not appear in `FA_GFX_ARCHS`)
11
-# Triton FA is used by default on ROCm now so this is unnecessary.
7
+# Whether to install CK-based flash-attention
8
+# If 0, will not install flash-attention
9
ARG BUILD_FA="1"
10
+# If `TRY_FA_WHEEL=1`, we will try installing flash-attention from `FA_WHEEL_URL`
11
+# If this succeeds, we use the downloaded wheel and skip building flash-attention.
12
+# Otherwise, ROCm flash-attention from `FA_BRANCH` will be built for the
13
+# architectures specified in `FA_GFX_ARCHS`
14
+ARG TRY_FA_WHEEL="1"
15
+ARG FA_WHEEL_URL="https://github.com/ROCm/flash-attention/releases/download/v2.5.9post1-cktile-vllm/flash_attn-2.5.9.post1-cp39-cp39-linux_x86_64.whl"
16
ARG FA_GFX_ARCHS="gfx90a;gfx942"
14
-ARG FA_BRANCH="ae7928c"
17
+ARG FA_BRANCH="23a2b1c2"
18
19
# Whether to build triton on rocm
20
ARG BUILD_TRITON="1"
18
-ARG TRITON_BRANCH="0ef1848"
21
+ARG TRITON_BRANCH="e0fc12c"
22
23
### Base image build stage
24
FROM $BASE_IMAGE AS base
@@ -43,15 +46,15 @@ RUN apt-get update && apt-get install -y \
46
ARG APP_MOUNT=/vllm-workspace
47
WORKDIR ${APP_MOUNT}
48
46
-RUN pip install --upgrade pip
49
+RUN python3 -m pip install --upgrade pip
50
# Remove sccache so it doesn't interfere with ccache
51
# TODO: implement sccache support across components
49
-RUN apt-get purge -y sccache; pip uninstall -y sccache; rm -f "$(which sccache)"
52
+RUN apt-get purge -y sccache; python3 -m pip uninstall -y sccache; rm -f "$(which sccache)"
53
# Install torch == 2.5.0 on ROCm
54
RUN case "$(ls /opt | grep -Po 'rocm-[0-9]\.[0-9]')" in \
55
*"rocm-6.1"*) \
53
- pip uninstall -y torch torchaudio torchvision \
54
- && pip install --no-cache-dir --pre \
56
+ python3 -m pip uninstall -y torch torchaudio torchvision \
57
+ && python3 -m pip install --no-cache-dir --pre \
58
torch==2.5.0.dev20240710 torchaudio==2.4.0.dev20240710 \
59
torchvision==0.20.0.dev20240710 \
60
--index-url https://download.pytorch.org/whl/nightly/rocm6.1;; \
@@ -70,24 +73,31 @@ ENV CCACHE_DIR=/root/.cache/ccache
73
FROM base AS build_amdsmi
74
# Build amdsmi wheel always
75
RUN cd /opt/rocm/share/amd_smi \
73
- && pip wheel . --wheel-dir=/install
76
+ && python3 -m pip wheel . --wheel-dir=/install
77
78
79
### Flash-Attention wheel build stage
80
FROM base AS build_fa
81
ARG BUILD_FA
82
+ARG TRY_FA_WHEEL
83
+ARG FA_WHEEL_URL
84
ARG FA_GFX_ARCHS
85
ARG FA_BRANCH
86
# Build ROCm flash-attention wheel if `BUILD_FA = 1`
87
RUN --mount=type=cache,target=${CCACHE_DIR} \
88
if [ "$BUILD_FA" = "1" ]; then \
84
- mkdir -p libs \
85
- && cd libs \
86
- && git clone https://github.com/ROCm/flash-attention.git \
87
- && cd flash-attention \
88
- && git checkout "${FA_BRANCH}" \
89
- && git submodule update --init \
90
- && GPU_ARCHS="${FA_GFX_ARCHS}" python3 setup.py bdist_wheel --dist-dir=/install; \
89
+ if [ "${TRY_FA_WHEEL}" = "1" ] && python3 -m pip install "${FA_WHEEL_URL}"; then \
90
+ # If a suitable wheel exists, we download it instead of building FA
91
+ mkdir -p /install && wget -N "${FA_WHEEL_URL}" -P /install; \
92
+ else \
93
+ mkdir -p libs \
94
+ && cd libs \
95
+ && git clone https://github.com/ROCm/flash-attention.git \
96
+ && cd flash-attention \
97
+ && git checkout "${FA_BRANCH}" \
98
+ && git submodule update --init \
99
+ && GPU_ARCHS="${FA_GFX_ARCHS}" python3 setup.py bdist_wheel --dist-dir=/install; \
100
+ fi; \
101
# Create an empty directory otherwise as later build stages expect one
102
else mkdir -p /install; \
103
fi
@@ -126,7 +136,7 @@ RUN case "$(which python3)" in \
136
137
# Package upgrades for useful functionality or to avoid dependency issues
138
RUN --mount=type=cache,target=/root/.cache/pip \
129
- pip install --upgrade numba scipy huggingface-hub[cli]
139
+ python3 -m pip install --upgrade numba scipy huggingface-hub[cli]
140
141
# Make sure punica kernels are built (for LoRA)
142
ENV VLLM_INSTALL_PUNICA_KERNELS=1
@@ -137,7 +147,7 @@ ENV TOKENIZERS_PARALLELISM=false
147
148
RUN --mount=type=cache,target=${CCACHE_DIR} \
149
--mount=type=cache,target=/root/.cache/pip \
140
- pip install -U -r requirements-rocm.txt \
150
+ python3 -m pip install -Ur requirements-rocm.txt \
151
&& case "$(ls /opt | grep -Po 'rocm-[0-9]\.[0-9]')" in \
152
*"rocm-6.1"*) \
153
# Bring in upgrades to HIP graph earlier than ROCm 6.2 for vLLM
@@ -153,7 +163,7 @@ RUN --mount=type=bind,from=build_amdsmi,src=/install,target=/install \
163
mkdir -p libs \
164
&& cp /install/*.whl libs \
165
# Preemptively uninstall to avoid same-version no-installs
156
- && pip uninstall -y amdsmi;
166
+ && python3 -m pip uninstall -y amdsmi;
167
168
# Copy triton wheel(s) into final image if they were built
169
RUN --mount=type=bind,from=build_triton,src=/install,target=/install \
@@ -161,7 +171,7 @@ RUN --mount=type=bind,from=build_triton,src=/install,target=/install \
171
&& if ls /install/*.whl; then \
172
cp /install/*.whl libs \
173
# Preemptively uninstall to avoid same-version no-installs
164
- && pip uninstall -y triton; fi
174
+ && python3 -m pip uninstall -y triton; fi
175
176
# Copy flash-attn wheel(s) into final image if they were built
177
RUN --mount=type=bind,from=build_fa,src=/install,target=/install \
@@ -169,11 +179,11 @@ RUN --mount=type=bind,from=build_fa,src=/install,target=/install \
179
&& if ls /install/*.whl; then \
180
cp /install/*.whl libs \
181
# Preemptively uninstall to avoid same-version no-installs
172
- && pip uninstall -y flash-attn; fi
182
+ && python3 -m pip uninstall -y flash-attn; fi
183
184
# Install wheels that were built to the final image
185
RUN --mount=type=cache,target=/root/.cache/pip \
186
if ls libs/*.whl; then \
177
- pip install libs/*.whl; fi
187
+ python3 -m pip install libs/*.whl; fi
188
189
CMD ["/bin/bash"]
docs/source/getting_started/amd-installation.rst
+4
-3
@@ -90,12 +90,12 @@ Alternatively, you can install PyTorch using PyTorch wheels. You can check PyTor
90
91
Install ROCm's Triton flash attention (the default triton-mlir branch) following the instructions from `ROCm/triton <https://github.com/ROCm/triton/blob/triton-mlir/README.md>`_
92
93
-2. Optionally, if you choose to use CK flash attention, you can install `flash attention for ROCm <https://github.com/ROCm/flash-attention/tree/flash_attention_for_rocm>`_
93
+2. Optionally, if you choose to use CK flash attention, you can install `flash attention for ROCm <https://github.com/ROCm/flash-attention/tree/ck_tile>`_
94
95
-Install ROCm's flash attention (v2.0.4) following the instructions from `ROCm/flash-attention <https://github.com/ROCm/flash-attention/tree/flash_attention_for_rocm#amd-gpurocm-support>`_
95
+Install ROCm's flash attention (v2.5.9.post1) following the instructions from `ROCm/flash-attention <https://github.com/ROCm/flash-attention/tree/ck_tile#amd-gpurocm-support>`_
96
+Alternatively, wheels intended for vLLM use can be accessed under the releases.
97
98
.. note::
98
- - ROCm's Flash-attention-2 (v2.0.4) does not support sliding windows attention.
99
- You might need to downgrade the "ninja" version to 1.10 it is not used when compiling flash-attention-2 (e.g. `pip install ninja==1.10.2.4`)
100
101
3. Build vLLM.
@@ -110,5 +110,6 @@ Install ROCm's flash attention (v2.0.4) following the instructions from `ROCm/fl
110
.. tip::
111
112
- Triton flash attention is used by default. For benchmarking purposes, it is recommended to run a warm up step before collecting perf numbers.
113
+ - Triton flash attention does not currently support sliding window attention. If using half precision, please use CK flash-attention for sliding window support.
114
- To use CK flash-attention or PyTorch naive attention, please use this flag ``export VLLM_USE_TRITON_FLASH_ATTN=0`` to turn off triton flash attention.
115
- The ROCm version of PyTorch, ideally, should match the ROCm driver version.
requirements-rocm.txt
+4
@@ -2,5 +2,9 @@
2
-r requirements-common.txt
3
4
# Dependencies for AMD GPUs
5
+awscli
6
+boto3
7
+botocore
8
ray >= 2.10.0
9
+peft
10
pytest-asyncio
tests/basic_correctness/test_cpu_offload.py
+7
-2
@@ -1,8 +1,13 @@
1
+from vllm.utils import is_hip
2
+
3
from ..utils import compare_two_settings
4
5
6
def test_cpu_offload():
7
compare_two_settings("meta-llama/Llama-2-7b-hf", [],
8
["--cpu-offload-gb", "4"])
7
- compare_two_settings("nm-testing/llama7b-one-shot-2_4-w4a16-marlin24-t",
8
- [], ["--cpu-offload-gb", "1"])
9
+ if not is_hip():
10
+ # compressed-tensors quantization is currently not supported in ROCm.
11
+ compare_two_settings(
12
+ "nm-testing/llama7b-one-shot-2_4-w4a16-marlin24-t", [],
13
+ ["--cpu-offload-gb", "1"])
tests/models/test_paligemma.py
+17
-1
@@ -1,3 +1,4 @@
1
+import os
2
from typing import List, Optional, Tuple, Type
3
4
import pytest
@@ -5,6 +6,7 @@ from transformers import AutoTokenizer
6
7
from vllm.multimodal.utils import rescale_image_size
8
from vllm.sequence import SampleLogprobs
9
+from vllm.utils import is_hip
10
11
from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
12
from .utils import check_logprobs_close
@@ -22,6 +24,12 @@ IMAGE_TOKEN_ID = 257152
24
25
models = ["google/paligemma-3b-mix-224"]
26
27
+# ROCm Triton FA can run into compilation issues with these models due to,
28
+# excessive use of shared memory. Use other backends in the meantime.
29
+# FIXME (mattwong, gshtrasb, hongxiayan)
30
+if is_hip():
31
+ os.environ["VLLM_USE_TRITON_FLASH_ATTN"] = "0"
32
+
33
34
def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
35
Optional[SampleLogprobs]],
@@ -130,7 +138,15 @@ def run_test(
138
[0.25, 0.5, 1.0],
139
],
140
)
133
-@pytest.mark.parametrize("dtype", ["float", "half"])
141
+@pytest.mark.parametrize("dtype", [
142
+ pytest.param(
143
+ "float",
144
+ marks=pytest.mark.skipif(
145
+ is_hip(),
146
+ reason=
147
+ "ROCm FA does not yet fully support 32-bit precision on PaliGemma")
148
+ ), "half"
149
+])
150
@pytest.mark.parametrize("max_tokens", [128])
151
@pytest.mark.parametrize("num_logprobs", [5])
152
def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
tests/models/test_phi3v.py
+8
-1
@@ -1,3 +1,4 @@
1
+import os
2
import re
3
from typing import List, Optional, Tuple, Type
4
@@ -6,7 +7,7 @@ from transformers import AutoTokenizer
7
8
from vllm.multimodal.utils import rescale_image_size
9
from vllm.sequence import SampleLogprobs
9
-from vllm.utils import is_cpu
10
+from vllm.utils import is_cpu, is_hip
11
12
from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
13
from .utils import check_logprobs_close
@@ -47,6 +48,12 @@ target_dtype = "half"
48
if is_cpu():
49
target_dtype = "bfloat16"
50
51
+# ROCm Triton FA can run into shared memory issues with these models,
52
+# use other backends in the meantime
53
+# FIXME (mattwong, gshtrasb, hongxiayan)
54
+if is_hip():
55
+ os.environ["VLLM_USE_TRITON_FLASH_ATTN"] = "0"
56
+
57
58
def run_test(
59
hf_runner: Type[HfRunner],
vllm/attention/backends/rocm_flash_attn.py
+8
@@ -275,6 +275,12 @@ class ROCmFlashAttentionImpl(AttentionImpl):
275
triton_attention)
276
self.attn_func = triton_attention
277
logger.debug("Using Triton FA in ROCmBackend")
278
+ if self.sliding_window != (-1, -1):
279
+ logger.warning("ROCm Triton FA does not currently support "
280
+ "sliding window attention. If using half "
281
+ "precision, please try using the ROCm CK "
282
+ "FA backend instead by setting the env var "
283
+ "`VLLM_USE_TRITON_FLASH_ATTN=0`")
284
else:
285
# if not using triton, navi3x/navi21/navi10 do not use flash-attn
286
# either
@@ -434,6 +440,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
440
max_seqlen_k=prefill_meta.max_prefill_seq_len,
441
softmax_scale=self.scale,
442
causal=True,
443
+ window_size=self.sliding_window,
444
+ alibi_slopes=self.alibi_slopes,
445
)
446
447
# common code for prefill
vllm/model_executor/models/__init__.py
+14
-3
@@ -87,13 +87,24 @@ _ROCM_UNSUPPORTED_MODELS: List[str] = []
87
88
# Models partially supported by ROCm.
89
# Architecture -> Reason.
90
+_ROCM_SWA_REASON = ("Sliding window attention (SWA) is not yet supported in "
91
+ "Triton flash attention. For half-precision SWA support, "
92
+ "please use CK flash attention by setting "
93
+ "`VLLM_USE_TRITON_FLASH_ATTN=0`")
94
_ROCM_PARTIALLY_SUPPORTED_MODELS: Dict[str, str] = {
95
"Qwen2ForCausalLM":
92
- "Sliding window attention is not yet supported in ROCm's flash attention",
96
+ _ROCM_SWA_REASON,
97
"MistralForCausalLM":
94
- "Sliding window attention is not yet supported in ROCm's flash attention",
98
+ _ROCM_SWA_REASON,
99
"MixtralForCausalLM":
96
- "Sliding window attention is not yet supported in ROCm's flash attention",
100
+ _ROCM_SWA_REASON,
101
+ "PaliGemmaForConditionalGeneration":
102
+ ("ROCm flash attention does not yet "
103
+ "fully support 32-bit precision on PaliGemma"),
104
+ "Phi3VForCausalLM":
105
+ ("ROCm Triton flash attention may run into compilation errors due to "
106
+ "excessive use of shared memory. If this happens, disable Triton FA "
107
+ "by setting `VLLM_USE_TRITON_FLASH_ATTN=0`")
108
}
109
110
vllm/spec_decode/draft_model_runner.py
+8
-1
@@ -3,7 +3,14 @@ from typing import List, Optional
3
import torch
4
5
from vllm import _custom_ops as ops
6
-from vllm.attention.backends.flash_attn import FlashAttentionMetadata
6
+
7
+try:
8
+ from vllm.attention.backends.flash_attn import FlashAttentionMetadata
9
+except ModuleNotFoundError:
10
+ # vllm_flash_attn is not installed, use the identical ROCm FA metadata
11
+ from vllm.attention.backends.rocm_flash_attn import (
12
+ ROCmFlashAttentionMetadata as FlashAttentionMetadata)
13
+
14
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
15
ModelConfig, MultiModalConfig, ParallelConfig,
16
PromptAdapterConfig, SchedulerConfig)