@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 06d6c5fe9f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Bugfix][CI/Build][Hardware][AMD] Fix AMD tests, add HF cache, update CK FA, add partially supported model notes (#6543)

Matt Wong committed Jul 20, 2024 at 11:39 UTC 06d6c5fe9fad0b798afe058c9e5e872193390456
12 files changed +116 -39
.buildkite/run-amd-test.sh
+7
@@ -66,11 +66,18 @@ trap remove_docker_container EXIT
66
67 echo "--- Running container"
68
69 +HF_CACHE="$(realpath ~)/huggingface"
70 +mkdir -p ${HF_CACHE}
71 +HF_MOUNT="/root/.cache/huggingface"
72 +
73 docker run \
74 --device /dev/kfd --device /dev/dri \
75 --network host \
76 + --shm-size=16gb \
77 --rm \
78 -e HF_TOKEN \
79 + -v ${HF_CACHE}:${HF_MOUNT} \
80 + -e HF_HOME=${HF_MOUNT} \
81 --name ${container_name} \
82 ${image_name} \
83 /bin/bash -c "${@}"
.buildkite/test-pipeline.yaml
+2 -1
@@ -44,7 +44,8 @@ steps:
44 mirror_hardwares: [amd]
45 fast_check: true
46 commands:
47 - - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.0.8/flashinfer-0.0.8+cu121torch2.3-cp310-cp310-linux_x86_64.whl
47 + # This flashinfer installation will fail on AMD ROCm, so it is set as optional.
48 + - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.0.8/flashinfer-0.0.8+cu121torch2.3-cp310-cp310-linux_x86_64.whl || true
49 - pytest -v -s basic_correctness/test_basic_correctness.py
50 - pytest -v -s basic_correctness/test_cpu_offload.py
51 - VLLM_ATTENTION_BACKEND=XFORMERS pytest -v -s basic_correctness/test_chunked_prefill.py
CMakeLists.txt
+2 -2
@@ -33,7 +33,7 @@ set(HIP_SUPPORTED_ARCHS "gfx906;gfx908;gfx90a;gfx940;gfx941;gfx942;gfx1030;gfx11
33 # versions are derived from Dockerfile.rocm
34 #
35 set(TORCH_SUPPORTED_VERSION_CUDA "2.3.1")
36 -set(TORCH_SUPPORTED_VERSION_ROCM "2.4.0")
36 +set(TORCH_SUPPORTED_VERSION_ROCM "2.5.0")
37
38 #
39 # Try to find python package with an executable that exactly matches
@@ -101,7 +101,7 @@ elseif(HIP_FOUND)
101 # ROCm 5.X and 6.X
102 if (ROCM_VERSION_DEV_MAJOR GREATER_EQUAL 5 AND
103 NOT Torch_VERSION VERSION_EQUAL ${TORCH_SUPPORTED_VERSION_ROCM})
104 - message(WARNING "Pytorch version ${TORCH_SUPPORTED_VERSION_ROCM} "
104 + message(WARNING "Pytorch version >= ${TORCH_SUPPORTED_VERSION_ROCM} "
105 "expected for ROCm build, saw ${Torch_VERSION} instead.")
106 endif()
107 else()
Dockerfile.rocm
+35 -25
@@ -4,18 +4,21 @@ ARG BASE_IMAGE="rocm/pytorch:rocm6.1.2_ubuntu20.04_py3.9_pytorch_staging"
4 # Default ROCm ARCHes to build vLLM for.
5 ARG PYTORCH_ROCM_ARCH="gfx908;gfx90a;gfx942;gfx1100"
6
7 -# Whether to build CK-based flash-attention
8 -# If 0, will not build flash attention
9 -# This is useful for gfx target where flash-attention is not supported
10 -# (i.e. those that do not appear in `FA_GFX_ARCHS`)
11 -# Triton FA is used by default on ROCm now so this is unnecessary.
7 +# Whether to install CK-based flash-attention
8 +# If 0, will not install flash-attention
9 ARG BUILD_FA="1"
10 +# If `TRY_FA_WHEEL=1`, we will try installing flash-attention from `FA_WHEEL_URL`
11 +# If this succeeds, we use the downloaded wheel and skip building flash-attention.
12 +# Otherwise, ROCm flash-attention from `FA_BRANCH` will be built for the
13 +# architectures specified in `FA_GFX_ARCHS`
14 +ARG TRY_FA_WHEEL="1"
15 +ARG FA_WHEEL_URL="https://github.com/ROCm/flash-attention/releases/download/v2.5.9post1-cktile-vllm/flash_attn-2.5.9.post1-cp39-cp39-linux_x86_64.whl"
16 ARG FA_GFX_ARCHS="gfx90a;gfx942"
14 -ARG FA_BRANCH="ae7928c"
17 +ARG FA_BRANCH="23a2b1c2"
18
19 # Whether to build triton on rocm
20 ARG BUILD_TRITON="1"
18 -ARG TRITON_BRANCH="0ef1848"
21 +ARG TRITON_BRANCH="e0fc12c"
22
23 ### Base image build stage
24 FROM $BASE_IMAGE AS base
@@ -43,15 +46,15 @@ RUN apt-get update && apt-get install -y \
46 ARG APP_MOUNT=/vllm-workspace
47 WORKDIR ${APP_MOUNT}
48
46 -RUN pip install --upgrade pip
49 +RUN python3 -m pip install --upgrade pip
50 # Remove sccache so it doesn't interfere with ccache
51 # TODO: implement sccache support across components
49 -RUN apt-get purge -y sccache; pip uninstall -y sccache; rm -f "$(which sccache)"
52 +RUN apt-get purge -y sccache; python3 -m pip uninstall -y sccache; rm -f "$(which sccache)"
53 # Install torch == 2.5.0 on ROCm
54 RUN case "$(ls /opt | grep -Po 'rocm-[0-9]\.[0-9]')" in \
55 *"rocm-6.1"*) \
53 - pip uninstall -y torch torchaudio torchvision \
54 - && pip install --no-cache-dir --pre \
56 + python3 -m pip uninstall -y torch torchaudio torchvision \
57 + && python3 -m pip install --no-cache-dir --pre \
58 torch==2.5.0.dev20240710 torchaudio==2.4.0.dev20240710 \
59 torchvision==0.20.0.dev20240710 \
60 --index-url https://download.pytorch.org/whl/nightly/rocm6.1;; \
@@ -70,24 +73,31 @@ ENV CCACHE_DIR=/root/.cache/ccache
73 FROM base AS build_amdsmi
74 # Build amdsmi wheel always
75 RUN cd /opt/rocm/share/amd_smi \
73 - && pip wheel . --wheel-dir=/install
76 + && python3 -m pip wheel . --wheel-dir=/install
77
78
79 ### Flash-Attention wheel build stage
80 FROM base AS build_fa
81 ARG BUILD_FA
82 +ARG TRY_FA_WHEEL
83 +ARG FA_WHEEL_URL
84 ARG FA_GFX_ARCHS
85 ARG FA_BRANCH
86 # Build ROCm flash-attention wheel if `BUILD_FA = 1`
87 RUN --mount=type=cache,target=${CCACHE_DIR} \
88 if [ "$BUILD_FA" = "1" ]; then \
84 - mkdir -p libs \
85 - && cd libs \
86 - && git clone https://github.com/ROCm/flash-attention.git \
87 - && cd flash-attention \
88 - && git checkout "${FA_BRANCH}" \
89 - && git submodule update --init \
90 - && GPU_ARCHS="${FA_GFX_ARCHS}" python3 setup.py bdist_wheel --dist-dir=/install; \
89 + if [ "${TRY_FA_WHEEL}" = "1" ] && python3 -m pip install "${FA_WHEEL_URL}"; then \
90 + # If a suitable wheel exists, we download it instead of building FA
91 + mkdir -p /install && wget -N "${FA_WHEEL_URL}" -P /install; \
92 + else \
93 + mkdir -p libs \
94 + && cd libs \
95 + && git clone https://github.com/ROCm/flash-attention.git \
96 + && cd flash-attention \
97 + && git checkout "${FA_BRANCH}" \
98 + && git submodule update --init \
99 + && GPU_ARCHS="${FA_GFX_ARCHS}" python3 setup.py bdist_wheel --dist-dir=/install; \
100 + fi; \
101 # Create an empty directory otherwise as later build stages expect one
102 else mkdir -p /install; \
103 fi
@@ -126,7 +136,7 @@ RUN case "$(which python3)" in \
136
137 # Package upgrades for useful functionality or to avoid dependency issues
138 RUN --mount=type=cache,target=/root/.cache/pip \
129 - pip install --upgrade numba scipy huggingface-hub[cli]
139 + python3 -m pip install --upgrade numba scipy huggingface-hub[cli]
140
141 # Make sure punica kernels are built (for LoRA)
142 ENV VLLM_INSTALL_PUNICA_KERNELS=1
@@ -137,7 +147,7 @@ ENV TOKENIZERS_PARALLELISM=false
147
148 RUN --mount=type=cache,target=${CCACHE_DIR} \
149 --mount=type=cache,target=/root/.cache/pip \
140 - pip install -U -r requirements-rocm.txt \
150 + python3 -m pip install -Ur requirements-rocm.txt \
151 && case "$(ls /opt | grep -Po 'rocm-[0-9]\.[0-9]')" in \
152 *"rocm-6.1"*) \
153 # Bring in upgrades to HIP graph earlier than ROCm 6.2 for vLLM
@@ -153,7 +163,7 @@ RUN --mount=type=bind,from=build_amdsmi,src=/install,target=/install \
163 mkdir -p libs \
164 && cp /install/*.whl libs \
165 # Preemptively uninstall to avoid same-version no-installs
156 - && pip uninstall -y amdsmi;
166 + && python3 -m pip uninstall -y amdsmi;
167
168 # Copy triton wheel(s) into final image if they were built
169 RUN --mount=type=bind,from=build_triton,src=/install,target=/install \
@@ -161,7 +171,7 @@ RUN --mount=type=bind,from=build_triton,src=/install,target=/install \
171 && if ls /install/*.whl; then \
172 cp /install/*.whl libs \
173 # Preemptively uninstall to avoid same-version no-installs
164 - && pip uninstall -y triton; fi
174 + && python3 -m pip uninstall -y triton; fi
175
176 # Copy flash-attn wheel(s) into final image if they were built
177 RUN --mount=type=bind,from=build_fa,src=/install,target=/install \
@@ -169,11 +179,11 @@ RUN --mount=type=bind,from=build_fa,src=/install,target=/install \
179 && if ls /install/*.whl; then \
180 cp /install/*.whl libs \
181 # Preemptively uninstall to avoid same-version no-installs
172 - && pip uninstall -y flash-attn; fi
182 + && python3 -m pip uninstall -y flash-attn; fi
183
184 # Install wheels that were built to the final image
185 RUN --mount=type=cache,target=/root/.cache/pip \
186 if ls libs/*.whl; then \
177 - pip install libs/*.whl; fi
187 + python3 -m pip install libs/*.whl; fi
188
189 CMD ["/bin/bash"]
docs/source/getting_started/amd-installation.rst
+4 -3
@@ -90,12 +90,12 @@ Alternatively, you can install PyTorch using PyTorch wheels. You can check PyTor
90
91 Install ROCm's Triton flash attention (the default triton-mlir branch) following the instructions from `ROCm/triton <https://github.com/ROCm/triton/blob/triton-mlir/README.md>`_
92
93 -2. Optionally, if you choose to use CK flash attention, you can install `flash attention for ROCm <https://github.com/ROCm/flash-attention/tree/flash_attention_for_rocm>`_
93 +2. Optionally, if you choose to use CK flash attention, you can install `flash attention for ROCm <https://github.com/ROCm/flash-attention/tree/ck_tile>`_
94
95 -Install ROCm's flash attention (v2.0.4) following the instructions from `ROCm/flash-attention <https://github.com/ROCm/flash-attention/tree/flash_attention_for_rocm#amd-gpurocm-support>`_
95 +Install ROCm's flash attention (v2.5.9.post1) following the instructions from `ROCm/flash-attention <https://github.com/ROCm/flash-attention/tree/ck_tile#amd-gpurocm-support>`_
96 +Alternatively, wheels intended for vLLM use can be accessed under the releases.
97
98 .. note::
98 - - ROCm's Flash-attention-2 (v2.0.4) does not support sliding windows attention.
99 - You might need to downgrade the "ninja" version to 1.10 it is not used when compiling flash-attention-2 (e.g. `pip install ninja==1.10.2.4`)
100
101 3. Build vLLM.
@@ -110,5 +110,6 @@ Install ROCm's flash attention (v2.0.4) following the instructions from `ROCm/fl
110 .. tip::
111
112 - Triton flash attention is used by default. For benchmarking purposes, it is recommended to run a warm up step before collecting perf numbers.
113 + - Triton flash attention does not currently support sliding window attention. If using half precision, please use CK flash-attention for sliding window support.
114 - To use CK flash-attention or PyTorch naive attention, please use this flag ``export VLLM_USE_TRITON_FLASH_ATTN=0`` to turn off triton flash attention.
115 - The ROCm version of PyTorch, ideally, should match the ROCm driver version.
requirements-rocm.txt
+4
@@ -2,5 +2,9 @@
2 -r requirements-common.txt
3
4 # Dependencies for AMD GPUs
5 +awscli
6 +boto3
7 +botocore
8 ray >= 2.10.0
9 +peft
10 pytest-asyncio
tests/basic_correctness/test_cpu_offload.py
+7 -2
@@ -1,8 +1,13 @@
1 +from vllm.utils import is_hip
2 +
3 from ..utils import compare_two_settings
4
5
6 def test_cpu_offload():
7 compare_two_settings("meta-llama/Llama-2-7b-hf", [],
8 ["--cpu-offload-gb", "4"])
7 - compare_two_settings("nm-testing/llama7b-one-shot-2_4-w4a16-marlin24-t",
8 - [], ["--cpu-offload-gb", "1"])
9 + if not is_hip():
10 + # compressed-tensors quantization is currently not supported in ROCm.
11 + compare_two_settings(
12 + "nm-testing/llama7b-one-shot-2_4-w4a16-marlin24-t", [],
13 + ["--cpu-offload-gb", "1"])
tests/models/test_paligemma.py
+17 -1
@@ -1,3 +1,4 @@
1 +import os
2 from typing import List, Optional, Tuple, Type
3
4 import pytest
@@ -5,6 +6,7 @@ from transformers import AutoTokenizer
6
7 from vllm.multimodal.utils import rescale_image_size
8 from vllm.sequence import SampleLogprobs
9 +from vllm.utils import is_hip
10
11 from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
12 from .utils import check_logprobs_close
@@ -22,6 +24,12 @@ IMAGE_TOKEN_ID = 257152
24
25 models = ["google/paligemma-3b-mix-224"]
26
27 +# ROCm Triton FA can run into compilation issues with these models due to,
28 +# excessive use of shared memory. Use other backends in the meantime.
29 +# FIXME (mattwong, gshtrasb, hongxiayan)
30 +if is_hip():
31 + os.environ["VLLM_USE_TRITON_FLASH_ATTN"] = "0"
32 +
33
34 def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
35 Optional[SampleLogprobs]],
@@ -130,7 +138,15 @@ def run_test(
138 [0.25, 0.5, 1.0],
139 ],
140 )
133 -@pytest.mark.parametrize("dtype", ["float", "half"])
141 +@pytest.mark.parametrize("dtype", [
142 + pytest.param(
143 + "float",
144 + marks=pytest.mark.skipif(
145 + is_hip(),
146 + reason=
147 + "ROCm FA does not yet fully support 32-bit precision on PaliGemma")
148 + ), "half"
149 +])
150 @pytest.mark.parametrize("max_tokens", [128])
151 @pytest.mark.parametrize("num_logprobs", [5])
152 def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
tests/models/test_phi3v.py
+8 -1
@@ -1,3 +1,4 @@
1 +import os
2 import re
3 from typing import List, Optional, Tuple, Type
4
@@ -6,7 +7,7 @@ from transformers import AutoTokenizer
7
8 from vllm.multimodal.utils import rescale_image_size
9 from vllm.sequence import SampleLogprobs
9 -from vllm.utils import is_cpu
10 +from vllm.utils import is_cpu, is_hip
11
12 from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
13 from .utils import check_logprobs_close
@@ -47,6 +48,12 @@ target_dtype = "half"
48 if is_cpu():
49 target_dtype = "bfloat16"
50
51 +# ROCm Triton FA can run into shared memory issues with these models,
52 +# use other backends in the meantime
53 +# FIXME (mattwong, gshtrasb, hongxiayan)
54 +if is_hip():
55 + os.environ["VLLM_USE_TRITON_FLASH_ATTN"] = "0"
56 +
57
58 def run_test(
59 hf_runner: Type[HfRunner],
vllm/attention/backends/rocm_flash_attn.py
+8
@@ -275,6 +275,12 @@ class ROCmFlashAttentionImpl(AttentionImpl):
275 triton_attention)
276 self.attn_func = triton_attention
277 logger.debug("Using Triton FA in ROCmBackend")
278 + if self.sliding_window != (-1, -1):
279 + logger.warning("ROCm Triton FA does not currently support "
280 + "sliding window attention. If using half "
281 + "precision, please try using the ROCm CK "
282 + "FA backend instead by setting the env var "
283 + "`VLLM_USE_TRITON_FLASH_ATTN=0`")
284 else:
285 # if not using triton, navi3x/navi21/navi10 do not use flash-attn
286 # either
@@ -434,6 +440,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
440 max_seqlen_k=prefill_meta.max_prefill_seq_len,
441 softmax_scale=self.scale,
442 causal=True,
443 + window_size=self.sliding_window,
444 + alibi_slopes=self.alibi_slopes,
445 )
446
447 # common code for prefill
vllm/model_executor/models/__init__.py
+14 -3
@@ -87,13 +87,24 @@ _ROCM_UNSUPPORTED_MODELS: List[str] = []
87
88 # Models partially supported by ROCm.
89 # Architecture -> Reason.
90 +_ROCM_SWA_REASON = ("Sliding window attention (SWA) is not yet supported in "
91 + "Triton flash attention. For half-precision SWA support, "
92 + "please use CK flash attention by setting "
93 + "`VLLM_USE_TRITON_FLASH_ATTN=0`")
94 _ROCM_PARTIALLY_SUPPORTED_MODELS: Dict[str, str] = {
95 "Qwen2ForCausalLM":
92 - "Sliding window attention is not yet supported in ROCm's flash attention",
96 + _ROCM_SWA_REASON,
97 "MistralForCausalLM":
94 - "Sliding window attention is not yet supported in ROCm's flash attention",
98 + _ROCM_SWA_REASON,
99 "MixtralForCausalLM":
96 - "Sliding window attention is not yet supported in ROCm's flash attention",
100 + _ROCM_SWA_REASON,
101 + "PaliGemmaForConditionalGeneration":
102 + ("ROCm flash attention does not yet "
103 + "fully support 32-bit precision on PaliGemma"),
104 + "Phi3VForCausalLM":
105 + ("ROCm Triton flash attention may run into compilation errors due to "
106 + "excessive use of shared memory. If this happens, disable Triton FA "
107 + "by setting `VLLM_USE_TRITON_FLASH_ATTN=0`")
108 }
109
110
vllm/spec_decode/draft_model_runner.py
+8 -1
@@ -3,7 +3,14 @@ from typing import List, Optional
3 import torch
4
5 from vllm import _custom_ops as ops
6 -from vllm.attention.backends.flash_attn import FlashAttentionMetadata
6 +
7 +try:
8 + from vllm.attention.backends.flash_attn import FlashAttentionMetadata
9 +except ModuleNotFoundError:
10 + # vllm_flash_attn is not installed, use the identical ROCm FA metadata
11 + from vllm.attention.backends.rocm_flash_attn import (
12 + ROCmFlashAttentionMetadata as FlashAttentionMetadata)
13 +
14 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
15 ModelConfig, MultiModalConfig, ParallelConfig,
16 PromptAdapterConfig, SchedulerConfig)