@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6b7de1a030
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[ROCm] add support to ROCm 6.0 and MI300 (#2274)

Hongxia Yang committed Jan 26, 2024 at 15:41 UTC 6b7de1a030e5bf8c32eb66a03a0fc70bb3c2da4a
8 files changed +96 -13
Dockerfile.rocm
+31 -5
@@ -1,4 +1,24 @@
1 -FROM rocm/pytorch:rocm5.7_ubuntu22.04_py3.10_pytorch_2.0.1
1 +# default base image
2 +ARG BASE_IMAGE="rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1"
3 +
4 +FROM $BASE_IMAGE
5 +
6 +ARG BASE_IMAGE="rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1"
7 +
8 +RUN echo "Base image is $BASE_IMAGE"
9 +
10 +# BASE_IMAGE for ROCm_5.7: "rocm/pytorch:rocm5.7_ubuntu22.04_py3.10_pytorch_2.0.1"
11 +# BASE_IMAGE for ROCm_6.0: "rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1"
12 +
13 +# this does not always work for all rocm versions
14 +RUN LLVM_GFX_ARCH=$(/opt/rocm/llvm/bin/amdgpu-offload-arch) && \
15 + echo "LLVM_GFX_ARCH is $LLVM_GFX_ARCH"
16 +
17 +ARG FA_GFX_ARCHS="gfx90a;gfx942"
18 +RUN echo "FA_GFX_ARCHS is $FA_GFX_ARCHS"
19 +
20 +ARG FA_BRANCH="3d2b6f5"
21 +RUN echo "FA_BRANCH is $FA_BRANCH"
22
23 # Install some basic utilities
24 RUN apt-get update && apt-get install python3 python3-pip -y
@@ -37,17 +57,23 @@ RUN mkdir libs \
57 && cd libs \
58 && git clone https://github.com/ROCmSoftwarePlatform/flash-attention.git \
59 && cd flash-attention \
40 - && git checkout 3d2b6f5 \
60 + && git checkout ${FA_BRANCH} \
61 && git submodule update --init \
42 - && export GPU_ARCHS=$(/opt/rocm/llvm/bin/amdgpu-offload-arch) \
43 - && patch /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/utils/hipify/hipify_python.py hipify_patch.patch \
62 + && export GPU_ARCHS=${FA_GFX_ARCHS} \
63 + && if [ "$BASE_IMAGE" = "rocm/pytorch:rocm5.7_ubuntu22.04_py3.10_pytorch_2.0.1" ]; then \
64 + patch /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/utils/hipify/hipify_python.py hipify_patch.patch; fi \
65 && python3 setup.py install \
66 && cd ..
67
68 COPY ./ /app/vllm
69
70 RUN python3 -m pip install --upgrade pip
50 -RUN pip install xformers==0.0.23 --no-deps
71 +RUN python3 -m pip install xformers==0.0.23 --no-deps
72 +
73 +# Error related to odd state for numpy 1.20.3 where there is no METADATA etc, but an extra LICENSES_bundled.txt.
74 +# Manually removed it so that later steps of numpy upgrade can continue
75 +RUN if [ "$BASE_IMAGE" = "rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1" ]; then \
76 + rm -rf /opt/conda/envs/py_3.9/lib/python3.9/site-packages/numpy-1.20.3.dist-info/; fi
77
78 RUN cd /app \
79 && cd vllm \
README.md
+2 -1
@@ -26,7 +26,8 @@ Please register [here](https://lu.ma/ygxbpzhl) and join us!
26 ---
27
28 *Latest News* 🔥
29 -- [2023/12] Added ROCm support to vLLM.
29 +- [2024/01] Added ROCm 6.0 support to vLLM.
30 +- [2023/12] Added ROCm 5.7 support to vLLM.
31 - [2023/10] We hosted [the first vLLM meetup](https://lu.ma/first-vllm-meetup) in SF! Please find the meetup slides [here](https://docs.google.com/presentation/d/1QL-XPFXiFpDBh86DbEegFXBXFXjix4v032GhShbKf3s/edit?usp=sharing).
32 - [2023/09] We created our [Discord server](https://discord.gg/jz7wjKhh6g)! Join us to discuss vLLM and LLM serving! We will also post the latest announcements and updates there.
33 - [2023/09] We released our [PagedAttention paper](https://arxiv.org/abs/2309.06180) on arXiv!
csrc/cuda_utils.h
+3
@@ -5,3 +5,6 @@
5 int get_device_attribute(
6 int attribute,
7 int device_id);
8 +
9 +int get_max_shared_memory_per_block_device_attribute(
10 + int device_id);
csrc/cuda_utils_kernels.cu
+18
@@ -1,5 +1,6 @@
1 #ifdef USE_ROCM
2 #include <hip/hip_runtime.h>
3 + #include <hip/hip_runtime_api.h>
4 #endif
5 int get_device_attribute(
6 int attribute,
@@ -15,3 +16,20 @@ int get_device_attribute(
16 cudaDeviceGetAttribute(&value, static_cast<cudaDeviceAttr>(attribute), device);
17 return value;
18 }
19 +
20 +
21 +int get_max_shared_memory_per_block_device_attribute(
22 + int device_id)
23 +{
24 +int attribute;
25 +// https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html
26 +// cudaDevAttrMaxSharedMemoryPerBlockOptin = 97 if not is_hip() else 74
27 +
28 +#ifdef USE_ROCM
29 + attribute = hipDeviceAttributeMaxSharedMemoryPerBlock;
30 +#else
31 + attribute = cudaDevAttrMaxSharedMemoryPerBlockOptin;
32 +#endif
33 +
34 + return get_device_attribute(attribute, device_id);
35 +}
csrc/pybind.cpp
+6
@@ -81,4 +81,10 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
81 "get_device_attribute",
82 &get_device_attribute,
83 "Gets the specified device attribute.");
84 +
85 + cuda_utils.def(
86 + "get_max_shared_memory_per_block_device_attribute",
87 + &get_max_shared_memory_per_block_device_attribute,
88 + "Gets the maximum shared memory per block device attribute.");
89 +
90 }
docs/source/getting_started/amd-installation.rst
+30 -3
@@ -11,10 +11,10 @@ Requirements
11 ------------
12
13 * OS: Linux
14 -* Python: 3.8 -- 3.11 (Verified on 3.10)
15 -* GPU: MI200s
14 +* Python: 3.8 -- 3.11
15 +* GPU: MI200s (gfx90a), MI300 (gfx942)
16 * Pytorch 2.0.1/2.1.1/2.2
17 -* ROCm 5.7
17 +* ROCm 5.7 (Verified on python 3.10) or ROCm 6.0 (Verified on python 3.9)
18
19 Installation options:
20
@@ -27,6 +27,8 @@ Installation options:
27 (Recommended) Option 1: Quick start with vLLM pre-installed in Docker Image
28 ---------------------------------------------------------------------------
29
30 +This option is for ROCm 5.7 only:
31 +
32 .. code-block:: console
33
34 $ docker pull embeddedllminfo/vllm-rocm:vllm-v0.2.4
@@ -50,6 +52,9 @@ Option 2: Build from source
52
53 You can build and install vLLM from source:
54
55 +Below instruction is for ROCm 5.7 only.
56 +At the time of this documentation update, PyTorch on ROCm 6.0 wheel is not yet available on the PyTorch website.
57 +
58 0. Install prerequisites (skip if you are already in an environment/docker with the following installed):
59
60 - `ROCm <https://rocm.docs.amd.com/en/latest/deploy/linux/index.html>`_
@@ -95,6 +100,23 @@ You can build and install vLLM from source:
100
101 Build a docker image from `Dockerfile.rocm`, and launch a docker container.
102
103 +The `Dokerfile.rocm` is designed to support both ROCm 5.7 and ROCm 6.0 and later versions. It provides flexibility to customize the build of docker image using the following arguments:
104 +
105 +* `BASE_IMAGE`: specifies the base image used when running ``docker build``, specifically the PyTorch on ROCm base image. We have tested ROCm 5.7 and ROCm 6.0. The default is `rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1`
106 +* `FX_GFX_ARCHS`: specifies the GFX architecture that is used to build flash-attention, for example, `gfx90a;gfx942` for MI200 and MI300. The default is `gfx90a;gfx942`
107 +* `FA_BRANCH`: specifies the branch used to build the flash-attention in `ROCmSoftwarePlatform's flash-attention repo <https://github.com/ROCmSoftwarePlatform/flash-attention>`_. The default is `3d2b6f5`
108 +
109 +Their values can be passed in when running ``docker build`` with ``--build-arg`` options.
110 +
111 +For example, to build docker image for vllm on ROCm 5.7, you can run:
112 +
113 +.. code-block:: console
114 +
115 + $ docker build --build-arg BASE_IMAGE="rocm/pytorch:rocm5.7_ubuntu22.04_py3.10_pytorch_2.0.1" \
116 + -f Dockerfile.rocm -t vllm-rocm .
117 +
118 +To build vllm on ROCm 6.0, you can use the default:
119 +
120 .. code-block:: console
121
122 $ docker build -f Dockerfile.rocm -t vllm-rocm .
@@ -142,3 +164,8 @@ Alternatively, if you plan to install vLLM-ROCm on a local machine or start from
164 $ cd vllm
165 $ pip install -U -r requirements-rocm.txt
166 $ python setup.py install # This may take 5-10 minutes.
167 +
168 +.. note::
169 +
170 + - You may need to turn on the ``--enforce-eager`` flag if you experience process hang when running the `benchmark_thoughput.py` script to test your installation.
171 +
setup.py
+2
@@ -51,6 +51,8 @@ if _is_hip():
51 "Cannot find ROCM_HOME. ROCm must be available to build the package."
52 )
53 NVCC_FLAGS += ["-DUSE_ROCM"]
54 + NVCC_FLAGS += [f"-U__HIP_NO_HALF_CONVERSIONS__"]
55 + NVCC_FLAGS += [f"-U__HIP_NO_HALF_OPERATORS__"]
56
57 if _is_cuda() and CUDA_HOME is None:
58 raise RuntimeError(
vllm/utils.py
+4 -4
@@ -112,10 +112,10 @@ def get_max_shared_memory_bytes(gpu: int = 0) -> int:
112 # the Neuron-X backend does not have the `cuda_utils` module.
113 from vllm._C import cuda_utils
114
115 - # https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html
116 - cudaDevAttrMaxSharedMemoryPerBlockOptin = 97 if not is_hip() else 74
117 - max_shared_mem = cuda_utils.get_device_attribute(
118 - cudaDevAttrMaxSharedMemoryPerBlockOptin, gpu)
115 + max_shared_mem = cuda_utils.get_max_shared_memory_per_block_device_attribute(
116 + gpu)
117 + # value 0 will cause MAX_SEQ_LEN become negative and test_attention.py will fail
118 + assert max_shared_mem > 0, "max_shared_mem can not be zero"
119 return int(max_shared_mem)
120
121