A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Hardware][AMD][CI/Build][Doc] Upgrade to ROCm 6.1, Dockerfile improvements, test fixes (#5422)
Matt Wong committed
Jun 25, 2024 at 17:56 UTC
dd793d1de59b5efad25f4794b68cb935824c7a11
15 files changed
+259
-122
CMakeLists.txt
+6
-14
@@ -32,8 +32,7 @@ set(HIP_SUPPORTED_ARCHS "gfx906;gfx908;gfx90a;gfx940;gfx941;gfx942;gfx1030;gfx11
32
# versions are derived from Dockerfile.rocm
33
#
34
set(TORCH_SUPPORTED_VERSION_CUDA "2.3.0")
35
-set(TORCH_SUPPORTED_VERSION_ROCM_5X "2.0.1")
36
-set(TORCH_SUPPORTED_VERSION_ROCM_6X "2.1.1")
35
+set(TORCH_SUPPORTED_VERSION_ROCM "2.4.0")
36
37
#
38
# Try to find python package with an executable that exactly matches
@@ -98,18 +97,11 @@ elseif(HIP_FOUND)
97
# .hip extension automatically, HIP must be enabled explicitly.
98
enable_language(HIP)
99
101
- # ROCm 5.x
102
- if (ROCM_VERSION_DEV_MAJOR EQUAL 5 AND
103
- NOT Torch_VERSION VERSION_EQUAL ${TORCH_SUPPORTED_VERSION_ROCM_5X})
104
- message(WARNING "Pytorch version ${TORCH_SUPPORTED_VERSION_ROCM_5X} "
105
- "expected for ROCMm 5.x build, saw ${Torch_VERSION} instead.")
106
- endif()
107
-
108
- # ROCm 6.x
109
- if (ROCM_VERSION_DEV_MAJOR EQUAL 6 AND
110
- NOT Torch_VERSION VERSION_EQUAL ${TORCH_SUPPORTED_VERSION_ROCM_6X})
111
- message(WARNING "Pytorch version ${TORCH_SUPPORTED_VERSION_ROCM_6X} "
112
- "expected for ROCMm 6.x build, saw ${Torch_VERSION} instead.")
100
+ # ROCm 5.X and 6.X
101
+ if (ROCM_VERSION_DEV_MAJOR GREATER_EQUAL 5 AND
102
+ NOT Torch_VERSION VERSION_EQUAL ${TORCH_SUPPORTED_VERSION_ROCM})
103
+ message(WARNING "Pytorch version ${TORCH_SUPPORTED_VERSION_ROCM} "
104
+ "expected for ROCm build, saw ${Torch_VERSION} instead.")
105
endif()
106
else()
107
message(FATAL_ERROR "Can't find CUDA or HIP installation.")
Dockerfile.rocm
+145
-64
@@ -1,34 +1,35 @@
1
-# default base image
2
-ARG BASE_IMAGE="rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1"
3
-
4
-FROM $BASE_IMAGE
5
-
6
-ARG BASE_IMAGE="rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1"
7
-
8
-RUN echo "Base image is $BASE_IMAGE"
9
-
10
-ARG ROCm_5_7_BASE="rocm/pytorch:rocm5.7_ubuntu22.04_py3.10_pytorch_2.0.1" \
11
- ROCm_6_0_BASE="rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1"
12
-
1
+# Default ROCm 6.1 base image
2
+ARG BASE_IMAGE="rocm/pytorch:rocm6.1.2_ubuntu20.04_py3.9_pytorch_staging"
3
+
4
+# Tested and supported base rocm/pytorch images
5
+ARG ROCm_5_7_BASE="rocm/pytorch:rocm5.7_ubuntu20.04_py3.9_pytorch_2.0.1" \
6
+ ROCm_6_0_BASE="rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1" \
7
+ ROCM_6_1_BASE="rocm/pytorch:rocm6.1.2_ubuntu20.04_py3.9_pytorch_staging"
8
+
9
+# Default ROCm ARCHes to build vLLM for.
10
+ARG PYTORCH_ROCM_ARCH="gfx908;gfx90a;gfx942;gfx1100"
11
+
12
+# Whether to build CK-based flash-attention
13
+# If 0, will not build flash attention
14
+# This is useful for gfx target where flash-attention is not supported
15
+# (i.e. those that do not appear in `FA_GFX_ARCHS`)
16
+# Triton FA is used by default on ROCm now so this is unnecessary.
17
+ARG BUILD_FA="1"
18
ARG FA_GFX_ARCHS="gfx90a;gfx942"
14
-RUN echo "FA_GFX_ARCHS is $FA_GFX_ARCHS"
15
-
19
ARG FA_BRANCH="ae7928c"
17
-RUN echo "FA_BRANCH is $FA_BRANCH"
20
19
-# whether to build flash-attention
20
-# if 0, will not build flash attention
21
-# this is useful for gfx target where flash-attention is not supported
22
-# In that case, we need to use the python reference attention implementation in vllm
23
-ARG BUILD_FA="1"
24
-
25
-# whether to build triton on rocm
21
+# Whether to build triton on rocm
22
ARG BUILD_TRITON="1"
23
+ARG TRITON_BRANCH="0ef1848"
24
28
-# Install some basic utilities
29
-RUN apt-get update && apt-get install python3 python3-pip -y
25
+### Base image build stage
26
+FROM $BASE_IMAGE AS base
27
+
28
+# Import arg(s) defined before this build stage
29
+ARG PYTORCH_ROCM_ARCH
30
31
# Install some basic utilities
32
+RUN apt-get update && apt-get install python3 python3-pip -y
33
RUN apt-get update && apt-get install -y \
34
curl \
35
ca-certificates \
@@ -39,79 +40,159 @@ RUN apt-get update && apt-get install -y \
40
build-essential \
41
wget \
42
unzip \
42
- nvidia-cuda-toolkit \
43
tmux \
44
ccache \
45
&& rm -rf /var/lib/apt/lists/*
46
47
-### Mount Point ###
48
-# When launching the container, mount the code directory to /app
47
+# When launching the container, mount the code directory to /vllm-workspace
48
ARG APP_MOUNT=/vllm-workspace
50
-VOLUME [ ${APP_MOUNT} ]
49
WORKDIR ${APP_MOUNT}
50
53
-RUN python3 -m pip install --upgrade pip
54
-RUN python3 -m pip install --no-cache-dir fastapi ninja tokenizers pandas
51
+RUN pip install --upgrade pip
52
+# Remove sccache so it doesn't interfere with ccache
53
+# TODO: implement sccache support across components
54
+RUN apt-get purge -y sccache; pip uninstall -y sccache; rm -f "$(which sccache)"
55
+# Install torch == 2.4.0 on ROCm
56
+RUN case "$(ls /opt | grep -Po 'rocm-[0-9]\.[0-9]')" in \
57
+ *"rocm-5.7"*) \
58
+ pip uninstall -y torch \
59
+ && pip install --no-cache-dir --pre torch==2.4.0.dev20240612 \
60
+ --index-url https://download.pytorch.org/whl/nightly/rocm5.7;; \
61
+ *"rocm-6.0"*) \
62
+ pip uninstall -y torch \
63
+ && pip install --no-cache-dir --pre torch==2.4.0.dev20240612 \
64
+ --index-url https://download.pytorch.org/whl/nightly/rocm6.0;; \
65
+ *"rocm-6.1"*) \
66
+ pip uninstall -y torch \
67
+ && pip install --no-cache-dir --pre torch==2.4.0.dev20240612 \
68
+ --index-url https://download.pytorch.org/whl/nightly/rocm6.1;; \
69
+ *) ;; esac
70
71
ENV LLVM_SYMBOLIZER_PATH=/opt/rocm/llvm/bin/llvm-symbolizer
72
ENV PATH=$PATH:/opt/rocm/bin:/libtorch/bin:
73
ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/rocm/lib/:/libtorch/lib:
74
ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:/libtorch/include:/libtorch/include/torch/csrc/api/include/:/opt/rocm/include/:
75
61
-# Install ROCm flash-attention
62
-RUN if [ "$BUILD_FA" = "1" ]; then \
63
- mkdir libs \
76
+ENV PYTORCH_ROCM_ARCH=${PYTORCH_ROCM_ARCH}
77
+ENV CCACHE_DIR=/root/.cache/ccache
78
+
79
+
80
+### AMD-SMI build stage
81
+FROM base AS build_amdsmi
82
+# Build amdsmi wheel always
83
+RUN cd /opt/rocm/share/amd_smi \
84
+ && pip wheel . --wheel-dir=/install
85
+
86
+
87
+### Flash-Attention wheel build stage
88
+FROM base AS build_fa
89
+ARG BUILD_FA
90
+ARG FA_GFX_ARCHS
91
+ARG FA_BRANCH
92
+# Build ROCm flash-attention wheel if `BUILD_FA = 1`
93
+RUN --mount=type=cache,target=${CCACHE_DIR} \
94
+ if [ "$BUILD_FA" = "1" ]; then \
95
+ mkdir -p libs \
96
&& cd libs \
97
&& git clone https://github.com/ROCm/flash-attention.git \
98
&& cd flash-attention \
67
- && git checkout ${FA_BRANCH} \
99
+ && git checkout "${FA_BRANCH}" \
100
&& git submodule update --init \
69
- && export GPU_ARCHS=${FA_GFX_ARCHS} \
70
- && if [ "$BASE_IMAGE" = "$ROCm_5_7_BASE" ]; then \
71
- patch /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/utils/hipify/hipify_python.py hipify_patch.patch; fi \
72
- && python3 setup.py install \
73
- && cd ..; \
101
+ && case "$(ls /opt | grep -Po 'rocm-[0-9]\.[0-9]')" in \
102
+ *"rocm-5.7"*) \
103
+ export VLLM_TORCH_PATH="$(python3 -c 'import torch; print(torch.__path__[0])')" \
104
+ && patch "${VLLM_TORCH_PATH}"/utils/hipify/hipify_python.py hipify_patch.patch;; \
105
+ *) ;; esac \
106
+ && GPU_ARCHS="${FA_GFX_ARCHS}" python3 setup.py bdist_wheel --dist-dir=/install; \
107
+ # Create an empty directory otherwise as later build stages expect one
108
+ else mkdir -p /install; \
109
fi
110
76
-# Error related to odd state for numpy 1.20.3 where there is no METADATA etc, but an extra LICENSES_bundled.txt.
77
-# Manually removed it so that later steps of numpy upgrade can continue
78
-RUN if [ "$BASE_IMAGE" = "$ROCm_6_0_BASE" ]; then \
79
- rm -rf /opt/conda/envs/py_3.9/lib/python3.9/site-packages/numpy-1.20.3.dist-info/; fi
111
81
-# build triton
82
-RUN if [ "$BUILD_TRITON" = "1" ]; then \
112
+### Triton wheel build stage
113
+FROM base AS build_triton
114
+ARG BUILD_TRITON
115
+ARG TRITON_BRANCH
116
+# Build triton wheel if `BUILD_TRITON = 1`
117
+RUN --mount=type=cache,target=${CCACHE_DIR} \
118
+ if [ "$BUILD_TRITON" = "1" ]; then \
119
mkdir -p libs \
120
&& cd libs \
85
- && pip uninstall -y triton \
86
- && git clone https://github.com/ROCm/triton.git \
87
- && cd triton/python \
88
- && pip3 install . \
89
- && cd ../..; \
121
+ && git clone https://github.com/OpenAI/triton.git \
122
+ && cd triton \
123
+ && git checkout "${TRITON_BRANCH}" \
124
+ && cd python \
125
+ && python3 setup.py bdist_wheel --dist-dir=/install; \
126
+ # Create an empty directory otherwise as later build stages expect one
127
+ else mkdir -p /install; \
128
fi
129
92
-WORKDIR /vllm-workspace
130
+
131
+### Final vLLM build stage
132
+FROM base AS final
133
+# Import the vLLM development directory from the build context
134
COPY . .
135
95
-#RUN python3 -m pip install pynvml # to be removed eventually
96
-RUN python3 -m pip install --upgrade pip numba
136
+# Error related to odd state for numpy 1.20.3 where there is no METADATA etc, but an extra LICENSES_bundled.txt.
137
+# Manually remove it so that later steps of numpy upgrade can continue
138
+RUN case "$(which python3)" in \
139
+ *"/opt/conda/envs/py_3.9"*) \
140
+ rm -rf /opt/conda/envs/py_3.9/lib/python3.9/site-packages/numpy-1.20.3.dist-info/;; \
141
+ *) ;; esac
142
+
143
+# Package upgrades for useful functionality or to avoid dependency issues
144
+RUN --mount=type=cache,target=/root/.cache/pip \
145
+ pip install --upgrade numba scipy huggingface-hub[cli]
146
98
-# make sure punica kernels are built (for LoRA)
147
+# Make sure punica kernels are built (for LoRA)
148
ENV VLLM_INSTALL_PUNICA_KERNELS=1
149
# Workaround for ray >= 2.10.0
150
ENV RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES=1
151
+# Silences the HF Tokenizers warning
152
+ENV TOKENIZERS_PARALLELISM=false
153
103
-ENV VLLM_NCCL_SO_PATH=/opt/rocm/lib/librccl.so
104
-
105
-ENV CCACHE_DIR=/root/.cache/ccache
106
-RUN --mount=type=cache,target=/root/.cache/ccache \
154
+RUN --mount=type=cache,target=${CCACHE_DIR} \
155
--mount=type=cache,target=/root/.cache/pip \
156
pip install -U -r requirements-rocm.txt \
109
- && if [ "$BASE_IMAGE" = "$ROCm_6_0_BASE" ]; then \
110
- patch /opt/rocm/include/hip/amd_detail/amd_hip_bf16.h ./rocm_patch/rocm_bf16.patch; fi \
111
- && python3 setup.py install \
112
- && export VLLM_PYTHON_VERSION=$(python -c "import sys; print(str(sys.version_info.major) + str(sys.version_info.minor))") \
113
- && cp build/lib.linux-x86_64-cpython-${VLLM_PYTHON_VERSION}/vllm/*.so vllm/ \
114
- && cd ..
157
+ && case "$(ls /opt | grep -Po 'rocm-[0-9]\.[0-9]')" in \
158
+ *"rocm-6.0"*) \
159
+ patch /opt/rocm/include/hip/amd_detail/amd_hip_bf16.h rocm_patch/rocm_bf16.patch;; \
160
+ *"rocm-6.1"*) \
161
+ # Bring in upgrades to HIP graph earlier than ROCm 6.2 for vLLM
162
+ wget -N https://github.com/ROCm/vllm/raw/fa78403/rocm_patch/libamdhip64.so.6 -P rocm_patch \
163
+ && cp rocm_patch/libamdhip64.so.6 /opt/rocm/lib/libamdhip64.so.6 \
164
+ # Prevent interference if torch bundles its own HIP runtime
165
+ && rm -f "$(python3 -c 'import torch; print(torch.__path__[0])')"/lib/libamdhip64.so* || true;; \
166
+ *) ;; esac \
167
+ && python3 setup.py clean --all \
168
+ && python3 setup.py develop
169
+
170
+# Copy amdsmi wheel into final image
171
+RUN --mount=type=bind,from=build_amdsmi,src=/install,target=/install \
172
+ mkdir -p libs \
173
+ && cp /install/*.whl libs \
174
+ # Preemptively uninstall to avoid same-version no-installs
175
+ && pip uninstall -y amdsmi;
176
177
+# Copy triton wheel(s) into final image if they were built
178
+RUN --mount=type=bind,from=build_triton,src=/install,target=/install \
179
+ mkdir -p libs \
180
+ && if ls /install/*.whl; then \
181
+ cp /install/*.whl libs \
182
+ # Preemptively uninstall to avoid same-version no-installs
183
+ && pip uninstall -y triton; fi
184
+
185
+# Copy flash-attn wheel(s) into final image if they were built
186
+RUN --mount=type=bind,from=build_fa,src=/install,target=/install \
187
+ mkdir -p libs \
188
+ && if ls /install/*.whl; then \
189
+ cp /install/*.whl libs \
190
+ # Preemptively uninstall to avoid same-version no-installs
191
+ && pip uninstall -y flash-attn; fi
192
+
193
+# Install wheels that were built to the final image
194
+RUN --mount=type=cache,target=/root/.cache/pip \
195
+ if ls libs/*.whl; then \
196
+ pip install libs/*.whl; fi
197
198
CMD ["/bin/bash"]
cmake/utils.cmake
+12
-8
@@ -147,19 +147,23 @@ macro(override_gpu_arches GPU_ARCHES GPU_LANG GPU_SUPPORTED_ARCHES)
147
if (${GPU_LANG} STREQUAL "HIP")
148
#
149
# `GPU_ARCHES` controls the `--offload-arch` flags.
150
- # `CMAKE_HIP_ARCHITECTURES` is set up by torch and can be controlled
151
- # via the `PYTORCH_ROCM_ARCH` env variable.
150
#
153
-
151
+ # If PYTORCH_ROCM_ARCH env variable exists, then we take it as a list,
152
+ # if not, then we use CMAKE_HIP_ARCHITECTURES which was generated by calling
153
+ # "rocm_agent_enumerator" in "enable_language(HIP)"
154
+ # (in file Modules/CMakeDetermineHIPCompiler.cmake)
155
+ #
156
+ if(DEFINED ENV{PYTORCH_ROCM_ARCH})
157
+ set(HIP_ARCHITECTURES $ENV{PYTORCH_ROCM_ARCH})
158
+ else()
159
+ set(HIP_ARCHITECTURES ${CMAKE_HIP_ARCHITECTURES})
160
+ endif()
161
#
162
# Find the intersection of the supported + detected architectures to
163
# set the module architecture flags.
164
#
158
-
159
- set(VLLM_ROCM_SUPPORTED_ARCHS "gfx908;gfx90a;gfx942;gfx1100")
160
-
165
set(${GPU_ARCHES})
162
- foreach (_ARCH ${VLLM_ROCM_SUPPORTED_ARCHS})
166
+ foreach (_ARCH ${HIP_ARCHITECTURES})
167
if (_ARCH IN_LIST _GPU_SUPPORTED_ARCHES_LIST)
168
list(APPEND ${GPU_ARCHES} ${_ARCH})
169
endif()
@@ -167,7 +171,7 @@ macro(override_gpu_arches GPU_ARCHES GPU_LANG GPU_SUPPORTED_ARCHES)
171
172
if(NOT ${GPU_ARCHES})
173
message(FATAL_ERROR
170
- "None of the detected ROCm architectures: ${CMAKE_HIP_ARCHITECTURES} is"
174
+ "None of the detected ROCm architectures: ${HIP_ARCHITECTURES} is"
175
" supported. Supported ROCm architectures are: ${_GPU_SUPPORTED_ARCHES_LIST}.")
176
endif()
177
docs/source/getting_started/amd-installation.rst
+3
-3
@@ -88,7 +88,7 @@ Option 2: Build from source
88
- `Pytorch <https://pytorch.org/>`_
89
- `hipBLAS <https://rocm.docs.amd.com/projects/hipBLAS/en/latest/install.html>`_
90
91
-For installing PyTorch, you can start from a fresh docker image, e.g, `rocm6.0.2_ubuntu22.04_py3.10_pytorch_2.1.2`, `rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1`, `rocm/pytorch-nightly`.
91
+For installing PyTorch, you can start from a fresh docker image, e.g, `rocm/pytorch:rocm6.1.2_ubuntu20.04_py3.9_pytorch_staging`, `rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1`, `rocm/pytorch-nightly`.
92
93
Alternatively, you can install pytorch using pytorch wheels. You can check Pytorch installation guild in Pytorch `Getting Started <https://pytorch.org/get-started/locally/>`_
94
@@ -126,12 +126,12 @@ Install ROCm's flash attention (v2.0.4) following the instructions from `ROCm/fl
126
127
$ cd vllm
128
$ pip install -U -r requirements-rocm.txt
129
- $ python setup.py install # This may take 5-10 minutes. Currently, `pip install .`` does not work for ROCm installation
129
+ $ python setup.py develop # This may take 5-10 minutes. Currently, `pip install .`` does not work for ROCm installation
130
131
132
.. tip::
133
134
- You may need to turn on the ``--enforce-eager`` flag if you experience process hang when running the `benchmark_thoughput.py` script to test your installation.
135
- Triton flash attention is used by default. For benchmarking purposes, it is recommended to run a warm up step before collecting perf numbers.
136
- - To use CK flash-attention, please use this flag ``export VLLM_USE_FLASH_ATTN_TRITON=0`` to turn off triton flash attention.
136
+ - To use CK flash-attention, please use this flag ``export VLLM_USE_TRITON_FLASH_ATTN=0`` to turn off triton flash attention.
137
- The ROCm version of pytorch, ideally, should match the ROCm driver version.
tests/async_engine/test_openapi_server_ray.py
+2
-2
@@ -4,7 +4,7 @@ import pytest
4
# and debugging.
5
import ray
6
7
-from ..utils import VLLM_PATH, RemoteOpenAIServer
7
+from ..utils import RemoteOpenAIServer
8
9
# any model with a chat template should work here
10
MODEL_NAME = "facebook/opt-125m"
@@ -12,7 +12,7 @@ MODEL_NAME = "facebook/opt-125m"
12
13
@pytest.fixture(scope="module")
14
def ray_ctx():
15
- ray.init(runtime_env={"working_dir": VLLM_PATH})
15
+ ray.init()
16
yield
17
ray.shutdown()
18
tests/distributed/test_utils.py
+11
-6
@@ -1,8 +1,8 @@
1
-import os
2
-
1
import ray
2
5
-from vllm.utils import cuda_device_count_stateless
3
+import vllm.envs as envs
4
+from vllm.utils import (cuda_device_count_stateless, is_hip,
5
+ update_environment_variables)
6
7
8
@ray.remote
@@ -12,16 +12,21 @@ class _CUDADeviceCountStatelessTestActor:
12
return cuda_device_count_stateless()
13
14
def set_cuda_visible_devices(self, cuda_visible_devices: str):
15
- os.environ["CUDA_VISIBLE_DEVICES"] = cuda_visible_devices
15
+ update_environment_variables(
16
+ {"CUDA_VISIBLE_DEVICES": cuda_visible_devices})
17
18
def get_cuda_visible_devices(self):
18
- return os.environ["CUDA_VISIBLE_DEVICES"]
19
+ return envs.CUDA_VISIBLE_DEVICES
20
21
22
def test_cuda_device_count_stateless():
23
"""Test that cuda_device_count_stateless changes return value if
24
CUDA_VISIBLE_DEVICES is changed."""
24
-
25
+ if is_hip():
26
+ # Set HIP_VISIBLE_DEVICES == CUDA_VISIBLE_DEVICES. Conversion
27
+ # is handled by `update_environment_variables`
28
+ update_environment_variables(
29
+ {"CUDA_VISIBLE_DEVICES": envs.CUDA_VISIBLE_DEVICES})
30
actor = _CUDADeviceCountStatelessTestActor.options( # type: ignore
31
num_gpus=2).remote()
32
assert sorted(ray.get(
tests/entrypoints/test_openai_embedding.py
+2
-2
@@ -2,7 +2,7 @@ import openai
2
import pytest
3
import ray
4
5
-from ..utils import VLLM_PATH, RemoteOpenAIServer
5
+from ..utils import RemoteOpenAIServer
6
7
EMBEDDING_MODEL_NAME = "intfloat/e5-mistral-7b-instruct"
8
@@ -11,7 +11,7 @@ pytestmark = pytest.mark.openai
11
12
@pytest.fixture(scope="module")
13
def ray_ctx():
14
- ray.init(runtime_env={"working_dir": VLLM_PATH})
14
+ ray.init()
15
yield
16
ray.shutdown()
17
tests/entrypoints/test_openai_server.py
+2
-2
@@ -16,7 +16,7 @@ from openai import BadRequestError
16
17
from vllm.transformers_utils.tokenizer import get_tokenizer
18
19
-from ..utils import VLLM_PATH, RemoteOpenAIServer
19
+from ..utils import RemoteOpenAIServer
20
21
# any model with a chat template should work here
22
MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
@@ -81,7 +81,7 @@ def zephyr_lora_files():
81
82
@pytest.fixture(scope="module")
83
def ray_ctx():
84
- ray.init(runtime_env={"working_dir": VLLM_PATH})
84
+ ray.init()
85
yield
86
ray.shutdown()
87
tests/entrypoints/test_openai_vision.py
+2
-2
@@ -8,7 +8,7 @@ import ray
8
9
from vllm.multimodal.utils import ImageFetchAiohttp, encode_image_base64
10
11
-from ..utils import VLLM_PATH, RemoteOpenAIServer
11
+from ..utils import RemoteOpenAIServer
12
13
MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
14
LLAVA_CHAT_TEMPLATE = (Path(__file__).parent.parent.parent /
@@ -27,7 +27,7 @@ pytestmark = pytest.mark.openai
27
28
@pytest.fixture(scope="module")
29
def ray_ctx():
30
- ray.init(runtime_env={"working_dir": VLLM_PATH})
30
+ ray.init()
31
yield
32
ray.shutdown()
33
tests/utils.py
+32
-6
@@ -15,9 +15,30 @@ from vllm.distributed import (ensure_model_parallel_initialized,
15
from vllm.entrypoints.openai.cli_args import make_arg_parser
16
from vllm.utils import get_open_port, is_hip
17
18
-if (not is_hip()):
18
+if is_hip():
19
+ from amdsmi import (amdsmi_get_gpu_vram_usage,
20
+ amdsmi_get_processor_handles, amdsmi_init,
21
+ amdsmi_shut_down)
22
+
23
+ @contextmanager
24
+ def _nvml():
25
+ try:
26
+ amdsmi_init()
27
+ yield
28
+ finally:
29
+ amdsmi_shut_down()
30
+else:
31
from pynvml import (nvmlDeviceGetHandleByIndex, nvmlDeviceGetMemoryInfo,
20
- nvmlInit)
32
+ nvmlInit, nvmlShutdown)
33
+
34
+ @contextmanager
35
+ def _nvml():
36
+ try:
37
+ nvmlInit()
38
+ yield
39
+ finally:
40
+ nvmlShutdown()
41
+
42
43
# Path to root of repository so that utilities can be imported by ray workers
44
VLLM_PATH = os.path.abspath(os.path.join(__file__, os.pardir, os.pardir))
@@ -160,20 +181,25 @@ def error_on_warning():
181
yield
182
183
184
+@_nvml()
185
def wait_for_gpu_memory_to_clear(devices: List[int],
186
threshold_bytes: int,
187
timeout_s: float = 120) -> None:
188
# Use nvml instead of pytorch to reduce measurement error from torch cuda
189
# context.
168
- nvmlInit()
190
start_time = time.time()
191
while True:
192
output: Dict[int, str] = {}
193
output_raw: Dict[int, float] = {}
194
for device in devices:
174
- dev_handle = nvmlDeviceGetHandleByIndex(device)
175
- mem_info = nvmlDeviceGetMemoryInfo(dev_handle)
176
- gb_used = mem_info.used / 2**30
195
+ if is_hip():
196
+ dev_handle = amdsmi_get_processor_handles()[device]
197
+ mem_info = amdsmi_get_gpu_vram_usage(dev_handle)
198
+ gb_used = mem_info["vram_used"] / 2**10
199
+ else:
200
+ dev_handle = nvmlDeviceGetHandleByIndex(device)
201
+ mem_info = nvmlDeviceGetMemoryInfo(dev_handle)
202
+ gb_used = mem_info.used / 2**30
203
output_raw[device] = gb_used
204
output[device] = f'{gb_used:.02f}'
205
vllm/config.py
+9
-1
@@ -7,13 +7,15 @@ from typing import (TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Tuple,
7
import torch
8
from transformers import PretrainedConfig, PreTrainedTokenizerBase
9
10
+import vllm.envs as envs
11
from vllm.logger import init_logger
12
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
13
from vllm.model_executor.models import ModelRegistry
14
from vllm.tracing import is_otel_installed
15
from vllm.transformers_utils.config import get_config, get_hf_text_config
16
from vllm.utils import (cuda_device_count_stateless, get_cpu_memory, is_cpu,
16
- is_hip, is_neuron, is_tpu, is_xpu)
17
+ is_hip, is_neuron, is_tpu, is_xpu,
18
+ update_environment_variables)
19
20
if TYPE_CHECKING:
21
from ray.util.placement_group import PlacementGroup
@@ -634,6 +636,12 @@ class ParallelConfig:
636
self.distributed_executor_backend = backend
637
logger.info("Defaulting to use %s for distributed inference",
638
backend)
639
+ # If CUDA_VISIBLE_DEVICES is set on ROCm prior to vLLM init,
640
+ # propagate changes to HIP_VISIBLE_DEVICES (conversion handled by
641
+ # the update_environment_variables function)
642
+ if is_hip() and envs.CUDA_VISIBLE_DEVICES:
643
+ update_environment_variables(
644
+ {"CUDA_VISIBLE_DEVICES": envs.CUDA_VISIBLE_DEVICES})
645
646
self._verify_args()
647
vllm/distributed/device_communicators/custom_all_reduce_utils.py
+7
-4
@@ -13,7 +13,8 @@ import torch.multiprocessing as mp
13
import vllm.envs as envs
14
from vllm.distributed.device_communicators.cuda_wrapper import CudaRTLibrary
15
from vllm.logger import init_logger
16
-from vllm.utils import cuda_device_count_stateless
16
+from vllm.utils import (cuda_device_count_stateless,
17
+ update_environment_variables)
18
19
logger = init_logger(__name__)
20
@@ -24,7 +25,8 @@ def producer(batch_src: Sequence[int],
25
result_queue,
26
cuda_visible_devices: Optional[str] = None):
27
if cuda_visible_devices is not None:
27
- os.environ["CUDA_VISIBLE_DEVICES"] = cuda_visible_devices
28
+ update_environment_variables(
29
+ {"CUDA_VISIBLE_DEVICES": cuda_visible_devices})
30
31
lib = CudaRTLibrary()
32
for i in batch_src:
@@ -56,7 +58,8 @@ def consumer(batch_tgt: Sequence[int],
58
result_queue,
59
cuda_visible_devices: Optional[str] = None):
60
if cuda_visible_devices is not None:
59
- os.environ["CUDA_VISIBLE_DEVICES"] = cuda_visible_devices
61
+ update_environment_variables(
62
+ {"CUDA_VISIBLE_DEVICES": cuda_visible_devices})
63
64
lib = CudaRTLibrary()
65
for j in batch_tgt:
@@ -123,7 +126,7 @@ def can_actually_p2p(
126
processes for testing all pairs of GPUs in batch. The trick is to reset
127
the device after each test (which is not available in PyTorch).
128
""" # noqa
126
- cuda_visible_devices = os.getenv('CUDA_VISIBLE_DEVICES', None)
129
+ cuda_visible_devices = envs.CUDA_VISIBLE_DEVICES
130
# pass the CUDA_VISIBLE_DEVICES to the child process
131
# to make sure they see the same set of GPUs
132
vllm/executor/multiproc_gpu_executor.py
+5
-3
@@ -11,7 +11,8 @@ from vllm.logger import init_logger
11
from vllm.sequence import ExecuteModelRequest, SamplerOutput
12
from vllm.utils import (cuda_device_count_stateless,
13
get_distributed_init_method, get_open_port,
14
- get_vllm_instance_id, make_async)
14
+ get_vllm_instance_id, make_async,
15
+ update_environment_variables)
16
17
logger = init_logger(__name__)
18
@@ -25,8 +26,9 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
26
27
# Set CUDA_VISIBLE_DEVICES for the driver, inherited by workers
28
if "CUDA_VISIBLE_DEVICES" not in os.environ:
28
- os.environ["CUDA_VISIBLE_DEVICES"] = (",".join(
29
- map(str, range(world_size))))
29
+ update_environment_variables({
30
+ "CUDA_VISIBLE_DEVICES": (",".join(map(str, range(world_size))))
31
+ })
32
33
# Ensure that VLLM_INSTANCE_ID is set, to be inherited by workers
34
os.environ["VLLM_INSTANCE_ID"] = get_vllm_instance_id()
vllm/utils.py
+12
-4
@@ -376,6 +376,10 @@ def get_open_port() -> int:
376
377
378
def update_environment_variables(envs: Dict[str, str]):
379
+ if is_hip() and "CUDA_VISIBLE_DEVICES" in envs:
380
+ # Propagate changes to CUDA_VISIBLE_DEVICES to
381
+ # ROCm's HIP_VISIBLE_DEVICES as well
382
+ envs["HIP_VISIBLE_DEVICES"] = envs["CUDA_VISIBLE_DEVICES"]
383
for k, v in envs.items():
384
if k in os.environ and os.environ[k] != v:
385
logger.warning(
@@ -779,9 +783,14 @@ def _cuda_device_count_stateless(
783
784
if not torch.cuda._is_compiled():
785
return 0
782
- # bypass _device_count_nvml() if rocm (not supported)
783
- nvml_count = -1 if torch.version.hip else torch.cuda._device_count_nvml()
784
- r = torch._C._cuda_getDeviceCount() if nvml_count < 0 else nvml_count
786
+ if is_hip():
787
+ # ROCm uses amdsmi instead of nvml for stateless device count
788
+ # This requires a sufficiently modern version of Torch 2.4.0
789
+ raw_count = torch.cuda._device_count_amdsmi() if (hasattr(
790
+ torch.cuda, "_device_count_amdsmi")) else -1
791
+ else:
792
+ raw_count = torch.cuda._device_count_nvml()
793
+ r = torch._C._cuda_getDeviceCount() if raw_count < 0 else raw_count
794
return r
795
796
@@ -795,7 +804,6 @@ def cuda_device_count_stateless() -> int:
804
805
# This can be removed and simply replaced with torch.cuda.get_device_count
806
# after https://github.com/pytorch/pytorch/pull/122815 is released.
798
-
807
return _cuda_device_count_stateless(envs.CUDA_VISIBLE_DEVICES)
808
809
vllm/worker/worker_base.py
+9
-1
@@ -6,7 +6,7 @@ from typing import Dict, List, Optional, Set, Tuple
6
from vllm.logger import init_logger
7
from vllm.lora.request import LoRARequest
8
from vllm.sequence import ExecuteModelRequest, SamplerOutput
9
-from vllm.utils import (enable_trace_function_call_for_thread,
9
+from vllm.utils import (enable_trace_function_call_for_thread, is_hip,
10
update_environment_variables)
11
12
logger = init_logger(__name__)
@@ -125,6 +125,14 @@ class WorkerWrapperBase:
125
# overwriting CUDA_VISIBLE_DEVICES is desired behavior
126
# suppress the warning in `update_environment_variables`
127
del os.environ[key]
128
+ if is_hip():
129
+ hip_env_var = "HIP_VISIBLE_DEVICES"
130
+ if hip_env_var in os.environ:
131
+ logger.warning(
132
+ "Ignoring pre-set environment variable `%s=%s` as "
133
+ "%s has also been set, which takes precedence.",
134
+ hip_env_var, os.environ[hip_env_var], key)
135
+ os.environ.pop(hip_env_var, None)
136
update_environment_variables(envs)
137
138
def init_worker(self, *args, **kwargs):