@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dd793d1de5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Hardware][AMD][CI/Build][Doc] Upgrade to ROCm 6.1, Dockerfile improvements, test fixes (#5422)

Matt Wong committed Jun 25, 2024 at 17:56 UTC dd793d1de59b5efad25f4794b68cb935824c7a11
15 files changed +259 -122
CMakeLists.txt
+6 -14
@@ -32,8 +32,7 @@ set(HIP_SUPPORTED_ARCHS "gfx906;gfx908;gfx90a;gfx940;gfx941;gfx942;gfx1030;gfx11
32 # versions are derived from Dockerfile.rocm
33 #
34 set(TORCH_SUPPORTED_VERSION_CUDA "2.3.0")
35 -set(TORCH_SUPPORTED_VERSION_ROCM_5X "2.0.1")
36 -set(TORCH_SUPPORTED_VERSION_ROCM_6X "2.1.1")
35 +set(TORCH_SUPPORTED_VERSION_ROCM "2.4.0")
36
37 #
38 # Try to find python package with an executable that exactly matches
@@ -98,18 +97,11 @@ elseif(HIP_FOUND)
97 # .hip extension automatically, HIP must be enabled explicitly.
98 enable_language(HIP)
99
101 - # ROCm 5.x
102 - if (ROCM_VERSION_DEV_MAJOR EQUAL 5 AND
103 - NOT Torch_VERSION VERSION_EQUAL ${TORCH_SUPPORTED_VERSION_ROCM_5X})
104 - message(WARNING "Pytorch version ${TORCH_SUPPORTED_VERSION_ROCM_5X} "
105 - "expected for ROCMm 5.x build, saw ${Torch_VERSION} instead.")
106 - endif()
107 -
108 - # ROCm 6.x
109 - if (ROCM_VERSION_DEV_MAJOR EQUAL 6 AND
110 - NOT Torch_VERSION VERSION_EQUAL ${TORCH_SUPPORTED_VERSION_ROCM_6X})
111 - message(WARNING "Pytorch version ${TORCH_SUPPORTED_VERSION_ROCM_6X} "
112 - "expected for ROCMm 6.x build, saw ${Torch_VERSION} instead.")
100 + # ROCm 5.X and 6.X
101 + if (ROCM_VERSION_DEV_MAJOR GREATER_EQUAL 5 AND
102 + NOT Torch_VERSION VERSION_EQUAL ${TORCH_SUPPORTED_VERSION_ROCM})
103 + message(WARNING "Pytorch version ${TORCH_SUPPORTED_VERSION_ROCM} "
104 + "expected for ROCm build, saw ${Torch_VERSION} instead.")
105 endif()
106 else()
107 message(FATAL_ERROR "Can't find CUDA or HIP installation.")
Dockerfile.rocm
+145 -64
@@ -1,34 +1,35 @@
1 -# default base image
2 -ARG BASE_IMAGE="rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1"
3 -
4 -FROM $BASE_IMAGE
5 -
6 -ARG BASE_IMAGE="rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1"
7 -
8 -RUN echo "Base image is $BASE_IMAGE"
9 -
10 -ARG ROCm_5_7_BASE="rocm/pytorch:rocm5.7_ubuntu22.04_py3.10_pytorch_2.0.1" \
11 - ROCm_6_0_BASE="rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1"
12 -
1 +# Default ROCm 6.1 base image
2 +ARG BASE_IMAGE="rocm/pytorch:rocm6.1.2_ubuntu20.04_py3.9_pytorch_staging"
3 +
4 +# Tested and supported base rocm/pytorch images
5 +ARG ROCm_5_7_BASE="rocm/pytorch:rocm5.7_ubuntu20.04_py3.9_pytorch_2.0.1" \
6 + ROCm_6_0_BASE="rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1" \
7 + ROCM_6_1_BASE="rocm/pytorch:rocm6.1.2_ubuntu20.04_py3.9_pytorch_staging"
8 +
9 +# Default ROCm ARCHes to build vLLM for.
10 +ARG PYTORCH_ROCM_ARCH="gfx908;gfx90a;gfx942;gfx1100"
11 +
12 +# Whether to build CK-based flash-attention
13 +# If 0, will not build flash attention
14 +# This is useful for gfx target where flash-attention is not supported
15 +# (i.e. those that do not appear in `FA_GFX_ARCHS`)
16 +# Triton FA is used by default on ROCm now so this is unnecessary.
17 +ARG BUILD_FA="1"
18 ARG FA_GFX_ARCHS="gfx90a;gfx942"
14 -RUN echo "FA_GFX_ARCHS is $FA_GFX_ARCHS"
15 -
19 ARG FA_BRANCH="ae7928c"
17 -RUN echo "FA_BRANCH is $FA_BRANCH"
20
19 -# whether to build flash-attention
20 -# if 0, will not build flash attention
21 -# this is useful for gfx target where flash-attention is not supported
22 -# In that case, we need to use the python reference attention implementation in vllm
23 -ARG BUILD_FA="1"
24 -
25 -# whether to build triton on rocm
21 +# Whether to build triton on rocm
22 ARG BUILD_TRITON="1"
23 +ARG TRITON_BRANCH="0ef1848"
24
28 -# Install some basic utilities
29 -RUN apt-get update && apt-get install python3 python3-pip -y
25 +### Base image build stage
26 +FROM $BASE_IMAGE AS base
27 +
28 +# Import arg(s) defined before this build stage
29 +ARG PYTORCH_ROCM_ARCH
30
31 # Install some basic utilities
32 +RUN apt-get update && apt-get install python3 python3-pip -y
33 RUN apt-get update && apt-get install -y \
34 curl \
35 ca-certificates \
@@ -39,79 +40,159 @@ RUN apt-get update && apt-get install -y \
40 build-essential \
41 wget \
42 unzip \
42 - nvidia-cuda-toolkit \
43 tmux \
44 ccache \
45 && rm -rf /var/lib/apt/lists/*
46
47 -### Mount Point ###
48 -# When launching the container, mount the code directory to /app
47 +# When launching the container, mount the code directory to /vllm-workspace
48 ARG APP_MOUNT=/vllm-workspace
50 -VOLUME [ ${APP_MOUNT} ]
49 WORKDIR ${APP_MOUNT}
50
53 -RUN python3 -m pip install --upgrade pip
54 -RUN python3 -m pip install --no-cache-dir fastapi ninja tokenizers pandas
51 +RUN pip install --upgrade pip
52 +# Remove sccache so it doesn't interfere with ccache
53 +# TODO: implement sccache support across components
54 +RUN apt-get purge -y sccache; pip uninstall -y sccache; rm -f "$(which sccache)"
55 +# Install torch == 2.4.0 on ROCm
56 +RUN case "$(ls /opt | grep -Po 'rocm-[0-9]\.[0-9]')" in \
57 + *"rocm-5.7"*) \
58 + pip uninstall -y torch \
59 + && pip install --no-cache-dir --pre torch==2.4.0.dev20240612 \
60 + --index-url https://download.pytorch.org/whl/nightly/rocm5.7;; \
61 + *"rocm-6.0"*) \
62 + pip uninstall -y torch \
63 + && pip install --no-cache-dir --pre torch==2.4.0.dev20240612 \
64 + --index-url https://download.pytorch.org/whl/nightly/rocm6.0;; \
65 + *"rocm-6.1"*) \
66 + pip uninstall -y torch \
67 + && pip install --no-cache-dir --pre torch==2.4.0.dev20240612 \
68 + --index-url https://download.pytorch.org/whl/nightly/rocm6.1;; \
69 + *) ;; esac
70
71 ENV LLVM_SYMBOLIZER_PATH=/opt/rocm/llvm/bin/llvm-symbolizer
72 ENV PATH=$PATH:/opt/rocm/bin:/libtorch/bin:
73 ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/rocm/lib/:/libtorch/lib:
74 ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:/libtorch/include:/libtorch/include/torch/csrc/api/include/:/opt/rocm/include/:
75
61 -# Install ROCm flash-attention
62 -RUN if [ "$BUILD_FA" = "1" ]; then \
63 - mkdir libs \
76 +ENV PYTORCH_ROCM_ARCH=${PYTORCH_ROCM_ARCH}
77 +ENV CCACHE_DIR=/root/.cache/ccache
78 +
79 +
80 +### AMD-SMI build stage
81 +FROM base AS build_amdsmi
82 +# Build amdsmi wheel always
83 +RUN cd /opt/rocm/share/amd_smi \
84 + && pip wheel . --wheel-dir=/install
85 +
86 +
87 +### Flash-Attention wheel build stage
88 +FROM base AS build_fa
89 +ARG BUILD_FA
90 +ARG FA_GFX_ARCHS
91 +ARG FA_BRANCH
92 +# Build ROCm flash-attention wheel if `BUILD_FA = 1`
93 +RUN --mount=type=cache,target=${CCACHE_DIR} \
94 + if [ "$BUILD_FA" = "1" ]; then \
95 + mkdir -p libs \
96 && cd libs \
97 && git clone https://github.com/ROCm/flash-attention.git \
98 && cd flash-attention \
67 - && git checkout ${FA_BRANCH} \
99 + && git checkout "${FA_BRANCH}" \
100 && git submodule update --init \
69 - && export GPU_ARCHS=${FA_GFX_ARCHS} \
70 - && if [ "$BASE_IMAGE" = "$ROCm_5_7_BASE" ]; then \
71 - patch /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/utils/hipify/hipify_python.py hipify_patch.patch; fi \
72 - && python3 setup.py install \
73 - && cd ..; \
101 + && case "$(ls /opt | grep -Po 'rocm-[0-9]\.[0-9]')" in \
102 + *"rocm-5.7"*) \
103 + export VLLM_TORCH_PATH="$(python3 -c 'import torch; print(torch.__path__[0])')" \
104 + && patch "${VLLM_TORCH_PATH}"/utils/hipify/hipify_python.py hipify_patch.patch;; \
105 + *) ;; esac \
106 + && GPU_ARCHS="${FA_GFX_ARCHS}" python3 setup.py bdist_wheel --dist-dir=/install; \
107 + # Create an empty directory otherwise as later build stages expect one
108 + else mkdir -p /install; \
109 fi
110
76 -# Error related to odd state for numpy 1.20.3 where there is no METADATA etc, but an extra LICENSES_bundled.txt.
77 -# Manually removed it so that later steps of numpy upgrade can continue
78 -RUN if [ "$BASE_IMAGE" = "$ROCm_6_0_BASE" ]; then \
79 - rm -rf /opt/conda/envs/py_3.9/lib/python3.9/site-packages/numpy-1.20.3.dist-info/; fi
111
81 -# build triton
82 -RUN if [ "$BUILD_TRITON" = "1" ]; then \
112 +### Triton wheel build stage
113 +FROM base AS build_triton
114 +ARG BUILD_TRITON
115 +ARG TRITON_BRANCH
116 +# Build triton wheel if `BUILD_TRITON = 1`
117 +RUN --mount=type=cache,target=${CCACHE_DIR} \
118 + if [ "$BUILD_TRITON" = "1" ]; then \
119 mkdir -p libs \
120 && cd libs \
85 - && pip uninstall -y triton \
86 - && git clone https://github.com/ROCm/triton.git \
87 - && cd triton/python \
88 - && pip3 install . \
89 - && cd ../..; \
121 + && git clone https://github.com/OpenAI/triton.git \
122 + && cd triton \
123 + && git checkout "${TRITON_BRANCH}" \
124 + && cd python \
125 + && python3 setup.py bdist_wheel --dist-dir=/install; \
126 + # Create an empty directory otherwise as later build stages expect one
127 + else mkdir -p /install; \
128 fi
129
92 -WORKDIR /vllm-workspace
130 +
131 +### Final vLLM build stage
132 +FROM base AS final
133 +# Import the vLLM development directory from the build context
134 COPY . .
135
95 -#RUN python3 -m pip install pynvml # to be removed eventually
96 -RUN python3 -m pip install --upgrade pip numba
136 +# Error related to odd state for numpy 1.20.3 where there is no METADATA etc, but an extra LICENSES_bundled.txt.
137 +# Manually remove it so that later steps of numpy upgrade can continue
138 +RUN case "$(which python3)" in \
139 + *"/opt/conda/envs/py_3.9"*) \
140 + rm -rf /opt/conda/envs/py_3.9/lib/python3.9/site-packages/numpy-1.20.3.dist-info/;; \
141 + *) ;; esac
142 +
143 +# Package upgrades for useful functionality or to avoid dependency issues
144 +RUN --mount=type=cache,target=/root/.cache/pip \
145 + pip install --upgrade numba scipy huggingface-hub[cli]
146
98 -# make sure punica kernels are built (for LoRA)
147 +# Make sure punica kernels are built (for LoRA)
148 ENV VLLM_INSTALL_PUNICA_KERNELS=1
149 # Workaround for ray >= 2.10.0
150 ENV RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES=1
151 +# Silences the HF Tokenizers warning
152 +ENV TOKENIZERS_PARALLELISM=false
153
103 -ENV VLLM_NCCL_SO_PATH=/opt/rocm/lib/librccl.so
104 -
105 -ENV CCACHE_DIR=/root/.cache/ccache
106 -RUN --mount=type=cache,target=/root/.cache/ccache \
154 +RUN --mount=type=cache,target=${CCACHE_DIR} \
155 --mount=type=cache,target=/root/.cache/pip \
156 pip install -U -r requirements-rocm.txt \
109 - && if [ "$BASE_IMAGE" = "$ROCm_6_0_BASE" ]; then \
110 - patch /opt/rocm/include/hip/amd_detail/amd_hip_bf16.h ./rocm_patch/rocm_bf16.patch; fi \
111 - && python3 setup.py install \
112 - && export VLLM_PYTHON_VERSION=$(python -c "import sys; print(str(sys.version_info.major) + str(sys.version_info.minor))") \
113 - && cp build/lib.linux-x86_64-cpython-${VLLM_PYTHON_VERSION}/vllm/*.so vllm/ \
114 - && cd ..
157 + && case "$(ls /opt | grep -Po 'rocm-[0-9]\.[0-9]')" in \
158 + *"rocm-6.0"*) \
159 + patch /opt/rocm/include/hip/amd_detail/amd_hip_bf16.h rocm_patch/rocm_bf16.patch;; \
160 + *"rocm-6.1"*) \
161 + # Bring in upgrades to HIP graph earlier than ROCm 6.2 for vLLM
162 + wget -N https://github.com/ROCm/vllm/raw/fa78403/rocm_patch/libamdhip64.so.6 -P rocm_patch \
163 + && cp rocm_patch/libamdhip64.so.6 /opt/rocm/lib/libamdhip64.so.6 \
164 + # Prevent interference if torch bundles its own HIP runtime
165 + && rm -f "$(python3 -c 'import torch; print(torch.__path__[0])')"/lib/libamdhip64.so* || true;; \
166 + *) ;; esac \
167 + && python3 setup.py clean --all \
168 + && python3 setup.py develop
169 +
170 +# Copy amdsmi wheel into final image
171 +RUN --mount=type=bind,from=build_amdsmi,src=/install,target=/install \
172 + mkdir -p libs \
173 + && cp /install/*.whl libs \
174 + # Preemptively uninstall to avoid same-version no-installs
175 + && pip uninstall -y amdsmi;
176
177 +# Copy triton wheel(s) into final image if they were built
178 +RUN --mount=type=bind,from=build_triton,src=/install,target=/install \
179 + mkdir -p libs \
180 + && if ls /install/*.whl; then \
181 + cp /install/*.whl libs \
182 + # Preemptively uninstall to avoid same-version no-installs
183 + && pip uninstall -y triton; fi
184 +
185 +# Copy flash-attn wheel(s) into final image if they were built
186 +RUN --mount=type=bind,from=build_fa,src=/install,target=/install \
187 + mkdir -p libs \
188 + && if ls /install/*.whl; then \
189 + cp /install/*.whl libs \
190 + # Preemptively uninstall to avoid same-version no-installs
191 + && pip uninstall -y flash-attn; fi
192 +
193 +# Install wheels that were built to the final image
194 +RUN --mount=type=cache,target=/root/.cache/pip \
195 + if ls libs/*.whl; then \
196 + pip install libs/*.whl; fi
197
198 CMD ["/bin/bash"]
cmake/utils.cmake
+12 -8
@@ -147,19 +147,23 @@ macro(override_gpu_arches GPU_ARCHES GPU_LANG GPU_SUPPORTED_ARCHES)
147 if (${GPU_LANG} STREQUAL "HIP")
148 #
149 # `GPU_ARCHES` controls the `--offload-arch` flags.
150 - # `CMAKE_HIP_ARCHITECTURES` is set up by torch and can be controlled
151 - # via the `PYTORCH_ROCM_ARCH` env variable.
150 #
153 -
151 + # If PYTORCH_ROCM_ARCH env variable exists, then we take it as a list,
152 + # if not, then we use CMAKE_HIP_ARCHITECTURES which was generated by calling
153 + # "rocm_agent_enumerator" in "enable_language(HIP)"
154 + # (in file Modules/CMakeDetermineHIPCompiler.cmake)
155 + #
156 + if(DEFINED ENV{PYTORCH_ROCM_ARCH})
157 + set(HIP_ARCHITECTURES $ENV{PYTORCH_ROCM_ARCH})
158 + else()
159 + set(HIP_ARCHITECTURES ${CMAKE_HIP_ARCHITECTURES})
160 + endif()
161 #
162 # Find the intersection of the supported + detected architectures to
163 # set the module architecture flags.
164 #
158 -
159 - set(VLLM_ROCM_SUPPORTED_ARCHS "gfx908;gfx90a;gfx942;gfx1100")
160 -
165 set(${GPU_ARCHES})
162 - foreach (_ARCH ${VLLM_ROCM_SUPPORTED_ARCHS})
166 + foreach (_ARCH ${HIP_ARCHITECTURES})
167 if (_ARCH IN_LIST _GPU_SUPPORTED_ARCHES_LIST)
168 list(APPEND ${GPU_ARCHES} ${_ARCH})
169 endif()
@@ -167,7 +171,7 @@ macro(override_gpu_arches GPU_ARCHES GPU_LANG GPU_SUPPORTED_ARCHES)
171
172 if(NOT ${GPU_ARCHES})
173 message(FATAL_ERROR
170 - "None of the detected ROCm architectures: ${CMAKE_HIP_ARCHITECTURES} is"
174 + "None of the detected ROCm architectures: ${HIP_ARCHITECTURES} is"
175 " supported. Supported ROCm architectures are: ${_GPU_SUPPORTED_ARCHES_LIST}.")
176 endif()
177
docs/source/getting_started/amd-installation.rst
+3 -3
@@ -88,7 +88,7 @@ Option 2: Build from source
88 - `Pytorch <https://pytorch.org/>`_
89 - `hipBLAS <https://rocm.docs.amd.com/projects/hipBLAS/en/latest/install.html>`_
90
91 -For installing PyTorch, you can start from a fresh docker image, e.g, `rocm6.0.2_ubuntu22.04_py3.10_pytorch_2.1.2`, `rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1`, `rocm/pytorch-nightly`.
91 +For installing PyTorch, you can start from a fresh docker image, e.g, `rocm/pytorch:rocm6.1.2_ubuntu20.04_py3.9_pytorch_staging`, `rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1`, `rocm/pytorch-nightly`.
92
93 Alternatively, you can install pytorch using pytorch wheels. You can check Pytorch installation guild in Pytorch `Getting Started <https://pytorch.org/get-started/locally/>`_
94
@@ -126,12 +126,12 @@ Install ROCm's flash attention (v2.0.4) following the instructions from `ROCm/fl
126
127 $ cd vllm
128 $ pip install -U -r requirements-rocm.txt
129 - $ python setup.py install # This may take 5-10 minutes. Currently, `pip install .`` does not work for ROCm installation
129 + $ python setup.py develop # This may take 5-10 minutes. Currently, `pip install .`` does not work for ROCm installation
130
131
132 .. tip::
133
134 - You may need to turn on the ``--enforce-eager`` flag if you experience process hang when running the `benchmark_thoughput.py` script to test your installation.
135 - Triton flash attention is used by default. For benchmarking purposes, it is recommended to run a warm up step before collecting perf numbers.
136 - - To use CK flash-attention, please use this flag ``export VLLM_USE_FLASH_ATTN_TRITON=0`` to turn off triton flash attention.
136 + - To use CK flash-attention, please use this flag ``export VLLM_USE_TRITON_FLASH_ATTN=0`` to turn off triton flash attention.
137 - The ROCm version of pytorch, ideally, should match the ROCm driver version.
tests/async_engine/test_openapi_server_ray.py
+2 -2
@@ -4,7 +4,7 @@ import pytest
4 # and debugging.
5 import ray
6
7 -from ..utils import VLLM_PATH, RemoteOpenAIServer
7 +from ..utils import RemoteOpenAIServer
8
9 # any model with a chat template should work here
10 MODEL_NAME = "facebook/opt-125m"
@@ -12,7 +12,7 @@ MODEL_NAME = "facebook/opt-125m"
12
13 @pytest.fixture(scope="module")
14 def ray_ctx():
15 - ray.init(runtime_env={"working_dir": VLLM_PATH})
15 + ray.init()
16 yield
17 ray.shutdown()
18
tests/distributed/test_utils.py
+11 -6
@@ -1,8 +1,8 @@
1 -import os
2 -
1 import ray
2
5 -from vllm.utils import cuda_device_count_stateless
3 +import vllm.envs as envs
4 +from vllm.utils import (cuda_device_count_stateless, is_hip,
5 + update_environment_variables)
6
7
8 @ray.remote
@@ -12,16 +12,21 @@ class _CUDADeviceCountStatelessTestActor:
12 return cuda_device_count_stateless()
13
14 def set_cuda_visible_devices(self, cuda_visible_devices: str):
15 - os.environ["CUDA_VISIBLE_DEVICES"] = cuda_visible_devices
15 + update_environment_variables(
16 + {"CUDA_VISIBLE_DEVICES": cuda_visible_devices})
17
18 def get_cuda_visible_devices(self):
18 - return os.environ["CUDA_VISIBLE_DEVICES"]
19 + return envs.CUDA_VISIBLE_DEVICES
20
21
22 def test_cuda_device_count_stateless():
23 """Test that cuda_device_count_stateless changes return value if
24 CUDA_VISIBLE_DEVICES is changed."""
24 -
25 + if is_hip():
26 + # Set HIP_VISIBLE_DEVICES == CUDA_VISIBLE_DEVICES. Conversion
27 + # is handled by `update_environment_variables`
28 + update_environment_variables(
29 + {"CUDA_VISIBLE_DEVICES": envs.CUDA_VISIBLE_DEVICES})
30 actor = _CUDADeviceCountStatelessTestActor.options( # type: ignore
31 num_gpus=2).remote()
32 assert sorted(ray.get(
tests/entrypoints/test_openai_embedding.py
+2 -2
@@ -2,7 +2,7 @@ import openai
2 import pytest
3 import ray
4
5 -from ..utils import VLLM_PATH, RemoteOpenAIServer
5 +from ..utils import RemoteOpenAIServer
6
7 EMBEDDING_MODEL_NAME = "intfloat/e5-mistral-7b-instruct"
8
@@ -11,7 +11,7 @@ pytestmark = pytest.mark.openai
11
12 @pytest.fixture(scope="module")
13 def ray_ctx():
14 - ray.init(runtime_env={"working_dir": VLLM_PATH})
14 + ray.init()
15 yield
16 ray.shutdown()
17
tests/entrypoints/test_openai_server.py
+2 -2
@@ -16,7 +16,7 @@ from openai import BadRequestError
16
17 from vllm.transformers_utils.tokenizer import get_tokenizer
18
19 -from ..utils import VLLM_PATH, RemoteOpenAIServer
19 +from ..utils import RemoteOpenAIServer
20
21 # any model with a chat template should work here
22 MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
@@ -81,7 +81,7 @@ def zephyr_lora_files():
81
82 @pytest.fixture(scope="module")
83 def ray_ctx():
84 - ray.init(runtime_env={"working_dir": VLLM_PATH})
84 + ray.init()
85 yield
86 ray.shutdown()
87
tests/entrypoints/test_openai_vision.py
+2 -2
@@ -8,7 +8,7 @@ import ray
8
9 from vllm.multimodal.utils import ImageFetchAiohttp, encode_image_base64
10
11 -from ..utils import VLLM_PATH, RemoteOpenAIServer
11 +from ..utils import RemoteOpenAIServer
12
13 MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
14 LLAVA_CHAT_TEMPLATE = (Path(__file__).parent.parent.parent /
@@ -27,7 +27,7 @@ pytestmark = pytest.mark.openai
27
28 @pytest.fixture(scope="module")
29 def ray_ctx():
30 - ray.init(runtime_env={"working_dir": VLLM_PATH})
30 + ray.init()
31 yield
32 ray.shutdown()
33
tests/utils.py
+32 -6
@@ -15,9 +15,30 @@ from vllm.distributed import (ensure_model_parallel_initialized,
15 from vllm.entrypoints.openai.cli_args import make_arg_parser
16 from vllm.utils import get_open_port, is_hip
17
18 -if (not is_hip()):
18 +if is_hip():
19 + from amdsmi import (amdsmi_get_gpu_vram_usage,
20 + amdsmi_get_processor_handles, amdsmi_init,
21 + amdsmi_shut_down)
22 +
23 + @contextmanager
24 + def _nvml():
25 + try:
26 + amdsmi_init()
27 + yield
28 + finally:
29 + amdsmi_shut_down()
30 +else:
31 from pynvml import (nvmlDeviceGetHandleByIndex, nvmlDeviceGetMemoryInfo,
20 - nvmlInit)
32 + nvmlInit, nvmlShutdown)
33 +
34 + @contextmanager
35 + def _nvml():
36 + try:
37 + nvmlInit()
38 + yield
39 + finally:
40 + nvmlShutdown()
41 +
42
43 # Path to root of repository so that utilities can be imported by ray workers
44 VLLM_PATH = os.path.abspath(os.path.join(__file__, os.pardir, os.pardir))
@@ -160,20 +181,25 @@ def error_on_warning():
181 yield
182
183
184 +@_nvml()
185 def wait_for_gpu_memory_to_clear(devices: List[int],
186 threshold_bytes: int,
187 timeout_s: float = 120) -> None:
188 # Use nvml instead of pytorch to reduce measurement error from torch cuda
189 # context.
168 - nvmlInit()
190 start_time = time.time()
191 while True:
192 output: Dict[int, str] = {}
193 output_raw: Dict[int, float] = {}
194 for device in devices:
174 - dev_handle = nvmlDeviceGetHandleByIndex(device)
175 - mem_info = nvmlDeviceGetMemoryInfo(dev_handle)
176 - gb_used = mem_info.used / 2**30
195 + if is_hip():
196 + dev_handle = amdsmi_get_processor_handles()[device]
197 + mem_info = amdsmi_get_gpu_vram_usage(dev_handle)
198 + gb_used = mem_info["vram_used"] / 2**10
199 + else:
200 + dev_handle = nvmlDeviceGetHandleByIndex(device)
201 + mem_info = nvmlDeviceGetMemoryInfo(dev_handle)
202 + gb_used = mem_info.used / 2**30
203 output_raw[device] = gb_used
204 output[device] = f'{gb_used:.02f}'
205
vllm/config.py
+9 -1
@@ -7,13 +7,15 @@ from typing import (TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Tuple,
7 import torch
8 from transformers import PretrainedConfig, PreTrainedTokenizerBase
9
10 +import vllm.envs as envs
11 from vllm.logger import init_logger
12 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
13 from vllm.model_executor.models import ModelRegistry
14 from vllm.tracing import is_otel_installed
15 from vllm.transformers_utils.config import get_config, get_hf_text_config
16 from vllm.utils import (cuda_device_count_stateless, get_cpu_memory, is_cpu,
16 - is_hip, is_neuron, is_tpu, is_xpu)
17 + is_hip, is_neuron, is_tpu, is_xpu,
18 + update_environment_variables)
19
20 if TYPE_CHECKING:
21 from ray.util.placement_group import PlacementGroup
@@ -634,6 +636,12 @@ class ParallelConfig:
636 self.distributed_executor_backend = backend
637 logger.info("Defaulting to use %s for distributed inference",
638 backend)
639 + # If CUDA_VISIBLE_DEVICES is set on ROCm prior to vLLM init,
640 + # propagate changes to HIP_VISIBLE_DEVICES (conversion handled by
641 + # the update_environment_variables function)
642 + if is_hip() and envs.CUDA_VISIBLE_DEVICES:
643 + update_environment_variables(
644 + {"CUDA_VISIBLE_DEVICES": envs.CUDA_VISIBLE_DEVICES})
645
646 self._verify_args()
647
vllm/distributed/device_communicators/custom_all_reduce_utils.py
+7 -4
@@ -13,7 +13,8 @@ import torch.multiprocessing as mp
13 import vllm.envs as envs
14 from vllm.distributed.device_communicators.cuda_wrapper import CudaRTLibrary
15 from vllm.logger import init_logger
16 -from vllm.utils import cuda_device_count_stateless
16 +from vllm.utils import (cuda_device_count_stateless,
17 + update_environment_variables)
18
19 logger = init_logger(__name__)
20
@@ -24,7 +25,8 @@ def producer(batch_src: Sequence[int],
25 result_queue,
26 cuda_visible_devices: Optional[str] = None):
27 if cuda_visible_devices is not None:
27 - os.environ["CUDA_VISIBLE_DEVICES"] = cuda_visible_devices
28 + update_environment_variables(
29 + {"CUDA_VISIBLE_DEVICES": cuda_visible_devices})
30
31 lib = CudaRTLibrary()
32 for i in batch_src:
@@ -56,7 +58,8 @@ def consumer(batch_tgt: Sequence[int],
58 result_queue,
59 cuda_visible_devices: Optional[str] = None):
60 if cuda_visible_devices is not None:
59 - os.environ["CUDA_VISIBLE_DEVICES"] = cuda_visible_devices
61 + update_environment_variables(
62 + {"CUDA_VISIBLE_DEVICES": cuda_visible_devices})
63
64 lib = CudaRTLibrary()
65 for j in batch_tgt:
@@ -123,7 +126,7 @@ def can_actually_p2p(
126 processes for testing all pairs of GPUs in batch. The trick is to reset
127 the device after each test (which is not available in PyTorch).
128 """ # noqa
126 - cuda_visible_devices = os.getenv('CUDA_VISIBLE_DEVICES', None)
129 + cuda_visible_devices = envs.CUDA_VISIBLE_DEVICES
130 # pass the CUDA_VISIBLE_DEVICES to the child process
131 # to make sure they see the same set of GPUs
132
vllm/executor/multiproc_gpu_executor.py
+5 -3
@@ -11,7 +11,8 @@ from vllm.logger import init_logger
11 from vllm.sequence import ExecuteModelRequest, SamplerOutput
12 from vllm.utils import (cuda_device_count_stateless,
13 get_distributed_init_method, get_open_port,
14 - get_vllm_instance_id, make_async)
14 + get_vllm_instance_id, make_async,
15 + update_environment_variables)
16
17 logger = init_logger(__name__)
18
@@ -25,8 +26,9 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
26
27 # Set CUDA_VISIBLE_DEVICES for the driver, inherited by workers
28 if "CUDA_VISIBLE_DEVICES" not in os.environ:
28 - os.environ["CUDA_VISIBLE_DEVICES"] = (",".join(
29 - map(str, range(world_size))))
29 + update_environment_variables({
30 + "CUDA_VISIBLE_DEVICES": (",".join(map(str, range(world_size))))
31 + })
32
33 # Ensure that VLLM_INSTANCE_ID is set, to be inherited by workers
34 os.environ["VLLM_INSTANCE_ID"] = get_vllm_instance_id()
vllm/utils.py
+12 -4
@@ -376,6 +376,10 @@ def get_open_port() -> int:
376
377
378 def update_environment_variables(envs: Dict[str, str]):
379 + if is_hip() and "CUDA_VISIBLE_DEVICES" in envs:
380 + # Propagate changes to CUDA_VISIBLE_DEVICES to
381 + # ROCm's HIP_VISIBLE_DEVICES as well
382 + envs["HIP_VISIBLE_DEVICES"] = envs["CUDA_VISIBLE_DEVICES"]
383 for k, v in envs.items():
384 if k in os.environ and os.environ[k] != v:
385 logger.warning(
@@ -779,9 +783,14 @@ def _cuda_device_count_stateless(
783
784 if not torch.cuda._is_compiled():
785 return 0
782 - # bypass _device_count_nvml() if rocm (not supported)
783 - nvml_count = -1 if torch.version.hip else torch.cuda._device_count_nvml()
784 - r = torch._C._cuda_getDeviceCount() if nvml_count < 0 else nvml_count
786 + if is_hip():
787 + # ROCm uses amdsmi instead of nvml for stateless device count
788 + # This requires a sufficiently modern version of Torch 2.4.0
789 + raw_count = torch.cuda._device_count_amdsmi() if (hasattr(
790 + torch.cuda, "_device_count_amdsmi")) else -1
791 + else:
792 + raw_count = torch.cuda._device_count_nvml()
793 + r = torch._C._cuda_getDeviceCount() if raw_count < 0 else raw_count
794 return r
795
796
@@ -795,7 +804,6 @@ def cuda_device_count_stateless() -> int:
804
805 # This can be removed and simply replaced with torch.cuda.get_device_count
806 # after https://github.com/pytorch/pytorch/pull/122815 is released.
798 -
807 return _cuda_device_count_stateless(envs.CUDA_VISIBLE_DEVICES)
808
809
vllm/worker/worker_base.py
+9 -1
@@ -6,7 +6,7 @@ from typing import Dict, List, Optional, Set, Tuple
6 from vllm.logger import init_logger
7 from vllm.lora.request import LoRARequest
8 from vllm.sequence import ExecuteModelRequest, SamplerOutput
9 -from vllm.utils import (enable_trace_function_call_for_thread,
9 +from vllm.utils import (enable_trace_function_call_for_thread, is_hip,
10 update_environment_variables)
11
12 logger = init_logger(__name__)
@@ -125,6 +125,14 @@ class WorkerWrapperBase:
125 # overwriting CUDA_VISIBLE_DEVICES is desired behavior
126 # suppress the warning in `update_environment_variables`
127 del os.environ[key]
128 + if is_hip():
129 + hip_env_var = "HIP_VISIBLE_DEVICES"
130 + if hip_env_var in os.environ:
131 + logger.warning(
132 + "Ignoring pre-set environment variable `%s=%s` as "
133 + "%s has also been set, which takes precedence.",
134 + hip_env_var, os.environ[hip_env_var], key)
135 + os.environ.pop(hip_env_var, None)
136 update_environment_variables(envs)
137
138 def init_worker(self, *args, **kwargs):