A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[ROCm] enable cupy in order to enable cudagraph mode for AMD GPUs (#3123)
Co-authored-by: lcskrishna <lollachaitanya@gmail.com>
Hongxia Yang committed
Mar 4, 2024 at 21:14 UTC
05af6da8d927f70d15ab1ed25b01df3c967ad961
2 files changed
+26
-8
Dockerfile.rocm
+25
-5
@@ -23,6 +23,9 @@ RUN echo "FA_BRANCH is $FA_BRANCH"
23
# In that case, we need to use the python reference attention implementation in vllm
24
ARG BUILD_FA="1"
25
26
+# whether to build cupy on rocm
27
+ARG BUILD_CUPY="1"
28
+
29
# Install some basic utilities
30
RUN apt-get update && apt-get install python3 python3-pip -y
31
@@ -70,16 +73,33 @@ RUN if [ "$BUILD_FA" = "1" ]; then \
73
&& cd ..; \
74
fi
75
73
-COPY ./ /app/vllm
74
-
75
-RUN python3 -m pip install --upgrade pip
76
-RUN python3 -m pip install xformers==0.0.23 --no-deps
77
-
76
# Error related to odd state for numpy 1.20.3 where there is no METADATA etc, but an extra LICENSES_bundled.txt.
77
# Manually removed it so that later steps of numpy upgrade can continue
78
RUN if [ "$BASE_IMAGE" = "rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1" ]; then \
79
rm -rf /opt/conda/envs/py_3.9/lib/python3.9/site-packages/numpy-1.20.3.dist-info/; fi
80
81
+# build cupy
82
+RUN if [ "$BUILD_CUPY" = "1" ]; then \
83
+ mkdir -p libs \
84
+ && cd libs \
85
+ && git clone -b hipgraph_enablement --recursive https://github.com/ROCm/cupy.git \
86
+ && cd cupy \
87
+ && pip install mpi4py-mpich \
88
+ && pip install scipy==1.9.3 \
89
+ && pip install cython==0.29.* \
90
+ && env CC=$MPI_HOME/bin/mpicc python -m pip install mpi4py \
91
+ && export CUPY_INSTALL_USE_HIP=1 \
92
+ && export ROCM_HOME=/opt/rocm \
93
+ && export HCC_AMDGPU_TARGET="gfx90a,gfx942,gfx1100" \
94
+ && pip install . \
95
+ && cd ..; \
96
+ fi
97
+
98
+COPY ./ /app/vllm
99
+
100
+RUN python3 -m pip install --upgrade pip
101
+RUN python3 -m pip install xformers==0.0.23 --no-deps
102
+
103
RUN cd /app \
104
&& cd vllm \
105
&& pip install -U -r requirements-rocm.txt \
vllm/worker/worker.py
+1
-3
@@ -19,7 +19,6 @@ from vllm.sequence import SamplerOutput, SequenceGroupMetadata
19
from vllm.worker.cache_engine import CacheEngine
20
from vllm.worker.model_runner import ModelRunner
21
from vllm.lora.request import LoRARequest
22
-from vllm.utils import is_hip
22
23
24
class Worker:
@@ -267,8 +266,7 @@ def init_distributed_environment(
266
"cupy.distributed is already initialized but the cupy world "
267
"size does not match parallel_config.world_size "
268
f"({cupy_world_size} vs. {parallel_config.world_size}).")
270
- elif (parallel_config.world_size > 1 and cupy_port is not None
271
- and not is_hip()):
269
+ elif (parallel_config.world_size > 1 and cupy_port is not None):
270
# NOTE(woosuk): We don't initialize CuPy process group when world size
271
# is 1.
272
# TODO(woosuk): Support multi-node connection.