@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 05af6da8d9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[ROCm] enable cupy in order to enable cudagraph mode for AMD GPUs (#3123)

Co-authored-by: lcskrishna <lollachaitanya@gmail.com>

Hongxia Yang committed Mar 4, 2024 at 21:14 UTC 05af6da8d927f70d15ab1ed25b01df3c967ad961
2 files changed +26 -8
Dockerfile.rocm
+25 -5
@@ -23,6 +23,9 @@ RUN echo "FA_BRANCH is $FA_BRANCH"
23 # In that case, we need to use the python reference attention implementation in vllm
24 ARG BUILD_FA="1"
25
26 +# whether to build cupy on rocm
27 +ARG BUILD_CUPY="1"
28 +
29 # Install some basic utilities
30 RUN apt-get update && apt-get install python3 python3-pip -y
31
@@ -70,16 +73,33 @@ RUN if [ "$BUILD_FA" = "1" ]; then \
73 && cd ..; \
74 fi
75
73 -COPY ./ /app/vllm
74 -
75 -RUN python3 -m pip install --upgrade pip
76 -RUN python3 -m pip install xformers==0.0.23 --no-deps
77 -
76 # Error related to odd state for numpy 1.20.3 where there is no METADATA etc, but an extra LICENSES_bundled.txt.
77 # Manually removed it so that later steps of numpy upgrade can continue
78 RUN if [ "$BASE_IMAGE" = "rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1" ]; then \
79 rm -rf /opt/conda/envs/py_3.9/lib/python3.9/site-packages/numpy-1.20.3.dist-info/; fi
80
81 +# build cupy
82 +RUN if [ "$BUILD_CUPY" = "1" ]; then \
83 + mkdir -p libs \
84 + && cd libs \
85 + && git clone -b hipgraph_enablement --recursive https://github.com/ROCm/cupy.git \
86 + && cd cupy \
87 + && pip install mpi4py-mpich \
88 + && pip install scipy==1.9.3 \
89 + && pip install cython==0.29.* \
90 + && env CC=$MPI_HOME/bin/mpicc python -m pip install mpi4py \
91 + && export CUPY_INSTALL_USE_HIP=1 \
92 + && export ROCM_HOME=/opt/rocm \
93 + && export HCC_AMDGPU_TARGET="gfx90a,gfx942,gfx1100" \
94 + && pip install . \
95 + && cd ..; \
96 + fi
97 +
98 +COPY ./ /app/vllm
99 +
100 +RUN python3 -m pip install --upgrade pip
101 +RUN python3 -m pip install xformers==0.0.23 --no-deps
102 +
103 RUN cd /app \
104 && cd vllm \
105 && pip install -U -r requirements-rocm.txt \
vllm/worker/worker.py
+1 -3
@@ -19,7 +19,6 @@ from vllm.sequence import SamplerOutput, SequenceGroupMetadata
19 from vllm.worker.cache_engine import CacheEngine
20 from vllm.worker.model_runner import ModelRunner
21 from vllm.lora.request import LoRARequest
22 -from vllm.utils import is_hip
22
23
24 class Worker:
@@ -267,8 +266,7 @@ def init_distributed_environment(
266 "cupy.distributed is already initialized but the cupy world "
267 "size does not match parallel_config.world_size "
268 f"({cupy_world_size} vs. {parallel_config.world_size}).")
270 - elif (parallel_config.world_size > 1 and cupy_port is not None
271 - and not is_hip()):
269 + elif (parallel_config.world_size > 1 and cupy_port is not None):
270 # NOTE(woosuk): We don't initialize CuPy process group when world size
271 # is 1.
272 # TODO(woosuk): Support multi-node connection.