A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Build/CI] Enabling AMD Entrypoints Test (#4834)
Co-authored-by: Alexey Kondratiev <alexey.kondratiev@amd.com>
Alexei-V-Ivanov-AMD committed
May 20, 2024 at 13:29 UTC
943e72ca56974b4d8b5a141182e717d2abd3a819
4 files changed
+16
-6
.buildkite/test-pipeline.yaml
+2
-1
@@ -60,7 +60,8 @@ steps:
60
command: pytest -v -s engine tokenization test_sequence.py test_config.py test_logger.py
61
62
- label: Entrypoints Test
63
- #mirror_hardwares: [amd]
63
+ mirror_hardwares: [amd]
64
+
65
commands:
66
# these tests have to be separated, because each one will allocate all posible GPU memory
67
- pytest -v -s entrypoints --ignore=entrypoints/test_server_oot_registration.py
Dockerfile.rocm
+6
-2
@@ -92,19 +92,23 @@ RUN if [ "$BUILD_TRITON" = "1" ]; then \
92
WORKDIR /vllm-workspace
93
COPY . .
94
95
+#RUN python3 -m pip install pynvml # to be removed eventually
96
RUN python3 -m pip install --upgrade pip numba
97
98
# make sure punica kernels are built (for LoRA)
99
ENV VLLM_INSTALL_PUNICA_KERNELS=1
100
+# Workaround for ray >= 2.10.0
101
+ENV RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES=1
102
+
103
+ENV VLLM_NCCL_SO_PATH=/opt/rocm/lib/librccl.so
104
105
RUN --mount=type=cache,target=/root/.cache/pip \
106
pip install -U -r requirements-rocm.txt \
107
&& patch /opt/rocm/include/hip/amd_detail/amd_hip_bf16.h ./rocm_patch/rocm_bf16.patch \
108
&& python3 setup.py install \
109
&& cp build/lib.linux-x86_64-cpython-39/vllm/_C.cpython-39-x86_64-linux-gnu.so vllm/ \
110
+ && cp build/lib.linux-x86_64-cpython-39/vllm/_punica_C.cpython-39-x86_64-linux-gnu.so vllm/ \
111
&& cd ..
112
107
-RUN python3 -m pip install --upgrade pip
108
-RUN python3 -m pip install --no-cache-dir ray[all]==2.9.3
113
114
CMD ["/bin/bash"]
requirements-rocm.txt
+2
-1
@@ -2,4 +2,5 @@
2
-r requirements-common.txt
3
4
# Dependencies for AMD GPUs
5
-ray == 2.9.3
5
+ray >= 2.10.0
6
+pytest-asyncio
tests/spec_decode/e2e/conftest.py
+6
-2
@@ -6,8 +6,12 @@ from typing import Dict, List, Optional, Tuple, Union
6
import pytest
7
import ray
8
import torch
9
-from pynvml import (nvmlDeviceGetHandleByIndex, nvmlDeviceGetMemoryInfo,
10
- nvmlInit)
9
+
10
+from vllm.utils import is_hip
11
+
12
+if (not is_hip()):
13
+ from pynvml import (nvmlDeviceGetHandleByIndex, nvmlDeviceGetMemoryInfo,
14
+ nvmlInit)
15
16
from vllm import LLM
17
from vllm.engine.arg_utils import AsyncEngineArgs