@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 943e72ca56
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Build/CI] Enabling AMD Entrypoints Test (#4834)

Co-authored-by: Alexey Kondratiev <alexey.kondratiev@amd.com>

Alexei-V-Ivanov-AMD committed May 20, 2024 at 13:29 UTC 943e72ca56974b4d8b5a141182e717d2abd3a819
4 files changed +16 -6
.buildkite/test-pipeline.yaml
+2 -1
@@ -60,7 +60,8 @@ steps:
60 command: pytest -v -s engine tokenization test_sequence.py test_config.py test_logger.py
61
62 - label: Entrypoints Test
63 - #mirror_hardwares: [amd]
63 + mirror_hardwares: [amd]
64 +
65 commands:
66 # these tests have to be separated, because each one will allocate all posible GPU memory
67 - pytest -v -s entrypoints --ignore=entrypoints/test_server_oot_registration.py
Dockerfile.rocm
+6 -2
@@ -92,19 +92,23 @@ RUN if [ "$BUILD_TRITON" = "1" ]; then \
92 WORKDIR /vllm-workspace
93 COPY . .
94
95 +#RUN python3 -m pip install pynvml # to be removed eventually
96 RUN python3 -m pip install --upgrade pip numba
97
98 # make sure punica kernels are built (for LoRA)
99 ENV VLLM_INSTALL_PUNICA_KERNELS=1
100 +# Workaround for ray >= 2.10.0
101 +ENV RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES=1
102 +
103 +ENV VLLM_NCCL_SO_PATH=/opt/rocm/lib/librccl.so
104
105 RUN --mount=type=cache,target=/root/.cache/pip \
106 pip install -U -r requirements-rocm.txt \
107 && patch /opt/rocm/include/hip/amd_detail/amd_hip_bf16.h ./rocm_patch/rocm_bf16.patch \
108 && python3 setup.py install \
109 && cp build/lib.linux-x86_64-cpython-39/vllm/_C.cpython-39-x86_64-linux-gnu.so vllm/ \
110 + && cp build/lib.linux-x86_64-cpython-39/vllm/_punica_C.cpython-39-x86_64-linux-gnu.so vllm/ \
111 && cd ..
112
107 -RUN python3 -m pip install --upgrade pip
108 -RUN python3 -m pip install --no-cache-dir ray[all]==2.9.3
113
114 CMD ["/bin/bash"]
requirements-rocm.txt
+2 -1
@@ -2,4 +2,5 @@
2 -r requirements-common.txt
3
4 # Dependencies for AMD GPUs
5 -ray == 2.9.3
5 +ray >= 2.10.0
6 +pytest-asyncio
tests/spec_decode/e2e/conftest.py
+6 -2
@@ -6,8 +6,12 @@ from typing import Dict, List, Optional, Tuple, Union
6 import pytest
7 import ray
8 import torch
9 -from pynvml import (nvmlDeviceGetHandleByIndex, nvmlDeviceGetMemoryInfo,
10 - nvmlInit)
9 +
10 +from vllm.utils import is_hip
11 +
12 +if (not is_hip()):
13 + from pynvml import (nvmlDeviceGetHandleByIndex, nvmlDeviceGetMemoryInfo,
14 + nvmlInit)
15
16 from vllm import LLM
17 from vllm.engine.arg_utils import AsyncEngineArgs