@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 260024a374
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Bugfix][Intel] Fix XPU Dockerfile Build (#7824)

Signed-off-by: tylertitsworth <tyler.titsworth@intel.com> Co-authored-by: youkaichao <youkaichao@126.com>

Tyler Titsworth committed Sep 27, 2024 at 23:45 UTC 260024a3749fb6856625dfee28560a98a92dd339
9 files changed +87 -14
.buildkite/run-xpu-test.sh
+1 -1
@@ -11,4 +11,4 @@ trap remove_docker_container EXIT
11 remove_docker_container
12
13 # Run the image and launch offline inference
14 -docker run --network host --name xpu-test --device /dev/dri -v /dev/dri/by-path:/dev/dri/by-path xpu-test python3 examples/offline_inference.py
14 +docker run --network host --name xpu-test --device /dev/dri -v /dev/dri/by-path:/dev/dri/by-path --entrypoint="" xpu-test python3 examples/offline_inference.py
.dockerignore
+3 -1
@@ -1,4 +1,6 @@
1 -vllm/*.so
1 +/.github/
2 /.venv
3 /build
4 dist
5 +Dockerfile*
6 +vllm/*.so
Dockerfile.xpu
+38 -9
@@ -1,4 +1,4 @@
1 -FROM intel/oneapi-basekit:2024.2.1-0-devel-ubuntu22.04
1 +FROM intel/oneapi-basekit:2024.2.1-0-devel-ubuntu22.04 AS vllm-base
2
3 RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/intel-oneapi-archive-keyring.gpg > /dev/null && \
4 echo "deb [signed-by=/usr/share/keyrings/intel-oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main " | tee /etc/apt/sources.list.d/oneAPI.list && \
@@ -7,20 +7,49 @@ RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRO
7 echo "deb [arch=amd64,i386 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/graphics/ubuntu jammy arc" | tee /etc/apt/sources.list.d/intel.gpu.jammy.list && \
8 chmod 644 /usr/share/keyrings/intel-graphics.gpg
9
10 -RUN apt-get update -y && \
11 - apt-get install -y curl libicu70 lsb-release git wget vim numactl python3 python3-pip ffmpeg libsm6 libxext6 libgl1
12 -
13 -COPY ./ /workspace/vllm
10 +RUN apt-get update -y && \
11 + apt-get install -y --no-install-recommends --fix-missing \
12 + curl \
13 + ffmpeg \
14 + git \
15 + libsndfile1 \
16 + libsm6 \
17 + libxext6 \
18 + libgl1 \
19 + lsb-release \
20 + numactl \
21 + python3 \
22 + python3-dev \
23 + python3-pip \
24 + # vim \
25 + wget
26
27 WORKDIR /workspace/vllm
28 +COPY requirements-xpu.txt /workspace/vllm/requirements-xpu.txt
29 +COPY requirements-common.txt /workspace/vllm/requirements-common.txt
30
31 RUN --mount=type=cache,target=/root/.cache/pip \
18 - pip install -v --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ \
19 - cmake>=3.26 ninja packaging setuptools-scm>=8 wheel jinja2 \
20 - -r requirements-xpu.txt
32 + pip install --no-cache-dir \
33 + --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ \
34 + -r requirements-xpu.txt
35 +
36 +COPY ./ /workspace/vllm
37 +
38 +ENV VLLM_TARGET_DEVICE=xpu
39
40 RUN --mount=type=cache,target=/root/.cache/pip \
41 --mount=type=bind,source=.git,target=.git \
24 - VLLM_TARGET_DEVICE=xpu python3 setup.py install
42 + python3 setup.py install
43
44 CMD ["/bin/bash"]
45 +
46 +FROM vllm-base AS vllm-openai
47 +
48 +# install additional dependencies for openai api server
49 +RUN --mount=type=cache,target=/root/.cache/pip \
50 + pip install accelerate hf_transfer 'modelscope!=1.15.0'
51 +
52 +ENV VLLM_USAGE_SOURCE production-docker-image \
53 + TRITON_XPU_PROFILE 1
54 +
55 +ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
requirements-common.txt
+1 -1
@@ -1,7 +1,7 @@
1 psutil
2 sentencepiece # Required for LLaMA tokenizer.
3 numpy < 2.0.0
4 -requests
4 +requests >= 2.26.0
5 tqdm
6 py-cpuinfo
7 transformers >= 4.45.0 # Required for Llama 3.2.
requirements-xpu.txt
+6 -2
@@ -1,9 +1,13 @@
1 # Common dependencies
2 -r requirements-common.txt
3
4 -setuptools < 70.0.0 # IPEX's torch have some dependency. to be removed.
5 -
4 ray >= 2.9
5 +cmake>=3.26
6 +ninja
7 +packaging
8 +setuptools-scm>=8
9 +wheel
10 +jinja2
11 # Following pkgs retrieved from https://pytorch-extension.intel.com/release-whl/stable/xpu/us/
12 torch == 2.3.1+cxx11.abi
13 intel-extension-for-pytorch == 2.3.110+xpu
setup.py
+2
@@ -415,6 +415,8 @@ def get_requirements() -> List[str]:
415 for line in requirements:
416 if line.startswith("-r "):
417 resolved_requirements += _read_requirements(line.split()[1])
418 + elif line.startswith("--"):
419 + continue
420 else:
421 resolved_requirements.append(line)
422 return resolved_requirements
vllm/platforms/__init__.py
+12
@@ -42,6 +42,15 @@ try:
42 except Exception:
43 pass
44
45 +is_xpu = False
46 +
47 +try:
48 + import torch
49 + if hasattr(torch, 'xpu') and torch.xpu.is_available():
50 + is_xpu = True
51 +except Exception:
52 + pass
53 +
54 is_cpu = False
55 try:
56 from importlib.metadata import version
@@ -60,6 +69,9 @@ elif is_cuda:
69 elif is_rocm:
70 from .rocm import RocmPlatform
71 current_platform = RocmPlatform()
72 +elif is_xpu:
73 + from .xpu import XPUPlatform
74 + current_platform = XPUPlatform()
75 elif is_cpu:
76 from .cpu import CpuPlatform
77 current_platform = CpuPlatform()
vllm/platforms/interface.py
+4
@@ -8,6 +8,7 @@ class PlatformEnum(enum.Enum):
8 CUDA = enum.auto()
9 ROCM = enum.auto()
10 TPU = enum.auto()
11 + XPU = enum.auto()
12 CPU = enum.auto()
13 UNSPECIFIED = enum.auto()
14
@@ -41,6 +42,9 @@ class Platform:
42 def is_tpu(self) -> bool:
43 return self._enum == PlatformEnum.TPU
44
45 + def is_xpu(self) -> bool:
46 + return self._enum == PlatformEnum.XPU
47 +
48 def is_cpu(self) -> bool:
49 return self._enum == PlatformEnum.CPU
50
vllm/platforms/xpu.py new
+20
@@ -0,0 +1,20 @@
1 +import torch
2 +
3 +from .interface import DeviceCapability, Platform, PlatformEnum
4 +
5 +
6 +class XPUPlatform(Platform):
7 + _enum = PlatformEnum.XPU
8 +
9 + @staticmethod
10 + def get_device_capability(device_id: int = 0) -> DeviceCapability:
11 + return DeviceCapability(major=int(
12 + torch.xpu.get_device_capability(device_id)['version'].split('.')
13 + [0]),
14 + minor=int(
15 + torch.xpu.get_device_capability(device_id)
16 + ['version'].split('.')[1]))
17 +
18 + @staticmethod
19 + def get_device_name(device_id: int = 0) -> str:
20 + return torch.xpu.get_device_name(device_id)