A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Bugfix][Intel] Fix XPU Dockerfile Build (#7824)
Signed-off-by: tylertitsworth <tyler.titsworth@intel.com> Co-authored-by: youkaichao <youkaichao@126.com>
Tyler Titsworth committed
Sep 27, 2024 at 23:45 UTC
260024a3749fb6856625dfee28560a98a92dd339
9 files changed
+87
-14
.buildkite/run-xpu-test.sh
+1
-1
@@ -11,4 +11,4 @@ trap remove_docker_container EXIT
11
remove_docker_container
12
13
# Run the image and launch offline inference
14
-docker run --network host --name xpu-test --device /dev/dri -v /dev/dri/by-path:/dev/dri/by-path xpu-test python3 examples/offline_inference.py
14
+docker run --network host --name xpu-test --device /dev/dri -v /dev/dri/by-path:/dev/dri/by-path --entrypoint="" xpu-test python3 examples/offline_inference.py
.dockerignore
+3
-1
@@ -1,4 +1,6 @@
1
-vllm/*.so
1
+/.github/
2
/.venv
3
/build
4
dist
5
+Dockerfile*
6
+vllm/*.so
Dockerfile.xpu
+38
-9
@@ -1,4 +1,4 @@
1
-FROM intel/oneapi-basekit:2024.2.1-0-devel-ubuntu22.04
1
+FROM intel/oneapi-basekit:2024.2.1-0-devel-ubuntu22.04 AS vllm-base
2
3
RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/intel-oneapi-archive-keyring.gpg > /dev/null && \
4
echo "deb [signed-by=/usr/share/keyrings/intel-oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main " | tee /etc/apt/sources.list.d/oneAPI.list && \
@@ -7,20 +7,49 @@ RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRO
7
echo "deb [arch=amd64,i386 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/graphics/ubuntu jammy arc" | tee /etc/apt/sources.list.d/intel.gpu.jammy.list && \
8
chmod 644 /usr/share/keyrings/intel-graphics.gpg
9
10
-RUN apt-get update -y && \
11
- apt-get install -y curl libicu70 lsb-release git wget vim numactl python3 python3-pip ffmpeg libsm6 libxext6 libgl1
12
-
13
-COPY ./ /workspace/vllm
10
+RUN apt-get update -y && \
11
+ apt-get install -y --no-install-recommends --fix-missing \
12
+ curl \
13
+ ffmpeg \
14
+ git \
15
+ libsndfile1 \
16
+ libsm6 \
17
+ libxext6 \
18
+ libgl1 \
19
+ lsb-release \
20
+ numactl \
21
+ python3 \
22
+ python3-dev \
23
+ python3-pip \
24
+ # vim \
25
+ wget
26
27
WORKDIR /workspace/vllm
28
+COPY requirements-xpu.txt /workspace/vllm/requirements-xpu.txt
29
+COPY requirements-common.txt /workspace/vllm/requirements-common.txt
30
31
RUN --mount=type=cache,target=/root/.cache/pip \
18
- pip install -v --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ \
19
- cmake>=3.26 ninja packaging setuptools-scm>=8 wheel jinja2 \
20
- -r requirements-xpu.txt
32
+ pip install --no-cache-dir \
33
+ --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ \
34
+ -r requirements-xpu.txt
35
+
36
+COPY ./ /workspace/vllm
37
+
38
+ENV VLLM_TARGET_DEVICE=xpu
39
40
RUN --mount=type=cache,target=/root/.cache/pip \
41
--mount=type=bind,source=.git,target=.git \
24
- VLLM_TARGET_DEVICE=xpu python3 setup.py install
42
+ python3 setup.py install
43
44
CMD ["/bin/bash"]
45
+
46
+FROM vllm-base AS vllm-openai
47
+
48
+# install additional dependencies for openai api server
49
+RUN --mount=type=cache,target=/root/.cache/pip \
50
+ pip install accelerate hf_transfer 'modelscope!=1.15.0'
51
+
52
+ENV VLLM_USAGE_SOURCE production-docker-image \
53
+ TRITON_XPU_PROFILE 1
54
+
55
+ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
requirements-common.txt
+1
-1
@@ -1,7 +1,7 @@
1
psutil
2
sentencepiece # Required for LLaMA tokenizer.
3
numpy < 2.0.0
4
-requests
4
+requests >= 2.26.0
5
tqdm
6
py-cpuinfo
7
transformers >= 4.45.0 # Required for Llama 3.2.
requirements-xpu.txt
+6
-2
@@ -1,9 +1,13 @@
1
# Common dependencies
2
-r requirements-common.txt
3
4
-setuptools < 70.0.0 # IPEX's torch have some dependency. to be removed.
5
-
4
ray >= 2.9
5
+cmake>=3.26
6
+ninja
7
+packaging
8
+setuptools-scm>=8
9
+wheel
10
+jinja2
11
# Following pkgs retrieved from https://pytorch-extension.intel.com/release-whl/stable/xpu/us/
12
torch == 2.3.1+cxx11.abi
13
intel-extension-for-pytorch == 2.3.110+xpu
setup.py
+2
@@ -415,6 +415,8 @@ def get_requirements() -> List[str]:
415
for line in requirements:
416
if line.startswith("-r "):
417
resolved_requirements += _read_requirements(line.split()[1])
418
+ elif line.startswith("--"):
419
+ continue
420
else:
421
resolved_requirements.append(line)
422
return resolved_requirements
vllm/platforms/__init__.py
+12
@@ -42,6 +42,15 @@ try:
42
except Exception:
43
pass
44
45
+is_xpu = False
46
+
47
+try:
48
+ import torch
49
+ if hasattr(torch, 'xpu') and torch.xpu.is_available():
50
+ is_xpu = True
51
+except Exception:
52
+ pass
53
+
54
is_cpu = False
55
try:
56
from importlib.metadata import version
@@ -60,6 +69,9 @@ elif is_cuda:
69
elif is_rocm:
70
from .rocm import RocmPlatform
71
current_platform = RocmPlatform()
72
+elif is_xpu:
73
+ from .xpu import XPUPlatform
74
+ current_platform = XPUPlatform()
75
elif is_cpu:
76
from .cpu import CpuPlatform
77
current_platform = CpuPlatform()
vllm/platforms/interface.py
+4
@@ -8,6 +8,7 @@ class PlatformEnum(enum.Enum):
8
CUDA = enum.auto()
9
ROCM = enum.auto()
10
TPU = enum.auto()
11
+ XPU = enum.auto()
12
CPU = enum.auto()
13
UNSPECIFIED = enum.auto()
14
@@ -41,6 +42,9 @@ class Platform:
42
def is_tpu(self) -> bool:
43
return self._enum == PlatformEnum.TPU
44
45
+ def is_xpu(self) -> bool:
46
+ return self._enum == PlatformEnum.XPU
47
+
48
def is_cpu(self) -> bool:
49
return self._enum == PlatformEnum.CPU
50
vllm/platforms/xpu.py
new
+20
@@ -0,0 +1,20 @@
1
+import torch
2
+
3
+from .interface import DeviceCapability, Platform, PlatformEnum
4
+
5
+
6
+class XPUPlatform(Platform):
7
+ _enum = PlatformEnum.XPU
8
+
9
+ @staticmethod
10
+ def get_device_capability(device_id: int = 0) -> DeviceCapability:
11
+ return DeviceCapability(major=int(
12
+ torch.xpu.get_device_capability(device_id)['version'].split('.')
13
+ [0]),
14
+ minor=int(
15
+ torch.xpu.get_device_capability(device_id)
16
+ ['version'].split('.')[1]))
17
+
18
+ @staticmethod
19
+ def get_device_name(device_id: int = 0) -> str:
20
+ return torch.xpu.get_device_name(device_id)