A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 17:15 UTC
Dockerfile.ppc64le changes to move to UBI (#15402)
Signed-off-by: Md. Shafi Hussain <Md.Shafi.Hussain@ibm.com>
Md. Shafi Hussain committed
Mar 25, 2025 at 15:45 UTC
3e2f37a69a659e8585f84d61afd4623525d70b20
2 files changed
+254
-24
Dockerfile.ppc64le
+251
-21
@@ -1,37 +1,267 @@
1
-FROM mambaorg/micromamba
2
-ARG MAMBA_DOCKERFILE_ACTIVATE=1
3
-USER root
1
+ARG BASE_UBI_IMAGE_TAG=9.5-1741850109
2
5
-ENV PATH="/usr/local/cargo/bin:$PATH:/opt/conda/bin/"
3
+###############################################################
4
+# base stage with basic dependencies
5
+###############################################################
6
7
-RUN apt-get update -y && apt-get install -y git wget kmod curl vim libnuma-dev libsndfile-dev libprotobuf-dev build-essential ffmpeg libsm6 libxext6 libgl1 libssl-dev
7
+FROM registry.access.redhat.com/ubi9/ubi-minimal:${BASE_UBI_IMAGE_TAG} AS base-builder
8
9
-# Some packages in requirements/cpu are installed here
10
-# IBM provides optimized packages for ppc64le processors in the open-ce project for mamba
11
-# Currently these may not be available for venv or pip directly
12
-RUN micromamba install -y -n base -c https://ftp.osuosl.org/pub/open-ce/1.11.0-p10/ -c defaults python=3.10 rust && micromamba clean --all --yes
9
+ARG PYTHON_VERSION=3.12
10
+ARG OPENBLAS_VERSION=0.3.29
11
+
12
+# Set Environment Variables for venv, cargo & openblas
13
+ENV VIRTUAL_ENV=/opt/vllm
14
+ENV PATH=${VIRTUAL_ENV}/bin:/root/.cargo/bin:$PATH
15
+ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig/
16
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib64:/usr/local/lib:/usr/lib64:/usr/lib
17
+ENV UV_LINK_MODE=copy
18
+
19
+# install gcc-13, python, rust, openblas
20
+# Note: A symlink for libatomic.so is created for gcc-13 (linker fails to find libatomic otherwise - reqd. for sentencepiece)
21
+# Note: A dummy file 'control' is created in /tmp/ to artificially create dependencies between stages when building stages in parallel
22
+# when `--jobs=<N>` is passed with podman build command
23
+RUN microdnf install -y openssl-devel dnf \
24
+ && dnf install -y https://mirror.stream.centos.org/9-stream/BaseOS/`arch`/os/Packages/centos-gpg-keys-9.0-24.el9.noarch.rpm \
25
+ https://mirror.stream.centos.org/9-stream/BaseOS/`arch`/os/Packages/centos-stream-repos-9.0-24.el9.noarch.rpm \
26
+ https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm \
27
+ && dnf config-manager --add-repo https://mirror.stream.centos.org/9-stream/BaseOS/`arch`/os \
28
+ && dnf config-manager --add-repo https://mirror.stream.centos.org/9-stream/AppStream/`arch`/os \
29
+ && dnf config-manager --set-enabled crb \
30
+ && dnf install -y \
31
+ git tar gcc-toolset-13 automake libtool numactl-devel lapack-devel \
32
+ pkgconfig xsimd zeromq-devel kmod findutils protobuf* \
33
+ libtiff-devel libjpeg-devel openjpeg2-devel zlib-devel \
34
+ freetype-devel lcms2-devel libwebp-devel tcl-devel tk-devel \
35
+ harfbuzz-devel fribidi-devel libraqm-devel libimagequant-devel libxcb-devel \
36
+ python${PYTHON_VERSION}-devel python${PYTHON_VERSION}-pip \
37
+ && dnf clean all \
38
+ && ln -sf /usr/lib64/libatomic.so.1 /usr/lib64/libatomic.so \
39
+ && python${PYTHON_VERSION} -m venv ${VIRTUAL_ENV} \
40
+ && python -m pip install -U pip uv \
41
+ && uv pip install wheel build "setuptools<70" setuptools_scm setuptools_rust meson-python cmake ninja cython scikit_build_core scikit_build \
42
+ && curl -sL https://ftp2.osuosl.org/pub/ppc64el/openblas/latest/Openblas_${OPENBLAS_VERSION}_ppc64le.tar.gz | tar xvf - -C /usr/local \
43
+ && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y \
44
+ && cd /tmp && touch control
45
+
46
+###############################################################
47
+# Stage to build torch family
48
+###############################################################
49
+
50
+FROM base-builder AS torch-builder
51
+
52
+ARG MAX_JOBS
53
+ARG TORCH_VERSION=2.6.0
54
+ARG _GLIBCXX_USE_CXX11_ABI=1
55
+RUN --mount=type=cache,target=/root/.cache/uv \
56
+ source /opt/rh/gcc-toolset-13/enable && \
57
+ git clone --recursive https://github.com/pytorch/pytorch.git -b v${TORCH_VERSION} && \
58
+ cd pytorch && \
59
+ uv pip install -r requirements.txt && \
60
+ python setup.py develop && \
61
+ rm -f dist/torch*+git*whl && \
62
+ MAX_JOBS=${MAX_JOBS:-$(nproc)} \
63
+ PYTORCH_BUILD_VERSION=${TORCH_VERSION} PYTORCH_BUILD_NUMBER=1 uv build --wheel --out-dir /torchwheels/
64
+
65
+ARG TORCHVISION_VERSION=0.21.0
66
+ARG TORCHVISION_USE_NVJPEG=0
67
+ARG TORCHVISION_USE_FFMPEG=0
68
+RUN --mount=type=cache,target=/root/.cache/uv \
69
+ source /opt/rh/gcc-toolset-13/enable && \
70
+ git clone --recursive https://github.com/pytorch/vision.git -b v${TORCHVISION_VERSION} && \
71
+ cd vision && \
72
+ MAX_JOBS=${MAX_JOBS:-$(nproc)} \
73
+ BUILD_VERSION=${TORCHVISION_VERSION} \
74
+ uv build --wheel --out-dir /torchwheels/ --no-build-isolation
75
+
76
+ARG TORCHAUDIO_VERSION=2.6.0
77
+ARG BUILD_SOX=1
78
+ARG BUILD_KALDI=1
79
+ARG BUILD_RNNT=1
80
+ARG USE_FFMPEG=0
81
+ARG USE_ROCM=0
82
+ARG USE_CUDA=0
83
+ARG TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_FFMPEG=1
84
+RUN --mount=type=cache,target=/root/.cache/uv \
85
+ source /opt/rh/gcc-toolset-13/enable && \
86
+ git clone --recursive https://github.com/pytorch/audio.git -b v${TORCHAUDIO_VERSION} && \
87
+ cd audio && \
88
+ MAX_JOBS=${MAX_JOBS:-$(nproc)} \
89
+ BUILD_VERSION=${TORCHAUDIO_VERSION} \
90
+ uv build --wheel --out-dir /torchwheels/ --no-build-isolation
91
+
92
+###############################################################
93
+# Stage to build pyarrow
94
+###############################################################
95
+
96
+FROM base-builder AS arrow-builder
97
+
98
+ARG MAX_JOBS
99
+ARG PYARROW_PARALLEL
100
+ARG PYARROW_VERSION=19.0.1
101
+RUN --mount=type=cache,target=/root/.cache/uv \
102
+ source /opt/rh/gcc-toolset-13/enable && \
103
+ git clone --recursive https://github.com/apache/arrow.git -b apache-arrow-${PYARROW_VERSION} && \
104
+ cd arrow/cpp && \
105
+ mkdir build && cd build && \
106
+ cmake -DCMAKE_BUILD_TYPE=release \
107
+ -DCMAKE_INSTALL_PREFIX=/usr/local \
108
+ -DARROW_PYTHON=ON \
109
+ -DARROW_BUILD_TESTS=OFF \
110
+ -DARROW_JEMALLOC=ON \
111
+ -DARROW_BUILD_STATIC="OFF" \
112
+ -DARROW_PARQUET=ON \
113
+ .. && \
114
+ make install -j ${MAX_JOBS:-$(nproc)} && \
115
+ cd ../../python/ && \
116
+ uv pip install -v -r requirements-wheel-build.txt && \
117
+ PYARROW_PARALLEL=${PYARROW_PARALLEL:-$(nproc)} \
118
+ python setup.py build_ext \
119
+ --build-type=release --bundle-arrow-cpp \
120
+ bdist_wheel --dist-dir /arrowwheels/
121
+
122
+###############################################################
123
+# Stage to build opencv
124
+###############################################################
125
+
126
+FROM base-builder AS cv-builder
127
+
128
+ARG MAX_JOBS
129
+ARG OPENCV_VERSION=84
130
+ARG ENABLE_HEADLESS=1
131
+RUN --mount=type=cache,target=/root/.cache/uv \
132
+ source /opt/rh/gcc-toolset-13/enable && \
133
+ git clone --recursive https://github.com/opencv/opencv-python.git -b ${OPENCV_VERSION} && \
134
+ cd opencv-python && \
135
+ sed -i 's/"setuptools==59.2.0",/"setuptools<70.0",/g' pyproject.toml && \
136
+ python -m build --wheel --installer=uv --outdir /opencvwheels/
137
+
138
+###############################################################
139
+# Stage to build vllm - this stage builds and installs
140
+# vllm, tensorizer and vllm-tgis-adapter and builds uv cache
141
+# for transitive dependencies - eg. grpcio
142
+###############################################################
143
+
144
+FROM base-builder AS vllmcache-builder
145
+
146
+COPY --from=torch-builder /tmp/control /dev/null
147
+COPY --from=arrow-builder /tmp/control /dev/null
148
+COPY --from=cv-builder /tmp/control /dev/null
149
+
150
+ARG VLLM_TARGET_DEVICE=cpu
151
+
152
+# this step installs vllm and populates uv cache
153
+# with all the transitive dependencies
154
+RUN --mount=type=cache,target=/root/.cache/uv \
155
+ --mount=type=bind,from=torch-builder,source=/torchwheels/,target=/torchwheels/,ro \
156
+ --mount=type=bind,from=arrow-builder,source=/arrowwheels/,target=/arrowwheels/,ro \
157
+ --mount=type=bind,from=cv-builder,source=/opencvwheels/,target=/opencvwheels/,ro \
158
+ --mount=type=bind,src=.,dst=/src/,rw \
159
+ source /opt/rh/gcc-toolset-13/enable && \
160
+ uv pip install /opencvwheels/*.whl /arrowwheels/*.whl /torchwheels/*.whl && \
161
+ sed -i -e 's/.*torch.*//g' /src/pyproject.toml /src/requirements/*.txt && \
162
+ uv pip install pandas pythran pybind11 && \
163
+ # sentencepiece.pc is in some pkgconfig inside uv cache
164
+ export PKG_CONFIG_PATH=$(find / -type d -name "pkgconfig" 2>/dev/null | tr '\n' ':') && \
165
+ uv pip install -r /src/requirements/common.txt -r /src/requirements/cpu.txt -r /src/requirements/build.txt --no-build-isolation && \
166
+ cd /src/ && \
167
+ uv build --wheel --out-dir /vllmwheel/ --no-build-isolation && \
168
+ uv pip install /vllmwheel/*.whl
169
14
-COPY ./ /workspace/vllm
170
171
+###############################################################
172
+# Stage to build numactl
173
+###############################################################
174
+
175
+FROM base-builder AS numa-builder
176
+
177
+# Note: Building numactl with gcc-11. Compiling with gcc-13 in this builder stage will
178
+# trigger recompilation with gcc-11 (and require libtool) in the final stage where we do not have gcc-13
179
+ARG MAX_JOBS
180
+ARG NUMACTL_VERSION=2.0.19
181
+RUN git clone --recursive https://github.com/numactl/numactl.git -b v${NUMACTL_VERSION} \
182
+ && cd numactl \
183
+ && autoreconf -i && ./configure \
184
+ && make -j ${MAX_JOBS:-$(nproc)}
185
+
186
+###############################################################
187
+# Stage to build lapack
188
+###############################################################
189
+
190
+FROM base-builder AS lapack-builder
191
+
192
+ARG MAX_JOBS
193
+ARG LAPACK_VERSION=3.12.1
194
+RUN git clone --recursive https://github.com/Reference-LAPACK/lapack.git -b v${LAPACK_VERSION} \
195
+ && cd lapack && source /opt/rh/gcc-toolset-13/enable \
196
+ && cmake -B build -S . \
197
+ && cmake --build build -j ${MAX_JOBS:-$(nproc)}
198
+
199
+
200
+###############################################################
201
+# FINAL VLLM IMAGE STAGE #
202
+###############################################################
203
+
204
+FROM registry.access.redhat.com/ubi9/ubi-minimal:${BASE_UBI_IMAGE_TAG} AS vllm-openai
205
+
206
+ARG PYTHON_VERSION=3.12
207
+ARG OPENBLAS_VERSION=0.3.29
208
+
209
+# Set Environment Variables for venv & openblas
210
+ENV VIRTUAL_ENV=/opt/vllm
211
+ENV PATH=${VIRTUAL_ENV}/bin:$PATH
212
+ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig/
213
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib64:/usr/local/lib:/usr/lib64:/usr/lib
214
+ENV UV_LINK_MODE=copy
215
+
216
+# create artificial dependencies between stages for independent stages to build in parallel
217
+COPY --from=torch-builder /tmp/control /dev/null
218
+COPY --from=arrow-builder /tmp/control /dev/null
219
+COPY --from=cv-builder /tmp/control /dev/null
220
+COPY --from=vllmcache-builder /tmp/control /dev/null
221
+COPY --from=numa-builder /tmp/control /dev/null
222
+COPY --from=lapack-builder /tmp/control /dev/null
223
+
224
+# install gcc-11, python, openblas, numactl, lapack
225
+RUN --mount=type=cache,target=/root/.cache/uv \
226
+ --mount=type=bind,from=numa-builder,source=/numactl/,target=/numactl/,rw \
227
+ --mount=type=bind,from=lapack-builder,source=/lapack/,target=/lapack/,rw \
228
+ rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && \
229
+ microdnf install --nodocs -y \
230
+ tar findutils openssl \
231
+ pkgconfig xsimd g++ gcc-fortran libsndfile \
232
+ libtiff libjpeg openjpeg2 zlib zeromq \
233
+ freetype lcms2 libwebp tcl tk utf8proc \
234
+ harfbuzz fribidi libraqm libimagequant libxcb \
235
+ python${PYTHON_VERSION}-devel python${PYTHON_VERSION}-pip \
236
+ && microdnf clean all \
237
+ && python${PYTHON_VERSION} -m venv ${VIRTUAL_ENV} \
238
+ && python -m pip install -U pip uv --no-cache \
239
+ && curl -sL https://ftp2.osuosl.org/pub/ppc64el/openblas/latest/Openblas_${OPENBLAS_VERSION}_ppc64le.tar.gz | tar xvf - -C /usr/local \
240
+ && make -C /numactl install \
241
+ && uv pip install cmake \
242
+ && cmake --install /lapack/build \
243
+ && uv pip uninstall cmake
244
+
245
+# consume previously built wheels (including vllm)
246
+RUN --mount=type=cache,target=/root/.cache/uv \
247
+ --mount=type=bind,from=torch-builder,source=/torchwheels/,target=/torchwheels/,ro \
248
+ --mount=type=bind,from=arrow-builder,source=/arrowwheels/,target=/arrowwheels/,ro \
249
+ --mount=type=bind,from=cv-builder,source=/opencvwheels/,target=/opencvwheels/,ro \
250
+ --mount=type=bind,from=vllmcache-builder,source=/vllmwheel/,target=/vllmwheel/,ro \
251
+ HOME=/root uv pip install /opencvwheels/*.whl /arrowwheels/*.whl /torchwheels/*.whl /vllmwheel/*.whl
252
+
253
+COPY ./ /workspace/vllm
254
WORKDIR /workspace/vllm
255
ARG GIT_REPO_CHECK=0
256
RUN --mount=type=bind,source=.git,target=.git \
257
if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh; fi
258
21
-RUN --mount=type=cache,target=/root/.cache/pip \
22
- RUSTFLAGS='-L /opt/conda/lib' pip install -v --prefer-binary --extra-index-url https://repo.fury.io/mgiessing \
23
- 'cmake>=3.26' ninja packaging 'setuptools-scm>=8' wheel jinja2 \
24
- -r requirements/cpu.txt \
25
- xformers uvloop==0.20.0
26
-
27
-RUN --mount=type=bind,source=.git,target=.git \
28
- VLLM_TARGET_DEVICE=cpu python3 setup.py install
29
-
259
# install development dependencies (for testing)
31
-RUN python3 -m pip install -e tests/vllm_test_utils
260
+RUN --mount=type=cache,target=/root/.cache/uv \
261
+ uv pip install -e tests/vllm_test_utils
262
263
WORKDIR /workspace/
264
265
RUN ln -s /workspace/vllm/tests && ln -s /workspace/vllm/examples && ln -s /workspace/vllm/benchmarks
266
37
-ENTRYPOINT ["/opt/conda/bin/python3", "-m", "vllm.entrypoints.openai.api_server"]
267
+ENTRYPOINT ["python", "-m", "vllm.entrypoints.openai.api_server"]
requirements/cpu.txt
+3
-3
@@ -4,14 +4,14 @@
4
# Dependencies for CPUs
5
torch==2.6.0+cpu; platform_machine == "x86_64"
6
torch==2.6.0; platform_system == "Darwin"
7
-torch==2.5.1; platform_machine == "ppc64le" or platform_machine == "aarch64"
7
+torch==2.6.0; platform_machine == "ppc64le" or platform_machine == "aarch64"
8
torch==2.7.0.dev20250304; platform_machine == "s390x"
9
10
# required for the image processor of minicpm-o-2_6, this must be updated alongside torch
11
torchaudio; platform_machine != "ppc64le" and platform_machine != "s390x"
12
-torchaudio==2.5.1; platform_machine == "ppc64le"
12
+torchaudio==2.6.0; platform_machine == "ppc64le"
13
14
# required for the image processor of phi3v, this must be updated alongside torch
15
torchvision; platform_machine != "ppc64le" and platform_machine != "s390x"
16
-torchvision==0.20.1; platform_machine == "ppc64le"
16
+torchvision==0.21.0; platform_machine == "ppc64le"
17
datasets # for benchmark scripts