@tduncan / Nvidia-Optimization-Stack-VLLM
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
s390x 271 lines 10.9 KB
Raw
1 # Base UBI image for s390x architecture
2 ARG BASE_UBI_IMAGE_TAG=9.6
3 ARG PYTHON_VERSION=3.12
4 FROM registry.access.redhat.com/ubi9/ubi-minimal:${BASE_UBI_IMAGE_TAG} AS base
5
6 # Install basic dependencies
7 ARG PYTHON_VERSION
8 ENV PYTHON_VERSION=${PYTHON_VERSION}
9
10 WORKDIR /workspace
11
12 ENV LANG=C.UTF-8 \
13 LC_ALL=C.UTF-8
14
15 # Install development utilities
16 RUN microdnf install -y \
17 which procps findutils tar vim git gcc-toolset-14 gcc-toolset-14-binutils gcc-toolset-14-libatomic-devel patch zlib-devel \
18 libjpeg-turbo-devel libtiff-devel libpng-devel libwebp-devel freetype-devel harfbuzz-devel \
19 openssl-devel openblas openblas-devel autoconf automake libtool cmake numpy libsndfile \
20 clang llvm-devel llvm-static clang-devel && \
21 microdnf clean all
22
23 ENV GCC_TOOLSET_ROOT=/opt/rh/gcc-toolset-14/root \
24 PATH=/opt/rh/gcc-toolset-14/root/usr/bin:/usr/local/bin:/usr/bin:/bin \
25 LD_LIBRARY_PATH=/opt/rh/gcc-toolset-14/root/usr/lib64:/usr/local/lib:/usr/lib64 \
26 LIBRARY_PATH=/opt/rh/gcc-toolset-14/root/usr/lib64 \
27 PKG_CONFIG_PATH=/opt/rh/gcc-toolset-14/root/usr/lib64/pkgconfig
28
29 # Python Installation
30 FROM base AS python-install
31 ARG PYTHON_VERSION
32
33 ENV VIRTUAL_ENV=/opt/vllm
34 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
35 ENV PYTHON_VERSION=${PYTHON_VERSION}
36 RUN microdnf install -y \
37 python${PYTHON_VERSION}-devel python${PYTHON_VERSION}-pip python${PYTHON_VERSION}-wheel && \
38 python${PYTHON_VERSION} -m venv $VIRTUAL_ENV && pip install --no-cache -U pip wheel uv && microdnf clean all
39
40 FROM python-install AS rust
41 ENV CARGO_HOME=/root/.cargo
42 ENV RUSTUP_HOME=/root/.rustup
43 ENV PATH="$CARGO_HOME/bin:$RUSTUP_HOME/bin:$PATH"
44
45 RUN curl https://sh.rustup.rs -sSf | sh -s -- -y && \
46 . "$CARGO_HOME/env" && \
47 rustup default stable && \
48 rustup show
49
50 FROM python-install AS numa-build
51 WORKDIR /tmp
52 RUN curl -LO https://github.com/numactl/numactl/archive/refs/tags/v2.0.19.tar.gz && \
53 tar -xvzf v2.0.19.tar.gz && \
54 cd numactl-2.0.19 && \
55 ./autogen.sh && \
56 ./configure && \
57 make
58
59 # Set include path
60 ENV C_INCLUDE_PATH="/usr/local/include:$C_INCLUDE_PATH"
61
62 FROM base AS gperftools-build
63 WORKDIR /tmp
64 ARG GPERFTOOLS_VERSION=2.16
65 RUN curl -LO https://github.com/gperftools/gperftools/releases/download/gperftools-${GPERFTOOLS_VERSION}/gperftools-${GPERFTOOLS_VERSION}.tar.gz && \
66 tar -xzf gperftools-${GPERFTOOLS_VERSION}.tar.gz && \
67 cd gperftools-${GPERFTOOLS_VERSION} && \
68 ./configure --enable-minimal && \
69 make -j"$(nproc)"
70
71 FROM python-install AS torch-vision
72 # Install torchvision
73 ARG TORCH_VISION_VERSION=v0.28.0
74 WORKDIR /tmp
75 RUN --mount=type=cache,target=/root/.cache/uv \
76 git clone https://github.com/pytorch/vision.git && \
77 cd vision && \
78 git checkout $TORCH_VISION_VERSION && \
79 uv pip install torch==2.13.0 --index-url https://download.pytorch.org/whl/cpu && \
80 python setup.py bdist_wheel
81
82 FROM python-install AS hf-xet-builder
83 # Install hf-xet
84 WORKDIR /tmp
85 ENV CARGO_HOME=/root/.cargo
86 ENV RUSTUP_HOME=/root/.rustup
87 ENV PATH="$CARGO_HOME/bin:$RUSTUP_HOME/bin:$PATH"
88 RUN --mount=type=cache,target=/root/.cache/uv \
89 --mount=type=bind,from=rust,source=/root/.cargo,target=/root/.cargo,rw \
90 --mount=type=bind,from=rust,source=/root/.rustup,target=/root/.rustup,rw \
91 git clone https://github.com/huggingface/xet-core.git && \
92 cd xet-core/hf_xet/ && \
93 uv pip install maturin patchelf && \
94 python -m maturin build --release --out dist && \
95 mkdir -p /tmp/hf-xet/dist && \
96 cp dist/*.whl /tmp/hf-xet/dist/
97
98 # Build LLVM 20 from source for llvmlite (system repos ship LLVM 21 which
99 # llvmlite v0.47 does not support; only SystemZ target is needed).
100 FROM base AS llvm20-build
101 ARG LLVM_VERSION=20.1.8
102 WORKDIR /tmp
103 RUN microdnf install -y ninja-build gcc gcc-c++ python3 xz && \
104 curl -LO https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/llvm-project-${LLVM_VERSION}.src.tar.xz && \
105 tar -xf llvm-project-${LLVM_VERSION}.src.tar.xz && \
106 cmake -G Ninja -S llvm-project-${LLVM_VERSION}.src/llvm -B build \
107 -DCMAKE_BUILD_TYPE=Release \
108 -DCMAKE_INSTALL_PREFIX=/opt/llvm20 \
109 -DLLVM_TARGETS_TO_BUILD="SystemZ" \
110 -DLLVM_ENABLE_RTTI=ON \
111 -DLLVM_BUILD_TOOLS=OFF \
112 -DLLVM_BUILD_UTILS=ON \
113 -DLLVM_BUILD_EXAMPLES=OFF \
114 -DLLVM_BUILD_TESTS=OFF \
115 -DLLVM_INCLUDE_TESTS=OFF \
116 -DLLVM_INCLUDE_EXAMPLES=OFF \
117 -DLLVM_INCLUDE_BENCHMARKS=OFF && \
118 ninja -C build install && \
119 rm -rf build llvm-project-${LLVM_VERSION}.src*
120
121 # Build numba
122 FROM python-install AS numba-builder
123
124 ARG MAX_JOBS
125 ARG NUMBA_VERSION=0.65.0
126
127 WORKDIR /tmp
128
129 # Clone all required dependencies
130 RUN --mount=type=cache,target=/root/.cache/uv \
131 --mount=type=bind,from=llvm20-build,source=/opt/llvm20,target=/opt/llvm20 \
132 microdnf install ninja-build gcc gcc-c++ -y && \
133 git clone --recursive https://github.com/numba/llvmlite.git -b v0.47.0 && \
134 git clone --recursive https://github.com/numba/numba.git -b ${NUMBA_VERSION} && \
135 cd llvmlite && \
136 uv pip install 'cmake<4' 'setuptools<70' numpy && \
137 CMAKE_PREFIX_PATH=/opt/llvm20 LLVM_CONFIG=/opt/llvm20/bin/llvm-config \
138 python setup.py bdist_wheel && \
139 cd ../numba && \
140 if ! grep '#include "dynamic_annotations.h"' numba/_dispatcher.cpp; then \
141 sed -i '/#include "internal\/pycore_atomic.h"/i\#include "dynamic_annotations.h"' numba/_dispatcher.cpp; \
142 fi && python setup.py bdist_wheel
143
144 # Shared FFmpeg for torchcodec video decoding (UBI has no ffmpeg package).
145 FROM base AS ffmpeg-build
146 ARG FFMPEG_VERSION=n6.1.2
147 WORKDIR /tmp
148 RUN microdnf install -y pkgconf-pkg-config xz && \
149 curl -LO https://github.com/FFmpeg/FFmpeg/archive/refs/tags/${FFMPEG_VERSION}.tar.gz && \
150 tar -xzf ${FFMPEG_VERSION}.tar.gz && \
151 cd FFmpeg-${FFMPEG_VERSION} && \
152 ./configure --prefix=/usr/local --libdir=/usr/local/lib \
153 --enable-shared --disable-static --enable-pic \
154 --disable-doc --disable-debug --disable-ffplay --disable-ffprobe \
155 --extra-cflags="-Wno-incompatible-pointer-types" && \
156 make -j"$(nproc)" && \
157 make install
158
159 # Torchcodec video decoder (no s390x wheels; CPU / FFmpeg only s390x does not support audio decoding).
160 FROM python-install AS torchcodec-builder
161 ARG TORCHCODEC_VERSION=v0.14.0
162 WORKDIR /tmp
163 ENV LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH"
164 ENV PKG_CONFIG_PATH="/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH"
165 RUN --mount=type=cache,target=/root/.cache/uv \
166 --mount=type=bind,from=ffmpeg-build,source=/usr/local,target=/usr/local \
167 git clone --depth 1 --branch ${TORCHCODEC_VERSION} \
168 https://github.com/pytorch/torchcodec.git && \
169 cd torchcodec && \
170 uv pip install torch==2.13.0 --index-url https://download.pytorch.org/whl/cpu && \
171 uv pip install pybind11 setuptools wheel && \
172 pybind11_DIR="$(python -c 'import pybind11; print(pybind11.get_cmake_dir())')" && \
173 export pybind11_DIR CMAKE_PREFIX_PATH="${pybind11_DIR}:${CMAKE_PREFIX_PATH}" && \
174 export I_CONFIRM_THIS_IS_NOT_A_LICENSE_VIOLATION=1 && \
175 export TORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR=1 && \
176 export TORCHCODEC_CMAKE_BUILD_DIR="${PWD}/build" && \
177 python setup.py bdist_wheel
178
179 # Build OpenCV from source for s390x
180 FROM python-install AS opencv-builder
181 WORKDIR /tmp
182 ARG MAX_JOBS
183 ARG OPENCV_VERSION=90
184 ARG ENABLE_HEADLESS=1
185 RUN --mount=type=cache,target=/root/.cache/uv \
186 uv pip install numpy setuptools wheel scikit_build build && \
187 git clone --recursive https://github.com/opencv/opencv-python.git -b ${OPENCV_VERSION} && \
188 cd opencv-python && \
189 python -m build --wheel --installer=uv --outdir /tmp/opencv-python/dist
190
191 # # Final build stage
192 FROM python-install AS vllm-cpu
193 ARG PYTHON_VERSION
194 ARG PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu"
195 # Set correct library path for torch and numactl
196 ENV LD_LIBRARY_PATH="/opt/vllm/lib64/python${PYTHON_VERSION}/site-packages/torch/lib:/usr/local/lib:/opt/rh/gcc-toolset-14/root/usr/lib64:$LD_LIBRARY_PATH"
197 ENV C_INCLUDE_PATH="/usr/local/include:$C_INCLUDE_PATH"
198 ENV UV_LINK_MODE=copy
199 ENV UV_COMPILE_BYTECODE=1
200 ENV CARGO_HOME=/root/.cargo
201 ENV RUSTUP_HOME=/root/.rustup
202 ENV GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1
203 ENV PCP_DIR=/opt/rh/gcc-toolset-14/root
204 ENV PKG_CONFIG_PATH="/opt/rh/gcc-toolset-14/root/usr/lib64/pkgconfig:/usr/local/lib/pkgconfig/"
205 ENV PATH="${VIRTUAL_ENV:+${VIRTUAL_ENV}/bin}:/opt/rh/gcc-toolset-14/root/usr/bin:/usr/local/bin:$CARGO_HOME/bin:$RUSTUP_HOME/bin:$PATH"
206 ENV PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
207 ENV UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
208 COPY . /workspace/vllm
209 WORKDIR /workspace/vllm
210
211 RUN --mount=type=bind,from=numa-build,src=/tmp/numactl-2.0.19,target=/numactl \
212 make -C /numactl install
213
214 RUN --mount=type=bind,from=gperftools-build,src=/tmp/gperftools-2.16,target=/gperftools \
215 make -C /gperftools install-libLTLIBRARIES && \
216 /sbin/ldconfig
217
218 RUN --mount=type=bind,from=ffmpeg-build,source=/usr/local,target=/ffmpeg-prefix \
219 cp -a /ffmpeg-prefix/lib/. /usr/local/lib/ && \
220 /sbin/ldconfig
221
222 ENV LD_PRELOAD=/usr/local/lib/libtcmalloc_minimal.so.4
223
224 # Install dependencies, including PyTorch and Apache Arrow
225 RUN --mount=type=cache,target=/root/.cache/uv \
226 --mount=type=bind,from=rust,source=/root/.cargo,target=/root/.cargo,rw \
227 --mount=type=bind,from=rust,source=/root/.rustup,target=/root/.rustup,rw \
228 --mount=type=bind,from=torch-vision,source=/tmp/vision/dist,target=/tmp/vision-wheels/ \
229 --mount=type=bind,from=hf-xet-builder,source=/tmp/hf-xet/dist,target=/tmp/hf-xet-wheels/ \
230 --mount=type=bind,from=numba-builder,source=/tmp/llvmlite/dist,target=/tmp/llvmlite-wheels/ \
231 --mount=type=bind,from=numba-builder,source=/tmp/numba/dist,target=/tmp/numba-wheels/ \
232 --mount=type=bind,from=opencv-builder,source=/tmp/opencv-python/dist,target=/tmp/opencv-wheels/ \
233 --mount=type=bind,from=torchcodec-builder,source=/tmp/torchcodec/dist,target=/tmp/torchcodec-wheels/ \
234 VISION_WHL_FILE=$(ls /tmp/vision-wheels/*.whl) && \
235 HF_XET_WHL_FILE=$(ls /tmp/hf-xet-wheels/*.whl) && \
236 LLVM_WHL_FILE=$(ls /tmp/llvmlite-wheels/*.whl) && \
237 NUMBA_WHL_FILE=$(ls /tmp/numba-wheels/*.whl) && \
238 OPENCV_WHL_FILE=$(ls /tmp/opencv-wheels/*.whl) && \
239 TORCHCODEC_WHL_FILE=$(ls /tmp/torchcodec-wheels/*.whl) && \
240 uv pip install -v \
241 $VISION_WHL_FILE \
242 $HF_XET_WHL_FILE \
243 $LLVM_WHL_FILE \
244 $NUMBA_WHL_FILE \
245 $OPENCV_WHL_FILE \
246 $TORCHCODEC_WHL_FILE \
247 --torch-backend cpu \
248 --index-strategy unsafe-best-match \
249 -r requirements/build/cpu.txt \
250 -r requirements/cpu.txt
251
252
253 # Build and install vllm
254 RUN --mount=type=cache,target=/root/.cache/uv \
255 VLLM_TARGET_DEVICE=cpu VLLM_CPU_MOE_PREPACK=0 python setup.py bdist_wheel && \
256 uv pip install "$(echo dist/*.whl)[tensorizer]"
257
258 # setup non-root user for vllm
259 RUN umask 002 && \
260 /usr/sbin/useradd --uid 2000 --gid 0 vllm && \
261 mkdir -p /home/vllm && \
262 chmod g+rwx /home/vllm
263
264 COPY LICENSE /licenses/vllm.md
265 COPY examples/*.jinja /app/data/template/
266
267 USER 2000
268 WORKDIR /home/vllm
269
270 # Set the default entrypoint
271 ENTRYPOINT ["vllm", "serve"]