A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Hardware][IBM Z]Enable v1 for s390x and s390x dockerfile fixes (#22725)
Signed-off-by: Nikhil Suryawanshi <suryawanshin74@gmail.com>
Nikhil Suryawanshi committed
Aug 19, 2025 at 10:10 UTC
78dba404adaa2b9814a4480249e2d9c0512364c1
7 files changed
+96
-18
docker/Dockerfile.s390x
+79
-8
@@ -16,7 +16,7 @@ ENV LANG=C.UTF-8 \
16
RUN microdnf install -y \
17
which procps findutils tar vim git gcc gcc-gfortran g++ make patch zlib-devel \
18
libjpeg-turbo-devel libtiff-devel libpng-devel libwebp-devel freetype-devel harfbuzz-devel \
19
- openssl-devel openblas openblas-devel autoconf automake libtool cmake numpy && \
19
+ openssl-devel openblas openblas-devel autoconf automake libtool cmake numpy libsndfile && \
20
microdnf clean all
21
22
# Python Installation
@@ -136,6 +136,71 @@ RUN --mount=type=cache,target=/root/.cache/uv \
136
mkdir -p /tmp/hf-xet/dist && \
137
cp dist/*.whl /tmp/hf-xet/dist/
138
139
+# Build numba
140
+FROM python-install AS numba-builder
141
+
142
+ARG MAX_JOBS
143
+ARG NUMBA_VERSION=0.61.2
144
+
145
+WORKDIR /tmp
146
+
147
+# Clone all required dependencies
148
+RUN --mount=type=cache,target=/root/.cache/uv \
149
+ microdnf install ninja-build gcc gcc-c++ -y && \
150
+ git clone --recursive https://github.com/llvm/llvm-project.git -b llvmorg-15.0.7 && \
151
+ git clone --recursive https://github.com/numba/llvmlite.git -b v0.44.0 && \
152
+ git clone --recursive https://github.com/numba/numba.git -b ${NUMBA_VERSION} && \
153
+ cd llvm-project && mkdir build && cd build && \
154
+ uv pip install 'cmake<4' setuptools numpy && \
155
+ export PREFIX=/usr/local && CMAKE_ARGS="${CMAKE_ARGS} -DLLVM_ENABLE_PROJECTS=lld;libunwind;compiler-rt" \
156
+ CFLAGS="$(echo $CFLAGS | sed 's/-fno-plt //g')" \
157
+ CXXFLAGS="$(echo $CXXFLAGS | sed 's/-fno-plt //g')" \
158
+ CMAKE_ARGS="${CMAKE_ARGS} -DFFI_INCLUDE_DIR=$PREFIX/include" \
159
+ CMAKE_ARGS="${CMAKE_ARGS} -DFFI_LIBRARY_DIR=$PREFIX/lib" \
160
+ cmake -DCMAKE_INSTALL_PREFIX="${PREFIX}" \
161
+ -DCMAKE_BUILD_TYPE=Release \
162
+ -DCMAKE_LIBRARY_PATH="${PREFIX}" \
163
+ -DLLVM_ENABLE_LIBEDIT=OFF \
164
+ -DLLVM_ENABLE_LIBXML2=OFF \
165
+ -DLLVM_ENABLE_RTTI=ON \
166
+ -DLLVM_ENABLE_TERMINFO=OFF \
167
+ -DLLVM_INCLUDE_BENCHMARKS=OFF \
168
+ -DLLVM_INCLUDE_DOCS=OFF \
169
+ -DLLVM_INCLUDE_EXAMPLES=OFF \
170
+ -DLLVM_INCLUDE_GO_TESTS=OFF \
171
+ -DLLVM_INCLUDE_TESTS=OFF \
172
+ -DLLVM_INCLUDE_UTILS=ON \
173
+ -DLLVM_INSTALL_UTILS=ON \
174
+ -DLLVM_UTILS_INSTALL_DIR=libexec/llvm \
175
+ -DLLVM_BUILD_LLVM_DYLIB=OFF \
176
+ -DLLVM_LINK_LLVM_DYLIB=OFF \
177
+ -DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=WebAssembly \
178
+ -DLLVM_ENABLE_FFI=ON \
179
+ -DLLVM_ENABLE_Z3_SOLVER=OFF \
180
+ -DLLVM_OPTIMIZED_TABLEGEN=ON \
181
+ -DCMAKE_POLICY_DEFAULT_CMP0111=NEW \
182
+ -DCOMPILER_RT_BUILD_BUILTINS=ON \
183
+ -DCOMPILER_RT_BUILTINS_HIDE_SYMBOLS=OFF \
184
+ -DCOMPILER_RT_BUILD_LIBFUZZER=OFF \
185
+ -DCOMPILER_RT_BUILD_CRT=OFF \
186
+ -DCOMPILER_RT_BUILD_MEMPROF=OFF \
187
+ -DCOMPILER_RT_BUILD_PROFILE=OFF \
188
+ -DCOMPILER_RT_BUILD_SANITIZERS=OFF \
189
+ -DCOMPILER_RT_BUILD_XRAY=OFF \
190
+ -DCOMPILER_RT_BUILD_GWP_ASAN=OFF \
191
+ -DCOMPILER_RT_BUILD_ORC=OFF \
192
+ -DCOMPILER_RT_INCLUDE_TESTS=OFF \
193
+ ${CMAKE_ARGS} -GNinja ../llvm \
194
+
195
+ && ninja install . && \
196
+ # build llvmlite
197
+ cd ../../llvmlite && python setup.py bdist_wheel && \
198
+ cd ../numba && \
199
+ if ! grep '#include "dynamic_annotations.h"' numba/_dispatcher.cpp; then \
200
+ sed -i '/#include "internal\/pycore_atomic.h"/i\#include "dynamic_annotations.h"' numba/_dispatcher.cpp; \
201
+ fi && python setup.py bdist_wheel
202
+
203
+
204
# Final build stage
205
FROM python-install AS vllm-cpu
206
ARG PYTHON_VERSION
@@ -163,23 +228,30 @@ RUN --mount=type=cache,target=/root/.cache/uv \
228
--mount=type=bind,from=torch-vision,source=/tmp/vision/dist,target=/tmp/vision-wheels/ \
229
--mount=type=bind,from=hf-xet-builder,source=/tmp/hf-xet/dist,target=/tmp/hf-xet-wheels/ \
230
--mount=type=bind,from=torch,source=/tmp/pytorch/dist,target=/tmp/torch-wheels/ \
231
+ --mount=type=bind,from=numba-builder,source=/tmp/llvmlite/dist,target=/tmp/llvmlite-wheels/ \
232
+ --mount=type=bind,from=numba-builder,source=/tmp/numba/dist,target=/tmp/numba-wheels/ \
233
sed -i '/^torch/d' requirements/build.txt && \
167
- ARROW_WHL_FILE=$(ls /tmp/arrow-wheels/pyarrow-*.whl | head -n 1) && \
168
- VISION_WHL_FILE=$(ls /tmp/vision-wheels/*.whl | head -n 1) && \
169
- HF_XET_WHL_FILE=$(ls /tmp/hf-xet-wheels/*.whl | head -n 1) && \
170
- TORCH_WHL_FILE=$(ls /tmp/torch-wheels/*.whl | head -n 1) && \
234
+ ARROW_WHL_FILE=$(ls /tmp/arrow-wheels/pyarrow-*.whl) && \
235
+ VISION_WHL_FILE=$(ls /tmp/vision-wheels/*.whl) && \
236
+ HF_XET_WHL_FILE=$(ls /tmp/hf-xet-wheels/*.whl) && \
237
+ TORCH_WHL_FILE=$(ls /tmp/torch-wheels/*.whl) && \
238
+ LLVM_WHL_FILE=$(ls /tmp/llvmlite-wheels/*.whl) && \
239
+ NUMBA_WHL_FILE=$(ls /tmp/numba-wheels/*.whl) && \
240
uv pip install -v \
241
$ARROW_WHL_FILE \
242
$VISION_WHL_FILE \
243
$HF_XET_WHL_FILE \
244
$TORCH_WHL_FILE \
245
+ $LLVM_WHL_FILE \
246
+ $NUMBA_WHL_FILE \
247
--index-strategy unsafe-best-match \
248
-r requirements/build.txt \
178
- -r requirements/cpu.txt
249
+ -r requirements/cpu.txt
250
+
251
252
# Build and install vllm
253
RUN --mount=type=cache,target=/root/.cache/uv \
182
- VLLM_TARGET_DEVICE=cpu python setup.py bdist_wheel && \
254
+ VLLM_TARGET_DEVICE=cpu VLLM_CPU_MOE_PREPACK=0 python setup.py bdist_wheel && \
255
uv pip install "$(echo dist/*.whl)[tensorizer]"
256
257
# setup non-root user for vllm
@@ -196,4 +268,3 @@ WORKDIR /home/vllm
268
269
# Set the default entrypoint
270
ENTRYPOINT ["python", "-m", "vllm.entrypoints.openai.api_server"]
199
-
requirements/common.txt
+2
-1
@@ -20,7 +20,8 @@ prometheus-fastapi-instrumentator >= 7.0.0
20
tiktoken >= 0.6.0 # Required for DBRX tokenizer
21
lm-format-enforcer >= 0.10.11, < 0.11
22
llguidance >= 0.7.11, < 0.8.0; platform_machine == "x86_64" or platform_machine == "arm64" or platform_machine == "aarch64"
23
-outlines_core == 0.2.10
23
+outlines_core == 0.2.10 ; platform_machine != "s390x"
24
+outlines == 0.1.11 ; platform_machine == "s390x"
25
# required for outlines backend disk cache
26
diskcache == 5.6.3
27
lark == 1.2.2
requirements/cpu.txt
+2
-2
@@ -1,8 +1,8 @@
1
# Common dependencies
2
-r common.txt
3
4
-numba == 0.60.0; python_version == '3.9' # v0.61 doesn't support Python 3.9. Required for N-gram speculative decoding
5
-numba == 0.61.2; python_version > '3.9'
4
+numba == 0.60.0; python_version == '3.9' and platform_machine != "s390x" # v0.61 doesn't support Python 3.9. Required for N-gram speculative decoding
5
+numba == 0.61.2; python_version > '3.9' and platform_machine != "s390x"
6
7
# Dependencies for CPUs
8
packaging>=24.2
vllm/engine/arg_utils.py
+4
-3
@@ -1076,12 +1076,13 @@ class EngineArgs:
1076
# Set default arguments for V0 or V1 Engine.
1077
if use_v1:
1078
self._set_default_args_v1(usage_context, model_config)
1079
- # Disable chunked prefill for POWER (ppc64le)/ARM CPUs in V1
1079
+ # Disable chunked prefill for POWER (ppc64le)/ARM/s390x CPUs in V1
1080
if current_platform.is_cpu(
1081
) and current_platform.get_cpu_architecture() in (
1082
- CpuArchEnum.POWERPC, CpuArchEnum.ARM):
1082
+ CpuArchEnum.POWERPC, CpuArchEnum.S390X, CpuArchEnum.ARM):
1083
logger.info(
1084
- "Chunked prefill is not supported for ARM and POWER CPUs; "
1084
+ "Chunked prefill is not supported for ARM and POWER "
1085
+ "and S390X CPUs; "
1086
"disabling it for V1 backend.")
1087
self.enable_chunked_prefill = False
1088
else:
vllm/platforms/cpu.py
+3
-2
@@ -332,5 +332,6 @@ class CpuPlatform(Platform):
332
supplied model configuration.
333
"""
334
arch = cls.get_cpu_architecture()
335
- return (cls.supports_v1(model_config) and arch
336
- in (CpuArchEnum.X86, CpuArchEnum.POWERPC, CpuArchEnum.ARM))
335
+ return (cls.supports_v1(model_config)
336
+ and arch in (CpuArchEnum.X86, CpuArchEnum.POWERPC,
337
+ CpuArchEnum.ARM, CpuArchEnum.S390X))
vllm/platforms/interface.py
+3
@@ -81,6 +81,7 @@ class CpuArchEnum(enum.Enum):
81
X86 = enum.auto()
82
ARM = enum.auto()
83
POWERPC = enum.auto()
84
+ S390X = enum.auto()
85
OTHER = enum.auto()
86
UNKNOWN = enum.auto()
87
@@ -377,6 +378,8 @@ class Platform:
378
return CpuArchEnum.ARM
379
elif machine.startswith("ppc"):
380
return CpuArchEnum.POWERPC
381
+ elif machine == "s390x":
382
+ return CpuArchEnum.S390X
383
384
return CpuArchEnum.OTHER if machine else CpuArchEnum.UNKNOWN
385
vllm/v1/worker/cpu_worker.py
+3
-2
@@ -43,8 +43,9 @@ class CPUWorker(Worker):
43
# Setup OpenMP threads affinity.
44
omp_cpuids = envs.VLLM_CPU_OMP_THREADS_BIND
45
if omp_cpuids == "auto" and platform.system() == "Linux":
46
- if current_platform.get_cpu_architecture() == CpuArchEnum.POWERPC:
47
- # For POWERPC SMT-8/4/2
46
+ cpu_arch = current_platform.get_cpu_architecture()
47
+ if cpu_arch in (CpuArchEnum.POWERPC, CpuArchEnum.S390X):
48
+ # For S390X/POWERPC SMT-8/4/2
49
self.local_omp_cpuid = self._get_autobind_cpu_ids(
50
lambda cpus: [cpu for cpu in cpus if cpu.id % 8 < 4])
51
elif current_platform.get_cpu_architecture() == CpuArchEnum.X86: