A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 02:15 UTC
[CPU][IBM Z][Dockefile][Docs] Fix s390x builds for torch 2.11 and update docs for s390x (#39910)
Signed-off-by: Rehan Khan <Rehan.Khan7@ibm.com>
R3hankhan committed
Apr 16, 2026 at 10:56 UTC
4b7ca37bd45467c6190dc139e58554b389d80ac1
5 files changed
+72
-47
csrc/cpu/cpu_attn_impl.hpp
+3
@@ -147,6 +147,9 @@ struct AttentionMetadata {
147
case ISA::NEON:
148
ss << "NEON, ";
149
break;
150
+ case ISA::VXE:
151
+ ss << "VXE, ";
152
+ break;
153
}
154
ss << "workitem_group_num: " << workitem_group_num
155
<< ", reduction_item_num: " << reduction_item_num
csrc/cpu/utils.hpp
+22
@@ -54,12 +54,34 @@ struct Counter {
54
};
55
56
inline int64_t get_available_l2_size() {
57
+#if defined(__s390x__)
58
+ static int64_t size = []() {
59
+ uint32_t l2_cache_size = 0;
60
+ auto caps = at::cpu::get_cpu_capabilities();
61
+ auto it = caps.find("l2_cache_size");
62
+ if (it != caps.end()) {
63
+ l2_cache_size = static_cast<uint32_t>(it->second.toInt());
64
+ }
65
+ if (l2_cache_size == 0) {
66
+ long sys_l2 = sysconf(_SC_LEVEL2_CACHE_SIZE);
67
+ if (sys_l2 > 0) {
68
+ l2_cache_size = static_cast<uint32_t>(sys_l2);
69
+ }
70
+ }
71
+ if (l2_cache_size == 0) {
72
+ l2_cache_size = 256 * 1024;
73
+ }
74
+ return static_cast<int64_t>(l2_cache_size) >> 1; // use 50% of L2 cache
75
+ }();
76
+ return size;
77
+#else
78
static int64_t size = []() {
79
auto caps = at::cpu::get_cpu_capabilities();
80
const uint32_t l2_cache_size = caps.at("l2_cache_size").toInt();
81
return l2_cache_size >> 1; // use 50% of L2 cache
82
}();
83
return size;
84
+#endif
85
}
86
87
template <int32_t alignment_v, typename T>
docker/Dockerfile.s390x
+34
-34
@@ -42,7 +42,7 @@ FROM python-install AS pyarrow
42
# Build Apache Arrow
43
WORKDIR /tmp
44
RUN --mount=type=cache,target=/root/.cache/uv \
45
- git clone https://github.com/apache/arrow.git && \
45
+ git clone https://github.com/apache/arrow.git -b maint-19.0.1 && \
46
cd arrow/cpp && \
47
mkdir release && cd release && \
48
cmake -DCMAKE_BUILD_TYPE=Release \
@@ -68,19 +68,6 @@ RUN --mount=type=cache,target=/root/.cache/uv \
68
uv pip install -r requirements-build.txt && \
69
python setup.py build_ext --build-type=$ARROW_BUILD_TYPE --bundle-arrow-cpp bdist_wheel
70
71
-FROM python-install AS numa-build
72
-# Install numactl (needed for numa.h dependency)
73
-WORKDIR /tmp
74
-RUN curl -LO https://github.com/numactl/numactl/archive/refs/tags/v2.0.16.tar.gz && \
75
- tar -xvzf v2.0.16.tar.gz && \
76
- cd numactl-2.0.16 && \
77
- ./autogen.sh && \
78
- ./configure && \
79
- make
80
-
81
-# Set include path
82
-ENV C_INCLUDE_PATH="/usr/local/include:$C_INCLUDE_PATH"
83
-
71
FROM python-install AS rust
72
ENV CARGO_HOME=/root/.cargo
73
ENV RUSTUP_HOME=/root/.rustup
@@ -91,6 +78,18 @@ RUN curl https://sh.rustup.rs -sSf | sh -s -- -y && \
78
rustup default stable && \
79
rustup show
80
81
+FROM python-install AS numa-build
82
+WORKDIR /tmp
83
+RUN curl -LO https://github.com/numactl/numactl/archive/refs/tags/v2.0.19.tar.gz && \
84
+ tar -xvzf v2.0.19.tar.gz && \
85
+ cd numactl-2.0.19 && \
86
+ ./autogen.sh && \
87
+ ./configure && \
88
+ make
89
+
90
+# Set include path
91
+ENV C_INCLUDE_PATH="/usr/local/include:$C_INCLUDE_PATH"
92
+
93
FROM python-install AS torch-vision
94
# Install torchvision
95
ARG TORCH_VISION_VERSION=v0.26.0
@@ -133,7 +132,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
132
git clone --recursive https://github.com/numba/llvmlite.git -b v0.44.0 && \
133
git clone --recursive https://github.com/numba/numba.git -b ${NUMBA_VERSION} && \
134
cd llvm-project && mkdir build && cd build && \
136
- uv pip install 'cmake<4' setuptools numpy && \
135
+ uv pip install 'cmake<4' 'setuptools<70' numpy && \
136
export PREFIX=/usr/local && CMAKE_ARGS="${CMAKE_ARGS} -DLLVM_ENABLE_PROJECTS=lld;libunwind;compiler-rt" \
137
CFLAGS="$(echo $CFLAGS | sed 's/-fno-plt //g')" \
138
CXXFLAGS="$(echo $CXXFLAGS | sed 's/-fno-plt //g')" \
@@ -193,27 +192,22 @@ RUN --mount=type=cache,target=/root/.cache/uv \
192
cd opencv-python && \
193
python -m build --wheel --installer=uv --outdir /tmp/opencv-python/dist
194
196
-# Build Outlines Core
197
-FROM python-install AS outlines-core-builder
195
+## Todo(r3hankhan123): Remove guidance-builder stage once vLLM upgrades to new version of llguidance that fixes s390x issues. See https://github.com/guidance-ai/llguidance/issues/330
196
+FROM python-install AS guidance-builder
197
WORKDIR /tmp
198
ENV CARGO_HOME=/root/.cargo
199
ENV RUSTUP_HOME=/root/.rustup
200
ENV PATH="$CARGO_HOME/bin:$RUSTUP_HOME/bin:$PATH"
202
-COPY requirements/common.txt /tmp/requirements/common.txt
203
-ARG OUTLINES_CORE_VERSION
201
RUN --mount=type=cache,target=/root/.cache/uv \
202
--mount=type=bind,from=rust,source=/root/.cargo,target=/root/.cargo,rw \
203
--mount=type=bind,from=rust,source=/root/.rustup,target=/root/.rustup,rw \
207
- OUTLINES_CORE_VERSION=${OUTLINES_CORE_VERSION:-$(grep -E '^outlines_core\s*==\s*[0-9.]+' /tmp/requirements/common.txt | grep -Eo '[0-9.]+')} && \
208
- if [ -z "${OUTLINES_CORE_VERSION}" ]; then echo "ERROR: Could not determine outlines_core version"; exit 1; fi && \
209
- git clone https://github.com/dottxt-ai/outlines-core.git && \
210
- cd outlines-core && \
211
- git checkout tags/${OUTLINES_CORE_VERSION} && \
212
- sed -i "s/version = \"0.0.0\"/version = \"${OUTLINES_CORE_VERSION}\"/" Cargo.toml && \
204
+ git clone https://github.com/guidance-ai/llguidance.git && \
205
+ cd llguidance && \
206
+ git checkout s390x-fix-v2 && \
207
uv pip install maturin && \
214
- python -m maturin build --release --out dist
208
+ python -m maturin build --release --out dist --compatibility linux
209
216
-# Final build stage
210
+# # Final build stage
211
FROM python-install AS vllm-cpu
212
ARG PYTHON_VERSION
213
ARG PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu"
@@ -229,10 +223,12 @@ ENV PKG_CONFIG_PATH="/opt/rh/gcc-toolset-14/root/usr/lib64/pkgconfig:/usr/local/
223
ENV PATH="${VIRTUAL_ENV:+${VIRTUAL_ENV}/bin}:/opt/rh/gcc-toolset-14/root/usr/bin:/usr/local/bin:$CARGO_HOME/bin:$RUSTUP_HOME/bin:$PATH"
224
ENV PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
225
ENV UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
226
+# Force pure Python protobuf to avoid s390x C++ extension crashes
227
+ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
228
COPY . /workspace/vllm
229
WORKDIR /workspace/vllm
230
235
-RUN --mount=type=bind,from=numa-build,src=/tmp/numactl-2.0.16,target=/numactl \
231
+RUN --mount=type=bind,from=numa-build,src=/tmp/numactl-2.0.19,target=/numactl \
232
make -C /numactl install
233
234
# Install dependencies, including PyTorch and Apache Arrow
@@ -245,22 +241,22 @@ RUN --mount=type=cache,target=/root/.cache/uv \
241
--mount=type=bind,from=numba-builder,source=/tmp/llvmlite/dist,target=/tmp/llvmlite-wheels/ \
242
--mount=type=bind,from=numba-builder,source=/tmp/numba/dist,target=/tmp/numba-wheels/ \
243
--mount=type=bind,from=opencv-builder,source=/tmp/opencv-python/dist,target=/tmp/opencv-wheels/ \
248
- --mount=type=bind,from=outlines-core-builder,source=/tmp/outlines-core/dist,target=/tmp/outlines-core/dist/ \
249
- ARROW_WHL_FILE=$(ls /tmp/arrow-wheels/pyarrow-*.whl) && \
244
+ --mount=type=bind,from=guidance-builder,source=/tmp/llguidance/dist,target=/tmp/guidance-wheels/ \
245
+ ARROW_WHL_FILE=$(ls /tmp/arrow-wheels/*.whl) && \
246
VISION_WHL_FILE=$(ls /tmp/vision-wheels/*.whl) && \
247
HF_XET_WHL_FILE=$(ls /tmp/hf-xet-wheels/*.whl) && \
248
LLVM_WHL_FILE=$(ls /tmp/llvmlite-wheels/*.whl) && \
249
NUMBA_WHL_FILE=$(ls /tmp/numba-wheels/*.whl) && \
250
OPENCV_WHL_FILE=$(ls /tmp/opencv-wheels/*.whl) && \
255
- OUTLINES_CORE_WHL_FILE=$(ls /tmp/outlines-core/dist/*.whl) && \
256
- uv pip install -v \
257
- $ARROW_WHL_FILE \
251
+ GUIDANCE_WHL_FILE=$(ls /tmp/guidance-wheels/*.whl) && \
252
+ uv pip install -v \
253
+ $ARROW_WHL_FILE \
254
$VISION_WHL_FILE \
255
$HF_XET_WHL_FILE \
256
$LLVM_WHL_FILE \
257
$NUMBA_WHL_FILE \
258
$OPENCV_WHL_FILE \
263
- $OUTLINES_CORE_WHL_FILE \
259
+ $GUIDANCE_WHL_FILE \
260
--index-strategy unsafe-best-match \
261
-r requirements/build/cpu.txt \
262
-r requirements/cpu.txt
@@ -271,6 +267,10 @@ RUN --mount=type=cache,target=/root/.cache/uv \
267
VLLM_TARGET_DEVICE=cpu VLLM_CPU_MOE_PREPACK=0 python setup.py bdist_wheel && \
268
uv pip install "$(echo dist/*.whl)[tensorizer]"
269
270
+# Remove protobuf C++ extension that crashes on s390x
271
+RUN rm -rf /opt/vllm/lib64/python${PYTHON_VERSION}/site-packages/google/_upb/*.so \
272
+ /opt/vllm/lib64/python${PYTHON_VERSION}/site-packages/google/protobuf/pyext/*.so 2>/dev/null || true
273
+
274
# setup non-root user for vllm
275
RUN umask 002 && \
276
/usr/sbin/useradd --uid 2000 --gid 0 vllm && \
docs/getting_started/installation/cpu.s390x.inc.md
+12
-12
@@ -3,15 +3,15 @@
3
4
vLLM has experimental support for s390x architecture on IBM Z platform. For now, users must build from source to natively run on IBM Z platform.
5
6
-Currently, the CPU implementation for s390x architecture supports FP32 datatype only.
6
+Currently, the CPU implementation for s390x architecture supports FP32, BF16 and FP16.
7
8
--8<-- [end:installation]
9
--8<-- [start:requirements]
10
11
- OS: `Linux`
12
-- SDK: `gcc/g++ >= 12.3.0` or later with Command Line Tools
12
+- SDK: `gcc/g++ >= 14.0.0` or later with Command Line Tools
13
- Instruction Set Architecture (ISA): VXE support is required. Works with Z14 and above.
14
-- Build install python packages: `pyarrow`, `torch` and `torchvision`
14
+- Build install python packages: `torchvision`, `llvmlite`, `numba`, `pyarrow (for testing)`, `opencv-headless`
15
16
--8<-- [end:requirements]
17
--8<-- [start:set-up-using-python]
@@ -24,13 +24,14 @@ Currently, there are no pre-built IBM Z CPU wheels.
24
--8<-- [end:pre-built-wheels]
25
--8<-- [start:build-wheel-from-source]
26
27
-Install the following packages from the package manager before building the vLLM. For example on RHEL 9.4:
27
+Install the following packages from the package manager before building the vLLM. For example on RHEL 9.6:
28
29
```bash
30
dnf install -y \
31
- which procps findutils tar vim git gcc g++ make patch make cython zlib-devel \
31
+ which procps findutils tar vim git gcc-toolset-14 gcc-toolset-14-binutils gcc-toolset-14-libatomic-devel zlib-devel \
32
libjpeg-turbo-devel libtiff-devel libpng-devel libwebp-devel freetype-devel harfbuzz-devel \
33
- openssl-devel openblas openblas-devel wget autoconf automake libtool cmake numactl-devel
33
+ openssl-devel openblas openblas-devel autoconf automake libtool cmake numpy libsndfile \
34
+ clang llvm-devel llvm-static clang-devel
35
```
36
37
Install rust>=1.80 which is needed for `outlines-core` and `uvloop` python packages installation.
@@ -43,13 +44,13 @@ curl https://sh.rustup.rs -sSf | sh -s -- -y && \
44
Execute the following commands to build and install vLLM from source.
45
46
!!! tip
46
- Please build the following dependencies, `torchvision`, `pyarrow` from source before building vLLM.
47
+ Please build the following dependencies, `torchvision`, `llvmlite`, `numba`, `llguidance`, `pyarrow`, `opencv-headless` from source before building vLLM.
48
49
```bash
49
- sed -i '/^torch/d' requirements/build/cuda.txt # remove torch from requirements/build/cuda.txt since we use nightly builds
50
uv pip install -v \
51
+ --extra-index-url https://download.pytorch.org/whl/cpu \
52
--torch-backend auto \
52
- -r requirements/build/cuda.txt \
53
+ -r requirements/build/cpu.txt \
54
-r requirements/cpu.txt \
55
VLLM_TARGET_DEVICE=cpu python setup.py bdist_wheel && \
56
uv pip install dist/*.whl
@@ -57,10 +58,9 @@ Execute the following commands to build and install vLLM from source.
58
59
??? console "pip"
60
```bash
60
- sed -i '/^torch/d' requirements/build/cuda.txt # remove torch from requirements/build/cuda.txt since we use nightly builds
61
pip install -v \
62
- --extra-index-url https://download.pytorch.org/whl/nightly/cpu \
63
- -r requirements/build/cuda.txt \
62
+ --extra-index-url https://download.pytorch.org/whl/cpu \
63
+ -r requirements/build/cpu.txt \
64
-r requirements/cpu.txt \
65
VLLM_TARGET_DEVICE=cpu python setup.py bdist_wheel && \
66
pip install dist/*.whl
requirements/common.txt
+1
-1
@@ -19,7 +19,7 @@ pillow # Required for image processing
19
prometheus-fastapi-instrumentator >= 7.0.0
20
tiktoken >= 0.6.0 # Required for DBRX tokenizer
21
lm-format-enforcer == 0.11.3
22
-llguidance >= 1.3.0, < 1.4.0; platform_machine == "x86_64" or platform_machine == "arm64" or platform_machine == "aarch64" or platform_machine == "s390x" or platform_machine == "ppc64le"
22
+llguidance >= 1.3.0, < 1.4.0; platform_machine == "x86_64" or platform_machine == "arm64" or platform_machine == "aarch64" or platform_machine == "ppc64le"
23
outlines_core == 0.2.11
24
# required for outlines backend disk cache
25
diskcache == 5.6.3