A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 06:15 UTC
[CPU][ARM] Add ARM BF16 cross-compilation support and improve documen… (#33079)
Signed-off-by: Maryam Tahhan <mtahhan@redhat.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
Maryam Tahhan committed
Feb 15, 2026 at 14:33 UTC
f07a128413ff6900f407e46fe2b36d93eb2a0c12
3 files changed
+81
-7
cmake/cpu_extension.cmake
+5
@@ -18,6 +18,7 @@ set(ENABLE_AVX512 $ENV{VLLM_CPU_AVX512})
18
set(ENABLE_AVX512BF16 $ENV{VLLM_CPU_AVX512BF16})
19
set(ENABLE_AVX512VNNI $ENV{VLLM_CPU_AVX512VNNI})
20
set(ENABLE_AMXBF16 $ENV{VLLM_CPU_AMXBF16})
21
+set(ENABLE_ARM_BF16 $ENV{VLLM_CPU_ARM_BF16})
22
23
include_directories("${CMAKE_SOURCE_DIR}/csrc")
24
@@ -115,6 +116,10 @@ else()
116
set(AVX512_FOUND ON)
117
message(STATUS "AVX512 support enabled via VLLM_CPU_AVX512 environment variable")
118
endif()
119
+ if (ENABLE_ARM_BF16)
120
+ set(ARM_BF16_FOUND ON)
121
+ message(STATUS "ARM BF16 support enabled via VLLM_CPU_ARM_BF16 environment variable")
122
+ endif()
123
endif()
124
125
if (AVX512_FOUND AND NOT AVX512_DISABLED)
docker/Dockerfile.cpu
+16
@@ -20,6 +20,7 @@
20
# VLLM_CPU_AVX512BF16=false (default)|true (for cross-compilation)
21
# VLLM_CPU_AVX512VNNI=false (default)|true (for cross-compilation)
22
# VLLM_CPU_AMXBF16=false (default)|true (for cross-compilation)
23
+# VLLM_CPU_ARM_BF16=false (default)|true (for cross-compilation)
24
#
25
26
######################### COMMON BASE IMAGE #########################
@@ -108,9 +109,22 @@ ENV VLLM_CPU_AVX512VNNI=${VLLM_CPU_AVX512VNNI}
109
# Support for building with AMXBF16 ISA: docker build --build-arg VLLM_CPU_AMXBF16="true" ...
110
ARG VLLM_CPU_AMXBF16=1
111
ENV VLLM_CPU_AMXBF16=${VLLM_CPU_AMXBF16}
112
+# Support for cross-compilation with ARM BF16 ISA: docker build --build-arg VLLM_CPU_ARM_BF16="true" ...
113
+ARG VLLM_CPU_ARM_BF16=0
114
+ENV VLLM_CPU_ARM_BF16=${VLLM_CPU_ARM_BF16}
115
116
WORKDIR /vllm-workspace
117
118
+# Validate build arguments - prevent mixing incompatible ISA flags
119
+RUN if [ "$TARGETARCH" = "arm64" ] && { [ "$VLLM_CPU_AVX2" != "0" ] || [ "$VLLM_CPU_AVX512" != "0" ] || [ "$VLLM_CPU_AVX512BF16" != "0" ] || [ "$VLLM_CPU_AVX512VNNI" != "0" ]; }; then \
120
+ echo "ERROR: Cannot use x86-specific ISA flags (AVX2, AVX512, etc.) when building for ARM64 (--platform=linux/arm64)"; \
121
+ exit 1; \
122
+ fi && \
123
+ if [ "$TARGETARCH" = "amd64" ] && [ "$VLLM_CPU_ARM_BF16" != "0" ]; then \
124
+ echo "ERROR: Cannot use ARM-specific ISA flags (ARM_BF16) when building for x86_64 (--platform=linux/amd64)"; \
125
+ exit 1; \
126
+ fi
127
+
128
# Copy build requirements
129
COPY requirements/cpu-build.txt requirements/build.txt
130
@@ -224,6 +238,7 @@ ARG VLLM_CPU_AVX512
238
ARG VLLM_CPU_AVX512BF16
239
ARG VLLM_CPU_AVX512VNNI
240
ARG VLLM_CPU_AMXBF16
241
+ARG VLLM_CPU_ARM_BF16
242
ARG PYTHON_VERSION
243
244
LABEL ai.vllm.build.target-arch="${TARGETARCH}"
@@ -233,6 +248,7 @@ LABEL ai.vllm.build.cpu-avx512="${VLLM_CPU_AVX512:-false}"
248
LABEL ai.vllm.build.cpu-avx512bf16="${VLLM_CPU_AVX512BF16:-false}"
249
LABEL ai.vllm.build.cpu-avx512vnni="${VLLM_CPU_AVX512VNNI:-false}"
250
LABEL ai.vllm.build.cpu-amxbf16="${VLLM_CPU_AMXBF16:-false}"
251
+LABEL ai.vllm.build.cpu-arm-bf16="${VLLM_CPU_ARM_BF16:-false}"
252
LABEL ai.vllm.build.python-version="${PYTHON_VERSION:-3.12}"
253
254
ENTRYPOINT ["vllm", "serve"]
docs/getting_started/installation/cpu.arm.inc.md
+60
-7
@@ -172,25 +172,78 @@ docker pull public.ecr.aws/q9t5s3a7/vllm-ci-postmerge-repo:${VLLM_COMMIT}-arm64-
172
173
# --8<-- [end:pre-built-images]
174
# --8<-- [start:build-image-from-source]
175
+
176
+## Building for your target ARM CPU
177
+
178
+```bash
179
+docker build -f docker/Dockerfile.cpu \
180
+ --platform=linux/arm64 \
181
+ --build-arg VLLM_CPU_ARM_BF16=<false (default)|true> \
182
+ --tag vllm-cpu-env \
183
+ --target vllm-openai .
184
+```
185
+
186
+!!! note "Auto-detection by default"
187
+ By default, ARM CPU instruction sets (BF16, NEON, etc.) are automatically detected from the build system's CPU flags. The `VLLM_CPU_ARM_BF16` build argument is used for cross-compilation:
188
+
189
+ - `VLLM_CPU_ARM_BF16=true` - Force-enable ARM BF16 support (build with BF16 regardless of build system capabilities)
190
+ - `VLLM_CPU_ARM_BF16=false` - Rely on auto-detection (default)
191
+
192
+### Examples
193
+
194
+**Auto-detection build (native ARM)**
195
+
196
```bash
197
+# Building on ARM64 system - platform auto-detected
198
docker build -f docker/Dockerfile.cpu \
177
- --tag vllm-cpu-env .
199
+ --tag vllm-cpu-arm64 \
200
+ --target vllm-openai .
201
+```
202
+
203
+**Cross-compile for ARM with BF16 support**
204
179
-# Launching OpenAI server
205
+```bash
206
+# Building on ARM64 for newer ARM CPUs with BF16
207
+docker build -f docker/Dockerfile.cpu \
208
+ --build-arg VLLM_CPU_ARM_BF16=true \
209
+ --tag vllm-cpu-arm64-bf16 \
210
+ --target vllm-openai .
211
+```
212
+
213
+**Cross-compile from x86_64 to ARM64 with BF16**
214
+
215
+```bash
216
+# Requires Docker buildx with ARM emulation (QEMU)
217
+docker buildx build -f docker/Dockerfile.cpu \
218
+ --platform=linux/arm64 \
219
+ --build-arg VLLM_CPU_ARM_BF16=true \
220
+ --build-arg max_jobs=4 \
221
+ --tag vllm-cpu-arm64-bf16 \
222
+ --target vllm-openai \
223
+ --load .
224
+```
225
+
226
+!!! note "ARM BF16 requirements"
227
+ ARM BF16 support requires ARMv8.6-A or later (FEAT_BF16). Supported on AWS Graviton3/4, AmpereOne, and other recent ARM processors.
228
+
229
+## Launching the OpenAI server
230
+
231
+```bash
232
docker run --rm \
181
- --privileged=true \
233
+ --security-opt seccomp=unconfined \
234
+ --cap-add SYS_NICE \
235
--shm-size=4g \
236
-p 8000:8000 \
237
-e VLLM_CPU_KVCACHE_SPACE=<KV cache space> \
238
-e VLLM_CPU_OMP_THREADS_BIND=<CPU cores for inference> \
186
- vllm-cpu-env \
187
- --model=meta-llama/Llama-3.2-1B-Instruct \
239
+ vllm-cpu-arm64 \
240
+ meta-llama/Llama-3.2-1B-Instruct \
241
--dtype=bfloat16 \
242
other vLLM OpenAI server arguments
243
```
244
192
-!!! tip
193
- An alternative of `--privileged=true` is `--cap-add SYS_NICE --security-opt seccomp=unconfined`.
245
+!!! tip "Alternative to --privileged"
246
+ Instead of `--privileged=true`, use `--cap-add SYS_NICE --security-opt seccomp=unconfined` for better security.
247
248
# --8<-- [end:build-image-from-source]
249
# --8<-- [start:extra-information]