A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 18:15 UTC
[UX] Improve UX of CPU backend (#36968)
Signed-off-by: jiang1.li <jiang1.li@intel.com> Signed-off-by: Li, Jiang <bigpyj64@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Li, Jiang committed
Mar 14, 2026 at 09:27 UTC
092ace9e3a21f90c9f4aba8defe69ecff4bab628
10 files changed
+173
-117
.buildkite/hardware_tests/cpu.yaml
+14
@@ -21,6 +21,20 @@ steps:
21
pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py
22
pytest -x -v -s tests/kernels/test_onednn.py"
23
24
+- label: CPU-Compatibility Tests
25
+ depends_on: []
26
+ soft_fail: true
27
+ device: intel_cpu
28
+ no_plugin: true
29
+ source_file_dependencies:
30
+ - cmake/cpu_extension.cmake
31
+ - setup.py
32
+ - vllm/platforms/cpu.py
33
+ commands:
34
+ - |
35
+ bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 20m "
36
+ bash .buildkite/scripts/hardware_ci/run-cpu-compatibility-test.sh"
37
+
38
- label: CPU-Language Generation and Pooling Model Tests
39
depends_on: []
40
soft_fail: true
.buildkite/image_build/image_build_cpu.sh
+1
-3
@@ -25,9 +25,7 @@ fi
25
docker build --file docker/Dockerfile.cpu \
26
--build-arg max_jobs=16 \
27
--build-arg buildkite_commit="$BUILDKITE_COMMIT" \
28
- --build-arg VLLM_CPU_AVX512BF16=true \
29
- --build-arg VLLM_CPU_AVX512VNNI=true \
30
- --build-arg VLLM_CPU_AMXBF16=true \
28
+ --build-arg VLLM_CPU_X86=true \
29
--tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-cpu \
30
--target vllm-test \
31
--progress plain .
.buildkite/release-pipeline.yaml
+2
-2
@@ -83,7 +83,7 @@ steps:
83
agents:
84
queue: cpu_queue_postmerge
85
commands:
86
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --build-arg VLLM_CPU_AVX512BF16=true --build-arg VLLM_CPU_AVX512VNNI=true --build-arg VLLM_CPU_AMXBF16=true --tag vllm-ci:build-image --target vllm-build --progress plain -f docker/Dockerfile.cpu ."
86
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --build-arg VLLM_CPU_X86=true --tag vllm-ci:build-image --target vllm-build --progress plain -f docker/Dockerfile.cpu ."
87
- "mkdir artifacts"
88
- "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
89
- "bash .buildkite/scripts/upload-nightly-wheels.sh manylinux_2_35"
@@ -152,7 +152,7 @@ steps:
152
queue: cpu_queue_postmerge
153
commands:
154
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
155
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --build-arg VLLM_CPU_AVX512BF16=true --build-arg VLLM_CPU_AVX512VNNI=true --build-arg VLLM_CPU_AMXBF16=true --tag public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:$(buildkite-agent meta-data get release-version) --tag public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:latest --progress plain --target vllm-openai -f docker/Dockerfile.cpu ."
155
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --build-arg VLLM_CPU_X86=true --tag public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:$(buildkite-agent meta-data get release-version) --tag public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:latest --progress plain --target vllm-openai -f docker/Dockerfile.cpu ."
156
- "docker push public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:latest"
157
- "docker push public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:$(buildkite-agent meta-data get release-version)"
158
env:
.buildkite/scripts/hardware_ci/run-cpu-compatibility-test.sh
new
+65
@@ -0,0 +1,65 @@
1
+#!/bin/bash
2
+set -euox pipefail
3
+
4
+export VLLM_CPU_KVCACHE_SPACE=1
5
+export VLLM_CPU_CI_ENV=1
6
+# Reduce sub-processes for acceleration
7
+export TORCH_COMPILE_DISABLE=1
8
+export VLLM_ENABLE_V1_MULTIPROCESSING=0
9
+
10
+SDE_ARCHIVE="sde-external-10.7.0-2026-02-18-lin.tar.xz"
11
+SDE_CHECKSUM="CA3D4086DE4ACB3FAEDF9F57B541C6936B7D5E19AE2BF763B6EA933573A0A217"
12
+wget "https://downloadmirror.intel.com/913594/${SDE_ARCHIVE}"
13
+echo "${SDE_CHECKSUM} ${SDE_ARCHIVE}" | sha256sum --check
14
+mkdir -p sde
15
+tar -xvf "./${SDE_ARCHIVE}" --strip-components=1 -C ./sde/
16
+
17
+wait_for_pid_and_check_log() {
18
+ local pid="$1"
19
+ local log_file="$2"
20
+ local exit_status
21
+
22
+ if [ -z "$pid" ] || [ -z "$log_file" ]; then
23
+ echo "Usage: wait_for_pid_and_check_log <PID> <LOG_FILE>"
24
+ return 1
25
+ fi
26
+
27
+ echo "Waiting for process $pid to finish..."
28
+
29
+ # Use the 'wait' command to pause the script until the specific PID exits.
30
+ # The 'wait' command's own exit status will be that of the waited-for process.
31
+ if wait "$pid"; then
32
+ exit_status=$?
33
+ echo "Process $pid finished with exit status $exit_status (Success)."
34
+ else
35
+ exit_status=$?
36
+ echo "Process $pid finished with exit status $exit_status (Failure)."
37
+ fi
38
+
39
+ if [ "$exit_status" -ne 0 ]; then
40
+ echo "Process exited with a non-zero status."
41
+ echo "--- Last few lines of log file: $log_file ---"
42
+ tail -n 50 "$log_file"
43
+ echo "---------------------------------------------"
44
+ return 1 # Indicate failure based on exit status
45
+ fi
46
+
47
+ echo "No errors detected in log file and process exited successfully."
48
+ return 0
49
+}
50
+
51
+# Test Sky Lake (AVX512F)
52
+./sde/sde64 -skl -- python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --dtype bfloat16 > test_0.log 2>&1 &
53
+PID_TEST_0=$!
54
+
55
+# Test Cascade Lake (AVX512F + VNNI)
56
+./sde/sde64 -clx -- python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --dtype bfloat16 > test_1.log 2>&1 &
57
+PID_TEST_1=$!
58
+
59
+# Test Cooper Lake (AVX512F + VNNI + BF16)
60
+./sde/sde64 -cpx -- python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --dtype bfloat16 > test_2.log 2>&1 &
61
+PID_TEST_2=$!
62
+
63
+wait_for_pid_and_check_log $PID_TEST_0 test_0.log
64
+wait_for_pid_and_check_log $PID_TEST_1 test_1.log
65
+wait_for_pid_and_check_log $PID_TEST_2 test_2.log
cmake/cpu_extension.cmake
+36
-14
@@ -102,11 +102,13 @@ if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64" OR ENABLE_X86_ISA)
102
"-mavx512f"
103
"-mavx512vl"
104
"-mavx512bw"
105
- "-mavx512dq"
106
- "-mavx512bf16"
107
- "-mavx512vnni"
105
+ "-mavx512dq")
106
+ list(APPEND CXX_COMPILE_FLAGS_AVX512_AMX
107
+ ${CXX_COMPILE_FLAGS_AVX512}
108
"-mamx-bf16"
109
- "-mamx-tile")
109
+ "-mamx-tile"
110
+ "-mavx512bf16"
111
+ "-mavx512vnni")
112
list(APPEND CXX_COMPILE_FLAGS_AVX2
113
"-mavx2")
114
elseif (POWER9_FOUND OR POWER10_FOUND OR POWER11_FOUND)
@@ -314,7 +316,8 @@ endif()
316
317
# TODO: Refactor this
318
if (ENABLE_X86_ISA)
317
- message(STATUS "CPU extension (AVX512) compile flags: ${CXX_COMPILE_FLAGS_AVX512}")
319
+ message(STATUS "CPU extension (AVX512F + BF16 + VNNI + AMX) compile flags: ${CXX_COMPILE_FLAGS_AVX512_AMX}")
320
+ message(STATUS "CPU extension (AVX512F) compile flags: ${CXX_COMPILE_FLAGS_AVX512}")
321
message(STATUS "CPU extension (AVX2) compile flags: ${CXX_COMPILE_FLAGS_AVX2}")
322
else()
323
message(STATUS "CPU extension compile flags: ${CXX_COMPILE_FLAGS}")
@@ -366,13 +369,15 @@ if(USE_ONEDNN)
369
endif()
370
371
if (ENABLE_X86_ISA)
369
- set(VLLM_EXT_SRC_AVX512
372
+ set(VLLM_EXT_SRC_SGL
373
"csrc/cpu/sgl-kernels/gemm.cpp"
374
"csrc/cpu/sgl-kernels/gemm_int8.cpp"
375
"csrc/cpu/sgl-kernels/gemm_fp8.cpp"
376
"csrc/cpu/sgl-kernels/moe.cpp"
377
"csrc/cpu/sgl-kernels/moe_int8.cpp"
375
- "csrc/cpu/sgl-kernels/moe_fp8.cpp"
378
+ "csrc/cpu/sgl-kernels/moe_fp8.cpp")
379
+
380
+ set(VLLM_EXT_SRC_AVX512
381
"csrc/cpu/shm.cpp"
382
"csrc/cpu/cpu_wna16.cpp"
383
"csrc/cpu/cpu_fused_moe.cpp"
@@ -398,31 +403,48 @@ if (ENABLE_X86_ISA)
403
"csrc/cpu/pos_encoding.cpp"
404
"csrc/moe/dynamic_4bit_int_moe_cpu.cpp")
405
401
- message(STATUS "CPU extension (AVX512) source files: ${VLLM_EXT_SRC_AVX512}")
406
+ message(STATUS "CPU extension (AVX512F + BF16 + VNNI + AMX) source files: ${VLLM_EXT_SRC_AVX512} ${VLLM_EXT_SRC_SGL}")
407
+ message(STATUS "CPU extension (AVX512F) source files: ${VLLM_EXT_SRC_AVX512}")
408
message(STATUS "CPU extension (AVX2) source files: ${VLLM_EXT_SRC_AVX2}")
409
410
+ set(_C_LIBS numa dnnl_ext)
411
+ set(_C_AVX512_LIBS numa dnnl_ext)
412
+ set(_C_AVX2_LIBS numa)
413
+
414
+ # AMX + AVX512F + AVX512BF16 + AVX512VNNI
415
define_extension_target(
416
_C
417
DESTINATION vllm
418
LANGUAGE CXX
408
- SOURCES ${VLLM_EXT_SRC_AVX512}
409
- LIBRARIES ${LIBS}
410
- COMPILE_FLAGS ${CXX_COMPILE_FLAGS_AVX512}
419
+ SOURCES ${VLLM_EXT_SRC_AVX512} ${VLLM_EXT_SRC_SGL}
420
+ LIBRARIES ${_C_LIBS}
421
+ COMPILE_FLAGS ${CXX_COMPILE_FLAGS_AVX512_AMX}
422
USE_SABI 3
423
WITH_SOABI
424
)
425
415
- # For SGL kernels
416
- target_compile_definitions(_C PRIVATE "-DCPU_CAPABILITY_AVX512")
426
# For AMX kernels
427
target_compile_definitions(_C PRIVATE "-DCPU_CAPABILITY_AMXBF16")
428
429
+ # AVX512F
430
+ define_extension_target(
431
+ _C_AVX512
432
+ DESTINATION vllm
433
+ LANGUAGE CXX
434
+ SOURCES ${VLLM_EXT_SRC_AVX512}
435
+ LIBRARIES ${_C_AVX512_LIBS}
436
+ COMPILE_FLAGS ${CXX_COMPILE_FLAGS_AVX512}
437
+ USE_SABI 3
438
+ WITH_SOABI
439
+ )
440
+
441
+ # AVX2
442
define_extension_target(
443
_C_AVX2
444
DESTINATION vllm
445
LANGUAGE CXX
446
SOURCES ${VLLM_EXT_SRC_AVX2}
425
- LIBRARIES ${LIBS}
447
+ LIBRARIES ${_C_AVX2_LIBS}
448
COMPILE_FLAGS ${CXX_COMPILE_FLAGS_AVX2}
449
USE_SABI 3
450
WITH_SOABI
docker/Dockerfile.cpu
+9
-39
@@ -14,12 +14,7 @@
14
#
15
# Build arguments:
16
# PYTHON_VERSION=3.13|3.12 (default)|3.11|3.10
17
-# VLLM_CPU_DISABLE_AVX512=false (default)|true
18
-# VLLM_CPU_AVX2=false (default)|true (for cross-compilation)
19
-# VLLM_CPU_AVX512=false (default)|true (for cross-compilation)
20
-# VLLM_CPU_AVX512BF16=false (default)|true (for cross-compilation)
21
-# VLLM_CPU_AVX512VNNI=false (default)|true (for cross-compilation)
22
-# VLLM_CPU_AMXBF16=false (default)|true (for cross-compilation)
17
+# VLLM_CPU_X86=false (default)|true (for cross-compilation)
18
# VLLM_CPU_ARM_BF16=false (default)|true (for cross-compilation)
19
#
20
@@ -36,7 +31,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
31
--mount=type=cache,target=/var/lib/apt,sharing=locked \
32
apt-get update -y \
33
&& apt-get install -y --no-install-recommends sudo ccache git curl wget ca-certificates \
39
- gcc-12 g++-12 libtcmalloc-minimal4 libnuma-dev ffmpeg libsm6 libxext6 libgl1 jq lsof \
34
+ gcc-12 g++-12 libtcmalloc-minimal4 libnuma-dev ffmpeg libsm6 libxext6 libgl1 jq lsof xz-utils \
35
&& update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12 \
36
&& curl -LsSf https://astral.sh/uv/install.sh | sh
37
@@ -91,24 +86,9 @@ ARG max_jobs=32
86
ENV MAX_JOBS=${max_jobs}
87
88
ARG GIT_REPO_CHECK=0
94
-# Support for building with non-AVX512 vLLM: docker build --build-arg VLLM_CPU_DISABLE_AVX512="true" ...
95
-ARG VLLM_CPU_DISABLE_AVX512=0
96
-ENV VLLM_CPU_DISABLE_AVX512=${VLLM_CPU_DISABLE_AVX512}
97
-# Support for cross-compilation with AVX2 ISA: docker build --build-arg VLLM_CPU_AVX2="1" ...
98
-ARG VLLM_CPU_AVX2=0
99
-ENV VLLM_CPU_AVX2=${VLLM_CPU_AVX2}
100
-# Support for cross-compilation with AVX512 ISA: docker build --build-arg VLLM_CPU_AVX512="1" ...
101
-ARG VLLM_CPU_AVX512=0
102
-ENV VLLM_CPU_AVX512=${VLLM_CPU_AVX512}
103
-# Support for building with AVX512BF16 ISA: docker build --build-arg VLLM_CPU_AVX512BF16="true" ...
104
-ARG VLLM_CPU_AVX512BF16=0
105
-ENV VLLM_CPU_AVX512BF16=${VLLM_CPU_AVX512BF16}
106
-# Support for building with AVX512VNNI ISA: docker build --build-arg VLLM_CPU_AVX512VNNI="true" ...
107
-ARG VLLM_CPU_AVX512VNNI=0
108
-ENV VLLM_CPU_AVX512VNNI=${VLLM_CPU_AVX512VNNI}
109
-# Support for building with AMXBF16 ISA: docker build --build-arg VLLM_CPU_AMXBF16="true" ...
110
-ARG VLLM_CPU_AMXBF16=1
111
-ENV VLLM_CPU_AMXBF16=${VLLM_CPU_AMXBF16}
89
+# Support for cross-compilation with x86 ISA including AVX2 and AVX512: docker build --build-arg VLLM_CPU_X86="true" ...
90
+ARG VLLM_CPU_X86=0
91
+ENV VLLM_CPU_X86=${VLLM_CPU_X86}
92
# Support for cross-compilation with ARM BF16 ISA: docker build --build-arg VLLM_CPU_ARM_BF16="true" ...
93
ARG VLLM_CPU_ARM_BF16=0
94
ENV VLLM_CPU_ARM_BF16=${VLLM_CPU_ARM_BF16}
@@ -116,7 +96,7 @@ ENV VLLM_CPU_ARM_BF16=${VLLM_CPU_ARM_BF16}
96
WORKDIR /vllm-workspace
97
98
# Validate build arguments - prevent mixing incompatible ISA flags
119
-RUN if [ "$TARGETARCH" = "arm64" ] && { [ "$VLLM_CPU_AVX2" != "0" ] || [ "$VLLM_CPU_AVX512" != "0" ] || [ "$VLLM_CPU_AVX512BF16" != "0" ] || [ "$VLLM_CPU_AVX512VNNI" != "0" ]; }; then \
99
+RUN if [ "$TARGETARCH" = "arm64" ] && [ "$VLLM_CPU_X86" != "0" ]; then \
100
echo "ERROR: Cannot use x86-specific ISA flags (AVX2, AVX512, etc.) when building for ARM64 (--platform=linux/arm64)"; \
101
exit 1; \
102
fi && \
@@ -174,7 +154,7 @@ WORKDIR /vllm-workspace
154
155
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
156
--mount=type=cache,target=/var/lib/apt,sharing=locked \
177
- apt-get install -y --no-install-recommends vim numactl xz-utils make clangd-14
157
+ apt-get install -y --no-install-recommends vim numactl make clangd-14
158
159
RUN ln -s /usr/bin/clangd-14 /usr/bin/clangd
160
@@ -232,22 +212,12 @@ LABEL org.opencontainers.image.source="https://github.com/vllm-project/vllm"
212
213
# Build configuration labels
214
ARG TARGETARCH
235
-ARG VLLM_CPU_DISABLE_AVX512
236
-ARG VLLM_CPU_AVX2
237
-ARG VLLM_CPU_AVX512
238
-ARG VLLM_CPU_AVX512BF16
239
-ARG VLLM_CPU_AVX512VNNI
240
-ARG VLLM_CPU_AMXBF16
215
+ARG VLLM_CPU_X86
216
ARG VLLM_CPU_ARM_BF16
217
ARG PYTHON_VERSION
218
219
LABEL ai.vllm.build.target-arch="${TARGETARCH}"
245
-LABEL ai.vllm.build.cpu-disable-avx512="${VLLM_CPU_DISABLE_AVX512:-false}"
246
-LABEL ai.vllm.build.cpu-avx2="${VLLM_CPU_AVX2:-false}"
247
-LABEL ai.vllm.build.cpu-avx512="${VLLM_CPU_AVX512:-false}"
248
-LABEL ai.vllm.build.cpu-avx512bf16="${VLLM_CPU_AVX512BF16:-false}"
249
-LABEL ai.vllm.build.cpu-avx512vnni="${VLLM_CPU_AVX512VNNI:-false}"
250
-LABEL ai.vllm.build.cpu-amxbf16="${VLLM_CPU_AMXBF16:-false}"
220
+LABEL ai.vllm.build.cpu-x86="${VLLM_CPU_X86:-false}"
221
LABEL ai.vllm.build.cpu-arm-bf16="${VLLM_CPU_ARM_BF16:-false}"
222
LABEL ai.vllm.build.python-version="${PYTHON_VERSION:-3.12}"
223
docs/getting_started/installation/cpu.x86.inc.md
+6
-48
@@ -7,7 +7,7 @@ vLLM supports basic model inferencing and serving on x86 CPU platform, with data
7
--8<-- [start:requirements]
8
9
- OS: Linux
10
-- CPU flags: `avx512f` (Recommended), `avx512_bf16` (Optional), `avx512_vnni` (Optional)
10
+- CPU flags: `avx512f` (Recommended), `avx2` (Limited features)
11
12
!!! tip
13
Use `lscpu` to check the CPU flags.
@@ -18,7 +18,7 @@ vLLM supports basic model inferencing and serving on x86 CPU platform, with data
18
--8<-- [end:set-up-using-python]
19
--8<-- [start:pre-built-wheels]
20
21
-Pre-built vLLM wheels for x86 with AVX512 are available since version 0.13.0. To install release wheels:
21
+Pre-built vLLM wheels for x86 with AVX512/AVX2 are available since version 0.17.0. To install release wheels:
22
23
```bash
24
export VLLM_VERSION=$(curl -s https://api.github.com/repos/vllm-project/vllm/releases/latest | jq -r .tag_name | sed 's/^v//')
@@ -108,13 +108,13 @@ VLLM_TARGET_DEVICE=cpu uv pip install . --no-build-isolation
108
If you want to develop vLLM, install it in editable mode instead.
109
110
```bash
111
-VLLM_TARGET_DEVICE=cpu uv pip install -e . --no-build-isolation
111
+VLLM_TARGET_DEVICE=cpu python3 setup.py develop
112
```
113
114
Optionally, build a portable wheel which you can then install elsewhere:
115
116
```bash
117
-VLLM_TARGET_DEVICE=cpu uv build --wheel
117
+VLLM_TARGET_DEVICE=cpu uv build --wheel --no-build-isolation
118
```
119
120
```bash
@@ -185,12 +185,9 @@ docker run \
185
-v ~/.cache/huggingface:/root/.cache/huggingface \
186
-p 8000:8000 \
187
--env "HF_TOKEN=<secret>" \
188
-vllm/vllm-openai-cpu:latest-x86_64 <args...>
188
+ vllm/vllm-openai-cpu:latest-x86_64 <args...>
189
```
190
191
-!!! warning
192
- If deploying the pre-built images on machines without `avx512f`, `avx512_bf16`, or `avx512_vnni` support, an `Illegal instruction` error may be raised. See the build-image-from-source section below for build arguments to match your target CPU capabilities.
193
-
191
--8<-- [end:pre-built-images]
192
--8<-- [start:build-image-from-source]
193
@@ -198,50 +195,11 @@ vllm/vllm-openai-cpu:latest-x86_64 <args...>
195
196
```bash
197
docker build -f docker/Dockerfile.cpu \
201
- --build-arg VLLM_CPU_DISABLE_AVX512=<false (default)|true> \
202
- --build-arg VLLM_CPU_AVX2=<false (default)|true> \
203
- --build-arg VLLM_CPU_AVX512=<false (default)|true> \
204
- --build-arg VLLM_CPU_AVX512BF16=<false (default)|true> \
205
- --build-arg VLLM_CPU_AVX512VNNI=<false (default)|true> \
206
- --build-arg VLLM_CPU_AMXBF16=<false|true (default)> \
198
+ --build-arg VLLM_CPU_X86=<false (default)|true> \ # For cross-compilation
199
--tag vllm-cpu-env \
200
--target vllm-openai .
201
```
202
211
-!!! note "Auto-detection by default"
212
- By default, CPU instruction sets (AVX512, AVX2, etc.) are automatically detected from the build system's CPU flags. Build arguments like `VLLM_CPU_AVX2`, `VLLM_CPU_AVX512`, `VLLM_CPU_AVX512BF16`, `VLLM_CPU_AVX512VNNI`, and `VLLM_CPU_AMXBF16` are used for cross-compilation:
213
-
214
- - `VLLM_CPU_{ISA}=true` - Force-enable the instruction set (build with ISA regardless of build system capabilities)
215
- - `VLLM_CPU_{ISA}=false` - Rely on auto-detection (default)
216
-
217
-##### Examples
218
-
219
-###### Auto-detection build (default)
220
-
221
-```bash
222
-docker build -f docker/Dockerfile.cpu --tag vllm-cpu-env --target vllm-openai .
223
-```
224
-
225
-###### Cross-compile for AVX512
226
-
227
-```bash
228
-docker build -f docker/Dockerfile.cpu \
229
- --build-arg VLLM_CPU_AVX512=true \
230
- --build-arg VLLM_CPU_AVX512BF16=true \
231
- --build-arg VLLM_CPU_AVX512VNNI=true \
232
- --tag vllm-cpu-avx512 \
233
- --target vllm-openai .
234
-```
235
-
236
-###### Cross-compile for AVX2
237
-
238
-```bash
239
-docker build -f docker/Dockerfile.cpu \
240
- --build-arg VLLM_CPU_AVX2=true \
241
- --tag vllm-cpu-avx2 \
242
- --target vllm-openai .
243
-```
244
-
203
#### Launching the OpenAI server
204
205
```bash
setup.py
+1
@@ -920,6 +920,7 @@ if _is_cpu():
920
921
if platform.machine() in ("x86_64", "AMD64"):
922
ext_modules.append(CMakeExtension(name="vllm._C"))
923
+ ext_modules.append(CMakeExtension(name="vllm._C_AVX512"))
924
ext_modules.append(CMakeExtension(name="vllm._C_AVX2"))
925
else:
926
ext_modules.append(CMakeExtension(name="vllm._C"))
vllm/platforms/cpu.py
+24
-11
@@ -252,6 +252,8 @@ class CpuPlatform(Platform):
252
if vllm_config.lora_config is not None:
253
compilation_config.mode = CompilationMode.NONE
254
255
+ vllm_config.profiler_config.torch_profiler_dump_cuda_time_total = False
256
+
257
assert vllm_config.device_config.device_type == "cpu"
258
259
#
@@ -470,21 +472,32 @@ class CpuPlatform(Platform):
472
@classmethod
473
def import_kernels(cls) -> None:
474
if Platform.get_cpu_architecture() in (CpuArchEnum.X86,):
473
- if torch._C._cpu._is_avx512_supported():
474
- try:
475
- import vllm._C # noqa: F401
476
- except ImportError as e:
477
- logger.warning("Failed to import from vllm._C: %r", e)
475
+ # Note: The lib name is _C_AVX2/AVX512, but the module name is _C.
476
+ # This will cause a exception "dynamic module does define
477
+ # module export function". But the library is imported
478
+ # successfully. So ignore the exception for now, until we find
479
+ # a solution.
480
+ ignored_msg = "dynamic module does not define module export function"
481
+ if torch.cpu._is_avx512_supported():
482
+ if torch.cpu._is_avx512_bf16_supported():
483
+ try:
484
+ import vllm._C # noqa: F401
485
+ except ImportError as e:
486
+ logger.warning("Failed to import from vllm._C: %r", e)
487
+ else:
488
+ try:
489
+ import vllm._C_AVX512 # noqa: F401
490
+ except ImportError as e:
491
+ if ignored_msg not in e.msg:
492
+ logger.warning(
493
+ "Failed to import from vllm._C_AVX512: %r", e
494
+ )
495
else:
479
- # Note: The lib name is _C_AVX2, but the module name is _C.
480
- # This will cause a exception "dynamic module does define
481
- # module export function". But the library is imported
482
- # successfully. So ignore the exception for now, until we find
483
- # a solution.
496
try:
497
import vllm._C_AVX2 # noqa: F401
498
except ImportError as e:
487
- logger.warning("Failed to import from vllm._C_AVX2: %r", e)
499
+ if ignored_msg not in e.msg:
500
+ logger.warning("Failed to import from vllm._C_AVX2: %r", e)
501
else:
502
try:
503
import vllm._C # noqa: F401
vllm/v1/worker/cpu_worker.py
+15
@@ -52,6 +52,21 @@ class CPUWorker(Worker):
52
)
53
54
def init_device(self):
55
+ # Check whether critical libraries are loaded
56
+ def check_preloaded_libs(name: str):
57
+ ld_preload_list = os.environ.get("LD_PRELOAD", "")
58
+ if name not in ld_preload_list:
59
+ raise RuntimeError(
60
+ f"{name} is not found in LD_PRELOAD. "
61
+ "Please follow the section `set LD_PRELOAD` in "
62
+ "https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ "
63
+ "to setup required pre-loaded libraries."
64
+ )
65
+
66
+ check_preloaded_libs("libtcmalloc")
67
+ if current_platform.get_cpu_architecture() == CpuArchEnum.X86:
68
+ check_preloaded_libs("libiomp")
69
+
70
# Setup OpenMP threads affinity.
71
omp_cpuids = envs.VLLM_CPU_OMP_THREADS_BIND
72
# Under numa binding some cores reserved for kv transfer in nixl_connector.py