A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 21:30 UTC
[CI/Build] Parallelize CPU CI tests (#33778)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
Li, Jiang committed
Feb 5, 2026 at 13:53 UTC
07daee132b30140bb7c5b28d7f8c856036d2baad
6 files changed
+157
-130
.buildkite/hardware_tests/arm.yaml
deleted
-8
@@ -1,8 +0,0 @@
1
-group: Hardware
2
-steps:
3
- - label: "Arm CPU Test"
4
- soft_fail: true
5
- device: arm_cpu
6
- no_plugin: true
7
- commands:
8
- - bash .buildkite/scripts/hardware_ci/run-cpu-test-arm.sh
.buildkite/hardware_tests/cpu.yaml
new
+100
@@ -0,0 +1,100 @@
1
+group: CPU
2
+depends_on: []
3
+steps:
4
+- label: CPU-Kernel Tests
5
+ depends_on: []
6
+ soft_fail: true
7
+ device: intel_cpu
8
+ no_plugin: true
9
+ source_file_dependencies:
10
+ - csrc/cpu/
11
+ - cmake/cpu_extension.cmake
12
+ - CMakeLists.txt
13
+ - vllm/_custom_ops.py
14
+ - tests/kernels/attention/test_cpu_attn.py
15
+ - tests/kernels/moe/test_cpu_fused_moe.py
16
+ - tests/kernels/test_onednn.py
17
+ commands:
18
+ - |
19
+ bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 20m "
20
+ pytest -x -v -s tests/kernels/attention/test_cpu_attn.py
21
+ pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py
22
+ pytest -x -v -s tests/kernels/test_onednn.py"
23
+
24
+- label: CPU-Language Generation and Pooling Model Tests
25
+ depends_on: []
26
+ soft_fail: true
27
+ device: intel_cpu
28
+ no_plugin: true
29
+ source_file_dependencies:
30
+ - csrc/cpu/
31
+ - vllm/
32
+ - tests/models/language/generation/
33
+ - tests/models/language/pooling/
34
+ commands:
35
+ - |
36
+ bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m "
37
+ pytest -x -v -s tests/models/language/generation -m cpu_model
38
+ pytest -x -v -s tests/models/language/pooling -m cpu_model"
39
+
40
+- label: CPU-Quantization Model Tests
41
+ depends_on: []
42
+ soft_fail: true
43
+ device: intel_cpu
44
+ no_plugin: true
45
+ source_file_dependencies:
46
+ - csrc/cpu/
47
+ - vllm/model_executor/layers/quantization/cpu_wna16.py
48
+ - vllm/model_executor/layers/quantization/gptq_marlin.py
49
+ - vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py
50
+ - vllm/model_executor/layers/quantization/kernels/scaled_mm/cpu.py
51
+ - vllm/model_executor/layers/quantization/kernels/mixed_precision/cpu.py
52
+ - tests/quantization/test_compressed_tensors.py
53
+ - tests/quantization/test_cpu_wna16.py
54
+ commands:
55
+ - |
56
+ bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 20m "
57
+ pytest -x -v -s tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_logprobs
58
+ pytest -x -v -s tests/quantization/test_cpu_wna16.py"
59
+
60
+- label: CPU-TP/DP/PP Tests
61
+ depends_on: []
62
+ soft_fail: true
63
+ device: intel_cpu
64
+ no_plugin: true
65
+ source_file_dependencies:
66
+ - csrc/cpu/shm.cpp
67
+ - vllm/v1/worker/cpu_worker.py
68
+ - vllm/v1/worker/gpu_worker.py
69
+ - vllm/v1/worker/cpu_model_runner.py
70
+ - vllm/v1/worker/gpu_model_runner.py
71
+ - vllm/platforms/cpu.py
72
+ - vllm/distributed/parallel_state.py
73
+ - vllm/distributed/device_communicators/cpu_communicator.py
74
+ commands:
75
+ - |
76
+ bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 10m "
77
+ bash .buildkite/scripts/hardware_ci/run-cpu-distributed-smoke-test.sh"
78
+
79
+- label: CPU-Multi-Modal Model Tests %N
80
+ depends_on: []
81
+ soft_fail: true
82
+ device: intel_cpu
83
+ no_plugin: true
84
+ source_file_dependencies:
85
+ # - vllm/
86
+ - vllm/model_executor/layers/rotary_embedding
87
+ - tests/models/multimodal/generation/
88
+ commands:
89
+ - |
90
+ bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 45m "
91
+ pytest -x -v -s tests/models/multimodal/generation --ignore=tests/models/multimodal/generation/test_pixtral.py -m cpu_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
92
+ parallelism: 2
93
+
94
+- label: "Arm CPU Test"
95
+ depends_on: []
96
+ soft_fail: true
97
+ device: arm_cpu
98
+ no_plugin: true
99
+ commands:
100
+ - bash .buildkite/scripts/hardware_ci/run-cpu-test-arm.sh
.buildkite/hardware_tests/intel.yaml
-7
@@ -1,13 +1,6 @@
1
group: Hardware
2
depends_on: ~
3
steps:
4
- - label: "Intel CPU Test"
5
- soft_fail: true
6
- device: intel_cpu
7
- no_plugin: true
8
- commands:
9
- - bash .buildkite/scripts/hardware_ci/run-cpu-test.sh
10
-
4
- label: "Intel HPU Test"
5
soft_fail: true
6
device: intel_hpu
.buildkite/scripts/hardware_ci/run-cpu-distributed-smoke-test.sh
new
+26
@@ -0,0 +1,26 @@
1
+#!/bin/bash
2
+set -euox pipefail
3
+
4
+echo "--- PP+TP"
5
+vllm serve meta-llama/Llama-3.2-3B-Instruct -tp=2 -pp=2 &
6
+server_pid=$!
7
+timeout 600 bash -c "until curl localhost:8000/v1/models; do sleep 1; done" || exit 1
8
+vllm bench serve \
9
+ --backend vllm \
10
+ --dataset-name random \
11
+ --model meta-llama/Llama-3.2-3B-Instruct \
12
+ --num-prompts 20 \
13
+ --endpoint /v1/completions
14
+kill -s SIGTERM $server_pid &
15
+
16
+echo "--- DP+TP"
17
+vllm serve meta-llama/Llama-3.2-3B-Instruct -tp=2 -dp=2 &
18
+server_pid=$!
19
+timeout 600 bash -c "until curl localhost:8000/v1/models; do sleep 1; done" || exit 1
20
+vllm bench serve \
21
+ --backend vllm \
22
+ --dataset-name random \
23
+ --model meta-llama/Llama-3.2-3B-Instruct \
24
+ --num-prompts 20 \
25
+ --endpoint /v1/completions
26
+kill -s SIGTERM $server_pid &
.buildkite/scripts/hardware_ci/run-cpu-test.sh
+9
-109
@@ -2,119 +2,19 @@
2
3
# This script build the CPU docker image and run the offline inference inside the container.
4
# It serves a sanity check for compilation and basic model usage.
5
-set -ex
5
+set -euox pipefail
6
7
# allow to bind to different cores
8
CORE_RANGE=${CORE_RANGE:-48-95}
9
-# used for TP/PP E2E test
10
-OMP_CORE_RANGE=${OMP_CORE_RANGE:-48-95}
9
NUMA_NODE=${NUMA_NODE:-1}
10
+IMAGE_NAME="cpu-test-$NUMA_NODE"
11
+TIMEOUT_VAL=$1
12
+TEST_COMMAND=$2
13
13
-export CMAKE_BUILD_PARALLEL_LEVEL=32
14
-
15
-# Setup cleanup
16
-remove_docker_container() {
17
- set -e;
18
- docker rm -f cpu-test-"$NUMA_NODE" cpu-test-"$NUMA_NODE"-avx2 || true;
19
-}
20
-trap remove_docker_container EXIT
21
-remove_docker_container
22
-
23
-# Try building the docker image
24
-numactl -C "$CORE_RANGE" -N "$NUMA_NODE" docker build --progress plain --tag cpu-test-"$NUMA_NODE" --target vllm-test -f docker/Dockerfile.cpu .
25
-numactl -C "$CORE_RANGE" -N "$NUMA_NODE" docker build --progress plain --build-arg VLLM_CPU_DISABLE_AVX512="true" --tag cpu-test-"$NUMA_NODE"-avx2 --target vllm-test -f docker/Dockerfile.cpu .
14
+# building the docker image
15
+echo "--- :docker: Building Docker image"
16
+docker build --progress plain --tag "$IMAGE_NAME" --target vllm-test -f docker/Dockerfile.cpu .
17
18
# Run the image, setting --shm-size=4g for tensor parallel.
28
-docker run -itd --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=16 --env VLLM_CPU_CI_ENV=1 -e E2E_OMP_THREADS="$OMP_CORE_RANGE" --shm-size=4g --name cpu-test-"$NUMA_NODE" cpu-test-"$NUMA_NODE"
29
-docker run -itd --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=16 --env VLLM_CPU_CI_ENV=1 -e E2E_OMP_THREADS="$OMP_CORE_RANGE" --shm-size=4g --name cpu-test-"$NUMA_NODE"-avx2 cpu-test-"$NUMA_NODE"-avx2
30
-
31
-function cpu_tests() {
32
- set -e
33
- export NUMA_NODE=$2
34
-
35
- # list packages
36
- docker exec cpu-test-"$NUMA_NODE"-avx2 bash -c "
37
- set -e
38
- pip list"
39
-
40
- docker exec cpu-test-"$NUMA_NODE" bash -c "
41
- set -e
42
- pip list"
43
-
44
- # offline inference
45
- docker exec cpu-test-"$NUMA_NODE"-avx2 bash -c "
46
- set -e
47
- python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m"
48
-
49
- # Run kernel tests
50
- docker exec cpu-test-"$NUMA_NODE" bash -c "
51
- set -e
52
- pytest -x -v -s tests/kernels/attention/test_cpu_attn.py
53
- pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py
54
- pytest -x -v -s tests/kernels/test_onednn.py"
55
-
56
- # Run basic model test
57
- docker exec cpu-test-"$NUMA_NODE" bash -c "
58
- set -e
59
- # Note: disable until supports V1
60
- # pytest -x -v -s tests/kernels/attention/test_cache.py -m cpu_model
61
- # pytest -x -v -s tests/kernels/attention/test_mla_decode_cpu.py -m cpu_model
62
-
63
- pytest -x -v -s tests/models/language/generation -m cpu_model
64
- VLLM_CPU_SGL_KERNEL=1 pytest -x -v -s tests/models/language/generation -m cpu_model
65
-
66
- pytest -x -v -s tests/models/language/pooling -m cpu_model
67
- pytest -x -v -s tests/models/multimodal/generation \
68
- --ignore=tests/models/multimodal/generation/test_pixtral.py \
69
- -m cpu_model"
70
-
71
- # Run compressed-tensor test
72
- docker exec cpu-test-"$NUMA_NODE" bash -c "
73
- set -e
74
- pytest -x -s -v \
75
- tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_logprobs"
76
-
77
- # Run AWQ/GPTQ test
78
- docker exec cpu-test-"$NUMA_NODE" bash -c "
79
- set -e
80
- pytest -x -s -v \
81
- tests/quantization/test_cpu_wna16.py"
82
-
83
- # Run multi-lora tests
84
- docker exec cpu-test-"$NUMA_NODE" bash -c "
85
- set -e
86
- pytest -x -s -v \
87
- tests/lora/test_qwenvl.py"
88
-
89
- # online serving: tp+pp
90
- docker exec cpu-test-"$NUMA_NODE" bash -c '
91
- set -e
92
- VLLM_CPU_OMP_THREADS_BIND=$E2E_OMP_THREADS VLLM_CPU_SGL_KERNEL=1 vllm serve meta-llama/Llama-3.2-3B-Instruct -tp=2 -pp=2 &
93
- server_pid=$!
94
- timeout 600 bash -c "until curl localhost:8000/v1/models; do sleep 1; done" || exit 1
95
- vllm bench serve \
96
- --backend vllm \
97
- --dataset-name random \
98
- --model meta-llama/Llama-3.2-3B-Instruct \
99
- --num-prompts 20 \
100
- --endpoint /v1/completions
101
- kill -s SIGTERM $server_pid &'
102
-
103
- # online serving: tp+dp
104
- docker exec cpu-test-"$NUMA_NODE" bash -c '
105
- set -e
106
- VLLM_CPU_OMP_THREADS_BIND=$E2E_OMP_THREADS VLLM_CPU_SGL_KERNEL=1 vllm serve meta-llama/Llama-3.2-3B-Instruct -tp=2 -dp=2 &
107
- server_pid=$!
108
- timeout 600 bash -c "until curl localhost:8000/v1/models; do sleep 1; done" || exit 1
109
- vllm bench serve \
110
- --backend vllm \
111
- --dataset-name random \
112
- --model meta-llama/Llama-3.2-3B-Instruct \
113
- --num-prompts 20 \
114
- --endpoint /v1/completions
115
- kill -s SIGTERM $server_pid &'
116
-}
117
-
118
-# All of CPU tests are expected to be finished less than 40 mins.
119
-export -f cpu_tests
120
-timeout 2.5h bash -c "cpu_tests $CORE_RANGE $NUMA_NODE"
19
+docker run --rm --cpuset-cpus=$CORE_RANGE --cpuset-mems=$NUMA_NODE -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true -e HF_TOKEN -e VLLM_CPU_KVCACHE_SPACE=16 -e VLLM_CPU_CI_ENV=1 -e VLLM_CPU_SIM_MULTI_NUMA=1 --shm-size=4g $IMAGE_NAME \
20
+ timeout $TIMEOUT_VAL bash -c "set -euox pipefail; echo \"--- Print packages\"; pip list; echo \"--- Running tests\"; ${TEST_COMMAND}"
vllm/v1/worker/cpu_worker.py
+22
-6
@@ -136,22 +136,38 @@ class CPUWorker(Worker):
136
the LogicalCPUInfo.id. A selected LogicalCPUInfo list should be
137
returned.
138
"""
139
+ # simulate multiple numa nodes, for testing
140
+ sim_multi_numa_nodes = os.environ.get("VLLM_CPU_SIM_MULTI_NUMA", "0") != "0"
141
142
allowed_numa_nodes, logical_cpu_list = (
143
CpuPlatform.get_allowed_cpu_core_node_list()
144
)
143
- assert len(allowed_numa_nodes) >= self.parallel_config.world_size, (
145
+ assert (
146
+ len(allowed_numa_nodes) >= self.parallel_config.world_size
147
+ or sim_multi_numa_nodes
148
+ ), (
149
f"Not enough allowed NUMA nodes to bind threads of "
150
f"{self.parallel_config.world_size} CPUWorkers. "
151
f"Allowed NUMA nodes are {allowed_numa_nodes}. "
152
"Please try to bind threads manually."
153
)
154
150
- # Get CPUs on NUMA node `allowed_numa_nodes[local_rank]`
151
- selected_numa_node = allowed_numa_nodes[self.local_rank] # type: ignore
152
- logical_cpu_list = [
153
- x for x in logical_cpu_list if x.numa_node == selected_numa_node
154
- ]
155
+ if not sim_multi_numa_nodes:
156
+ # Get CPUs on NUMA node `allowed_numa_nodes[local_rank]`
157
+ selected_numa_node = allowed_numa_nodes[self.local_rank] # type: ignore
158
+ logical_cpu_list = [
159
+ x for x in logical_cpu_list if x.numa_node == selected_numa_node
160
+ ]
161
+ else:
162
+ assert len(logical_cpu_list) >= self.parallel_config.world_size
163
+ logical_cpu_list = sorted(logical_cpu_list, key=lambda x: x.numa_node)
164
+ sim_cpu_num_per_node = (
165
+ len(logical_cpu_list) // self.parallel_config.world_size
166
+ )
167
+ start_idx = self.local_rank * sim_cpu_num_per_node
168
+ logical_cpu_list = logical_cpu_list[
169
+ start_idx : (start_idx + sim_cpu_num_per_node)
170
+ ]
171
172
# Select CPUs from each physical core via cpu_selector
173
core_to_cpus: dict[int, list[LogicalCPUInfo]] = {}