A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 07:15 UTC
[CPU][CI] Re-enable the CPU CI tests (#19046)
Signed-off-by: jiang.li <jiang1.li@intel.com>
Li, Jiang committed
Jun 3, 2025 at 11:46 UTC
8655f47f37750eb5d00992d39305d6705659983f
3 files changed
+29
-26
.buildkite/scripts/hardware_ci/run-cpu-test.sh
+20
-22
@@ -8,67 +8,65 @@ set -ex
8
CORE_RANGE=${CORE_RANGE:-48-95}
9
NUMA_NODE=${NUMA_NODE:-1}
10
11
+export CMAKE_BUILD_PARALLEL_LEVEL=32
12
+
13
# Setup cleanup
14
remove_docker_container() {
15
set -e;
14
- docker rm -f cpu-test-"$BUILDKITE_BUILD_NUMBER"-"$NUMA_NODE" cpu-test-"$BUILDKITE_BUILD_NUMBER"-avx2-"$NUMA_NODE" || true;
15
- docker image rm cpu-test-"$BUILDKITE_BUILD_NUMBER" cpu-test-"$BUILDKITE_BUILD_NUMBER"-avx2 || true;
16
+ docker rm -f cpu-test-"$NUMA_NODE" cpu-test-"$NUMA_NODE"-avx2 || true;
17
}
18
trap remove_docker_container EXIT
19
remove_docker_container
20
21
# Try building the docker image
21
-numactl -C "$CORE_RANGE" -N "$NUMA_NODE" docker build --tag cpu-test-"$BUILDKITE_BUILD_NUMBER" --target vllm-test -f docker/Dockerfile.cpu .
22
-numactl -C "$CORE_RANGE" -N "$NUMA_NODE" docker build --build-arg VLLM_CPU_DISABLE_AVX512="true" --tag cpu-test-"$BUILDKITE_BUILD_NUMBER"-avx2 --target vllm-test -f docker/Dockerfile.cpu .
22
+numactl -C "$CORE_RANGE" -N "$NUMA_NODE" docker build --tag cpu-test-"$NUMA_NODE" --target vllm-test -f docker/Dockerfile.cpu .
23
+numactl -C "$CORE_RANGE" -N "$NUMA_NODE" docker build --build-arg VLLM_CPU_DISABLE_AVX512="true" --tag cpu-test-"$NUMA_NODE"-avx2 --target vllm-test -f docker/Dockerfile.cpu .
24
25
# Run the image, setting --shm-size=4g for tensor parallel.
26
docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --cpuset-cpus="$CORE_RANGE" \
26
- --cpuset-mems="$NUMA_NODE" --privileged=true -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test-"$BUILDKITE_BUILD_NUMBER"-"$NUMA_NODE" cpu-test-"$BUILDKITE_BUILD_NUMBER"
27
+ --cpuset-mems="$NUMA_NODE" --privileged=true -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test-"$NUMA_NODE" cpu-test-"$NUMA_NODE"
28
docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --cpuset-cpus="$CORE_RANGE" \
28
- --cpuset-mems="$NUMA_NODE" --privileged=true -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test-"$BUILDKITE_BUILD_NUMBER"-avx2-"$NUMA_NODE" cpu-test-"$BUILDKITE_BUILD_NUMBER"-avx2
29
+ --cpuset-mems="$NUMA_NODE" --privileged=true -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test-"$NUMA_NODE"-avx2 cpu-test-"$NUMA_NODE"-avx2
30
31
function cpu_tests() {
32
set -e
33
export NUMA_NODE=$2
33
- export BUILDKITE_BUILD_NUMBER=$3
34
35
# offline inference
36
- docker exec cpu-test-"$BUILDKITE_BUILD_NUMBER"-avx2-"$NUMA_NODE" bash -c "
36
+ docker exec cpu-test-"$NUMA_NODE"-avx2 bash -c "
37
set -e
38
python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m"
39
40
# Run basic model test
41
- docker exec cpu-test-"$BUILDKITE_BUILD_NUMBER"-"$NUMA_NODE" bash -c "
41
+ docker exec cpu-test-"$NUMA_NODE" bash -c "
42
set -e
43
- pytest -v -s tests/kernels/test_cache.py -m cpu_model
44
- pytest -v -s tests/kernels/test_mla_decode_cpu.py -m cpu_model
45
- pytest -v -s tests/models/decoder_only/language -m cpu_model
46
- pytest -v -s tests/models/embedding/language -m cpu_model
47
- pytest -v -s tests/models/encoder_decoder/language -m cpu_model
48
- pytest -v -s tests/models/decoder_only/audio_language -m cpu_model
49
- pytest -v -s tests/models/decoder_only/vision_language -m cpu_model"
43
+ pytest -v -s tests/kernels/attention/test_cache.py -m cpu_model
44
+ pytest -v -s tests/kernels/attention/test_mla_decode_cpu.py -m cpu_model
45
+ pytest -v -s tests/models/language/generation -m cpu_model
46
+ pytest -v -s tests/models/language/pooling -m cpu_model
47
+ pytest -v -s tests/models/multimodal/generation --ignore=tests/models/multimodal/generation/test_mllama.py -m cpu_model"
48
49
# Run compressed-tensor test
52
- docker exec cpu-test-"$BUILDKITE_BUILD_NUMBER"-"$NUMA_NODE" bash -c "
50
+ docker exec cpu-test-"$NUMA_NODE" bash -c "
51
set -e
52
pytest -s -v \
53
tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_static_setup \
54
tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_dynamic_per_token"
55
56
# Run AWQ test
59
- docker exec cpu-test-"$BUILDKITE_BUILD_NUMBER"-"$NUMA_NODE" bash -c "
57
+ docker exec cpu-test-"$NUMA_NODE" bash -c "
58
set -e
59
pytest -s -v \
60
tests/quantization/test_ipex_quant.py"
61
62
# Run chunked-prefill and prefix-cache test
65
- docker exec cpu-test-"$BUILDKITE_BUILD_NUMBER"-"$NUMA_NODE" bash -c "
63
+ docker exec cpu-test-"$NUMA_NODE" bash -c "
64
set -e
65
pytest -s -v -k cpu_model \
66
tests/basic_correctness/test_chunked_prefill.py"
67
68
# online serving
71
- docker exec cpu-test-"$BUILDKITE_BUILD_NUMBER"-"$NUMA_NODE" bash -c "
69
+ docker exec cpu-test-"$NUMA_NODE" bash -c "
70
set -e
71
export VLLM_CPU_KVCACHE_SPACE=10
72
export VLLM_CPU_OMP_THREADS_BIND=$1
@@ -83,7 +81,7 @@ function cpu_tests() {
81
--tokenizer facebook/opt-125m"
82
83
# Run multi-lora tests
86
- docker exec cpu-test-"$BUILDKITE_BUILD_NUMBER"-"$NUMA_NODE" bash -c "
84
+ docker exec cpu-test-"$NUMA_NODE" bash -c "
85
set -e
86
pytest -s -v \
87
tests/lora/test_qwen2vl.py"
@@ -91,4 +89,4 @@ function cpu_tests() {
89
90
# All of CPU tests are expected to be finished less than 40 mins.
91
export -f cpu_tests
94
-timeout 40m bash -c "cpu_tests $CORE_RANGE $NUMA_NODE $BUILDKITE_BUILD_NUMBER"
92
+timeout 40m bash -c "cpu_tests $CORE_RANGE $NUMA_NODE"
docker/Dockerfile.cpu
+7
-3
@@ -75,6 +75,7 @@ RUN --mount=type=bind,source=.git,target=.git \
75
76
RUN --mount=type=cache,target=/root/.cache/uv \
77
--mount=type=cache,target=/root/.cache/ccache \
78
+ --mount=type=cache,target=/workspace/vllm/.deps,sharing=locked \
79
--mount=type=bind,source=.git,target=.git \
80
VLLM_TARGET_DEVICE=cpu python3 setup.py bdist_wheel
81
@@ -85,7 +86,7 @@ WORKDIR /workspace/vllm
86
87
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
88
--mount=type=cache,target=/var/lib/apt,sharing=locked \
88
- apt-get install -y --no-install-recommends vim numactl
89
+ apt-get install -y --no-install-recommends vim numactl xz-utils
90
91
# install development dependencies (for testing)
92
RUN --mount=type=cache,target=/root/.cache/uv \
@@ -108,8 +109,11 @@ FROM base AS vllm-test
109
WORKDIR /workspace/
110
111
RUN --mount=type=cache,target=/root/.cache/uv \
111
- --mount=type=bind,src=requirements/test.txt,target=requirements/test.txt \
112
- uv pip install -r requirements/test.txt
112
+ --mount=type=bind,src=requirements/test.in,target=requirements/test.in \
113
+ cp requirements/test.in requirements/test-cpu.in && \
114
+ sed -i '/mamba_ssm/d' requirements/test-cpu.in && \
115
+ uv pip compile requirements/test-cpu.in -o requirements/cpu-test.txt && \
116
+ uv pip install -r requirements/cpu-test.txt
117
118
RUN --mount=type=cache,target=/root/.cache/uv \
119
--mount=type=bind,from=vllm-build,src=/workspace/vllm/dist,target=dist \
vllm/distributed/parallel_state.py
+2
-1
@@ -1203,7 +1203,8 @@ def cleanup_dist_env_and_memory(shutdown_ray: bool = False):
1203
if empty_cache is not None:
1204
empty_cache()
1205
try:
1206
- torch._C._host_emptyCache()
1206
+ if not current_platform.is_cpu():
1207
+ torch._C._host_emptyCache()
1208
except AttributeError:
1209
logger.warning(
1210
"torch._C._host_emptyCache() only available in Pytorch >=2.5")