A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[CI][CPU]refactor CPU tests to allow to bind with different cores (#10222)
Signed-off-by: Yuan Zhou <yuan.zhou@intel.com>
Yuan committed
Nov 12, 2024 at 18:07 UTC
d201d419730dec120b0ecb60ae212f08c0b68be0
1 file changed
+11
-7
.buildkite/run-cpu-test.sh
+11
-7
@@ -4,9 +4,13 @@
4
# It serves a sanity check for compilation and basic model usage.
5
set -ex
6
7
+# allow to bind to different cores
8
+CORE_RANGE=${CORE_RANGE:-48-95}
9
+NUMA_NODE=${NUMA_NODE:-1}
10
+
11
# Try building the docker image
8
-numactl -C 48-95 -N 1 docker build -t cpu-test -f Dockerfile.cpu .
9
-numactl -C 48-95 -N 1 docker build --build-arg VLLM_CPU_DISABLE_AVX512="true" -t cpu-test-avx2 -f Dockerfile.cpu .
12
+numactl -C $CORE_RANGE -N $NUMA_NODE docker build -t cpu-test -f Dockerfile.cpu .
13
+numactl -C $CORE_RANGE -N $NUMA_NODE docker build --build-arg VLLM_CPU_DISABLE_AVX512="true" -t cpu-test-avx2 -f Dockerfile.cpu .
14
15
# Setup cleanup
16
remove_docker_container() { docker rm -f cpu-test cpu-test-avx2 || true; }
@@ -14,10 +18,10 @@ trap remove_docker_container EXIT
18
remove_docker_container
19
20
# Run the image, setting --shm-size=4g for tensor parallel.
17
-docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --cpuset-cpus=48-95 \
18
- --cpuset-mems=1 --privileged=true --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test cpu-test
19
-docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --cpuset-cpus=48-95 \
20
- --cpuset-mems=1 --privileged=true --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test-avx2 cpu-test-avx2
21
+docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --cpuset-cpus=$CORE_RANGE \
22
+ --cpuset-mems=$NUMA_NODE --privileged=true --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test cpu-test
23
+docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --cpuset-cpus=$CORE_RANGE \
24
+ --cpuset-mems=$NUMA_NODE --privileged=true --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test-avx2 cpu-test-avx2
25
26
function cpu_tests() {
27
set -e
@@ -57,7 +61,7 @@ function cpu_tests() {
61
docker exec cpu-test bash -c "
62
set -e
63
export VLLM_CPU_KVCACHE_SPACE=10
60
- export VLLM_CPU_OMP_THREADS_BIND=48-92
64
+ export VLLM_CPU_OMP_THREADS_BIND=$CORE_RANGE
65
python3 -m vllm.entrypoints.openai.api_server --model facebook/opt-125m --dtype half &
66
timeout 600 bash -c 'until curl localhost:8000/v1/models; do sleep 1; done' || exit 1
67
python3 benchmarks/benchmark_serving.py \