@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d201d41973
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[CI][CPU]refactor CPU tests to allow to bind with different cores (#10222)

Signed-off-by: Yuan Zhou <yuan.zhou@intel.com>

Yuan committed Nov 12, 2024 at 18:07 UTC d201d419730dec120b0ecb60ae212f08c0b68be0
1 file changed +11 -7
.buildkite/run-cpu-test.sh
+11 -7
@@ -4,9 +4,13 @@
4 # It serves a sanity check for compilation and basic model usage.
5 set -ex
6
7 +# allow to bind to different cores
8 +CORE_RANGE=${CORE_RANGE:-48-95}
9 +NUMA_NODE=${NUMA_NODE:-1}
10 +
11 # Try building the docker image
8 -numactl -C 48-95 -N 1 docker build -t cpu-test -f Dockerfile.cpu .
9 -numactl -C 48-95 -N 1 docker build --build-arg VLLM_CPU_DISABLE_AVX512="true" -t cpu-test-avx2 -f Dockerfile.cpu .
12 +numactl -C $CORE_RANGE -N $NUMA_NODE docker build -t cpu-test -f Dockerfile.cpu .
13 +numactl -C $CORE_RANGE -N $NUMA_NODE docker build --build-arg VLLM_CPU_DISABLE_AVX512="true" -t cpu-test-avx2 -f Dockerfile.cpu .
14
15 # Setup cleanup
16 remove_docker_container() { docker rm -f cpu-test cpu-test-avx2 || true; }
@@ -14,10 +18,10 @@ trap remove_docker_container EXIT
18 remove_docker_container
19
20 # Run the image, setting --shm-size=4g for tensor parallel.
17 -docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --cpuset-cpus=48-95 \
18 - --cpuset-mems=1 --privileged=true --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test cpu-test
19 -docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --cpuset-cpus=48-95 \
20 - --cpuset-mems=1 --privileged=true --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test-avx2 cpu-test-avx2
21 +docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --cpuset-cpus=$CORE_RANGE \
22 + --cpuset-mems=$NUMA_NODE --privileged=true --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test cpu-test
23 +docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --cpuset-cpus=$CORE_RANGE \
24 + --cpuset-mems=$NUMA_NODE --privileged=true --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test-avx2 cpu-test-avx2
25
26 function cpu_tests() {
27 set -e
@@ -57,7 +61,7 @@ function cpu_tests() {
61 docker exec cpu-test bash -c "
62 set -e
63 export VLLM_CPU_KVCACHE_SPACE=10
60 - export VLLM_CPU_OMP_THREADS_BIND=48-92
64 + export VLLM_CPU_OMP_THREADS_BIND=$CORE_RANGE
65 python3 -m vllm.entrypoints.openai.api_server --model facebook/opt-125m --dtype half &
66 timeout 600 bash -c 'until curl localhost:8000/v1/models; do sleep 1; done' || exit 1
67 python3 benchmarks/benchmark_serving.py \