@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 03025c023f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[CI/Build] Fix CPU CI online inference timeout (#10314)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed Nov 14, 2024 at 16:45 UTC 03025c023f99bea58652e9b5a8a4a8b50af6bdd0
1 file changed +2 -2
.buildkite/run-cpu-test.sh
+2 -2
@@ -61,7 +61,7 @@ function cpu_tests() {
61 docker exec cpu-test bash -c "
62 set -e
63 export VLLM_CPU_KVCACHE_SPACE=10
64 - export VLLM_CPU_OMP_THREADS_BIND=$CORE_RANGE
64 + export VLLM_CPU_OMP_THREADS_BIND=$1
65 python3 -m vllm.entrypoints.openai.api_server --model facebook/opt-125m --dtype half &
66 timeout 600 bash -c 'until curl localhost:8000/v1/models; do sleep 1; done' || exit 1
67 python3 benchmarks/benchmark_serving.py \
@@ -75,4 +75,4 @@ function cpu_tests() {
75
76 # All of CPU tests are expected to be finished less than 25 mins.
77 export -f cpu_tests
78 -timeout 25m bash -c "cpu_tests"
78 +timeout 25m bash -c "cpu_tests $CORE_RANGE"