A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[CI/Build] Fix CPU CI online inference timeout (#10314)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Nov 14, 2024 at 16:45 UTC
03025c023f99bea58652e9b5a8a4a8b50af6bdd0
1 file changed
+2
-2
.buildkite/run-cpu-test.sh
+2
-2
@@ -61,7 +61,7 @@ function cpu_tests() {
61
docker exec cpu-test bash -c "
62
set -e
63
export VLLM_CPU_KVCACHE_SPACE=10
64
- export VLLM_CPU_OMP_THREADS_BIND=$CORE_RANGE
64
+ export VLLM_CPU_OMP_THREADS_BIND=$1
65
python3 -m vllm.entrypoints.openai.api_server --model facebook/opt-125m --dtype half &
66
timeout 600 bash -c 'until curl localhost:8000/v1/models; do sleep 1; done' || exit 1
67
python3 benchmarks/benchmark_serving.py \
@@ -75,4 +75,4 @@ function cpu_tests() {
75
76
# All of CPU tests are expected to be finished less than 25 mins.
77
export -f cpu_tests
78
-timeout 25m bash -c "cpu_tests"
78
+timeout 25m bash -c "cpu_tests $CORE_RANGE"