A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[CI/Build] Adding timeout in CPU CI to avoid CPU test queue blocking (#6892)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
Li, Jiang committed
Nov 9, 2024 at 11:27 UTC
d7edca1dee96e6caeeadcee4914a6b00d1c99fd5
2 files changed
+79
-69
.buildkite/run-cpu-test-ppc64le.sh
+31
-26
@@ -17,30 +17,35 @@ source /etc/environment
17
#docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test cpu-test
18
docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true --network host -e HF_TOKEN="$HF_TOKEN" --name cpu-test cpu-test
19
20
-# Run basic model test
21
-docker exec cpu-test bash -c "
22
- set -e
23
- pip install pytest pytest-asyncio \
24
- decord einops librosa peft Pillow sentence-transformers soundfile \
25
- transformers_stream_generator matplotlib datamodel_code_generator
26
- pip install torchvision --index-url https://download.pytorch.org/whl/cpu
27
- # Embedding models are not supported for CPU yet
28
- # pytest -v -s tests/models/embedding/language
29
- pytest -v -s tests/models/encoder_decoder/language
30
- pytest -v -s tests/models/decoder_only/language/test_models.py
31
- # Chunked prefill not supported for CPU yet
32
- # pytest -v -s tests/models/decoder_only/audio_language -m cpu_model
33
- pytest -v -s tests/models/decoder_only/vision_language -m cpu_model"
20
+function cpu_tests() {
21
+ # Run basic model test
22
+ docker exec cpu-test bash -c "
23
+ set -e
24
+ pip install pytest pytest-asyncio \
25
+ decord einops librosa peft Pillow sentence-transformers soundfile \
26
+ transformers_stream_generator matplotlib datamodel_code_generator
27
+ pip install torchvision --index-url https://download.pytorch.org/whl/cpu
28
+ # Embedding models are not supported for CPU yet
29
+ # pytest -v -s tests/models/embedding/language
30
+ pytest -v -s tests/models/encoder_decoder/language
31
+ pytest -v -s tests/models/decoder_only/language/test_models.py
32
+ pytest -v -s tests/models/decoder_only/audio_language -m cpu_model
33
+ pytest -v -s tests/models/decoder_only/vision_language -m cpu_model"
34
35
-# online inference
36
-docker exec cpu-test bash -c "
37
- set -e
38
- python3 -m vllm.entrypoints.openai.api_server --model facebook/opt-125m &
39
- timeout 600 bash -c 'until curl localhost:8000/v1/models; do sleep 1; done' || exit 1
40
- python3 benchmarks/benchmark_serving.py \
41
- --backend vllm \
42
- --dataset-name random \
43
- --model facebook/opt-125m \
44
- --num-prompts 20 \
45
- --endpoint /v1/completions \
46
- --tokenizer facebook/opt-125m"
35
+ # online inference
36
+ docker exec cpu-test bash -c "
37
+ set -e
38
+ python3 -m vllm.entrypoints.openai.api_server --model facebook/opt-125m &
39
+ timeout 600 bash -c 'until curl localhost:8000/v1/models; do sleep 1; done' || exit 1
40
+ python3 benchmarks/benchmark_serving.py \
41
+ --backend vllm \
42
+ --dataset-name random \
43
+ --model facebook/opt-125m \
44
+ --num-prompts 20 \
45
+ --endpoint /v1/completions \
46
+ --tokenizer facebook/opt-125m"
47
+}
48
+
49
+# All of CPU tests are expected to be finished less than 25 mins.
50
+export -f cpu_tests
51
+timeout 25m bash -c "cpu_tests"
.buildkite/run-cpu-test.sh
+48
-43
@@ -19,50 +19,55 @@ docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/hugg
19
docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --cpuset-cpus=48-95 \
20
--cpuset-mems=1 --privileged=true --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test-avx2 cpu-test-avx2
21
22
-# offline inference
23
-docker exec cpu-test-avx2 bash -c "
24
- set -e
25
- python3 examples/offline_inference.py"
22
+function cpu_tests() {
23
+ # offline inference
24
+ docker exec cpu-test-avx2 bash -c "
25
+ set -e
26
+ python3 examples/offline_inference.py"
27
27
-# Run basic model test
28
-docker exec cpu-test bash -c "
29
- set -e
30
- pip install pytest pytest-asyncio \
31
- decord einops librosa peft Pillow sentence-transformers soundfile \
32
- transformers_stream_generator matplotlib datamodel_code_generator
33
- pip install torchvision --index-url https://download.pytorch.org/whl/cpu
34
- # Embedding models are not supported for CPU yet
35
- # pytest -v -s tests/models/embedding/language
36
- pytest -v -s tests/models/encoder_decoder/language
37
- pytest -v -s tests/models/decoder_only/language/test_models.py
38
- # Chunked prefill not supported for CPU yet
39
- # pytest -v -s tests/models/decoder_only/audio_language -m cpu_model
40
- pytest -v -s tests/models/decoder_only/vision_language -m cpu_model"
28
+ # Run basic model test
29
+ docker exec cpu-test bash -c "
30
+ set -e
31
+ pip install pytest pytest-asyncio \
32
+ decord einops librosa peft Pillow sentence-transformers soundfile \
33
+ transformers_stream_generator matplotlib datamodel_code_generator
34
+ pip install torchvision --index-url https://download.pytorch.org/whl/cpu
35
+ # Embedding models are not supported for CPU yet
36
+ # pytest -v -s tests/models/embedding/language
37
+ pytest -v -s tests/models/encoder_decoder/language
38
+ pytest -v -s tests/models/decoder_only/language/test_models.py
39
+ pytest -v -s tests/models/decoder_only/audio_language -m cpu_model
40
+ pytest -v -s tests/models/decoder_only/vision_language -m cpu_model"
41
42
-# Run compressed-tensor test
43
-docker exec cpu-test bash -c "
44
- set -e
45
- pytest -s -v \
46
- tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_static_setup \
47
- tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_dynamic_per_token"
42
+ # Run compressed-tensor test
43
+ docker exec cpu-test bash -c "
44
+ set -e
45
+ pytest -s -v \
46
+ tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_static_setup \
47
+ tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_dynamic_per_token"
48
49
-# Run AWQ test
50
-docker exec cpu-test bash -c "
51
- set -e
52
- pytest -s -v \
53
- tests/quantization/test_ipex_quant.py"
49
+ # Run AWQ test
50
+ docker exec cpu-test bash -c "
51
+ set -e
52
+ pytest -s -v \
53
+ tests/quantization/test_ipex_quant.py"
54
55
-# online inference
56
-docker exec cpu-test bash -c "
57
- set -e
58
- export VLLM_CPU_KVCACHE_SPACE=10
59
- export VLLM_CPU_OMP_THREADS_BIND=48-92
60
- python3 -m vllm.entrypoints.openai.api_server --model facebook/opt-125m --dtype half &
61
- timeout 600 bash -c 'until curl localhost:8000/v1/models; do sleep 1; done' || exit 1
62
- python3 benchmarks/benchmark_serving.py \
63
- --backend vllm \
64
- --dataset-name random \
65
- --model facebook/opt-125m \
66
- --num-prompts 20 \
67
- --endpoint /v1/completions \
68
- --tokenizer facebook/opt-125m"
55
+ # online inference
56
+ docker exec cpu-test bash -c "
57
+ set -e
58
+ export VLLM_CPU_KVCACHE_SPACE=10
59
+ export VLLM_CPU_OMP_THREADS_BIND=48-92
60
+ python3 -m vllm.entrypoints.openai.api_server --model facebook/opt-125m --dtype half &
61
+ timeout 600 bash -c 'until curl localhost:8000/v1/models; do sleep 1; done' || exit 1
62
+ python3 benchmarks/benchmark_serving.py \
63
+ --backend vllm \
64
+ --dataset-name random \
65
+ --model facebook/opt-125m \
66
+ --num-prompts 20 \
67
+ --endpoint /v1/completions \
68
+ --tokenizer facebook/opt-125m"
69
+}
70
+
71
+# All of CPU tests are expected to be finished less than 25 mins.
72
+export -f cpu_tests
73
+timeout 25m bash -c "cpu_tests"