@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 651f6c31ac
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

For ppc64le, disabled tests for now and addressed space issues (#10538)

Nishidha committed Nov 23, 2024 at 15:03 UTC 651f6c31ac86f29aa72fa682ef6c34349bcc75db
1 file changed +3 -41
.buildkite/run-cpu-test-ppc64le.sh
+3 -41
@@ -4,49 +4,11 @@
4 # It serves a sanity check for compilation and basic model usage.
5 set -ex
6
7 -# Try building the docker image
8 -docker build -t cpu-test -f Dockerfile.ppc64le .
9 -
7 # Setup cleanup
11 -remove_docker_container() { docker rm -f cpu-test || true; }
8 +remove_docker_container() { docker rm -f cpu-test || true; docker system prune -f; }
9 trap remove_docker_container EXIT
10 remove_docker_container
11
15 -# Run the image, setting --shm-size=4g for tensor parallel.
16 -source /etc/environment
17 -#docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test cpu-test
18 -docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true --network host -e HF_TOKEN="$HF_TOKEN" --name cpu-test cpu-test
19 -
20 -function cpu_tests() {
21 - set -e
22 -
23 - # Run basic model test
24 - docker exec cpu-test bash -c "
25 - set -e
26 - pip install pytest pytest-asyncio \
27 - decord einops librosa peft Pillow sentence-transformers soundfile \
28 - transformers_stream_generator matplotlib datamodel_code_generator
29 - pip install torchvision --index-url https://download.pytorch.org/whl/cpu
30 - pytest -v -s tests/models/decoder_only/language -m cpu_model
31 - pytest -v -s tests/models/embedding/language -m cpu_model
32 - pytest -v -s tests/models/encoder_decoder/language -m cpu_model
33 - pytest -v -s tests/models/decoder_only/audio_language -m cpu_model
34 - pytest -v -s tests/models/decoder_only/vision_language -m cpu_model"
35 -
36 - # online inference
37 - docker exec cpu-test bash -c "
38 - set -e
39 - python3 -m vllm.entrypoints.openai.api_server --model facebook/opt-125m &
40 - timeout 600 bash -c 'until curl localhost:8000/v1/models; do sleep 1; done' || exit 1
41 - python3 benchmarks/benchmark_serving.py \
42 - --backend vllm \
43 - --dataset-name random \
44 - --model facebook/opt-125m \
45 - --num-prompts 20 \
46 - --endpoint /v1/completions \
47 - --tokenizer facebook/opt-125m"
48 -}
12 +# Try building the docker image
13 +docker build -t cpu-test -f Dockerfile.ppc64le .
14
50 -# All of CPU tests are expected to be finished less than 25 mins.
51 -export -f cpu_tests
52 -timeout 25m bash -c "cpu_tests"