@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 00efdc84ba
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Add benchmark serving to CI (#2505)

Simon Mo committed Jan 19, 2024 at 20:20 UTC 00efdc84baf313cb775ca99a011b0e9a13539bdd
1 file changed +32 -4
.buildkite/run-benchmarks.sh
+32 -4
@@ -1,10 +1,13 @@
1 # This script is run by buildkite to run the benchmarks and upload the results to buildkite
2
3 set -ex
4 +set -o pipefail
5
6 # cd into parent directory of this file
7 cd "$(dirname "${BASH_SOURCE[0]}")/.."
8
9 +(wget && curl) || (apt-get update && apt-get install -y wget curl)
10 +
11 # run benchmarks and upload the result to buildkite
12 python3 benchmarks/benchmark_latency.py 2>&1 | tee benchmark_latency.txt
13 bench_latency_exit_code=$?
@@ -12,15 +15,36 @@ bench_latency_exit_code=$?
15 python3 benchmarks/benchmark_throughput.py --input-len 256 --output-len 256 2>&1 | tee benchmark_throughput.txt
16 bench_throughput_exit_code=$?
17
18 +python3 -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-2-7b-chat-hf &
19 +server_pid=$!
20 +wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json
21 +
22 +# wait for server to start, timeout after 600 seconds
23 +timeout 600 bash -c 'until curl localhost:8000/v1/models; do sleep 1; done' || exit 1
24 +python3 benchmarks/benchmark_serving.py \
25 + --dataset ./ShareGPT_V3_unfiltered_cleaned_split.json \
26 + --model meta-llama/Llama-2-7b-chat-hf \
27 + --num-prompts 20 \
28 + --endpoint /v1/completions \
29 + --tokenizer meta-llama/Llama-2-7b-chat-hf 2>&1 | tee benchmark_serving.txt
30 +bench_serving_exit_code=$?
31 +kill $server_pid
32 +
33 # write the results into a markdown file
34 echo "### Latency Benchmarks" >> benchmark_results.md
17 -sed -n '1p' benchmark_latency.txt >> benchmark_results.md
35 +sed -n '1p' benchmark_latency.txt >> benchmark_results.md # first line
36 echo "" >> benchmark_results.md
19 -sed -n '$p' benchmark_latency.txt >> benchmark_results.md
37 +sed -n '$p' benchmark_latency.txt >> benchmark_results.md # last line
38 +
39 echo "### Throughput Benchmarks" >> benchmark_results.md
21 -sed -n '1p' benchmark_throughput.txt >> benchmark_results.md
40 +sed -n '1p' benchmark_throughput.txt >> benchmark_results.md # first line
41 echo "" >> benchmark_results.md
23 -sed -n '$p' benchmark_throughput.txt >> benchmark_results.md
42 +sed -n '$p' benchmark_throughput.txt >> benchmark_results.md # last line
43 +
44 +echo "### Serving Benchmarks" >> benchmark_results.md
45 +sed -n '1p' benchmark_serving.txt >> benchmark_results.md # first line
46 +echo "" >> benchmark_results.md
47 +tail -n 5 benchmark_serving.txt >> benchmark_results.md # last 5 lines
48
49 # upload the results to buildkite
50 /workspace/buildkite-agent annotate --style "info" --context "benchmark-results" < benchmark_results.md
@@ -33,3 +57,7 @@ fi
57 if [ $bench_throughput_exit_code -ne 0 ]; then
58 exit $bench_throughput_exit_code
59 fi
60 +
61 +if [ $bench_serving_exit_code -ne 0 ]; then
62 + exit $bench_serving_exit_code
63 +fi