A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Add benchmark serving to CI (#2505)
Simon Mo committed
Jan 19, 2024 at 20:20 UTC
00efdc84baf313cb775ca99a011b0e9a13539bdd
1 file changed
+32
-4
.buildkite/run-benchmarks.sh
+32
-4
@@ -1,10 +1,13 @@
1
# This script is run by buildkite to run the benchmarks and upload the results to buildkite
2
3
set -ex
4
+set -o pipefail
5
6
# cd into parent directory of this file
7
cd "$(dirname "${BASH_SOURCE[0]}")/.."
8
9
+(wget && curl) || (apt-get update && apt-get install -y wget curl)
10
+
11
# run benchmarks and upload the result to buildkite
12
python3 benchmarks/benchmark_latency.py 2>&1 | tee benchmark_latency.txt
13
bench_latency_exit_code=$?
@@ -12,15 +15,36 @@ bench_latency_exit_code=$?
15
python3 benchmarks/benchmark_throughput.py --input-len 256 --output-len 256 2>&1 | tee benchmark_throughput.txt
16
bench_throughput_exit_code=$?
17
18
+python3 -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-2-7b-chat-hf &
19
+server_pid=$!
20
+wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json
21
+
22
+# wait for server to start, timeout after 600 seconds
23
+timeout 600 bash -c 'until curl localhost:8000/v1/models; do sleep 1; done' || exit 1
24
+python3 benchmarks/benchmark_serving.py \
25
+ --dataset ./ShareGPT_V3_unfiltered_cleaned_split.json \
26
+ --model meta-llama/Llama-2-7b-chat-hf \
27
+ --num-prompts 20 \
28
+ --endpoint /v1/completions \
29
+ --tokenizer meta-llama/Llama-2-7b-chat-hf 2>&1 | tee benchmark_serving.txt
30
+bench_serving_exit_code=$?
31
+kill $server_pid
32
+
33
# write the results into a markdown file
34
echo "### Latency Benchmarks" >> benchmark_results.md
17
-sed -n '1p' benchmark_latency.txt >> benchmark_results.md
35
+sed -n '1p' benchmark_latency.txt >> benchmark_results.md # first line
36
echo "" >> benchmark_results.md
19
-sed -n '$p' benchmark_latency.txt >> benchmark_results.md
37
+sed -n '$p' benchmark_latency.txt >> benchmark_results.md # last line
38
+
39
echo "### Throughput Benchmarks" >> benchmark_results.md
21
-sed -n '1p' benchmark_throughput.txt >> benchmark_results.md
40
+sed -n '1p' benchmark_throughput.txt >> benchmark_results.md # first line
41
echo "" >> benchmark_results.md
23
-sed -n '$p' benchmark_throughput.txt >> benchmark_results.md
42
+sed -n '$p' benchmark_throughput.txt >> benchmark_results.md # last line
43
+
44
+echo "### Serving Benchmarks" >> benchmark_results.md
45
+sed -n '1p' benchmark_serving.txt >> benchmark_results.md # first line
46
+echo "" >> benchmark_results.md
47
+tail -n 5 benchmark_serving.txt >> benchmark_results.md # last 5 lines
48
49
# upload the results to buildkite
50
/workspace/buildkite-agent annotate --style "info" --context "benchmark-results" < benchmark_results.md
@@ -33,3 +57,7 @@ fi
57
if [ $bench_throughput_exit_code -ne 0 ]; then
58
exit $bench_throughput_exit_code
59
fi
60
+
61
+if [ $bench_serving_exit_code -ne 0 ]; then
62
+ exit $bench_serving_exit_code
63
+fi