| 1 | #!/bin/bash |
| 2 | |
| 3 | # This script is run by buildkite to run the benchmarks and upload the results to buildkite |
| 4 | |
| 5 | set -ex |
| 6 | set -o pipefail |
| 7 | |
| 8 | # cd 2 levels into the working directory |
| 9 | cd "$(dirname "${BASH_SOURCE[0]}")/../.." |
| 10 | |
| 11 | (which wget && which curl) || (apt-get update && apt-get install -y wget curl) |
| 12 | |
| 13 | # run python-based benchmarks and upload the result to buildkite |
| 14 | vllm bench latency --output-json latency_results.json 2>&1 | tee benchmark_latency.txt |
| 15 | bench_latency_exit_code=$? |
| 16 | |
| 17 | vllm bench throughput --input-len 256 --output-len 256 --output-json throughput_results.json 2>&1 | tee benchmark_throughput.txt |
| 18 | bench_throughput_exit_code=$? |
| 19 | |
| 20 | # run server-based benchmarks and upload the result to buildkite |
| 21 | vllm serve meta-llama/Llama-2-7b-chat-hf & |
| 22 | server_pid=$! |
| 23 | wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json |
| 24 | |
| 25 | # wait for server to start, timeout after 600 seconds |
| 26 | timeout 600 bash -c 'until curl localhost:8000/v1/models; do sleep 1; done' || exit 1 |
| 27 | vllm bench serve \ |
| 28 | --backend vllm \ |
| 29 | --dataset-name sharegpt \ |
| 30 | --dataset-path ./ShareGPT_V3_unfiltered_cleaned_split.json \ |
| 31 | --model meta-llama/Llama-2-7b-chat-hf \ |
| 32 | --num-prompts 20 \ |
| 33 | --endpoint /v1/completions \ |
| 34 | --tokenizer meta-llama/Llama-2-7b-chat-hf \ |
| 35 | --save-result \ |
| 36 | 2>&1 | tee benchmark_serving.txt |
| 37 | bench_serving_exit_code=$? |
| 38 | kill $server_pid |
| 39 | |
| 40 | # write the results into a markdown file |
| 41 | echo "### Latency Benchmarks" >> benchmark_results.md |
| 42 | sed -n '1p' benchmark_latency.txt >> benchmark_results.md # first line |
| 43 | echo "" >> benchmark_results.md |
| 44 | sed -n '$p' benchmark_latency.txt >> benchmark_results.md # last line |
| 45 | |
| 46 | echo "### Throughput Benchmarks" >> benchmark_results.md |
| 47 | sed -n '1p' benchmark_throughput.txt >> benchmark_results.md # first line |
| 48 | echo "" >> benchmark_results.md |
| 49 | sed -n '$p' benchmark_throughput.txt >> benchmark_results.md # last line |
| 50 | |
| 51 | echo "### Serving Benchmarks" >> benchmark_results.md |
| 52 | sed -n '1p' benchmark_serving.txt >> benchmark_results.md # first line |
| 53 | echo "" >> benchmark_results.md |
| 54 | echo '```' >> benchmark_results.md |
| 55 | tail -n 24 benchmark_serving.txt >> benchmark_results.md # last 24 lines |
| 56 | echo '```' >> benchmark_results.md |
| 57 | |
| 58 | # if the agent binary is not found, skip uploading the results, exit 0 |
| 59 | if [ ! -f /usr/bin/buildkite-agent ]; then |
| 60 | exit 0 |
| 61 | fi |
| 62 | |
| 63 | # upload the results to buildkite |
| 64 | buildkite-agent annotate --style "info" --context "benchmark-results" < benchmark_results.md |
| 65 | |
| 66 | # exit with the exit code of the benchmarks |
| 67 | if [ $bench_latency_exit_code -ne 0 ]; then |
| 68 | exit $bench_latency_exit_code |
| 69 | fi |
| 70 | |
| 71 | if [ $bench_throughput_exit_code -ne 0 ]; then |
| 72 | exit $bench_throughput_exit_code |
| 73 | fi |
| 74 | |
| 75 | if [ $bench_serving_exit_code -ne 0 ]; then |
| 76 | exit $bench_serving_exit_code |
| 77 | fi |
| 78 | |
| 79 | rm ShareGPT_V3_unfiltered_cleaned_split.json |
| 80 | buildkite-agent artifact upload "*.json" |