A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 07:15 UTC
[Misc][Tools][Benchmark] Fix and improve auto tune script (#19163)
Signed-off-by: Chenyaaang <chenyangli@google.com>
Chenyaaang committed
Jun 6, 2025 at 16:31 UTC
441b65d8c7818bff8385a28c72ad8354bd51d53f
1 file changed
+98
-66
benchmarks/auto_tune.sh
+98
-66
@@ -10,11 +10,15 @@
10
# 3. Set variables (ALL REQUIRED)
11
# BASE: your directory for vllm repo
12
# MODEL: the model served by vllm
13
+# TP: ways of tensor parallelism
14
# DOWNLOAD_DIR: directory to download and load model weights.
15
# INPUT_LEN: request input len
16
# OUTPUT_LEN: request output len
17
# MIN_CACHE_HIT_PCT: prefix cache rate
18
# MAX_LATENCY_ALLOWED_MS: (e2e) latency requirement. If there's no latency requirement, set it to a large number like 1000000000
19
+# NUM_SEQS_LIST: a list of `max-num-seqs` you want to loop with.
20
+# NUM_BATCHED_TOKENS_LIST: a list of `max-num-batched-tokens` you want to loop with.
21
+# Note that the default NUM_SEQS_LIST and NUM_BATCHED_TOKENS_LIST are set for medium size input/output len, for extra short context (such as 20:20), you might need to include larger numbers in NUM_SEQS_LIST.
22
# 4. Run the script, it might take a long time, you can use tmux to avoid the script stop if disconnection happens.
23
# 5. The final result will be saved in RESULT file.
24
@@ -30,31 +34,27 @@
34
TAG=$(date +"%Y_%m_%d_%H_%M")
35
BASE=""
36
MODEL="meta-llama/Llama-3.1-8B-Instruct"
37
+TP=1
38
DOWNLOAD_DIR=""
39
INPUT_LEN=4000
40
OUTPUT_LEN=16
36
-MIN_CACHE_HIT_PCT_PCT=0
41
+MIN_CACHE_HIT_PCT=0
42
MAX_LATENCY_ALLOWED_MS=100000000000
43
+NUM_SEQS_LIST="128 256"
44
+NUM_BATCHED_TOKENS_LIST="512 1024 2048 4096"
45
46
LOG_FOLDER="$BASE/auto-benchmark/$TAG"
47
RESULT="$LOG_FOLDER/result.txt"
48
42
-echo "result file$ $RESULT"
49
+echo "result file: $RESULT"
50
echo "model: $MODEL"
44
-echo
51
52
rm -rf $LOG_FOLDER
53
mkdir -p $LOG_FOLDER
54
55
cd "$BASE/vllm"
50
-# create sonnet-4x.txt so that we can sample 2048 tokens for input
51
-echo "" > benchmarks/sonnet_4x.txt
52
-for _ in {1..4}
53
-do
54
-cat benchmarks/sonnet.txt >> benchmarks/sonnet_4x.txt
55
-done
56
57
-pip install datasets
57
+pip install -q datasets
58
59
current_hash=$(git rev-parse HEAD)
60
echo "hash:$current_hash" >> "$RESULT"
@@ -64,53 +64,69 @@ best_throughput=0
64
best_max_num_seqs=0
65
best_num_batched_tokens=0
66
best_goodput=0
67
-run_benchmark() {
68
- local max_num_seqs=$1
69
- local max_num_batched_tokens=$2
70
- echo "max_num_seq: $max_num_seqs, max_num_batched_tokens: $max_num_batched_tokens"
71
- local vllm_log="$LOG_FOLDER/vllm_log_${max_num_seqs}_${max_num_batched_tokens}.txt"
72
- echo "vllm_log: $vllm_log"
73
- echo
74
- rm -f $vllm_log
67
76
- # start the server
68
+start_server() {
69
+ local gpu_memory_utilization=$1
70
+ local max_num_seqs=$2
71
+ local max_num_batched_tokens=$3
72
+ local vllm_log=$4
73
+
74
+ pkill -f vllm
75
+
76
VLLM_USE_V1=1 VLLM_SERVER_DEV_MODE=1 vllm serve $MODEL \
77
--disable-log-requests \
78
--port 8004 \
80
- --gpu-memory-utilization 0.98 \
79
+ --gpu-memory-utilization $gpu_memory_utilization \
80
--max-num-seqs $max_num_seqs \
81
--max-num-batched-tokens $max_num_batched_tokens \
83
- --tensor-parallel-size 1 \
82
+ --tensor-parallel-size $TP \
83
--enable-prefix-caching \
84
--load-format dummy \
86
- --download-dir $DOWNLOAD_DIR \
85
+ --download-dir "$DOWNLOAD_DIR" \
86
--max-model-len $(( INPUT_LEN+OUTPUT_LEN )) > "$vllm_log" 2>&1 &
88
- echo "wait for 10 minutes.."
89
- echo
87
+
88
# wait for 10 minutes...
89
server_started=0
92
- for i in {1..60}; do
93
- if grep -Fq "Application startup complete" "$vllm_log"; then
94
- echo "Application started"
90
+ for i in {1..60}; do
91
+ RESPONSE=$(curl -s -X GET "http://0.0.0.0:8004/health" -w "%{http_code}" -o /dev/stdout)
92
+ STATUS_CODE=$(echo "$RESPONSE" | tail -n 1)
93
+ if [[ "$STATUS_CODE" -eq 200 ]]; then
94
server_started=1
95
break
96
else
98
- # echo "wait for 10 seconds..."
97
sleep 10
98
fi
99
done
102
-
100
if (( ! server_started )); then
104
- echo "server did not start within 10 minutes, terminate the benchmarking. Please check server log at $vllm_log"
105
- echo "pkill -f vllm"
106
- echo
107
- pkill vllm
108
- sleep 10
101
+ echo "server did not start within 10 minutes. Please check server log at $vllm_log".
102
return 1
103
+ else
104
+ return 0
105
fi
106
+}
107
+
108
+run_benchmark() {
109
+ local max_num_seqs=$1
110
+ local max_num_batched_tokens=$2
111
+ local gpu_memory_utilization=$3
112
+ echo "max_num_seq: $max_num_seqs, max_num_batched_tokens: $max_num_batched_tokens"
113
+ local vllm_log="$LOG_FOLDER/vllm_log_${max_num_seqs}_${max_num_batched_tokens}.txt"
114
+ echo "vllm_log: $vllm_log"
115
+ echo
116
+ rm -f $vllm_log
117
+ pkill -f vllm
118
+
119
+ echo "starting server..."
120
+ start_server $gpu_memory_utilization $max_num_seqs $max_num_batched_tokens $vllm_log
121
+ result=$?
122
+ if [[ "$result" -eq 1 ]]; then
123
+ echo "server failed to start. gpu_memory_utilization:$gpu_memory_utilization, max_num_seqs:$max_num_seqs, max_num_batched_tokens: $max_num_batched_tokens"
124
+ else
125
+ echo "server started."
126
+ fi
127
+ echo
128
129
echo "run benchmark test..."
113
- echo
130
meet_latency_requirement=0
131
# get a basic qps by using request-rate inf
132
bm_log="$LOG_FOLDER/bm_log_${max_num_seqs}_${max_num_batched_tokens}_requestrate_inf.txt"
@@ -118,29 +134,29 @@ run_benchmark() {
134
python benchmarks/benchmark_serving.py \
135
--backend vllm \
136
--model $MODEL \
121
- --dataset-name sonnet \
122
- --dataset-path benchmarks/sonnet_4x.txt \
123
- --sonnet-input-len $INPUT_LEN \
124
- --sonnet-output-len $OUTPUT_LEN \
137
+ --dataset-name random \
138
+ --random-input-len $INPUT_LEN \
139
+ --random-output-len $OUTPUT_LEN \
140
--ignore-eos \
141
--disable-tqdm \
142
--request-rate inf \
143
--percentile-metrics ttft,tpot,itl,e2el \
144
--goodput e2el:$MAX_LATENCY_ALLOWED_MS \
130
- --num-prompts 100 \
131
- --sonnet-prefix-len $prefix_len \
132
- --port 8004 > "$bm_log"
133
- through_put=$(grep "Request throughput (req/s):" "$bm_log" | sed 's/[^0-9.]//g')
145
+ --num-prompts 1000 \
146
+ --random-prefix-len $prefix_len \
147
+ --port 8004 &> "$bm_log"
148
+ throughput=$(grep "Request throughput (req/s):" "$bm_log" | sed 's/[^0-9.]//g')
149
e2el=$(grep "P99 E2EL (ms):" "$bm_log" | awk '{print $NF}')
150
goodput=$(grep "Request goodput (req/s):" "$bm_log" | sed 's/[^0-9.]//g')
151
152
if (( $(echo "$e2el <= $MAX_LATENCY_ALLOWED_MS" | bc -l) )); then
153
meet_latency_requirement=1
154
+ request_rate=inf
155
fi
156
157
if (( ! meet_latency_requirement )); then
142
- # start from request-rate as int(through_put) + 1
143
- request_rate=$((${through_put%.*} + 1))
158
+ # start from request-rate as int(throughput) + 1
159
+ request_rate=$((${throughput%.*} + 1))
160
while ((request_rate > 0)); do
161
# clear prefix cache
162
curl -X POST http://0.0.0.0:8004/reset_prefix_cache
@@ -149,19 +165,18 @@ run_benchmark() {
165
python benchmarks/benchmark_serving.py \
166
--backend vllm \
167
--model $MODEL \
152
- --dataset-name sonnet \
153
- --dataset-path benchmarks/sonnet_4x.txt \
154
- --sonnet-input-len $INPUT_LEN \
155
- --sonnet-output-len $OUTPUT_LEN \
156
- --ignore_eos \
168
+ --dataset-name random \
169
+ --random-input-len $INPUT_LEN \
170
+ --random-output-len $OUTPUT_LEN \
171
+ --ignore-eos \
172
--disable-tqdm \
173
--request-rate $request_rate \
174
--percentile-metrics ttft,tpot,itl,e2el \
175
--goodput e2el:$MAX_LATENCY_ALLOWED_MS \
176
--num-prompts 100 \
162
- --sonnet-prefix-len $prefix_len \
163
- --port 8004 > "$bm_log"
164
- through_put=$(grep "Request throughput (req/s):" "$bm_log" | sed 's/[^0-9.]//g')
177
+ --random-prefix-len $prefix_len \
178
+ --port 8004 &> "$bm_log"
179
+ throughput=$(grep "Request throughput (req/s):" "$bm_log" | sed 's/[^0-9.]//g')
180
e2el=$(grep "P99 E2EL (ms):" "$bm_log" | awk '{print $NF}')
181
goodput=$(grep "Request goodput (req/s):" "$bm_log" | sed 's/[^0-9.]//g')
182
if (( $(echo "$e2el <= $MAX_LATENCY_ALLOWED_MS" | bc -l) )); then
@@ -173,10 +188,10 @@ run_benchmark() {
188
fi
189
# write the results and update the best result.
190
if ((meet_latency_requirement)); then
176
- echo "max_num_seqs: $max_num_seqs, max_num_batched_tokens: $max_num_batched_tokens, request_rate: $request_rate, e2el: $e2el, through put: $through_put, goodput: $goodput"
177
- echo "max_num_seqs: $max_num_seqs, max_num_batched_tokens: $max_num_batched_tokens, request_rate: $request_rate, e2el: $e2el, through put: $through_put, goodput: $goodput" >> "$RESULT"
178
- if (( $(echo "$through_put > $best_throughput" | bc -l) )); then
179
- best_throughput=$through_put
191
+ echo "max_num_seqs: $max_num_seqs, max_num_batched_tokens: $max_num_batched_tokens, request_rate: $request_rate, e2el: $e2el, throughput: $throughput, goodput: $goodput"
192
+ echo "max_num_seqs: $max_num_seqs, max_num_batched_tokens: $max_num_batched_tokens, request_rate: $request_rate, e2el: $e2el, throughput: $throughput, goodput: $goodput" >> "$RESULT"
193
+ if (( $(echo "$throughput > $best_throughput" | bc -l) )); then
194
+ best_throughput=$throughput
195
best_max_num_seqs=$max_num_seqs
196
best_num_batched_tokens=$max_num_batched_tokens
197
best_goodput=$goodput
@@ -188,22 +203,39 @@ run_benchmark() {
203
204
echo "best_max_num_seqs: $best_max_num_seqs, best_num_batched_tokens: $best_num_batched_tokens, best_throughput: $best_throughput"
205
191
- echo "pkill -f vllm"
192
- echo
206
pkill vllm
207
sleep 10
195
- rm -f $vllm_log
208
printf '=%.0s' $(seq 1 20)
209
return 0
210
}
211
212
+read -r -a num_seqs_list <<< "$NUM_SEQS_LIST"
213
+read -r -a num_batched_tokens_list <<< "$NUM_BATCHED_TOKENS_LIST"
214
+
215
+# first find out the max gpu-memory-utilization without HBM OOM.
216
+gpu_memory_utilization=0.98
217
+find_gpu_memory_utilization=0
218
+while (( $(echo "$gpu_memory_utilization >= 0.9" | bc -l) )); do
219
+ start_server $gpu_memory_utilization "${num_seqs_list[-1]}" "${num_batched_tokens_list[-1]}" "$LOG_FOLDER/vllm_log_gpu_memory_utilization_$gpu_memory_utilization.log"
220
+ result=$?
221
+ if [[ "$result" -eq 0 ]]; then
222
+ find_gpu_memory_utilization=1
223
+ break
224
+ else
225
+ gpu_memory_utilization=$(echo "$gpu_memory_utilization - 0.01" | bc)
226
+ fi
227
+done
228
+
229
+if [[ "$find_gpu_memory_utilization" -eq 1 ]]; then
230
+ echo "Using gpu_memory_utilization=$gpu_memory_utilization to serve model."
231
+else
232
+ echo "Cannot find a proper gpu_memory_utilization over 0.9 to serve the model, please check logs in $LOG_FOLDER."
233
+ exit 1
234
+fi
235
201
-num_seqs_list="128 256"
202
-num_batched_tokens_list="512 1024 2048 4096"
203
-for num_seqs in $num_seqs_list; do
204
- for num_batched_tokens in $num_batched_tokens_list; do
205
- run_benchmark $num_seqs $num_batched_tokens
206
- exit 0
236
+for num_seqs in "${num_seqs_list[@]}"; do
237
+ for num_batched_tokens in "${num_batched_tokens_list[@]}"; do
238
+ run_benchmark $num_seqs $num_batched_tokens $gpu_memory_utilization
239
done
240
done
241
echo "finish permutations"