A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bug] Update auto_tune.sh to separate benchmarking and profiling. (#21629)
Signed-off-by: Eric Hanley <ericehanley@google.com>
ericehanley committed
Aug 4, 2025 at 10:12 UTC
309c1bb822c94436e8beff60d68404b4cecd62b8
1 file changed
+80
-43
benchmarks/auto_tune/auto_tune.sh
+80
-43
@@ -49,6 +49,7 @@ best_throughput=0
49
best_max_num_seqs=0
50
best_num_batched_tokens=0
51
best_goodput=0
52
+best_request_rate=0
53
54
start_server() {
55
local gpu_memory_utilization=$1
@@ -57,18 +58,35 @@ start_server() {
58
local vllm_log=$4
59
local profile_dir=$5
60
60
- pkill -f vllm
61
+ pkill -if vllm
62
62
- VLLM_USE_V1=1 VLLM_SERVER_DEV_MODE=1 VLLM_TORCH_PROFILER_DIR=$profile_dir vllm serve $MODEL \
63
- --port 8004 \
64
- --gpu-memory-utilization $gpu_memory_utilization \
65
- --max-num-seqs $max_num_seqs \
66
- --max-num-batched-tokens $max_num_batched_tokens \
67
- --tensor-parallel-size $TP \
68
- --enable-prefix-caching \
69
- --load-format dummy \
70
- --download-dir "$DOWNLOAD_DIR" \
71
- --max-model-len $MAX_MODEL_LEN > "$vllm_log" 2>&1 &
63
+ # Define the common arguments as a bash array.
64
+ # Each argument and its value are separate elements.
65
+ local common_args_array=(
66
+ "$MODEL"
67
+ "--disable-log-requests"
68
+ "--port" "8004"
69
+ "--gpu-memory-utilization" "$gpu_memory_utilization"
70
+ "--max-num-seqs" "$max_num_seqs"
71
+ "--max-num-batched-tokens" "$max_num_batched_tokens"
72
+ "--tensor-parallel-size" "$TP"
73
+ "--enable-prefix-caching"
74
+ "--load-format" "dummy"
75
+ "--download-dir" "$DOWNLOAD_DIR"
76
+ "--max-model-len" "$MAX_MODEL_LEN"
77
+ )
78
+
79
+ # Use the array expansion "${common_args_array[@]}"
80
+ # This correctly passes each element as a separate argument.
81
+ if [[ -n "$profile_dir" ]]; then
82
+ # Start server with profiling enabled
83
+ VLLM_USE_V1=1 VLLM_SERVER_DEV_MODE=1 VLLM_TORCH_PROFILER_DIR=$profile_dir \
84
+ vllm serve "${common_args_array[@]}" > "$vllm_log" 2>&1 &
85
+ else
86
+ # Start server without profiling
87
+ VLLM_USE_V1=1 VLLM_SERVER_DEV_MODE=1 \
88
+ vllm serve "${common_args_array[@]}" > "$vllm_log" 2>&1 &
89
+ fi
90
91
# wait for 10 minutes...
92
server_started=0
@@ -82,6 +100,7 @@ start_server() {
100
sleep 10
101
fi
102
done
103
+
104
if (( ! server_started )); then
105
echo "server did not start within 10 minutes. Please check server log at $vllm_log".
106
return 1
@@ -90,37 +109,20 @@ start_server() {
109
fi
110
}
111
93
-update_best_profile() {
94
- local profile_dir=$1
95
- local profile_index=$2
96
- sorted_paths=($(find "$profile_dir" -maxdepth 1 -not -path "$profile_dir" | sort))
97
- selected_profile_file=
98
- if [[ "$SYSTEM" == "TPU" ]]; then
99
- selected_profile_file="${sorted_paths[$profile_index]}/*.xplane.pb"
100
- fi
101
- if [[ "$SYSTEM" == "GPU" ]]; then
102
- selected_profile_file="${sorted_paths[$profile_index]}"
103
- fi
104
- rm -f $PROFILE_PATH/*
105
- cp $selected_profile_file $PROFILE_PATH
106
-}
107
-
112
run_benchmark() {
113
local max_num_seqs=$1
114
local max_num_batched_tokens=$2
115
local gpu_memory_utilization=$3
116
echo "max_num_seq: $max_num_seqs, max_num_batched_tokens: $max_num_batched_tokens"
117
local vllm_log="$LOG_FOLDER/vllm_log_${max_num_seqs}_${max_num_batched_tokens}.txt"
114
- local profile_dir="$LOG_FOLDER/profile_${max_num_seqs}_${max_num_batched_tokens}"
118
echo "vllm_log: $vllm_log"
119
echo
120
rm -f $vllm_log
118
- mkdir -p $profile_dir
119
- pkill -f vllm
120
- local profile_index=0
121
+ pkill -if vllm
122
123
echo "starting server..."
123
- start_server $gpu_memory_utilization $max_num_seqs $max_num_batched_tokens $vllm_log $profile_dir
124
+ # Call start_server without a profile_dir to avoid profiling overhead
125
+ start_server $gpu_memory_utilization $max_num_seqs $max_num_batched_tokens $vllm_log ""
126
result=$?
127
if [[ "$result" -eq 1 ]]; then
128
echo "server failed to start. gpu_memory_utilization:$gpu_memory_utilization, max_num_seqs:$max_num_seqs, max_num_batched_tokens: $max_num_batched_tokens"
@@ -134,7 +136,8 @@ run_benchmark() {
136
# get a basic qps by using request-rate inf
137
bm_log="$LOG_FOLDER/bm_log_${max_num_seqs}_${max_num_batched_tokens}_requestrate_inf.txt"
138
prefix_len=$(( INPUT_LEN * MIN_CACHE_HIT_PCT / 100 ))
137
-adjusted_input_len=$(( INPUT_LEN - prefix_len ))
139
+ adjusted_input_len=$(( INPUT_LEN - prefix_len ))
140
+ # --profile flag is removed from this call
141
vllm bench serve \
142
--backend vllm \
143
--model $MODEL \
@@ -148,8 +151,7 @@ adjusted_input_len=$(( INPUT_LEN - prefix_len ))
151
--goodput e2el:$MAX_LATENCY_ALLOWED_MS \
152
--num-prompts 1000 \
153
--random-prefix-len $prefix_len \
151
- --port 8004 \
152
- --profile &> "$bm_log"
154
+ --port 8004 &> "$bm_log"
155
throughput=$(grep "Request throughput (req/s):" "$bm_log" | sed 's/[^0-9.]//g')
156
e2el=$(grep "P99 E2EL (ms):" "$bm_log" | awk '{print $NF}')
157
goodput=$(grep "Request goodput (req/s):" "$bm_log" | sed 's/[^0-9.]//g')
@@ -163,7 +165,6 @@ adjusted_input_len=$(( INPUT_LEN - prefix_len ))
165
# start from request-rate as int(throughput) + 1
166
request_rate=$((${throughput%.*} + 1))
167
while ((request_rate > 0)); do
166
- profile_index=$((profile_index+1))
168
# clear prefix cache
169
curl -X POST http://0.0.0.0:8004/reset_prefix_cache
170
sleep 5
@@ -201,12 +202,7 @@ adjusted_input_len=$(( INPUT_LEN - prefix_len ))
202
best_max_num_seqs=$max_num_seqs
203
best_num_batched_tokens=$max_num_batched_tokens
204
best_goodput=$goodput
204
- if [[ "$SYSTEM" == "TPU" ]]; then
205
- update_best_profile "$profile_dir/plugins/profile" $profile_index
206
- fi
207
- if [[ "$SYSTEM" == "GPU" ]]; then
208
- update_best_profile "$profile_dir" $profile_index
209
- fi
205
+ best_request_rate=$request_rate
206
fi
207
else
208
echo "max_num_seqs: $max_num_seqs, max_num_batched_tokens: $max_num_batched_tokens does not meet latency requirement ${MAX_LATENCY_ALLOWED_MS}"
@@ -215,7 +211,7 @@ adjusted_input_len=$(( INPUT_LEN - prefix_len ))
211
212
echo "best_max_num_seqs: $best_max_num_seqs, best_num_batched_tokens: $best_num_batched_tokens, best_throughput: $best_throughput"
213
218
- pkill vllm
214
+ pkill -if vllm
215
sleep 10
216
printf '=%.0s' $(seq 1 20)
217
return 0
@@ -228,7 +224,8 @@ read -r -a num_batched_tokens_list <<< "$NUM_BATCHED_TOKENS_LIST"
224
gpu_memory_utilization=0.98
225
find_gpu_memory_utilization=0
226
while (( $(echo "$gpu_memory_utilization >= 0.9" | bc -l) )); do
231
- start_server $gpu_memory_utilization "${num_seqs_list[-1]}" "${num_batched_tokens_list[-1]}" "$LOG_FOLDER/vllm_log_gpu_memory_utilization_$gpu_memory_utilization.log"
227
+ # Pass empty string for profile_dir argument
228
+ start_server $gpu_memory_utilization "${num_seqs_list[-1]}" "${num_batched_tokens_list[-1]}" "$LOG_FOLDER/vllm_log_gpu_memory_utilization_$gpu_memory_utilization.log" ""
229
result=$?
230
if [[ "$result" -eq 0 ]]; then
231
find_gpu_memory_utilization=1
@@ -251,5 +248,45 @@ for num_seqs in "${num_seqs_list[@]}"; do
248
done
249
done
250
echo "finish permutations"
251
+
252
+# =================================================================================
253
+# FINAL PROFILING RUN FOR THE BEST CONFIGURATION
254
+# =================================================================================
255
+if (( $(echo "$best_throughput > 0" | bc -l) )); then
256
+ echo
257
+ echo "Benchmark tuning finished. Now running profiling on the best configuration found..."
258
+ echo "Best config: max_num_seqs: $best_max_num_seqs, max_num_batched_tokens: $best_num_batched_tokens, throughput: $best_throughput"
259
+ echo
260
+
261
+ vllm_log="$LOG_FOLDER/vllm_log_BEST_PROFILE.txt"
262
+ bm_log="$LOG_FOLDER/bm_log_BEST_PROFILE.txt"
263
+
264
+ # Start server with the best params and profiling ENABLED
265
+ echo "Starting server for profiling..."
266
+ start_server $gpu_memory_utilization $best_max_num_seqs $best_num_batched_tokens "$vllm_log" "$PROFILE_PATH"
267
+
268
+ # Run benchmark with the best params and the --profile flag
269
+ echo "Running benchmark with profiling..."
270
+ prefix_len=$(( INPUT_LEN * MIN_CACHE_HIT_PCT / 100 ))
271
+ adjusted_input_len=$(( INPUT_LEN - prefix_len ))
272
+ vllm bench serve \
273
+ --backend vllm \
274
+ --model $MODEL \
275
+ --dataset-name random \
276
+ --random-input-len $adjusted_input_len \
277
+ --random-output-len $OUTPUT_LEN \
278
+ --ignore-eos \
279
+ --disable-tqdm \
280
+ --request-rate $best_request_rate \
281
+ --percentile-metrics ttft,tpot,itl,e2el \
282
+ --goodput e2el:$MAX_LATENCY_ALLOWED_MS \
283
+ --num-prompts 100 \
284
+ --random-prefix-len $prefix_len \
285
+ --port 8004 \
286
+ --profile &> "$bm_log"
287
+else
288
+ echo "No configuration met the latency requirements. Skipping final profiling run."
289
+fi
290
+pkill -if vllm
291
echo "best_max_num_seqs: $best_max_num_seqs, best_num_batched_tokens: $best_num_batched_tokens, best_throughput: $best_throughput, profile saved in: $PROFILE_PATH"
292
echo "best_max_num_seqs: $best_max_num_seqs, best_num_batched_tokens: $best_num_batched_tokens, best_throughput: $best_throughput, profile saved in: $PROFILE_PATH" >> "$RESULT"