A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 10:00 UTC
[Benchmarks] `auto_tune.sh`: Use hostname variable for server requests (#30529)
Signed-off-by: Kevin Musgrave <kevin.musgrave@gmail.com> Signed-off-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Kevin Musgrave committed
Dec 15, 2025 at 17:00 UTC
c01d589813f40c9ea25db3cdaa2c6c2144ab4e53
1 file changed
+11
-2
benchmarks/auto_tune/auto_tune.sh
+11
-2
@@ -18,6 +18,11 @@ MIN_CACHE_HIT_PCT=${MIN_CACHE_HIT_PCT:-0}
18
MAX_LATENCY_ALLOWED_MS=${MAX_LATENCY_ALLOWED_MS:-100000000000}
19
NUM_SEQS_LIST=${NUM_SEQS_LIST:-"128 256"}
20
NUM_BATCHED_TOKENS_LIST=${NUM_BATCHED_TOKENS_LIST:-"512 1024 2048 4096"}
21
+HOSTNAME=$(hostname)
22
+if [[ -z "$HOSTNAME" ]]; then
23
+ echo "Error: Failed to determine hostname." >&2
24
+ exit 1
25
+fi
26
27
LOG_FOLDER="$BASE/auto-benchmark/$TAG"
28
RESULT="$LOG_FOLDER/result.txt"
@@ -82,6 +87,7 @@ start_server() {
87
"$MODEL"
88
"--disable-log-requests"
89
"--port" "8004"
90
+ "--host" "$HOSTNAME"
91
"--gpu-memory-utilization" "$gpu_memory_utilization"
92
"--max-num-seqs" "$max_num_seqs"
93
"--max-num-batched-tokens" "$max_num_batched_tokens"
@@ -113,7 +119,7 @@ start_server() {
119
# since that we should always have permission to send signal to the server process.
120
kill -0 $server_pid 2> /dev/null || break
121
116
- RESPONSE=$(curl -s -X GET "http://0.0.0.0:8004/health" -w "%{http_code}" -o /dev/stdout)
122
+ RESPONSE=$(curl -s -X GET "http://${HOSTNAME}:8004/health" -w "%{http_code}" -o /dev/stdout)
123
STATUS_CODE=$(echo "$RESPONSE" | tail -n 1)
124
if [[ "$STATUS_CODE" -eq 200 ]]; then
125
server_started=1
@@ -173,6 +179,7 @@ run_benchmark() {
179
--goodput e2el:$MAX_LATENCY_ALLOWED_MS \
180
--num-prompts 1000 \
181
--random-prefix-len $prefix_len \
182
+ --host "$HOSTNAME" \
183
--port 8004 &> "$bm_log"
184
throughput=$(grep "Request throughput (req/s):" "$bm_log" | sed 's/[^0-9.]//g')
185
e2el=$(grep "P99 E2EL (ms):" "$bm_log" | awk '{print $NF}')
@@ -188,7 +195,7 @@ run_benchmark() {
195
request_rate=$((${throughput%.*} + 1))
196
while ((request_rate > 0)); do
197
# clear prefix cache
191
- curl -X POST http://0.0.0.0:8004/reset_prefix_cache
198
+ curl -X POST http://${HOSTNAME}:8004/reset_prefix_cache
199
sleep 5
200
bm_log="$LOG_FOLDER/bm_log_${max_num_seqs}_${max_num_batched_tokens}_requestrate_${request_rate}.txt"
201
vllm bench serve \
@@ -204,6 +211,7 @@ run_benchmark() {
211
--goodput e2el:$MAX_LATENCY_ALLOWED_MS \
212
--num-prompts 100 \
213
--random-prefix-len $prefix_len \
214
+ --host "$HOSTNAME" \
215
--port 8004 &> "$bm_log"
216
throughput=$(grep "Request throughput (req/s):" "$bm_log" | sed 's/[^0-9.]//g')
217
e2el=$(grep "P99 E2EL (ms):" "$bm_log" | awk '{print $NF}')
@@ -304,6 +312,7 @@ if (( $(echo "$best_throughput > 0" | bc -l) )); then
312
--goodput e2el:$MAX_LATENCY_ALLOWED_MS \
313
--num-prompts 100 \
314
--random-prefix-len $prefix_len \
315
+ --host "$HOSTNAME" \
316
--port 8004 \
317
--profile &> "$bm_log"
318
else