A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Improve flexibility of auto_tune.sh execution. (#23766)
Signed-off-by: Anthony Su <50185138+anthonsu@users.noreply.github.com> Signed-off-by: anthonsu <50185138+anthonsu@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
anthonsu committed
Sep 4, 2025 at 02:41 UTC
04f3c35cff1a3ea240d3977842b23ae5e853eb9f
2 files changed
+36
-14
benchmarks/auto_tune/README.md
+6
@@ -31,6 +31,12 @@ cd vllm
31
32
You must set the following variables at the top of the script before execution.
33
34
+ Note: You can also override the default values below via environment variables when running the script.
35
+
36
+```bash
37
+MODEL=meta-llama/Llama-3.3-70B-Instruct SYSTEM=TPU TP=8 DOWNLOAD_DIR='' INPUT_LEN=128 OUTPUT_LEN=2048 MAX_MODEL_LEN=2300 MIN_CACHE_HIT_PCT=0 MAX_LATENCY_ALLOWED_MS=100000000000 NUM_SEQS_LIST="128 256" NUM_BATCHED_TOKENS_LIST="1024 2048 4096" VLLM_LOGGING_LEVEL=DEBUG bash auto_tune.sh
38
+```
39
+
40
| Variable | Description | Example Value |
41
| --- | --- | --- |
42
| `BASE` | **Required.** The absolute path to the parent directory of your vLLM repository directory. | `"$HOME"` |
benchmarks/auto_tune/auto_tune.sh
+30
-14
@@ -5,25 +5,41 @@
5
6
TAG=$(date +"%Y_%m_%d_%H_%M")
7
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
8
-BASE="$SCRIPT_DIR/../../.."
9
-MODEL="meta-llama/Llama-3.1-8B-Instruct"
10
-SYSTEM="TPU"
11
-TP=1
12
-DOWNLOAD_DIR=""
13
-INPUT_LEN=4000
14
-OUTPUT_LEN=16
15
-MAX_MODEL_LEN=4096
16
-MIN_CACHE_HIT_PCT=0
17
-MAX_LATENCY_ALLOWED_MS=100000000000
18
-NUM_SEQS_LIST="128 256"
19
-NUM_BATCHED_TOKENS_LIST="512 1024 2048 4096"
8
+VLLM_LOGGING_LEVEL=${VLLM_LOGGING_LEVEL:-INFO}
9
+BASE=${BASE:-"$SCRIPT_DIR/../../.."}
10
+MODEL=${MODEL:-"meta-llama/Llama-3.1-8B-Instruct"}
11
+SYSTEM=${SYSTEM:-"TPU"}
12
+TP=${TP:-1}
13
+DOWNLOAD_DIR=${DOWNLOAD_DIR:-""}
14
+INPUT_LEN=${INPUT_LEN:-4000}
15
+OUTPUT_LEN=${OUTPUT_LEN:-16}
16
+MAX_MODEL_LEN=${MAX_MODEL_LEN:-4096}
17
+MIN_CACHE_HIT_PCT=${MIN_CACHE_HIT_PCT:-0}
18
+MAX_LATENCY_ALLOWED_MS=${MAX_LATENCY_ALLOWED_MS:-100000000000}
19
+NUM_SEQS_LIST=${NUM_SEQS_LIST:-"128 256"}
20
+NUM_BATCHED_TOKENS_LIST=${NUM_BATCHED_TOKENS_LIST:-"512 1024 2048 4096"}
21
22
LOG_FOLDER="$BASE/auto-benchmark/$TAG"
23
RESULT="$LOG_FOLDER/result.txt"
24
PROFILE_PATH="$LOG_FOLDER/profile"
25
25
-echo "result file: $RESULT"
26
-echo "model: $MODEL"
26
+echo "====================== AUTO TUNE PARAMETERS ===================="
27
+echo "SCRIPT_DIR=$SCRIPT_DIR"
28
+echo "BASE=$BASE"
29
+echo "MODEL=$MODEL"
30
+echo "SYSTEM=$SYSTEM"
31
+echo "TP=$TP"
32
+echo "DOWNLOAD_DIR=$DOWNLOAD_DIR"
33
+echo "INPUT_LEN=$INPUT_LEN"
34
+echo "OUTPUT_LEN=$OUTPUT_LEN"
35
+echo "MAX_MODEL_LEN=$MAX_MODEL_LEN"
36
+echo "MIN_CACHE_HIT_PCT=$MIN_CACHE_HIT_PCT"
37
+echo "MAX_LATENCY_ALLOWED_MS=$MAX_LATENCY_ALLOWED_MS"
38
+echo "NUM_SEQS_LIST=$NUM_SEQS_LIST"
39
+echo "NUM_BATCHED_TOKENS_LIST=$NUM_BATCHED_TOKENS_LIST"
40
+echo "VLLM_LOGGING_LEVEL=$VLLM_LOGGING_LEVEL"
41
+echo "RESULT_FILE=$RESULT"
42
+echo "====================== AUTO TUNEPARAMETERS ===================="
43
44
rm -rf $LOG_FOLDER
45
rm -rf $PROFILE_PATH