A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 23:45 UTC
[Misc][Tools] make max-model-len a parameter in auto_tune script (#21321)
Signed-off-by: Chengji Yao <chengjiyao@google.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Chengji Yao committed
Jul 24, 2025 at 22:46 UTC
947edd099e259a072e4cafadce04ccbdebc49441
2 files changed
+15
-3
benchmarks/auto_tune/README.md
+4
@@ -39,6 +39,7 @@ You must set the following variables at the top of the script before execution.
39
| `DOWNLOAD_DIR` | **Required.** Directory to download and load model weights from. | `""` (default download path) |
40
| `INPUT_LEN` | **Required.** Request input length. | `4000` |
41
| `OUTPUT_LEN` | **Required.** Request output length. | `16` |
42
+| `MAX_MODEL_LEN` | **Required.** Max model length. | `4096` |
43
| `MIN_CACHE_HIT_PCT` | Prefix cache hit rate in percentage (0-100). Set to `0` to disable. | `60` |
44
| `MAX_LATENCY_ALLOWED_MS` | The maximum allowed P99 end-to-end latency in milliseconds. Set to a very large number (e.g., `100000000000`) to effectively ignore the latency constraint. | `500` |
45
| `NUM_SEQS_LIST` | A space-separated string of `max-num-seqs` values to test. | `"128 256"` |
@@ -69,6 +70,7 @@ Here are a few examples of how to configure the script for different goals:
70
```bash
71
INPUT_LEN=1800
72
OUTPUT_LEN=20
73
+MAX_MODEL_LEN=2048
74
MIN_CACHE_HIT_PCT=0
75
MAX_LATENCY_ALLOWED_MS=100000000000 # A very large number
76
```
@@ -80,6 +82,7 @@ MAX_LATENCY_ALLOWED_MS=100000000000 # A very large number
82
```bash
83
INPUT_LEN=1800
84
OUTPUT_LEN=20
85
+MAX_MODEL_LEN=2048
86
MIN_CACHE_HIT_PCT=0
87
MAX_LATENCY_ALLOWED_MS=500
88
```
@@ -91,6 +94,7 @@ MAX_LATENCY_ALLOWED_MS=500
94
```bash
95
INPUT_LEN=1800
96
OUTPUT_LEN=20
97
+MAX_MODEL_LEN=2048
98
MIN_CACHE_HIT_PCT=60
99
MAX_LATENCY_ALLOWED_MS=500
100
```
benchmarks/auto_tune/auto_tune.sh
+11
-3
@@ -4,13 +4,15 @@
4
# See details in README (benchmarks/auto_tune/README.md).
5
6
TAG=$(date +"%Y_%m_%d_%H_%M")
7
-BASE=""
7
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
8
+BASE="$SCRIPT_DIR/../../.."
9
MODEL="meta-llama/Llama-3.1-8B-Instruct"
10
SYSTEM="TPU"
11
TP=1
12
DOWNLOAD_DIR=""
13
INPUT_LEN=4000
14
OUTPUT_LEN=16
15
+MAX_MODEL_LEN=4096
16
MIN_CACHE_HIT_PCT=0
17
MAX_LATENCY_ALLOWED_MS=100000000000
18
NUM_SEQS_LIST="128 256"
@@ -36,6 +38,13 @@ current_hash=$(git rev-parse HEAD)
38
echo "hash:$current_hash" >> "$RESULT"
39
echo "current_hash: $current_hash"
40
41
+TOTAL_LEN=$((INPUT_LEN + OUTPUT_LEN))
42
+RED='\033[0;31m'
43
+if (( TOTAL_LEN > MAX_MODEL_LEN )); then
44
+ echo -e "${RED}FAILED: INPUT_LEN($INPUT_LEN) + OUTPUT_LEN($OUTPUT_LEN) = $TOTAL_LEN, which is > MAX_MODEL_LEN = $MAX_MODEL_LEN.\033[0m" >&2
45
+ exit 1
46
+fi
47
+
48
best_throughput=0
49
best_max_num_seqs=0
50
best_num_batched_tokens=0
@@ -60,7 +69,7 @@ start_server() {
69
--enable-prefix-caching \
70
--load-format dummy \
71
--download-dir "$DOWNLOAD_DIR" \
63
- --max-model-len $(( INPUT_LEN+OUTPUT_LEN )) > "$vllm_log" 2>&1 &
72
+ --max-model-len $MAX_MODEL_LEN > "$vllm_log" 2>&1 &
73
74
# wait for 10 minutes...
75
server_started=0
@@ -245,4 +254,3 @@ done
254
echo "finish permutations"
255
echo "best_max_num_seqs: $best_max_num_seqs, best_num_batched_tokens: $best_num_batched_tokens, best_throughput: $best_throughput, profile saved in: $PROFILE_PATH"
256
echo "best_max_num_seqs: $best_max_num_seqs, best_num_batched_tokens: $best_num_batched_tokens, best_throughput: $best_throughput, profile saved in: $PROFILE_PATH" >> "$RESULT"
248
-