A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 01:00 UTC
[BugFix] Update python to python3 calls for image; fix prefix & input calculations. (#21391)
Signed-off-by: Eric Hanley <ericehanley@google.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
ericehanley committed
Jul 22, 2025 at 22:33 UTC
4f76a05f4f6c70a821e4a092d69331c993731614
1 file changed
+5
-4
benchmarks/auto_tune/auto_tune.sh
+5
-4
@@ -126,11 +126,12 @@ run_benchmark() {
126
# get a basic qps by using request-rate inf
127
bm_log="$LOG_FOLDER/bm_log_${max_num_seqs}_${max_num_batched_tokens}_requestrate_inf.txt"
128
prefix_len=$(( INPUT_LEN * MIN_CACHE_HIT_PCT / 100 ))
129
- python benchmarks/benchmark_serving.py \
129
+adjusted_input_len=$(( INPUT_LEN - prefix_len ))
130
+ python3 benchmarks/benchmark_serving.py \
131
--backend vllm \
132
--model $MODEL \
133
--dataset-name random \
133
- --random-input-len $INPUT_LEN \
134
+ --random-input-len $adjusted_input_len \
135
--random-output-len $OUTPUT_LEN \
136
--ignore-eos \
137
--disable-tqdm \
@@ -159,11 +160,11 @@ run_benchmark() {
160
curl -X POST http://0.0.0.0:8004/reset_prefix_cache
161
sleep 5
162
bm_log="$LOG_FOLDER/bm_log_${max_num_seqs}_${max_num_batched_tokens}_requestrate_${request_rate}.txt"
162
- python benchmarks/benchmark_serving.py \
163
+ python3 benchmarks/benchmark_serving.py \
164
--backend vllm \
165
--model $MODEL \
166
--dataset-name random \
166
- --random-input-len $INPUT_LEN \
167
+ --random-input-len $adjusted_input_len \
168
--random-output-len $OUTPUT_LEN \
169
--ignore-eos \
170
--disable-tqdm \