A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 11:15 UTC
[Benchmark] Add startup benchmarking to buildkite run (#33183)
Signed-off-by: Bin Bao <binbao@meta.com>
Bin Bao committed
Jan 28, 2026 at 11:03 UTC
392c5af4fe1aa767e669070e8b1a94fa48011714
1 file changed
+35
-88
.buildkite/performance-benchmarks/scripts/run-performance-benchmarks.sh
+35
-88
@@ -25,9 +25,9 @@ check_gpus() {
25
echo "Need at least 1 GPU to run benchmarking."
26
exit 1
27
fi
28
-
28
+
29
declare -g arch_suffix=''
30
-
30
+
31
if command -v nvidia-smi; then
32
declare -g gpu_type=$(nvidia-smi --query-gpu=name --format=csv,noheader | awk '{print $2}')
33
elif command -v amd-smi; then
@@ -181,19 +181,20 @@ upload_to_buildkite() {
181
$BUILDKITE_AGENT_COMMAND artifact upload "$RESULTS_FOLDER/*"
182
}
183
184
-run_latency_tests() {
185
- # run latency tests using `vllm bench latency` command
186
- # $1: a json file specifying latency test cases
184
+run_benchmark_tests() {
185
+ # run benchmark tests using `vllm bench <test_type>` command
186
+ # $1: test type (latency or throughput)
187
+ # $2: a json file specifying test cases
188
188
- local latency_test_file
189
- latency_test_file=$1
189
+ local test_type=$1
190
+ local test_file=$2
191
191
- # Iterate over latency tests
192
- jq -c '.[]' "$latency_test_file" | while read -r params; do
192
+ # Iterate over tests
193
+ jq -c '.[]' "$test_file" | while read -r params; do
194
# get the test name, and append the GPU type back to it.
195
test_name=$(echo "$params" | jq -r '.test_name')
195
- if [[ ! "$test_name" =~ ^latency_ ]]; then
196
- echo "In latency-test.json, test_name must start with \"latency_\"."
196
+ if [[ ! "$test_name" =~ ^${test_type}_ ]]; then
197
+ echo "In ${test_type}-test.json, test_name must start with \"${test_type}_\"."
198
exit 1
199
fi
200
@@ -204,15 +205,15 @@ run_latency_tests() {
205
fi
206
207
# get arguments
207
- latency_params=$(echo "$params" | jq -r '.parameters')
208
- latency_args=$(json2args "$latency_params")
209
- latency_environment_variables=$(echo "$params" | jq -r '.environment_variables')
210
- latency_envs=$(json2envs "$latency_environment_variables")
208
+ bench_params=$(echo "$params" | jq -r '.parameters')
209
+ bench_args=$(json2args "$bench_params")
210
+ bench_environment_variables=$(echo "$params" | jq -r '.environment_variables')
211
+ bench_envs=$(json2envs "$bench_environment_variables")
212
213
# check if there is enough GPU to run the test
213
- tp=$(echo "$latency_params" | jq -r '.tensor_parallel_size')
214
+ tp=$(echo "$bench_params" | jq -r '.tensor_parallel_size')
215
if [[ "$ON_CPU" == "1" ]]; then
215
- pp=$(echo "$latency_params" | jq -r '.pipeline_parallel_size // 1')
216
+ pp=$(echo "$bench_params" | jq -r '.pipeline_parallel_size // 1')
217
world_size=$(($tp*$pp))
218
if [[ $numa_count -lt $world_size && -z "${REMOTE_HOST}" ]]; then
219
echo "Required world-size $world_size but only $numa_count NUMA nodes found. Skip testcase $test_name."
@@ -225,97 +226,42 @@ run_latency_tests() {
226
fi
227
fi
228
228
- latency_command=" $latency_envs vllm bench latency \
229
+ bench_command=" $bench_envs vllm bench $test_type \
230
--output-json $RESULTS_FOLDER/${test_name}.json \
230
- $latency_args"
231
+ $bench_args"
232
233
echo "Running test case $test_name"
233
- echo "Latency command: $latency_command"
234
+ echo "${test_type^} command: $bench_command"
235
235
- # recoding benchmarking command ang GPU command
236
+ # recording benchmarking command and GPU command
237
jq_output=$(jq -n \
237
- --arg latency "$latency_command" \
238
+ --arg command "$bench_command" \
239
--arg gpu "$gpu_type" \
240
+ --arg test_type "$test_type" \
241
'{
240
- latency_command: $latency,
242
+ ($test_type + "_command"): $command,
243
gpu_type: $gpu
244
}')
245
echo "$jq_output" >"$RESULTS_FOLDER/$test_name.commands"
246
247
# run the benchmark
246
- eval "$latency_command"
248
+ eval "$bench_command"
249
250
kill_gpu_processes
251
252
done
253
}
254
253
-run_throughput_tests() {
254
- # run throughput tests using `vllm bench throughput`
255
- # $1: a json file specifying throughput test cases
256
-
257
- local throughput_test_file
258
- throughput_test_file=$1
259
-
260
- # Iterate over throughput tests
261
- jq -c '.[]' "$throughput_test_file" | while read -r params; do
262
- # get the test name, and append the GPU type back to it.
263
- test_name=$(echo "$params" | jq -r '.test_name')
264
- if [[ ! "$test_name" =~ ^throughput_ ]]; then
265
- echo "In throughput-test.json, test_name must start with \"throughput_\"."
266
- exit 1
267
- fi
268
-
269
- # if TEST_SELECTOR is set, only run the test cases that match the selector
270
- if [[ -n "$TEST_SELECTOR" ]] && [[ ! "$test_name" =~ $TEST_SELECTOR ]]; then
271
- echo "Skip test case $test_name."
272
- continue
273
- fi
274
-
275
- # get arguments
276
- throughput_params=$(echo "$params" | jq -r '.parameters')
277
- throughput_args=$(json2args "$throughput_params")
278
- throughput_environment_variables=$(echo "$params" | jq -r '.environment_variables')
279
- throughput_envs=$(json2envs "$throughput_environment_variables")
280
-
281
- # check if there is enough GPU to run the test
282
- tp=$(echo "$throughput_params" | jq -r '.tensor_parallel_size')
283
- if [[ "$ON_CPU" == "1" ]]; then
284
- pp=$(echo "$throughput_params" | jq -r '.pipeline_parallel_size // 1')
285
- world_size=$(($tp*$pp))
286
- if [[ $numa_count -lt $world_size && -z "${REMOTE_HOST}" ]]; then
287
- echo "Required world-size $world_size but only $numa_count NUMA nodes found. Skip testcase $test_name."
288
- continue
289
- fi
290
- else
291
- if [[ $gpu_count -lt $tp ]]; then
292
- echo "Required tensor-parallel-size $tp but only $gpu_count GPU found. Skip testcase $test_name."
293
- continue
294
- fi
295
- fi
296
-
297
- throughput_command=" $throughput_envs vllm bench throughput \
298
- --output-json $RESULTS_FOLDER/${test_name}.json \
299
- $throughput_args"
300
-
301
- echo "Running test case $test_name"
302
- echo "Throughput command: $throughput_command"
303
- # recoding benchmarking command ang GPU command
304
- jq_output=$(jq -n \
305
- --arg command "$throughput_command" \
306
- --arg gpu "$gpu_type" \
307
- '{
308
- throughput_command: $command,
309
- gpu_type: $gpu
310
- }')
311
- echo "$jq_output" >"$RESULTS_FOLDER/$test_name.commands"
312
-
313
- # run the benchmark
314
- eval "$throughput_command"
255
+run_latency_tests() {
256
+ run_benchmark_tests "latency" "$1"
257
+}
258
316
- kill_gpu_processes
259
+run_startup_tests() {
260
+ run_benchmark_tests "startup" "$1"
261
+}
262
318
- done
263
+run_throughput_tests() {
264
+ run_benchmark_tests "throughput" "$1"
265
}
266
267
run_serving_tests() {
@@ -534,6 +480,7 @@ main() {
480
# benchmarking
481
run_serving_tests $QUICK_BENCHMARK_ROOT/tests/"${SERVING_JSON:-serving-tests$ARCH.json}"
482
run_latency_tests $QUICK_BENCHMARK_ROOT/tests/"${LATENCY_JSON:-latency-tests$ARCH.json}"
483
+ run_startup_tests $QUICK_BENCHMARK_ROOT/tests/"${STARTUP_JSON:-startup-tests$ARCH.json}"
484
run_throughput_tests $QUICK_BENCHMARK_ROOT/tests/"${THROUGHPUT_JSON:-throughput-tests$ARCH.json}"
485
486
# postprocess benchmarking results