| 1 | #!/bin/bash |
| 2 | # This script assumes that we are already inside the vllm/ directory |
| 3 | # Benchmarking results will be available inside vllm/benchmarks/results/ |
| 4 | |
| 5 | # Do not set -e, as the mixtral 8x22B model tends to crash occasionally |
| 6 | # and we still want to see other benchmarking results even when mixtral crashes. |
| 7 | set -x |
| 8 | set -o pipefail |
| 9 | |
| 10 | # Environment-driven debug controls (like ON_CPU=1) |
| 11 | DRY_RUN="${DRY_RUN:-0}" |
| 12 | MODEL_FILTER="${MODEL_FILTER:-}" |
| 13 | DTYPE_FILTER="${DTYPE_FILTER:-}" |
| 14 | |
| 15 | # Adaptive search controls |
| 16 | ENABLE_ADAPTIVE_CONCURRENCY="${ENABLE_ADAPTIVE_CONCURRENCY:-0}" |
| 17 | SLA_TTFT_MS="${SLA_TTFT_MS:-3000}" |
| 18 | SLA_TPOT_MS="${SLA_TPOT_MS:-100}" |
| 19 | ADAPTIVE_MAX_PROBES="${ADAPTIVE_MAX_PROBES:-8}" |
| 20 | ADAPTIVE_MAX_CONCURRENCY="${ADAPTIVE_MAX_CONCURRENCY:-1024}" |
| 21 | |
| 22 | check_gpus() { |
| 23 | if command -v nvidia-smi; then |
| 24 | # check the number of GPUs and GPU type. |
| 25 | declare -g gpu_count=$(nvidia-smi --list-gpus | grep -c . || true) |
| 26 | elif command -v amd-smi; then |
| 27 | declare -g gpu_count=$(amd-smi list | grep -c 'GPU' || true) |
| 28 | elif command -v hl-smi; then |
| 29 | declare -g gpu_count=$(hl-smi --list | grep -ci "Module ID" || true) |
| 30 | fi |
| 31 | |
| 32 | if [[ $gpu_count -gt 0 ]]; then |
| 33 | echo "GPU found." |
| 34 | else |
| 35 | echo "Need at least 1 GPU to run benchmarking." |
| 36 | exit 1 |
| 37 | fi |
| 38 | |
| 39 | declare -g arch_suffix='' |
| 40 | |
| 41 | if command -v nvidia-smi; then |
| 42 | declare -g gpu_type=$(nvidia-smi --query-gpu=name --format=csv,noheader | awk '{print $2}') |
| 43 | elif command -v amd-smi; then |
| 44 | declare -g gpu_type=$(amd-smi static -g 0 -a | grep 'MARKET_NAME' | awk '{print $2}') |
| 45 | elif command -v hl-smi; then |
| 46 | declare -g gpu_type=$(hl-smi -q | grep "Product Name" | head -n 1 | awk -F ':' '{print $2}' | sed 's/^ *//') |
| 47 | arch_suffix='-hpu' |
| 48 | fi |
| 49 | echo "GPU type is $gpu_type" |
| 50 | } |
| 51 | |
| 52 | check_cpus() { |
| 53 | # check the number of CPUs and NUMA Node and GPU type. |
| 54 | declare -g numa_count=$(lscpu | grep "NUMA node(s):" | awk '{print $3}') |
| 55 | if [[ $numa_count -gt 0 ]]; then |
| 56 | echo "NUMA found." |
| 57 | echo "$numa_count" |
| 58 | else |
| 59 | echo "Need at least 1 NUMA to run benchmarking." |
| 60 | exit 1 |
| 61 | fi |
| 62 | if [[ "$(uname -m)" == "aarch64" ]] || [[ "$(uname -m)" == "arm64" ]]; then |
| 63 | declare -g gpu_type="arm64-cpu" |
| 64 | else |
| 65 | declare -g gpu_type="cpu" |
| 66 | fi |
| 67 | echo "GPU type is $gpu_type" |
| 68 | } |
| 69 | |
| 70 | check_hf_token() { |
| 71 | # check if HF_TOKEN is available and valid |
| 72 | if [[ -z "$HF_TOKEN" ]]; then |
| 73 | echo "Error: HF_TOKEN is not set." |
| 74 | exit 1 |
| 75 | elif [[ ! "$HF_TOKEN" =~ ^hf_ ]]; then |
| 76 | echo "Error: HF_TOKEN does not start with 'hf_'." |
| 77 | exit 1 |
| 78 | else |
| 79 | echo "HF_TOKEN is set and valid." |
| 80 | fi |
| 81 | } |
| 82 | |
| 83 | ensure_sharegpt_downloaded() { |
| 84 | local FILE=ShareGPT_V3_unfiltered_cleaned_split.json |
| 85 | if [ ! -f "$FILE" ]; then |
| 86 | wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/$FILE |
| 87 | else |
| 88 | echo "$FILE already exists." |
| 89 | fi |
| 90 | } |
| 91 | |
| 92 | json2args() { |
| 93 | # transforms the JSON string to command line args, and '_' is replaced to '-' |
| 94 | # example: |
| 95 | # input: { "model": "meta-llama/Llama-2-7b-chat-hf", "tensor_parallel_size": 1 } |
| 96 | # output: --model meta-llama/Llama-2-7b-chat-hf --tensor-parallel-size 1 |
| 97 | local json_string=$1 |
| 98 | local args=$( |
| 99 | echo "$json_string" | jq -r ' |
| 100 | to_entries | |
| 101 | map("--" + (.key | gsub("_"; "-")) + " " + (.value | tostring)) | |
| 102 | join(" ") |
| 103 | ' |
| 104 | ) |
| 105 | echo "$args" |
| 106 | } |
| 107 | |
| 108 | json2envs() { |
| 109 | # transforms the JSON string to environment variables. |
| 110 | # example: |
| 111 | # input: { "VLLM_CPU_KVCACHE_SPACE": 5 } |
| 112 | # output: VLLM_CPU_KVCACHE_SPACE=5 |
| 113 | local json_string=$1 |
| 114 | local args=$( |
| 115 | echo "$json_string" | jq -r ' |
| 116 | to_entries | |
| 117 | map((.key ) + "=" + (.value | tostring)) | |
| 118 | join(" ") |
| 119 | ' |
| 120 | ) |
| 121 | echo "$args" |
| 122 | } |
| 123 | |
| 124 | wait_for_server() { |
| 125 | local timeout_val="1200" |
| 126 | timeout "$timeout_val" bash -c ' |
| 127 | until curl -sf http://localhost:8000/v1/models >/dev/null; do |
| 128 | sleep 1 |
| 129 | done |
| 130 | ' |
| 131 | } |
| 132 | |
| 133 | kill_processes_launched_by_current_bash() { |
| 134 | # Kill all python processes launched from current bash script |
| 135 | current_shell_pid=$$ |
| 136 | processes=$(ps -eo pid,ppid,command | awk -v ppid="$current_shell_pid" -v proc="$1" '$2 == ppid && $3 ~ proc {print $1}') |
| 137 | if [ -n "$processes" ]; then |
| 138 | echo "Killing the following processes matching '$1':" |
| 139 | echo "$processes" |
| 140 | echo "$processes" | xargs kill -9 |
| 141 | else |
| 142 | echo "No processes found matching '$1'." |
| 143 | fi |
| 144 | } |
| 145 | |
| 146 | kill_gpu_processes() { |
| 147 | |
| 148 | ps -aux |
| 149 | lsof -t -i:8000 | xargs -r kill -9 |
| 150 | pgrep python3 | xargs -r kill -9 |
| 151 | # vLLM now names the process with VLLM prefix after https://github.com/vllm-project/vllm/pull/21445 |
| 152 | pgrep VLLM | xargs -r kill -9 |
| 153 | |
| 154 | # wait until GPU memory usage smaller than 1GB |
| 155 | if command -v nvidia-smi; then |
| 156 | while [ "$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -n 1)" -ge 1000 ]; do |
| 157 | sleep 1 |
| 158 | done |
| 159 | elif command -v amd-smi; then |
| 160 | while [ "$(amd-smi metric -g 0 | grep 'USED_VRAM' | awk '{print $2}')" -ge 1000 ]; do |
| 161 | sleep 1 |
| 162 | done |
| 163 | elif command -v hl-smi; then |
| 164 | while [ "$(hl-smi -q | grep "Used" | head -n 1 | awk '{print $3}')" -ge 1000 ]; do |
| 165 | sleep 1 |
| 166 | done |
| 167 | fi |
| 168 | |
| 169 | # remove vllm config file |
| 170 | rm -rf ~/.config/vllm |
| 171 | |
| 172 | } |
| 173 | |
| 174 | upload_to_buildkite() { |
| 175 | # upload the benchmarking results to buildkite |
| 176 | |
| 177 | # if the agent binary is not found, skip uploading the results, exit 0 |
| 178 | # Check if buildkite-agent is available in the PATH or at /workspace/buildkite-agent |
| 179 | if command -v buildkite-agent >/dev/null 2>&1; then |
| 180 | BUILDKITE_AGENT_COMMAND="buildkite-agent" |
| 181 | elif [ -f /workspace/buildkite-agent ]; then |
| 182 | BUILDKITE_AGENT_COMMAND="/workspace/buildkite-agent" |
| 183 | else |
| 184 | echo "buildkite-agent binary not found. Skip uploading the results." |
| 185 | return 0 |
| 186 | fi |
| 187 | |
| 188 | # Use the determined command to annotate and upload artifacts |
| 189 | $BUILDKITE_AGENT_COMMAND annotate --style "info" --context "$BUILDKITE_LABEL-benchmark-results" < "$RESULTS_FOLDER/benchmark_results.md" |
| 190 | $BUILDKITE_AGENT_COMMAND artifact upload "$RESULTS_FOLDER/*" |
| 191 | } |
| 192 | |
| 193 | # ------------------------------- |
| 194 | # Adaptive concurrency helpers |
| 195 | # ------------------------------- |
| 196 | result_json_path_for_serving() { |
| 197 | local test_name=$1 |
| 198 | local qps=$2 |
| 199 | local max_concurrency=$3 |
| 200 | echo "$RESULTS_FOLDER/${test_name}_qps_${qps}_concurrency_${max_concurrency}.json" |
| 201 | } |
| 202 | |
| 203 | extract_metric_ms() { |
| 204 | local metric_name=$1 |
| 205 | local json_file=$2 |
| 206 | |
| 207 | [[ -f "$json_file" ]] || return 0 |
| 208 | |
| 209 | if [[ "$metric_name" == "ttft" ]]; then |
| 210 | jq -r ' |
| 211 | [ |
| 212 | .ttft_ms.p99?, |
| 213 | .metrics.ttft_ms.p99?, |
| 214 | .ttft.p99?, |
| 215 | .metrics.ttft.p99?, |
| 216 | .p99_ttft_ms?, |
| 217 | .ttft_ms.mean?, |
| 218 | .metrics.ttft_ms.mean?, |
| 219 | .ttft.mean?, |
| 220 | .metrics.ttft.mean?, |
| 221 | .mean_ttft_ms? |
| 222 | ] | map(select(. != null)) | .[0] // empty |
| 223 | ' "$json_file" |
| 224 | else |
| 225 | jq -r ' |
| 226 | [ |
| 227 | .tpot_ms.p99?, |
| 228 | .metrics.tpot_ms.p99?, |
| 229 | .tpot.p99?, |
| 230 | .metrics.tpot.p99?, |
| 231 | .p99_tpot_ms?, |
| 232 | .itl_ms.p99?, |
| 233 | .metrics.itl_ms.p99?, |
| 234 | .inter_token_latency_ms.p99?, |
| 235 | .tpot_ms.mean?, |
| 236 | .metrics.tpot_ms.mean?, |
| 237 | .tpot.mean?, |
| 238 | .metrics.tpot.mean?, |
| 239 | .itl_ms.mean?, |
| 240 | .metrics.itl_ms.mean?, |
| 241 | .mean_tpot_ms?, |
| 242 | .mean_itl_ms? |
| 243 | ] | map(select(. != null)) | .[0] // empty |
| 244 | ' "$json_file" |
| 245 | fi |
| 246 | } |
| 247 | |
| 248 | evaluate_sla_from_json() { |
| 249 | local json_file=$1 |
| 250 | local ttft |
| 251 | local tpot |
| 252 | local pass |
| 253 | |
| 254 | [[ -f "$json_file" ]] || return 2 |
| 255 | |
| 256 | ttft=$(extract_metric_ms ttft "$json_file") |
| 257 | tpot=$(extract_metric_ms tpot "$json_file") |
| 258 | |
| 259 | [[ -n "$ttft" && -n "$tpot" ]] || return 2 |
| 260 | |
| 261 | pass=$(jq -n \ |
| 262 | --argjson ttft "$ttft" \ |
| 263 | --argjson tpot "$tpot" \ |
| 264 | --argjson sla_ttft "$SLA_TTFT_MS" \ |
| 265 | --argjson sla_tpot "$SLA_TPOT_MS" \ |
| 266 | '($ttft <= $sla_ttft) and ($tpot <= $sla_tpot)') |
| 267 | |
| 268 | [[ "$pass" == "true" ]] |
| 269 | } |
| 270 | |
| 271 | write_adaptive_summary_json() { |
| 272 | local summary_file=$1 |
| 273 | local test_name=$2 |
| 274 | local qps=$3 |
| 275 | local static_last_pass=$4 |
| 276 | local static_first_fail=$5 |
| 277 | local final_last_pass=$6 |
| 278 | local final_first_fail=$7 |
| 279 | |
| 280 | jq -n \ |
| 281 | --arg test_name "$test_name" \ |
| 282 | --arg qps "$qps" \ |
| 283 | --argjson sla_ttft "$SLA_TTFT_MS" \ |
| 284 | --argjson sla_tpot "$SLA_TPOT_MS" \ |
| 285 | --arg static_last_pass "${static_last_pass:-}" \ |
| 286 | --arg static_first_fail "${static_first_fail:-}" \ |
| 287 | --arg final_last_pass "${final_last_pass:-}" \ |
| 288 | --arg final_first_fail "${final_first_fail:-}" \ |
| 289 | '{ |
| 290 | test_name: $test_name, |
| 291 | qps: $qps, |
| 292 | sla_ttft_ms: $sla_ttft, |
| 293 | sla_tpot_ms: $sla_tpot, |
| 294 | static_last_pass: (if $static_last_pass == "" then null else ($static_last_pass | tonumber) end), |
| 295 | static_first_fail: (if $static_first_fail == "" then null else ($static_first_fail | tonumber) end), |
| 296 | final_last_pass: (if $final_last_pass == "" then null else ($final_last_pass | tonumber) end), |
| 297 | final_first_fail: (if $final_first_fail == "" then null else ($final_first_fail | tonumber) end) |
| 298 | }' > "$summary_file" |
| 299 | } |
| 300 | |
| 301 | run_single_serving_probe() { |
| 302 | local test_name=$1 |
| 303 | local qps=$2 |
| 304 | local max_concurrency=$3 |
| 305 | local tp=$4 |
| 306 | local compilation_config_mode=$5 |
| 307 | local optimization_level=$6 |
| 308 | local client_args_effective=$7 |
| 309 | local client_remote_args=$8 |
| 310 | local server_command=$9 |
| 311 | |
| 312 | local new_test_name="${test_name}_qps_${qps}_concurrency_${max_concurrency}" |
| 313 | local result_json |
| 314 | local num_prompts_arg="" |
| 315 | local client_command |
| 316 | |
| 317 | result_json=$(result_json_path_for_serving "$test_name" "$qps" "$max_concurrency") |
| 318 | |
| 319 | if [[ -f "$result_json" ]]; then |
| 320 | evaluate_sla_from_json "$result_json" |
| 321 | return $? |
| 322 | fi |
| 323 | |
| 324 | if [[ -n "${PROMPTS_PER_CONCURRENCY}" ]]; then |
| 325 | num_prompts=$(( max_concurrency * PROMPTS_PER_CONCURRENCY )) |
| 326 | if (( num_prompts < MIN_NUM_PROMPTS )); then num_prompts=$MIN_NUM_PROMPTS; fi |
| 327 | if (( num_prompts > MAX_NUM_PROMPTS )); then num_prompts=$MAX_NUM_PROMPTS; fi |
| 328 | num_prompts_arg="--num-prompts $num_prompts" |
| 329 | fi |
| 330 | |
| 331 | client_command="vllm bench serve \ |
| 332 | --save-result \ |
| 333 | --result-dir $RESULTS_FOLDER \ |
| 334 | --result-filename ${new_test_name}.json \ |
| 335 | --request-rate $qps \ |
| 336 | --max-concurrency $max_concurrency \ |
| 337 | $num_prompts_arg \ |
| 338 | --metadata tensor_parallel_size=$tp compilation_config.mode=$compilation_config_mode optimization_level=$optimization_level adaptive_search=1 \ |
| 339 | $client_args_effective $client_remote_args " |
| 340 | |
| 341 | echo "Adaptive probe: $client_command" |
| 342 | |
| 343 | if [[ "${DRY_RUN:-0}" != "1" ]]; then |
| 344 | bash -c "$client_command" |
| 345 | fi |
| 346 | |
| 347 | jq_output=$(jq -n \ |
| 348 | --arg server "$server_command" \ |
| 349 | --arg client "$client_command" \ |
| 350 | --arg gpu "$gpu_type" \ |
| 351 | '{ |
| 352 | server_command: $server, |
| 353 | client_command: $client, |
| 354 | gpu_type: $gpu, |
| 355 | adaptive_search: true |
| 356 | }') |
| 357 | echo "$jq_output" > "$RESULTS_FOLDER/${new_test_name}.commands" |
| 358 | |
| 359 | evaluate_sla_from_json "$result_json" |
| 360 | } |
| 361 | |
| 362 | adaptive_refine_from_static_results() { |
| 363 | local test_name=$1 |
| 364 | local qps=$2 |
| 365 | local max_concurrency_list_raw=$3 |
| 366 | local tp=$4 |
| 367 | local compilation_config_mode=$5 |
| 368 | local optimization_level=$6 |
| 369 | local client_args_effective=$7 |
| 370 | local client_remote_args=$8 |
| 371 | local server_command=$9 |
| 372 | |
| 373 | local sorted_points |
| 374 | local point |
| 375 | local rc |
| 376 | local static_last_pass="" |
| 377 | local static_first_fail="" |
| 378 | local largest_static="" |
| 379 | local step_hint=1 |
| 380 | local previous_point="" |
| 381 | local low |
| 382 | local high |
| 383 | local mid |
| 384 | local probes=0 |
| 385 | local summary_file="$RESULTS_FOLDER/${test_name}_qps_${qps}_sla_summary.json" |
| 386 | |
| 387 | [[ "${ENABLE_ADAPTIVE_CONCURRENCY}" == "1" ]] || return 0 |
| 388 | [[ "${DRY_RUN:-0}" != "1" ]] || return 0 |
| 389 | |
| 390 | sorted_points=$(for point in $max_concurrency_list_raw; do printf '%s\n' "$point"; done | tr -d "'" | awk '/^[0-9]+$/' | sort -n | uniq) |
| 391 | [[ -n "$sorted_points" ]] || return 0 |
| 392 | |
| 393 | while read -r point; do |
| 394 | [[ -z "$point" ]] && continue |
| 395 | largest_static="$point" |
| 396 | evaluate_sla_from_json "$(result_json_path_for_serving "$test_name" "$qps" "$point")" |
| 397 | rc=$? |
| 398 | if (( rc == 0 )); then |
| 399 | static_last_pass="$point" |
| 400 | elif (( rc == 1 )); then |
| 401 | if [[ -n "$static_last_pass" ]]; then |
| 402 | static_first_fail="$point" |
| 403 | break |
| 404 | fi |
| 405 | fi |
| 406 | |
| 407 | if [[ -n "$previous_point" ]]; then |
| 408 | step_hint=$(( point - previous_point )) |
| 409 | if (( step_hint < 1 )); then step_hint=1; fi |
| 410 | fi |
| 411 | previous_point="$point" |
| 412 | done <<< "$sorted_points" |
| 413 | |
| 414 | if [[ -z "$static_last_pass" ]]; then |
| 415 | write_adaptive_summary_json "$summary_file" "$test_name" "$qps" "" "$static_first_fail" "" "$static_first_fail" |
| 416 | return 0 |
| 417 | fi |
| 418 | |
| 419 | if [[ -n "$static_first_fail" ]]; then |
| 420 | low=$static_last_pass |
| 421 | high=$static_first_fail |
| 422 | while (( low + 1 < high )) && (( probes < ADAPTIVE_MAX_PROBES )); do |
| 423 | mid=$(( (low + high) / 2 )) |
| 424 | probes=$(( probes + 1 )) |
| 425 | run_single_serving_probe \ |
| 426 | "$test_name" "$qps" "$mid" "$tp" \ |
| 427 | "$compilation_config_mode" "$optimization_level" \ |
| 428 | "$client_args_effective" "$client_remote_args" "$server_command" |
| 429 | rc=$? |
| 430 | if (( rc == 0 )); then |
| 431 | low=$mid |
| 432 | elif (( rc == 1 )); then |
| 433 | high=$mid |
| 434 | else |
| 435 | break |
| 436 | fi |
| 437 | done |
| 438 | write_adaptive_summary_json "$summary_file" "$test_name" "$qps" "$static_last_pass" "$static_first_fail" "$low" "$high" |
| 439 | return 0 |
| 440 | fi |
| 441 | |
| 442 | low=$largest_static |
| 443 | high="" |
| 444 | while (( probes < ADAPTIVE_MAX_PROBES )); do |
| 445 | point=$(( low + step_hint )) |
| 446 | if (( point > ADAPTIVE_MAX_CONCURRENCY )); then |
| 447 | point=$ADAPTIVE_MAX_CONCURRENCY |
| 448 | fi |
| 449 | (( point > low )) || break |
| 450 | probes=$(( probes + 1 )) |
| 451 | run_single_serving_probe \ |
| 452 | "$test_name" "$qps" "$point" "$tp" \ |
| 453 | "$compilation_config_mode" "$optimization_level" \ |
| 454 | "$client_args_effective" "$client_remote_args" "$server_command" |
| 455 | rc=$? |
| 456 | if (( rc == 0 )); then |
| 457 | low=$point |
| 458 | (( point == ADAPTIVE_MAX_CONCURRENCY )) && break |
| 459 | step_hint=$(( step_hint * 2 )) |
| 460 | if (( step_hint < 1 )); then step_hint=1; fi |
| 461 | elif (( rc == 1 )); then |
| 462 | high=$point |
| 463 | break |
| 464 | else |
| 465 | break |
| 466 | fi |
| 467 | done |
| 468 | |
| 469 | if [[ -n "$high" ]]; then |
| 470 | while (( low + 1 < high )) && (( probes < ADAPTIVE_MAX_PROBES )); do |
| 471 | mid=$(( (low + high) / 2 )) |
| 472 | probes=$(( probes + 1 )) |
| 473 | run_single_serving_probe \ |
| 474 | "$test_name" "$qps" "$mid" "$tp" \ |
| 475 | "$compilation_config_mode" "$optimization_level" \ |
| 476 | "$client_args_effective" "$client_remote_args" "$server_command" |
| 477 | rc=$? |
| 478 | if (( rc == 0 )); then |
| 479 | low=$mid |
| 480 | elif (( rc == 1 )); then |
| 481 | high=$mid |
| 482 | else |
| 483 | break |
| 484 | fi |
| 485 | done |
| 486 | fi |
| 487 | |
| 488 | write_adaptive_summary_json "$summary_file" "$test_name" "$qps" "$static_last_pass" "" "$low" "$high" |
| 489 | } |
| 490 | |
| 491 | run_benchmark_tests() { |
| 492 | # run benchmark tests using `vllm bench <test_type>` command |
| 493 | # $1: test type (latency or throughput) |
| 494 | # $2: a json file specifying test cases |
| 495 | |
| 496 | local test_type=$1 |
| 497 | local test_file=$2 |
| 498 | |
| 499 | # Iterate over tests |
| 500 | jq -c '.[]' "$test_file" | while read -r params; do |
| 501 | # get the test name, and append the GPU type back to it. |
| 502 | test_name=$(echo "$params" | jq -r '.test_name') |
| 503 | if [[ ! "$test_name" =~ ^${test_type}_ ]]; then |
| 504 | echo "In ${test_type}-test.json, test_name must start with \"${test_type}_\"." |
| 505 | exit 1 |
| 506 | fi |
| 507 | |
| 508 | # if TEST_SELECTOR is set, only run the test cases that match the selector |
| 509 | if [[ -n "$TEST_SELECTOR" ]] && [[ ! "$test_name" =~ $TEST_SELECTOR ]]; then |
| 510 | echo "Skip test case $test_name." |
| 511 | continue |
| 512 | fi |
| 513 | |
| 514 | # get arguments |
| 515 | bench_params=$(echo "$params" | jq -r '.parameters') |
| 516 | bench_args=$(json2args "$bench_params") |
| 517 | bench_environment_variables=$(echo "$params" | jq -r '.environment_variables') |
| 518 | bench_envs=$(json2envs "$bench_environment_variables") |
| 519 | |
| 520 | # check if there is enough GPU to run the test |
| 521 | tp=$(echo "$bench_params" | jq -r '.tensor_parallel_size') |
| 522 | if [[ "$ON_CPU" == "1" ]]; then |
| 523 | pp=$(echo "$bench_params" | jq -r '.pipeline_parallel_size // 1') |
| 524 | world_size=$(($tp*$pp)) |
| 525 | if [[ $numa_count -lt $world_size && -z "${REMOTE_HOST}" ]]; then |
| 526 | echo "Required world-size $world_size but only $numa_count NUMA nodes found. Skip testcase $test_name." |
| 527 | continue |
| 528 | fi |
| 529 | else |
| 530 | if [[ $gpu_count -lt $tp ]]; then |
| 531 | echo "Required tensor-parallel-size $tp but only $gpu_count GPU found. Skip testcase $test_name." |
| 532 | continue |
| 533 | fi |
| 534 | fi |
| 535 | |
| 536 | bench_command=" $bench_envs vllm bench $test_type \ |
| 537 | --output-json $RESULTS_FOLDER/${test_name}.json \ |
| 538 | $bench_args" |
| 539 | |
| 540 | echo "Running test case $test_name" |
| 541 | echo "${test_type^} command: $bench_command" |
| 542 | |
| 543 | # recording benchmarking command and GPU command |
| 544 | jq_output=$(jq -n \ |
| 545 | --arg command "$bench_command" \ |
| 546 | --arg gpu "$gpu_type" \ |
| 547 | --arg test_type "$test_type" \ |
| 548 | '{ |
| 549 | ($test_type + "_command"): $command, |
| 550 | gpu_type: $gpu |
| 551 | }') |
| 552 | echo "$jq_output" >"$RESULTS_FOLDER/$test_name.commands" |
| 553 | |
| 554 | # run the benchmark |
| 555 | eval "$bench_command" |
| 556 | |
| 557 | kill_gpu_processes |
| 558 | |
| 559 | done |
| 560 | } |
| 561 | |
| 562 | run_latency_tests() { run_benchmark_tests "latency" "$1"; } |
| 563 | run_startup_tests() { run_benchmark_tests "startup" "$1"; } |
| 564 | run_throughput_tests() { run_benchmark_tests "throughput" "$1"; } |
| 565 | |
| 566 | merge_serving_tests_stream() { |
| 567 | # Emit merged serving test objects, optionally filtered by MODEL_FILTER/DTYPE_FILTER in DRY_RUN mode. |
| 568 | # This helper does NOT modify JSON; it only filters the stream in dry-run mode. |
| 569 | local serving_test_file="$1" |
| 570 | # shellcheck disable=SC2016 |
| 571 | local merged=' |
| 572 | if type == "array" then |
| 573 | # Plain format: test cases array |
| 574 | .[] |
| 575 | elif (type == "object" and has("tests")) then |
| 576 | # merge the default parameters into each test cases |
| 577 | . as $root |
| 578 | | ($root.defaults // {}) as $d |
| 579 | | ($root.tests // [])[] |
| 580 | # default qps / max_concurrency from defaults if missing |
| 581 | | .qps_list = (.qps_list // $d.qps_list) |
| 582 | | .max_concurrency_list = (.max_concurrency_list // $d.max_concurrency_list) |
| 583 | # merge envs / params: test overrides defaults |
| 584 | | .server_environment_variables = |
| 585 | (($d.server_environment_variables // {}) + (.server_environment_variables // {})) |
| 586 | | .server_parameters = |
| 587 | (($d.server_parameters // {}) + (.server_parameters // {})) |
| 588 | | .client_parameters = |
| 589 | (($d.client_parameters // {}) + (.client_parameters // {})) |
| 590 | else |
| 591 | error("Unsupported serving test file format: must be array or object with .tests") |
| 592 | end |
| 593 | ' |
| 594 | |
| 595 | jq -c "$merged" "$serving_test_file" | \ |
| 596 | if [[ "${DRY_RUN:-0}" == "1" && ( "${MODEL_FILTER}${DTYPE_FILTER}" != "" ) ]]; then |
| 597 | jq -c --arg model "$MODEL_FILTER" --arg dtype "$DTYPE_FILTER" ' |
| 598 | select((($model|length)==0) |
| 599 | or ((.server_parameters.model // "") == $model) |
| 600 | or ((.client_parameters.model // "") == $model)) |
| 601 | | select((($dtype|length)==0) or ((.server_parameters.dtype // "") == $dtype)) |
| 602 | ' |
| 603 | else |
| 604 | cat |
| 605 | fi |
| 606 | } |
| 607 | |
| 608 | run_serving_tests() { |
| 609 | # run serving tests using `vllm bench serve` command |
| 610 | # $1: a json file specifying serving test cases |
| 611 | # |
| 612 | # Supported JSON formats: |
| 613 | # 1) Plain format: top-level array |
| 614 | # [ { "test_name": "...", "server_parameters": {...}, ... }, ... ] |
| 615 | # |
| 616 | # 2) Default parameters field + plain format tests |
| 617 | # { |
| 618 | # "defaults": { ... }, |
| 619 | # "tests": [ { "test_name": "...", "server_parameters": {...}, ... }, ... ] |
| 620 | # } |
| 621 | |
| 622 | local serving_test_file |
| 623 | serving_test_file=$1 |
| 624 | |
| 625 | # In dry-run mode, if filters are provided but no tests match, fail fast. |
| 626 | if [[ "${DRY_RUN:-0}" == "1" && ( "${MODEL_FILTER}${DTYPE_FILTER}" != "" ) ]]; then |
| 627 | local count |
| 628 | count=$(merge_serving_tests_stream "$serving_test_file" | wc -l | tr -d ' ') |
| 629 | if [[ "$count" -eq 0 ]]; then |
| 630 | echo "No matching serving tests found in $serving_test_file for model='$MODEL_FILTER' dtype='$DTYPE_FILTER'." >&2 |
| 631 | return 0 |
| 632 | fi |
| 633 | fi |
| 634 | |
| 635 | # Iterate over serving tests (merged + optional filtered stream) |
| 636 | merge_serving_tests_stream "$serving_test_file" | while read -r params; do |
| 637 | # get the test name, and append the GPU type back to it. |
| 638 | test_name=$(echo "$params" | jq -r '.test_name') |
| 639 | if [[ ! "$test_name" =~ ^serving_ ]]; then |
| 640 | echo "In serving-test.json, test_name must start with \"serving_\"." |
| 641 | exit 1 |
| 642 | fi |
| 643 | |
| 644 | # if TEST_SELECTOR is set, only run the test cases that match the selector |
| 645 | if [[ -n "$TEST_SELECTOR" ]] && [[ ! "$test_name" =~ $TEST_SELECTOR ]]; then |
| 646 | echo "Skip test case $test_name." |
| 647 | continue |
| 648 | fi |
| 649 | |
| 650 | # get client and server arguments (after merged the default parameters) |
| 651 | server_params=$(echo "$params" | jq -r '.server_parameters') |
| 652 | server_envs=$(echo "$params" | jq -r '.server_environment_variables') |
| 653 | client_params=$(echo "$params" | jq -r '.client_parameters') |
| 654 | |
| 655 | # vLLM serve CLI: model must be positional (no --model). Convert server_parameters accordingly. |
| 656 | server_model=$(echo "$server_params" | jq -r '.model // empty') |
| 657 | if [[ -z "$server_model" || "$server_model" == "null" ]]; then |
| 658 | echo "Error: serving test '$test_name' is missing server_parameters.model" >&2 |
| 659 | exit 1 |
| 660 | fi |
| 661 | server_params_no_model=$(echo "$server_params" | jq -c 'del(.model)') |
| 662 | server_args=$(json2args "$server_params_no_model") |
| 663 | |
| 664 | server_envs=$(json2envs "$server_envs") |
| 665 | client_args=$(json2args "$client_params") |
| 666 | |
| 667 | # ------------------------------------------------------------ |
| 668 | # Option 1: Dynamic num-prompts scaling based on max_concurrency |
| 669 | # |
| 670 | # If PROMPTS_PER_CONCURRENCY is set, override JSON num_prompts with: |
| 671 | # num_prompts = max_concurrency * PROMPTS_PER_CONCURRENCY |
| 672 | # |
| 673 | # If PROMPTS_PER_CONCURRENCY is NOT set, keep JSON num_prompts behavior |
| 674 | # unchanged (i.e., whatever is in serving-tests-*.json). |
| 675 | # ------------------------------------------------------------ |
| 676 | PROMPTS_PER_CONCURRENCY="${PROMPTS_PER_CONCURRENCY-}" # no default on purpose |
| 677 | MIN_NUM_PROMPTS="${MIN_NUM_PROMPTS:-1}" |
| 678 | MAX_NUM_PROMPTS="${MAX_NUM_PROMPTS:-1000000}" |
| 679 | |
| 680 | if [[ -n "${PROMPTS_PER_CONCURRENCY}" ]]; then |
| 681 | # Remove any fixed --num-prompts from JSON-derived args (avoid duplicates) |
| 682 | # Remove any fixed --num-prompts from JSON-derived args (avoid duplicates) |
| 683 | # Handles: --num-prompts 123 and --num-prompts=123 |
| 684 | client_args_no_np="$( |
| 685 | printf ' %s ' "$client_args" \ |
| 686 | | sed -E \ |
| 687 | -e 's/[[:space:]]--num-prompts=([^[:space:]]+)([[:space:]]|$)/ /g' \ |
| 688 | -e 's/[[:space:]]--num-prompts[[:space:]]+([^[:space:]]+)([[:space:]]|$)/ /g' |
| 689 | )" |
| 690 | # normalize whitespace |
| 691 | client_args_no_np="$(echo "$client_args_no_np" | tr -s ' ' | sed -E 's/^ //; s/ $//')" |
| 692 | client_args_no_np="$(echo "$client_args_no_np" | xargs)" |
| 693 | client_args_effective="$client_args_no_np" |
| 694 | else |
| 695 | client_args_effective="$client_args" |
| 696 | fi |
| 697 | # qps_list |
| 698 | qps_list=$(echo "$params" | jq -r '.qps_list') |
| 699 | qps_list=$(echo "$qps_list" | jq -r '.[] | @sh') |
| 700 | echo "Running over qps list $qps_list" |
| 701 | |
| 702 | # max_concurrency_list (fallback to num_prompts if missing) |
| 703 | max_concurrency_list=$(echo "$params" | jq -r '.max_concurrency_list') |
| 704 | if [[ -z "$max_concurrency_list" || "$max_concurrency_list" == "null" ]]; then |
| 705 | num_prompts=$(echo "$client_params" | jq -r '.num_prompts') |
| 706 | max_concurrency_list="[$num_prompts]" |
| 707 | fi |
| 708 | max_concurrency_list=$(echo "$max_concurrency_list" | jq -r '.[] | @sh') |
| 709 | echo "Running over max concurrency list $max_concurrency_list" |
| 710 | |
| 711 | # check if there is enough resources to run the test |
| 712 | tp=$(echo "$server_params" | jq -r '.tensor_parallel_size') |
| 713 | if [[ "$ON_CPU" == "1" ]]; then |
| 714 | pp=$(echo "$server_params" | jq -r '.pipeline_parallel_size // 1') |
| 715 | world_size=$(($tp*$pp)) |
| 716 | if [[ $numa_count -lt $world_size && -z "${REMOTE_HOST}" ]]; then |
| 717 | echo "Required world-size $world_size but only $numa_count NUMA nodes found. Skip testcase $test_name." |
| 718 | continue |
| 719 | fi |
| 720 | else |
| 721 | if [[ $gpu_count -lt $tp ]]; then |
| 722 | echo "Required tensor-parallel-size $tp but only $gpu_count GPU found. Skip testcase $test_name." |
| 723 | continue |
| 724 | fi |
| 725 | fi |
| 726 | |
| 727 | # check if server model and client model is aligned |
| 728 | client_model=$(echo "$client_params" | jq -r '.model') |
| 729 | if [[ $server_model != "$client_model" ]]; then |
| 730 | echo "Server model and client model must be the same. Skip testcase $test_name." |
| 731 | continue |
| 732 | fi |
| 733 | |
| 734 | server_command="$server_envs vllm serve $server_model \ |
| 735 | $server_args" |
| 736 | |
| 737 | # run the server |
| 738 | echo "Running test case $test_name" |
| 739 | echo "Server command: $server_command" |
| 740 | # support remote vllm server |
| 741 | client_remote_args="" |
| 742 | if [[ -z "${REMOTE_HOST}" && "${DRY_RUN:-0}" != "1" ]]; then |
| 743 | bash -c "$server_command" & |
| 744 | server_pid=$! |
| 745 | # wait until the server is alive |
| 746 | if wait_for_server; then |
| 747 | echo "" |
| 748 | echo "vLLM server is up and running." |
| 749 | else |
| 750 | echo "" |
| 751 | echo "vLLM failed to start within the timeout period." |
| 752 | fi |
| 753 | elif [[ "${DRY_RUN:-0}" == "1" ]]; then |
| 754 | # dry-run: don't start server |
| 755 | echo "Dry Run." |
| 756 | else |
| 757 | server_command="Using Remote Server $REMOTE_HOST $REMOTE_PORT" |
| 758 | if [[ ${REMOTE_PORT} ]]; then |
| 759 | client_remote_args=" --host=$REMOTE_HOST --port=$REMOTE_PORT " |
| 760 | else |
| 761 | client_remote_args=" --host=$REMOTE_HOST " |
| 762 | fi |
| 763 | fi |
| 764 | |
| 765 | # save the compilation mode and optimization level on the serving results |
| 766 | # whenever they are set |
| 767 | compilation_config_mode=$(echo "$server_params" | jq -r '."compilation_config.mode" // empty') |
| 768 | optimization_level=$(echo "$server_params" | jq -r '.optimization_level // empty') |
| 769 | |
| 770 | # iterate over different QPS |
| 771 | for qps in $qps_list; do |
| 772 | # remove the surrounding single quote from qps |
| 773 | if [[ "$qps" == *"inf"* ]]; then |
| 774 | qps="inf" |
| 775 | fi |
| 776 | |
| 777 | # iterate over different max_concurrency |
| 778 | for max_concurrency in $max_concurrency_list; do |
| 779 | new_test_name="${test_name}_qps_${qps}_concurrency_${max_concurrency}" |
| 780 | echo " new test name $new_test_name" |
| 781 | # If PROMPTS_PER_CONCURRENCY is set, compute per-concurrency --num-prompts. |
| 782 | num_prompts_arg="" |
| 783 | if [[ -n "${PROMPTS_PER_CONCURRENCY}" ]]; then |
| 784 | num_prompts=$(( max_concurrency * PROMPTS_PER_CONCURRENCY )) |
| 785 | if (( num_prompts < MIN_NUM_PROMPTS )); then num_prompts=$MIN_NUM_PROMPTS; fi |
| 786 | if (( num_prompts > MAX_NUM_PROMPTS )); then num_prompts=$MAX_NUM_PROMPTS; fi |
| 787 | num_prompts_arg="--num-prompts $num_prompts" |
| 788 | fi |
| 789 | # pass the tensor parallel size, the compilation mode, and the optimization |
| 790 | # level to the client so that they can be used on the benchmark dashboard |
| 791 | client_command="vllm bench serve \ |
| 792 | --save-result \ |
| 793 | --result-dir $RESULTS_FOLDER \ |
| 794 | --result-filename ${new_test_name}.json \ |
| 795 | --request-rate $qps \ |
| 796 | --max-concurrency $max_concurrency \ |
| 797 | $num_prompts_arg \ |
| 798 | --metadata tensor_parallel_size=$tp compilation_config.mode=$compilation_config_mode optimization_level=$optimization_level \ |
| 799 | $client_args_effective $client_remote_args " |
| 800 | |
| 801 | echo "Running test case $test_name with qps $qps" |
| 802 | echo "Client command: $client_command" |
| 803 | |
| 804 | if [[ "${DRY_RUN:-0}" != "1" ]]; then |
| 805 | bash -c "$client_command" |
| 806 | fi |
| 807 | |
| 808 | # record the benchmarking commands |
| 809 | jq_output=$(jq -n \ |
| 810 | --arg server "$server_command" \ |
| 811 | --arg client "$client_command" \ |
| 812 | --arg gpu "$gpu_type" \ |
| 813 | '{ |
| 814 | server_command: $server, |
| 815 | client_command: $client, |
| 816 | gpu_type: $gpu |
| 817 | }') |
| 818 | echo "$jq_output" >"$RESULTS_FOLDER/${new_test_name}.commands" |
| 819 | |
| 820 | done |
| 821 | |
| 822 | adaptive_refine_from_static_results \ |
| 823 | "$test_name" "$qps" "$max_concurrency_list" "$tp" \ |
| 824 | "$compilation_config_mode" "$optimization_level" \ |
| 825 | "$client_args_effective" "$client_remote_args" "$server_command" |
| 826 | done |
| 827 | |
| 828 | # clean up |
| 829 | if [[ "${DRY_RUN:-0}" != "1" ]]; then |
| 830 | kill -9 "$server_pid" |
| 831 | kill_gpu_processes |
| 832 | fi |
| 833 | done |
| 834 | } |
| 835 | |
| 836 | main() { |
| 837 | |
| 838 | local ARCH |
| 839 | ARCH='' |
| 840 | if [[ "$ON_CPU" == "1" ]]; then |
| 841 | check_cpus |
| 842 | ARCH="-$gpu_type" |
| 843 | else |
| 844 | check_gpus |
| 845 | ARCH="$arch_suffix" |
| 846 | fi |
| 847 | |
| 848 | # DRY_RUN does not execute vLLM; do not require HF_TOKEN. |
| 849 | if [[ "${DRY_RUN:-0}" != "1" ]]; then |
| 850 | check_hf_token |
| 851 | else |
| 852 | echo "DRY_RUN=1 -> skip HF_TOKEN validation" |
| 853 | fi |
| 854 | |
| 855 | # dependencies |
| 856 | (which wget && which curl) || (apt-get update && apt-get install -y wget curl) |
| 857 | (which jq) || (apt-get update && apt-get -y install jq) |
| 858 | (which lsof) || (apt-get update && apt-get install -y lsof) |
| 859 | |
| 860 | # get the current IP address, required by `vllm bench serve` command |
| 861 | export VLLM_HOST_IP=$(hostname -I | awk '{print $1}') |
| 862 | # turn of the reporting of the status of each request, to clean up the terminal output |
| 863 | export VLLM_LOGGING_LEVEL="WARNING" |
| 864 | |
| 865 | # prepare for benchmarking |
| 866 | cd benchmarks || exit 1 |
| 867 | ensure_sharegpt_downloaded |
| 868 | declare -g RESULTS_FOLDER=results/ |
| 869 | mkdir -p $RESULTS_FOLDER |
| 870 | QUICK_BENCHMARK_ROOT=../.buildkite/performance-benchmarks/ |
| 871 | |
| 872 | # dump vllm info via vllm collect-env |
| 873 | env_output=$(vllm collect-env) |
| 874 | echo "$env_output" >"$RESULTS_FOLDER/vllm_env.txt" |
| 875 | |
| 876 | # benchmarking |
| 877 | run_serving_tests $QUICK_BENCHMARK_ROOT/tests/"${SERVING_JSON:-serving-tests$ARCH.json}" || exit $? |
| 878 | |
| 879 | if [[ "${DRY_RUN:-0}" == "1" ]]; then |
| 880 | echo "DRY_RUN=1 -> skip latency/startup/throughput suites" |
| 881 | exit 0 |
| 882 | fi |
| 883 | |
| 884 | run_latency_tests $QUICK_BENCHMARK_ROOT/tests/"${LATENCY_JSON:-latency-tests$ARCH.json}" |
| 885 | run_startup_tests $QUICK_BENCHMARK_ROOT/tests/"${STARTUP_JSON:-startup-tests$ARCH.json}" |
| 886 | run_throughput_tests $QUICK_BENCHMARK_ROOT/tests/"${THROUGHPUT_JSON:-throughput-tests$ARCH.json}" |
| 887 | |
| 888 | # postprocess benchmarking results |
| 889 | pip install tabulate pandas |
| 890 | python3 $QUICK_BENCHMARK_ROOT/scripts/convert-results-json-to-markdown.py |
| 891 | python3 $QUICK_BENCHMARK_ROOT/scripts/compare-json-results.py -f $RESULTS_FOLDER/benchmark_results.json |
| 892 | |
| 893 | upload_to_buildkite |
| 894 | } |
| 895 | |
| 896 | main "$@" |