@tduncan / Nvidia-Optimization-Stack-VLLM
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
sh 131 lines 3.73 KB
Raw
1 #!/bin/bash
2
3 # default values
4 MODEL=${MODEL:-"Qwen/Qwen2.5-7B-Instruct"}
5 BACKEND=${BACKEND:-"vllm"}
6 DATASET=${DATASET:-"xgrammar_bench"}
7 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
8 OUTPUT_DIR=${OUTPUT_DIR:-"$SCRIPT_DIR/structured_output_benchmark_results"}
9 PORT=${PORT:-8000}
10 STRUCTURED_OUTPUT_RATIO=${STRUCTURED_OUTPUT_RATIO:-1}
11 TOTAL_SECONDS=${TOTAL_SECONDS:-90}
12 MAX_NEW_TOKENS=${MAX_NEW_TOKENS:-300}
13 TOKENIZER_MODE=${TOKENIZER_MODE:-"auto"}
14
15 usage() {
16 echo "Usage: $0 [options]"
17 echo "Options:"
18 echo " --model MODEL Model to benchmark (default: $MODEL)"
19 echo " --backend BACKEND Backend to use (default: $BACKEND)"
20 echo " --dataset DATASET Dataset to use (default: $DATASET)"
21 echo " --max-new-tokens N Maximum number of tokens to generate (default: $MAX_NEW_TOKENS)"
22 echo " --output-dir DIR Output directory for results (default: $OUTPUT_DIR)"
23 echo " --port PORT Port to use (default: $PORT)"
24 echo " --structured-output-ratio N Ratio of structured outputs (default: $STRUCTURED_OUTPUT_RATIO)"
25 echo " --tokenizer-mode MODE Tokenizer mode to use (default: $TOKENIZER_MODE)"
26 echo " --total-seconds N Total seconds to run the benchmark (default: $TOTAL_SECONDS)"
27 echo " -h, --help Show this help message and exit"
28 exit 0
29 }
30
31 # parse command line arguments
32 while [[ $# -gt 0 ]]; do
33 case $1 in
34 --model)
35 MODEL="$2"
36 shift 2
37 ;;
38 --backend)
39 BACKEND="$2"
40 shift 2
41 ;;
42 --dataset)
43 DATASET="$2"
44 shift 2
45 ;;
46 --max-new-tokens)
47 MAX_NEW_TOKENS="$2"
48 shift 2
49 ;;
50 --output-dir)
51 OUTPUT_DIR="$2"
52 shift 2
53 ;;
54 --port)
55 PORT="$2"
56 shift 2
57 ;;
58 --structured-output-ratio)
59 STRUCTURED_OUTPUT_RATIO="$2"
60 shift 2
61 ;;
62 --tokenizer-mode)
63 TOKENIZER_MODE="$2"
64 shift 2
65 ;;
66 --total-seconds)
67 TOTAL_SECONDS="$2"
68 shift 2
69 ;;
70 -h|--help)
71 usage
72 ;;
73 *)
74 printf "Unknown argument: %s\n" "$1"
75 usage
76 ;;
77 esac
78 done
79
80 # Create output directory if it doesn't exist
81 mkdir -p "$OUTPUT_DIR"
82
83 # Define QPS values to test
84 QPS_VALUES=(25 20 15 10 5 1)
85
86 # Common parameters
87 COMMON_PARAMS=(
88 --backend "$BACKEND"
89 --model "$MODEL"
90 --dataset "$DATASET"
91 --structured-output-ratio "$STRUCTURED_OUTPUT_RATIO"
92 --save-results
93 --result-dir "$OUTPUT_DIR"
94 --output-len "$MAX_NEW_TOKENS"
95 --port "$PORT"
96 --tokenizer-mode "$TOKENIZER_MODE"
97 )
98
99 echo "Starting structured output benchmark with model: $MODEL"
100 echo "Backend: $BACKEND"
101 echo "Dataset: $DATASET"
102 echo "Results will be saved to: $OUTPUT_DIR"
103 echo "----------------------------------------"
104
105 # Run benchmarks with different QPS values
106 for qps in "${QPS_VALUES[@]}"; do
107 echo "Running benchmark with QPS: $qps"
108
109 # Get git hash and branch for the filename
110 GIT_HASH=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown")
111 GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown")
112
113 # Construct filename for this run
114 FILENAME="${BACKEND}_${qps}qps_$(basename "$MODEL")_${DATASET}_${GIT_HASH}_${GIT_BRANCH}.json"
115
116 NUM_PROMPTS=$(echo "$TOTAL_SECONDS * $qps" | bc)
117 NUM_PROMPTS=${NUM_PROMPTS%.*} # Remove fractional part
118 echo "Running benchmark with $NUM_PROMPTS prompts"
119
120 # Run the benchmark
121 python "$SCRIPT_DIR/benchmark_serving_structured_output.py" "${COMMON_PARAMS[@]}" \
122 --request-rate "$qps" \
123 --result-filename "$FILENAME" \
124 --num-prompts "$NUM_PROMPTS"
125
126 echo "Completed benchmark with QPS: $qps"
127 echo "----------------------------------------"
128 done
129
130 echo "All benchmarks completed!"
131 echo "Results saved to: $OUTPUT_DIR"