@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3d8a5f063d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[CI] Organizing performance benchmark files (#7616)

Kuntai Du committed Aug 19, 2024 at 22:43 UTC 3d8a5f063d8a96ccfb8fc14d1d43b93cea0411a0
5 files changed +36 -24
.buildkite/nightly-benchmarks/README.md
+5 -4
@@ -34,17 +34,18 @@ See [vLLM performance dashboard](https://perf.vllm.ai) for the latest performan
34
35 Performance benchmark will be triggered when:
36 - A PR being merged into vllm.
37 -- Every commit for those PRs with `perf-benchmarks` label.
37 +- Every commit for those PRs with `perf-benchmarks` label AND `ready` label.
38
39 Nightly benchmark will be triggered when:
40 -- Every commit for those PRs with `nightly-benchmarks` label.
40 +- Every commit for those PRs with `perf-benchmarks` label and `nightly-benchmarks` label.
41
42
43
44
45 ## Performance benchmark details
46
47 -See [descriptions.md](tests/descriptions.md) for detailed descriptions, and use `tests/latency-tests.json`, `tests/throughput-tests.json`, `tests/serving-tests.json` to configure the test cases.
47 +
48 +See [performance-benchmarks-descriptions.md](performance-benchmarks-descriptions.md) for detailed descriptions, and use `tests/latency-tests.json`, `tests/throughput-tests.json`, `tests/serving-tests.json` to configure the test cases.
49
50
51 #### Latency test
@@ -68,7 +69,7 @@ Here is an example of one test inside `latency-tests.json`:
69
70 In this example:
71 - The `test_name` attributes is a unique identifier for the test. In `latency-tests.json`, it must start with `latency_`.
71 -- The `parameters` attribute control the command line arguments to be used for `benchmark_latency.py`. Note that please use underline `_` instead of the dash `-` when specifying the command line arguments, and `run-benchmarks-suite.sh` will convert the underline to dash when feeding the arguments to `benchmark_latency.py`. For example, the corresponding command line arguments for `benchmark_latency.py` will be `--model meta-llama/Meta-Llama-3-8B --tensor-parallel-size 1 --load-format dummy --num-iters-warmup 5 --num-iters 15`
72 +- The `parameters` attribute control the command line arguments to be used for `benchmark_latency.py`. Note that please use underline `_` instead of the dash `-` when specifying the command line arguments, and `run-performance-benchmarks.sh` will convert the underline to dash when feeding the arguments to `benchmark_latency.py`. For example, the corresponding command line arguments for `benchmark_latency.py` will be `--model meta-llama/Meta-Llama-3-8B --tensor-parallel-size 1 --load-format dummy --num-iters-warmup 5 --num-iters 15`
73
74 Note that the performance numbers are highly sensitive to the value of the parameters. Please make sure the parameters are set correctly.
75
.buildkite/nightly-benchmarks/benchmark-pipeline.yaml
+1 -1
@@ -21,7 +21,7 @@ steps:
21 containers:
22 - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
23 command:
24 - - bash .buildkite/nightly-benchmarks/run-benchmarks-suite.sh
24 + - bash .buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh
25 resources:
26 limits:
27 nvidia.com/gpu: 8
.buildkite/nightly-benchmarks/performance-benchmarks-descriptions.md renamed
.buildkite/nightly-benchmarks/scripts/convert-results-json-to-markdown.py
+2 -2
@@ -174,8 +174,8 @@ if __name__ == "__main__":
174 # document the result
175 with open(results_folder / "benchmark_results.md", "w") as f:
176
177 - results = read_markdown(
178 - "../.buildkite/nightly-benchmarks/tests/descriptions.md")
177 + results = read_markdown("../.buildkite/nightly-benchmarks/" +
178 + "performance-benchmarks-descriptions.md")
179 results = results.format(
180 latency_tests_markdown_table=latency_md_table,
181 throughput_tests_markdown_table=throughput_md_table,
.buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh renamed
+28 -17
@@ -37,9 +37,9 @@ check_hf_token() {
37 ensure_sharegpt_downloaded() {
38 local FILE=ShareGPT_V3_unfiltered_cleaned_split.json
39 if [ ! -f "$FILE" ]; then
40 - wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/$FILE
40 + wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/$FILE
41 else
42 - echo "$FILE already exists."
42 + echo "$FILE already exists."
43 fi
44 }
45
@@ -68,11 +68,29 @@ wait_for_server() {
68 done' && return 0 || return 1
69 }
70
71 +kill_processes_launched_by_current_bash() {
72 + # Kill all python processes launched from current bash script
73 + current_shell_pid=$$
74 + processes=$(ps -eo pid,ppid,command | awk -v ppid="$current_shell_pid" -v proc="$1" '$2 == ppid && $3 ~ proc {print $1}')
75 + if [ -n "$processes" ]; then
76 + echo "Killing the following processes matching '$1':"
77 + echo "$processes"
78 + echo "$processes" | xargs kill -9
79 + else
80 + echo "No processes found matching '$1'."
81 + fi
82 +}
83 +
84 kill_gpu_processes() {
72 - # kill all processes on GPU.
85
74 - ps aux | grep python | grep openai | awk '{print $2}' | xargs -r kill -9
75 - ps -e | grep pt_main_thread | awk '{print $1}' | xargs kill -9
86 + ps -aux
87 + lsof -t -i:8000 | xargs -r kill -9
88 + pkill -f pt_main_thread
89 + # this line doesn't work now
90 + # ps aux | grep python | grep openai | awk '{print $2}' | xargs -r kill -9
91 + pkill -f python3
92 + pkill -f /usr/bin/python3
93 +
94
95 # wait until GPU memory usage smaller than 1GB
96 while [ $(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -n 1) -ge 1000 ]; do
@@ -82,11 +100,6 @@ kill_gpu_processes() {
100 # remove vllm config file
101 rm -rf ~/.config/vllm
102
85 - # Print the GPU memory usage
86 - # so that we know if all GPU processes are killed.
87 - gpu_memory_usage=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i 0)
88 - # The memory usage should be 0 MB.
89 - echo "GPU 0 Memory Usage: $gpu_memory_usage MB"
103 }
104
105 upload_to_buildkite() {
@@ -104,7 +117,7 @@ upload_to_buildkite() {
117 fi
118
119 # Use the determined command to annotate and upload artifacts
107 - $BUILDKITE_AGENT_COMMAND annotate --style "info" --context "$BUILDKITE_LABEL-benchmark-results" < $RESULTS_FOLDER/benchmark_results.md
120 + $BUILDKITE_AGENT_COMMAND annotate --style "info" --context "$BUILDKITE_LABEL-benchmark-results" <$RESULTS_FOLDER/benchmark_results.md
121 $BUILDKITE_AGENT_COMMAND artifact upload "$RESULTS_FOLDER/*"
122 }
123
@@ -156,7 +169,7 @@ run_latency_tests() {
169 latency_command: $latency,
170 gpu_type: $gpu
171 }')
159 - echo "$jq_output" > "$RESULTS_FOLDER/$test_name.commands"
172 + echo "$jq_output" >"$RESULTS_FOLDER/$test_name.commands"
173
174 # run the benchmark
175 eval "$latency_command"
@@ -166,7 +179,6 @@ run_latency_tests() {
179 done
180 }
181
169 -
182 run_throughput_tests() {
183 # run throughput tests using `benchmark_throughput.py`
184 # $1: a json file specifying throughput test cases
@@ -214,7 +226,7 @@ run_throughput_tests() {
226 throughput_command: $command,
227 gpu_type: $gpu
228 }')
217 - echo "$jq_output" > "$RESULTS_FOLDER/$test_name.commands"
229 + echo "$jq_output" >"$RESULTS_FOLDER/$test_name.commands"
230
231 # run the benchmark
232 eval "$throughput_command"
@@ -246,7 +258,6 @@ run_serving_tests() {
258 continue
259 fi
260
249 -
261 # get client and server arguments
262 server_params=$(echo "$params" | jq -r '.server_parameters')
263 client_params=$(echo "$params" | jq -r '.client_parameters')
@@ -324,7 +335,7 @@ run_serving_tests() {
335 client_command: $client,
336 gpu_type: $gpu
337 }')
327 - echo "$jq_output" > "$RESULTS_FOLDER/${new_test_name}.commands"
338 + echo "$jq_output" >"$RESULTS_FOLDER/${new_test_name}.commands"
339
340 done
341
@@ -341,6 +352,7 @@ main() {
352 # dependencies
353 (which wget && which curl) || (apt-get update && apt-get install -y wget curl)
354 (which jq) || (apt-get update && apt-get -y install jq)
355 + (which lsof) || (apt-get update && apt-get install -y lsof)
356
357 # get the current IP address, required by benchmark_serving.py
358 export VLLM_HOST_IP=$(hostname -I | awk '{print $1}')
@@ -359,7 +371,6 @@ main() {
371 run_latency_tests $QUICK_BENCHMARK_ROOT/tests/latency-tests.json
372 run_throughput_tests $QUICK_BENCHMARK_ROOT/tests/throughput-tests.json
373
362 -
374 # postprocess benchmarking results
375 pip install tabulate pandas
376 python3 $QUICK_BENCHMARK_ROOT/scripts/convert-results-json-to-markdown.py