A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[CI] Organizing performance benchmark files (#7616)
Kuntai Du committed
Aug 19, 2024 at 22:43 UTC
3d8a5f063d8a96ccfb8fc14d1d43b93cea0411a0
5 files changed
+36
-24
.buildkite/nightly-benchmarks/README.md
+5
-4
@@ -34,17 +34,18 @@ See [vLLM performance dashboard](https://perf.vllm.ai) for the latest performan
34
35
Performance benchmark will be triggered when:
36
- A PR being merged into vllm.
37
-- Every commit for those PRs with `perf-benchmarks` label.
37
+- Every commit for those PRs with `perf-benchmarks` label AND `ready` label.
38
39
Nightly benchmark will be triggered when:
40
-- Every commit for those PRs with `nightly-benchmarks` label.
40
+- Every commit for those PRs with `perf-benchmarks` label and `nightly-benchmarks` label.
41
42
43
44
45
## Performance benchmark details
46
47
-See [descriptions.md](tests/descriptions.md) for detailed descriptions, and use `tests/latency-tests.json`, `tests/throughput-tests.json`, `tests/serving-tests.json` to configure the test cases.
47
+
48
+See [performance-benchmarks-descriptions.md](performance-benchmarks-descriptions.md) for detailed descriptions, and use `tests/latency-tests.json`, `tests/throughput-tests.json`, `tests/serving-tests.json` to configure the test cases.
49
50
51
#### Latency test
@@ -68,7 +69,7 @@ Here is an example of one test inside `latency-tests.json`:
69
70
In this example:
71
- The `test_name` attributes is a unique identifier for the test. In `latency-tests.json`, it must start with `latency_`.
71
-- The `parameters` attribute control the command line arguments to be used for `benchmark_latency.py`. Note that please use underline `_` instead of the dash `-` when specifying the command line arguments, and `run-benchmarks-suite.sh` will convert the underline to dash when feeding the arguments to `benchmark_latency.py`. For example, the corresponding command line arguments for `benchmark_latency.py` will be `--model meta-llama/Meta-Llama-3-8B --tensor-parallel-size 1 --load-format dummy --num-iters-warmup 5 --num-iters 15`
72
+- The `parameters` attribute control the command line arguments to be used for `benchmark_latency.py`. Note that please use underline `_` instead of the dash `-` when specifying the command line arguments, and `run-performance-benchmarks.sh` will convert the underline to dash when feeding the arguments to `benchmark_latency.py`. For example, the corresponding command line arguments for `benchmark_latency.py` will be `--model meta-llama/Meta-Llama-3-8B --tensor-parallel-size 1 --load-format dummy --num-iters-warmup 5 --num-iters 15`
73
74
Note that the performance numbers are highly sensitive to the value of the parameters. Please make sure the parameters are set correctly.
75
.buildkite/nightly-benchmarks/benchmark-pipeline.yaml
+1
-1
@@ -21,7 +21,7 @@ steps:
21
containers:
22
- image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
23
command:
24
- - bash .buildkite/nightly-benchmarks/run-benchmarks-suite.sh
24
+ - bash .buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh
25
resources:
26
limits:
27
nvidia.com/gpu: 8
.buildkite/nightly-benchmarks/performance-benchmarks-descriptions.md
renamed
.buildkite/nightly-benchmarks/scripts/convert-results-json-to-markdown.py
+2
-2
@@ -174,8 +174,8 @@ if __name__ == "__main__":
174
# document the result
175
with open(results_folder / "benchmark_results.md", "w") as f:
176
177
- results = read_markdown(
178
- "../.buildkite/nightly-benchmarks/tests/descriptions.md")
177
+ results = read_markdown("../.buildkite/nightly-benchmarks/" +
178
+ "performance-benchmarks-descriptions.md")
179
results = results.format(
180
latency_tests_markdown_table=latency_md_table,
181
throughput_tests_markdown_table=throughput_md_table,
.buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh
renamed
+28
-17
@@ -37,9 +37,9 @@ check_hf_token() {
37
ensure_sharegpt_downloaded() {
38
local FILE=ShareGPT_V3_unfiltered_cleaned_split.json
39
if [ ! -f "$FILE" ]; then
40
- wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/$FILE
40
+ wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/$FILE
41
else
42
- echo "$FILE already exists."
42
+ echo "$FILE already exists."
43
fi
44
}
45
@@ -68,11 +68,29 @@ wait_for_server() {
68
done' && return 0 || return 1
69
}
70
71
+kill_processes_launched_by_current_bash() {
72
+ # Kill all python processes launched from current bash script
73
+ current_shell_pid=$$
74
+ processes=$(ps -eo pid,ppid,command | awk -v ppid="$current_shell_pid" -v proc="$1" '$2 == ppid && $3 ~ proc {print $1}')
75
+ if [ -n "$processes" ]; then
76
+ echo "Killing the following processes matching '$1':"
77
+ echo "$processes"
78
+ echo "$processes" | xargs kill -9
79
+ else
80
+ echo "No processes found matching '$1'."
81
+ fi
82
+}
83
+
84
kill_gpu_processes() {
72
- # kill all processes on GPU.
85
74
- ps aux | grep python | grep openai | awk '{print $2}' | xargs -r kill -9
75
- ps -e | grep pt_main_thread | awk '{print $1}' | xargs kill -9
86
+ ps -aux
87
+ lsof -t -i:8000 | xargs -r kill -9
88
+ pkill -f pt_main_thread
89
+ # this line doesn't work now
90
+ # ps aux | grep python | grep openai | awk '{print $2}' | xargs -r kill -9
91
+ pkill -f python3
92
+ pkill -f /usr/bin/python3
93
+
94
95
# wait until GPU memory usage smaller than 1GB
96
while [ $(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -n 1) -ge 1000 ]; do
@@ -82,11 +100,6 @@ kill_gpu_processes() {
100
# remove vllm config file
101
rm -rf ~/.config/vllm
102
85
- # Print the GPU memory usage
86
- # so that we know if all GPU processes are killed.
87
- gpu_memory_usage=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i 0)
88
- # The memory usage should be 0 MB.
89
- echo "GPU 0 Memory Usage: $gpu_memory_usage MB"
103
}
104
105
upload_to_buildkite() {
@@ -104,7 +117,7 @@ upload_to_buildkite() {
117
fi
118
119
# Use the determined command to annotate and upload artifacts
107
- $BUILDKITE_AGENT_COMMAND annotate --style "info" --context "$BUILDKITE_LABEL-benchmark-results" < $RESULTS_FOLDER/benchmark_results.md
120
+ $BUILDKITE_AGENT_COMMAND annotate --style "info" --context "$BUILDKITE_LABEL-benchmark-results" <$RESULTS_FOLDER/benchmark_results.md
121
$BUILDKITE_AGENT_COMMAND artifact upload "$RESULTS_FOLDER/*"
122
}
123
@@ -156,7 +169,7 @@ run_latency_tests() {
169
latency_command: $latency,
170
gpu_type: $gpu
171
}')
159
- echo "$jq_output" > "$RESULTS_FOLDER/$test_name.commands"
172
+ echo "$jq_output" >"$RESULTS_FOLDER/$test_name.commands"
173
174
# run the benchmark
175
eval "$latency_command"
@@ -166,7 +179,6 @@ run_latency_tests() {
179
done
180
}
181
169
-
182
run_throughput_tests() {
183
# run throughput tests using `benchmark_throughput.py`
184
# $1: a json file specifying throughput test cases
@@ -214,7 +226,7 @@ run_throughput_tests() {
226
throughput_command: $command,
227
gpu_type: $gpu
228
}')
217
- echo "$jq_output" > "$RESULTS_FOLDER/$test_name.commands"
229
+ echo "$jq_output" >"$RESULTS_FOLDER/$test_name.commands"
230
231
# run the benchmark
232
eval "$throughput_command"
@@ -246,7 +258,6 @@ run_serving_tests() {
258
continue
259
fi
260
249
-
261
# get client and server arguments
262
server_params=$(echo "$params" | jq -r '.server_parameters')
263
client_params=$(echo "$params" | jq -r '.client_parameters')
@@ -324,7 +335,7 @@ run_serving_tests() {
335
client_command: $client,
336
gpu_type: $gpu
337
}')
327
- echo "$jq_output" > "$RESULTS_FOLDER/${new_test_name}.commands"
338
+ echo "$jq_output" >"$RESULTS_FOLDER/${new_test_name}.commands"
339
340
done
341
@@ -341,6 +352,7 @@ main() {
352
# dependencies
353
(which wget && which curl) || (apt-get update && apt-get install -y wget curl)
354
(which jq) || (apt-get update && apt-get -y install jq)
355
+ (which lsof) || (apt-get update && apt-get install -y lsof)
356
357
# get the current IP address, required by benchmark_serving.py
358
export VLLM_HOST_IP=$(hostname -I | awk '{print $1}')
@@ -359,7 +371,6 @@ main() {
371
run_latency_tests $QUICK_BENCHMARK_ROOT/tests/latency-tests.json
372
run_throughput_tests $QUICK_BENCHMARK_ROOT/tests/throughput-tests.json
373
362
-
374
# postprocess benchmarking results
375
pip install tabulate pandas
376
python3 $QUICK_BENCHMARK_ROOT/scripts/convert-results-json-to-markdown.py