A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[CI] Fix crashes of performance benchmark (#7500)
Kuntai Du committed
Aug 16, 2024 at 08:08 UTC
6fc5b0f249396c6fb3a63b3175cd9892e7fedd9b
5 files changed
+21
-36
.buildkite/nightly-benchmarks/run-benchmarks-suite.sh
+4
-14
@@ -70,23 +70,13 @@ wait_for_server() {
70
71
kill_gpu_processes() {
72
# kill all processes on GPU.
73
- pids=$(nvidia-smi --query-compute-apps=pid --format=csv,noheader)
74
- if [ -z "$pids" ]; then
75
- echo "No GPU processes found."
76
- else
77
- for pid in $pids; do
78
- kill -9 "$pid"
79
- echo "Killed process with PID: $pid"
80
- done
73
82
- echo "All GPU processes have been killed."
83
- fi
74
+ ps aux | grep python | grep openai | awk '{print $2}' | xargs -r kill -9
75
+ ps -e | grep pt_main_thread | awk '{print $1}' | xargs kill -9
76
85
- # waiting for GPU processes to be fully killed
86
- # loop while nvidia-smi returns any processes
87
- while [ -n "$(nvidia-smi --query-compute-apps=pid --format=csv,noheader)" ]; do
77
+ # wait until GPU memory usage smaller than 1GB
78
+ while [ $(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -n 1) -ge 1000 ]; do
79
sleep 1
89
- echo "Waiting for GPU processes to be killed"
80
done
81
82
# remove vllm config file
.buildkite/nightly-benchmarks/tests/descriptions.md
+7
-12
@@ -1,47 +1,42 @@
1
2
## Latency tests
3
4
-This test suite aims to test vllm's end-to-end latency under a controlled setup.
5
-
4
- Input length: 32 tokens.
5
- Output length: 128 tokens.
6
- Batch size: fixed (8).
9
-- Models: llama-3 8B, llama-3 70B, mixtral 8x7B.
7
+- Models: llama-3.1 8B, llama-3 70B, mixtral 8x7B.
8
- Evaluation metrics: end-to-end latency (mean, median, p99).
9
12
-### Latency benchmarking results
10
11
{latency_tests_markdown_table}
12
16
-## Throughput tests
13
18
-This test suite aims to test vllm's throughput.
14
+## Throughput tests
15
16
- Input length: randomly sample 200 prompts from ShareGPT dataset (with fixed random seed).
17
- Output length: the corresponding output length of these 200 prompts.
18
- Batch size: dynamically determined by vllm to achieve maximum throughput.
23
-- Models: llama-3 8B, llama-3 70B, mixtral 8x7B.
19
+- Models: llama-3.1 8B, llama-3 70B, mixtral 8x7B.
20
- Evaluation metrics: throughput.
21
26
-### Throughput benchmarking results
22
23
{throughput_tests_markdown_table}
24
30
-## Serving tests
25
32
-This test suite aims to test vllm's real serving metrics.
26
+## Serving tests
27
28
- Input length: randomly sample 200 prompts from ShareGPT dataset (with fixed random seed).
29
- Output length: the corresponding output length of these 200 prompts.
30
- Batch size: dynamically determined by vllm and the arrival pattern of the requests.
31
- **Average QPS (query per second)**: 1, 4, 16 and inf. QPS = inf means all requests come at once. For other QPS values, the arrival time of each query is determined using a random Poisson process (with fixed random seed).
38
-- Models: llama-3 8B, llama-3 70B, mixtral 8x7B.
32
+- Models: llama-3.1 8B, llama-3 70B, mixtral 8x7B.
33
+- We also added a speculative decoding test for llama-3 70B, under QPS 2
34
- Evaluation metrics: throughput, TTFT (time to the first token, with mean, median and p99), ITL (inter-token latency, with mean, median and p99).
35
41
-### Serving benchmarking results
36
37
{serving_tests_markdown_table}
38
39
+
40
## json version of the benchmarking tables
41
42
This section contains the data of the markdown tables above in JSON format.
.buildkite/nightly-benchmarks/tests/latency-tests.json
+2
-2
@@ -2,7 +2,7 @@
2
{
3
"test_name": "latency_llama8B_tp1",
4
"parameters": {
5
- "model": "meta-llama/Meta-Llama-3-8B",
5
+ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
6
"tensor_parallel_size": 1,
7
"load_format": "dummy",
8
"num_iters_warmup": 5,
@@ -12,7 +12,7 @@
12
{
13
"test_name": "latency_llama70B_tp4",
14
"parameters": {
15
- "model": "meta-llama/Meta-Llama-3-70B-Instruct",
15
+ "model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
16
"tensor_parallel_size": 4,
17
"load_format": "dummy",
18
"num-iters-warmup": 5,
.buildkite/nightly-benchmarks/tests/serving-tests.json
+6
-6
@@ -3,7 +3,7 @@
3
"test_name": "serving_llama8B_tp1_sharegpt",
4
"qps_list": [1, 4, 16, "inf"],
5
"server_parameters": {
6
- "model": "meta-llama/Meta-Llama-3-8B",
6
+ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
7
"tensor_parallel_size": 1,
8
"swap_space": 16,
9
"disable_log_stats": "",
@@ -11,7 +11,7 @@
11
"load_format": "dummy"
12
},
13
"client_parameters": {
14
- "model": "meta-llama/Meta-Llama-3-8B",
14
+ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
15
"backend": "vllm",
16
"dataset_name": "sharegpt",
17
"dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
@@ -22,7 +22,7 @@
22
"test_name": "serving_llama70B_tp4_sharegpt",
23
"qps_list": [1, 4, 16, "inf"],
24
"server_parameters": {
25
- "model": "meta-llama/Meta-Llama-3-70B-Instruct",
25
+ "model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
26
"tensor_parallel_size": 4,
27
"swap_space": 16,
28
"disable_log_stats": "",
@@ -30,7 +30,7 @@
30
"load_format": "dummy"
31
},
32
"client_parameters": {
33
- "model": "meta-llama/Meta-Llama-3-70B-Instruct",
33
+ "model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
34
"backend": "vllm",
35
"dataset_name": "sharegpt",
36
"dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
@@ -60,7 +60,7 @@
60
"test_name": "serving_llama70B_tp4_sharegpt_specdecode",
61
"qps_list": [2],
62
"server_parameters": {
63
- "model": "meta-llama/Meta-Llama-3-70B-Instruct",
63
+ "model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
64
"disable_log_requests": "",
65
"tensor_parallel_size": 4,
66
"swap_space": 16,
@@ -70,7 +70,7 @@
70
"use_v2_block_manager": ""
71
},
72
"client_parameters": {
73
- "model": "meta-llama/Meta-Llama-3-70B-Instruct",
73
+ "model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
74
"backend": "vllm",
75
"dataset_name": "sharegpt",
76
"dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
.buildkite/nightly-benchmarks/tests/throughput-tests.json
+2
-2
@@ -2,7 +2,7 @@
2
{
3
"test_name": "throughput_llama8B_tp1",
4
"parameters": {
5
- "model": "meta-llama/Meta-Llama-3-8B",
5
+ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
6
"tensor_parallel_size": 1,
7
"load_format": "dummy",
8
"dataset": "./ShareGPT_V3_unfiltered_cleaned_split.json",
@@ -13,7 +13,7 @@
13
{
14
"test_name": "throughput_llama70B_tp4",
15
"parameters": {
16
- "model": "meta-llama/Meta-Llama-3-70B-Instruct",
16
+ "model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
17
"tensor_parallel_size": 4,
18
"load_format": "dummy",
19
"dataset": "./ShareGPT_V3_unfiltered_cleaned_split.json",