@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6fc5b0f249
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[CI] Fix crashes of performance benchmark (#7500)

Kuntai Du committed Aug 16, 2024 at 08:08 UTC 6fc5b0f249396c6fb3a63b3175cd9892e7fedd9b
5 files changed +21 -36
.buildkite/nightly-benchmarks/run-benchmarks-suite.sh
+4 -14
@@ -70,23 +70,13 @@ wait_for_server() {
70
71 kill_gpu_processes() {
72 # kill all processes on GPU.
73 - pids=$(nvidia-smi --query-compute-apps=pid --format=csv,noheader)
74 - if [ -z "$pids" ]; then
75 - echo "No GPU processes found."
76 - else
77 - for pid in $pids; do
78 - kill -9 "$pid"
79 - echo "Killed process with PID: $pid"
80 - done
73
82 - echo "All GPU processes have been killed."
83 - fi
74 + ps aux | grep python | grep openai | awk '{print $2}' | xargs -r kill -9
75 + ps -e | grep pt_main_thread | awk '{print $1}' | xargs kill -9
76
85 - # waiting for GPU processes to be fully killed
86 - # loop while nvidia-smi returns any processes
87 - while [ -n "$(nvidia-smi --query-compute-apps=pid --format=csv,noheader)" ]; do
77 + # wait until GPU memory usage smaller than 1GB
78 + while [ $(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -n 1) -ge 1000 ]; do
79 sleep 1
89 - echo "Waiting for GPU processes to be killed"
80 done
81
82 # remove vllm config file
.buildkite/nightly-benchmarks/tests/descriptions.md
+7 -12
@@ -1,47 +1,42 @@
1
2 ## Latency tests
3
4 -This test suite aims to test vllm's end-to-end latency under a controlled setup.
5 -
4 - Input length: 32 tokens.
5 - Output length: 128 tokens.
6 - Batch size: fixed (8).
9 -- Models: llama-3 8B, llama-3 70B, mixtral 8x7B.
7 +- Models: llama-3.1 8B, llama-3 70B, mixtral 8x7B.
8 - Evaluation metrics: end-to-end latency (mean, median, p99).
9
12 -### Latency benchmarking results
10
11 {latency_tests_markdown_table}
12
16 -## Throughput tests
13
18 -This test suite aims to test vllm's throughput.
14 +## Throughput tests
15
16 - Input length: randomly sample 200 prompts from ShareGPT dataset (with fixed random seed).
17 - Output length: the corresponding output length of these 200 prompts.
18 - Batch size: dynamically determined by vllm to achieve maximum throughput.
23 -- Models: llama-3 8B, llama-3 70B, mixtral 8x7B.
19 +- Models: llama-3.1 8B, llama-3 70B, mixtral 8x7B.
20 - Evaluation metrics: throughput.
21
26 -### Throughput benchmarking results
22
23 {throughput_tests_markdown_table}
24
30 -## Serving tests
25
32 -This test suite aims to test vllm's real serving metrics.
26 +## Serving tests
27
28 - Input length: randomly sample 200 prompts from ShareGPT dataset (with fixed random seed).
29 - Output length: the corresponding output length of these 200 prompts.
30 - Batch size: dynamically determined by vllm and the arrival pattern of the requests.
31 - **Average QPS (query per second)**: 1, 4, 16 and inf. QPS = inf means all requests come at once. For other QPS values, the arrival time of each query is determined using a random Poisson process (with fixed random seed).
38 -- Models: llama-3 8B, llama-3 70B, mixtral 8x7B.
32 +- Models: llama-3.1 8B, llama-3 70B, mixtral 8x7B.
33 +- We also added a speculative decoding test for llama-3 70B, under QPS 2
34 - Evaluation metrics: throughput, TTFT (time to the first token, with mean, median and p99), ITL (inter-token latency, with mean, median and p99).
35
41 -### Serving benchmarking results
36
37 {serving_tests_markdown_table}
38
39 +
40 ## json version of the benchmarking tables
41
42 This section contains the data of the markdown tables above in JSON format.
.buildkite/nightly-benchmarks/tests/latency-tests.json
+2 -2
@@ -2,7 +2,7 @@
2 {
3 "test_name": "latency_llama8B_tp1",
4 "parameters": {
5 - "model": "meta-llama/Meta-Llama-3-8B",
5 + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
6 "tensor_parallel_size": 1,
7 "load_format": "dummy",
8 "num_iters_warmup": 5,
@@ -12,7 +12,7 @@
12 {
13 "test_name": "latency_llama70B_tp4",
14 "parameters": {
15 - "model": "meta-llama/Meta-Llama-3-70B-Instruct",
15 + "model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
16 "tensor_parallel_size": 4,
17 "load_format": "dummy",
18 "num-iters-warmup": 5,
.buildkite/nightly-benchmarks/tests/serving-tests.json
+6 -6
@@ -3,7 +3,7 @@
3 "test_name": "serving_llama8B_tp1_sharegpt",
4 "qps_list": [1, 4, 16, "inf"],
5 "server_parameters": {
6 - "model": "meta-llama/Meta-Llama-3-8B",
6 + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
7 "tensor_parallel_size": 1,
8 "swap_space": 16,
9 "disable_log_stats": "",
@@ -11,7 +11,7 @@
11 "load_format": "dummy"
12 },
13 "client_parameters": {
14 - "model": "meta-llama/Meta-Llama-3-8B",
14 + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
15 "backend": "vllm",
16 "dataset_name": "sharegpt",
17 "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
@@ -22,7 +22,7 @@
22 "test_name": "serving_llama70B_tp4_sharegpt",
23 "qps_list": [1, 4, 16, "inf"],
24 "server_parameters": {
25 - "model": "meta-llama/Meta-Llama-3-70B-Instruct",
25 + "model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
26 "tensor_parallel_size": 4,
27 "swap_space": 16,
28 "disable_log_stats": "",
@@ -30,7 +30,7 @@
30 "load_format": "dummy"
31 },
32 "client_parameters": {
33 - "model": "meta-llama/Meta-Llama-3-70B-Instruct",
33 + "model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
34 "backend": "vllm",
35 "dataset_name": "sharegpt",
36 "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
@@ -60,7 +60,7 @@
60 "test_name": "serving_llama70B_tp4_sharegpt_specdecode",
61 "qps_list": [2],
62 "server_parameters": {
63 - "model": "meta-llama/Meta-Llama-3-70B-Instruct",
63 + "model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
64 "disable_log_requests": "",
65 "tensor_parallel_size": 4,
66 "swap_space": 16,
@@ -70,7 +70,7 @@
70 "use_v2_block_manager": ""
71 },
72 "client_parameters": {
73 - "model": "meta-llama/Meta-Llama-3-70B-Instruct",
73 + "model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
74 "backend": "vllm",
75 "dataset_name": "sharegpt",
76 "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
.buildkite/nightly-benchmarks/tests/throughput-tests.json
+2 -2
@@ -2,7 +2,7 @@
2 {
3 "test_name": "throughput_llama8B_tp1",
4 "parameters": {
5 - "model": "meta-llama/Meta-Llama-3-8B",
5 + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
6 "tensor_parallel_size": 1,
7 "load_format": "dummy",
8 "dataset": "./ShareGPT_V3_unfiltered_cleaned_split.json",
@@ -13,7 +13,7 @@
13 {
14 "test_name": "throughput_llama70B_tp4",
15 "parameters": {
16 - "model": "meta-llama/Meta-Llama-3-70B-Instruct",
16 + "model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
17 "tensor_parallel_size": 4,
18 "load_format": "dummy",
19 "dataset": "./ShareGPT_V3_unfiltered_cleaned_split.json",