A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[CI] the readability of benchmarking and prepare for dashboard (#5571)
[CI] Improve the readability of performance benchmarking results and prepare for upcoming performance dashboard (#5571)
Kuntai Du committed
Jun 17, 2024 at 11:41 UTC
9e4e6fe2073ff5e4a747d5ce2a08d321268b7254
8 files changed
+232
-130
.buildkite/nightly-benchmarks/README.md
+13
-8
@@ -13,9 +13,17 @@ This benchmark will be *triggered* upon:
13
14
**Benchmarking Duration**: about 1hr.
15
16
-## Configuring the workload for the quick benchmark
16
+**For benchmarking developers**: please try your best to constraint the duration of benchmarking to less than 1.5 hr so that it won't take forever to run.
17
18
-The workload of the quick benchmark contains two parts: latency tests in `latency-tests.json`, throughput tests in `throughput-tests.json` and serving tests in `serving-tests.json`.
18
+
19
+## Configuring the workload
20
+
21
+The benchmarking workload contains three parts:
22
+- Latency tests in `latency-tests.json`.
23
+- Throughput tests in `throughput-tests.json`.
24
+- Serving tests in `serving-tests.json`.
25
+
26
+See [descriptions.md](tests/descriptions.md) for detailed descriptions.
27
28
### Latency test
29
@@ -23,7 +31,6 @@ Here is an example of one test inside `latency-tests.json`:
31
32
```json
33
[
26
- ...
34
{
35
"test_name": "latency_llama8B_tp1",
36
"parameters": {
@@ -34,7 +41,6 @@ Here is an example of one test inside `latency-tests.json`:
41
"num_iters": 15
42
}
43
},
37
- ...
44
]
45
```
46
@@ -57,7 +63,6 @@ We test the throughput by using `benchmark_serving.py` with request rate = inf t
63
64
```
65
[
60
- ...
66
{
67
"test_name": "serving_llama8B_tp1_sharegpt",
68
"qps_list": [1, 4, 16, "inf"],
@@ -77,7 +82,6 @@ We test the throughput by using `benchmark_serving.py` with request rate = inf t
82
"num_prompts": 200
83
}
84
},
80
- ...
85
]
86
```
87
@@ -92,7 +96,8 @@ The number of this test is less stable compared to the delay and latency benchma
96
WARNING: The benchmarking script will save json results by itself, so please do not configure `--save-results` or other results-saving-related parameters in `serving-tests.json`.
97
98
## Visualizing the results
95
-The `convert-results-json-to-markdown.py` helps you put the benchmarking results inside a markdown table.
99
+The `convert-results-json-to-markdown.py` helps you put the benchmarking results inside a markdown table, by formatting [descriptions.md](tests/descriptions.md) with real benchmarking results.
100
You can find the result presented as a table inside the `buildkite/performance-benchmark` job page.
101
If you do not see the table, please wait till the benchmark finish running.
98
-The JSON file is also attached within each buildkite job for further analysis.
\ No newline at end of file
102
+The json version of the table (together with the json version of the benchmark) will be also attached to the markdown file.
103
+The raw benchmarking results (in the format of json files) are in the `Artifacts` tab of the benchmarking.
.buildkite/nightly-benchmarks/run-benchmarks-suite.sh
+3
-3
@@ -343,9 +343,9 @@ main() {
343
QUICK_BENCHMARK_ROOT=../.buildkite/nightly-benchmarks/
344
345
# benchmarking
346
- run_serving_tests $QUICK_BENCHMARK_ROOT/serving-tests.json
347
- run_latency_tests $QUICK_BENCHMARK_ROOT/latency-tests.json
348
- run_throughput_tests $QUICK_BENCHMARK_ROOT/throughput-tests.json
346
+ run_serving_tests $QUICK_BENCHMARK_ROOT/tests/serving-tests.json
347
+ run_latency_tests $QUICK_BENCHMARK_ROOT/tests/latency-tests.json
348
+ run_throughput_tests $QUICK_BENCHMARK_ROOT/tests/throughput-tests.json
349
350
351
# postprocess benchmarking results
.buildkite/nightly-benchmarks/scripts/convert-results-json-to-markdown.py
+145
-115
@@ -1,4 +1,5 @@
1
import json
2
+import os
3
from pathlib import Path
4
5
import pandas as pd
@@ -11,12 +12,13 @@ latency_results = []
12
latency_column_mapping = {
13
"test_name": "Test name",
14
"gpu_type": "GPU",
14
- "avg_latency": "Average latency (s)",
15
- "P10": "P10 (s)",
16
- "P25": "P25 (s)",
17
- "P50": "P50 (s)",
18
- "P75": "P75 (s)",
19
- "P90": "P90 (s)",
15
+ "avg_latency": "Mean latency (ms)",
16
+ # "P10": "P10 (s)",
17
+ # "P25": "P25 (s)",
18
+ "P50": "Median",
19
+ # "P75": "P75 (s)",
20
+ # "P90": "P90 (s)",
21
+ "P99": "P99",
22
}
23
24
# thoughput tests and the keys that will be printed into markdown
@@ -24,11 +26,11 @@ throughput_results = []
26
throughput_results_column_mapping = {
27
"test_name": "Test name",
28
"gpu_type": "GPU",
27
- "num_requests": "# of req.",
28
- "total_num_tokens": "Total # of tokens",
29
- "elapsed_time": "Elapsed time (s)",
29
+ # "num_requests": "# of req.",
30
+ # "total_num_tokens": "Total # of tokens",
31
+ # "elapsed_time": "Elapsed time (s)",
32
"requests_per_second": "Tput (req/s)",
31
- "tokens_per_second": "Tput (tok/s)",
33
+ # "tokens_per_second": "Tput (tok/s)",
34
}
35
36
# serving results and the keys that will be printed into markdown
@@ -36,120 +38,148 @@ serving_results = []
38
serving_column_mapping = {
39
"test_name": "Test name",
40
"gpu_type": "GPU",
39
- "completed": "# of req.",
41
+ # "completed": "# of req.",
42
"request_throughput": "Tput (req/s)",
41
- "input_throughput": "Input Tput (tok/s)",
42
- "output_throughput": "Output Tput (tok/s)",
43
+ # "input_throughput": "Input Tput (tok/s)",
44
+ # "output_throughput": "Output Tput (tok/s)",
45
"mean_ttft_ms": "Mean TTFT (ms)",
46
# do not say TTFT again to avoid the table getting too wide
47
"median_ttft_ms": "Median",
48
"p99_ttft_ms": "P99",
47
- "mean_tpot_ms": "Mean TPOT (ms)",
48
- "median_tpot_ms": "Median",
49
- "p99_tpot_ms": "P99",
49
+ # "mean_tpot_ms": "Mean TPOT (ms)",
50
+ # "median_tpot_ms": "Median",
51
+ # "p99_tpot_ms": "P99",
52
"mean_itl_ms": "Mean ITL (ms)",
53
"median_itl_ms": "Median",
54
"p99_itl_ms": "P99",
55
}
56
55
-for test_file in results_folder.glob("*.json"):
56
-
57
- with open(test_file, "r") as f:
58
- raw_result = json.loads(f.read())
59
-
60
- if "serving" in str(test_file):
61
- # this result is generated via `benchmark_serving.py`
62
-
63
- # attach the benchmarking command to raw_result
64
- with open(test_file.with_suffix(".commands"), "r") as f:
65
- command = json.loads(f.read())
66
- raw_result.update(command)
67
-
68
- # update the test name of this result
69
- raw_result.update({"test_name": test_file.stem})
70
-
71
- # add the result to raw_result
72
- serving_results.append(raw_result)
73
- continue
74
-
75
- elif "latency" in f.name:
76
- # this result is generated via `benchmark_latency.py`
77
-
78
- # attach the benchmarking command to raw_result
79
- with open(test_file.with_suffix(".commands"), "r") as f:
80
- command = json.loads(f.read())
81
- raw_result.update(command)
82
-
83
- # update the test name of this result
84
- raw_result.update({"test_name": test_file.stem})
85
-
86
- # get different percentiles
87
- for perc in [10, 25, 50, 75, 90]:
88
- raw_result.update(
89
- {f"P{perc}": raw_result["percentiles"][str(perc)]})
90
-
91
- # add the result to raw_result
92
- latency_results.append(raw_result)
93
- continue
94
-
95
- elif "throughput" in f.name:
96
- # this result is generated via `benchmark_throughput.py`
97
-
98
- # attach the benchmarking command to raw_result
99
- with open(test_file.with_suffix(".commands"), "r") as f:
100
- command = json.loads(f.read())
101
- raw_result.update(command)
102
-
103
- # update the test name of this result
104
- raw_result.update({"test_name": test_file.stem})
105
-
106
- # add the result to raw_result
107
- throughput_results.append(raw_result)
108
- continue
109
-
110
- print(f"Skipping {test_file}")
111
-
112
-latency_results = pd.DataFrame.from_dict(latency_results)
113
-serving_results = pd.DataFrame.from_dict(serving_results)
114
-throughput_results = pd.DataFrame.from_dict(throughput_results)
115
-
116
-# remapping the key, for visualization purpose
117
-if not latency_results.empty:
118
- latency_results = latency_results[list(
119
- latency_column_mapping.keys())].rename(columns=latency_column_mapping)
120
-if not serving_results.empty:
121
- serving_results = serving_results[list(
122
- serving_column_mapping.keys())].rename(columns=serving_column_mapping)
123
-if not throughput_results.empty:
124
- throughput_results = throughput_results[list(
125
- throughput_results_column_mapping.keys())].rename(
126
- columns=throughput_results_column_mapping)
127
-
128
-# get markdown tables
129
-latency_md_table = tabulate(latency_results,
130
- headers='keys',
131
- tablefmt='pipe',
132
- showindex=False)
133
-serving_md_table = tabulate(serving_results,
134
- headers='keys',
135
- tablefmt='pipe',
136
- showindex=False)
137
-throughput_md_table = tabulate(throughput_results,
138
- headers='keys',
139
- tablefmt='pipe',
140
- showindex=False)
141
-
142
-# document the result
143
-with open(results_folder / "benchmark_results.md", "w") as f:
57
+
58
+def read_markdown(file):
59
+ if os.path.exists(file):
60
+ with open(file, "r") as f:
61
+ return f.read() + "\n"
62
+ else:
63
+ return f"{file} not found.\n"
64
+
65
+
66
+def results_to_json(latency, throughput, serving):
67
+ return json.dumps({
68
+ 'latency': latency.to_dict(),
69
+ 'throughput': throughput.to_dict(),
70
+ 'serving': serving.to_dict()
71
+ })
72
+
73
+
74
+if __name__ == "__main__":
75
+
76
+ # collect results
77
+ for test_file in results_folder.glob("*.json"):
78
+
79
+ with open(test_file, "r") as f:
80
+ raw_result = json.loads(f.read())
81
+
82
+ if "serving" in str(test_file):
83
+ # this result is generated via `benchmark_serving.py`
84
+
85
+ # attach the benchmarking command to raw_result
86
+ with open(test_file.with_suffix(".commands"), "r") as f:
87
+ command = json.loads(f.read())
88
+ raw_result.update(command)
89
+
90
+ # update the test name of this result
91
+ raw_result.update({"test_name": test_file.stem})
92
+
93
+ # add the result to raw_result
94
+ serving_results.append(raw_result)
95
+ continue
96
+
97
+ elif "latency" in f.name:
98
+ # this result is generated via `benchmark_latency.py`
99
+
100
+ # attach the benchmarking command to raw_result
101
+ with open(test_file.with_suffix(".commands"), "r") as f:
102
+ command = json.loads(f.read())
103
+ raw_result.update(command)
104
+
105
+ # update the test name of this result
106
+ raw_result.update({"test_name": test_file.stem})
107
+
108
+ # get different percentiles
109
+ for perc in [10, 25, 50, 75, 90, 99]:
110
+ # Multiply 1000 to convert the time unit from s to ms
111
+ raw_result.update(
112
+ {f"P{perc}": 1000 * raw_result["percentiles"][str(perc)]})
113
+ raw_result["avg_latency"] = raw_result["avg_latency"] * 1000
114
+
115
+ # add the result to raw_result
116
+ latency_results.append(raw_result)
117
+ continue
118
+
119
+ elif "throughput" in f.name:
120
+ # this result is generated via `benchmark_throughput.py`
121
+
122
+ # attach the benchmarking command to raw_result
123
+ with open(test_file.with_suffix(".commands"), "r") as f:
124
+ command = json.loads(f.read())
125
+ raw_result.update(command)
126
+
127
+ # update the test name of this result
128
+ raw_result.update({"test_name": test_file.stem})
129
+
130
+ # add the result to raw_result
131
+ throughput_results.append(raw_result)
132
+ continue
133
+
134
+ print(f"Skipping {test_file}")
135
+
136
+ latency_results = pd.DataFrame.from_dict(latency_results)
137
+ serving_results = pd.DataFrame.from_dict(serving_results)
138
+ throughput_results = pd.DataFrame.from_dict(throughput_results)
139
+
140
+ raw_results_json = results_to_json(latency_results, throughput_results,
141
+ serving_results)
142
+
143
+ # remapping the key, for visualization purpose
144
if not latency_results.empty:
145
- f.write("## Latency tests\n")
146
- f.write(latency_md_table)
147
- f.write("\n")
148
- if not throughput_results.empty:
149
- f.write("## Throughput tests\n")
150
- f.write(throughput_md_table)
151
- f.write("\n")
145
+ latency_results = latency_results[list(
146
+ latency_column_mapping.keys())].rename(
147
+ columns=latency_column_mapping)
148
if not serving_results.empty:
153
- f.write("## Serving tests\n")
154
- f.write(serving_md_table)
155
- f.write("\n")
149
+ serving_results = serving_results[list(
150
+ serving_column_mapping.keys())].rename(
151
+ columns=serving_column_mapping)
152
+ if not throughput_results.empty:
153
+ throughput_results = throughput_results[list(
154
+ throughput_results_column_mapping.keys())].rename(
155
+ columns=throughput_results_column_mapping)
156
+
157
+ processed_results_json = results_to_json(latency_results,
158
+ throughput_results,
159
+ serving_results)
160
+
161
+ # get markdown tables
162
+ latency_md_table = tabulate(latency_results,
163
+ headers='keys',
164
+ tablefmt='pipe',
165
+ showindex=False)
166
+ serving_md_table = tabulate(serving_results,
167
+ headers='keys',
168
+ tablefmt='pipe',
169
+ showindex=False)
170
+ throughput_md_table = tabulate(throughput_results,
171
+ headers='keys',
172
+ tablefmt='pipe',
173
+ showindex=False)
174
+
175
+ # document the result
176
+ with open(results_folder / "benchmark_results.md", "w") as f:
177
+
178
+ results = read_markdown(
179
+ "../.buildkite/nightly-benchmarks/tests/descriptions.md")
180
+ results = results.format(
181
+ latency_tests_markdown_table=latency_md_table,
182
+ throughput_tests_markdown_table=throughput_md_table,
183
+ serving_tests_markdown_table=serving_md_table,
184
+ benchmarking_results_in_json_string=processed_results_json)
185
+ f.write(results)
.buildkite/nightly-benchmarks/tests/descriptions.md
new
+67
@@ -0,0 +1,67 @@
1
+
2
+## Latency tests
3
+
4
+This test suite aims to test vllm's end-to-end latency under a controlled setup.
5
+
6
+- Input length: 32 tokens.
7
+- Output length: 128 tokens.
8
+- Batch size: fixed (8).
9
+- Models: llama-3 8B, llama-3 70B, mixtral 8x7B.
10
+- Evaluation metrics: end-to-end latency (mean, median, p99).
11
+
12
+### Latency benchmarking results
13
+
14
+{latency_tests_markdown_table}
15
+
16
+## Throughput tests
17
+
18
+This test suite aims to test vllm's throughput.
19
+
20
+- Input length: randomly sample 200 prompts from ShareGPT dataset (with fixed random seed).
21
+- Output length: the corresponding output length of these 200 prompts.
22
+- Batch size: dynamically determined by vllm to achieve maximum throughput.
23
+- Models: llama-3 8B, llama-3 70B, mixtral 8x7B.
24
+- Evaluation metrics: throughput.
25
+
26
+### Throughput benchmarking results
27
+
28
+{throughput_tests_markdown_table}
29
+
30
+## Serving tests
31
+
32
+This test suite aims to test vllm's real serving metrics.
33
+
34
+- Input length: randomly sample 200 prompts from ShareGPT dataset (with fixed random seed).
35
+- Output length: the corresponding output length of these 200 prompts.
36
+- Batch size: dynamically determined by vllm and the arrival pattern of the requests.
37
+- **Average QPS (query per second)**: 1, 4, 16 and inf. QPS = inf means all requests come at once. For other QPS values, the arrival time of each query is determined using a random Poisson process (with fixed random seed).
38
+- Models: llama-3 8B, llama-3 70B, mixtral 8x7B.
39
+- Evaluation metrics: throughput, TTFT (time to the first token, with mean, median and p99), ITL (inter-token latency, with mean, median and p99).
40
+
41
+### Serving benchmarking results
42
+
43
+{serving_tests_markdown_table}
44
+
45
+## json version of the benchmarking tables
46
+
47
+This section contains the data of the markdown tables above in JSON format.
48
+You can load the benchmarking tables into pandas dataframes as follows:
49
+
50
+```python
51
+import json
52
+import pandas as pd
53
+
54
+benchmarking_results_json = """The json string"""
55
+benchmarking_results = json.loads(benchmarking_results_json)
56
+latency_results = pd.DataFrame.from_dict(benchmarking_results["latency"])
57
+throughput_results = pd.DataFrame.from_dict(benchmarking_results["throughput"])
58
+serving_results = pd.DataFrame.from_dict(benchmarking_results["serving"])
59
+```
60
+
61
+The json string for all benchmarking tables:
62
+```json
63
+{benchmarking_results_in_json_string}
64
+```
65
+
66
+You can also check the raw experiment data in the Artifact tab of the Buildkite page.
67
+
.buildkite/nightly-benchmarks/tests/latency-tests.json
renamed
+1
-1
@@ -29,4 +29,4 @@
29
"num-iters": 15
30
}
31
}
32
-]
32
+]
\ No newline at end of file
.buildkite/nightly-benchmarks/tests/serving-tests.json
renamed
+1
-1
@@ -56,4 +56,4 @@
56
"num_prompts": 200
57
}
58
}
59
-]
59
+]
\ No newline at end of file
.buildkite/nightly-benchmarks/tests/throughput-tests.json
renamed
+1
-1
@@ -32,4 +32,4 @@
32
"backend": "vllm"
33
}
34
}
35
-]
35
+]
\ No newline at end of file
benchmarks/benchmark_latency.py
+1
-1
@@ -98,7 +98,7 @@ def main(args: argparse.Namespace):
98
for _ in tqdm(range(args.num_iters), desc="Profiling iterations"):
99
latencies.append(run_to_completion(profile_dir=None))
100
latencies = np.array(latencies)
101
- percentages = [10, 25, 50, 75, 90]
101
+ percentages = [10, 25, 50, 75, 90, 99]
102
percentiles = np.percentile(latencies, percentages)
103
print(f'Avg latency: {np.mean(latencies)} seconds')
104
for percentage, percentile in zip(percentages, percentiles):