@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9e4e6fe207
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[CI] the readability of benchmarking and prepare for dashboard (#5571)

[CI] Improve the readability of performance benchmarking results and prepare for upcoming performance dashboard (#5571)

Kuntai Du committed Jun 17, 2024 at 11:41 UTC 9e4e6fe2073ff5e4a747d5ce2a08d321268b7254
8 files changed +232 -130
.buildkite/nightly-benchmarks/README.md
+13 -8
@@ -13,9 +13,17 @@ This benchmark will be *triggered* upon:
13
14 **Benchmarking Duration**: about 1hr.
15
16 -## Configuring the workload for the quick benchmark
16 +**For benchmarking developers**: please try your best to constraint the duration of benchmarking to less than 1.5 hr so that it won't take forever to run.
17
18 -The workload of the quick benchmark contains two parts: latency tests in `latency-tests.json`, throughput tests in `throughput-tests.json` and serving tests in `serving-tests.json`.
18 +
19 +## Configuring the workload
20 +
21 +The benchmarking workload contains three parts:
22 +- Latency tests in `latency-tests.json`.
23 +- Throughput tests in `throughput-tests.json`.
24 +- Serving tests in `serving-tests.json`.
25 +
26 +See [descriptions.md](tests/descriptions.md) for detailed descriptions.
27
28 ### Latency test
29
@@ -23,7 +31,6 @@ Here is an example of one test inside `latency-tests.json`:
31
32 ```json
33 [
26 - ...
34 {
35 "test_name": "latency_llama8B_tp1",
36 "parameters": {
@@ -34,7 +41,6 @@ Here is an example of one test inside `latency-tests.json`:
41 "num_iters": 15
42 }
43 },
37 - ...
44 ]
45 ```
46
@@ -57,7 +63,6 @@ We test the throughput by using `benchmark_serving.py` with request rate = inf t
63
64 ```
65 [
60 - ...
66 {
67 "test_name": "serving_llama8B_tp1_sharegpt",
68 "qps_list": [1, 4, 16, "inf"],
@@ -77,7 +82,6 @@ We test the throughput by using `benchmark_serving.py` with request rate = inf t
82 "num_prompts": 200
83 }
84 },
80 - ...
85 ]
86 ```
87
@@ -92,7 +96,8 @@ The number of this test is less stable compared to the delay and latency benchma
96 WARNING: The benchmarking script will save json results by itself, so please do not configure `--save-results` or other results-saving-related parameters in `serving-tests.json`.
97
98 ## Visualizing the results
95 -The `convert-results-json-to-markdown.py` helps you put the benchmarking results inside a markdown table.
99 +The `convert-results-json-to-markdown.py` helps you put the benchmarking results inside a markdown table, by formatting [descriptions.md](tests/descriptions.md) with real benchmarking results.
100 You can find the result presented as a table inside the `buildkite/performance-benchmark` job page.
101 If you do not see the table, please wait till the benchmark finish running.
98 -The JSON file is also attached within each buildkite job for further analysis.
\ No newline at end of file
102 +The json version of the table (together with the json version of the benchmark) will be also attached to the markdown file.
103 +The raw benchmarking results (in the format of json files) are in the `Artifacts` tab of the benchmarking.
.buildkite/nightly-benchmarks/run-benchmarks-suite.sh
+3 -3
@@ -343,9 +343,9 @@ main() {
343 QUICK_BENCHMARK_ROOT=../.buildkite/nightly-benchmarks/
344
345 # benchmarking
346 - run_serving_tests $QUICK_BENCHMARK_ROOT/serving-tests.json
347 - run_latency_tests $QUICK_BENCHMARK_ROOT/latency-tests.json
348 - run_throughput_tests $QUICK_BENCHMARK_ROOT/throughput-tests.json
346 + run_serving_tests $QUICK_BENCHMARK_ROOT/tests/serving-tests.json
347 + run_latency_tests $QUICK_BENCHMARK_ROOT/tests/latency-tests.json
348 + run_throughput_tests $QUICK_BENCHMARK_ROOT/tests/throughput-tests.json
349
350
351 # postprocess benchmarking results
.buildkite/nightly-benchmarks/scripts/convert-results-json-to-markdown.py
+145 -115
@@ -1,4 +1,5 @@
1 import json
2 +import os
3 from pathlib import Path
4
5 import pandas as pd
@@ -11,12 +12,13 @@ latency_results = []
12 latency_column_mapping = {
13 "test_name": "Test name",
14 "gpu_type": "GPU",
14 - "avg_latency": "Average latency (s)",
15 - "P10": "P10 (s)",
16 - "P25": "P25 (s)",
17 - "P50": "P50 (s)",
18 - "P75": "P75 (s)",
19 - "P90": "P90 (s)",
15 + "avg_latency": "Mean latency (ms)",
16 + # "P10": "P10 (s)",
17 + # "P25": "P25 (s)",
18 + "P50": "Median",
19 + # "P75": "P75 (s)",
20 + # "P90": "P90 (s)",
21 + "P99": "P99",
22 }
23
24 # thoughput tests and the keys that will be printed into markdown
@@ -24,11 +26,11 @@ throughput_results = []
26 throughput_results_column_mapping = {
27 "test_name": "Test name",
28 "gpu_type": "GPU",
27 - "num_requests": "# of req.",
28 - "total_num_tokens": "Total # of tokens",
29 - "elapsed_time": "Elapsed time (s)",
29 + # "num_requests": "# of req.",
30 + # "total_num_tokens": "Total # of tokens",
31 + # "elapsed_time": "Elapsed time (s)",
32 "requests_per_second": "Tput (req/s)",
31 - "tokens_per_second": "Tput (tok/s)",
33 + # "tokens_per_second": "Tput (tok/s)",
34 }
35
36 # serving results and the keys that will be printed into markdown
@@ -36,120 +38,148 @@ serving_results = []
38 serving_column_mapping = {
39 "test_name": "Test name",
40 "gpu_type": "GPU",
39 - "completed": "# of req.",
41 + # "completed": "# of req.",
42 "request_throughput": "Tput (req/s)",
41 - "input_throughput": "Input Tput (tok/s)",
42 - "output_throughput": "Output Tput (tok/s)",
43 + # "input_throughput": "Input Tput (tok/s)",
44 + # "output_throughput": "Output Tput (tok/s)",
45 "mean_ttft_ms": "Mean TTFT (ms)",
46 # do not say TTFT again to avoid the table getting too wide
47 "median_ttft_ms": "Median",
48 "p99_ttft_ms": "P99",
47 - "mean_tpot_ms": "Mean TPOT (ms)",
48 - "median_tpot_ms": "Median",
49 - "p99_tpot_ms": "P99",
49 + # "mean_tpot_ms": "Mean TPOT (ms)",
50 + # "median_tpot_ms": "Median",
51 + # "p99_tpot_ms": "P99",
52 "mean_itl_ms": "Mean ITL (ms)",
53 "median_itl_ms": "Median",
54 "p99_itl_ms": "P99",
55 }
56
55 -for test_file in results_folder.glob("*.json"):
56 -
57 - with open(test_file, "r") as f:
58 - raw_result = json.loads(f.read())
59 -
60 - if "serving" in str(test_file):
61 - # this result is generated via `benchmark_serving.py`
62 -
63 - # attach the benchmarking command to raw_result
64 - with open(test_file.with_suffix(".commands"), "r") as f:
65 - command = json.loads(f.read())
66 - raw_result.update(command)
67 -
68 - # update the test name of this result
69 - raw_result.update({"test_name": test_file.stem})
70 -
71 - # add the result to raw_result
72 - serving_results.append(raw_result)
73 - continue
74 -
75 - elif "latency" in f.name:
76 - # this result is generated via `benchmark_latency.py`
77 -
78 - # attach the benchmarking command to raw_result
79 - with open(test_file.with_suffix(".commands"), "r") as f:
80 - command = json.loads(f.read())
81 - raw_result.update(command)
82 -
83 - # update the test name of this result
84 - raw_result.update({"test_name": test_file.stem})
85 -
86 - # get different percentiles
87 - for perc in [10, 25, 50, 75, 90]:
88 - raw_result.update(
89 - {f"P{perc}": raw_result["percentiles"][str(perc)]})
90 -
91 - # add the result to raw_result
92 - latency_results.append(raw_result)
93 - continue
94 -
95 - elif "throughput" in f.name:
96 - # this result is generated via `benchmark_throughput.py`
97 -
98 - # attach the benchmarking command to raw_result
99 - with open(test_file.with_suffix(".commands"), "r") as f:
100 - command = json.loads(f.read())
101 - raw_result.update(command)
102 -
103 - # update the test name of this result
104 - raw_result.update({"test_name": test_file.stem})
105 -
106 - # add the result to raw_result
107 - throughput_results.append(raw_result)
108 - continue
109 -
110 - print(f"Skipping {test_file}")
111 -
112 -latency_results = pd.DataFrame.from_dict(latency_results)
113 -serving_results = pd.DataFrame.from_dict(serving_results)
114 -throughput_results = pd.DataFrame.from_dict(throughput_results)
115 -
116 -# remapping the key, for visualization purpose
117 -if not latency_results.empty:
118 - latency_results = latency_results[list(
119 - latency_column_mapping.keys())].rename(columns=latency_column_mapping)
120 -if not serving_results.empty:
121 - serving_results = serving_results[list(
122 - serving_column_mapping.keys())].rename(columns=serving_column_mapping)
123 -if not throughput_results.empty:
124 - throughput_results = throughput_results[list(
125 - throughput_results_column_mapping.keys())].rename(
126 - columns=throughput_results_column_mapping)
127 -
128 -# get markdown tables
129 -latency_md_table = tabulate(latency_results,
130 - headers='keys',
131 - tablefmt='pipe',
132 - showindex=False)
133 -serving_md_table = tabulate(serving_results,
134 - headers='keys',
135 - tablefmt='pipe',
136 - showindex=False)
137 -throughput_md_table = tabulate(throughput_results,
138 - headers='keys',
139 - tablefmt='pipe',
140 - showindex=False)
141 -
142 -# document the result
143 -with open(results_folder / "benchmark_results.md", "w") as f:
57 +
58 +def read_markdown(file):
59 + if os.path.exists(file):
60 + with open(file, "r") as f:
61 + return f.read() + "\n"
62 + else:
63 + return f"{file} not found.\n"
64 +
65 +
66 +def results_to_json(latency, throughput, serving):
67 + return json.dumps({
68 + 'latency': latency.to_dict(),
69 + 'throughput': throughput.to_dict(),
70 + 'serving': serving.to_dict()
71 + })
72 +
73 +
74 +if __name__ == "__main__":
75 +
76 + # collect results
77 + for test_file in results_folder.glob("*.json"):
78 +
79 + with open(test_file, "r") as f:
80 + raw_result = json.loads(f.read())
81 +
82 + if "serving" in str(test_file):
83 + # this result is generated via `benchmark_serving.py`
84 +
85 + # attach the benchmarking command to raw_result
86 + with open(test_file.with_suffix(".commands"), "r") as f:
87 + command = json.loads(f.read())
88 + raw_result.update(command)
89 +
90 + # update the test name of this result
91 + raw_result.update({"test_name": test_file.stem})
92 +
93 + # add the result to raw_result
94 + serving_results.append(raw_result)
95 + continue
96 +
97 + elif "latency" in f.name:
98 + # this result is generated via `benchmark_latency.py`
99 +
100 + # attach the benchmarking command to raw_result
101 + with open(test_file.with_suffix(".commands"), "r") as f:
102 + command = json.loads(f.read())
103 + raw_result.update(command)
104 +
105 + # update the test name of this result
106 + raw_result.update({"test_name": test_file.stem})
107 +
108 + # get different percentiles
109 + for perc in [10, 25, 50, 75, 90, 99]:
110 + # Multiply 1000 to convert the time unit from s to ms
111 + raw_result.update(
112 + {f"P{perc}": 1000 * raw_result["percentiles"][str(perc)]})
113 + raw_result["avg_latency"] = raw_result["avg_latency"] * 1000
114 +
115 + # add the result to raw_result
116 + latency_results.append(raw_result)
117 + continue
118 +
119 + elif "throughput" in f.name:
120 + # this result is generated via `benchmark_throughput.py`
121 +
122 + # attach the benchmarking command to raw_result
123 + with open(test_file.with_suffix(".commands"), "r") as f:
124 + command = json.loads(f.read())
125 + raw_result.update(command)
126 +
127 + # update the test name of this result
128 + raw_result.update({"test_name": test_file.stem})
129 +
130 + # add the result to raw_result
131 + throughput_results.append(raw_result)
132 + continue
133 +
134 + print(f"Skipping {test_file}")
135 +
136 + latency_results = pd.DataFrame.from_dict(latency_results)
137 + serving_results = pd.DataFrame.from_dict(serving_results)
138 + throughput_results = pd.DataFrame.from_dict(throughput_results)
139 +
140 + raw_results_json = results_to_json(latency_results, throughput_results,
141 + serving_results)
142 +
143 + # remapping the key, for visualization purpose
144 if not latency_results.empty:
145 - f.write("## Latency tests\n")
146 - f.write(latency_md_table)
147 - f.write("\n")
148 - if not throughput_results.empty:
149 - f.write("## Throughput tests\n")
150 - f.write(throughput_md_table)
151 - f.write("\n")
145 + latency_results = latency_results[list(
146 + latency_column_mapping.keys())].rename(
147 + columns=latency_column_mapping)
148 if not serving_results.empty:
153 - f.write("## Serving tests\n")
154 - f.write(serving_md_table)
155 - f.write("\n")
149 + serving_results = serving_results[list(
150 + serving_column_mapping.keys())].rename(
151 + columns=serving_column_mapping)
152 + if not throughput_results.empty:
153 + throughput_results = throughput_results[list(
154 + throughput_results_column_mapping.keys())].rename(
155 + columns=throughput_results_column_mapping)
156 +
157 + processed_results_json = results_to_json(latency_results,
158 + throughput_results,
159 + serving_results)
160 +
161 + # get markdown tables
162 + latency_md_table = tabulate(latency_results,
163 + headers='keys',
164 + tablefmt='pipe',
165 + showindex=False)
166 + serving_md_table = tabulate(serving_results,
167 + headers='keys',
168 + tablefmt='pipe',
169 + showindex=False)
170 + throughput_md_table = tabulate(throughput_results,
171 + headers='keys',
172 + tablefmt='pipe',
173 + showindex=False)
174 +
175 + # document the result
176 + with open(results_folder / "benchmark_results.md", "w") as f:
177 +
178 + results = read_markdown(
179 + "../.buildkite/nightly-benchmarks/tests/descriptions.md")
180 + results = results.format(
181 + latency_tests_markdown_table=latency_md_table,
182 + throughput_tests_markdown_table=throughput_md_table,
183 + serving_tests_markdown_table=serving_md_table,
184 + benchmarking_results_in_json_string=processed_results_json)
185 + f.write(results)
.buildkite/nightly-benchmarks/tests/descriptions.md new
+67
@@ -0,0 +1,67 @@
1 +
2 +## Latency tests
3 +
4 +This test suite aims to test vllm's end-to-end latency under a controlled setup.
5 +
6 +- Input length: 32 tokens.
7 +- Output length: 128 tokens.
8 +- Batch size: fixed (8).
9 +- Models: llama-3 8B, llama-3 70B, mixtral 8x7B.
10 +- Evaluation metrics: end-to-end latency (mean, median, p99).
11 +
12 +### Latency benchmarking results
13 +
14 +{latency_tests_markdown_table}
15 +
16 +## Throughput tests
17 +
18 +This test suite aims to test vllm's throughput.
19 +
20 +- Input length: randomly sample 200 prompts from ShareGPT dataset (with fixed random seed).
21 +- Output length: the corresponding output length of these 200 prompts.
22 +- Batch size: dynamically determined by vllm to achieve maximum throughput.
23 +- Models: llama-3 8B, llama-3 70B, mixtral 8x7B.
24 +- Evaluation metrics: throughput.
25 +
26 +### Throughput benchmarking results
27 +
28 +{throughput_tests_markdown_table}
29 +
30 +## Serving tests
31 +
32 +This test suite aims to test vllm's real serving metrics.
33 +
34 +- Input length: randomly sample 200 prompts from ShareGPT dataset (with fixed random seed).
35 +- Output length: the corresponding output length of these 200 prompts.
36 +- Batch size: dynamically determined by vllm and the arrival pattern of the requests.
37 +- **Average QPS (query per second)**: 1, 4, 16 and inf. QPS = inf means all requests come at once. For other QPS values, the arrival time of each query is determined using a random Poisson process (with fixed random seed).
38 +- Models: llama-3 8B, llama-3 70B, mixtral 8x7B.
39 +- Evaluation metrics: throughput, TTFT (time to the first token, with mean, median and p99), ITL (inter-token latency, with mean, median and p99).
40 +
41 +### Serving benchmarking results
42 +
43 +{serving_tests_markdown_table}
44 +
45 +## json version of the benchmarking tables
46 +
47 +This section contains the data of the markdown tables above in JSON format.
48 +You can load the benchmarking tables into pandas dataframes as follows:
49 +
50 +```python
51 +import json
52 +import pandas as pd
53 +
54 +benchmarking_results_json = """The json string"""
55 +benchmarking_results = json.loads(benchmarking_results_json)
56 +latency_results = pd.DataFrame.from_dict(benchmarking_results["latency"])
57 +throughput_results = pd.DataFrame.from_dict(benchmarking_results["throughput"])
58 +serving_results = pd.DataFrame.from_dict(benchmarking_results["serving"])
59 +```
60 +
61 +The json string for all benchmarking tables:
62 +```json
63 +{benchmarking_results_in_json_string}
64 +```
65 +
66 +You can also check the raw experiment data in the Artifact tab of the Buildkite page.
67 +
.buildkite/nightly-benchmarks/tests/latency-tests.json renamed
+1 -1
@@ -29,4 +29,4 @@
29 "num-iters": 15
30 }
31 }
32 -]
32 +]
\ No newline at end of file
.buildkite/nightly-benchmarks/tests/serving-tests.json renamed
+1 -1
@@ -56,4 +56,4 @@
56 "num_prompts": 200
57 }
58 }
59 -]
59 +]
\ No newline at end of file
.buildkite/nightly-benchmarks/tests/throughput-tests.json renamed
+1 -1
@@ -32,4 +32,4 @@
32 "backend": "vllm"
33 }
34 }
35 -]
35 +]
\ No newline at end of file
benchmarks/benchmark_latency.py
+1 -1
@@ -98,7 +98,7 @@ def main(args: argparse.Namespace):
98 for _ in tqdm(range(args.num_iters), desc="Profiling iterations"):
99 latencies.append(run_to_completion(profile_dir=None))
100 latencies = np.array(latencies)
101 - percentages = [10, 25, 50, 75, 90]
101 + percentages = [10, 25, 50, 75, 90, 99]
102 percentiles = np.percentile(latencies, percentages)
103 print(f'Avg latency: {np.mean(latencies)} seconds')
104 for percentage, percentile in zip(percentages, percentiles):