A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
Add JSON output support for benchmark_latency and benchmark_throughput (#4848)
Simon Mo committed
May 16, 2024 at 10:02 UTC
f09edd8a25d54c48eb804abe391e98d0b85b9ea2
3 files changed
+39
-5
.buildkite/run-benchmarks.sh
+4
-3
@@ -9,10 +9,10 @@ cd "$(dirname "${BASH_SOURCE[0]}")/.."
9
(which wget && which curl) || (apt-get update && apt-get install -y wget curl)
10
11
# run python-based benchmarks and upload the result to buildkite
12
-python3 benchmarks/benchmark_latency.py 2>&1 | tee benchmark_latency.txt
12
+python3 benchmarks/benchmark_latency.py --output-json latency_results.json 2>&1 | tee benchmark_latency.txt
13
bench_latency_exit_code=$?
14
15
-python3 benchmarks/benchmark_throughput.py --input-len 256 --output-len 256 2>&1 | tee benchmark_throughput.txt
15
+python3 benchmarks/benchmark_throughput.py --input-len 256 --output-len 256 --output-json throughput_results.json 2>&1 | tee benchmark_throughput.txt
16
bench_throughput_exit_code=$?
17
18
# run server-based benchmarks and upload the result to buildkite
@@ -74,4 +74,5 @@ if [ $bench_serving_exit_code -ne 0 ]; then
74
exit $bench_serving_exit_code
75
fi
76
77
-/workspace/buildkite-agent artifact upload openai-*.json
77
+rm ShareGPT_V3_unfiltered_cleaned_split.json
78
+/workspace/buildkite-agent artifact upload "*.json"
benchmarks/benchmark_latency.py
+18
-2
@@ -1,5 +1,6 @@
1
"""Benchmark the latency of processing a single batch of requests."""
2
import argparse
3
+import json
4
import time
5
from pathlib import Path
6
from typing import Optional
@@ -96,6 +97,16 @@ def main(args: argparse.Namespace):
97
for percentage, percentile in zip(percentages, percentiles):
98
print(f'{percentage}% percentile latency: {percentile} seconds')
99
100
+ # Output JSON results if specified
101
+ if args.output_json:
102
+ results = {
103
+ "avg_latency": np.mean(latencies),
104
+ "latencies": latencies.tolist(),
105
+ "percentiles": dict(zip(percentages, percentiles.tolist())),
106
+ }
107
+ with open(args.output_json, "w") as f:
108
+ json.dump(results, f, indent=4)
109
+
110
111
if __name__ == '__main__':
112
parser = argparse.ArgumentParser(
@@ -149,8 +160,8 @@ if __name__ == '__main__':
160
help=
161
'Data type for kv cache storage. If "auto", will use model data type. '
162
'FP8_E5M2 (without scaling) is only supported on cuda version greater '
152
- 'than 11.8. On ROCm (AMD GPU), FP8_E4M3 is instead supported for '
153
- 'common inference criteria.')
163
+ 'than 11.8. On ROCm (AMD GPU), FP8_E4M3 is '
164
+ 'instead supported for common inference criteria.')
165
parser.add_argument(
166
'--quantization-param-path',
167
type=str,
@@ -197,5 +208,10 @@ if __name__ == '__main__':
208
default=None,
209
help='directory to download and load the weights, '
210
'default to the default cache dir of huggingface')
211
+ parser.add_argument(
212
+ '--output-json',
213
+ type=str,
214
+ default=None,
215
+ help='Path to save the latency results in JSON format.')
216
args = parser.parse_args()
217
main(args)
benchmarks/benchmark_throughput.py
+17
@@ -242,6 +242,18 @@ def main(args: argparse.Namespace):
242
print(f"Throughput: {len(requests) / elapsed_time:.2f} requests/s, "
243
f"{total_num_tokens / elapsed_time:.2f} tokens/s")
244
245
+ # Output JSON results if specified
246
+ if args.output_json:
247
+ results = {
248
+ "elapsed_time": elapsed_time,
249
+ "num_requests": len(requests),
250
+ "total_num_tokens": total_num_tokens,
251
+ "requests_per_second": len(requests) / elapsed_time,
252
+ "tokens_per_second": total_num_tokens / elapsed_time,
253
+ }
254
+ with open(args.output_json, "w") as f:
255
+ json.dump(results, f, indent=4)
256
+
257
258
if __name__ == "__main__":
259
parser = argparse.ArgumentParser(description="Benchmark the throughput.")
@@ -353,6 +365,11 @@ if __name__ == "__main__":
365
default=None,
366
help='directory to download and load the weights, '
367
'default to the default cache dir of huggingface')
368
+ parser.add_argument(
369
+ '--output-json',
370
+ type=str,
371
+ default=None,
372
+ help='Path to save the throughput results in JSON format.')
373
args = parser.parse_args()
374
if args.tokenizer is None:
375
args.tokenizer = args.model