@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f09edd8a25
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

Add JSON output support for benchmark_latency and benchmark_throughput (#4848)

Simon Mo committed May 16, 2024 at 10:02 UTC f09edd8a25d54c48eb804abe391e98d0b85b9ea2
3 files changed +39 -5
.buildkite/run-benchmarks.sh
+4 -3
@@ -9,10 +9,10 @@ cd "$(dirname "${BASH_SOURCE[0]}")/.."
9 (which wget && which curl) || (apt-get update && apt-get install -y wget curl)
10
11 # run python-based benchmarks and upload the result to buildkite
12 -python3 benchmarks/benchmark_latency.py 2>&1 | tee benchmark_latency.txt
12 +python3 benchmarks/benchmark_latency.py --output-json latency_results.json 2>&1 | tee benchmark_latency.txt
13 bench_latency_exit_code=$?
14
15 -python3 benchmarks/benchmark_throughput.py --input-len 256 --output-len 256 2>&1 | tee benchmark_throughput.txt
15 +python3 benchmarks/benchmark_throughput.py --input-len 256 --output-len 256 --output-json throughput_results.json 2>&1 | tee benchmark_throughput.txt
16 bench_throughput_exit_code=$?
17
18 # run server-based benchmarks and upload the result to buildkite
@@ -74,4 +74,5 @@ if [ $bench_serving_exit_code -ne 0 ]; then
74 exit $bench_serving_exit_code
75 fi
76
77 -/workspace/buildkite-agent artifact upload openai-*.json
77 +rm ShareGPT_V3_unfiltered_cleaned_split.json
78 +/workspace/buildkite-agent artifact upload "*.json"
benchmarks/benchmark_latency.py
+18 -2
@@ -1,5 +1,6 @@
1 """Benchmark the latency of processing a single batch of requests."""
2 import argparse
3 +import json
4 import time
5 from pathlib import Path
6 from typing import Optional
@@ -96,6 +97,16 @@ def main(args: argparse.Namespace):
97 for percentage, percentile in zip(percentages, percentiles):
98 print(f'{percentage}% percentile latency: {percentile} seconds')
99
100 + # Output JSON results if specified
101 + if args.output_json:
102 + results = {
103 + "avg_latency": np.mean(latencies),
104 + "latencies": latencies.tolist(),
105 + "percentiles": dict(zip(percentages, percentiles.tolist())),
106 + }
107 + with open(args.output_json, "w") as f:
108 + json.dump(results, f, indent=4)
109 +
110
111 if __name__ == '__main__':
112 parser = argparse.ArgumentParser(
@@ -149,8 +160,8 @@ if __name__ == '__main__':
160 help=
161 'Data type for kv cache storage. If "auto", will use model data type. '
162 'FP8_E5M2 (without scaling) is only supported on cuda version greater '
152 - 'than 11.8. On ROCm (AMD GPU), FP8_E4M3 is instead supported for '
153 - 'common inference criteria.')
163 + 'than 11.8. On ROCm (AMD GPU), FP8_E4M3 is '
164 + 'instead supported for common inference criteria.')
165 parser.add_argument(
166 '--quantization-param-path',
167 type=str,
@@ -197,5 +208,10 @@ if __name__ == '__main__':
208 default=None,
209 help='directory to download and load the weights, '
210 'default to the default cache dir of huggingface')
211 + parser.add_argument(
212 + '--output-json',
213 + type=str,
214 + default=None,
215 + help='Path to save the latency results in JSON format.')
216 args = parser.parse_args()
217 main(args)
benchmarks/benchmark_throughput.py
+17
@@ -242,6 +242,18 @@ def main(args: argparse.Namespace):
242 print(f"Throughput: {len(requests) / elapsed_time:.2f} requests/s, "
243 f"{total_num_tokens / elapsed_time:.2f} tokens/s")
244
245 + # Output JSON results if specified
246 + if args.output_json:
247 + results = {
248 + "elapsed_time": elapsed_time,
249 + "num_requests": len(requests),
250 + "total_num_tokens": total_num_tokens,
251 + "requests_per_second": len(requests) / elapsed_time,
252 + "tokens_per_second": total_num_tokens / elapsed_time,
253 + }
254 + with open(args.output_json, "w") as f:
255 + json.dump(results, f, indent=4)
256 +
257
258 if __name__ == "__main__":
259 parser = argparse.ArgumentParser(description="Benchmark the throughput.")
@@ -353,6 +365,11 @@ if __name__ == "__main__":
365 default=None,
366 help='directory to download and load the weights, '
367 'default to the default cache dir of huggingface')
368 + parser.add_argument(
369 + '--output-json',
370 + type=str,
371 + default=None,
372 + help='Path to save the throughput results in JSON format.')
373 args = parser.parse_args()
374 if args.tokenizer is None:
375 args.tokenizer = args.model