A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[CI/Benchmark] add more iteration and use median for robust latency benchmark (#3889)
youkaichao committed
Apr 6, 2024 at 14:32 UTC
e4be7d70bbac47b2c309caa20e68fc43e05e0550
1 file changed
+12
-2
benchmarks/benchmark_latency.py
+12
-2
@@ -68,7 +68,8 @@ def main(args: argparse.Namespace):
68
return latency
69
70
print("Warming up...")
71
- run_to_completion(profile_dir=None)
71
+ for _ in tqdm(range(args.num_iters_warmup), desc="Warmup iterations"):
72
+ run_to_completion(profile_dir=None)
73
74
if args.profile:
75
profile_dir = args.profile_result_dir
@@ -84,7 +85,12 @@ def main(args: argparse.Namespace):
85
latencies = []
86
for _ in tqdm(range(args.num_iters), desc="Profiling iterations"):
87
latencies.append(run_to_completion(profile_dir=None))
88
+ latencies = np.array(latencies)
89
+ percentages = [10, 25, 50, 75, 90]
90
+ percentiles = np.percentile(latencies, percentages)
91
print(f'Avg latency: {np.mean(latencies)} seconds')
92
+ for percentage, percentile in zip(percentages, percentiles):
93
+ print(f'{percentage}% percentile latency: {percentile} seconds')
94
95
96
if __name__ == '__main__':
@@ -106,9 +112,13 @@ if __name__ == '__main__':
112
default=1,
113
help='Number of generated sequences per prompt.')
114
parser.add_argument('--use-beam-search', action='store_true')
115
+ parser.add_argument('--num-iters-warmup',
116
+ type=int,
117
+ default=10,
118
+ help='Number of iterations to run for warmup.')
119
parser.add_argument('--num-iters',
120
type=int,
111
- default=3,
121
+ default=30,
122
help='Number of iterations to run.')
123
parser.add_argument('--trust-remote-code',
124
action='store_true',