@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e4be7d70bb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[CI/Benchmark] add more iteration and use median for robust latency benchmark (#3889)

youkaichao committed Apr 6, 2024 at 14:32 UTC e4be7d70bbac47b2c309caa20e68fc43e05e0550
1 file changed +12 -2
benchmarks/benchmark_latency.py
+12 -2
@@ -68,7 +68,8 @@ def main(args: argparse.Namespace):
68 return latency
69
70 print("Warming up...")
71 - run_to_completion(profile_dir=None)
71 + for _ in tqdm(range(args.num_iters_warmup), desc="Warmup iterations"):
72 + run_to_completion(profile_dir=None)
73
74 if args.profile:
75 profile_dir = args.profile_result_dir
@@ -84,7 +85,12 @@ def main(args: argparse.Namespace):
85 latencies = []
86 for _ in tqdm(range(args.num_iters), desc="Profiling iterations"):
87 latencies.append(run_to_completion(profile_dir=None))
88 + latencies = np.array(latencies)
89 + percentages = [10, 25, 50, 75, 90]
90 + percentiles = np.percentile(latencies, percentages)
91 print(f'Avg latency: {np.mean(latencies)} seconds')
92 + for percentage, percentile in zip(percentages, percentiles):
93 + print(f'{percentage}% percentile latency: {percentile} seconds')
94
95
96 if __name__ == '__main__':
@@ -106,9 +112,13 @@ if __name__ == '__main__':
112 default=1,
113 help='Number of generated sequences per prompt.')
114 parser.add_argument('--use-beam-search', action='store_true')
115 + parser.add_argument('--num-iters-warmup',
116 + type=int,
117 + default=10,
118 + help='Number of iterations to run for warmup.')
119 parser.add_argument('--num-iters',
120 type=int,
111 - default=3,
121 + default=30,
122 help='Number of iterations to run.')
123 parser.add_argument('--trust-remote-code',
124 action='store_true',