@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5dd80d3777
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Fix latency benchmark script (#2035)

Woosuk Kwon committed Dec 11, 2023 at 11:19 UTC 5dd80d3777306e489cb17cb4df1ea4c902d32645
1 file changed +1 -1
benchmarks/benchmark_latency.py
+1 -1
@@ -72,7 +72,7 @@ def main(args: argparse.Namespace):
72 # Benchmark.
73 latencies = []
74 for _ in tqdm(range(args.num_iters), desc="Profiling iterations"):
75 - latencies.append(run_to_completion(profile=False))
75 + latencies.append(run_to_completion(profile_dir=None))
76 print(f'Avg latency: {np.mean(latencies)} seconds')
77
78