A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Fix latency benchmark script (#2035)
Woosuk Kwon committed
Dec 11, 2023 at 11:19 UTC
5dd80d3777306e489cb17cb4df1ea4c902d32645
1 file changed
+1
-1
benchmarks/benchmark_latency.py
+1
-1
@@ -72,7 +72,7 @@ def main(args: argparse.Namespace):
72
# Benchmark.
73
latencies = []
74
for _ in tqdm(range(args.num_iters), desc="Profiling iterations"):
75
- latencies.append(run_to_completion(profile=False))
75
+ latencies.append(run_to_completion(profile_dir=None))
76
print(f'Avg latency: {np.mean(latencies)} seconds')
77
78