A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
[Minor] Fix benchmark_latency script (#2765)
Woosuk Kwon committed
Feb 5, 2024 at 12:45 UTC
72d3a30c6327e70de3595d00f04e2d577fcbbb68
1 file changed
+5
-2
benchmarks/benchmark_latency.py
+5
-2
@@ -37,7 +37,10 @@ def main(args: argparse.Namespace):
37
max_tokens=args.output_len,
38
)
39
print(sampling_params)
40
- dummy_prompt_token_ids = [[0] * args.input_len] * args.batch_size
40
+ dummy_prompt_token_ids = np.random.randint(10000,
41
+ size=(args.batch_size,
42
+ args.input_len))
43
+ dummy_prompt_token_ids = dummy_prompt_token_ids.tolist()
44
45
def run_to_completion(profile_dir: Optional[str] = None):
46
if profile_dir:
@@ -71,7 +74,7 @@ def main(args: argparse.Namespace):
74
"."
75
) / "vllm_benchmark_result" / f"latency_result_{time.time()}"
76
print(f"Profiling (results will be saved to '{profile_dir}')...")
74
- run_to_completion(profile_dir=args.profile_result_dir)
77
+ run_to_completion(profile_dir=profile_dir)
78
return
79
80
# Benchmark.