@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 72d3a30c63
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[Minor] Fix benchmark_latency script (#2765)

Woosuk Kwon committed Feb 5, 2024 at 12:45 UTC 72d3a30c6327e70de3595d00f04e2d577fcbbb68
1 file changed +5 -2
benchmarks/benchmark_latency.py
+5 -2
@@ -37,7 +37,10 @@ def main(args: argparse.Namespace):
37 max_tokens=args.output_len,
38 )
39 print(sampling_params)
40 - dummy_prompt_token_ids = [[0] * args.input_len] * args.batch_size
40 + dummy_prompt_token_ids = np.random.randint(10000,
41 + size=(args.batch_size,
42 + args.input_len))
43 + dummy_prompt_token_ids = dummy_prompt_token_ids.tolist()
44
45 def run_to_completion(profile_dir: Optional[str] = None):
46 if profile_dir:
@@ -71,7 +74,7 @@ def main(args: argparse.Namespace):
74 "."
75 ) / "vllm_benchmark_result" / f"latency_result_{time.time()}"
76 print(f"Profiling (results will be saved to '{profile_dir}')...")
74 - run_to_completion(profile_dir=args.profile_result_dir)
77 + run_to_completion(profile_dir=profile_dir)
78 return
79
80 # Benchmark.