@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 717f4bcea0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

Feature/add benchmark testing (#5947)

Co-authored-by: Roger Wang <ywang@roblox.com>

Haichuan committed Jul 8, 2024 at 15:52 UTC 717f4bcea036a049e86802b3a05dd6f7cd17efc8
1 file changed +3 -3
benchmarks/benchmark_serving.py
+3 -3
@@ -483,10 +483,10 @@ def main(args: argparse.Namespace):
483
484 elif args.dataset_name == "random":
485 input_requests = sample_random_requests(
486 - input_len=args.input_len,
487 - output_len=args.output_len,
486 + input_len=args.random_input_len,
487 + output_len=args.random_output_len,
488 num_prompts=args.num_prompts,
489 - range_ratio=args.range_ratio,
489 + range_ratio=args.random_range_ratio,
490 tokenizer=tokenizer,
491 )
492