@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c52ec5f034
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Bugfix] fixing sonnet benchmark bug in benchmark_serving.py (#8616)

Kuntai Du committed Sep 18, 2024 at 22:24 UTC c52ec5f03471008fa1312d82fb17d40b95a3ca5d
1 file changed +5 -5
benchmarks/benchmark_serving.py
+5 -5
@@ -626,9 +626,9 @@ def main(args: argparse.Namespace):
626 prefix_len=args.sonnet_prefix_len,
627 tokenizer=tokenizer,
628 )
629 - input_requests = [(prompt, prompt_len, output_len)
629 + input_requests = [(prompt, prompt_len, output_len, None)
630 for prompt, prompt_formatted, prompt_len,
631 - output_len in input_requests]
631 + output_len, _ in input_requests]
632 else:
633 assert (
634 tokenizer.chat_template or tokenizer.default_chat_template
@@ -641,9 +641,9 @@ def main(args: argparse.Namespace):
641 prefix_len=args.sonnet_prefix_len,
642 tokenizer=tokenizer,
643 )
644 - input_requests = [(prompt_formatted, prompt_len, output_len)
644 + input_requests = [(prompt_formatted, prompt_len, output_len, None)
645 for prompt, prompt_formatted, prompt_len,
646 - output_len in input_requests]
646 + output_len, _ in input_requests]
647
648 elif args.dataset_name == "hf":
649 input_requests = sample_hf_requests(
@@ -963,4 +963,4 @@ if __name__ == "__main__":
963 )
964
965 args = parser.parse_args()
966 - main(args)
966 + main(args)
\ No newline at end of file