A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Bugfix] fixing sonnet benchmark bug in benchmark_serving.py (#8616)
Kuntai Du committed
Sep 18, 2024 at 22:24 UTC
c52ec5f03471008fa1312d82fb17d40b95a3ca5d
1 file changed
+5
-5
benchmarks/benchmark_serving.py
+5
-5
@@ -626,9 +626,9 @@ def main(args: argparse.Namespace):
626
prefix_len=args.sonnet_prefix_len,
627
tokenizer=tokenizer,
628
)
629
- input_requests = [(prompt, prompt_len, output_len)
629
+ input_requests = [(prompt, prompt_len, output_len, None)
630
for prompt, prompt_formatted, prompt_len,
631
- output_len in input_requests]
631
+ output_len, _ in input_requests]
632
else:
633
assert (
634
tokenizer.chat_template or tokenizer.default_chat_template
@@ -641,9 +641,9 @@ def main(args: argparse.Namespace):
641
prefix_len=args.sonnet_prefix_len,
642
tokenizer=tokenizer,
643
)
644
- input_requests = [(prompt_formatted, prompt_len, output_len)
644
+ input_requests = [(prompt_formatted, prompt_len, output_len, None)
645
for prompt, prompt_formatted, prompt_len,
646
- output_len in input_requests]
646
+ output_len, _ in input_requests]
647
648
elif args.dataset_name == "hf":
649
input_requests = sample_hf_requests(
@@ -963,4 +963,4 @@ if __name__ == "__main__":
963
)
964
965
args = parser.parse_args()
966
- main(args)
966
+ main(args)
\ No newline at end of file