@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 819a309c0f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Bugfix] Fix args in benchmark_serving (#3836)

Co-authored-by: Roger Wang <ywang@roblox.com>

Chang Su committed Apr 4, 2024 at 00:41 UTC 819a309c0fcf84d9b023a96d6836e7501309d6b1
1 file changed +13 -10
benchmarks/benchmark_serving.py
+13 -10
@@ -110,7 +110,9 @@ def sample_sonnet_requests(
110 prefix_len: int,
111 tokenizer: PreTrainedTokenizerBase,
112 ) -> List[Tuple[str, str, int, int]]:
113 - assert input_len > prefix_len, "input_len must be greater than prefix_len."
113 + assert (
114 + input_len > prefix_len
115 + ), "'args.sonnet-input-len' must be greater than 'args.prefix-input-len'."
116
117 # Load the dataset.
118 with open(dataset_path) as f:
@@ -131,8 +133,9 @@ def sample_sonnet_requests(
133 base_message, add_generation_prompt=True, tokenize=False)
134 base_prompt_offset = len(tokenizer(base_prompt_formatted).input_ids)
135
134 - assert (input_len > base_prompt_offset
135 - ), f"Please set 'args.input-len' higher than {base_prompt_offset}."
136 + assert (
137 + input_len > base_prompt_offset
138 + ), f"Please set 'args.sonnet-input-len' higher than {base_prompt_offset}."
139 num_input_lines = round(
140 (input_len - base_prompt_offset) / average_poem_len)
141
@@ -140,7 +143,7 @@ def sample_sonnet_requests(
143 # prompt are fixed poem lines.
144 assert (
145 prefix_len > base_prompt_offset
143 - ), f"Please set 'args.prefix-len' higher than {base_prompt_offset}."
146 + ), f"Please set 'args.sonnet-prefix-len' higher than {base_prompt_offset}."
147
148 num_prefix_lines = round(
149 (prefix_len - base_prompt_offset) / average_poem_len)
@@ -373,9 +376,9 @@ def main(args: argparse.Namespace):
376 input_requests = sample_sonnet_requests(
377 dataset_path=args.dataset_path,
378 num_requests=args.num_prompts,
376 - input_len=args.input_len,
377 - output_len=args.output_len,
378 - prefix_len=args.prefix_len,
379 + input_len=args.sonnet_input_len,
380 + output_len=args.sonnet_output_len,
381 + prefix_len=args.sonnet_prefix_len,
382 tokenizer=tokenizer,
383 )
384 input_requests = [(prompt, prompt_len, output_len)
@@ -388,9 +391,9 @@ def main(args: argparse.Namespace):
391 input_requests = sample_sonnet_requests(
392 dataset_path=args.dataset_path,
393 num_requests=args.num_prompts,
391 - input_len=args.input_len,
392 - output_len=args.output_len,
393 - prefix_len=args.prefix_len,
394 + input_len=args.sonnet_input_len,
395 + output_len=args.sonnet_output_len,
396 + prefix_len=args.sonnet_prefix_len,
397 tokenizer=tokenizer,
398 )
399 input_requests = [(prompt_formatted, prompt_len, output_len)