A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Bugfix] Fix args in benchmark_serving (#3836)
Co-authored-by: Roger Wang <ywang@roblox.com>
Chang Su committed
Apr 4, 2024 at 00:41 UTC
819a309c0fcf84d9b023a96d6836e7501309d6b1
1 file changed
+13
-10
benchmarks/benchmark_serving.py
+13
-10
@@ -110,7 +110,9 @@ def sample_sonnet_requests(
110
prefix_len: int,
111
tokenizer: PreTrainedTokenizerBase,
112
) -> List[Tuple[str, str, int, int]]:
113
- assert input_len > prefix_len, "input_len must be greater than prefix_len."
113
+ assert (
114
+ input_len > prefix_len
115
+ ), "'args.sonnet-input-len' must be greater than 'args.prefix-input-len'."
116
117
# Load the dataset.
118
with open(dataset_path) as f:
@@ -131,8 +133,9 @@ def sample_sonnet_requests(
133
base_message, add_generation_prompt=True, tokenize=False)
134
base_prompt_offset = len(tokenizer(base_prompt_formatted).input_ids)
135
134
- assert (input_len > base_prompt_offset
135
- ), f"Please set 'args.input-len' higher than {base_prompt_offset}."
136
+ assert (
137
+ input_len > base_prompt_offset
138
+ ), f"Please set 'args.sonnet-input-len' higher than {base_prompt_offset}."
139
num_input_lines = round(
140
(input_len - base_prompt_offset) / average_poem_len)
141
@@ -140,7 +143,7 @@ def sample_sonnet_requests(
143
# prompt are fixed poem lines.
144
assert (
145
prefix_len > base_prompt_offset
143
- ), f"Please set 'args.prefix-len' higher than {base_prompt_offset}."
146
+ ), f"Please set 'args.sonnet-prefix-len' higher than {base_prompt_offset}."
147
148
num_prefix_lines = round(
149
(prefix_len - base_prompt_offset) / average_poem_len)
@@ -373,9 +376,9 @@ def main(args: argparse.Namespace):
376
input_requests = sample_sonnet_requests(
377
dataset_path=args.dataset_path,
378
num_requests=args.num_prompts,
376
- input_len=args.input_len,
377
- output_len=args.output_len,
378
- prefix_len=args.prefix_len,
379
+ input_len=args.sonnet_input_len,
380
+ output_len=args.sonnet_output_len,
381
+ prefix_len=args.sonnet_prefix_len,
382
tokenizer=tokenizer,
383
)
384
input_requests = [(prompt, prompt_len, output_len)
@@ -388,9 +391,9 @@ def main(args: argparse.Namespace):
391
input_requests = sample_sonnet_requests(
392
dataset_path=args.dataset_path,
393
num_requests=args.num_prompts,
391
- input_len=args.input_len,
392
- output_len=args.output_len,
393
- prefix_len=args.prefix_len,
394
+ input_len=args.sonnet_input_len,
395
+ output_len=args.sonnet_output_len,
396
+ prefix_len=args.sonnet_prefix_len,
397
tokenizer=tokenizer,
398
)
399
input_requests = [(prompt_formatted, prompt_len, output_len)