A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Bugfix] Generate exactly input_len tokens in benchmark_throughput (#9592)
Chen Zhang committed
Oct 22, 2024 at 17:45 UTC
65050a40e63fb8d57f383ea833d8869f77e85c89
1 file changed
+10
-1
benchmarks/benchmark_throughput.py
+10
-1
@@ -233,7 +233,16 @@ def main(args: argparse.Namespace):
233
args.tokenizer, trust_remote_code=args.trust_remote_code)
234
if args.dataset is None:
235
# Synthesize a prompt with the given input length.
236
- prompt = "hi" * (args.input_len - 1)
236
+ # As tokenizer may add additional tokens like BOS, we need to try
237
+ # different lengths to get the desired input length.
238
+ for i in range(-10, 10):
239
+ prompt = "hi " * (args.input_len + i)
240
+ tokenized_prompt = tokenizer(prompt).input_ids
241
+ if len(tokenized_prompt) == args.input_len:
242
+ break
243
+ else:
244
+ raise ValueError(
245
+ f"Failed to synthesize a prompt with {args.input_len} tokens.")
246
requests = [(prompt, args.input_len, args.output_len)
247
for _ in range(args.num_prompts)]
248
else: