@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 65050a40e6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Bugfix] Generate exactly input_len tokens in benchmark_throughput (#9592)

Chen Zhang committed Oct 22, 2024 at 17:45 UTC 65050a40e63fb8d57f383ea833d8869f77e85c89
1 file changed +10 -1
benchmarks/benchmark_throughput.py
+10 -1
@@ -233,7 +233,16 @@ def main(args: argparse.Namespace):
233 args.tokenizer, trust_remote_code=args.trust_remote_code)
234 if args.dataset is None:
235 # Synthesize a prompt with the given input length.
236 - prompt = "hi" * (args.input_len - 1)
236 + # As tokenizer may add additional tokens like BOS, we need to try
237 + # different lengths to get the desired input length.
238 + for i in range(-10, 10):
239 + prompt = "hi " * (args.input_len + i)
240 + tokenized_prompt = tokenizer(prompt).input_ids
241 + if len(tokenized_prompt) == args.input_len:
242 + break
243 + else:
244 + raise ValueError(
245 + f"Failed to synthesize a prompt with {args.input_len} tokens.")
246 requests = [(prompt, args.input_len, args.output_len)
247 for _ in range(args.num_prompts)]
248 else: