@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4433195ab7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Bugfix] Prevent benchmark_throughput.py from using duplicated random prompts (#10753)

Michael Goin committed Dec 2, 2024 at 21:26 UTC 4433195ab75e2bb367303ba5f34c97521c5677ce
1 file changed +30 -17
benchmarks/benchmark_throughput.py
+30 -17
@@ -294,23 +294,36 @@ def main(args: argparse.Namespace):
294 tokenizer = AutoTokenizer.from_pretrained(
295 args.tokenizer, trust_remote_code=args.trust_remote_code)
296 if args.dataset is None:
297 - # Synthesize a prompt with the given input length.
298 - # As tokenizer may add additional tokens like BOS, we need to try
299 - # different lengths to get the desired input length.
300 - for i in range(-10, 10):
301 - prompt = "hi " * (args.input_len + i)
302 - tokenized_prompt = tokenizer(prompt).input_ids
303 - if len(tokenized_prompt) == args.input_len:
304 - break
305 - else:
306 - raise ValueError(
307 - f"Failed to synthesize a prompt with {args.input_len} tokens.")
308 - requests = [
309 - SampleRequest(prompt=prompt,
310 - prompt_len=args.input_len,
311 - expected_output_len=args.output_len)
312 - for _ in range(args.num_prompts)
313 - ]
297 + vocab_size = tokenizer.vocab_size
298 + requests = []
299 + for _ in range(args.num_prompts):
300 + # Synthesize a prompt with the given input length.
301 + candidate_ids = [
302 + random.randint(0, vocab_size - 1)
303 + for _ in range(args.input_len)
304 + ]
305 + # As tokenizer may add additional tokens like BOS, we need to try
306 + # different lengths to get the desired input length.
307 + for _ in range(5): # Max attempts to correct
308 + candidate_prompt = tokenizer.decode(candidate_ids)
309 + tokenized_len = len(tokenizer.encode(candidate_prompt))
310 +
311 + if tokenized_len == args.input_len:
312 + break
313 +
314 + # Adjust length based on difference
315 + diff = args.input_len - tokenized_len
316 + if diff > 0:
317 + candidate_ids.extend([
318 + random.randint(100, vocab_size - 100)
319 + for _ in range(diff)
320 + ])
321 + else:
322 + candidate_ids = candidate_ids[:diff]
323 + requests.append(
324 + SampleRequest(prompt=candidate_prompt,
325 + prompt_len=args.input_len,
326 + expected_output_len=args.output_len))
327 else:
328 requests = sample_requests(tokenizer, args)
329