A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Bugfix] Prevent benchmark_throughput.py from using duplicated random prompts (#10753)
Michael Goin committed
Dec 2, 2024 at 21:26 UTC
4433195ab75e2bb367303ba5f34c97521c5677ce
1 file changed
+30
-17
benchmarks/benchmark_throughput.py
+30
-17
@@ -294,23 +294,36 @@ def main(args: argparse.Namespace):
294
tokenizer = AutoTokenizer.from_pretrained(
295
args.tokenizer, trust_remote_code=args.trust_remote_code)
296
if args.dataset is None:
297
- # Synthesize a prompt with the given input length.
298
- # As tokenizer may add additional tokens like BOS, we need to try
299
- # different lengths to get the desired input length.
300
- for i in range(-10, 10):
301
- prompt = "hi " * (args.input_len + i)
302
- tokenized_prompt = tokenizer(prompt).input_ids
303
- if len(tokenized_prompt) == args.input_len:
304
- break
305
- else:
306
- raise ValueError(
307
- f"Failed to synthesize a prompt with {args.input_len} tokens.")
308
- requests = [
309
- SampleRequest(prompt=prompt,
310
- prompt_len=args.input_len,
311
- expected_output_len=args.output_len)
312
- for _ in range(args.num_prompts)
313
- ]
297
+ vocab_size = tokenizer.vocab_size
298
+ requests = []
299
+ for _ in range(args.num_prompts):
300
+ # Synthesize a prompt with the given input length.
301
+ candidate_ids = [
302
+ random.randint(0, vocab_size - 1)
303
+ for _ in range(args.input_len)
304
+ ]
305
+ # As tokenizer may add additional tokens like BOS, we need to try
306
+ # different lengths to get the desired input length.
307
+ for _ in range(5): # Max attempts to correct
308
+ candidate_prompt = tokenizer.decode(candidate_ids)
309
+ tokenized_len = len(tokenizer.encode(candidate_prompt))
310
+
311
+ if tokenized_len == args.input_len:
312
+ break
313
+
314
+ # Adjust length based on difference
315
+ diff = args.input_len - tokenized_len
316
+ if diff > 0:
317
+ candidate_ids.extend([
318
+ random.randint(100, vocab_size - 100)
319
+ for _ in range(diff)
320
+ ])
321
+ else:
322
+ candidate_ids = candidate_ids[:diff]
323
+ requests.append(
324
+ SampleRequest(prompt=candidate_prompt,
325
+ prompt_len=args.input_len,
326
+ expected_output_len=args.output_len))
327
else:
328
requests = sample_requests(tokenizer, args)
329