A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
check input length of sonnet samples (#16423)
Signed-off-by: alexey-belyakov <alexey.belyakov@intel.com>
Alexey Belyakov committed
Apr 11, 2025 at 03:15 UTC
3e397a948431755b668fb95af679ec0a33daea89
1 file changed
+9
-8
benchmarks/benchmark_dataset.py
+9
-8
@@ -489,7 +489,7 @@ class SonnetDataset(BenchmarkDataset):
489
prefix_lines = self.data[:num_prefix_lines]
490
491
samples = []
492
- for _ in range(num_requests):
492
+ while len(samples) < num_requests:
493
extra_lines = random.choices(self.data,
494
k=num_input_lines - num_prefix_lines)
495
prompt = f"{base_prompt}{''.join(prefix_lines + extra_lines)}"
@@ -497,13 +497,14 @@ class SonnetDataset(BenchmarkDataset):
497
prompt_formatted = tokenizer.apply_chat_template(
498
msg, add_generation_prompt=True, tokenize=False)
499
prompt_len = len(tokenizer(prompt_formatted).input_ids)
500
- samples.append(
501
- SampleRequest(
502
- prompt=prompt_formatted
503
- if return_prompt_formatted else prompt,
504
- prompt_len=prompt_len,
505
- expected_output_len=output_len,
506
- ))
500
+ if prompt_len <= input_len:
501
+ samples.append(
502
+ SampleRequest(
503
+ prompt=prompt_formatted
504
+ if return_prompt_formatted else prompt,
505
+ prompt_len=prompt_len,
506
+ expected_output_len=output_len,
507
+ ))
508
return samples
509
510