A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
fix sonnet dataset sample when prefix len is very small (#16379)
Signed-off-by: Chenyaaang <chenyangli@google.com>
Chenyaaang committed
Apr 9, 2025 at 22:35 UTC
417bcefbae024a10ddedb18e2dab58e7a4c535b3
1 file changed
+1
-1
benchmarks/benchmark_dataset.py
+1
-1
@@ -472,7 +472,7 @@ class SonnetDataset(BenchmarkDataset):
472
473
# Determine how many poem lines to use.
474
num_input_lines = round((input_len - base_offset) / avg_len)
475
- num_prefix_lines = round((prefix_len - base_offset) / avg_len)
475
+ num_prefix_lines = max(round((prefix_len - base_offset) / avg_len), 0)
476
prefix_lines = self.data[:num_prefix_lines]
477
478
samples = []