A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 02:15 UTC
[Bugfix][Benchmark] Make sure the output length > 0 when testing prefill workload. (#20786)
Signed-off-by: KuntaiDu <kuntai@uchicago.edu> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Kuntai Du committed
Jul 10, 2025 at 14:52 UTC
5b6fe23d05851b7bd353dbaf2a322f479215d726
1 file changed
+3
benchmarks/benchmark_dataset.py
+3
@@ -324,6 +324,9 @@ class RandomDataset(BenchmarkDataset):
324
input_low = int(real_input_len * (1 - range_ratio))
325
input_high = int(real_input_len * (1 + range_ratio))
326
output_low = int(output_len * (1 - range_ratio))
327
+ # Ensure the lower bound for output length is at least 1 to prevent
328
+ # sampling 0 tokens, which can cause request failures.
329
+ output_low = max(output_low, 1)
330
output_high = int(output_len * (1 + range_ratio))
331
332
# Add logging for debugging