A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bug] Add Assertion for `random-input-len` / `random-output-len` (#26834)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
Wentao Ye committed
Oct 16, 2025 at 17:36 UTC
23583ee28cd383a0a961f31b53725b32770be5e7
1 file changed
+16
vllm/benchmarks/datasets.py
+16
@@ -478,6 +478,22 @@ class RandomDataset(BenchmarkDataset):
478
batchsize: int = 1,
479
**kwargs,
480
) -> list[SampleRequest]:
481
+ # validate total input tokens (prefix + sampled) is at least 1.
482
+ num_special = int(tokenizer.num_special_tokens_to_add())
483
+ real_input_len = max(0, int(input_len) - num_special)
484
+ min_sampled_input = math.floor(real_input_len * (1.0 - float(range_ratio)))
485
+ min_total_input = int(prefix_len) + min_sampled_input
486
+ if min_total_input < 1:
487
+ raise ValueError(
488
+ "--random-input-len is too small: with tokenizer special "
489
+ f"tokens {num_special} and --random-range-ratio {range_ratio}, "
490
+ "the minimum possible total input tokens (prefix + sampled) is "
491
+ f"{min_total_input}. Increase --random-input-len and/or "
492
+ "--random-prefix-len, or decrease --random-range-ratio so that "
493
+ "prefix_len + floor(max(0, random_input_len - num_special)) "
494
+ "* (1 - range_ratio) >= 1."
495
+ )
496
+
497
input_lens, output_lens, offsets = self.get_sampling_params(
498
num_requests, range_ratio, input_len, output_len, tokenizer
499
)