A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 10:00 UTC
[Bugfix] Fix prefix_repetition routing in bench throughput (#29663)
Signed-off-by: Junru Shen <jrshen.sjr@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Junru Shen committed
Dec 16, 2025 at 17:37 UTC
676db55eecf8b6d9ec38ea243cf6f35ea8378ec6
1 file changed
+4
-1
vllm/benchmarks/throughput.py
+4
-1
@@ -346,7 +346,10 @@ def get_requests(args, tokenizer):
346
"output_len": args.output_len,
347
}
348
349
- if args.dataset_path is None or args.dataset_name == "random":
349
+ if args.dataset_name == "random" or (
350
+ args.dataset_path is None
351
+ and args.dataset_name not in {"prefix_repetition", "random-mm", "random-rerank"}
352
+ ):
353
sample_kwargs["range_ratio"] = args.random_range_ratio
354
sample_kwargs["prefix_len"] = args.prefix_len
355
dataset_cls = RandomDataset