A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
[Benchmark][Bugfix] Fix SonnetDataset default values in benchmark_throughput.py (#16556)
Jennifer Zhao committed
Apr 13, 2025 at 17:20 UTC
63d2705edbe7fbf4d581ef49503725f3481e04c7
1 file changed
+16
-8
benchmarks/benchmark_throughput.py
+16
-8
@@ -597,18 +597,26 @@ if __name__ == "__main__":
597
parser.add_argument(
598
"--prefix-len",
599
type=int,
600
- default=0,
601
- help="Number of fixed prefix tokens before the random "
602
- "context in a request (default: 0).",
603
- )
600
+ default=None,
601
+ help=f"Number of prefix tokens to be used in RandomDataset "
602
+ "and SonnetDataset. For RandomDataset, the total input "
603
+ "length is the sum of prefix-len (default: "
604
+ f"{RandomDataset.DEFAULT_PREFIX_LEN}) and a random context length "
605
+ "sampled from [input_len * (1 - range_ratio), "
606
+ "input_len * (1 + range_ratio)]. For SonnetDataset, "
607
+ f"prefix_len (default: {SonnetDataset.DEFAULT_PREFIX_LEN}) "
608
+ "controls how much of the input is fixed lines versus "
609
+ "random lines, but the total input length remains approximately "
610
+ "input_len tokens.")
611
# random dataset
612
parser.add_argument(
613
"--random-range-ratio",
614
type=float,
608
- default=0.0,
609
- help="Range ratio for sampling input/output length, "
610
- "used only for RandomDataset. Must be in the range [0, 1) to define "
611
- "a symmetric sampling range "
615
+ default=None,
616
+ help=f"Range ratio (default : {RandomDataset.DEFAULT_RANGE_RATIO}) "
617
+ "for sampling input/output length, "
618
+ "used only for RandomDataset. Must be in the range [0, 1) to "
619
+ "define a symmetric sampling range "
620
"[length * (1 - range_ratio), length * (1 + range_ratio)].",
621
)
622