A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 23:45 UTC
[bench] Support common prefix len config (for decode-only bench) (#29934)
Signed-off-by: Ming Yang <minos.future@gmail.com>
Ming Yang committed
Dec 5, 2025 at 02:26 UTC
f16356fe361df9ccef3a9b46d6a43d43a854e2e0
2 files changed
+7
vllm/benchmarks/datasets.py
+1
@@ -1842,6 +1842,7 @@ def get_samples(args, tokenizer) -> list[SampleRequest]:
1842
random_seed=args.seed,
1843
dataset_path=args.dataset_path,
1844
disable_shuffle=args.disable_shuffle,
1845
+ prefix_len=args.common_prefix_len,
1846
).sample(
1847
tokenizer=tokenizer,
1848
num_requests=args.num_prompts,
vllm/benchmarks/serve.py
+6
@@ -1221,6 +1221,12 @@ def add_cli_args(parser: argparse.ArgumentParser):
1221
help="Repetition penalty sampling parameter. Only has effect on "
1222
"openai-compatible backends.",
1223
)
1224
+ sampling_group.add_argument(
1225
+ "--common-prefix-len",
1226
+ type=int,
1227
+ default=None,
1228
+ help="Common prefix length shared by all prompts (used by random dataset)",
1229
+ )
1230
1231
parser.add_argument(
1232
"--tokenizer-mode",