A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Misc] Add penalties sampling parameters to serve tool (#25974)
Signed-off-by: Sergei Skvortsov <sergeyskv@nebius.com> Co-authored-by: Sergei Skvortsov <sergeyskv@nebius.com>
Sergei Skvortsov committed
Oct 3, 2025 at 23:43 UTC
b71fcd490559e204a2a9e30993ec2d6abba52638
1 file changed
+24
vllm/benchmarks/serve.py
+24
@@ -1077,6 +1077,27 @@ def add_cli_args(parser: argparse.ArgumentParser):
1077
"openai-compatible backends. If not specified, default to greedy "
1078
"decoding (i.e. temperature==0.0).",
1079
)
1080
+ sampling_group.add_argument(
1081
+ "--frequency-penalty",
1082
+ type=float,
1083
+ default=None,
1084
+ help="Frequency penalty sampling parameter. Only has effect on "
1085
+ "openai-compatible backends.",
1086
+ )
1087
+ sampling_group.add_argument(
1088
+ "--presence-penalty",
1089
+ type=float,
1090
+ default=None,
1091
+ help="Presence penalty sampling parameter. Only has effect on "
1092
+ "openai-compatible backends.",
1093
+ )
1094
+ sampling_group.add_argument(
1095
+ "--repetition-penalty",
1096
+ type=float,
1097
+ default=None,
1098
+ help="Repetition penalty sampling parameter. Only has effect on "
1099
+ "openai-compatible backends.",
1100
+ )
1101
1102
parser.add_argument(
1103
'--tokenizer-mode',
@@ -1211,6 +1232,9 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
1232
"top_k": args.top_k,
1233
"min_p": args.min_p,
1234
"temperature": args.temperature,
1235
+ "frequency_penalty": args.frequency_penalty,
1236
+ "presence_penalty": args.presence_penalty,
1237
+ "repetition_penalty": args.repetition_penalty,
1238
}.items() if v is not None
1239
}
1240