A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Feature] default --extra-body param to disable thinking in vllm bench serve (#26784)
Signed-off-by: rongfu.leng <rongfu.leng@daocloud.io>
rongfu.leng committed
Oct 15, 2025 at 12:23 UTC
a27b288e4a389e3ece5e178bc0219c6c0e1db7d1
1 file changed
+13
-1
vllm/benchmarks/serve.py
+13
-1
@@ -1230,6 +1230,15 @@ def add_cli_args(parser: argparse.ArgumentParser):
1230
"the ready check will be skipped.",
1231
)
1232
1233
+ parser.add_argument(
1234
+ "--extra-body",
1235
+ help="A JSON string representing extra body parameters to include "
1236
+ "in each request."
1237
+ 'Example: \'{"chat_template_kwargs":{"enable_thinking":false}}\'',
1238
+ type=json.loads,
1239
+ default=None,
1240
+ )
1241
+
1242
1243
def main(args: argparse.Namespace) -> dict[str, Any]:
1244
return asyncio.run(main_async(args))
@@ -1330,6 +1339,9 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
1339
else:
1340
sampling_params = {}
1341
1342
+ extra_body = args.extra_body or {}
1343
+ extra_body = {**sampling_params, **extra_body}
1344
+
1345
# Avoid GC processing "static" data - reduce pause times.
1346
gc.collect()
1347
gc.freeze()
@@ -1355,7 +1367,7 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
1367
max_concurrency=args.max_concurrency,
1368
lora_modules=args.lora_modules,
1369
extra_headers=headers,
1358
- extra_body=sampling_params,
1370
+ extra_body=extra_body,
1371
ramp_up_strategy=args.ramp_up_strategy,
1372
ramp_up_start_rps=args.ramp_up_start_rps,
1373
ramp_up_end_rps=args.ramp_up_end_rps,