A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 19:30 UTC
[Benchmark] Don't default to `temperature==0` in `vllm bench serve` (#32723)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Nick Hill committed
Jan 22, 2026 at 02:03 UTC
098b2d66fe310197cf7d70b7f2d43612b59f20c9
2 files changed
+7
-6
vllm/benchmarks/lib/endpoint_request_func.py
-3
@@ -160,7 +160,6 @@ async def async_request_openai_completions(
160
if request_func_input.model_name
161
else request_func_input.model,
162
"prompt": request_func_input.prompt,
163
- "temperature": 0.0,
163
"repetition_penalty": 1.0,
164
"max_tokens": request_func_input.output_len,
165
"logprobs": request_func_input.logprobs,
@@ -294,7 +293,6 @@ async def async_request_openai_chat_completions(
293
"messages": [
294
{"role": "user", "content": content},
295
],
297
- "temperature": 0.0,
296
"max_completion_tokens": request_func_input.output_len,
297
"stream": True,
298
"stream_options": {
@@ -389,7 +387,6 @@ async def async_request_openai_audio(
387
"model": request_func_input.model_name
388
if request_func_input.model_name
389
else request_func_input.model,
392
- "temperature": 0.0,
390
"max_completion_tokens": request_func_input.output_len,
391
"stream": True,
392
"language": "en",
vllm/benchmarks/serve.py
+7
-3
@@ -1419,8 +1419,7 @@ def add_cli_args(parser: argparse.ArgumentParser):
1419
type=float,
1420
default=None,
1421
help="Temperature sampling parameter. Only has effect on "
1422
- "openai-compatible backends. If not specified, default to greedy "
1423
- "decoding (i.e. temperature==0.0).",
1422
+ "openai-compatible backends.",
1423
)
1424
sampling_group.add_argument(
1425
"--frequency-penalty",
@@ -1634,7 +1633,12 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
1633
)
1634
1635
if "temperature" not in sampling_params:
1637
- sampling_params["temperature"] = 0.0 # Default to greedy decoding.
1636
+ print(
1637
+ "WARNING: vllm bench serve no longer sets temperature==0 (greedy) "
1638
+ "in requests by default. The default will be determined on the "
1639
+ "server side and can be model/API specific. "
1640
+ "For the old behavior, include --temperature=0."
1641
+ )
1642
1643
default_percentile_metrics = "ttft,tpot,itl"
1644
else: