[Rust][Benchmark] Warn when temperature is left to the server default (#59247)
Idder Ghanbaja committed
Oct 1, 2026 at 15:03 UTC
f3b77eff640b5ed5cc9221a4f5f402ab65adc021
1 file changed
+59
rust/src/bench/src/config.rs
+59
@@ -309,6 +309,14 @@ impl BenchConfig {
309
}
310
}
311
312
+ if uses_server_default_temperature(args.backend, extra_body.as_ref()) {
313
+ tracing::warn!(
314
+ "vllm-bench does not set temperature==0 (greedy) in requests by default. \
315
+ The default will be determined on the server side and can be \
316
+ model/API specific. For greedy decoding, include --temperature=0."
317
+ );
318
+ }
319
+
320
// Parse metadata
321
let metadata = match &args.metadata {
322
None => None,
@@ -862,6 +870,21 @@ fn parse_ramp_up(args: &BenchServeArgs) -> Result<Option<RampUpConfig>> {
870
}))
871
}
872
873
+/// Whether generation requests leave `temperature` to the server-side default.
874
+///
875
+/// Python `vllm bench serve` defaulted to greedy decoding before v0.15, so
876
+/// results are only comparable with older runs when temperature is set explicitly.
877
+/// An explicit `"temperature": null` also falls back to the server default.
878
+fn uses_server_default_temperature(
879
+ backend: BackendKind,
880
+ extra_body: Option<&serde_json::Value>,
881
+) -> bool {
882
+ !backend.is_pooling()
883
+ && extra_body
884
+ .and_then(|b| b.get("temperature"))
885
+ .is_none_or(serde_json::Value::is_null)
886
+}
887
+
888
#[cfg(test)]
889
mod tests {
890
use clap::Parser;
@@ -910,6 +933,42 @@ mod tests {
933
assert_eq!(args.speed_bench_output_len, 4096);
934
}
935
936
+ /// Temperature left unset is flagged, since older Python versions defaulted to greedy.
937
+ #[test]
938
+ fn test_uses_server_default_temperature() {
939
+ let check = |backend, extra: &[&str]| {
940
+ let mut argv = vec!["vllm-bench", "--model", "test-model", "--backend", backend];
941
+ argv.extend_from_slice(extra);
942
+ let config = BenchConfig::from_args(&parse_args(argv)).unwrap();
943
+ uses_server_default_temperature(config.backend, config.extra_body.as_ref())
944
+ };
945
+
946
+ assert!(check("openai-chat", &[]));
947
+ assert!(check("vllm", &["--extra-body", r#"{"top_p": 0.9}"#]));
948
+ assert!(!check("openai-chat", &["--temperature", "0"]));
949
+ assert!(!check(
950
+ "openai",
951
+ &["--extra-body", r#"{"temperature": 0.6}"#]
952
+ ));
953
+ assert!(!check("openai-embeddings", &[]));
954
+
955
+ // `"temperature": null` leaves it to the server, and `--extra-body`
956
+ // overrides `--temperature`, so both cases must still warn.
957
+ assert!(check(
958
+ "openai-chat",
959
+ &["--extra-body", r#"{"temperature": null}"#]
960
+ ));
961
+ assert!(check(
962
+ "openai-chat",
963
+ &[
964
+ "--temperature",
965
+ "0",
966
+ "--extra-body",
967
+ r#"{"temperature": null}"#
968
+ ]
969
+ ));
970
+ }
971
+
972
#[test]
973
fn test_prefix_sharing_defaults_to_zero() {
974
let args = base_multi_turn_args();