@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 9db222c6be

[Rust Frontend] Align OpenAI request and response edge cases (#53218)

Co-authored-by: Nick Hill <nickhill123@gmail.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>

Bugen Zhao committed Aug 28, 2026 at 05:23 UTC 9db222c6bea2623e18ce09a8f270ac9cb7d86981
16 files changed +419 -69
rust/src/server/src/routes/openai/chat_completions.rs
+2 -2
@@ -63,7 +63,8 @@ pub async fn chat_completions(
63 ) -> Response {
64 let stream = body.stream;
65 let request_context = resolve_request_context(&headers, body.request_id.as_deref());
66 - let lora_resolution = state.resolve_model_with_loras(Some(&body.model)).await;
66 + let requested_model = body.model.as_deref().filter(|model| !model.is_empty());
67 + let lora_resolution = state.resolve_model_with_loras(requested_model).await;
68
69 let prepared = match prepare_chat_request(body, &lora_resolution, request_context) {
70 Ok(prepared) => prepared,
@@ -196,7 +197,6 @@ async fn collect_chat_completion(
197 Some(prompt_logprobs_to_maps(
198 prompt_logprobs.as_ref(),
199 &prompt_token_ids,
199 - return_tokens_as_token_ids,
200 )?)
201 } else {
202 None
rust/src/server/src/routes/openai/chat_completions/convert.rs
+118 -3
@@ -433,6 +433,7 @@ mod tests {
433 use expect_test::expect;
434 use llm_multimodal::ImageDetail;
435 use serde_json::json;
436 + use validator::Validate;
437 use vllm_chat::{
438 AssistantContentBlock, AssistantToolCall, ChatContentPart, ChatMessage as VllmChatMessage,
439 ChatRenderer, ChatTool as VllmChatTool, ChatToolChoice, GenerationPromptMode,
@@ -466,7 +467,7 @@ mod tests {
467
468 fn base_request() -> ChatCompletionRequest {
469 ChatCompletionRequest {
469 - model: "Qwen/Qwen1.5-0.5B-Chat".to_string(),
470 + model: Some("Qwen/Qwen1.5-0.5B-Chat".to_string()),
471 messages: vec![ChatMessage::User {
472 content: MessageContent::Text("hello".to_string()),
473 name: None,
@@ -476,6 +477,120 @@ mod tests {
477 }
478 }
479
480 + #[test]
481 + fn chat_http_request_defaults_missing_or_null_model() {
482 + for model in [None, Some(serde_json::Value::Null)] {
483 + let mut value = json!({
484 + "messages": [{"role": "user", "content": "hello"}],
485 + "stream": true,
486 + });
487 + if let Some(model) = model {
488 + value
489 + .as_object_mut()
490 + .expect("request object")
491 + .insert("model".to_string(), model);
492 + }
493 +
494 + let request: ChatCompletionRequest =
495 + serde_json::from_value(value).expect("parse request without model");
496 + assert!(request.model.is_none());
497 + }
498 +
499 + let request: ChatCompletionRequest = serde_json::from_value(json!({
500 + "messages": [{"role": "user", "content": "hello"}],
501 + "model": "",
502 + }))
503 + .expect("parse empty model");
504 + assert_eq!(request.model.as_deref(), Some(""));
505 + }
506 +
507 + #[test]
508 + fn prepare_chat_request_normalizes_top_k_sentinels() {
509 + for (top_k, expected) in [
510 + (serde_json::Value::Null, None),
511 + (json!(-1), Some(0)),
512 + (json!(0), Some(0)),
513 + (json!(20), Some(20)),
514 + ] {
515 + let request: ChatCompletionRequest = serde_json::from_value(json!({
516 + "model": "Qwen/Qwen1.5-0.5B-Chat",
517 + "messages": [{"role": "user", "content": "hello"}],
518 + "stream": true,
519 + "top_k": top_k,
520 + }))
521 + .expect("parse top_k");
522 + let prepared = prepare_chat_request(
523 + request,
524 + &served(&["Qwen/Qwen1.5-0.5B-Chat"]),
525 + ResolvedRequestContext::default(),
526 + )
527 + .expect("prepare top_k");
528 +
529 + assert_eq!(prepared.chat_request.sampling_params.top_k, expected);
530 + }
531 + }
532 +
533 + #[test]
534 + fn prepare_chat_request_accepts_zero_min_tokens() {
535 + let request: ChatCompletionRequest = serde_json::from_value(json!({
536 + "model": "Qwen/Qwen1.5-0.5B-Chat",
537 + "messages": [{"role": "user", "content": "hello"}],
538 + "stream": true,
539 + "min_tokens": 0,
540 + }))
541 + .expect("parse zero min_tokens");
542 + request.validate().expect("validate zero min_tokens");
543 +
544 + let prepared = prepare_chat_request(
545 + request,
546 + &served(&["Qwen/Qwen1.5-0.5B-Chat"]),
547 + ResolvedRequestContext::default(),
548 + )
549 + .expect("prepare zero min_tokens");
550 +
551 + assert_eq!(prepared.chat_request.sampling_params.min_tokens, Some(0));
552 + assert_eq!(prepared.chat_request.decode_options.min_tokens, 0);
553 + }
554 +
555 + #[test]
556 + fn prepare_chat_request_defaults_function_tool_fields() {
557 + let request: ChatCompletionRequest = serde_json::from_value(json!({
558 + "model": "Qwen/Qwen1.5-0.5B-Chat",
559 + "messages": [{"role": "user", "content": "hello"}],
560 + "tools": [{"function": {"name": "lookup"}}],
561 + }))
562 + .expect("parse tool defaults");
563 +
564 + let prepared = prepare_chat_request(
565 + request,
566 + &served(&["Qwen/Qwen1.5-0.5B-Chat"]),
567 + ResolvedRequestContext::default(),
568 + )
569 + .expect("prepare tool defaults");
570 +
571 + assert_eq!(
572 + prepared.chat_request.tools(),
573 + &[VllmChatTool {
574 + name: "lookup".to_string(),
575 + description: None,
576 + parameters: serde_json::Value::Null,
577 + strict: None,
578 + }]
579 + );
580 + }
581 +
582 + #[test]
583 + fn chat_http_request_rejects_empty_cache_salt() {
584 + let request: ChatCompletionRequest = serde_json::from_value(json!({
585 + "model": "Qwen/Qwen1.5-0.5B-Chat",
586 + "messages": [{"role": "user", "content": "hello"}],
587 + "cache_salt": "",
588 + }))
589 + .expect("parse cache_salt");
590 +
591 + assert!(request.validate().is_err());
592 + }
593 +
594 #[test]
595 fn prepare_chat_request_maps_parallel_tool_calls() {
596 let mut request = base_request();
@@ -506,7 +621,7 @@ mod tests {
621 #[test]
622 fn prepare_chat_request_passes_response_format_to_kimi_k3_renderer() {
623 let mut request = base_request();
509 - request.model = "moonshotai/Kimi-K3".to_string();
624 + request.model = Some("moonshotai/Kimi-K3".to_string());
625 let response_format = ResponseFormat::JsonSchema {
626 json_schema: JsonSchemaFormat {
627 name: "answer".to_string(),
@@ -913,7 +1028,7 @@ mod tests {
1028 #[test]
1029 fn prepare_chat_request_accepts_audio_content_parts() {
1030 let request = ChatCompletionRequest {
916 - model: "Qwen/Qwen3-ASR-1.7B".to_string(),
1031 + model: Some("Qwen/Qwen3-ASR-1.7B".to_string()),
1032 messages: vec![ChatMessage::User {
1033 content: MessageContent::Parts(vec![
1034 ContentPart::InputAudio {
rust/src/server/src/routes/openai/chat_completions/types.rs
+8 -12
@@ -13,9 +13,9 @@ use vllm_engine_core_client::protocol::sampling::RepetitionDetectionParams;
13
14 use crate::routes::openai::utils::structured_outputs::ResponseFormat;
15 use crate::routes::openai::utils::types::{
16 - ChatLogProbs, ChatMessage, Normalizable, StreamOptions, StringOrArray, Tool, ToolCall,
17 - ToolCallDelta, ToolChoice, UNKNOWN_MODEL_ID, Usage, default_true, validate_messages,
18 - validate_stop, validate_top_p_value,
16 + ChatLogProbs, ChatMessage, Normalizable, PromptLogprobs, StreamOptions, StringOrArray, Tool,
17 + ToolCall, ToolCallDelta, ToolChoice, Usage, default_true, deserialize_request_top_k,
18 + validate_messages, validate_stop, validate_top_p_value,
19 };
20
21 /// vLLM-compatible request type for the Chat Completions API.
@@ -33,8 +33,7 @@ pub struct ChatCompletionRequest {
33 pub messages: Vec<ChatMessage>,
34
35 /// ID of the model to use
36 - #[serde(default = "default_model")]
37 - pub model: String,
36 + pub model: Option<String>,
37
38 /// Number between -2.0 and 2.0. Positive values penalize new tokens based
39 /// on their existing frequency in the text so far
@@ -120,6 +119,7 @@ pub struct ChatCompletionRequest {
119 pub use_beam_search: bool,
120
121 /// Top-k sampling parameter
122 + #[serde(default, deserialize_with = "deserialize_request_top_k")]
123 pub top_k: Option<u32>,
124
125 /// Min-p nucleus sampling parameter
@@ -145,7 +145,6 @@ pub struct ChatCompletionRequest {
145 pub ignore_eos: bool,
146
147 /// Minimum number of tokens to generate
148 - #[validate(range(min = 1))]
148 pub min_tokens: Option<u32>,
149
150 /// Skip special tokens during detokenization
@@ -233,6 +232,7 @@ pub struct ChatCompletionRequest {
232 pub return_token_ids: Option<bool>,
233
234 /// Salt for prefix cache isolation in multi-user environments
235 + #[validate(length(min = 1))]
236 pub cache_salt: Option<String>,
237
238 /// KV transfer parameters for disaggregated serving
@@ -254,7 +254,7 @@ impl Default for ChatCompletionRequest {
254 fn default() -> Self {
255 Self {
256 messages: Vec::new(),
257 - model: default_model(),
257 + model: None,
258 frequency_penalty: None,
259 logit_bias: None,
260 logprobs: false,
@@ -341,7 +341,7 @@ pub(super) struct ChatCompletionResponse {
341 pub choices: Vec<ChatCompletionChoice>,
342 pub usage: Option<Usage>,
343 pub system_fingerprint: Option<String>,
344 - pub prompt_logprobs: Option<Vec<Option<HashMap<String, f32>>>>,
344 + pub prompt_logprobs: Option<PromptLogprobs>,
345 pub prompt_token_ids: Option<Vec<u32>>,
346 pub kv_transfer_params: Option<Value>,
347 pub ec_transfer_params: Option<Value>,
@@ -429,10 +429,6 @@ pub(super) struct ChatMessageDelta {
429 pub reasoning: Option<String>,
430 }
431
432 -fn default_model() -> String {
433 - UNKNOWN_MODEL_ID.to_string()
434 -}
435 -
432 /// Schema-level validation for cross-field dependencies
433 fn validate_chat_cross_parameters(
434 req: &ChatCompletionRequest,
rust/src/server/src/routes/openai/chat_completions/validate.rs
+17 -3
@@ -10,8 +10,10 @@ pub(super) fn validate_request_compat(
10 request: &ChatCompletionRequest,
11 served_model_names: &[String],
12 ) -> Result<(), ApiError> {
13 - if !served_model_names.iter().any(|n| n == &request.model) {
14 - return Err(ApiError::model_not_found(request.model.clone()));
13 + if let Some(model) = request.model.as_ref().filter(|model| !model.is_empty())
14 + && !served_model_names.iter().any(|name| name == model)
15 + {
16 + return Err(ApiError::model_not_found(model.clone()));
17 }
18
19 if request.stream_options.is_some() && !request.stream {
@@ -142,7 +144,7 @@ mod tests {
144
145 fn base_request() -> ChatCompletionRequest {
146 ChatCompletionRequest {
145 - model: "Qwen/Qwen1.5-0.5B-Chat".to_string(),
147 + model: Some("Qwen/Qwen1.5-0.5B-Chat".to_string()),
148 messages: vec![ChatMessage::User {
149 content: MessageContent::Text("hello".to_string()),
150 name: None,
@@ -163,6 +165,18 @@ mod tests {
165 .expect("stop strings should be accepted");
166 }
167
168 + #[test]
169 + fn validate_request_compat_accepts_default_model_inputs() {
170 + for model in [None, Some(String::new())] {
171 + let request = ChatCompletionRequest {
172 + model,
173 + ..base_request()
174 + };
175 + validate_request_compat(&request, &served(&["served-model"]))
176 + .expect("default model input should select the primary served model");
177 + }
178 + }
179 +
180 #[test]
181 fn validate_request_compat_accepts_non_zero_penalties_and_function_tools() {
182 let request = ChatCompletionRequest {
rust/src/server/src/routes/openai/completions.rs
+2 -2
@@ -66,7 +66,8 @@ pub async fn completions(
66 ) -> Response {
67 let stream = body.stream;
68 let request_context = resolve_request_context(&headers, body.request_id.as_deref());
69 - let lora_resolution = state.resolve_model_with_loras(Some(&body.model)).await;
69 + let requested_model = body.model.as_deref().filter(|model| !model.is_empty());
70 + let lora_resolution = state.resolve_model_with_loras(requested_model).await;
71
72 let tokenizer = state.chat.text().tokenizer();
73 let prepared = match prepare_completion_request(
@@ -186,7 +187,6 @@ async fn collect_completion(
187 Some(prompt_logprobs_to_maps(
188 collected.prompt_logprobs.as_ref(),
189 collected.prompt_token_ids.as_ref(),
189 - return_tokens_as_token_ids,
190 )?)
191 } else {
192 None
rust/src/server/src/routes/openai/completions/convert.rs
+92 -11
@@ -40,7 +40,7 @@ pub(super) struct ResponseOptions {
40 /// Prompt text that should be echoed back northbound when `echo=true`.
41 pub echo: Option<String>,
42 /// Whether the caller requested output logprobs on completion choices.
43 - pub requested_logprobs: Option<u32>,
43 + pub requested_logprobs: Option<i32>,
44 /// Whether the caller requested choice-level prompt logprobs.
45 pub include_prompt_logprobs: bool,
46 /// Whether to include token IDs alongside generated text.
@@ -69,15 +69,7 @@ pub(super) fn prepare_completion_request(
69 .map(|request| request.lora_name.clone())
70 .unwrap_or_else(|| lora_resolution.model_names.first().cloned().unwrap_or_default());
71
72 - let logprobs = match request.logprobs {
73 - Some(logprobs) => Some(i32::try_from(logprobs).map_err(|_| {
74 - ApiError::invalid_request(
75 - "`logprobs` must fit within a signed 32-bit integer.".to_string(),
76 - Some("logprobs"),
77 - )
78 - })?),
79 - None => None,
80 - };
72 + let logprobs = request.logprobs;
73 let prompt_only = request.echo && request.max_tokens == Some(0);
74 let prompt_logprobs =
75 request.prompt_logprobs.or(if request.echo && (!request.stream || prompt_only) {
@@ -207,6 +199,7 @@ fn completion_echo_text(
199 mod tests {
200 use axum::http::HeaderMap;
201 use serde_json::json;
202 + use validator::Validate;
203 use vllm_text::Prompt;
204 use vllm_tokenizer::test_utils::TestTokenizer;
205
@@ -245,7 +238,35 @@ mod tests {
238 serde_json::from_value(base_request_json()).expect("parse request");
239
240 assert_eq!(request.prompt, Prompt::Text("hello".to_string()));
248 - assert_eq!(request.model, "Qwen/Qwen1.5-0.5B-Chat");
241 + assert_eq!(request.model.as_deref(), Some("Qwen/Qwen1.5-0.5B-Chat"));
242 + }
243 +
244 + #[test]
245 + fn completion_http_request_defaults_missing_or_null_model() {
246 + for model in [None, Some(serde_json::Value::Null)] {
247 + let mut value = base_request_json();
248 + let object = value.as_object_mut().expect("request object");
249 + match model {
250 + Some(model) => {
251 + object.insert("model".to_string(), model);
252 + }
253 + None => {
254 + object.remove("model");
255 + }
256 + }
257 +
258 + let request: CompletionRequest =
259 + serde_json::from_value(value).expect("parse request without model");
260 + assert!(request.model.is_none());
261 + }
262 +
263 + let mut value = base_request_json();
264 + value
265 + .as_object_mut()
266 + .expect("request object")
267 + .insert("model".to_string(), json!(""));
268 + let request: CompletionRequest = serde_json::from_value(value).expect("parse empty model");
269 + assert_eq!(request.model.as_deref(), Some(""));
270 }
271
272 #[test]
@@ -264,6 +285,44 @@ mod tests {
285 assert!(request.ignore_eos);
286 }
287
288 + #[test]
289 + fn completion_http_request_rejects_empty_cache_salt() {
290 + let mut value = base_request_json();
291 + value
292 + .as_object_mut()
293 + .expect("request object")
294 + .insert("cache_salt".to_string(), json!(""));
295 + let request: CompletionRequest = serde_json::from_value(value).expect("parse cache_salt");
296 +
297 + assert!(request.validate().is_err());
298 + }
299 +
300 + #[test]
301 + fn prepare_completion_request_normalizes_top_k_sentinels() {
302 + for (top_k, expected) in [
303 + (serde_json::Value::Null, None),
304 + (json!(-1), Some(0)),
305 + (json!(0), Some(0)),
306 + (json!(20), Some(20)),
307 + ] {
308 + let mut value = base_request_json();
309 + value
310 + .as_object_mut()
311 + .expect("request object")
312 + .insert("top_k".to_string(), top_k);
313 + let request: CompletionRequest = serde_json::from_value(value).expect("parse top_k");
314 + let prepared = prepare_completion_request(
315 + request,
316 + &served(&["Qwen/Qwen1.5-0.5B-Chat"]),
317 + ResolvedRequestContext::default(),
318 + &test_tokenizer(),
319 + )
320 + .expect("prepare top_k");
321 +
322 + assert_eq!(prepared.text_request.sampling_params.top_k, expected);
323 + }
324 + }
325 +
326 #[test]
327 fn normalize_coerces_null_max_tokens_to_default() {
328 // An absent `max_tokens` already gets the serde default.
@@ -572,6 +631,28 @@ mod tests {
631 );
632 }
633
634 + #[test]
635 + fn prepare_completion_request_accepts_full_vocab_logprobs() {
636 + let request: CompletionRequest = serde_json::from_value(json!({
637 + "model": "Qwen/Qwen1.5-0.5B-Chat",
638 + "prompt": "hello",
639 + "stream": false,
640 + "logprobs": -1
641 + }))
642 + .expect("parse request");
643 +
644 + let prepared = prepare_completion_request(
645 + request,
646 + &served(&["Qwen/Qwen1.5-0.5B-Chat"]),
647 + ResolvedRequestContext::default(),
648 + &test_tokenizer(),
649 + )
650 + .expect("prepare full-vocabulary logprobs");
651 +
652 + assert_eq!(prepared.text_request.sampling_params.logprobs, Some(-1));
653 + assert_eq!(prepared.options.requested_logprobs, Some(-1));
654 + }
655 +
656 #[test]
657 fn prepare_completion_request_threads_data_parallel_rank() {
658 let request: CompletionRequest = serde_json::from_value(json!({
rust/src/server/src/routes/openai/completions/types.rs
+7 -4
@@ -10,7 +10,8 @@ use vllm_engine_core_client::protocol::sampling::RepetitionDetectionParams;
10 use vllm_text::Prompt;
11
12 use crate::routes::openai::utils::types::{
13 - LogProbs, Normalizable, StreamOptions, StringOrArray, Usage, default_true, validate_stop,
13 + LogProbs, Normalizable, PromptLogprobs, StreamOptions, StringOrArray, Usage, default_true,
14 + deserialize_request_top_k, validate_stop,
15 };
16
17 /// Serde default for `CompletionRequest::max_tokens`, matching the Python vLLM
@@ -30,7 +31,7 @@ fn default_completion_max_tokens() -> Option<u32> {
31 pub struct CompletionRequest {
32 // -------- Standard OpenAI API Parameters --------
33 /// ID of the model to use
33 - pub model: String,
34 + pub model: Option<String>,
35
36 /// The prompt(s) to generate completions for.
37 ///
@@ -49,7 +50,7 @@ pub struct CompletionRequest {
50 pub logit_bias: Option<HashMap<String, f32>>,
51
52 /// Include the log probabilities on the logprobs most likely tokens
52 - pub logprobs: Option<u32>,
53 + pub logprobs: Option<i32>,
54
55 /// The maximum number of tokens to generate (defaults to 16 when absent,
56 /// matching the Python vLLM / OpenAI API convention)
@@ -96,6 +97,7 @@ pub struct CompletionRequest {
97 pub use_beam_search: bool,
98
99 /// Top-k sampling parameter
100 + #[serde(default, deserialize_with = "deserialize_request_top_k")]
101 pub top_k: Option<u32>,
102
103 /// Min-p nucleus sampling parameter
@@ -175,6 +177,7 @@ pub struct CompletionRequest {
177 pub return_token_ids: Option<bool>,
178
179 /// Salt for prefix cache isolation in multi-user environments
180 + #[validate(length(min = 1))]
181 pub cache_salt: Option<String>,
182
183 /// KV transfer parameters for disaggregated serving
@@ -229,7 +232,7 @@ pub(super) struct CompletionChoice {
232 pub logprobs: Option<LogProbs>,
233 pub finish_reason: Option<String>,
234 pub stop_reason: Option<Value>,
232 - pub prompt_logprobs: Option<Vec<Option<HashMap<String, f32>>>>,
235 + pub prompt_logprobs: Option<PromptLogprobs>,
236 pub token_ids: Option<Vec<u32>>,
237 pub prompt_token_ids: Option<Vec<u32>>,
238 }
rust/src/server/src/routes/openai/completions/validate.rs
+40 -4
@@ -12,8 +12,10 @@ pub(super) fn validate_request_compat(
12 // This path is intentionally scoped to the minimum surface needed by
13 // `vllm-bench` random workload compatibility, so unsupported legacy
14 // completions features fail early here.
15 - if !served_model_names.iter().any(|n| n == &request.model) {
16 - return Err(ApiError::model_not_found(request.model.clone()));
15 + if let Some(model) = request.model.as_ref().filter(|model| !model.is_empty())
16 + && !served_model_names.iter().any(|name| name == model)
17 + {
18 + return Err(ApiError::model_not_found(model.clone()));
19 }
20
21 if request.stream_options.is_some() && !request.stream {
@@ -39,11 +41,12 @@ pub(super) fn validate_request_compat(
41 }
42
43 if let Some(logprobs) = request.logprobs
42 - && logprobs > i32::MAX as u32
44 + && logprobs < 0
45 + && logprobs != -1
46 {
47 bail_invalid_request!(
48 param = "logprobs",
46 - "`logprobs` must fit within a signed 32-bit integer."
49 + "`logprobs` must be a non-negative value or -1."
50 );
51 }
52
@@ -123,6 +126,28 @@ mod tests {
126 );
127 }
128
129 + #[test]
130 + fn validate_request_compat_accepts_full_vocab_logprobs() {
131 + let request = CompletionRequest {
132 + logprobs: Some(-1),
133 + ..base_request()
134 + };
135 + assert!(
136 + validate_request_compat(&request, &served_names(&["Qwen/Qwen1.5-0.5B-Chat"])).is_ok()
137 + );
138 + }
139 +
140 + #[test]
141 + fn validate_request_compat_rejects_other_negative_logprobs() {
142 + let request = CompletionRequest {
143 + logprobs: Some(-2),
144 + ..base_request()
145 + };
146 + assert!(
147 + validate_request_compat(&request, &served_names(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err()
148 + );
149 + }
150 +
151 #[test]
152 fn validate_request_compat_accepts_any_served_name() {
153 let request = base_request();
@@ -135,6 +160,17 @@ mod tests {
160 );
161 }
162
163 + #[test]
164 + fn validate_request_compat_accepts_default_model_inputs() {
165 + for model in [None, Some(String::new())] {
166 + let request = CompletionRequest {
167 + model,
168 + ..base_request()
169 + };
170 + assert!(validate_request_compat(&request, &served_names(&["served-model"])).is_ok());
171 + }
172 + }
173 +
174 #[test]
175 fn validate_request_compat_rejects_unknown_model() {
176 let request = base_request();
rust/src/server/src/routes/openai/utils/logprobs.rs
+43 -8
@@ -9,7 +9,9 @@ use vllm_text::{
9 DecodedTokenLogprob,
10 };
11
12 -use super::types::{ChatLogProbs, ChatLogProbsContent, LogProbs, TopLogProb};
12 +use super::types::{
13 + ChatLogProbs, ChatLogProbsContent, LogProbs, PromptLogprob, PromptLogprobs, TopLogProb,
14 +};
15 use crate::error::{ApiError, server_error};
16
17 /// Convert decoded token-position logprobs into the OpenAI completions
@@ -105,15 +107,26 @@ pub fn decoded_prompt_logprobs_to_openai(
107 pub fn prompt_logprobs_to_maps(
108 prompt_logprobs: Option<&DecodedPromptLogprobs>,
109 prompt_token_ids: &[u32],
108 - return_tokens_as_token_ids: bool,
109 -) -> Result<Vec<Option<HashMap<String, f32>>>, ApiError> {
110 +) -> Result<PromptLogprobs, ApiError> {
111 if let Some(prompt_logprobs) = prompt_logprobs {
112 return Ok(std::iter::once(None)
113 .chain(prompt_logprobs.scored_positions.iter().map(|position| {
113 - Some(position_top_logprobs_map(
114 - position,
115 - return_tokens_as_token_ids,
116 - ))
114 + Some(
115 + position
116 + .entries
117 + .iter()
118 + .map(|entry| {
119 + (
120 + entry.token_id,
121 + PromptLogprob {
122 + logprob: clamp_logprob(entry.logprob),
123 + rank: entry.rank,
124 + decoded_token: entry.token.clone(),
125 + },
126 + )
127 + })
128 + .collect(),
129 + )
130 }))
131 .collect());
132 }
@@ -290,10 +303,32 @@ mod tests {
303
304 #[test]
305 fn prompt_logprobs_maps_reject_missing_multi_token_payload() {
293 - prompt_logprobs_to_maps(None, &[9707, 11], false)
306 + prompt_logprobs_to_maps(None, &[9707, 11])
307 .expect_err("multi-token prompt without payload is an engine failure");
308 }
309
310 + #[test]
311 + fn prompt_logprobs_maps_preserve_token_ids_and_metadata() {
312 + let prompt_logprobs = vllm_text::DecodedPromptLogprobs {
313 + first_token_id: 10,
314 + first_token: "first".to_string(),
315 + scored_positions: sample_logprobs().positions,
316 + };
317 +
318 + let maps = prompt_logprobs_to_maps(Some(&prompt_logprobs), &[10, 1]).expect("maps");
319 + assert_eq!(
320 + serde_json::to_value(maps).expect("serialize prompt logprobs"),
321 + serde_json::json!([
322 + null,
323 + {
324 + "1": {"logprob": -0.10000000149011612_f64, "rank": 1, "decoded_token": "A"},
325 + "2": {"logprob": -1.0, "rank": 2, "decoded_token": "B"},
326 + "3": {"logprob": -2.0, "rank": 3, "decoded_token": "C"},
327 + }
328 + ])
329 + );
330 + }
331 +
332 fn sample_logprobs() -> DecodedLogprobs {
333 DecodedLogprobs {
334 positions: vec![DecodedPositionLogprobs {
rust/src/server/src/routes/openai/utils/types.rs
+34 -4
@@ -5,7 +5,7 @@ use std::collections::HashMap;
5 use std::slice;
6
7 use llm_multimodal::ImageDetail;
8 -use serde::{Deserialize, Serialize};
8 +use serde::{Deserialize, Deserializer, Serialize};
9 use serde_json::Value;
10 use vllm_llm::TokenUsage;
11
@@ -13,9 +13,6 @@ use vllm_llm::TokenUsage;
13 // Constants
14 // ============================================================================
15
16 -/// Default model identifier used when no model is specified.
17 -pub const UNKNOWN_MODEL_ID: &str = "unknown";
18 -
16 // ============================================================================
17 // Default value helpers
18 // ============================================================================
@@ -25,6 +22,23 @@ pub fn default_true() -> bool {
22 true
23 }
24
25 +/// Deserialize an OpenAI request `top_k` while preserving explicit disable.
26 +///
27 +/// Null remains `None` so model generation defaults apply. Explicit `-1` and
28 +/// `0` become `Some(0)` so the request overrides those defaults and disables
29 +/// top-k sampling. Positive limits are preserved.
30 +pub fn deserialize_request_top_k<'de, D>(deserializer: D) -> Result<Option<u32>, D::Error>
31 +where
32 + D: Deserializer<'de>,
33 +{
34 + match Option::<i64>::deserialize(deserializer)? {
35 + None => Ok(None),
36 + Some(value) => vllm_text::normalize_top_k(value)
37 + .map(|value| Some(value.unwrap_or(0)))
38 + .map_err(serde::de::Error::custom),
39 + }
40 +}
41 +
42 // ============================================================================
43 // String/Array Utilities
44 // ============================================================================
@@ -201,15 +215,21 @@ pub struct StreamOptions {
215 #[derive(Debug, Clone, Deserialize, Serialize)]
216 pub struct Tool {
217 #[serde(rename = "type")]
218 + #[serde(default = "default_function_tool_type")]
219 pub tool_type: String,
220 pub function: Function,
221 }
222
223 +fn default_function_tool_type() -> String {
224 + "function".to_string()
225 +}
226 +
227 #[serde_with::skip_serializing_none]
228 #[derive(Debug, Clone, Deserialize, Serialize)]
229 pub struct Function {
230 pub name: String,
231 pub description: Option<String>,
232 + #[serde(default)]
233 pub parameters: Value,
234 /// Whether to enable strict schema adherence (OpenAI structured outputs).
235 pub strict: Option<bool>,
@@ -480,6 +500,16 @@ pub struct LogProbs {
500 pub text_offset: Vec<u32>,
501 }
502
503 +/// vLLM prompt-logprob metadata keyed by vocabulary token ID.
504 +#[derive(Debug, Clone, PartialEq, Serialize)]
505 +pub struct PromptLogprob {
506 + pub logprob: f32,
507 + pub rank: u32,
508 + pub decoded_token: String,
509 +}
510 +
511 +pub type PromptLogprobs = Vec<Option<HashMap<u32, PromptLogprob>>>;
512 +
513 /// Mirrors the Python vLLM `ChatCompletionLogProbs` class.
514 #[derive(Debug, Clone, Serialize)]
515 pub struct ChatLogProbs {
rust/src/server/src/routes/render.rs
+10 -2
@@ -68,6 +68,14 @@ fn model_resolution(state: &RenderState) -> LoraModelResolution {
68 }
69 }
70
71 +fn response_model(state: &RenderState, requested_model: Option<&str>) -> String {
72 + requested_model
73 + .filter(|model| !model.is_empty())
74 + .or_else(|| state.served_model_names.first().map(String::as_str))
75 + .unwrap_or_default()
76 + .to_string()
77 +}
78 +
79 fn lower_render_request(
80 state: &RenderState,
81 text_request: TextRequest,
@@ -108,7 +116,7 @@ async fn render_chat(
116 headers: HeaderMap,
117 ValidatedJson(body): ValidatedJson<ChatCompletionRequest>,
118 ) -> Result<Json<GenerateRequest>, ApiError> {
111 - let model = body.model.clone();
119 + let model = response_model(&state, body.model.as_deref());
120 let stream = body.stream;
121 let stream_options = body.stream_options.clone();
122 let request_context = resolve_request_context(&headers, body.request_id.as_deref());
@@ -132,7 +140,7 @@ async fn render_completion(
140 headers: HeaderMap,
141 ValidatedJson(body): ValidatedJson<CompletionRequest>,
142 ) -> Result<Json<Vec<GenerateRequest>>, ApiError> {
135 - let model = body.model.clone();
143 + let model = response_model(&state, body.model.as_deref());
144 let stream = body.stream;
145 let stream_options = body.stream_options.clone();
146 let request_context = resolve_request_context(&headers, body.request_id.as_deref());
rust/src/server/src/routes/tests.rs
+4 -1
@@ -3654,7 +3654,10 @@ async fn non_stream_completions_include_prompt_logprobs() {
3654 );
3655 assert_eq!(
3656 json["choices"][0]["prompt_logprobs"][1],
3657 - json!({"a": -0.5, "e": -0.3})
3657 + json!({
3658 + "97": {"logprob": -0.5, "rank": 1, "decoded_token": "a"},
3659 + "101": {"logprob": -0.3, "rank": 1, "decoded_token": "e"},
3660 + })
3661 );
3662 }
3663
rust/src/text/src/backend/hf/config.rs
+9 -12
@@ -9,6 +9,7 @@ use serde::{Deserialize, Serialize};
9 use thiserror_ext::AsReport as _;
10
11 use crate::error::{Error, Result};
12 +use crate::normalize_top_k;
13
14 /// Minimal subset of `tokenizer_config.json` needed by chat/EOS handling.
15 #[derive(Debug, Default, Deserialize)]
@@ -118,23 +119,19 @@ pub(super) struct GenerationConfig {
119 pub max_new_tokens: Option<u32>,
120 }
121
121 -/// Deserialize vLLM-compatible `top_k` values from generation configs.
122 +/// Deserialize a generation-config `top_k` into the text sampling model.
123 ///
123 -/// Both `-1` and `0` disable top-k sampling and become `None`; positive values
124 -/// are preserved.
124 +/// Null, `-1`, and `0` all disable top-k sampling; positive limits are
125 +/// preserved.
126 fn deserialize_top_k<'de, D>(deserializer: D) -> std::result::Result<Option<u32>, D::Error>
127 where
128 D: serde::Deserializer<'de>,
129 {
129 - match Option::<i64>::deserialize(deserializer)? {
130 - None | Some(-1) | Some(0) => Ok(None),
131 - Some(value) if value > 0 => {
132 - u32::try_from(value).map(Some).map_err(serde::de::Error::custom)
133 - }
134 - Some(value) => Err(serde::de::Error::custom(format!(
135 - "top_k must be -1, 0, or a positive integer, got {value}"
136 - ))),
137 - }
130 + Option::<i64>::deserialize(deserializer)?
131 + .map(normalize_top_k)
132 + .transpose()
133 + .map(Option::flatten)
134 + .map_err(serde::de::Error::custom)
135 }
136
137 /// HF generation configs allow either one EOS id or a list of EOS ids.
rust/src/text/src/lib.rs
+1 -1
@@ -21,7 +21,7 @@ pub use output::{
21 CollectedTextOutput, DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs,
22 DecodedTextEvent, DecodedTokenLogprob, Finished, TextDecodeOptions, TextOutputStreamExt,
23 };
24 -pub use request::{Prompt, SamplingParams, TextRequest};
24 +pub use request::{Prompt, SamplingParams, TextRequest, normalize_top_k};
25 use trait_set::trait_set;
26 use vllm_engine_core_client::EngineCoreClient;
27 pub use vllm_llm::FinishReason;
rust/src/text/src/lower.rs
+14
@@ -434,6 +434,20 @@ mod tests {
434 ));
435 }
436
437 + #[test]
438 + fn lower_sampling_params_preserves_zero_min_tokens() {
439 + let params = lower_sampling_params_with_limits(
440 + SamplingParams {
441 + min_tokens: Some(0),
442 + ..SamplingParams::default()
443 + },
444 + sample_sampling_limits(),
445 + )
446 + .expect("lower zero min_tokens");
447 +
448 + assert_eq!(params.min_tokens, 0);
449 + }
450 +
451 #[test]
452 fn lower_sampling_params_validates_repetition_detection() {
453 let lower = |repetition_detection| {
rust/src/text/src/request.rs
+18
@@ -15,6 +15,24 @@ use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsPara
15 use crate::error::{Error, Result};
16 use crate::output::TextDecodeOptions;
17
18 +/// Normalize vLLM's signed `top_k` domain into [`SamplingParams::top_k`].
19 +///
20 +/// vLLM uses both `-1` and `0` to disable top-k sampling. The text sampling
21 +/// model represents that state as `None` and preserves positive limits as
22 +/// `Some(k)`. Values below `-1` and values above `u32::MAX` are rejected.
23 +///
24 +/// Callers that need to distinguish an omitted value from an explicit disable
25 +/// should preserve that request-level distinction around this normalization.
26 +pub fn normalize_top_k(value: i64) -> std::result::Result<Option<u32>, String> {
27 + match value {
28 + -1 | 0 => Ok(None),
29 + value if value > 0 => u32::try_from(value).map(Some).map_err(|error| error.to_string()),
30 + value => Err(format!(
31 + "top_k must be -1, 0, or a positive integer, got {value}"
32 + )),
33 + }
34 +}
35 +
36 /// One raw text-generation prompt.
37 ///
38 /// This supports either ordinary text that still needs tokenization or