@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dac9e9a640
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

[Rust Frontend] Extract shared options in route helper params (#44884)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>

Bugen Zhao committed Jun 10, 2026 at 08:02 UTC dac9e9a6401103384aaef307970bcfb7ceec50b4
9 files changed +224 -279
rust/src/server/src/lib.rs
+2 -1
@@ -14,7 +14,8 @@ mod utils;
14 use std::sync::{Arc, OnceLock};
15
16 use anyhow::{Context as _, Result};
17 -use axum::{Router, serve::ListenerExt as _};
17 +use axum::Router;
18 +use axum::serve::ListenerExt as _;
19 pub use config::{Config, CoordinatorMode, HttpListenerMode};
20 use tokio::net::TcpListener;
21 use tokio::time::{Instant, sleep_until};
rust/src/server/src/routes/inference/generate.rs
+44 -30
@@ -22,7 +22,7 @@ use vllm_llm::{
22 CollectedGenerateOutput, FinishReason, GenerateOutput, GenerateOutputStreamExt as _,
23 };
24
25 -use self::convert::prepare_generate_request;
25 +use self::convert::{ResponseOptions, prepare_generate_request};
26 use self::types::{
27 GenerateLogprob, GenerateRequest, GenerateResponse, GenerateResponseChoice,
28 GenerateResponseStreamChoice, GenerateStreamResponse,
@@ -54,10 +54,7 @@ pub async fn generate(
54 );
55
56 let log_request = state.enable_log_requests;
57 - let include_logprobs = prepared.include_logprobs;
58 - let include_prompt_logprobs = prepared.include_prompt_logprobs;
57 let stream = prepared.stream;
60 -
58 let raw_stream = match state
59 .chat
60 .text()
@@ -80,9 +77,7 @@ pub async fn generate(
77 raw_stream,
78 prepared.request_id,
79 log_request,
83 - prepared.include_usage,
84 - prepared.include_continuous_usage,
85 - include_logprobs,
80 + prepared.options,
81 );
82 let sse_stream = generate_sse_stream(chunk_stream).instrument(request_span);
83
@@ -100,21 +95,11 @@ pub async fn generate(
95 }
96 };
97
103 - if log_request {
104 - info!(
105 - parent: &request_span,
106 - prompt_tokens = collected.prompt_token_ids.len(),
107 - output_tokens = collected.token_ids.len(),
108 - finish_reason = collected.finish_reason.as_str(),
109 - "generate finished"
110 - );
111 - }
112 -
98 let response = match collect_generate(
99 collected,
100 prepared.request_id,
116 - include_logprobs,
117 - include_prompt_logprobs,
101 + log_request,
102 + prepared.options,
103 ) {
104 Ok(response) => response,
105 Err(error) => return error.into_response(),
@@ -128,9 +113,13 @@ async fn generate_chunk_stream(
113 stream: impl Stream<Item = vllm_llm::Result<GenerateOutput>>,
114 request_id: String,
115 log_request: bool,
131 - include_usage: bool,
132 - include_continuous_usage: bool,
133 - include_logprobs: bool,
116 + ResponseOptions {
117 + include_usage,
118 + include_continuous_usage,
119 + include_logprobs,
120 + // Ignored: raw generate streaming has no prompt-logprobs wire shape.
121 + include_prompt_logprobs: _,
122 + }: ResponseOptions,
123 mut y: TryYielder<GenerateStreamResponse, ApiError>,
124 ) -> Result<(), ApiError> {
125 pin_mut!(stream);
@@ -222,8 +211,15 @@ async fn generate_chunk_stream(
211 fn collect_generate(
212 collected: CollectedGenerateOutput,
213 request_id: String,
225 - include_logprobs: bool,
226 - include_prompt_logprobs: bool,
214 + log_request: bool,
215 + ResponseOptions {
216 + // Ignored: non-streaming raw generate responses do not include usage.
217 + include_usage: _,
218 + // Ignored: continuous usage is a streaming-only option.
219 + include_continuous_usage: _,
220 + include_logprobs,
221 + include_prompt_logprobs,
222 + }: ResponseOptions,
223 ) -> Result<GenerateResponse, ApiError> {
224 let logprobs = if include_logprobs {
225 let logprobs = collected.logprobs.as_ref().ok_or_else(|| {
@@ -246,13 +242,23 @@ fn collect_generate(
242 } else {
243 None
244 };
245 + let finish_reason = collected.finish_reason.as_str().to_string();
246 +
247 + if log_request {
248 + info!(
249 + prompt_tokens = collected.prompt_token_ids.len(),
250 + output_tokens = collected.token_ids.len(),
251 + %finish_reason,
252 + "generate finished"
253 + );
254 + }
255
256 Ok(GenerateResponse {
257 request_id,
258 choices: vec![GenerateResponseChoice {
259 index: 0,
260 logprobs,
255 - finish_reason: Some(collected.finish_reason.as_str().to_string()),
261 + finish_reason: Some(finish_reason),
262 token_ids: collected.token_ids,
263 }],
264 prompt_logprobs,
@@ -408,11 +414,19 @@ mod tests {
414 }),
415 ]);
416
411 - let chunks: Vec<_> =
412 - generate_chunk_stream(stream, "raw-stream".to_string(), false, true, true, false)
413 - .try_collect()
414 - .await
415 - .expect("collect chunks");
417 + let chunks: Vec<_> = generate_chunk_stream(
418 + stream,
419 + "raw-stream".to_string(),
420 + false,
421 + ResponseOptions {
422 + include_usage: true,
423 + include_continuous_usage: true,
424 + ..Default::default()
425 + },
426 + )
427 + .try_collect()
428 + .await
429 + .expect("collect chunks");
430
431 assert_eq!(chunks.len(), 2);
432 assert_eq!(
rust/src/server/src/routes/inference/generate/convert.rs
+20 -8
@@ -8,19 +8,29 @@ use crate::utils::{ResolvedRequestContext, merge_kv_transfer_params};
8
9 /// Lowered generate request plus the response request ID.
10 #[derive(Debug, Clone, PartialEq)]
11 -pub struct PreparedRequest {
11 +pub(super) struct PreparedRequest {
12 pub request_id: String,
13 pub text_request: TextRequest,
14 pub stream: bool,
15 + /// Public response rendering options for route-layer helpers.
16 + pub options: ResponseOptions,
17 +}
18 +
19 +#[derive(Debug, Clone, Copy, Default, PartialEq)]
20 +pub(super) struct ResponseOptions {
21 + /// Whether the caller asked for the final streamed usage chunk.
22 pub include_usage: bool,
23 + /// Whether the caller asked for usage on every streamed chunk.
24 pub include_continuous_usage: bool,
25 + /// Whether the caller requested output logprobs on generate choices.
26 pub include_logprobs: bool,
27 + /// Whether the caller requested top-level prompt logprobs.
28 pub include_prompt_logprobs: bool,
29 }
30
31 /// Validate and lower one raw generate request into the internal
32 /// text-generation format.
23 -pub fn prepare_generate_request(
33 +pub(super) fn prepare_generate_request(
34 request: GenerateRequest,
35 lora_resolution: &LoraModelResolution,
36 ctx: ResolvedRequestContext,
@@ -65,10 +75,12 @@ pub fn prepare_generate_request(
75 request_id: ctx.request_id,
76 text_request,
77 stream,
68 - include_usage,
69 - include_continuous_usage,
70 - include_logprobs,
71 - include_prompt_logprobs,
78 + options: ResponseOptions {
79 + include_usage,
80 + include_continuous_usage,
81 + include_logprobs,
82 + include_prompt_logprobs,
83 + },
84 })
85 }
86
@@ -158,7 +170,7 @@ mod tests {
170 )
171 .expect("prepare");
172
161 - assert!(!prepared.include_usage);
162 - assert!(!prepared.include_continuous_usage);
173 + assert!(!prepared.options.include_usage);
174 + assert!(!prepared.options.include_continuous_usage);
175 }
176 }
rust/src/server/src/routes/openai/chat_completions.rs
+62 -75
@@ -1,4 +1,4 @@
1 -pub mod convert;
1 +mod convert;
2 mod types;
3 mod validate;
4
@@ -23,8 +23,8 @@ use vllm_chat::{
23 };
24 use vllm_engine_core_client::protocol::StopReason;
25
26 +use self::convert::{ResponseOptions, prepare_chat_request};
27 use crate::error::{ApiError, bail_server_error, server_error};
27 -use crate::routes::openai::chat_completions::convert::prepare_chat_request;
28 use crate::routes::openai::chat_completions::types::{
29 AssistantRole, ChatCompletionChoice, ChatCompletionMessage, ChatCompletionRequest,
30 ChatCompletionResponse, ChatCompletionStreamChoice, ChatCompletionStreamResponse,
@@ -83,12 +83,7 @@ pub async fn chat_completions(
83 prepared.response_model,
84 created,
85 log_request,
86 - prepared.include_usage,
87 - prepared.requested_logprobs,
88 - prepared.include_reasoning,
89 - prepared.echo,
90 - prepared.return_token_ids,
91 - prepared.return_tokens_as_token_ids,
86 + prepared.options,
87 );
88 let sse_stream = chat_completion_sse_stream(chunk_stream).instrument(request_span);
89
@@ -99,12 +94,8 @@ pub async fn chat_completions(
94 prepared.request_id,
95 prepared.response_model,
96 created,
102 - prepared.requested_logprobs,
103 - prepared.include_prompt_logprobs,
104 - prepared.include_reasoning,
105 - prepared.echo,
106 - prepared.return_token_ids,
107 - prepared.return_tokens_as_token_ids,
97 + log_request,
98 + prepared.options,
99 )
100 .instrument(request_span.clone())
101 .await
@@ -113,18 +104,6 @@ pub async fn chat_completions(
104 Err(error) => return error.into_response(),
105 };
106
116 - if log_request {
117 - let usage = response.usage.as_ref();
118 - info!(
119 - parent: &request_span,
120 - model = %response.model,
121 - prompt_tokens = usage.map_or(0, |u| u.prompt_tokens),
122 - output_tokens = usage.and_then(|u| u.completion_tokens).unwrap_or(0),
123 - finish_reason = response.choices.first().and_then(|c| c.finish_reason.as_deref()).unwrap_or("unknown"),
124 - "chat completion finished"
125 - );
126 - }
127 -
107 Json(response).into_response()
108 }
109 }
@@ -134,12 +113,17 @@ async fn collect_chat_completion(
113 request_id: String,
114 response_model: String,
115 created: u64,
137 - requested_logprobs: bool,
138 - include_prompt_logprobs: bool,
139 - include_reasoning: bool,
140 - echo: Option<String>,
141 - return_token_ids: bool,
142 - return_tokens_as_token_ids: bool,
116 + log_request: bool,
117 + ResponseOptions {
118 + // Ignored: non-streaming responses always include usage.
119 + include_usage: _,
120 + requested_logprobs,
121 + include_prompt_logprobs,
122 + include_reasoning,
123 + echo,
124 + return_token_ids,
125 + return_tokens_as_token_ids,
126 + }: ResponseOptions,
127 ) -> Result<ChatCompletionResponse, ApiError> {
128 let collected = stream.collect_message().await.map_err(|error| {
129 server_error!(
@@ -201,6 +185,16 @@ async fn collect_chat_completion(
185 };
186 let usage = Usage::from_counts(prompt_token_count as u32, output_token_count as u32);
187
188 + if log_request {
189 + info!(
190 + model = %response_model,
191 + prompt_tokens = usage.prompt_tokens,
192 + output_tokens = usage.completion_tokens.unwrap_or(0),
193 + finish_reason = %finish_reason,
194 + "chat completion finished"
195 + );
196 + }
197 +
198 Ok(ChatCompletionResponse {
199 id: request_id,
200 object: "chat.completion".to_string(),
@@ -238,12 +232,16 @@ async fn chat_completion_chunk_stream(
232 response_model: String,
233 created: u64,
234 log_request: bool,
241 - include_usage: bool,
242 - requested_logprobs: bool,
243 - include_reasoning: bool,
244 - echo: Option<String>,
245 - return_token_ids: bool,
246 - return_tokens_as_token_ids: bool,
235 + ResponseOptions {
236 + include_usage,
237 + requested_logprobs,
238 + // Ignored: chat streaming prompt logprobs are rejected for Python parity.
239 + include_prompt_logprobs: _,
240 + include_reasoning,
241 + echo,
242 + return_token_ids,
243 + return_tokens_as_token_ids,
244 + }: ResponseOptions,
245 mut y: TryYielder<ChatCompletionStreamResponse, ApiError>,
246 ) -> Result<(), ApiError> {
247 let mut saw_tool_calls = false;
@@ -806,7 +804,7 @@ mod tests {
804 use vllm_engine_core_client::protocol::StopReason;
805 use vllm_text::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTokenLogprob};
806
809 - use super::{block_delta_chunk, chat_completion_chunk_stream, final_chunk};
807 + use super::{ResponseOptions, block_delta_chunk, chat_completion_chunk_stream, final_chunk};
808
809 #[test]
810 fn text_chunk_uses_content_only_delta() {
@@ -932,12 +930,11 @@ mod tests {
930 "model".to_string(),
931 1,
932 false,
935 - false,
936 - true,
937 - true,
938 - None,
939 - false,
940 - false,
933 + ResponseOptions {
934 + requested_logprobs: true,
935 + include_reasoning: true,
936 + ..Default::default()
937 + },
938 )
939 .collect::<Vec<_>>()
940 .await
@@ -996,12 +993,11 @@ mod tests {
993 "model".to_string(),
994 1,
995 false,
999 - false,
1000 - true,
1001 - true,
1002 - None,
1003 - false,
1004 - false,
996 + ResponseOptions {
997 + requested_logprobs: true,
998 + include_reasoning: true,
999 + ..Default::default()
1000 + },
1001 )
1002 .collect::<Vec<_>>()
1003 .await
@@ -1049,12 +1045,7 @@ mod tests {
1045 "model".to_string(),
1046 1,
1047 false,
1052 - false,
1053 - false,
1054 - false,
1055 - None,
1056 - false,
1057 - false,
1048 + ResponseOptions::default(),
1049 )
1050 .collect::<Vec<_>>()
1051 .await
@@ -1132,12 +1123,11 @@ mod tests {
1123 "model".to_string(),
1124 1,
1125 false,
1135 - false,
1136 - true,
1137 - false,
1138 - None,
1139 - true,
1140 - false,
1126 + ResponseOptions {
1127 + requested_logprobs: true,
1128 + return_token_ids: true,
1129 + ..Default::default()
1130 + },
1131 )
1132 .collect::<Vec<_>>()
1133 .await
@@ -1263,12 +1253,11 @@ mod tests {
1253 "model".to_string(),
1254 1,
1255 false,
1266 - false,
1267 - true,
1268 - false,
1269 - None,
1270 - true,
1271 - false,
1256 + ResponseOptions {
1257 + requested_logprobs: true,
1258 + return_token_ids: true,
1259 + ..Default::default()
1260 + },
1261 )
1262 .collect::<Vec<_>>()
1263 .await
@@ -1342,12 +1331,10 @@ mod tests {
1331 "model".to_string(),
1332 1,
1333 false,
1345 - false,
1346 - false,
1347 - true,
1348 - None,
1349 - false,
1350 - false,
1334 + ResponseOptions {
1335 + include_reasoning: true,
1336 + ..Default::default()
1337 + },
1338 )
1339 .collect::<Vec<_>>()
1340 .await
rust/src/server/src/routes/openai/chat_completions/convert.rs
+23 -15
@@ -18,11 +18,19 @@ use crate::utils::{ResolvedRequestContext, convert_logit_bias, merge_kv_transfer
18 /// Lowered chat request plus the public response metadata carried by every SSE
19 /// chunk.
20 #[derive(Debug, Clone, PartialEq)]
21 -pub struct PreparedRequest {
21 +pub(super) struct PreparedRequest {
22 /// Stable OpenAI-style request ID, reused as the external chat request ID.
23 pub request_id: String,
24 /// Public model ID echoed back to the client.
25 pub response_model: String,
26 + /// Public response rendering options for route-layer helpers.
27 + pub options: ResponseOptions,
28 + /// Lowered chat request for `vllm-chat`.
29 + pub chat_request: ChatRequest,
30 +}
31 +
32 +#[derive(Debug, Clone, Default, PartialEq)]
33 +pub(super) struct ResponseOptions {
34 /// Whether the caller asked for the final streamed usage chunk.
35 pub include_usage: bool,
36 /// Whether the caller requested output logprobs on chat choices.
@@ -31,8 +39,6 @@ pub struct PreparedRequest {
39 pub include_prompt_logprobs: bool,
40 /// Whether to include reasoning content in OpenAI responses.
41 pub include_reasoning: bool,
34 - /// Lowered chat request for `vllm-chat`.
35 - pub chat_request: ChatRequest,
42 /// Last assistant-role message content to echo back when `echo=true`.
43 pub echo: Option<String>,
44 /// Whether to include token IDs alongside generated text.
@@ -46,7 +52,7 @@ pub struct PreparedRequest {
52 ///
53 /// `lora_resolution.model_names` must be non-empty; the first entry is used as
54 /// the base `model` field in responses when no LoRA adapter is selected.
49 -pub(crate) fn prepare_chat_request(
55 +pub(super) fn prepare_chat_request(
56 request: ChatCompletionRequest,
57 lora_resolution: &LoraModelResolution,
58 ctx: ResolvedRequestContext,
@@ -146,14 +152,16 @@ pub(crate) fn prepare_chat_request(
152 Ok(PreparedRequest {
153 request_id,
154 response_model,
149 - include_usage,
150 - requested_logprobs,
151 - include_prompt_logprobs,
152 - include_reasoning,
155 + options: ResponseOptions {
156 + include_usage,
157 + requested_logprobs,
158 + include_prompt_logprobs,
159 + include_reasoning,
160 + echo,
161 + return_token_ids: request.return_token_ids.unwrap_or(false),
162 + return_tokens_as_token_ids: request.return_tokens_as_token_ids.unwrap_or(false),
163 + },
164 chat_request,
154 - echo,
155 - return_token_ids: request.return_token_ids.unwrap_or(false),
156 - return_tokens_as_token_ids: request.return_tokens_as_token_ids.unwrap_or(false),
165 })
166 }
167 pub(crate) fn normalize_generation_prompt_mode(
@@ -497,7 +505,7 @@ mod tests {
505 )
506 .expect("request is valid");
507
500 - assert!(!prepared.include_reasoning);
508 + assert!(!prepared.options.include_reasoning);
509 }
510
511 #[test]
@@ -867,8 +875,8 @@ mod tests {
875 )
876 .expect("request is valid");
877
870 - assert!(prepared.requested_logprobs);
871 - assert!(prepared.include_prompt_logprobs);
878 + assert!(prepared.options.requested_logprobs);
879 + assert!(prepared.options.include_prompt_logprobs);
880 assert_eq!(prepared.chat_request.sampling_params.logprobs, Some(0));
881 assert_eq!(
882 prepared.chat_request.sampling_params.prompt_logprobs,
@@ -894,7 +902,7 @@ mod tests {
902
903 assert_eq!(prepared.chat_request.sampling_params.logprobs, Some(3));
904 assert_eq!(prepared.chat_request.sampling_params.prompt_logprobs, None);
897 - assert!(!prepared.include_prompt_logprobs);
905 + assert!(!prepared.options.include_prompt_logprobs);
906 }
907
908 #[test]
rust/src/server/src/routes/openai/completions.rs
+45 -47
@@ -18,13 +18,13 @@ use tracing::{debug, error, info, trace};
18 use tracing_futures::Instrument as _;
19 use vllm_text::{DecodedTextEvent, FinishReason, TextOutputStream, TextOutputStreamExt as _};
20
21 +use self::convert::{ResponseOptions, prepare_completion_request};
22 use super::utils::logprobs::{
23 collected_logprobs_to_openai, decoded_logprobs_to_openai, decoded_prompt_logprobs_to_maps,
24 text_len,
25 };
26 use super::utils::types::Usage;
27 use crate::error::{ApiError, bail_server_error, server_error};
27 -use crate::routes::openai::completions::convert::prepare_completion_request;
28 use crate::routes::openai::completions::types::{
29 CompletionChoice, CompletionRequest, CompletionResponse, CompletionSseChunk,
30 CompletionStreamChoice, CompletionStreamResponse,
@@ -42,7 +42,6 @@ pub async fn completions(
42 ValidatedJson(body): ValidatedJson<CompletionRequest>,
43 ) -> Response {
44 let stream = body.stream;
45 - let logprobs = body.logprobs;
45 let request_context = resolve_request_context(&headers, body.request_id.as_deref());
46 let lora_resolution = state.resolve_model_with_loras(Some(&body.model)).await;
47
@@ -57,9 +56,7 @@ pub async fn completions(
56 );
57
58 let created = unix_timestamp();
60 - let include_prompt_logprobs = prepared.text_request.sampling_params.prompt_logprobs.is_some();
59 let log_request = state.enable_log_requests;
62 -
60 let text_stream = match state
61 .chat
62 .text()
@@ -84,11 +81,7 @@ pub async fn completions(
81 prepared.response_model,
82 created,
83 log_request,
87 - prepared.include_usage,
88 - prepared.echo,
89 - logprobs,
90 - prepared.return_token_ids,
91 - prepared.return_tokens_as_token_ids,
84 + prepared.options,
85 );
86 let sse_stream = completion_sse_stream(chunk_stream).instrument(request_span);
87
@@ -99,11 +92,8 @@ pub async fn completions(
92 prepared.request_id,
93 prepared.response_model,
94 created,
102 - prepared.echo,
103 - logprobs,
104 - include_prompt_logprobs,
105 - prepared.return_token_ids,
106 - prepared.return_tokens_as_token_ids,
95 + log_request,
96 + prepared.options,
97 )
98 .instrument(request_span.clone())
99 .await
@@ -112,18 +102,6 @@ pub async fn completions(
102 Err(error) => return error.into_response(),
103 };
104
115 - if log_request {
116 - let usage = response.usage.as_ref();
117 - info!(
118 - parent: &request_span,
119 - model = %response.model,
120 - prompt_tokens = usage.map_or(0, |u| u.prompt_tokens),
121 - output_tokens = usage.and_then(|u| u.completion_tokens).unwrap_or(0),
122 - finish_reason = response.choices.first().and_then(|c| c.finish_reason.as_deref()).unwrap_or("unknown"),
123 - "completion finished"
124 - );
125 - }
126 -
105 Json(response).into_response()
106 }
107 }
@@ -133,11 +111,16 @@ async fn collect_completion(
111 request_id: String,
112 response_model: String,
113 created: u64,
136 - echo: Option<String>,
137 - requested_logprobs: Option<u32>,
138 - include_prompt_logprobs: bool,
139 - return_token_ids: bool,
140 - return_tokens_as_token_ids: bool,
114 + log_request: bool,
115 + ResponseOptions {
116 + // Ignored: non-streaming responses always include usage.
117 + include_usage: _,
118 + echo,
119 + requested_logprobs,
120 + include_prompt_logprobs,
121 + return_token_ids,
122 + return_tokens_as_token_ids,
123 + }: ResponseOptions,
124 ) -> Result<CompletionResponse, ApiError> {
125 let collected = stream
126 .collect_output()
@@ -175,6 +158,21 @@ async fn collect_completion(
158 None => collected.text,
159 Some(prompt) => format!("{prompt}{}", collected.text),
160 };
161 + let finish_reason = completion_finish_reason_to_openai(finish_reason)?.to_string();
162 + let usage = Usage::from_counts(
163 + collected.prompt_token_ids.len() as u32,
164 + collected.token_ids.len() as u32,
165 + );
166 +
167 + if log_request {
168 + info!(
169 + model = %response_model,
170 + prompt_tokens = usage.prompt_tokens,
171 + output_tokens = usage.completion_tokens.unwrap_or(0),
172 + %finish_reason,
173 + "completion finished"
174 + );
175 + }
176
177 Ok(CompletionResponse {
178 id: request_id,
@@ -185,16 +183,13 @@ async fn collect_completion(
183 index: 0,
184 text,
185 logprobs,
188 - finish_reason: Some(completion_finish_reason_to_openai(finish_reason)?.into()),
186 + finish_reason: Some(finish_reason),
187 stop_reason,
188 prompt_logprobs,
189 token_ids: return_token_ids.then(|| collected.token_ids.clone()),
190 prompt_token_ids: return_token_ids.then(|| collected.prompt_token_ids.to_vec()),
191 }],
194 - usage: Some(Usage::from_counts(
195 - collected.prompt_token_ids.len() as u32,
196 - collected.token_ids.len() as u32,
197 - )),
192 + usage: Some(usage),
193 system_fingerprint: None,
194 kv_transfer_params: collected.kv_transfer_params,
195 })
@@ -208,11 +203,15 @@ async fn completion_chunk_stream(
203 response_model: String,
204 created: u64,
205 log_request: bool,
211 - include_usage: bool,
212 - echo: Option<String>,
213 - requested_logprobs: Option<u32>,
214 - return_token_ids: bool,
215 - return_tokens_as_token_ids: bool,
206 + ResponseOptions {
207 + include_usage,
208 + echo,
209 + requested_logprobs,
210 + // Ignored: streaming prompt logprobs are rejected for Python parity.
211 + include_prompt_logprobs: _,
212 + return_token_ids,
213 + return_tokens_as_token_ids,
214 + }: ResponseOptions,
215 mut y: TryYielder<CompletionSseChunk, ApiError>,
216 ) -> Result<(), ApiError> {
217 pin_mut!(stream);
@@ -432,7 +431,7 @@ mod tests {
431 FinishReason, Finished,
432 };
433
435 - use super::{CompletionSseChunk, completion_chunk_stream, final_chunk};
434 + use super::{CompletionSseChunk, ResponseOptions, completion_chunk_stream, final_chunk};
435
436 #[test]
437 fn final_chunk_maps_stop_finish_reason() {
@@ -527,11 +526,10 @@ mod tests {
526 "model".to_string(),
527 1,
528 false,
530 - false,
531 - None,
532 - Some(1),
533 - false,
534 - false,
529 + ResponseOptions {
530 + requested_logprobs: Some(1),
531 + ..Default::default()
532 + },
533 )
534 .collect::<Vec<_>>()
535 .await;
rust/src/server/src/routes/openai/completions/convert.rs
+25 -10
@@ -10,18 +10,28 @@ use crate::utils::{ResolvedRequestContext, convert_logit_bias, merge_kv_transfer
10 /// Lowered completion request plus the public response metadata carried by
11 /// every SSE chunk.
12 #[derive(Debug, Clone, PartialEq)]
13 -pub struct PreparedRequest {
13 +pub(super) struct PreparedRequest {
14 /// Stable OpenAI-style request ID, reused as the external text request ID.
15 pub request_id: String,
16 /// Public model ID echoed back to the client.
17 pub response_model: String,
18 - /// Whether the caller asked for the final streamed usage chunk.
19 - pub include_usage: bool,
18 + /// Public response rendering options for route-layer helpers.
19 + pub options: ResponseOptions,
20 /// Lowered text request for the shared `vllm-text` facade.
21 pub text_request: TextRequest,
22 +}
23 +
24 +#[derive(Debug, Clone, Default, PartialEq)]
25 +pub(super) struct ResponseOptions {
26 + /// Whether the caller asked for the final streamed usage chunk.
27 + pub include_usage: bool,
28 /// Original text prompt that should be echoed back northbound when
29 /// `echo=true`.
30 pub echo: Option<String>,
31 + /// Whether the caller requested output logprobs on completion choices.
32 + pub requested_logprobs: Option<u32>,
33 + /// Whether the caller requested choice-level prompt logprobs.
34 + pub include_prompt_logprobs: bool,
35 /// Whether to include token IDs alongside generated text.
36 pub return_token_ids: bool,
37 /// Whether to format logprob tokens as `token_id:{id}`.
@@ -33,7 +43,7 @@ pub struct PreparedRequest {
43 ///
44 /// `lora_resolution.model_names` must be non-empty; the first entry is used as
45 /// the base `model` field in responses when no LoRA adapter is selected.
36 -pub(crate) fn prepare_completion_request(
46 +pub(super) fn prepare_completion_request(
47 request: CompletionRequest,
48 lora_resolution: &LoraModelResolution,
49 ctx: ResolvedRequestContext,
@@ -64,6 +74,7 @@ pub(crate) fn prepare_completion_request(
74 let include_usage = (request.stream_options.as_ref())
75 .and_then(|options| options.include_usage)
76 .unwrap_or(false);
77 + let include_prompt_logprobs = prompt_logprobs.is_some();
78 let echo = request.echo.then(|| request.prompt.as_text().cloned()).flatten();
79
80 let structured_outputs =
@@ -116,11 +127,15 @@ pub(crate) fn prepare_completion_request(
127 Ok(PreparedRequest {
128 request_id,
129 response_model,
119 - include_usage,
130 + options: ResponseOptions {
131 + include_usage,
132 + echo,
133 + requested_logprobs: request.logprobs,
134 + include_prompt_logprobs,
135 + return_token_ids: request.return_token_ids.unwrap_or(false),
136 + return_tokens_as_token_ids: request.return_tokens_as_token_ids.unwrap_or(false),
137 + },
138 text_request,
121 - echo,
122 - return_token_ids: request.return_token_ids.unwrap_or(false),
123 - return_tokens_as_token_ids: request.return_tokens_as_token_ids.unwrap_or(false),
139 })
140 }
141
@@ -206,7 +221,7 @@ mod tests {
221 )
222 .expect("prepare");
223
209 - assert!(prepared.include_usage);
224 + assert!(prepared.options.include_usage);
225 assert_eq!(
226 prepared.text_request.prompt,
227 Prompt::TokenIds(vec![11, 22, 33])
@@ -250,7 +265,7 @@ mod tests {
265 )
266 .expect("prepare");
267
253 - assert_eq!(prepared.echo, Some("hello".to_string()));
268 + assert_eq!(prepared.options.echo, Some("hello".to_string()));
269 assert_eq!(prepared.text_request.sampling_params.max_tokens, Some(7));
270 }
271
rust/src/server/src/routes/tests.rs
+3 -92
@@ -14,15 +14,14 @@ use std::{fmt, fs};
14 use axum::body::{Body, to_bytes};
15 use axum::http::{Request, StatusCode};
16 use bytes::Bytes;
17 -use futures::StreamExt as _;
17 use rmpv::Value;
18 use serde_json::json;
19 use serial_test::serial;
20 use tower::{Service as _, ServiceExt as _};
21 use vllm_chat::{
23 - ChatBackend, ChatContent, ChatContentPart, ChatEvent, ChatLlm, ChatMessage, ChatRenderer,
24 - ChatRequest, ChatRole, ChatTextBackend, DefaultChatOutputProcessor, DynChatOutputProcessor,
25 - DynChatRenderer, NewChatOutputProcessorOptions, SamplingParams,
22 + ChatBackend, ChatContent, ChatContentPart, ChatLlm, ChatMessage, ChatRenderer, ChatRequest,
23 + ChatTextBackend, DefaultChatOutputProcessor, DynChatOutputProcessor, DynChatRenderer,
24 + NewChatOutputProcessorOptions,
25 };
26 use vllm_engine_core_client::protocol::logprobs::{
27 Logprobs, MaybeWireLogprobs, PositionLogprobs, TokenLogprob,
@@ -44,8 +43,6 @@ use zeromq::prelude::{SocketRecv, SocketSend};
43 use zeromq::{DealerSocket, PushSocket, ZmqMessage};
44
45 use super::{build_router, build_router_with_dev_mode, build_router_with_dev_mode_and_lora};
47 -use crate::lora::LoraModelResolution;
48 -use crate::routes::openai::chat_completions::convert::prepare_chat_request;
46 use crate::state::AppState;
47
48 fn request_output(
@@ -3478,92 +3475,6 @@ async fn completions_echo_stream_emits_separate_prompt_chunk() {
3475 assert_eq!(usage_chunk["usage"]["completion_tokens"], 3);
3476 }
3477
3481 -#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3482 -#[serial]
3483 -async fn chat_harness_streams_text_events() {
3484 - let (chat, engine_task) = test_chat_with_engine_handle().await;
3485 - let mut stream = chat
3486 - .chat(ChatRequest {
3487 - messages: vec![ChatMessage::text(ChatRole::User, "hello")],
3488 - sampling_params: SamplingParams {
3489 - max_tokens: Some(8),
3490 - ..Default::default()
3491 - },
3492 - request_id: "chat-harness".to_string(),
3493 - ..ChatRequest::for_test()
3494 - })
3495 - .await
3496 - .expect("submit chat request");
3497 -
3498 - let mut saw_text = false;
3499 - let mut saw_done = false;
3500 - while let Some(event) = stream.next().await {
3501 - match event.expect("chat event") {
3502 - ChatEvent::BlockDelta { .. } => saw_text = true,
3503 - ChatEvent::Done { .. } => {
3504 - saw_done = true;
3505 - break;
3506 - }
3507 - ChatEvent::Start { .. }
3508 - | ChatEvent::LogprobsDelta { .. }
3509 - | ChatEvent::BlockStart { .. }
3510 - | ChatEvent::BlockEnd { .. }
3511 - | ChatEvent::ToolCallStart { .. }
3512 - | ChatEvent::ToolCallArgumentsDelta { .. }
3513 - | ChatEvent::ToolCallEnd { .. } => {}
3514 - }
3515 - }
3516 - engine_task.await.expect("mock engine task");
3517 -
3518 - assert!(saw_text);
3519 - assert!(saw_done);
3520 -}
3521 -
3522 -#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3523 -#[serial]
3524 -async fn prepared_openai_request_streams_text_events() {
3525 - let (chat, engine_task) = test_chat_with_engine_handle().await;
3526 - let prepared = prepare_chat_request(
3527 - serde_json::from_value(json!({
3528 - "model": "Qwen/Qwen1.5-0.5B-Chat",
3529 - "stream": true,
3530 - "messages": [{"role": "user", "content": "hello"}]
3531 - }))
3532 - .expect("decode request"),
3533 - &LoraModelResolution {
3534 - model_names: vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
3535 - lora_request: None,
3536 - },
3537 - crate::utils::ResolvedRequestContext::default(),
3538 - )
3539 - .expect("prepare request");
3540 -
3541 - let mut stream = chat.chat(prepared.chat_request).await.expect("submit chat request");
3542 -
3543 - let mut saw_text = false;
3544 - let mut saw_done = false;
3545 - while let Some(event) = stream.next().await {
3546 - match event.expect("chat event") {
3547 - ChatEvent::BlockDelta { .. } => saw_text = true,
3548 - ChatEvent::Done { .. } => {
3549 - saw_done = true;
3550 - break;
3551 - }
3552 - ChatEvent::Start { .. }
3553 - | ChatEvent::LogprobsDelta { .. }
3554 - | ChatEvent::BlockStart { .. }
3555 - | ChatEvent::BlockEnd { .. }
3556 - | ChatEvent::ToolCallStart { .. }
3557 - | ChatEvent::ToolCallArgumentsDelta { .. }
3558 - | ChatEvent::ToolCallEnd { .. } => {}
3559 - }
3560 - }
3561 - engine_task.await.expect("mock engine task");
3562 -
3563 - assert!(saw_text);
3564 - assert!(saw_done);
3565 -}
3566 -
3478 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3479 #[serial]
3480 async fn reasoning_blocks_are_mapped_to_reasoning_sse_chunks() {
rust/src/server/src/state.rs
-1
@@ -10,7 +10,6 @@ use vllm_engine_core_client::EngineCoreClient;
10 use vllm_engine_core_client::protocol::lora::LoraRequest;
11
12 use crate::lora::{LoadLoraError, LoraManager, LoraModelResolution, UnloadLoraError};
13 -
13 use crate::server_info::{ServerInfoConfigFormat, ServerInfoSnapshot};
14
15 const SHUTDOWN_REFCOUNT_POLL_INTERVAL: Duration = Duration::from_millis(100);