@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e64237ae82

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 1, 21:15 UTC

[Rust Frontend] Support include_reasoning=false (#44391)

Signed-off-by: RickyChen / 陳昭儒 <ricky.chen@infinirc.com>

Chao-Ju Chen committed Jun 5, 2026 at 16:47 UTC e64237ae82c9fa8abf9fabf96c2c4a9dcbfcd040

4 files changed +544 -26

rust/src/server/src/routes/openai/chat_completions.rs

+348 -20

@@ -85,6 +85,7 @@ pub async fn chat_completions(

85 log_request,

86 prepared.include_usage,

87 prepared.requested_logprobs,

88 + prepared.include_reasoning,

89 prepared.echo,

90 prepared.return_token_ids,

91 prepared.return_tokens_as_token_ids,

@@ -100,6 +101,7 @@ pub async fn chat_completions(

101 created,

102 prepared.requested_logprobs,

103 prepared.include_prompt_logprobs,

104 + prepared.include_reasoning,

105 prepared.echo,

106 prepared.return_token_ids,

107 prepared.return_tokens_as_token_ids,

@@ -134,6 +136,7 @@ async fn collect_chat_completion(

136 created: u64,

137 requested_logprobs: bool,

138 include_prompt_logprobs: bool,

139 + include_reasoning: bool,

140 echo: Option<String>,

141 return_token_ids: bool,

142 return_tokens_as_token_ids: bool,

@@ -157,6 +160,11 @@ async fn collect_chat_completion(

160 } = collected;

161 let stop_reason = finish_reason.as_stop_reason().map(stop_reason_to_json);

162 let saw_tool_calls = message.tool_calls().next().is_some();

163 + let reasoning = message.reasoning();

164 + // Output logprobs and token IDs cover the complete generated token stream.

165 + // When reasoning is hidden, omit them rather than leaking hidden reasoning

166 + // tokens through per-token metadata.

167 + let include_output_metadata = include_reasoning || reasoning.is_none();

168 let finish_reason = chat_finish_reason_to_openai(&finish_reason, saw_tool_calls)?.to_string();

169 let tool_calls = message

170 .tool_calls()

@@ -169,7 +177,7 @@ async fn collect_chat_completion(

177 },

178 })

179 .collect::<Vec<_>>();

172 - let logprobs = if requested_logprobs {

180 + let logprobs = if requested_logprobs && include_output_metadata {

181 Some(decoded_logprobs_to_openai_chat(

182 logprobs.as_ref().ok_or_else(|| {

183 server_error!("chat response requested logprobs but generation returned none")

@@ -207,12 +215,12 @@ async fn collect_chat_completion(

215 None => Some(message.text()).filter(|t| !t.is_empty()),

216 },

217 tool_calls: Some(tool_calls).filter(|calls| !calls.is_empty()),

210 - reasoning: message.reasoning(),

218 + reasoning: if include_reasoning { reasoning } else { None },

219 },

220 logprobs,

221 finish_reason: Some(finish_reason),

222 stop_reason,

215 - token_ids: return_token_ids.then_some(token_ids),

223 + token_ids: (return_token_ids && include_output_metadata).then_some(token_ids),

224 }],

225 usage: Some(usage),

226 system_fingerprint: None,

@@ -232,12 +240,18 @@ async fn chat_completion_chunk_stream(

240 log_request: bool,

241 include_usage: bool,

242 requested_logprobs: bool,

243 + include_reasoning: bool,

244 echo: Option<String>,

245 return_token_ids: bool,

246 return_tokens_as_token_ids: bool,

247 mut y: TryYielder<ChatCompletionStreamResponse, ApiError>,

248 ) -> Result<(), ApiError> {

249 let mut saw_tool_calls = false;

250 + // `LogprobsDelta` is emitted after all chat events for one decoded update.

251 + // If that update contains hidden reasoning, including delimiter-only block

252 + // starts or ends, omit its token metadata as well as its visible delta.

253 + let mut inside_hidden_reasoning = false;

254 + let mut suppress_current_update_metadata = false;

255

256 // If the client requested logprobs or token_ids, we need to buffer chunks until

257 // we receive the separate `LogprobsDelta` event, so that we can emit one

@@ -268,29 +282,44 @@ async fn chat_completion_chunk_stream(

282 }

283 }

284 Ok(ChatEvent::BlockDelta { kind, delta, .. }) => {

271 - if let Some(pending_chunk) = pending_chunk.as_mut() {

272 - pending_chunk.push_block_delta(kind, delta);

285 + let include_delta =

286 + include_reasoning || !matches!(kind, AssistantBlockKind::Reasoning);

287 + if include_delta {

288 + if let Some(pending_chunk) = pending_chunk.as_mut() {

289 + pending_chunk.push_block_delta(kind, delta);

290 + } else {

291 + y.yield_ok(block_delta_chunk(

292 + &request_id,

293 + &response_model,

294 + created,

295 + kind,

296 + delta,

297 + ))

298 + .await;

299 + }

300 } else {

274 - y.yield_ok(block_delta_chunk(

275 - &request_id,

276 - &response_model,

277 - created,

278 - kind,

279 - delta,

280 - ))

281 - .await;

301 + suppress_current_update_metadata = true;

302 }

303 }

304 Ok(ChatEvent::LogprobsDelta {

305 logprobs,

306 token_ids,

307 }) => {

288 - let openai_logprobs = logprobs

289 - .as_ref()

290 - .map(|lp| decoded_logprobs_to_openai_chat(lp, return_tokens_as_token_ids))

291 - .transpose()?;

292 - let openai_token_ids =

293 - return_token_ids.then_some(token_ids).filter(|t| !t.is_empty());

308 + let include_metadata =

309 + !suppress_current_update_metadata && !inside_hidden_reasoning;

310 + suppress_current_update_metadata = false;

311 + let openai_logprobs = if include_metadata {

312 + logprobs

313 + .as_ref()

314 + .map(|lp| decoded_logprobs_to_openai_chat(lp, return_tokens_as_token_ids))

315 + .transpose()?

316 + } else {

317 + None

318 + };

319 + let openai_token_ids = include_metadata

320 + .then_some(token_ids)

321 + .and_then(|token_ids| return_token_ids.then_some(token_ids))

322 + .filter(|t| !t.is_empty());

323 if let Some(pending_chunk) = pending_chunk.as_mut() {

324 pending_chunk.logprobs = openai_logprobs;

325 pending_chunk.token_ids = openai_token_ids;

@@ -311,9 +340,17 @@ async fn chat_completion_chunk_stream(

340 }

341 Ok(ChatEvent::BlockStart { kind, .. }) => {

342 debug!(?kind, "starting new block");

343 + if !include_reasoning && matches!(kind, AssistantBlockKind::Reasoning) {

344 + inside_hidden_reasoning = true;

345 + suppress_current_update_metadata = true;

346 + }

347 }

348 Ok(ChatEvent::BlockEnd { .. }) => {

349 debug!("ending current block");

350 + if inside_hidden_reasoning {

351 + inside_hidden_reasoning = false;

352 + suppress_current_update_metadata = true;

353 + }

354 }

355 Ok(ChatEvent::ToolCallStart { index, id, name }) => {

356 let tool_index = index as u32;

@@ -763,7 +800,9 @@ fn stop_reason_to_json(stop_reason: &StopReason) -> Value {

800 mod tests {

801 use futures::{StreamExt as _, stream};

802 use serde_json::json;

766 - use vllm_chat::{AssistantBlockKind, AssistantToolCall, ChatEvent, FinishReason};

803 + use vllm_chat::{

804 + AssistantBlockKind, AssistantContentBlock, AssistantToolCall, ChatEvent, FinishReason,

805 + };

806 use vllm_engine_core_client::protocol::StopReason;

807 use vllm_text::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTokenLogprob};

808

@@ -895,6 +934,7 @@ mod tests {

934 false,

935 false,

936 true,

937 + true,

938 None,

939 false,

940 false,

@@ -958,6 +998,7 @@ mod tests {

998 false,

999 false,

1000 true,

1001 + true,

1002 None,

1003 false,

1004 false,

@@ -976,6 +1017,292 @@ mod tests {

1017 assert!(chunks[1].choices[0].logprobs.is_some());

1018 }

1019

1020 + #[tokio::test]

1021 + async fn chunk_stream_omits_reasoning_delta_when_disabled() {

1022 + let stream = stream::iter(vec![

1023 + Ok(ChatEvent::Start {

1024 + prompt_token_ids: vec![].into(),

1025 + prompt_logprobs: None,

1026 + }),

1027 + Ok(ChatEvent::BlockDelta {

1028 + index: 0,

1029 + kind: AssistantBlockKind::Reasoning,

1030 + delta: "think".to_string(),

1031 + }),

1032 + Ok(ChatEvent::BlockDelta {

1033 + index: 1,

1034 + kind: AssistantBlockKind::Text,

1035 + delta: "answer".to_string(),

1036 + }),

1037 + Ok(ChatEvent::Done {

1038 + message: Default::default(),

1039 + prompt_token_count: 1,

1040 + output_token_count: 2,

1041 + finish_reason: FinishReason::stop_eos(),

1042 + kv_transfer_params: None,

1043 + }),

1044 + ]);

1045 +

1046 + let chunks = chat_completion_chunk_stream(

1047 + stream,

1048 + "chatcmpl-1".to_string(),

1049 + "model".to_string(),

1050 + 1,

1051 + false,

1052 + false,

1053 + false,

1054 + false,

1055 + None,

1056 + false,

1057 + false,

1058 + )

1059 + .collect::<Vec<_>>()

1060 + .await

1061 + .into_iter()

1062 + .collect::<Result<Vec<_>, _>>()

1063 + .expect("stream chunks");

1064 +

1065 + assert_eq!(chunks.len(), 3);

1066 + assert_eq!(

1067 + chunks[1].choices[0].delta.content.as_deref(),

1068 + Some("answer")

1069 + );

1070 + assert!(

1071 + chunks

1072 + .iter()

1073 + .all(|chunk| chunk.choices.iter().all(|choice| choice.delta.reasoning.is_none()))

1074 + );

1075 + }

1076 +

1077 + #[tokio::test]

1078 + async fn chunk_stream_omits_logprobs_for_suppressed_reasoning() {

1079 + let stream = stream::iter(vec![

1080 + Ok(ChatEvent::Start {

1081 + prompt_token_ids: vec![].into(),

1082 + prompt_logprobs: None,

1083 + }),

1084 + Ok(ChatEvent::BlockDelta {

1085 + index: 0,

1086 + kind: AssistantBlockKind::Reasoning,

1087 + delta: "think".to_string(),

1088 + }),

1089 + Ok(ChatEvent::LogprobsDelta {

1090 + logprobs: Some(DecodedLogprobs {

1091 + positions: vec![DecodedPositionLogprobs {

1092 + entries: vec![DecodedTokenLogprob {

1093 + token_id: 11,

1094 + token: "think".to_string(),

1095 + logprob: -0.1,

1096 + rank: 1,

1097 + }],

1098 + }],

1099 + }),

1100 + token_ids: vec![11],

1101 + }),

1102 + Ok(ChatEvent::BlockDelta {

1103 + index: 1,

1104 + kind: AssistantBlockKind::Text,

1105 + delta: "answer".to_string(),

1106 + }),

1107 + Ok(ChatEvent::LogprobsDelta {

1108 + logprobs: Some(DecodedLogprobs {

1109 + positions: vec![DecodedPositionLogprobs {

1110 + entries: vec![DecodedTokenLogprob {

1111 + token_id: 22,

1112 + token: "answer".to_string(),

1113 + logprob: -0.2,

1114 + rank: 1,

1115 + }],

1116 + }],

1117 + }),

1118 + token_ids: vec![22],

1119 + }),

1120 + Ok(ChatEvent::Done {

1121 + message: Default::default(),

1122 + prompt_token_count: 1,

1123 + output_token_count: 2,

1124 + finish_reason: FinishReason::stop_eos(),

1125 + kv_transfer_params: None,

1126 + }),

1127 + ]);

1128 +

1129 + let chunks = chat_completion_chunk_stream(

1130 + stream,

1131 + "chatcmpl-1".to_string(),

1132 + "model".to_string(),

1133 + 1,

1134 + false,

1135 + false,

1136 + true,

1137 + false,

1138 + None,

1139 + true,

1140 + false,

1141 + )

1142 + .collect::<Vec<_>>()

1143 + .await

1144 + .into_iter()

1145 + .collect::<Result<Vec<_>, _>>()

1146 + .expect("stream chunks");

1147 +

1148 + assert_eq!(chunks.len(), 3);

1149 + let choice = &chunks[1].choices[0];

1150 + assert_eq!(choice.delta.content.as_deref(), Some("answer"));

1151 + assert_eq!(choice.token_ids.as_deref(), Some(&[22][..]));

1152 + let logprobs = choice.logprobs.as_ref().expect("answer logprobs");

1153 + let content = logprobs.content.as_ref().expect("logprobs content");

1154 + assert_eq!(content[0].token, "answer");

1155 + assert!(chunks.iter().all(|chunk| {

1156 + chunk.choices.iter().all(|choice| {

1157 + choice.delta.reasoning.is_none()

1158 + && choice.token_ids.as_deref() != Some(&[11][..])

1159 + && choice

1160 + .logprobs

1161 + .as_ref()

1162 + .and_then(|logprobs| logprobs.content.as_ref())

1163 + .is_none_or(|content| content.iter().all(|entry| entry.token != "think"))

1164 + })

1165 + }));

1166 + }

1167 +

1168 + #[tokio::test]

1169 + async fn chunk_stream_omits_logprobs_for_hidden_reasoning_delimiters() {

1170 + let stream = stream::iter(vec![

1171 + Ok(ChatEvent::Start {

1172 + prompt_token_ids: vec![].into(),

1173 + prompt_logprobs: None,

1174 + }),

1175 + Ok(ChatEvent::BlockStart {

1176 + index: 0,

1177 + kind: AssistantBlockKind::Reasoning,

1178 + }),

1179 + Ok(ChatEvent::LogprobsDelta {

1180 + logprobs: Some(DecodedLogprobs {

1181 + positions: vec![DecodedPositionLogprobs {

1182 + entries: vec![DecodedTokenLogprob {

1183 + token_id: 11,

1184 + token: "<think>".to_string(),

1185 + logprob: -0.1,

1186 + rank: 1,

1187 + }],

1188 + }],

1189 + }),

1190 + token_ids: vec![11],

1191 + }),

1192 + Ok(ChatEvent::BlockDelta {

1193 + index: 0,

1194 + kind: AssistantBlockKind::Reasoning,

1195 + delta: "think".to_string(),

1196 + }),

1197 + Ok(ChatEvent::LogprobsDelta {

1198 + logprobs: Some(DecodedLogprobs {

1199 + positions: vec![DecodedPositionLogprobs {

1200 + entries: vec![DecodedTokenLogprob {

1201 + token_id: 12,

1202 + token: "think".to_string(),

1203 + logprob: -0.2,

1204 + rank: 1,

1205 + }],

1206 + }],

1207 + }),

1208 + token_ids: vec![12],

1209 + }),

1210 + Ok(ChatEvent::BlockEnd {

1211 + index: 0,

1212 + block: AssistantContentBlock::Reasoning {

1213 + text: "think".to_string(),

1214 + },

1215 + }),

1216 + Ok(ChatEvent::LogprobsDelta {

1217 + logprobs: Some(DecodedLogprobs {

1218 + positions: vec![DecodedPositionLogprobs {

1219 + entries: vec![DecodedTokenLogprob {

1220 + token_id: 13,

1221 + token: "</think>".to_string(),

1222 + logprob: -0.3,

1223 + rank: 1,

1224 + }],

1225 + }],

1226 + }),

1227 + token_ids: vec![13],

1228 + }),

1229 + Ok(ChatEvent::BlockStart {

1230 + index: 1,

1231 + kind: AssistantBlockKind::Text,

1232 + }),

1233 + Ok(ChatEvent::BlockDelta {

1234 + index: 1,

1235 + kind: AssistantBlockKind::Text,

1236 + delta: "answer".to_string(),

1237 + }),

1238 + Ok(ChatEvent::LogprobsDelta {

1239 + logprobs: Some(DecodedLogprobs {

1240 + positions: vec![DecodedPositionLogprobs {

1241 + entries: vec![DecodedTokenLogprob {

1242 + token_id: 22,

1243 + token: "answer".to_string(),

1244 + logprob: -0.4,

1245 + rank: 1,

1246 + }],

1247 + }],

1248 + }),

1249 + token_ids: vec![22],

1250 + }),

1251 + Ok(ChatEvent::Done {

1252 + message: Default::default(),

1253 + prompt_token_count: 1,

1254 + output_token_count: 4,

1255 + finish_reason: FinishReason::stop_eos(),

1256 + kv_transfer_params: None,

1257 + }),

1258 + ]);

1259 +

1260 + let chunks = chat_completion_chunk_stream(

1261 + stream,

1262 + "chatcmpl-1".to_string(),

1263 + "model".to_string(),

1264 + 1,

1265 + false,

1266 + false,

1267 + true,

1268 + false,

1269 + None,

1270 + true,

1271 + false,

1272 + )

1273 + .collect::<Vec<_>>()

1274 + .await

1275 + .into_iter()

1276 + .collect::<Result<Vec<_>, _>>()

1277 + .expect("stream chunks");

1278 +

1279 + assert_eq!(chunks.len(), 3);

1280 + let choice = &chunks[1].choices[0];

1281 + assert_eq!(choice.delta.content.as_deref(), Some("answer"));

1282 + assert_eq!(choice.token_ids.as_deref(), Some(&[22][..]));

1283 + let logprobs = choice.logprobs.as_ref().expect("answer logprobs");

1284 + let content = logprobs.content.as_ref().expect("logprobs content");

1285 + assert_eq!(content[0].token, "answer");

1286 + assert!(chunks.iter().all(|chunk| {

1287 + chunk.choices.iter().all(|choice| {

1288 + choice.delta.reasoning.is_none()

1289 + && !choice

1290 + .token_ids

1291 + .as_ref()

1292 + .is_some_and(|ids| matches!(ids.as_slice(), [11] | [12] | [13]))

1293 + && choice

1294 + .logprobs

1295 + .as_ref()

1296 + .and_then(|logprobs| logprobs.content.as_ref())

1297 + .is_none_or(|content| {

1298 + content.iter().all(|entry| {

1299 + !matches!(entry.token.as_str(), "<think>" | "think" | "</think>")

1300 + })

1301 + })

1302 + })

1303 + }));

1304 + }

1305 +

1306 #[tokio::test]

1307 async fn chunk_stream_preserves_tool_call_index_and_omits_id_from_arguments_delta() {

1308 let stream = stream::iter(vec![

@@ -1017,6 +1344,7 @@ mod tests {

1344 false,

1345 false,

1346 false,

1347 + true,

1348 None,

1349 false,

1350 false,

rust/src/server/src/routes/openai/chat_completions/convert.rs

+21

@@ -29,6 +29,8 @@ pub struct PreparedRequest {

29 pub requested_logprobs: bool,

30 /// Whether the caller requested top-level prompt logprobs.

31 pub include_prompt_logprobs: bool,

32 + /// Whether to include reasoning content in OpenAI responses.

33 + pub include_reasoning: bool,

34 /// Lowered chat request for `vllm-chat`.

35 pub chat_request: ChatRequest,

36 /// Last assistant-role message content to echo back when `echo=true`.

@@ -57,6 +59,7 @@ pub(crate) fn prepare_chat_request(

59 .as_ref()

60 .map(|request| request.lora_name.clone())

61 .unwrap_or_else(|| lora_resolution.model_names.first().cloned().unwrap_or_default());

62 + let include_reasoning = request.include_reasoning;

63 let echo = request

64 .echo

65 .then(|| extract_last_assistant_content(&request.messages))

@@ -146,6 +149,7 @@ pub(crate) fn prepare_chat_request(

149 include_usage,

150 requested_logprobs,

151 include_prompt_logprobs,

152 + include_reasoning,

153 chat_request,

154 echo,

155 return_token_ids: request.return_token_ids.unwrap_or(false),

@@ -480,6 +484,23 @@ mod tests {

484 assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::Auto);

485 }

486

487 + #[test]

488 + fn prepare_chat_request_preserves_include_reasoning_false() {

489 + let request = ChatCompletionRequest {

490 + include_reasoning: false,

491 + ..base_request()

492 + };

493 +

494 + let prepared = prepare_chat_request(

495 + request,

496 + &served(&["Qwen/Qwen1.5-0.5B-Chat"]),

497 + ResolvedRequestContext::default(),

498 + )

499 + .expect("request is valid");

500 +

501 + assert!(!prepared.include_reasoning);

502 + }

503 +

504 #[test]

505 fn prepare_chat_request_preserves_sampling_passthrough_fields() {

506 let request = ChatCompletionRequest {

rust/src/server/src/routes/openai/chat_completions/validate.rs

+11 -6

@@ -120,12 +120,6 @@ pub(super) fn validate_request_compat(

120 "thinking_token_budget",

121 "thinking_token_budget is not supported.",

122 )?;

123 - if !request.include_reasoning {

124 - bail_invalid_request!(

125 - param = "include_reasoning",

126 - "include_reasoning is not supported."

127 - );

128 - }

123 reject_non_default(

124 request.media_io_kwargs.as_ref(),

125 "media_io_kwargs",

@@ -312,6 +306,17 @@ mod tests {

306 .expect("reasoning_effort should be accepted");

307 }

308

309 + #[test]

310 + fn validate_request_compat_accepts_include_reasoning_false() {

311 + let request = ChatCompletionRequest {

312 + include_reasoning: false,

313 + ..base_request()

314 + };

315 +

316 + validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]))

317 + .expect("include_reasoning=false should be accepted");

318 + }

319 +

320 #[test]

321 fn validate_request_compat_rejects_top_logprobs_without_logprobs() {

322 let request = ChatCompletionRequest {

rust/src/server/src/routes/tests.rs

+164

@@ -3492,6 +3492,170 @@ async fn reasoning_blocks_are_mapped_to_reasoning_sse_chunks() {

3492 assert!(text.contains("\"content\":\"answer\""), "{text}");

3493 }

3494

3495 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]

3496 +#[serial]

3497 +async fn include_reasoning_false_suppresses_reasoning_in_non_stream_chat() {

3498 + let (app, engine_task) = test_app_with_backend_and_stream_output_specs(

3499 + Arc::new(FakeChatBackend::with_model_id("Qwen/Qwen3-0.6B")),

3500 + vec![

3501 + (bytes_to_token_ids(b"<think>think</think>"), None),

3502 + (

3503 + bytes_to_token_ids(b"answer"),

3504 + Some(EngineCoreFinishReason::Length),

3505 + ),

3506 + ],

3507 + )

3508 + .await;

3509 +

3510 + let response = app

3511 + .clone()

3512 + .call(

3513 + Request::builder()

3514 + .method("POST")

3515 + .uri("/v1/chat/completions")

3516 + .header("content-type", "application/json")

3517 + .body(Body::from(

3518 + json!({

3519 + "model": "Qwen/Qwen1.5-0.5B-Chat",

3520 + "stream": false,

3521 + "include_reasoning": false,

3522 + "messages": [{"role": "user", "content": "hello"}]

3523 + })

3524 + .to_string(),

3525 + ))

3526 + .expect("build request"),

3527 + )

3528 + .await

3529 + .expect("call app");

3530 +

3531 + assert_eq!(response.status(), StatusCode::OK);

3532 + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");

3533 + engine_task.await.expect("mock engine task");

3534 + let text = String::from_utf8(body.to_vec()).expect("utf8 body");

3535 + let json: serde_json::Value = serde_json::from_str(&text).expect("decode json");

3536 +

3537 + assert_eq!(json["choices"][0]["message"]["content"], "answer");

3538 + assert!(

3539 + json["choices"][0]["message"]

3540 + .as_object()

3541 + .is_some_and(|message| !message.contains_key("reasoning")),

3542 + "{text}"

3543 + );

3544 +}

3545 +

3546 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]

3547 +#[serial]

3548 +async fn include_reasoning_false_suppresses_non_stream_output_metadata() {

3549 + let ipc = IpcNamespace::new().expect("create ipc namespace");

3550 + let handshake_address = ipc.handshake_endpoint();

3551 + let engine_id = b"engine-openai-hidden-reasoning-logprobs".to_vec();

3552 +

3553 + let engine_task = MockEngineTask::new(spawn_mock_engine_task(

3554 + handshake_address.clone(),

3555 + engine_id.clone(),

3556 + |dealer, push| {

3557 + boxed_test_future(async move {

3558 + let add = recv_engine_message(dealer).await;

3559 + let request: EngineCoreRequest =

3560 + rmp_serde::from_slice(&add[1]).expect("decode request");

3561 + let reasoning_token_ids = bytes_to_token_ids(b"<think>think</think>");

3562 + let answer_token_ids = bytes_to_token_ids(b"answer");

3563 +

3564 + send_outputs(

3565 + push,

3566 + EngineCoreOutputs {

3567 + engine_index: 0,

3568 + outputs: vec![

3569 + request_output_with_logprobs(

3570 + &request.request_id,

3571 + reasoning_token_ids.clone(),

3572 + None,

3573 + None,

3574 + Some(sample_logprobs_for_tokens(&reasoning_token_ids)),

3575 + None,

3576 + ),

3577 + request_output_with_logprobs(

3578 + &request.request_id,

3579 + answer_token_ids.clone(),

3580 + Some(EngineCoreFinishReason::Length),

3581 + None,

3582 + Some(sample_logprobs_for_tokens(&answer_token_ids)),

3583 + None,

3584 + ),

3585 + ],

3586 + scheduler_stats: None,

3587 + timestamp: 0.0,

3588 + utility_output: None,

3589 + finished_requests: None,

3590 + wave_complete: None,

3591 + start_wave: None,

3592 + },

3593 + )

3594 + .await;

3595 + })

3596 + },

3597 + ));

3598 +

3599 + let client = EngineCoreClient::connect(

3600 + EngineCoreClientConfig::new_single(handshake_address)

3601 + .with_model_name("test-model")

3602 + .with_local_input_output_addresses(

3603 + Some(ipc.input_endpoint()),

3604 + Some(ipc.output_endpoint()),

3605 + ),

3606 + )

3607 + .await

3608 + .expect("connect client");

3609 + let chat = ChatLlm::from_shared_backend(

3610 + test_llm(client),

3611 + Arc::new(FakeChatBackend::with_model_id("Qwen/Qwen3-0.6B")),

3612 + );

3613 + let mut app = build_router(Arc::new(AppState::new(

3614 + vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],

3615 + chat,

3616 + )));

3617 +

3618 + let response = app

3619 + .call(

3620 + Request::builder()

3621 + .method("POST")

3622 + .uri("/v1/chat/completions")

3623 + .header("content-type", "application/json")

3624 + .body(Body::from(

3625 + json!({

3626 + "model": "Qwen/Qwen1.5-0.5B-Chat",

3627 + "stream": false,

3628 + "include_reasoning": false,

3629 + "logprobs": true,

3630 + "return_token_ids": true,

3631 + "messages": [{"role": "user", "content": "hello"}]

3632 + })

3633 + .to_string(),

3634 + ))

3635 + .expect("build request"),

3636 + )

3637 + .await

3638 + .expect("call app");

3639 +

3640 + assert_eq!(response.status(), StatusCode::OK);

3641 + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");

3642 + engine_task.await.expect("mock engine task");

3643 + let text = String::from_utf8(body.to_vec()).expect("utf8 body");

3644 + let json: serde_json::Value = serde_json::from_str(&text).expect("decode json");

3645 + let choice = json["choices"][0].as_object().expect("choice object");

3646 +

3647 + assert_eq!(json["choices"][0]["message"]["content"], "answer");

3648 + assert!(

3649 + json["choices"][0]["message"]

3650 + .as_object()

3651 + .is_some_and(|message| !message.contains_key("reasoning")),

3652 + "{text}"

3653 + );

3654 + assert!(!choice.contains_key("logprobs"), "{text}");

3655 + assert!(!choice.contains_key("token_ids"), "{text}");

3656 + assert!(json["prompt_token_ids"].is_array(), "{text}");

3657 +}

3658 +

3659 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]

3660 #[serial]

3661 async fn tool_calls_are_mapped_to_tool_call_sse_chunks() {