A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 04:45 UTC
[Rust Frontend] Cover different thinking modes in roundtrip tests (#44320)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Bugen Zhao committed
Jun 2, 2026 at 22:51 UTC
586201ebdc5c8b180a3e2f90174f152f4f8d65a0
1 file changed
+75
-21
rust/src/chat/tests/roundtrip.rs
+75
-21
@@ -20,6 +20,7 @@ use vllm_chat::{
20
use vllm_text::{DecodedTextEvent, Finished, Prompt};
21
22
/// One model/parser configuration used to run the fixed roundtrip fixtures.
23
+#[derive(Clone)]
24
struct RoundtripCase {
25
/// Hugging Face model id resolved through the production backend loader.
26
model_id: &'static str,
@@ -31,11 +32,45 @@ struct RoundtripCase {
32
tool_call_parser: ParserSelection,
33
/// Reasoning parser selection used by the output processor.
34
reasoning_parser: ParserSelection,
35
+ /// How this model's chat template handles thinking mode.
36
+ thinking_behavior: ThinkingBehavior,
37
/// JSON formatting expected after this model's template has materialized
38
/// tool-call arguments.
39
json_fmt: JsonFmt,
40
}
41
42
+#[derive(Clone, Copy)]
43
+enum ThinkingBehavior {
44
+ /// The chat template accepts explicit thinking on/off kwargs, and uses
45
+ /// `default` when the request does not specify either kwarg.
46
+ Toggleable { default: bool },
47
+ /// The chat template always behaves as `value` for this fixture.
48
+ Always { value: bool },
49
+}
50
+
51
+impl ThinkingBehavior {
52
+ fn default(self) -> bool {
53
+ match self {
54
+ Self::Toggleable { default } => default,
55
+ Self::Always { value } => value,
56
+ }
57
+ }
58
+
59
+ fn fixtures(self) -> Vec<Option<bool>> {
60
+ match self {
61
+ Self::Toggleable { .. } => vec![
62
+ Some(true), // explicitly enable thinking
63
+ Some(false), // explicitly disable thinking
64
+ None, // use default template behavior
65
+ ],
66
+ Self::Always { value } => vec![
67
+ Some(value), // explicitly request the supported thinking behavior
68
+ None, // use default template behavior
69
+ ],
70
+ }
71
+ }
72
+}
73
+
74
impl RoundtripCase {
75
/// Qwen3 XML tool-call format with `qwen3` reasoning tags.
76
fn qwen3() -> Self {
@@ -44,6 +79,7 @@ impl RoundtripCase {
79
assistant_stop_suffix: "<|im_end|>\n",
80
tool_call_parser: ParserSelection::Auto,
81
reasoning_parser: ParserSelection::Auto,
82
+ thinking_behavior: ThinkingBehavior::Toggleable { default: true },
83
json_fmt: spaced_json_fmt(),
84
}
85
}
@@ -55,6 +91,7 @@ impl RoundtripCase {
91
assistant_stop_suffix: "<|im_end|>\n",
92
tool_call_parser: ParserSelection::Auto,
93
reasoning_parser: ParserSelection::Auto,
94
+ thinking_behavior: ThinkingBehavior::Toggleable { default: true },
95
json_fmt: compact_json_fmt(),
96
}
97
}
@@ -66,6 +103,7 @@ impl RoundtripCase {
103
assistant_stop_suffix: "[e~[\n",
104
tool_call_parser: ParserSelection::Auto,
105
reasoning_parser: ParserSelection::Auto,
106
+ thinking_behavior: ThinkingBehavior::Always { value: true },
107
json_fmt: compact_json_fmt(),
108
}
109
}
@@ -77,6 +115,7 @@ impl RoundtripCase {
115
assistant_stop_suffix: "<｜end▁of▁sentence｜>",
116
tool_call_parser: ParserSelection::Auto,
117
reasoning_parser: ParserSelection::Auto,
118
+ thinking_behavior: ThinkingBehavior::Toggleable { default: false },
119
json_fmt: compact_json_fmt(),
120
}
121
}
@@ -88,6 +127,7 @@ impl RoundtripCase {
127
assistant_stop_suffix: "",
128
tool_call_parser: ParserSelection::Auto,
129
reasoning_parser: ParserSelection::Auto,
130
+ thinking_behavior: ThinkingBehavior::Toggleable { default: true },
131
json_fmt: compact_json_fmt(),
132
}
133
}
@@ -100,6 +140,7 @@ impl RoundtripCase {
140
assistant_stop_suffix: "<|im_end|>",
141
tool_call_parser: ParserSelection::Auto,
142
reasoning_parser: ParserSelection::Auto,
143
+ thinking_behavior: ThinkingBehavior::Toggleable { default: true },
144
json_fmt: spaced_json_fmt(),
145
}
146
}
@@ -135,35 +176,44 @@ roundtrip_tests! {
176
177
/// Run the fixed reasoning+content fixture for one model/parser case.
178
async fn run_roundtrip_reasoning_and_content(case: RoundtripCase) -> Result<()> {
179
+ for thinking in case.thinking_behavior.fixtures() {
180
+ run_roundtrip_reasoning_and_content_inner(case.clone(), thinking).await?;
181
+ }
182
+ Ok(())
183
+}
184
+
185
+async fn run_roundtrip_reasoning_and_content_inner(
186
+ case: RoundtripCase,
187
+ thinking: Option<bool>,
188
+) -> Result<()> {
189
let backends = load_roundtrip_backends(&case).await?;
190
let request = roundtrip_request(
191
"roundtrip-reasoning-content",
192
vec![ChatMessage::text(ChatRole::User, "What is 2 + 2?")],
193
Vec::new(),
194
+ thinking,
195
);
196
let expected_reasoning = "Need compute 2 + 2 directly.";
197
let expected_text = "The answer is 4.";
198
+ let effective_thinking = thinking.unwrap_or(case.thinking_behavior.default());
199
147
- let result = run_roundtrip(
148
- &case,
149
- &backends,
150
- &request,
151
- AssistantMessage {
152
- content: vec![
153
- AssistantContentBlock::Reasoning {
154
- text: expected_reasoning.to_string(),
155
- },
156
- AssistantContentBlock::Text {
157
- text: expected_text.to_string(),
158
- },
159
- ],
160
- },
161
- )
162
- .await?;
200
+ let assistant = {
201
+ let mut content = Vec::new();
202
+ if effective_thinking {
203
+ content.push(AssistantContentBlock::Reasoning {
204
+ text: expected_reasoning.to_string(),
205
+ });
206
+ }
207
+ content.push(AssistantContentBlock::Text {
208
+ text: expected_text.to_string(),
209
+ });
210
+ AssistantMessage { content }
211
+ };
212
+ let result = run_roundtrip(&case, &backends, &request, assistant).await?;
213
214
assert_eq!(
215
result.parsed_message.reasoning().as_deref().map(str::trim),
166
- Some(expected_reasoning)
216
+ effective_thinking.then_some(expected_reasoning)
217
);
218
assert_eq!(result.parsed_message.text().trim(), expected_text);
219
assert_eq!(result.parsed_message.tool_calls().count(), 0);
@@ -186,6 +236,7 @@ async fn run_roundtrip_tool_call_mix(case: RoundtripCase) -> Result<()> {
236
"Check Shanghai weather and add 1.00 plus 2.",
237
)],
238
test_tools(),
239
+ Some(true), // always enable thinking in this fixture
240
);
241
let expected_reasoning = "Need call the weather and add tools.";
242
let expected_text = "I will call the tools.";
@@ -487,6 +538,7 @@ fn roundtrip_request(
538
request_id: impl Into<String>,
539
messages: Vec<ChatMessage>,
540
tools: Vec<ChatTool>,
541
+ thinking: Option<bool>,
542
) -> ChatRequest {
543
let mut request = ChatRequest {
544
request_id: request_id.into(),
@@ -500,10 +552,12 @@ fn roundtrip_request(
552
..ChatRequest::for_test()
553
};
554
503
- // Enable thinking for some models so that rendering and parsing the reasoning block is
504
- // exercised in the roundtrip.
505
- for key in ["thinking", "enable_thinking"] {
506
- request.chat_options.template_kwargs.insert(key.to_string(), true.into());
555
+ // Explicitly enable or disable thinking so that rendering and parsing the reasoning block is
556
+ // exercised or skipped in the roundtrip. If unspecified, use the default template behavior.
557
+ if let Some(thinking) = thinking {
558
+ for key in ["thinking", "enable_thinking"] {
559
+ request.chat_options.template_kwargs.insert(key.to_string(), thinking.into());
560
+ }
561
}
562
563
request