[Bugfix][DeepSeek V4] Handle trailing system messages in prompt rendering (#51262)
Signed-off-by: jiahaoliang <gzliangjiahao@gmail.com> Signed-off-by: Jiahao Liang <gzliangjiahao@gmail.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
Jiahao Liang committed
Aug 25, 2026 at 15:49 UTC
59d7fc92ea038544959aae020a8b28fc34159545
4 files changed
+150
-3
rust/src/chat/src/renderer/deepseek_v4/encoding.rs
+2
-2
@@ -110,7 +110,7 @@ pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
110
}
111
}
112
113
- if is_user_like_entry(message)
113
+ if (is_user_like_entry(message) || matches!(message, ChatMessage::System { .. }))
114
&& next_rendered_entry_is_assistant_or_end(request.messages.as_slice(), message_index)
115
{
116
write_assistant_transition(
@@ -423,7 +423,7 @@ fn write_tool_result(out: &mut String, content: &ChatContent) -> Result<()> {
423
Ok(())
424
}
425
426
-/// Append the assistant transition token after a user-like turn.
426
+/// Append the assistant transition token after a user-like or system turn.
427
fn write_assistant_transition(
428
out: &mut String,
429
thinking_mode: ThinkingMode,
rust/src/chat/src/renderer/deepseek_v4/tests.rs
+62
@@ -195,6 +195,68 @@ fn reasoning_effort_none_disables_thinking() {
195
.assert_eq(&rendered);
196
}
197
198
+#[test]
199
+fn system_to_assistant_transitions_in_chat_and_thinking_modes() {
200
+ let rendered = [false, true].map(|enable_thinking| {
201
+ let mut request = ChatRequest {
202
+ messages: vec![
203
+ ChatMessage::system("rules"),
204
+ ChatMessage::assistant_blocks(vec![
205
+ AssistantContentBlock::Reasoning {
206
+ text: "reason".to_string(),
207
+ },
208
+ AssistantContentBlock::Text {
209
+ text: "answer".to_string(),
210
+ },
211
+ ]),
212
+ ],
213
+ ..ChatRequest::for_test()
214
+ };
215
+ request
216
+ .chat_options
217
+ .template_kwargs
218
+ .insert("enable_thinking".to_string(), Value::Bool(enable_thinking));
219
+ request.chat_options.reasoning_effort = Some(ReasoningEffort::Low);
220
+ render_request(&request)
221
+ });
222
+
223
+ expect![[r#"
224
+ [
225
+ "<｜begin▁of▁sentence｜>rules<｜Assistant｜></think>answer<｜end▁of▁sentence｜>",
226
+ "<｜begin▁of▁sentence｜>rules<｜Assistant｜><think>reason</think>answer<｜end▁of▁sentence｜>",
227
+ ]
228
+ "#]]
229
+ .assert_debug_eq(&rendered);
230
+}
231
+
232
+#[test]
233
+fn trailing_system_transitions_in_chat_and_thinking_modes() {
234
+ let rendered = [false, true].map(|enable_thinking| {
235
+ let mut request = ChatRequest {
236
+ messages: vec![
237
+ ChatMessage::system("rules"),
238
+ ChatMessage::user("question"),
239
+ ChatMessage::system("final rules"),
240
+ ],
241
+ ..ChatRequest::for_test()
242
+ };
243
+ request
244
+ .chat_options
245
+ .template_kwargs
246
+ .insert("enable_thinking".to_string(), Value::Bool(enable_thinking));
247
+ request.chat_options.reasoning_effort = Some(ReasoningEffort::Low);
248
+ render_request(&request)
249
+ });
250
+
251
+ expect![[r#"
252
+ [
253
+ "<｜begin▁of▁sentence｜>rules<｜User｜>questionfinal rules<｜Assistant｜></think>",
254
+ "<｜begin▁of▁sentence｜>rules<｜User｜>questionfinal rules<｜Assistant｜><think>",
255
+ ]
256
+ "#]]
257
+ .assert_debug_eq(&rendered);
258
+}
259
+
260
#[test]
261
fn reasoning_effort_template_kwarg_is_ignored() {
262
let mut request = ChatRequest {
tests/tokenizers_/test_deepseek_v4.py
+77
@@ -113,6 +113,83 @@ def test_deepseek_v4_explicitly_disables_thinking(kwargs):
113
assert prompt == ("<｜begin▁of▁sentence｜><｜User｜>Hello<｜Assistant｜></think>")
114
115
116
+@pytest.mark.parametrize(
117
+ ("enable_thinking", "thinking_token"),
118
+ [(False, "</think>"), (True, "<think>")],
119
+)
120
+def test_deepseek_v4_appends_assistant_transition_after_trailing_system(
121
+ enable_thinking, thinking_token
122
+):
123
+ prompt = _tokenizer().apply_chat_template(
124
+ [
125
+ {"role": "system", "content": "You are helpful."},
126
+ {"role": "user", "content": "What is 2+2?"},
127
+ {"role": "system", "content": "Be concise."},
128
+ ],
129
+ tokenize=False,
130
+ enable_thinking=enable_thinking,
131
+ reasoning_effort="low",
132
+ )
133
+
134
+ assert prompt == (
135
+ "<｜begin▁of▁sentence｜>You are helpful."
136
+ f"<｜User｜>What is 2+2?Be concise.<｜Assistant｜>{thinking_token}"
137
+ )
138
+
139
+
140
+def test_deepseek_v4_does_not_transition_after_mid_conversation_system():
141
+ prompt = _tokenizer().apply_chat_template(
142
+ [
143
+ {"role": "user", "content": "What is 3+3?"},
144
+ {"role": "assistant", "content": "6"},
145
+ {"role": "system", "content": "Answer in French."},
146
+ {"role": "user", "content": "What is 5+5?"},
147
+ ],
148
+ tokenize=False,
149
+ enable_thinking=False,
150
+ )
151
+
152
+ assert prompt == (
153
+ "<｜begin▁of▁sentence｜><｜User｜>What is 3+3?"
154
+ "<｜Assistant｜></think>6<｜end▁of▁sentence｜>"
155
+ "Answer in French.<｜User｜>What is 5+5?"
156
+ "<｜Assistant｜></think>"
157
+ )
158
+
159
+
160
+@pytest.mark.parametrize(
161
+ ("enable_thinking", "expected"),
162
+ [
163
+ (
164
+ False,
165
+ "<｜begin▁of▁sentence｜>rules<｜Assistant｜></think>answer"
166
+ "<｜end▁of▁sentence｜>",
167
+ ),
168
+ (
169
+ True,
170
+ "<｜begin▁of▁sentence｜>rules<｜Assistant｜><think>reason</think>answer"
171
+ "<｜end▁of▁sentence｜>",
172
+ ),
173
+ ],
174
+)
175
+def test_deepseek_v4_transitions_from_system_to_assistant(enable_thinking, expected):
176
+ prompt = _tokenizer().apply_chat_template(
177
+ [
178
+ {"role": "system", "content": "rules"},
179
+ {
180
+ "role": "assistant",
181
+ "reasoning": "reason",
182
+ "content": "answer",
183
+ },
184
+ ],
185
+ tokenize=False,
186
+ enable_thinking=enable_thinking,
187
+ reasoning_effort="low",
188
+ )
189
+
190
+ assert prompt == expected
191
+
192
+
193
def test_deepseek_v4_unknown_role_raises_value_error():
194
# Invalid roles are client errors: they must surface as ValueError
195
# (mapped to HTTP 400 by the OpenAI serving layer), not
vllm/tokenizers/deepseek_v4_encoding.py
+9
-1
@@ -374,7 +374,15 @@ def render_message(
374
prompt += thinking_end_token if thinking_mode != "thinking" else thinking_start_token
375
prompt += task_sp_token
376
377
- elif messages[index].get("role") in ["user", "developer"]:
377
+ # A trailing system message opens generation, while a system message
378
+ # followed by assistant opens that assistant history turn.
379
+ elif role in ["user", "developer"] or (
380
+ role == "system"
381
+ and (
382
+ index == len(messages) - 1
383
+ or messages[index + 1].get("role") == "assistant"
384
+ )
385
+ ):
386
# Normal generation: append Assistant + thinking token
387
prompt += ASSISTANT_SP_TOKEN
388
if not drop_thinking and thinking_mode == "thinking":