[Bugfix][Multimodal] Preserve DeepSeek V4 image block spacing (#56882)
Signed-off-by: Xucheng Zhou <aden1350@outlook.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
adenzhou1350 committed
Sep 18, 2026 at 18:44 UTC
cc09352e5c50ca6b7d2c6d79b29d6d3b18df3046
5 files changed
+60
-12
tests/tokenizers_/fixtures/deepseek_v4/test_input_5.json
new
+39
@@ -0,0 +1,39 @@
1
+{
2
+ "thinking_mode": "thinking",
3
+ "reasoning_effort": "max",
4
+ "messages": [
5
+ {
6
+ "role": "system",
7
+ "content": "You are a helpful vision assistant."
8
+ },
9
+ {
10
+ "role": "user",
11
+ "content": [
12
+ {
13
+ "type": "text",
14
+ "text": "请按“第一张、第二张”的顺序回答：第一张图"
15
+ },
16
+ {
17
+ "type": "image_url",
18
+ "image_url": {
19
+ "url": "examples/images/carrots.jpeg"
20
+ }
21
+ },
22
+ {
23
+ "type": "text",
24
+ "text": "和第二张图"
25
+ },
26
+ {
27
+ "type": "image_url",
28
+ "image_url": {
29
+ "url": "examples/images/corn.jpeg"
30
+ }
31
+ },
32
+ {
33
+ "type": "text",
34
+ "text": "中分别是什么食材？它们通常食用的部位分别是什么？"
35
+ }
36
+ ]
37
+ }
38
+ ]
39
+}
tests/tokenizers_/fixtures/deepseek_v4/test_output_5.txt
new
+9
@@ -0,0 +1,9 @@
1
+<｜begin▁of▁sentence｜>You are a helpful vision assistant.<｜User｜>请按“第一张、第二张”的顺序回答：第一张图
2
+
3
+<｜deepseek_image｜>
4
+
5
+和第二张图
6
+
7
+<｜deepseek_image｜>
8
+
9
+中分别是什么食材？它们通常食用的部位分别是什么？<｜Assistant｜></think>
\ No newline at end of file
tests/tokenizers_/test_deepseek_v4.py
+7
-7
@@ -54,13 +54,9 @@ def _load_reference_case(case_id: int):
54
55
def _render_reference_case(case_id: int, **kwargs):
56
messages, tools = _load_reference_case(case_id)
57
- conversation, _, _ = parse_chat_messages(
58
- messages,
59
- _model_config(),
60
- content_format="string",
61
- )
57
+ # Preserve content blocks for encoding without fetching fixture media.
58
return _tokenizer().apply_chat_template(
63
- conversation=conversation,
59
+ conversation=messages,
60
messages=messages,
61
tools=tools,
62
tokenize=False,
@@ -397,6 +393,7 @@ def test_deepseek_v4_maps_xhigh_to_high_reasoning_effort():
393
(2, {"thinking": True, "reasoning_effort": "low"}),
394
(3, {"thinking": True, "reasoning_effort": "low"}),
395
(4, {"thinking": False}),
396
+ (5, {"thinking": False}),
397
],
398
)
399
def test_deepseek_v4_matches_reference_golden_fixtures(case_id, kwargs):
@@ -447,7 +444,10 @@ def test_deepseek_v4_image_blocks_become_placeholders():
444
thinking=False,
445
)
446
450
- assert "<｜User｜>first:<｜deepseek_image｜>second:<｜deepseek_image｜>" in prompt
447
+ assert (
448
+ "<｜User｜>first:\n\n<｜deepseek_image｜>\n\n"
449
+ "second:\n\n<｜deepseek_image｜>" in prompt
450
+ )
451
452
453
def test_deepseek_v4_image_sentinel_ids_match_tokenizer():
vllm/renderers/deepseek_v4.py
+2
-2
@@ -40,7 +40,7 @@ class DeepseekV4Renderer(BaseRenderer[DeepseekV4Tokenizer]):
40
conversation, mm_data, mm_uuids = parse_chat_messages(
41
messages,
42
self.model_config,
43
- content_format="string",
43
+ content_format="openai",
44
media_io_kwargs=params.media_io_kwargs,
45
mm_processor_kwargs=params.mm_processor_kwargs,
46
)
@@ -67,7 +67,7 @@ class DeepseekV4Renderer(BaseRenderer[DeepseekV4Tokenizer]):
67
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
68
messages,
69
self.model_config,
70
- content_format="string",
70
+ content_format="openai",
71
media_io_kwargs=params.media_io_kwargs,
72
mm_processor_kwargs=params.mm_processor_kwargs,
73
)
vllm/tokenizers/deepseek_v4_encoding.py
+3
-3
@@ -224,7 +224,7 @@ def flatten_content_blocks(content: Any) -> Any:
224
raise ValueError(
225
f"Unsupported content block type: {block.get('type')!r}"
226
)
227
- return "".join(parts)
227
+ return "\n\n".join(parts)
228
229
230
def render_message(
@@ -267,7 +267,7 @@ def render_message(
267
)
268
269
role = msg.get("role")
270
- content = msg.get("content")
270
+ content = flatten_content_blocks(msg.get("content"))
271
tools = msg.get("tools")
272
response_format = msg.get("response_format")
273
tool_calls = msg.get("tool_calls")
@@ -337,7 +337,7 @@ def render_message(
337
parts.append(f"[Unsupported {block_type}]")
338
prompt += "\n\n".join(parts)
339
else:
340
- prompt += flatten_content_blocks(content) or ""
340
+ prompt += content or ""
341
342
elif role == "latest_reminder":
343
prompt += LATEST_REMINDER_SP_TOKEN + latest_reminder_msg_template.format(content=content)