[Bugfix][Tokenizer] Replace bare asserts in the DeepSeek V4 encoder (#53747)
Signed-off-by: mhuzaifa3 <mhuzaifa3@outlook.com>
Moe Huzaifa committed
Aug 25, 2026 at 11:30 UTC
19406fae287380991908d2f27fe3975ba047147d
2 files changed
+41
-8
tests/tokenizers_/test_deepseek_v4.py
+24
@@ -403,3 +403,27 @@ def test_deepseek_v4_matches_reference_golden_fixtures(case_id, kwargs):
403
404
expected = (FIXTURES_DIR / f"test_output_{case_id}.txt").read_text()
405
assert prompt == expected
406
+
407
+
408
+def test_deepseek_v4_rejects_empty_developer_content():
409
+ with pytest.raises(ValueError):
410
+ _tokenizer().apply_chat_template(
411
+ [{"role": "developer", "content": ""}],
412
+ tokenize=False,
413
+ enable_thinking=True,
414
+ reasoning_effort="low",
415
+ )
416
+
417
+
418
+@pytest.mark.parametrize(
419
+ "kwargs",
420
+ [
421
+ {"thinking_mode": "bogus"},
422
+ {"thinking_mode": "thinking", "reasoning_effort": "turbo"},
423
+ ],
424
+)
425
+def test_deepseek_v4_encode_messages_rejects_invalid_arguments(kwargs):
426
+ from vllm.tokenizers.deepseek_v4_encoding import encode_messages
427
+
428
+ with pytest.raises(ValueError):
429
+ encode_messages([{"role": "user", "content": "Hello"}], **kwargs)
vllm/tokenizers/deepseek_v4_encoding.py
+17
-8
@@ -225,8 +225,12 @@ def render_message(
225
Returns:
226
Encoded string for this message.
227
"""
228
- assert 0 <= index < len(messages)
229
- assert thinking_mode in ["chat", "thinking"], f"Invalid thinking_mode `{thinking_mode}`"
228
+ if not (0 <= index < len(messages)):
229
+ raise ValueError(
230
+ f"Index {index} out of range for messages list of length {len(messages)}"
231
+ )
232
+ if thinking_mode not in ["chat", "thinking"]:
233
+ raise ValueError(f"Invalid thinking_mode `{thinking_mode}`")
234
235
prompt = ""
236
msg = messages[index]
@@ -248,10 +252,11 @@ def render_message(
252
tool_calls = tool_calls_from_openai_format(tool_calls)
253
254
reasoning_effort = reasoning_effort or DEFAULT_REASONING_EFFORT
251
- assert reasoning_effort in REASONING_EFFORT_PROMPTS, (
252
- f"Invalid reasoning effort: {reasoning_effort}, expected one of "
253
- f"{list(REASONING_EFFORT_PROMPTS)}"
254
- )
255
+ if reasoning_effort not in REASONING_EFFORT_PROMPTS:
256
+ raise ValueError(
257
+ f"Invalid reasoning effort: {reasoning_effort}, expected one of "
258
+ f"{list(REASONING_EFFORT_PROMPTS)}"
259
+ )
260
if index == 0 and thinking_mode == "thinking":
261
prompt += REASONING_EFFORT_PROMPTS[reasoning_effort]
262
@@ -263,7 +268,8 @@ def render_message(
268
prompt += "\n\n" + response_format_template.format(schema=to_json(response_format))
269
270
elif role == "developer":
266
- assert content, f"Invalid message for role `{role}`: {msg}"
271
+ if not content:
272
+ raise ValueError(f"Invalid message for role `{role}`: {msg}")
273
274
content_developer = USER_SP_TOKEN
275
content_developer += content
@@ -362,7 +368,10 @@ def render_message(
368
task = messages[index].get("task")
369
if task is not None:
370
# Task special token for internal classification tasks
365
- assert task in VALID_TASKS, f"Invalid task: '{task}'. Valid tasks are: {list(VALID_TASKS)}"
371
+ if task not in VALID_TASKS:
372
+ raise ValueError(
373
+ f"Invalid task: '{task}'. Valid tasks are: {list(VALID_TASKS)}"
374
+ )
375
task_sp_token = DS_TASK_SP_TOKENS[task]
376
377
if task != "action":