[Bugfix] Fix double BOS in LLM.chat() for multimodal models (#55288)
Signed-off-by: summer <128961079+zhang-keliang@users.noreply.github.com>
summer committed
Sep 4, 2026 at 22:23 UTC
a85d0738da311ce4ad5814787e61a12826e232df
2 files changed
+140
-3
tests/entrypoints/multimodal/llm/test_mm_processor_kwargs.py
+130
@@ -286,3 +286,133 @@ def test_preprocess_chat_omits_mm_processor_kwargs_when_no_override() -> None:
286
assert call_args.args[1].mm_processor_kwargs is None
287
assert call_args.args[2] == "tok-params"
288
assert call_args.kwargs["prompt_extras"] is None
289
+
290
+
291
+def test_preprocess_chat_defaults_add_special_tokens_to_false() -> None:
292
+ # Matches `ChatCompletionRequest.add_special_tokens` on the server.
293
+ llm = _make_mock_llm()
294
+ messages = [[{"role": "user", "content": "hi"}]]
295
+
296
+ renderer = Mock()
297
+ renderer.tokenizer = object()
298
+ renderer.default_chat_tok_params = Mock()
299
+ renderer.default_chat_tok_params.with_kwargs.return_value = "tok-params"
300
+ renderer.render_chat.return_value = (messages, ["engine-input"])
301
+ llm.renderer = renderer
302
+
303
+ llm._preprocess_chat(messages)
304
+
305
+ renderer.default_chat_tok_params.with_kwargs.assert_called_once_with(
306
+ add_special_tokens=False
307
+ )
308
+
309
+
310
+def test_preprocess_chat_tokenization_kwargs_override_add_special_tokens() -> None:
311
+ llm = _make_mock_llm()
312
+ messages = [[{"role": "user", "content": "hi"}]]
313
+
314
+ renderer = Mock()
315
+ renderer.tokenizer = object()
316
+ renderer.default_chat_tok_params = Mock()
317
+ renderer.default_chat_tok_params.with_kwargs.return_value = "tok-params"
318
+ renderer.render_chat.return_value = (messages, ["engine-input"])
319
+ llm.renderer = renderer
320
+
321
+ llm._preprocess_chat(
322
+ messages,
323
+ tokenization_kwargs={"add_special_tokens": True, "truncate_prompt_tokens": 8},
324
+ )
325
+
326
+ renderer.default_chat_tok_params.with_kwargs.assert_called_once_with(
327
+ add_special_tokens=True, truncate_prompt_tokens=8
328
+ )
329
+
330
+
331
+@pytest.fixture(scope="module")
332
+def llava_llm():
333
+ from vllm.config import ModelConfig, VllmConfig
334
+ from vllm.renderers.hf import HfRenderer
335
+ from vllm.tokenizers import cached_tokenizer_from_config
336
+
337
+ # A real multimodal model (Llama tokenizer, BOS=1) so that the real
338
+ # processor default (`add_special_tokens=True`) is in play. Only
339
+ # config/tokenizer/processor files are fetched, no weights.
340
+ model_config = ModelConfig(model="llava-hf/llava-1.5-7b-hf", max_model_len=128)
341
+ renderer = HfRenderer(
342
+ VllmConfig(model_config=model_config),
343
+ cached_tokenizer_from_config(model_config),
344
+ )
345
+ assert renderer.default_chat_tok_params.add_special_tokens is True
346
+
347
+ llm = _make_mock_llm()
348
+ # `_preprocess_cmpl` parses prompts against the real model config.
349
+ llm.model_config = model_config
350
+ llm.renderer = renderer
351
+ return llm
352
+
353
+
354
+class TestChatAddSpecialTokensDefault:
355
+ """`LLM.chat()` renders the chat template to text and then tokenizes it.
356
+ The multimodal processor default is `add_special_tokens=True`, so a
357
+ template that emits `bos_token` used to get a second BOS from the
358
+ tokenizer. `_preprocess_chat` now defaults `add_special_tokens=False`
359
+ like the online chat API does (`ChatCompletionRequest`), unless the
360
+ caller overrides it via `tokenization_kwargs`.
361
+ """
362
+
363
+ # Mirrors the Gemma 3 chat template and the bundled deepseek_vl2 /
364
+ # deepseek_ocr templates, whose rendered output starts with the BOS token.
365
+ BOS_TEMPLATE = (
366
+ "{{ bos_token }}{% for m in messages %}{{ m['content'] }}{% endfor %}"
367
+ )
368
+ MESSAGES = [[{"role": "user", "content": "hi"}]]
369
+
370
+ def _chat_token_ids(self, llm, **kwargs):
371
+ (engine_input,) = llm._preprocess_chat(
372
+ self.MESSAGES,
373
+ chat_template=self.BOS_TEMPLATE,
374
+ add_generation_prompt=False,
375
+ **kwargs,
376
+ )
377
+ return engine_input["prompt_token_ids"]
378
+
379
+ def test_chat_does_not_duplicate_template_bos(self, llava_llm):
380
+ bos = llava_llm.renderer.tokenizer.bos_token_id
381
+ prompt_token_ids = self._chat_token_ids(llava_llm)
382
+
383
+ assert prompt_token_ids[0] == bos
384
+ assert prompt_token_ids.count(bos) == 1
385
+
386
+ def test_chat_matches_online_chat_api(self, llava_llm):
387
+ from vllm.renderers.params import ChatParams, TokenizeParams
388
+
389
+ # `ChatCompletionRequest.add_special_tokens` defaults to `False`.
390
+ online_tok_params = TokenizeParams(
391
+ max_total_tokens=llava_llm.renderer.model_config.max_model_len,
392
+ add_special_tokens=False,
393
+ )
394
+ chat_params = ChatParams(
395
+ chat_template=self.BOS_TEMPLATE,
396
+ chat_template_kwargs=dict(tokenize=False, add_generation_prompt=False),
397
+ )
398
+ _, (online_input,) = llava_llm.renderer.render_chat(
399
+ self.MESSAGES, chat_params, online_tok_params
400
+ )
401
+
402
+ assert self._chat_token_ids(llava_llm) == online_input["prompt_token_ids"]
403
+
404
+ def test_explicit_tokenization_kwargs_override_default(self, llava_llm):
405
+ bos = llava_llm.renderer.tokenizer.bos_token_id
406
+ prompt_token_ids = self._chat_token_ids(
407
+ llava_llm, tokenization_kwargs={"add_special_tokens": True}
408
+ )
409
+
410
+ assert prompt_token_ids.count(bos) == 2
411
+
412
+ def test_generate_keeps_processor_default(self, llava_llm):
413
+ # Raw prompts have no chat template to emit BOS, so `LLM.generate()`
414
+ # must keep letting the tokenizer add it.
415
+ bos = llava_llm.renderer.tokenizer.bos_token_id
416
+ (engine_input,) = llava_llm._preprocess_cmpl(["hi"])
417
+
418
+ assert engine_input["prompt_token_ids"][0] == bos
vllm/entrypoints/offline_utils.py
+10
-3
@@ -196,9 +196,16 @@ class OfflineInferenceMixin:
196
),
197
mm_processor_kwargs=mm_processor_kwargs,
198
)
199
- tok_params = renderer.default_chat_tok_params.with_kwargs(
200
- **(tokenization_kwargs or {})
201
- )
199
+ # The chat template is responsible for emitting BOS/EOS, so do not let
200
+ # the tokenizer add them again unless the caller asks for it. This
201
+ # matches the online chat API (`ChatCompletionRequest.add_special_tokens`
202
+ # defaults to `False`) and avoids a double BOS for multimodal models,
203
+ # whose processor default is `add_special_tokens=True` (#55197).
204
+ tokenization_kwargs = {
205
+ "add_special_tokens": False,
206
+ **(tokenization_kwargs or {}),
207
+ }
208
+ tok_params = renderer.default_chat_tok_params.with_kwargs(**tokenization_kwargs)
209
prompt_extras = (
210
None
211
if mm_processor_kwargs is None