A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 02:15 UTC
[Model] Fix Gemma 4 token repetition by dynamic BOS injection for PT models (#39842)
Signed-off-by: Luciano Martins <lucianommartins@users.noreply.github.com> Co-authored-by: Luciano Martins <lucianommartins@users.noreply.github.com>
Luciano Martins committed
Apr 15, 2026 at 20:13 UTC
6dc9491406938463e44e631610ce83df1d20f8cd
1 file changed
+7
-2
vllm/model_executor/models/gemma4_mm.py
+7
-2
@@ -167,10 +167,15 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
167
168
Setting ``add_special_tokens=False`` here prevents the duplicate and
169
ensures both ``llm.generate()`` and the chat/completions API behave
170
- correctly.
170
+ correctly for IT models. For PT models (without chat template), we
171
+ keep the default (True) to ensure BOS is added for raw prompts.
172
"""
173
+ tokenizer = self.ctx.get_tokenizer()
174
+ has_chat_template = getattr(tokenizer, "chat_template", None) is not None
175
+
176
params = super().get_default_tok_params()
173
- params = params.with_kwargs(add_special_tokens=False)
177
+ if has_chat_template:
178
+ params = params.with_kwargs(add_special_tokens=False)
179
return params
180
181
def get_hf_processor(self, **kwargs: object) -> Gemma4Processor: