A high-throughput and memory-efficient inference and serving engine for LLMs
[Bugfix][Gemma4] Fix parsing when thinking is disabled (#45832)
Signed-off-by: Federico Iezzi <fiezzi@google.com>
Federico committed
Jun 17, 2026 at 04:41 UTC
b831374cf1dbd6e9cc63d98a891c7b73773adb84
2 files changed
+70
-28
tests/tool_use/test_gemma4_responses_adjust_request.py
+57
-21
@@ -68,28 +68,33 @@ def _build_responses_request(*, tool_choice: str | dict[str, Any]) -> ResponsesR
68
)
69
70
71
-def _build_chat_request(*, tool_choice: str | dict[str, Any]) -> ChatCompletionRequest:
72
- return ChatCompletionRequest.model_validate(
73
- {
74
- "model": "gemma4-test",
75
- "messages": [{"role": "user", "content": "What is the weather in Hanoi?"}],
76
- "tools": [
77
- {
78
- "type": "function",
79
- "function": {
80
- "name": "get_weather",
81
- "description": "Get current weather for a city",
82
- "parameters": {
83
- "type": "object",
84
- "properties": {"city": {"type": "string"}},
85
- "required": ["city"],
86
- },
71
+def _build_chat_request(
72
+ *,
73
+ tool_choice: str | dict[str, Any],
74
+ chat_template_kwargs: dict[str, Any] | None = None,
75
+) -> ChatCompletionRequest:
76
+ data: dict[str, Any] = {
77
+ "model": "gemma4-test",
78
+ "messages": [{"role": "user", "content": "What is the weather in Hanoi?"}],
79
+ "tools": [
80
+ {
81
+ "type": "function",
82
+ "function": {
83
+ "name": "get_weather",
84
+ "description": "Get current weather for a city",
85
+ "parameters": {
86
+ "type": "object",
87
+ "properties": {"city": {"type": "string"}},
88
+ "required": ["city"],
89
},
88
- }
89
- ],
90
- "tool_choice": tool_choice,
91
- }
92
- )
90
+ },
91
+ }
92
+ ],
93
+ "tool_choice": tool_choice,
94
+ }
95
+ if chat_template_kwargs is not None:
96
+ data["chat_template_kwargs"] = chat_template_kwargs
97
+ return ChatCompletionRequest.model_validate(data)
98
99
100
class _StubTokenizer:
@@ -212,3 +217,34 @@ def test_gemma4_named_skips_structured_outputs_responses() -> None:
217
218
assert request.text is None
219
assert request.skip_special_tokens is False
220
+
221
+
222
+def test_gemma4_keeps_special_tokens_with_tools_thinking_disabled() -> None:
223
+ """tools active + thinking disabled: ``skip_special_tokens`` must stay
224
+ False so ``<|tool_call>`` delimiters reach the extractor. The merged
225
+ enable_thinking early-return stripped them, breaking tool calling when
226
+ thinking is off.
227
+ """
228
+ parser = Gemma4ToolParser(_StubTokenizer())
229
+ request = _build_chat_request(
230
+ tool_choice="auto", chat_template_kwargs={"enable_thinking": False}
231
+ )
232
+
233
+ parser.adjust_request(request)
234
+
235
+ assert request.skip_special_tokens is False
236
+
237
+
238
+def test_gemma4_strips_special_tokens_when_nothing_to_preserve() -> None:
239
+ """No active tools + thinking disabled: keep the default
240
+ (``skip_special_tokens=True``) so stray delimiters do not leak into
241
+ content.
242
+ """
243
+ parser = Gemma4ToolParser(_StubTokenizer())
244
+ request = _build_chat_request(
245
+ tool_choice="none", chat_template_kwargs={"enable_thinking": False}
246
+ )
247
+
248
+ parser.adjust_request(request)
249
+
250
+ assert request.skip_special_tokens is True
vllm/parser/gemma4.py
+13
-7
@@ -443,16 +443,22 @@ class Gemma4Parser(ParserEngine):
443
self,
444
request: ChatCompletionRequest | ResponsesRequest,
445
) -> ChatCompletionRequest | ResponsesRequest:
446
- """Skip ``skip_special_tokens=False`` when thinking is disabled.
446
+ """Keep special tokens when thinking or tool calls need them.
447
448
- When there are no reasoning channel tokens to preserve,
449
- keeping the default prevents tool-call delimiter tokens
450
- from leaking into content (e.g. with ``tool_choice="none"``).
448
+ ``skip_special_tokens`` must stay ``False`` when there is something to
449
+ preserve: reasoning channel tokens (thinking enabled) or tool-call
450
+ delimiters (tools active). Otherwise keep the default so stray
451
+ delimiters do not leak into content (e.g. ``tool_choice="none"`` with
452
+ thinking disabled).
453
"""
454
+ request = super().adjust_request(request)
455
chat_template_kwargs = getattr(request, "chat_template_kwargs", None) or {}
453
- if not chat_template_kwargs.get("enable_thinking", True):
454
- return request
455
- return super().adjust_request(request)
456
+ enable_thinking = chat_template_kwargs.get("enable_thinking", True)
457
+ has_tools = bool(getattr(request, "tools", None))
458
+ tools_active = has_tools and request.tool_choice != "none"
459
+ if not enable_thinking and not tools_active:
460
+ request.skip_special_tokens = True
461
+ return request
462
463
def _reset(self, initial_state=None) -> None:
464
super()._reset(initial_state=initial_state)