A high-throughput and memory-efficient inference and serving engine for LLMs
[Bugfix] Gemma4: skip forced JSON for required/named tool choice (#45795)
Signed-off-by: Federico Iezzi <fiezzi@google.com>
Federico committed
Jun 16, 2026 at 22:38 UTC
b9684d99e9ba50e06e375d7f1c63083e1e9286f7
2 files changed
+120
-1
tests/tool_use/test_gemma4_responses_adjust_request.py
+92
-1
@@ -20,6 +20,13 @@ calling for parsers relying on special-token delimiters (Gemma4):
20
tracked in ``__fields_set__``, which can drop the nested config from
21
``model_dump``. It also passed a ``description`` kwarg carrying the
22
wrong-purpose string ``"Response format for tool calling"``.
23
+
24
+3. :class:`Gemma4EngineToolParser` (the engine-based parser, #45588) sets
25
+ ``supports_required_and_named=False`` but did not skip the forced
26
+ ``structured_outputs`` JSON for ``required``/named tool choice. The model
27
+ was constrained to JSON the native parser cannot read, so the call leaked
28
+ as content with empty ``tool_calls``. ``adjust_request`` now skips that
29
+ constraint so Gemma4 emits its native ``<|tool_call>`` syntax.
30
"""
31
32
from __future__ import annotations
@@ -28,6 +35,7 @@ from typing import Any
35
36
from openai.types.responses.tool_param import FunctionToolParam
37
38
+from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
39
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
40
from vllm.tool_parsers.abstract_tool_parser import ToolParser
41
from vllm.tool_parsers.gemma4_engine_tool_parser import (
@@ -49,7 +57,7 @@ def _get_weather_tool() -> FunctionToolParam:
57
)
58
59
52
-def _build_responses_request(*, tool_choice: str) -> ResponsesRequest:
60
+def _build_responses_request(*, tool_choice: str | dict[str, Any]) -> ResponsesRequest:
61
return ResponsesRequest(
62
model="gemma4-test",
63
input=[{"role": "user", "content": "What is the weather in Hanoi?"}],
@@ -60,6 +68,30 @@ def _build_responses_request(*, tool_choice: str) -> ResponsesRequest:
68
)
69
70
71
+def _build_chat_request(*, tool_choice: str | dict[str, Any]) -> ChatCompletionRequest:
72
+ return ChatCompletionRequest.model_validate(
73
+ {
74
+ "model": "gemma4-test",
75
+ "messages": [{"role": "user", "content": "What is the weather in Hanoi?"}],
76
+ "tools": [
77
+ {
78
+ "type": "function",
79
+ "function": {
80
+ "name": "get_weather",
81
+ "description": "Get current weather for a city",
82
+ "parameters": {
83
+ "type": "object",
84
+ "properties": {"city": {"type": "string"}},
85
+ "required": ["city"],
86
+ },
87
+ },
88
+ }
89
+ ],
90
+ "tool_choice": tool_choice,
91
+ }
92
+ )
93
+
94
+
95
class _StubTokenizer:
96
"""Minimal tokenizer stub to satisfy ``Gemma4EngineToolParser.__init__``."""
97
@@ -121,3 +153,62 @@ def test_tool_parser_adjust_request_builds_valid_response_text_config() -> None:
153
# The old code passed a wrong-purpose string; valid field should now
154
# either be absent or None (the openai-python default).
155
assert fmt.get("description") in (None, "")
156
+
157
+
158
+def test_gemma4_required_skips_structured_outputs_chatcompletion() -> None:
159
+ """required + ChatCompletion: ``Gemma4EngineToolParser`` must skip the
160
+ forced JSON ``structured_outputs`` so the model emits its native
161
+ ``<|tool_call>`` syntax. The base parser constrained output to JSON the
162
+ native parser cannot read, leaking it as content with empty
163
+ ``tool_calls`` (regression after #45588).
164
+ """
165
+ parser = Gemma4ToolParser(_StubTokenizer())
166
+ request = _build_chat_request(tool_choice="required")
167
+
168
+ parser.adjust_request(request)
169
+
170
+ assert request.structured_outputs is None
171
+ assert request.skip_special_tokens is False
172
+
173
+
174
+def test_gemma4_named_skips_structured_outputs_chatcompletion() -> None:
175
+ """named + ChatCompletion: the forced single-function JSON schema must be
176
+ skipped, same as ``required``.
177
+ """
178
+ parser = Gemma4ToolParser(_StubTokenizer())
179
+ request = _build_chat_request(
180
+ tool_choice={"type": "function", "function": {"name": "get_weather"}}
181
+ )
182
+
183
+ parser.adjust_request(request)
184
+
185
+ assert request.structured_outputs is None
186
+ assert request.skip_special_tokens is False
187
+
188
+
189
+def test_gemma4_required_skips_structured_outputs_responses() -> None:
190
+ """required + Responses: the forced JSON schema (``request.text``) must be
191
+ skipped so the native delimiters reach the extractor.
192
+ """
193
+ parser = Gemma4ToolParser(_StubTokenizer())
194
+ request = _build_responses_request(tool_choice="required")
195
+
196
+ parser.adjust_request(request)
197
+
198
+ assert request.text is None
199
+ assert request.skip_special_tokens is False
200
+
201
+
202
+def test_gemma4_named_skips_structured_outputs_responses() -> None:
203
+ """named (``ToolChoiceFunction``) + Responses: the forced single-function
204
+ JSON schema must be skipped.
205
+ """
206
+ parser = Gemma4ToolParser(_StubTokenizer())
207
+ request = _build_responses_request(
208
+ tool_choice={"type": "function", "name": "get_weather"}
209
+ )
210
+
211
+ parser.adjust_request(request)
212
+
213
+ assert request.text is None
214
+ assert request.skip_special_tokens is False
vllm/tool_parsers/gemma4_engine_tool_parser.py
+28
@@ -1,8 +1,36 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+from openai.types.responses import ToolChoiceFunction
5
+
6
+from vllm.entrypoints.openai.chat_completion.protocol import (
7
+ ChatCompletionNamedToolChoiceParam,
8
+ ChatCompletionRequest,
9
+)
10
+from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
11
from vllm.parser.engine.registered_adapters import Gemma4ParserToolAdapter
12
13
14
class Gemma4EngineToolParser(Gemma4ParserToolAdapter): # type: ignore[valid-type, misc]
15
supports_required_and_named = False
16
+
17
+ def adjust_request(
18
+ self, request: ChatCompletionRequest | ResponsesRequest
19
+ ) -> ChatCompletionRequest | ResponsesRequest:
20
+ """Skip structured-output JSON for required/named tool choice.
21
+
22
+ Gemma4 emits its native ``<|tool_call>call:...`` syntax, which the
23
+ parser extracts directly. The base ``ToolParser.adjust_request`` would
24
+ set ``structured_outputs`` for required/named and force JSON via guided
25
+ decoding, conflicting with that native syntax (it leaks as content and
26
+ crashes EngineCore under speculative decoding). Skip it so the model
27
+ emits its native format (mirrors the GLM4 parser).
28
+ """
29
+ if request.tools:
30
+ tc = request.tool_choice
31
+ if tc == "required" or isinstance(
32
+ tc, (ChatCompletionNamedToolChoiceParam, ToolChoiceFunction)
33
+ ):
34
+ request.skip_special_tokens = False
35
+ return request
36
+ return super().adjust_request(request)