A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 23:45 UTC
[Bugfix][Responses API] Fix streaming tool calls on /v1/responses (#39892)
Signed-off-by: Hoang Nguyen <118159510+hnt2601@users.noreply.github.com> Co-authored-by: Claude <noreply@anthropic.com>
Hoang Nguyen committed
Apr 20, 2026 at 10:24 UTC
6e10cb54f69bf26cd4986cda3faa5eb07f25875e
3 files changed
+137
-13
tests/tool_use/test_gemma4_responses_adjust_request.py
new
+116
@@ -0,0 +1,116 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Regression tests for Responses API tool-calling request adjustment.
4
+
5
+Covers two bugs on the ``/v1/responses`` path that broke streaming tool
6
+calling for parsers relying on special-token delimiters (Gemma4):
7
+
8
+1. :class:`Gemma4ToolParser.adjust_request` used an
9
+ ``isinstance(request, ChatCompletionRequest)`` guard, so a
10
+ :class:`ResponsesRequest` with tools never had
11
+ ``skip_special_tokens`` flipped to ``False``. The default (``True``)
12
+ stripped ``<|tool_call>`` / ``<tool_call|>`` delimiters, causing
13
+ :meth:`Gemma4ToolParser.extract_tool_calls_streaming` to fall through
14
+ to the content branch and leak the raw ``call:fn{...}`` body via
15
+ ``response.output_text.delta``.
16
+
17
+2. :meth:`ToolParser.adjust_request` built
18
+ :class:`ResponseTextConfig` in two steps (bare constructor then
19
+ ``.format = ...``). Under Pydantic v2 the later assignment is not
20
+ tracked in ``__fields_set__``, which can drop the nested config from
21
+ ``model_dump``. It also passed a ``description`` kwarg carrying the
22
+ wrong-purpose string ``"Response format for tool calling"``.
23
+"""
24
+
25
+from __future__ import annotations
26
+
27
+from typing import Any
28
+
29
+from openai.types.responses.tool_param import FunctionToolParam
30
+
31
+from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
32
+from vllm.tool_parsers.abstract_tool_parser import ToolParser
33
+from vllm.tool_parsers.gemma4_tool_parser import Gemma4ToolParser
34
+
35
+
36
+def _get_weather_tool() -> FunctionToolParam:
37
+ return FunctionToolParam(
38
+ type="function",
39
+ name="get_weather",
40
+ description="Get current weather for a city",
41
+ parameters={
42
+ "type": "object",
43
+ "properties": {"city": {"type": "string"}},
44
+ "required": ["city"],
45
+ },
46
+ strict=True,
47
+ )
48
+
49
+
50
+def _build_responses_request(*, tool_choice: str) -> ResponsesRequest:
51
+ return ResponsesRequest(
52
+ model="gemma4-test",
53
+ input=[{"role": "user", "content": "What is the weather in Hanoi?"}],
54
+ tools=[_get_weather_tool()],
55
+ tool_choice=tool_choice,
56
+ stream=True,
57
+ max_output_tokens=200,
58
+ )
59
+
60
+
61
+class _StubTokenizer:
62
+ """Minimal tokenizer stub to satisfy ``Gemma4ToolParser.__init__``."""
63
+
64
+ def get_vocab(self) -> dict[str, int]:
65
+ return {"<|tool_call>": 256_000, "<tool_call|>": 256_001, '<|"|>': 52}
66
+
67
+
68
+def test_gemma4_adjust_request_sets_skip_special_tokens_on_responses() -> None:
69
+ """``Gemma4ToolParser.adjust_request`` must flip
70
+ ``skip_special_tokens=False`` for both ``ChatCompletionRequest`` and
71
+ ``ResponsesRequest`` so that ``<|tool_call>`` delimiters reach the
72
+ streaming extractor. The previous
73
+ ``isinstance(ChatCompletionRequest)`` guard omitted the Responses
74
+ path, causing raw ``call:fn{...}`` text to leak via
75
+ ``response.output_text.delta``.
76
+ """
77
+ parser = Gemma4ToolParser.__new__(Gemma4ToolParser)
78
+ parser.model_tokenizer = _StubTokenizer()
79
+
80
+ request = _build_responses_request(tool_choice="auto")
81
+ assert request.skip_special_tokens is True, (
82
+ "Precondition: ResponsesRequest.skip_special_tokens default is True"
83
+ )
84
+
85
+ Gemma4ToolParser.adjust_request(parser, request)
86
+
87
+ assert request.skip_special_tokens is False
88
+
89
+
90
+def test_tool_parser_adjust_request_builds_valid_response_text_config() -> None:
91
+ """``ToolParser.adjust_request`` must produce a ``ResponseTextConfig``
92
+ whose dumped form contains the JSON schema under the ``schema`` alias
93
+ and does not leak the unrelated ``"Response format for tool calling"``
94
+ description string that the previous two-step construction injected.
95
+ """
96
+ parser = ToolParser.__new__(ToolParser)
97
+ parser.model_tokenizer = None
98
+
99
+ request = _build_responses_request(tool_choice="required")
100
+ ToolParser.adjust_request(parser, request)
101
+
102
+ assert request.text is not None
103
+ assert request.text.format is not None
104
+ assert request.text.format.type == "json_schema"
105
+
106
+ dump: dict[str, Any] = request.text.model_dump(mode="json", by_alias=True)
107
+ fmt = dump.get("format") or {}
108
+ assert fmt.get("type") == "json_schema"
109
+ assert fmt.get("name") == "tool_calling_response"
110
+ assert fmt.get("strict") is True
111
+ # Nested config must be present under the alias. Two-step Pydantic v2
112
+ # construction could drop it from __fields_set__.
113
+ assert "schema" in fmt and isinstance(fmt["schema"], dict)
114
+ # The old code passed a wrong-purpose string; valid field should now
115
+ # either be absent or None (the openai-python default).
116
+ assert fmt.get("description") in (None, "")
vllm/tool_parsers/abstract_tool_parser.py
+14
-7
@@ -103,13 +103,20 @@ class ToolParser:
103
)
104
request.response_format = None
105
if isinstance(request, ResponsesRequest):
106
- request.text = ResponseTextConfig()
107
- request.text.format = ResponseFormatTextJSONSchemaConfig(
108
- name="tool_calling_response",
109
- schema=json_schema_from_tool,
110
- type="json_schema",
111
- description="Response format for tool calling",
112
- strict=True,
106
+ # Single-shot construction so Pydantic v2 tracks `format`
107
+ # in __fields_set__ — assigning to `.format` after the bare
108
+ # `ResponseTextConfig()` constructor does not, which can
109
+ # drop the nested config from `model_dump`. Also drop the
110
+ # `description` kwarg: it is not a field on
111
+ # ResponseFormatTextJSONSchemaConfig and was being silently
112
+ # passed through as extra.
113
+ request.text = ResponseTextConfig(
114
+ format=ResponseFormatTextJSONSchemaConfig(
115
+ type="json_schema",
116
+ name="tool_calling_response",
117
+ schema=json_schema_from_tool,
118
+ strict=True,
119
+ )
120
)
121
122
return request
vllm/tool_parsers/gemma4_tool_parser.py
+7
-6
@@ -360,12 +360,13 @@ class Gemma4ToolParser(ToolParser):
360
self, request: ChatCompletionRequest | ResponsesRequest
361
) -> ChatCompletionRequest | ResponsesRequest:
362
request = super().adjust_request(request)
363
- if (
364
- isinstance(request, ChatCompletionRequest)
365
- and request.tools
366
- and request.tool_choice != "none"
367
- ):
368
- # Don't skip special tokens — <|tool_call> etc. are needed
363
+ if request.tools and request.tool_choice != "none":
364
+ # Don't skip special tokens — <|tool_call> etc. are needed for
365
+ # the parser to detect tool calls. Apply to BOTH
366
+ # ChatCompletionRequest and ResponsesRequest (the previous
367
+ # isinstance(ChatCompletionRequest) guard caused tool-call
368
+ # delimiters to be stripped on /v1/responses, leaking raw
369
+ # `call:fn{...}` text via output_text.delta).
370
request.skip_special_tokens = False
371
return request
372