| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Regression tests for Responses API tool-calling request adjustment. |
| 4 | |
| 5 | Covers two bugs on the ``/v1/responses`` path that broke streaming tool |
| 6 | calling for parsers relying on special-token delimiters (Gemma4): |
| 7 | |
| 8 | 1. :class:`Gemma4ToolParser.adjust_request` used an |
| 9 | ``isinstance(request, ChatCompletionRequest)`` guard, so a |
| 10 | :class:`ResponsesRequest` with tools never had |
| 11 | ``skip_special_tokens`` flipped to ``False``. The default (``True``) |
| 12 | stripped ``<|tool_call>`` / ``<tool_call|>`` delimiters, causing |
| 13 | :meth:`Gemma4ToolParser.extract_tool_calls_streaming` to fall through |
| 14 | to the content branch and leak the raw ``call:fn{...}`` body via |
| 15 | ``response.output_text.delta``. |
| 16 | |
| 17 | 2. :meth:`ToolParser.adjust_request` built |
| 18 | :class:`ResponseTextConfig` in two steps (bare constructor then |
| 19 | ``.format = ...``). Under Pydantic v2 the later assignment is not |
| 20 | tracked in ``__fields_set__``, which can drop the nested config from |
| 21 | ``model_dump``. It also passed a ``description`` kwarg carrying the |
| 22 | wrong-purpose string ``"Response format for tool calling"``. |
| 23 | |
| 24 | 3. :class:`Gemma4EngineToolParser` (the engine-based parser, #45588) sets |
| 25 | ``supports_required_and_named=False`` but did not skip the forced |
| 26 | ``structured_outputs`` JSON for ``required``/named tool choice. The model |
| 27 | was constrained to JSON the native parser cannot read, so the call leaked |
| 28 | as content with empty ``tool_calls``. ``adjust_request`` now skips that |
| 29 | constraint so Gemma4 emits its native ``<|tool_call>`` syntax. |
| 30 | """ |
| 31 | |
| 32 | from __future__ import annotations |
| 33 | |
| 34 | from typing import Any |
| 35 | |
| 36 | from openai.types.responses.tool_param import FunctionToolParam |
| 37 | |
| 38 | from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest |
| 39 | from vllm.entrypoints.openai.responses.protocol import ResponsesRequest |
| 40 | from vllm.tool_parsers.abstract_tool_parser import ToolParser |
| 41 | from vllm.tool_parsers.gemma4_engine_tool_parser import ( |
| 42 | Gemma4EngineToolParser as Gemma4ToolParser, |
| 43 | ) |
| 44 | |
| 45 | |
| 46 | def _get_weather_tool() -> FunctionToolParam: |
| 47 | return FunctionToolParam( |
| 48 | type="function", |
| 49 | name="get_weather", |
| 50 | description="Get current weather for a city", |
| 51 | parameters={ |
| 52 | "type": "object", |
| 53 | "properties": {"city": {"type": "string"}}, |
| 54 | "required": ["city"], |
| 55 | }, |
| 56 | strict=True, |
| 57 | ) |
| 58 | |
| 59 | |
| 60 | def _build_responses_request(*, tool_choice: str | dict[str, Any]) -> ResponsesRequest: |
| 61 | return ResponsesRequest( |
| 62 | model="gemma4-test", |
| 63 | input=[{"role": "user", "content": "What is the weather in Hanoi?"}], |
| 64 | tools=[_get_weather_tool()], |
| 65 | tool_choice=tool_choice, |
| 66 | stream=True, |
| 67 | max_output_tokens=200, |
| 68 | ) |
| 69 | |
| 70 | |
| 71 | def _build_chat_request( |
| 72 | *, |
| 73 | tool_choice: str | dict[str, Any], |
| 74 | chat_template_kwargs: dict[str, Any] | None = None, |
| 75 | ) -> ChatCompletionRequest: |
| 76 | data: dict[str, Any] = { |
| 77 | "model": "gemma4-test", |
| 78 | "messages": [{"role": "user", "content": "What is the weather in Hanoi?"}], |
| 79 | "tools": [ |
| 80 | { |
| 81 | "type": "function", |
| 82 | "function": { |
| 83 | "name": "get_weather", |
| 84 | "description": "Get current weather for a city", |
| 85 | "parameters": { |
| 86 | "type": "object", |
| 87 | "properties": {"city": {"type": "string"}}, |
| 88 | "required": ["city"], |
| 89 | }, |
| 90 | }, |
| 91 | } |
| 92 | ], |
| 93 | "tool_choice": tool_choice, |
| 94 | } |
| 95 | if chat_template_kwargs is not None: |
| 96 | data["chat_template_kwargs"] = chat_template_kwargs |
| 97 | return ChatCompletionRequest.model_validate(data) |
| 98 | |
| 99 | |
| 100 | class _StubTokenizer: |
| 101 | """Minimal tokenizer stub to satisfy ``Gemma4EngineToolParser.__init__``.""" |
| 102 | |
| 103 | _VOCAB: dict[str, int] = { |
| 104 | "<|tool_call>": 256_000, |
| 105 | "<tool_call|>": 256_001, |
| 106 | '<|"|>': 52, |
| 107 | "<|channel>": 256_002, |
| 108 | "<channel|>": 256_003, |
| 109 | } |
| 110 | |
| 111 | def get_vocab(self) -> dict[str, int]: |
| 112 | return dict(self._VOCAB) |
| 113 | |
| 114 | @property |
| 115 | def all_special_tokens(self) -> list[str]: |
| 116 | return list(self._VOCAB.keys()) |
| 117 | |
| 118 | @property |
| 119 | def all_special_ids(self) -> list[int]: |
| 120 | return list(self._VOCAB.values()) |
| 121 | |
| 122 | |
| 123 | def test_gemma4_adjust_request_sets_skip_special_tokens_on_responses() -> None: |
| 124 | """``Gemma4ToolParser.adjust_request`` must flip |
| 125 | ``skip_special_tokens=False`` for both ``ChatCompletionRequest`` and |
| 126 | ``ResponsesRequest`` so that ``<|tool_call>`` delimiters reach the |
| 127 | streaming extractor. The previous |
| 128 | ``isinstance(ChatCompletionRequest)`` guard omitted the Responses |
| 129 | path, causing raw ``call:fn{...}`` text to leak via |
| 130 | ``response.output_text.delta``. |
| 131 | """ |
| 132 | parser = Gemma4ToolParser(_StubTokenizer()) |
| 133 | |
| 134 | request = _build_responses_request(tool_choice="auto") |
| 135 | assert request.skip_special_tokens is True, ( |
| 136 | "Precondition: ResponsesRequest.skip_special_tokens default is True" |
| 137 | ) |
| 138 | |
| 139 | parser.adjust_request(request) |
| 140 | |
| 141 | assert request.skip_special_tokens is False |
| 142 | |
| 143 | |
| 144 | def test_tool_parser_adjust_request_builds_valid_response_text_config() -> None: |
| 145 | """``ToolParser.adjust_request`` must produce a ``ResponseTextConfig`` |
| 146 | whose dumped form contains the JSON schema under the ``schema`` alias |
| 147 | and does not leak the unrelated ``"Response format for tool calling"`` |
| 148 | description string that the previous two-step construction injected. |
| 149 | """ |
| 150 | parser = ToolParser.__new__(ToolParser) |
| 151 | parser.model_tokenizer = None |
| 152 | |
| 153 | request = _build_responses_request(tool_choice="required") |
| 154 | ToolParser.adjust_request(parser, request) |
| 155 | |
| 156 | assert request.text is not None |
| 157 | assert request.text.format is not None |
| 158 | assert request.text.format.type == "json_schema" |
| 159 | |
| 160 | dump: dict[str, Any] = request.text.model_dump(mode="json", by_alias=True) |
| 161 | fmt = dump.get("format") or {} |
| 162 | assert fmt.get("type") == "json_schema" |
| 163 | assert fmt.get("name") == "tool_calling_response" |
| 164 | assert fmt.get("strict") is True |
| 165 | # Nested config must be present under the alias. Two-step Pydantic v2 |
| 166 | # construction could drop it from __fields_set__. |
| 167 | assert "schema" in fmt and isinstance(fmt["schema"], dict) |
| 168 | # The old code passed a wrong-purpose string; valid field should now |
| 169 | # either be absent or None (the openai-python default). |
| 170 | assert fmt.get("description") in (None, "") |
| 171 | |
| 172 | |
| 173 | def test_gemma4_required_skips_structured_outputs_chatcompletion() -> None: |
| 174 | """Required + ChatCompletion: ``Gemma4EngineToolParser`` must skip the |
| 175 | forced JSON ``structured_outputs`` so the model emits its native |
| 176 | ``<|tool_call>`` syntax. The base parser constrained output to JSON the |
| 177 | native parser cannot read, leaking it as content with empty |
| 178 | ``tool_calls`` (regression after #45588). |
| 179 | """ |
| 180 | parser = Gemma4ToolParser(_StubTokenizer()) |
| 181 | request = _build_chat_request(tool_choice="required") |
| 182 | |
| 183 | parser.adjust_request(request) |
| 184 | |
| 185 | assert request.structured_outputs is None |
| 186 | assert request.skip_special_tokens is False |
| 187 | |
| 188 | |
| 189 | def test_gemma4_named_skips_structured_outputs_chatcompletion() -> None: |
| 190 | """Named + ChatCompletion: the forced single-function JSON schema must be |
| 191 | skipped, same as ``required``. |
| 192 | """ |
| 193 | parser = Gemma4ToolParser(_StubTokenizer()) |
| 194 | request = _build_chat_request( |
| 195 | tool_choice={"type": "function", "function": {"name": "get_weather"}} |
| 196 | ) |
| 197 | |
| 198 | parser.adjust_request(request) |
| 199 | |
| 200 | assert request.structured_outputs is None |
| 201 | assert request.skip_special_tokens is False |
| 202 | |
| 203 | |
| 204 | def test_gemma4_required_skips_structured_outputs_responses() -> None: |
| 205 | """Required + Responses: the forced JSON schema (``request.text``) must be |
| 206 | skipped so the native delimiters reach the extractor. |
| 207 | """ |
| 208 | parser = Gemma4ToolParser(_StubTokenizer()) |
| 209 | request = _build_responses_request(tool_choice="required") |
| 210 | |
| 211 | parser.adjust_request(request) |
| 212 | |
| 213 | assert request.text is None |
| 214 | assert request.skip_special_tokens is False |
| 215 | |
| 216 | |
| 217 | def test_gemma4_named_skips_structured_outputs_responses() -> None: |
| 218 | """Named (``ToolChoiceFunction``) + Responses: the forced single-function |
| 219 | JSON schema must be skipped. |
| 220 | """ |
| 221 | parser = Gemma4ToolParser(_StubTokenizer()) |
| 222 | request = _build_responses_request( |
| 223 | tool_choice={"type": "function", "name": "get_weather"} |
| 224 | ) |
| 225 | |
| 226 | parser.adjust_request(request) |
| 227 | |
| 228 | assert request.text is None |
| 229 | assert request.skip_special_tokens is False |
| 230 | |
| 231 | |
| 232 | def test_gemma4_keeps_special_tokens_with_tools_thinking_disabled() -> None: |
| 233 | """Tools active + thinking disabled: ``skip_special_tokens`` must stay |
| 234 | False so ``<|tool_call>`` delimiters reach the extractor. The merged |
| 235 | enable_thinking early-return stripped them, breaking tool calling when |
| 236 | thinking is off. |
| 237 | """ |
| 238 | parser = Gemma4ToolParser(_StubTokenizer()) |
| 239 | request = _build_chat_request( |
| 240 | tool_choice="auto", chat_template_kwargs={"enable_thinking": False} |
| 241 | ) |
| 242 | |
| 243 | parser.adjust_request(request) |
| 244 | |
| 245 | assert request.skip_special_tokens is False |
| 246 | |
| 247 | |
| 248 | def test_gemma4_keeps_skip_special_tokens_false_when_nothing_to_preserve() -> None: |
| 249 | """No active tools + thinking disabled: ``skip_special_tokens`` stays |
| 250 | ``False`` because the parser engine's ``__DROP__`` terminal mechanism |
| 251 | strips unconfigured special tokens automatically. |
| 252 | """ |
| 253 | parser = Gemma4ToolParser(_StubTokenizer()) |
| 254 | request = _build_chat_request( |
| 255 | tool_choice="none", chat_template_kwargs={"enable_thinking": False} |
| 256 | ) |
| 257 | |
| 258 | parser.adjust_request(request) |
| 259 | |
| 260 | assert request.skip_special_tokens is False |