| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import json |
| 5 | from typing import Any |
| 6 | from unittest.mock import MagicMock, patch |
| 7 | |
| 8 | import pytest |
| 9 | from xgrammar import Grammar |
| 10 | from xgrammar.testing import _is_grammar_accept_string |
| 11 | |
| 12 | from vllm.entrypoints.openai.chat_completion.protocol import ( |
| 13 | ChatCompletionRequest, |
| 14 | ChatCompletionToolsParam, |
| 15 | ) |
| 16 | from vllm.parser.abstract_parser import DelegatingParser |
| 17 | from vllm.sampling_params import StructuredOutputsParams |
| 18 | from vllm.tool_parsers.abstract_tool_parser import ToolParser |
| 19 | from vllm.tool_parsers.qwen3_engine_tool_parser import Qwen3EngineToolParser |
| 20 | from vllm.tool_parsers.structural_tag_registry import ToolChoice |
| 21 | |
| 22 | |
| 23 | class TestToolChoice_Plus_ResponseFormat: |
| 24 | """Note(arpera): |
| 25 | Test cases for tool_choice={auto,required} + response_format |
| 26 | To keep it short: |
| 27 | DelegatingParser.adjust_request behavior in some corner cases is checked there |
| 28 | |
| 29 | Initial bug report: |
| 30 | https://github.com/vllm-project/vllm/issues/39929 |
| 31 | And PR that fixed this: |
| 32 | https://github.com/vllm-project/vllm/pull/56086 |
| 33 | """ |
| 34 | |
| 35 | # ================================ |
| 36 | # Helper methods |
| 37 | # ================================ |
| 38 | |
| 39 | @staticmethod |
| 40 | def _tools(strict: bool) -> list[ChatCompletionToolsParam]: |
| 41 | """Single get_weather tool, optionally marked as strict""" |
| 42 | function: dict[str, Any] = { |
| 43 | "name": "get_weather", |
| 44 | "parameters": { |
| 45 | "type": "object", |
| 46 | "properties": {"city": {"type": "string"}}, |
| 47 | "required": ["city"], |
| 48 | }, |
| 49 | } |
| 50 | if strict: |
| 51 | function["strict"] = True |
| 52 | return [ChatCompletionToolsParam(type="function", function=function)] |
| 53 | |
| 54 | @staticmethod |
| 55 | def _qwen_tool_call() -> str: |
| 56 | """Tool call for Qwen model that do support structural tag""" |
| 57 | return ( |
| 58 | "<tool_call>\n<function=get_weather>\n<parameter=city>\nParis\n" |
| 59 | "</parameter>\n</function>\n</tool_call>" |
| 60 | ) |
| 61 | |
| 62 | @staticmethod |
| 63 | def _json_schema_response_format() -> dict: |
| 64 | return { |
| 65 | "type": "json_schema", |
| 66 | "json_schema": { |
| 67 | "name": "answer", |
| 68 | "schema": { |
| 69 | "type": "object", |
| 70 | "properties": {"text": {"type": "string"}}, |
| 71 | "required": ["text"], |
| 72 | }, |
| 73 | }, |
| 74 | } |
| 75 | |
| 76 | @staticmethod |
| 77 | def _setup_request( |
| 78 | tools: list[ChatCompletionToolsParam], |
| 79 | tool_choice: ToolChoice, |
| 80 | response_format: dict, |
| 81 | ) -> ChatCompletionRequest: |
| 82 | request = ChatCompletionRequest( |
| 83 | messages=[], # for our test cases it's always empty |
| 84 | model="m", # Just a placeholder, don't pay much attention |
| 85 | tools=tools, |
| 86 | tool_choice=tool_choice, |
| 87 | response_format=response_format, |
| 88 | ) |
| 89 | return request |
| 90 | |
| 91 | @staticmethod |
| 92 | def _setup_abstract_parser( |
| 93 | tools: list[ChatCompletionToolsParam], |
| 94 | ) -> DelegatingParser: |
| 95 | """Construct parser that does NOT support structural tag""" |
| 96 | |
| 97 | class TestParser(DelegatingParser): |
| 98 | tool_parser_cls = ToolParser |
| 99 | |
| 100 | return TestParser(MagicMock(), tools=tools) |
| 101 | |
| 102 | @staticmethod |
| 103 | def _setup_qwen_parser( |
| 104 | tools: list[ChatCompletionToolsParam], |
| 105 | ) -> DelegatingParser: |
| 106 | """Construct parser that supports structural tag""" |
| 107 | |
| 108 | class TestParser(DelegatingParser): |
| 109 | tool_parser_cls = Qwen3EngineToolParser |
| 110 | |
| 111 | return TestParser(MagicMock(), tools=tools) |
| 112 | |
| 113 | # ================================ |
| 114 | # Test cases |
| 115 | # tool_choice=auto + response_format |
| 116 | # ================================ |
| 117 | |
| 118 | @pytest.mark.parametrize( |
| 119 | # In this test we check that for response_format |
| 120 | # resulting grammar accepts @compliant_output and rejects @non_compliant_output |
| 121 | # You can add more examples here if you see some corner cases not covered |
| 122 | ("response_format", "compliant_output", "non_compliant_output"), |
| 123 | [ |
| 124 | (_json_schema_response_format(), '{"text": "hi"}', '{"foo": 1}'), |
| 125 | ({"type": "json_object"}, '{"any": 1}', "[1, 2]"), |
| 126 | ], |
| 127 | # We test here two different response_format types: |
| 128 | ids=["json_schema", "json_object"], |
| 129 | ) |
| 130 | def test_auto_with_strict_tools( |
| 131 | self, |
| 132 | response_format: dict, |
| 133 | compliant_output: str, |
| 134 | non_compliant_output: str, |
| 135 | ): |
| 136 | tools = self._tools(strict=True) |
| 137 | request = self._setup_request( |
| 138 | tools=tools, |
| 139 | tool_choice="auto", |
| 140 | response_format=response_format, |
| 141 | ) |
| 142 | parser = self._setup_qwen_parser(tools) |
| 143 | out = parser.adjust_request(request) |
| 144 | |
| 145 | # Now check that request does not have response_format anymore |
| 146 | # but instead has structured_outputs set as structural tag |
| 147 | # And this structural tag is OR operation |
| 148 | assert out.tool_choice == "auto" |
| 149 | assert out.response_format is None |
| 150 | assert out.structured_outputs is not None |
| 151 | tag = json.loads(out.structured_outputs.structural_tag) |
| 152 | assert tag["format"]["type"] == "or" |
| 153 | grammar = Grammar.from_structural_tag(out.structured_outputs.structural_tag) |
| 154 | |
| 155 | assert _is_grammar_accept_string(grammar, compliant_output) |
| 156 | assert not _is_grammar_accept_string(grammar, non_compliant_output) |
| 157 | |
| 158 | # Also tool call must be accepted by grammar |
| 159 | assert _is_grammar_accept_string(grammar, self._qwen_tool_call()) |
| 160 | |
| 161 | # IMPORTANT(arpera): Regression test |
| 162 | # If we in adjust_request implementation by mistake |
| 163 | # construct structural tag using tool_choice=auto |
| 164 | # then such a structural tag would allow plain text as well |
| 165 | # We need to be sure that plain text is NOT accepted in our case |
| 166 | assert not _is_grammar_accept_string(grammar, "Hello") |
| 167 | |
| 168 | def test_auto_without_strict_tools(self): |
| 169 | tools = self._tools(strict=False) |
| 170 | request = self._setup_request( |
| 171 | tools=tools, |
| 172 | tool_choice="auto", |
| 173 | response_format=self._json_schema_response_format(), |
| 174 | ) |
| 175 | parser = self._setup_qwen_parser(tools) |
| 176 | |
| 177 | # There must be a warning that tool calls are disabled |
| 178 | # Consume that warning |
| 179 | with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn: |
| 180 | out = parser.adjust_request(request) |
| 181 | |
| 182 | assert out.response_format is not None |
| 183 | assert out.structured_outputs is None |
| 184 | mock_warn.assert_called_once() |
| 185 | |
| 186 | def test_when_model_does_not_have_structural_tag(self): |
| 187 | """Note(arpera): |
| 188 | When model does NOT have structural tag support |
| 189 | we apply constraint only for response_format |
| 190 | """ |
| 191 | tools = self._tools(strict=True) |
| 192 | request = self._setup_request( |
| 193 | tools=tools, |
| 194 | tool_choice="auto", |
| 195 | response_format=self._json_schema_response_format(), |
| 196 | ) |
| 197 | # SIC! use parser whose model does NOT support structural tag |
| 198 | parser = self._setup_abstract_parser(tools) |
| 199 | |
| 200 | with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn: |
| 201 | out = parser.adjust_request(request) |
| 202 | |
| 203 | assert out.response_format is not None |
| 204 | assert out.structured_outputs is None |
| 205 | mock_warn.assert_called_once() |
| 206 | |
| 207 | def test_auto_with_xgrammar_unsupported_schema(self): |
| 208 | """Conrer case based on Vadim's feedback in PR #56086 |
| 209 | https://github.com/vllm-project/vllm/pull/56086#pullrequestreview-5329203459 |
| 210 | """ |
| 211 | tools = self._tools(strict=True) |
| 212 | request = self._setup_request( |
| 213 | tools=tools, |
| 214 | tool_choice="auto", |
| 215 | response_format={ |
| 216 | "type": "json_schema", |
| 217 | "json_schema": { |
| 218 | "name": "answer", |
| 219 | "schema": {"type": "integer", "multipleOf": 2}, |
| 220 | }, |
| 221 | }, |
| 222 | ) |
| 223 | parser = self._setup_qwen_parser(tools) |
| 224 | |
| 225 | with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn: |
| 226 | out = parser.adjust_request(request) |
| 227 | |
| 228 | assert out.response_format is not None |
| 229 | assert out.structured_outputs is None |
| 230 | mock_warn.assert_called_once() |
| 231 | |
| 232 | def test_auto_with_xgrammar_unsupported_nested_schema(self): |
| 233 | """Conrer case based on Vadim's feedback in PR #56086 |
| 234 | https://github.com/vllm-project/vllm/pull/56086#pullrequestreview-5329203459 |
| 235 | """ |
| 236 | tools = self._tools(strict=True) |
| 237 | request = self._setup_request( |
| 238 | tools=tools, |
| 239 | tool_choice="auto", |
| 240 | response_format={ |
| 241 | "type": "json_schema", |
| 242 | "json_schema": { |
| 243 | "name": "answer", |
| 244 | "schema": { |
| 245 | "type": "object", |
| 246 | "properties": {"value": {"type": "number", "multipleOf": 0.5}}, |
| 247 | "required": ["value"], |
| 248 | }, |
| 249 | }, |
| 250 | }, |
| 251 | ) |
| 252 | parser = self._setup_qwen_parser(tools) |
| 253 | |
| 254 | with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn: |
| 255 | out = parser.adjust_request(request) |
| 256 | |
| 257 | assert out.response_format is not None |
| 258 | assert out.structured_outputs is None |
| 259 | mock_warn.assert_called_once() |
| 260 | |
| 261 | def test_auto_with_lark_grammar(self): |
| 262 | """Conrer case based on Vadim's feedback in PR #56086 |
| 263 | https://github.com/vllm-project/vllm/pull/56086#pullrequestreview-5329203459 |
| 264 | Lark grammars are converted to EBNF before being merged into the tag. |
| 265 | """ |
| 266 | tools = self._tools(strict=True) |
| 267 | request = ChatCompletionRequest( |
| 268 | messages=[], |
| 269 | model="m", |
| 270 | tools=tools, |
| 271 | tool_choice="auto", |
| 272 | structured_outputs=StructuredOutputsParams(grammar='start: "ok"'), |
| 273 | ) |
| 274 | parser = self._setup_qwen_parser(tools) |
| 275 | out = parser.adjust_request(request) |
| 276 | |
| 277 | assert out.structured_outputs is not None |
| 278 | grammar = Grammar.from_structural_tag(out.structured_outputs.structural_tag) |
| 279 | assert _is_grammar_accept_string(grammar, "ok") |
| 280 | assert not _is_grammar_accept_string(grammar, "bad") |
| 281 | assert _is_grammar_accept_string(grammar, self._qwen_tool_call()) |
| 282 | |
| 283 | # ================================ |
| 284 | # Test cases |
| 285 | # tool_choice=required + response_format |
| 286 | # ================================ |
| 287 | |
| 288 | def test_required(self): |
| 289 | tools = self._tools(strict=True) |
| 290 | request = self._setup_request( |
| 291 | tools=tools, |
| 292 | tool_choice="required", |
| 293 | response_format=self._json_schema_response_format(), |
| 294 | ) |
| 295 | parser = self._setup_qwen_parser(tools) |
| 296 | |
| 297 | with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn: |
| 298 | out = parser.adjust_request(request) |
| 299 | |
| 300 | assert out.response_format is None |
| 301 | assert out.structured_outputs is not None |
| 302 | grammar = Grammar.from_structural_tag(out.structured_outputs.structural_tag) |
| 303 | assert _is_grammar_accept_string(grammar, self._qwen_tool_call()) |
| 304 | assert not _is_grammar_accept_string(grammar, '{"text": "hi"}') |
| 305 | mock_warn.assert_called_once() |