@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7c572544e4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[GPT-OSS] Structure_Tag support for gpt-oss tool-call in cot (#25515)

Signed-off-by: Hanchenli <lihanc2002@gmail.com> Signed-off-by: Hanchenli <61769611+Hanchenli@users.noreply.github.com> Signed-off-by: Wei Wei <wwei6@meta.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Wei Wei <wwei6@meta.com> Co-authored-by: Wei Wei <weiweinpu@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>

Hanchenli committed Oct 17, 2025 at 21:55 UTC 7c572544e4292cb97e2f4f5a68c43f02406ad1d6
14 files changed +911 -32
tests/entrypoints/openai/test_gptoss_structural_tags_integration.py new
+280
@@ -0,0 +1,280 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +"""Integration tests for GPT-OSS structural tags functionality (PR #25515)."""
5 +
6 +import json
7 +from unittest.mock import Mock
8 +
9 +import pytest
10 +
11 +from vllm.entrypoints.openai.protocol import (
12 + StructuredOutputsParams,
13 +)
14 +from vllm.entrypoints.tool_server import ToolServer
15 +from vllm.reasoning.gptoss_reasoning_parser import (
16 + GptOssReasoningParser,
17 +)
18 +
19 +
20 +class TestGptOssStructuralTagsIntegration:
21 + """Integration tests for structural tags in GPT-OSS tool calls."""
22 +
23 + @pytest.fixture
24 + def mock_tokenizer(self):
25 + """Create a mock tokenizer."""
26 + tokenizer = Mock()
27 + tokenizer.encode = Mock(return_value=[1, 2, 3, 4, 5])
28 + return tokenizer
29 +
30 + @pytest.fixture
31 + def gptoss_parser(self, mock_tokenizer):
32 + """Create a real GptOssReasoningParser instance."""
33 + return GptOssReasoningParser(mock_tokenizer)
34 +
35 + @pytest.fixture
36 + def tool_server_with_python(self):
37 + """Create a tool server with Python tool enabled."""
38 + tool_server = Mock(spec=ToolServer)
39 + tool_server.has_tool = Mock(side_effect=lambda tool: tool == "python")
40 + return tool_server
41 +
42 + @pytest.fixture
43 + def tool_server_empty(self):
44 + """Create a tool server with no tools."""
45 + tool_server = Mock(spec=ToolServer)
46 + tool_server.has_tool = Mock(return_value=False)
47 + return tool_server
48 +
49 + def test_end_to_end_no_tools(self, gptoss_parser):
50 + """Test end-to-end flow when no tools are available."""
51 + # Test the parser directly
52 + result = gptoss_parser.prepare_structured_tag(None, None)
53 + parsed_result = json.loads(result)
54 +
55 + # Verify basic structure
56 + assert parsed_result["type"] == "structural_tag"
57 + assert parsed_result["format"]["type"] == "triggered_tags"
58 + assert len(parsed_result["format"]["tags"]) == 1
59 +
60 + # Verify only analysis channel is allowed
61 + analysis_tag = parsed_result["format"]["tags"][0]
62 + assert analysis_tag["begin"] == "<|channel|>analysis<|message|>"
63 + assert analysis_tag["content"]["type"] == "any_text"
64 + assert analysis_tag["end"] == "<|end|>"
65 +
66 + # Verify triggers
67 + assert parsed_result["format"]["triggers"] == ["<|channel|>analysis"]
68 + assert parsed_result["format"]["stop_after_first"] is False
69 +
70 + def test_end_to_end_with_python_tool(self, gptoss_parser, tool_server_with_python):
71 + """Test end-to-end flow with Python tool enabled."""
72 + result = gptoss_parser.prepare_structured_tag(None, tool_server_with_python)
73 + parsed_result = json.loads(result)
74 +
75 + # Should have analysis tag + 2 python tags
76 + assert len(parsed_result["format"]["tags"]) == 3
77 +
78 + # Verify all expected tags are present
79 + tag_begins = [tag["begin"] for tag in parsed_result["format"]["tags"]]
80 + expected_begins = [
81 + "<|channel|>analysis<|message|>",
82 + "<|channel|>commentary to=python",
83 + "<|channel|>analysis to=python",
84 + ]
85 +
86 + for expected in expected_begins:
87 + assert expected in tag_begins
88 +
89 + # Verify triggers include commentary
90 + assert "<|channel|>analysis" in parsed_result["format"]["triggers"]
91 + assert "<|channel|>commentary to=" in parsed_result["format"]["triggers"]
92 +
93 + def test_structured_outputs_params_integration(
94 + self, gptoss_parser, tool_server_with_python
95 + ):
96 + """Test integration with StructuredOutputsParams."""
97 + # Generate structural tag
98 + structural_tag = gptoss_parser.prepare_structured_tag(
99 + None, tool_server_with_python
100 + )
101 +
102 + # Create StructuredOutputsParams
103 + params = StructuredOutputsParams(structural_tag=structural_tag)
104 +
105 + # Verify the tag is properly stored and accessible
106 + assert params.structural_tag == structural_tag
107 +
108 + # Verify the tag is valid JSON
109 + parsed_tag = json.loads(params.structural_tag)
110 + assert parsed_tag["type"] == "structural_tag"
111 +
112 + @pytest.mark.parametrize(
113 + "browser, python, container, expected_tags",
114 + [
115 + # No tools
116 + (False, False, False, 1),
117 + # Single tool
118 + (True, False, False, 3),
119 + # Multiple tools
120 + (True, True, False, 5),
121 + # All tools
122 + (True, True, True, 7),
123 + ],
124 + )
125 + def test_tool_server_interaction_flow(
126 + self, gptoss_parser, browser, python, container, expected_tags
127 + ):
128 + """Test the complete tool server interaction flow."""
129 +
130 + # Create a mock ToolServer
131 + tool_server = Mock(spec=ToolServer)
132 +
133 + # Simulate tool availability based on parameters
134 + tool_server.has_tool = Mock(
135 + side_effect=lambda tool: {
136 + "browser": browser,
137 + "python": python,
138 + "container": container,
139 + }.get(tool, False)
140 + )
141 +
142 + # Run the parser and verify results
143 + result = gptoss_parser.prepare_structured_tag(None, tool_server)
144 + parsed_result = json.loads(result)
145 +
146 + # Validate number of tags
147 + assert len(parsed_result["format"]["tags"]) == expected_tags
148 +
149 + # Verify tool-specific tags exist for enabled tools
150 + tag_begins = [tag["begin"] for tag in parsed_result["format"]["tags"]]
151 + for tool, enabled in {
152 + "browser": browser,
153 + "python": python,
154 + "container": container,
155 + }.items():
156 + if enabled:
157 + assert f"<|channel|>commentary to={tool}" in tag_begins
158 + assert f"<|channel|>analysis to={tool}" in tag_begins
159 +
160 + def test_original_tag_preservation(self, gptoss_parser, tool_server_with_python):
161 + """Test that original tags are preserved when provided."""
162 + original_tag = '{"type": "custom_tag", "data": "preserved"}'
163 +
164 + result = gptoss_parser.prepare_structured_tag(
165 + original_tag, tool_server_with_python
166 + )
167 +
168 + # Should return original tag unchanged
169 + assert result == original_tag
170 +
171 + @pytest.mark.parametrize(
172 + "tools",
173 + [
174 + [],
175 + ["browser"],
176 + ["python"],
177 + ["container"],
178 + ["browser", "python"],
179 + ["browser", "container"],
180 + ["python", "container"],
181 + ["browser", "python", "container"],
182 + ],
183 + )
184 + def test_json_validity_comprehensive(self, gptoss_parser, tools):
185 + """Test JSON validity across all possible tool combinations."""
186 +
187 + tool_server = Mock(spec=ToolServer)
188 + tool_server.has_tool = Mock(side_effect=lambda tool: tool in tools)
189 +
190 + result = gptoss_parser.prepare_structured_tag(None, tool_server)
191 +
192 + # Should be valid JSON
193 + parsed_result = json.loads(result)
194 +
195 + # Should have correct structure
196 + assert parsed_result["type"] == "structural_tag"
197 + assert "format" in parsed_result
198 + assert "tags" in parsed_result["format"]
199 + assert "triggers" in parsed_result["format"]
200 +
201 + # Tag count should be: 1 (analysis) + 2 * len(tools)
202 + expected_tag_count = 1 + (2 * len(tools))
203 + assert len(parsed_result["format"]["tags"]) == expected_tag_count
204 +
205 + def test_error_handling_invalid_tool_server(self, gptoss_parser):
206 + """Test error handling with invalid tool server."""
207 + # Tool server that raises exceptions
208 + tool_server = Mock(spec=ToolServer)
209 + tool_server.has_tool = Mock(side_effect=Exception("Tool server error"))
210 +
211 + # Should handle gracefully and still return a valid tag
212 + with pytest.raises(Exception, match="Tool server error"):
213 + gptoss_parser.prepare_structured_tag(None, tool_server)
214 +
215 + def test_concurrent_requests_isolation(self, gptoss_parser):
216 + """Test that concurrent requests don't interfere with each other."""
217 + # Simulate concurrent requests with different tool servers
218 + tool_server_1 = Mock(spec=ToolServer)
219 + tool_server_1.has_tool = Mock(side_effect=lambda tool: tool == "python")
220 +
221 + tool_server_2 = Mock(spec=ToolServer)
222 + tool_server_2.has_tool = Mock(side_effect=lambda tool: tool == "browser")
223 +
224 + # Generate tags concurrently
225 + result_1 = gptoss_parser.prepare_structured_tag(None, tool_server_1)
226 + result_2 = gptoss_parser.prepare_structured_tag(None, tool_server_2)
227 +
228 + # Parse results
229 + parsed_1 = json.loads(result_1)
230 + parsed_2 = json.loads(result_2)
231 +
232 + # Verify they have different tool configurations
233 + tags_1 = [tag["begin"] for tag in parsed_1["format"]["tags"]]
234 + tags_2 = [tag["begin"] for tag in parsed_2["format"]["tags"]]
235 +
236 + # Result 1 should have python tags
237 + assert "<|channel|>commentary to=python" in tags_1
238 + assert "<|channel|>commentary to=browser" not in tags_1
239 +
240 + # Result 2 should have browser tags
241 + assert "<|channel|>commentary to=browser" in tags_2
242 + assert "<|channel|>commentary to=python" not in tags_2
243 +
244 + def test_tag_format_consistency(self, gptoss_parser):
245 + """Test that all generated tags follow consistent format."""
246 + tool_server = Mock(spec=ToolServer)
247 + tool_server.has_tool = Mock(
248 + side_effect=lambda tool: tool in ["python", "browser"]
249 + )
250 +
251 + result = gptoss_parser.prepare_structured_tag(None, tool_server)
252 + parsed_result = json.loads(result)
253 +
254 + # Verify all tags have required fields
255 + for tag in parsed_result["format"]["tags"]:
256 + assert "begin" in tag
257 + assert "content" in tag
258 + assert "end" in tag
259 + assert tag["content"]["type"] == "any_text"
260 + assert tag["end"] == "<|end|>"
261 +
262 + # Verify begin format
263 + assert tag["begin"].startswith("<|channel|>")
264 +
265 + def test_trigger_configuration(self, gptoss_parser):
266 + """Test trigger configuration for different tool setups."""
267 + # Test with no tools
268 + result_no_tools = gptoss_parser.prepare_structured_tag(None, None)
269 + parsed_no_tools = json.loads(result_no_tools)
270 + assert parsed_no_tools["format"]["triggers"] == ["<|channel|>analysis"]
271 +
272 + # Test with tools
273 + tool_server = Mock(spec=ToolServer)
274 + tool_server.has_tool = Mock(side_effect=lambda tool: tool == "python")
275 +
276 + result_with_tools = gptoss_parser.prepare_structured_tag(None, tool_server)
277 + parsed_with_tools = json.loads(result_with_tools)
278 +
279 + expected_triggers = ["<|channel|>analysis", "<|channel|>commentary to="]
280 + assert set(parsed_with_tools["format"]["triggers"]) == set(expected_triggers)
tests/v1/entrypoints/llm/test_struct_output_generate.py
+46
@@ -864,3 +864,49 @@ def test_structured_output_batched_with_non_structured_outputs_requests(
864 # non-structured outputs requests should not return a valid JSON here
865 with pytest.raises(ValueError):
866 output_json = json.loads(generated_text)
867 +
868 +
869 +@pytest.mark.parametrize("guided_decoding_backend", ["xgrammar"])
870 +def test_structured_output_with_structural_tag(
871 + monkeypatch: pytest.MonkeyPatch,
872 + guided_decoding_backend: str,
873 +):
874 + monkeypatch.setenv("VLLM_USE_V1", "1")
875 +
876 + llm = LLM(
877 + model="Qwen/Qwen2.5-1.5B-Instruct",
878 + guided_decoding_backend=guided_decoding_backend,
879 + )
880 +
881 + structural_tag_config = {
882 + "type": "structural_tag",
883 + "format": {
884 + "type": "triggered_tags",
885 + "tags": [
886 + {"begin": "hello_flag", "content": {"type": "any_text"}, "end": "hello"}
887 + ],
888 + "triggers": ["hello"],
889 + "stop_after_first": False,
890 + },
891 + }
892 +
893 + sampling_params = SamplingParams(
894 + temperature=0.0,
895 + max_tokens=500,
896 + guided_decoding=StructuredOutputsParams(
897 + structural_tag=json.dumps(structural_tag_config)
898 + ),
899 + )
900 +
901 + prompt = "Hello and repete hello 10 times, do not say anything else. Only say hello hello hello, now start"
902 + outputs = llm.generate(prompt, sampling_params=sampling_params, use_tqdm=True)
903 + assert outputs is not None
904 + for output in outputs:
905 + assert output is not None
906 + assert isinstance(output, RequestOutput)
907 + prompt = output.prompt
908 + generated_text = output.outputs[0].text
909 + assert generated_text is not None
910 + assert "hello_flag" in generated_text, (
911 + f"Expected 'hello_flag' to be in generated text, but got: {generated_text}"
912 + )
tests/v1/structured_output/test_gptoss_structural_tags.py new
+172
@@ -0,0 +1,172 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +"""Unit tests for GPT-OSS structural tag support in reasoning (PR #25515)."""
5 +
6 +import json
7 +from unittest.mock import Mock
8 +
9 +import pytest
10 +
11 +from vllm.entrypoints.tool_server import ToolServer
12 +from vllm.reasoning.gptoss_reasoning_parser import (
13 + GptOssReasoningParser,
14 + from_builtin_tool_to_tag,
15 + no_func_reaonsing_tag,
16 + tag_with_builtin_funcs,
17 +)
18 +
19 +
20 +class TestGptOssReasoningParser:
21 + """Test cases for GptOssReasoningParser structural tag functionality."""
22 +
23 + @pytest.fixture
24 + def mock_tokenizer(self):
25 + """Create a mock tokenizer for testing."""
26 + tokenizer = Mock()
27 + tokenizer.encode = Mock(return_value=[1, 2, 3, 4, 5])
28 + return tokenizer
29 +
30 + @pytest.fixture
31 + def reasoning_parser(self, mock_tokenizer):
32 + """Create a GptOssReasoningParser instance."""
33 + return GptOssReasoningParser(mock_tokenizer)
34 +
35 + @pytest.fixture
36 + def mock_tool_server_empty(self):
37 + """Create a mock ToolServer with no tools."""
38 + tool_server = Mock(spec=ToolServer)
39 + tool_server.has_tool = Mock(return_value=False)
40 + return tool_server
41 +
42 + @pytest.fixture
43 + def mock_tool_server_with_browser(self):
44 + """Create a mock ToolServer with browser tool."""
45 + tool_server = Mock(spec=ToolServer)
46 + tool_server.has_tool = Mock(side_effect=lambda tool: tool == "browser")
47 + return tool_server
48 +
49 + @pytest.fixture
50 + def mock_tool_server_with_all_tools(self):
51 + """Create a mock ToolServer with all builtin tools."""
52 + tool_server = Mock(spec=ToolServer)
53 + tool_server.has_tool = Mock(
54 + side_effect=lambda tool: tool in ["browser", "python", "container"]
55 + )
56 + return tool_server
57 +
58 + def test_prepare_structured_tag_no_tool_server(self, reasoning_parser):
59 + """Test prepare_structured_tag with no tool server."""
60 + result = reasoning_parser.prepare_structured_tag(None, None)
61 + expected = json.dumps(no_func_reaonsing_tag)
62 +
63 + assert result == expected
64 +
65 + # Verify the structure is correct
66 + parsed = json.loads(result)
67 + assert parsed["type"] == "structural_tag"
68 + assert parsed["format"]["type"] == "triggered_tags"
69 + assert len(parsed["format"]["tags"]) == 1
70 + assert parsed["format"]["tags"][0]["begin"] == "<|channel|>analysis<|message|>"
71 + assert parsed["format"]["triggers"] == ["<|channel|>analysis"]
72 +
73 + def test_prepare_structured_tag_with_all_tools(
74 + self, reasoning_parser, mock_tool_server_with_all_tools
75 + ):
76 + """Test prepare_structured_tag with all builtin tools."""
77 + result = reasoning_parser.prepare_structured_tag(
78 + None, mock_tool_server_with_all_tools
79 + )
80 + parsed = json.loads(result)
81 +
82 + # Should have analysis tag + tags for all 3 tools (2 tags each)
83 + assert len(parsed["format"]["tags"]) == 7 # 1 analysis + 6 tool tags
84 +
85 + # Check all tool tags are present
86 + tag_begins = [tag["begin"] for tag in parsed["format"]["tags"]]
87 + for tool in ["browser", "python", "container"]:
88 + assert f"<|channel|>commentary to={tool}" in tag_begins
89 + assert f"<|channel|>analysis to={tool}" in tag_begins
90 +
91 + def test_prepare_structured_tag_with_original_tag(self, reasoning_parser):
92 + """Test prepare_structured_tag when original_tag is provided."""
93 + original_tag = '{"custom": "tag"}'
94 + result = reasoning_parser.prepare_structured_tag(original_tag, None)
95 +
96 + # Should return the original tag unchanged
97 + assert result == original_tag
98 +
99 + def test_from_builtin_tool_to_tag(self):
100 + """Test from_builtin_tool_to_tag function."""
101 + tags = from_builtin_tool_to_tag("python")
102 +
103 + assert len(tags) == 2
104 + assert tags[0]["begin"] == "<|channel|>commentary to=python"
105 + assert tags[0]["content"]["type"] == "any_text"
106 + assert tags[0]["end"] == "<|end|>"
107 +
108 + assert tags[1]["begin"] == "<|channel|>analysis to=python"
109 + assert tags[1]["content"]["type"] == "any_text"
110 + assert tags[1]["end"] == "<|end|>"
111 +
112 + def test_tag_with_builtin_funcs(self):
113 + """Test tag_with_builtin_funcs function."""
114 + builtin_tools = ["browser", "python"]
115 + result = tag_with_builtin_funcs(no_func_reaonsing_tag, builtin_tools)
116 +
117 + assert result["type"] == "structural_tag"
118 + # Should have original analysis tag + 2 tags per tool
119 + assert len(result["format"]["tags"]) == 5 # 1 + 2*2
120 +
121 + # Should have added commentary trigger
122 + assert "<|channel|>commentary to=" in result["format"]["triggers"]
123 + assert "<|channel|>analysis" in result["format"]["triggers"]
124 +
125 + def test_tag_structure_invariants(self):
126 + """Test that the basic tag structure follows expected format."""
127 + # Test the base no_func_reaonsing_tag structure
128 + assert no_func_reaonsing_tag["type"] == "structural_tag"
129 + assert no_func_reaonsing_tag["format"]["type"] == "triggered_tags"
130 + assert no_func_reaonsing_tag["format"]["stop_after_first"] is False
131 +
132 + # Verify analysis tag structure
133 + analysis_tag = no_func_reaonsing_tag["format"]["tags"][0]
134 + assert analysis_tag["begin"] == "<|channel|>analysis<|message|>"
135 + assert analysis_tag["content"]["type"] == "any_text"
136 + assert analysis_tag["end"] == "<|end|>"
137 +
138 + def test_json_serialization_valid(
139 + self, reasoning_parser, mock_tool_server_with_all_tools
140 + ):
141 + """Test that all generated tags produce valid JSON."""
142 + # Test with no tool server
143 + result1 = reasoning_parser.prepare_structured_tag(None, None)
144 + json.loads(result1) # Should not raise
145 +
146 + # Test with empty tool server
147 + empty_server = Mock(spec=ToolServer)
148 + empty_server.has_tool = Mock(return_value=False)
149 + result2 = reasoning_parser.prepare_structured_tag(None, empty_server)
150 + json.loads(result2) # Should not raise
151 +
152 + # Test with tools
153 + result3 = reasoning_parser.prepare_structured_tag(
154 + None, mock_tool_server_with_all_tools
155 + )
156 + json.loads(result3) # Should not raise
157 +
158 + @pytest.mark.parametrize("tool_name", ["browser", "python", "container"])
159 + def test_single_tool_integration(self, reasoning_parser, tool_name):
160 + """Test integration with individual tools."""
161 + tool_server = Mock(spec=ToolServer)
162 + tool_server.has_tool = Mock(side_effect=lambda tool: tool == tool_name)
163 +
164 + result = reasoning_parser.prepare_structured_tag(None, tool_server)
165 + parsed = json.loads(result)
166 +
167 + # Should have 1 analysis + 2 tool-specific tags
168 + assert len(parsed["format"]["tags"]) == 3
169 +
170 + tag_begins = [tag["begin"] for tag in parsed["format"]["tags"]]
171 + assert f"<|channel|>commentary to={tool_name}" in tag_begins
172 + assert f"<|channel|>analysis to={tool_name}" in tag_begins
tests/v1/structured_output/test_reasoning_structured_output.py new
+207
@@ -0,0 +1,207 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +"""Unit tests for reasoning-aware structured output functionality (PR #25515)."""
5 +
6 +from unittest.mock import Mock
7 +
8 +import pytest
9 +
10 +from vllm.config import ModelConfig, SchedulerConfig, VllmConfig
11 +from vllm.reasoning import ReasoningParser
12 +from vllm.v1.request import Request
13 +from vllm.v1.structured_output import StructuredOutputManager
14 +
15 +
16 +class TestReasoningStructuredOutput:
17 + """Test reasoning-aware structured output functionality."""
18 +
19 + @pytest.fixture
20 + def mock_model_config(self):
21 + """Create a mock ModelConfig."""
22 + config = Mock(spec=ModelConfig)
23 + config.skip_tokenizer_init = True # Skip tokenizer init to avoid network calls
24 + config.get_vocab_size = Mock(return_value=50000)
25 + # Add missing runner_type attribute that tokenizer initialization expects
26 + config.runner_type = "generate"
27 + # Add other attributes that tokenizer initialization might need
28 + config.tokenizer = "test-tokenizer"
29 + config.tokenizer_mode = "auto"
30 + config.trust_remote_code = False
31 + config.tokenizer_revision = None
32 + return config
33 +
34 + @pytest.fixture
35 + def mock_scheduler_config(self):
36 + """Create a mock SchedulerConfig."""
37 + config = Mock(spec=SchedulerConfig)
38 + config.max_num_seqs = 128
39 + return config
40 +
41 + @pytest.fixture
42 + def mock_vllm_config(self, mock_model_config, mock_scheduler_config):
43 + """Create a mock VllmConfig."""
44 + config = Mock(spec=VllmConfig)
45 + config.model_config = mock_model_config
46 + config.scheduler_config = mock_scheduler_config
47 + config.structured_outputs_config = Mock()
48 + config.structured_outputs_config.reasoning_parser = None
49 + config.structured_outputs_config.enable_in_reasoning = False
50 + config.speculative_config = None
51 + return config
52 +
53 + @pytest.fixture
54 + def mock_reasoning_parser(self):
55 + """Create a mock ReasoningParser."""
56 + parser = Mock(spec=ReasoningParser)
57 + parser.is_reasoning_end = Mock(return_value=False)
58 + return parser
59 +
60 + @pytest.fixture
61 + def mock_request_with_structured_output(self):
62 + """Create a mock request with structured output."""
63 + request = Mock(spec=Request)
64 + request.structured_output_request = Mock()
65 + request.structured_output_request.reasoning_ended = None
66 + request.structured_output_request.grammar = Mock()
67 + request.structured_output_request.grammar.is_terminated = Mock(
68 + return_value=False
69 + )
70 + request.use_structured_output = True
71 + request.prompt_token_ids = [1, 2, 3, 4, 5]
72 + request.all_token_ids = [1, 2, 3, 4, 5, 6, 7, 8]
73 + return request
74 +
75 + def test_should_fill_bitmask_with_enable_in_reasoning(
76 + self, mock_vllm_config, mock_request_with_structured_output
77 + ):
78 + """Test should_fill_bitmask when enable_in_reasoning is True."""
79 + # Enable enable_in_reasoning
80 + mock_vllm_config.structured_outputs_config.enable_in_reasoning = True
81 +
82 + manager = StructuredOutputManager(mock_vllm_config)
83 +
84 + # Should always return True when enable_in_reasoning is enabled
85 + result = manager.should_fill_bitmask(mock_request_with_structured_output)
86 + assert result is True
87 +
88 + def test_should_fill_bitmask_without_enable_in_reasoning(
89 + self,
90 + mock_vllm_config,
91 + mock_request_with_structured_output,
92 + mock_reasoning_parser,
93 + ):
94 + """Test should_fill_bitmask when enable_in_reasoning is False."""
95 + # Keep enable_in_reasoning as False (default)
96 + config = mock_vllm_config.structured_outputs_config
97 + assert config.enable_in_reasoning is False
98 +
99 + manager = StructuredOutputManager(mock_vllm_config)
100 + manager.reasoner = mock_reasoning_parser
101 +
102 + # Mock reasoning not ended
103 + mock_reasoning_parser.is_reasoning_end.return_value = False
104 +
105 + result = manager.should_fill_bitmask(mock_request_with_structured_output)
106 +
107 + # Should set reasoning_ended and return its value
108 + assert (
109 + mock_request_with_structured_output.structured_output_request.reasoning_ended
110 + is False
111 + )
112 + assert result is False
113 +
114 + def test_should_fill_bitmask_no_reasoner(
115 + self, mock_vllm_config, mock_request_with_structured_output
116 + ):
117 + """Test should_fill_bitmask when no reasoner is configured."""
118 + manager = StructuredOutputManager(mock_vllm_config)
119 + manager.reasoner = None
120 +
121 + result = manager.should_fill_bitmask(mock_request_with_structured_output)
122 +
123 + # Should default to True when no reasoner
124 + assert result is True
125 +
126 + def test_should_advance_with_enable_in_reasoning(
127 + self,
128 + mock_vllm_config,
129 + mock_request_with_structured_output,
130 + mock_reasoning_parser,
131 + ):
132 + """Test should_advance when enable_in_reasoning is True."""
133 + # Enable enable_in_reasoning
134 + mock_vllm_config.structured_outputs_config.enable_in_reasoning = True
135 +
136 + manager = StructuredOutputManager(mock_vllm_config)
137 + manager.reasoner = mock_reasoning_parser
138 +
139 + # Should always return True when enable_in_reasoning is enabled
140 + result = manager.should_advance(mock_request_with_structured_output)
141 + assert result is True
142 +
143 + def test_should_advance_reasoning_not_ended(
144 + self,
145 + mock_vllm_config,
146 + mock_request_with_structured_output,
147 + mock_reasoning_parser,
148 + ):
149 + """Test should_advance when reasoning has not ended."""
150 + manager = StructuredOutputManager(mock_vllm_config)
151 + manager.reasoner = mock_reasoning_parser
152 +
153 + # Set reasoning as not ended
154 + (
155 + mock_request_with_structured_output.structured_output_request
156 + ).reasoning_ended = False
157 + mock_reasoning_parser.is_reasoning_end.return_value = False
158 +
159 + result = manager.should_advance(mock_request_with_structured_output)
160 +
161 + # Should return False since reasoning hasn't ended
162 + assert result is False
163 +
164 + def test_should_advance_reasoning_just_ended(
165 + self,
166 + mock_vllm_config,
167 + mock_request_with_structured_output,
168 + mock_reasoning_parser,
169 + ):
170 + """Test should_advance when reasoning ends in current step."""
171 + manager = StructuredOutputManager(mock_vllm_config)
172 + manager.reasoner = mock_reasoning_parser
173 +
174 + # Set reasoning as not ended initially, but ends in this step
175 + (
176 + mock_request_with_structured_output.structured_output_request
177 + ).reasoning_ended = False
178 + mock_reasoning_parser.is_reasoning_end.return_value = True
179 +
180 + result = manager.should_advance(mock_request_with_structured_output)
181 +
182 + # Should set reasoning_ended to True but return False for this step
183 + assert (
184 + mock_request_with_structured_output.structured_output_request.reasoning_ended
185 + is True
186 + )
187 + assert result is False
188 +
189 + def test_should_advance_reasoning_already_ended(
190 + self,
191 + mock_vllm_config,
192 + mock_request_with_structured_output,
193 + mock_reasoning_parser,
194 + ):
195 + """Test should_advance when reasoning has already ended."""
196 + manager = StructuredOutputManager(mock_vllm_config)
197 + manager.reasoner = mock_reasoning_parser
198 +
199 + # Set reasoning as already ended
200 + (
201 + mock_request_with_structured_output.structured_output_request
202 + ).reasoning_ended = True
203 +
204 + result = manager.should_advance(mock_request_with_structured_output)
205 +
206 + # Should return True since reasoning has ended
207 + assert result is True
vllm/config/structured_outputs.py
+2
@@ -35,6 +35,8 @@ class StructuredOutputsConfig:
35 reasoning_parser: str = ""
36 """Select the reasoning parser depending on the model that you're using.
37 This is used to parse the reasoning content into OpenAI API format."""
38 + enable_in_reasoning: bool = False
39 + """Whether to use structured input for reasoning."""
40
41 def compute_hash(self) -> str:
42 """
vllm/engine/arg_utils.py
+1
@@ -479,6 +479,7 @@ class EngineArgs:
479 VllmConfig, "structured_outputs_config"
480 )
481 reasoning_parser: str = StructuredOutputsConfig.reasoning_parser
482 +
483 # Deprecated guided decoding fields
484 guided_decoding_backend: str | None = None
485 guided_decoding_disable_fallback: bool | None = None
vllm/entrypoints/openai/protocol.py
+21 -5
@@ -200,7 +200,7 @@ class JsonSchemaResponseFormat(OpenAIBaseModel):
200 strict: bool | None = None
201
202
203 -class StructuralTag(OpenAIBaseModel):
203 +class LegacyStructuralTag(OpenAIBaseModel):
204 begin: str
205 # schema is the field, but that causes conflicts with pydantic so
206 # instead use structural_tag_schema with an alias
@@ -208,19 +208,31 @@ class StructuralTag(OpenAIBaseModel):
208 end: str
209
210
211 -class StructuralTagResponseFormat(OpenAIBaseModel):
211 +class LegacyStructuralTagResponseFormat(OpenAIBaseModel):
212 type: Literal["structural_tag"]
213 - structures: list[StructuralTag]
213 + structures: list[LegacyStructuralTag]
214 triggers: list[str]
215
216
217 +class StructuralTagResponseFormat(OpenAIBaseModel):
218 + type: Literal["structural_tag"]
219 + format: Any
220 +
221 +
222 +AnyStructuralTagResponseFormat: TypeAlias = (
223 + LegacyStructuralTagResponseFormat | StructuralTagResponseFormat
224 +)
225 +
226 +
227 class ResponseFormat(OpenAIBaseModel):
228 # type must be "json_schema", "json_object", or "text"
229 type: Literal["text", "json_object", "json_schema"]
230 json_schema: JsonSchemaResponseFormat | None = None
231
232
223 -AnyResponseFormat: TypeAlias = ResponseFormat | StructuralTagResponseFormat
233 +AnyResponseFormat: TypeAlias = (
234 + ResponseFormat | StructuralTagResponseFormat | LegacyStructuralTagResponseFormat
235 +)
236
237
238 class StreamOptions(OpenAIBaseModel):
@@ -823,7 +835,11 @@ class ChatCompletionRequest(OpenAIBaseModel):
835 elif response_format.type == "structural_tag":
836 structural_tag = response_format
837 assert structural_tag is not None and isinstance(
826 - structural_tag, StructuralTagResponseFormat
838 + structural_tag,
839 + (
840 + LegacyStructuralTagResponseFormat,
841 + StructuralTagResponseFormat,
842 + ),
843 )
844 s_tag_obj = structural_tag.model_dump(by_alias=True)
845 self.structured_outputs.structural_tag = json.dumps(s_tag_obj)
vllm/entrypoints/openai/serving_responses.py
+15 -1
@@ -98,7 +98,7 @@ from vllm.logger import init_logger
98 from vllm.logprobs import Logprob as SampleLogprob
99 from vllm.logprobs import SampleLogprobs
100 from vllm.outputs import CompletionOutput
101 -from vllm.sampling_params import SamplingParams
101 +from vllm.sampling_params import SamplingParams, StructuredOutputsParams
102 from vllm.transformers_utils.tokenizer import AnyTokenizer
103 from vllm.utils import random_uuid
104
@@ -365,6 +365,19 @@ class OpenAIServingResponses(OpenAIServing):
365 context = HarmonyContext(messages, available_tools)
366 else:
367 context = SimpleContext()
368 +
369 + if self.reasoning_parser is not None:
370 + reasoning_parser = self.reasoning_parser(tokenizer)
371 + if sampling_params.structured_outputs is None:
372 + sampling_params.structured_outputs = StructuredOutputsParams()
373 + struct_out = sampling_params.structured_outputs
374 + if struct_out.all_non_structural_tag_constraints_none():
375 + sampling_params.structured_outputs.structural_tag = (
376 + reasoning_parser.prepare_structured_tag(
377 + sampling_params.structured_outputs.structural_tag,
378 + self.tool_server,
379 + )
380 + )
381 generator = self._generate_with_builtin_tools(
382 request_id=request.request_id,
383 request_prompt=request_prompts[i],
@@ -1919,6 +1932,7 @@ class OpenAIServingResponses(OpenAIServing):
1932 processer = self._process_harmony_streaming_events
1933 else:
1934 processer = self._process_simple_streaming_events
1935 + # TODO Hanchen make sampling params to include the structural tag
1936
1937 initial_response = ResponsesResponse.from_request(
1938 request,
vllm/reasoning/abs_reasoning_parsers.py
+12
@@ -7,6 +7,7 @@ from collections.abc import Callable, Sequence
7 from functools import cached_property
8 from typing import TYPE_CHECKING, Any
9
10 +from vllm.entrypoints.tool_server import ToolServer
11 from vllm.logger import init_logger
12 from vllm.utils.collections import is_list_of
13 from vllm.utils.import_utils import import_from_path
@@ -115,6 +116,17 @@ class ReasoningParser:
116 previously been parsed and extracted (see constructor)
117 """
118
119 + def prepare_structured_tag(
120 + self,
121 + original_tag: str | None,
122 + tool_server: ToolServer | None,
123 + ) -> str:
124 + """
125 + Instance method that is implemented for preparing the structured tag
126 + Otherwise, None is returned
127 + """
128 + return None
129 +
130
131 class ReasoningParserManager:
132 reasoning_parsers: dict[str, type] = {}
vllm/reasoning/gptoss_reasoning_parser.py
+75 -1
@@ -1,17 +1,61 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
3 +import json
4 from collections.abc import Sequence
5
6 from transformers import PreTrainedTokenizerBase
7
8 from vllm.entrypoints.harmony_utils import parse_chat_output
9 from vllm.entrypoints.openai.protocol import ChatCompletionRequest, DeltaMessage
10 +from vllm.entrypoints.tool_server import ToolServer
11 from vllm.logger import init_logger
12 from vllm.reasoning import ReasoningParser, ReasoningParserManager
13
14 logger = init_logger(__name__)
15
16 +no_func_reaonsing_tag = {
17 + "type": "structural_tag",
18 + "format": {
19 + "type": "triggered_tags",
20 + "tags": [
21 + {
22 + "begin": "<|channel|>analysis<|message|>",
23 + "content": {"type": "any_text"},
24 + "end": "<|end|>",
25 + }
26 + ],
27 + "triggers": ["<|channel|>analysis"],
28 + "stop_after_first": False,
29 + },
30 +}
31 +
32 +
33 +def from_builtin_tool_to_tag(tool: str) -> list[dict]:
34 + tag = [
35 + {
36 + "begin": f"<|channel|>commentary to={tool}",
37 + "content": {"type": "any_text"},
38 + "end": "<|end|>",
39 + },
40 + {
41 + "begin": f"<|channel|>analysis to={tool}",
42 + "content": {"type": "any_text"},
43 + "end": "<|end|>",
44 + },
45 + ]
46 + return tag
47 +
48 +
49 +def tag_with_builtin_funcs(no_func_reaonsing_tag, builtin_tool_list: list[str]) -> dict:
50 + import copy
51 +
52 + new_tag = copy.deepcopy(no_func_reaonsing_tag)
53 + new_tag["format"]["triggers"].append("<|channel|>commentary to=")
54 +
55 + for tool in builtin_tool_list:
56 + new_tag["format"]["tags"].extend(from_builtin_tool_to_tag(tool))
57 + return new_tag
58 +
59
60 @ReasoningParserManager.register_module("openai_gptoss")
61 class GptOssReasoningParser(ReasoningParser):
@@ -81,3 +125,33 @@ class GptOssReasoningParser(ReasoningParser):
125 raise NotImplementedError(
126 "gpt-oss has a special branch for parsing reasoning in non-streaming mode. This method shouldn't be used." # noqa: E501
127 )
128 +
129 + # This function prepares the structural tag to format reasoning output
130 + def prepare_structured_tag(
131 + self, original_tag: str | None, tool_server: ToolServer | None
132 + ) -> str:
133 + if original_tag is None:
134 + if tool_server is None:
135 + return json.dumps(no_func_reaonsing_tag)
136 + else:
137 + builtin_tool_list: list[str] = []
138 + if tool_server.has_tool("browser"):
139 + builtin_tool_list.append("browser")
140 + if tool_server.has_tool("python"):
141 + builtin_tool_list.append("python")
142 + if tool_server.has_tool("container"):
143 + builtin_tool_list.append("container")
144 +
145 + if len(builtin_tool_list) > 0:
146 + logger.info("Builtin_tool_list: %s", builtin_tool_list)
147 + func_tag = json.dumps(
148 + tag_with_builtin_funcs(no_func_reaonsing_tag, builtin_tool_list)
149 + )
150 + else:
151 + logger.info("Builtin_tool_list is empty")
152 + func_tag = json.dumps(no_func_reaonsing_tag)
153 +
154 + return func_tag
155 + else:
156 + # There is potential risk for appending the tag to the original tag
157 + return original_tag
vllm/sampling_params.py
+32
@@ -58,6 +58,7 @@ class StructuredOutputsParams:
58 self.choice is not None,
59 self.grammar is not None,
60 self.json_object is not None,
61 + self.structural_tag is not None,
62 ]
63 )
64 if count > 1:
@@ -66,6 +67,37 @@ class StructuredOutputsParams:
67 f"but multiple are specified: {self.__dict__}"
68 )
69
70 + def all_constraints_none(self) -> bool:
71 + """
72 + Returns True if all structured-output constraint fields are None.
73 + """
74 + return all(
75 + getattr(self, field) is None
76 + for field in (
77 + "json",
78 + "regex",
79 + "choice",
80 + "grammar",
81 + "json_object",
82 + "structural_tag",
83 + )
84 + )
85 +
86 + def all_non_structural_tag_constraints_none(self) -> bool:
87 + """
88 + Returns True if all structured-output constraint fields are None.
89 + """
90 + return all(
91 + getattr(self, field) is None
92 + for field in (
93 + "json",
94 + "regex",
95 + "choice",
96 + "grammar",
97 + "json_object",
98 + )
99 + )
100 +
101
102 @dataclass
103 class GuidedDecodingParams(StructuredOutputsParams):
vllm/v1/structured_output/__init__.py
+14
@@ -73,6 +73,10 @@ class StructuredOutputManager:
73 )
74 self.reasoner = reasoner_cls(tokenizer=self.tokenizer)
75
76 + self.enable_in_reasoning = (
77 + self.vllm_config.structured_outputs_config.enable_in_reasoning
78 + )
79 +
80 def grammar_init(self, request: Request) -> None:
81 if request.structured_output_request is None:
82 return
@@ -274,7 +278,13 @@ class StructuredOutputManager:
278 return bitmask_tensor.numpy()
279
280 def should_fill_bitmask(self, request: Request) -> bool:
281 + # NOTE (Hanchen) if enable_in_reasoning is True, it means that
282 + # the model needs to be constrained in reasoning. So we should always
283 + # enable the bitmask filling.
284 +
285 if self.reasoner is not None:
286 + if self.enable_in_reasoning:
287 + return True
288 assert request.structured_output_request is not None
289 if request.structured_output_request.reasoning_ended is None:
290 request.structured_output_request.reasoning_ended = (
@@ -297,6 +307,10 @@ class StructuredOutputManager:
307 if self.reasoner is None:
308 return True
309
310 + # if the model needs structured in reasoning, we should advance
311 + if self.enable_in_reasoning:
312 + return True
313 +
314 structured_req = request.structured_output_request
315 if structured_req.reasoning_ended:
316 return True
vllm/v1/structured_output/backend_xgrammar.py
+28 -24
@@ -91,18 +91,20 @@ class XgrammarBackend(StructuredOutputBackend):
91 ctx = self.compiler.compile_regex(grammar_spec)
92 elif request_type == StructuredOutputOptions.STRUCTURAL_TAG:
93 s_tag = json.loads(grammar_spec)
94 - tags = [
95 - xgr.StructuralTagItem(
96 - begin=s["begin"],
97 - schema=json.dumps(s["schema"]),
98 - end=s["end"],
99 - )
100 - for s in s_tag["structures"]
101 - ]
102 - structural_tag = xgr.StructuralTag.from_legacy_structural_tag(
103 - tags, s_tag["triggers"]
104 - )
105 - ctx = self.compiler.compile_structural_tag(structural_tag)
94 + if "structures" in s_tag:
95 + # Falling back to deprecated method of compiling structural tag
96 + tags = [
97 + xgr.StructuralTagItem(
98 + begin=s["begin"],
99 + schema=json.dumps(s["schema"]),
100 + end=s["end"],
101 + )
102 + for s in s_tag["structures"]
103 + ]
104 + ctx = self.compiler.compile_structural_tag(tags, s_tag["triggers"])
105 + else:
106 + logger.info("Compiling structural tag grammar_spec: %s", grammar_spec)
107 + ctx = self.compiler.compile_structural_tag(grammar_spec)
108 else:
109 logger.error(
110 "Validation should have already occurred. Please file an issue."
@@ -320,17 +322,19 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:
322 if so_params.structural_tag:
323 try:
324 s_tag = json.loads(so_params.structural_tag)
323 - tags = [
324 - xgr.StructuralTagItem(
325 - begin=s["begin"],
326 - schema=json.dumps(s["schema"]),
327 - end=s["end"],
328 - )
329 - for s in s_tag["structures"]
330 - ]
331 - structural_tag = xgr.StructuralTag.from_legacy_structural_tag(
332 - tags, s_tag["triggers"]
333 - )
334 - xgr.Grammar.from_structural_tag(structural_tag)
325 +
326 + # Using the deprecated method of compiling structural tag
327 + if "structures" in s_tag:
328 + tags = [
329 + xgr.StructuralTagItem(
330 + begin=s["begin"],
331 + schema=json.dumps(s["schema"]),
332 + end=s["end"],
333 + )
334 + for s in s_tag["structures"]
335 + ]
336 + xgr.Grammar.from_structural_tag(tags, s_tag["triggers"])
337 + else:
338 + xgr.Grammar.from_structural_tag(so_params.structural_tag)
339 except Exception as e:
340 raise ValueError("Invalid structural tag specification.") from e
vllm/v1/structured_output/request.py
+6 -1
@@ -28,7 +28,12 @@ class StructuredOutputRequest:
28 if sampling_params is None:
29 return None
30 params = sampling_params.structured_outputs
31 - return StructuredOutputRequest(params=params) if params else None
31 + if params:
32 + if params.all_constraints_none():
33 + return None
34 + else:
35 + return StructuredOutputRequest(params=params)
36 + return None
37
38 def _check_grammar_completion(self) -> bool:
39 # NOTE: We have to lazy import to gate circular imports