@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7a49742b88
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 6, 14:30 UTC

[CI/Build] Add common tool call parser test suite (#27599)

Signed-off-by: Ben Browning <bbrownin@redhat.com>

Ben Browning committed Mar 16, 2026 at 19:46 UTC 7a49742b8867e7d310abfd85c944e54d090e9301
16 files changed +1201 -5
tests/entrypoints/openai/tool_parsers/test_gigachat3_tool_parser.py
+1 -1
@@ -5,7 +5,7 @@ import json
5
6 import pytest
7
8 -from tests.entrypoints.openai.tool_parsers.utils import (
8 +from tests.tool_parsers.utils import (
9 run_tool_extraction,
10 run_tool_extraction_streaming,
11 )
tests/entrypoints/openai/tool_parsers/test_hunyuan_a13b_tool_parser.py
+1 -1
@@ -7,7 +7,7 @@ from unittest.mock import MagicMock
7
8 import pytest
9
10 -from tests.entrypoints.openai.tool_parsers.utils import (
10 +from tests.tool_parsers.utils import (
11 run_tool_extraction,
12 run_tool_extraction_streaming,
13 )
tests/entrypoints/openai/tool_parsers/test_llama4_pythonic_tool_parser.py
+1 -1
@@ -5,7 +5,7 @@ from unittest.mock import MagicMock, patch
5
6 import pytest
7
8 -from tests.entrypoints.openai.tool_parsers.utils import (
8 +from tests.tool_parsers.utils import (
9 run_tool_extraction,
10 run_tool_extraction_streaming,
11 )
tests/entrypoints/openai/tool_parsers/test_olmo3_tool_parser.py
+1 -1
@@ -5,7 +5,7 @@ from unittest.mock import MagicMock, patch
5
6 import pytest
7
8 -from tests.entrypoints.openai.tool_parsers.utils import (
8 +from tests.tool_parsers.utils import (
9 run_tool_extraction,
10 run_tool_extraction_streaming,
11 )
tests/entrypoints/openai/tool_parsers/test_pythonic_tool_parser.py
+1 -1
@@ -5,7 +5,7 @@ from unittest.mock import MagicMock, patch
5
6 import pytest
7
8 -from tests.entrypoints.openai.tool_parsers.utils import (
8 +from tests.tool_parsers.utils import (
9 run_tool_extraction,
10 run_tool_extraction_streaming,
11 )
tests/tool_parsers/common_tests.py new
+378
@@ -0,0 +1,378 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import json
5 +from dataclasses import dataclass, field
6 +from types import NoneType
7 +from typing import Any
8 +
9 +import pytest
10 +
11 +from tests.tool_parsers.utils import run_tool_extraction
12 +from vllm.tokenizers import TokenizerLike
13 +from vllm.tool_parsers import ToolParserManager
14 +
15 +
16 +@dataclass
17 +class ToolParserTestConfig:
18 + """Configuration for a tool parser's common tests.
19 +
20 + This dataclass contains all the test data and expected results needed
21 + to run the common test suite for a parser. Each parser test file
22 + creates one instance of this config with parser-specific values.
23 +
24 + Attributes:
25 + parser_name: Name used with ToolParserManager (e.g., "mistral")
26 +
27 + Test data (model outputs):
28 + no_tool_calls_output: Plain text without any tool syntax
29 + single_tool_call_output: One tool call with simple arguments
30 + parallel_tool_calls_output: Multiple tool calls in one response
31 + various_data_types_output: Tool with various data types
32 + empty_arguments_output: Tool call with no parameters
33 + surrounding_text_output: Tool call mixed with regular text
34 + escaped_strings_output: Tool call with escaped chars
35 + malformed_input_outputs: List of invalid inputs
36 +
37 + Expected results:
38 + single_tool_call_expected_name: Expected function name
39 + single_tool_call_expected_args: Expected arguments dict
40 + parallel_tool_calls_count: Number of tools in parallel test
41 + parallel_tool_calls_names: Function names in order
42 + single_tool_call_expected_content: Content field when tool called
43 + parallel_tool_calls_expected_content: Content for parallel test
44 +
45 + xfail markers:
46 + xfail_streaming: Mapping test name to xfail reason (streaming only)
47 + xfail_nonstreaming: Mapping test name to xfail reason (non-streaming)
48 +
49 + Special flags:
50 + allow_empty_or_json_empty_args: True if "" or "{}" both valid for empty args
51 + supports_typed_arguments: True if the parser supports typed function arguments
52 + """
53 +
54 + # Parser identification
55 + parser_name: str
56 +
57 + # Test data - model outputs for each common test
58 + no_tool_calls_output: str
59 + single_tool_call_output: str
60 + parallel_tool_calls_output: str
61 + various_data_types_output: str
62 + empty_arguments_output: str
63 + surrounding_text_output: str
64 + escaped_strings_output: str
65 + malformed_input_outputs: list[str]
66 +
67 + # Expected results for specific tests (optional overrides)
68 + single_tool_call_expected_name: str = "get_weather"
69 + single_tool_call_expected_args: dict[str, Any] = field(
70 + default_factory=lambda: {"city": "Tokyo"}
71 + )
72 + parallel_tool_calls_count: int = 2
73 + parallel_tool_calls_names: list[str] = field(
74 + default_factory=lambda: ["get_weather", "get_time"]
75 + )
76 +
77 + # xfail configuration - maps test name to xfail reason
78 + xfail_streaming: dict[str, str] = field(default_factory=dict)
79 + xfail_nonstreaming: dict[str, str] = field(default_factory=dict)
80 +
81 + # Content expectations (some parsers strip content, others don't)
82 + single_tool_call_expected_content: str | None = None
83 + parallel_tool_calls_expected_content: str | None = None
84 +
85 + # Special assertions for edge cases
86 + allow_empty_or_json_empty_args: bool = True # "{}" or "" for empty args
87 + supports_typed_arguments: bool = True
88 +
89 +
90 +class ToolParserTests:
91 + """Mixin class providing common test suite for tool parsers.
92 +
93 + To use this mixin in a parser test file:
94 +
95 + 1. Create a test_config fixture that returns a ToolParserTestConfig instance
96 + 2. Inherit from this class
97 + 3. Add parser-specific tests as additional methods
98 +
99 + Example:
100 + class TestMistralToolParser(ToolParserTests):
101 + @pytest.fixture
102 + def test_config(self) -> ToolParserTestConfig:
103 + return ToolParserTestConfig(
104 + parser_name="mistral",
105 + no_tool_calls_output="Plain text...",
106 + # ... other config ...
107 + )
108 +
109 + # Parser-specific tests
110 + def test_mistral_specific_feature(self, tool_parser):
111 + # Custom test logic
112 + pass
113 + """
114 +
115 + @pytest.fixture
116 + def test_config(self) -> ToolParserTestConfig:
117 + """Override this to provide parser-specific configuration."""
118 + raise NotImplementedError(
119 + "Subclass must provide test_config fixture returning ToolParserTestConfig"
120 + )
121 +
122 + @pytest.fixture
123 + def tokenizer(self, default_tokenizer: TokenizerLike) -> TokenizerLike:
124 + """Override this to provide parser-specific tokenizer."""
125 + return default_tokenizer
126 +
127 + @pytest.fixture
128 + def tool_parser(self, test_config: ToolParserTestConfig, tokenizer: TokenizerLike):
129 + return ToolParserManager.get_tool_parser(test_config.parser_name)(tokenizer)
130 +
131 + @pytest.fixture(params=[True, False])
132 + def streaming(self, request: pytest.FixtureRequest) -> bool:
133 + return request.param
134 +
135 + def test_no_tool_calls(
136 + self,
137 + request: pytest.FixtureRequest,
138 + tool_parser: Any,
139 + test_config: ToolParserTestConfig,
140 + streaming: bool,
141 + ):
142 + """Verify parser handles plain text without tool syntax."""
143 + # Apply xfail markers if configured
144 + test_name = "test_no_tool_calls"
145 + self.apply_xfail_mark(request, test_config, test_name, streaming)
146 +
147 + content, tool_calls = run_tool_extraction(
148 + tool_parser, test_config.no_tool_calls_output, streaming=streaming
149 + )
150 + assert content == test_config.no_tool_calls_output, (
151 + f"Expected content to match input, got {content}"
152 + )
153 + assert len(tool_calls) == 0, f"Expected no tool calls, got {len(tool_calls)}"
154 +
155 + def test_single_tool_call_simple_args(
156 + self,
157 + request: pytest.FixtureRequest,
158 + tool_parser: Any,
159 + test_config: ToolParserTestConfig,
160 + streaming: bool,
161 + ):
162 + """Verify parser extracts one tool with simple arguments."""
163 + # Apply xfail markers if configured
164 + test_name = "test_single_tool_call_simple_args"
165 + self.apply_xfail_mark(request, test_config, test_name, streaming)
166 +
167 + content, tool_calls = run_tool_extraction(
168 + tool_parser, test_config.single_tool_call_output, streaming=streaming
169 + )
170 +
171 + # Content check (some parsers strip it)
172 + if test_config.single_tool_call_expected_content is not None:
173 + assert content == test_config.single_tool_call_expected_content
174 +
175 + assert len(tool_calls) == 1, f"Expected 1 tool call, got {len(tool_calls)}"
176 + assert tool_calls[0].type == "function"
177 + assert tool_calls[0].function.name == test_config.single_tool_call_expected_name
178 +
179 + args = json.loads(tool_calls[0].function.arguments)
180 + for key, value in test_config.single_tool_call_expected_args.items():
181 + assert args.get(key) == value, (
182 + f"Expected {key}={value}, got {args.get(key)}"
183 + )
184 +
185 + def test_parallel_tool_calls(
186 + self,
187 + request: pytest.FixtureRequest,
188 + tool_parser: Any,
189 + test_config: ToolParserTestConfig,
190 + streaming: bool,
191 + ):
192 + """Verify parser handles multiple tools in one response."""
193 + # Apply xfail markers if configured
194 + test_name = "test_parallel_tool_calls"
195 + self.apply_xfail_mark(request, test_config, test_name, streaming)
196 +
197 + content, tool_calls = run_tool_extraction(
198 + tool_parser,
199 + test_config.parallel_tool_calls_output,
200 + streaming=streaming,
201 + )
202 +
203 + assert len(tool_calls) == test_config.parallel_tool_calls_count, (
204 + f"Expected {test_config.parallel_tool_calls_count} "
205 + f"tool calls, got {len(tool_calls)}"
206 + )
207 +
208 + # Verify tool names match expected
209 + for i, expected_name in enumerate(test_config.parallel_tool_calls_names):
210 + assert tool_calls[i].type == "function"
211 + assert tool_calls[i].function.name == expected_name
212 +
213 + # Verify unique IDs
214 + ids = [tc.id for tc in tool_calls]
215 + assert len(ids) == len(set(ids)), "Tool call IDs should be unique"
216 +
217 + def test_various_data_types(
218 + self,
219 + request: pytest.FixtureRequest,
220 + tool_parser: Any,
221 + test_config: ToolParserTestConfig,
222 + streaming: bool,
223 + ):
224 + """Verify parser handles all JSON types in arguments."""
225 + # Apply xfail markers if configured
226 + test_name = "test_various_data_types"
227 + self.apply_xfail_mark(request, test_config, test_name, streaming)
228 +
229 + content, tool_calls = run_tool_extraction(
230 + tool_parser,
231 + test_config.various_data_types_output,
232 + streaming=streaming,
233 + )
234 + assert len(tool_calls) == 1, f"Expected 1 tool call, got {len(tool_calls)}"
235 +
236 + args = json.loads(tool_calls[0].function.arguments)
237 + # Verify all expected fields present
238 + required_fields_types = {
239 + "string_field": str,
240 + "int_field": int,
241 + "float_field": float,
242 + "bool_field": bool,
243 + "null_field": NoneType,
244 + "array_field": list,
245 + "object_field": dict,
246 + }
247 + for required_field, expected_type in required_fields_types.items():
248 + assert required_field in args, (
249 + f"Expected field '{required_field}' in arguments"
250 + )
251 + if test_config.supports_typed_arguments:
252 + found_type = type(args[required_field])
253 + assert found_type is expected_type, (
254 + f"Expected field '{required_field}' to have type {expected_type}, "
255 + f"got {found_type}"
256 + )
257 +
258 + def test_empty_arguments(
259 + self,
260 + request: pytest.FixtureRequest,
261 + tool_parser: Any,
262 + test_config: ToolParserTestConfig,
263 + streaming: bool,
264 + ):
265 + """Verify parser handles parameterless tool calls."""
266 + # Apply xfail markers if configured
267 + test_name = "test_empty_arguments"
268 + self.apply_xfail_mark(request, test_config, test_name, streaming)
269 +
270 + content, tool_calls = run_tool_extraction(
271 + tool_parser, test_config.empty_arguments_output, streaming=streaming
272 + )
273 + assert len(tool_calls) == 1, f"Expected 1 tool call, got {len(tool_calls)}"
274 +
275 + args = tool_calls[0].function.arguments
276 + if test_config.allow_empty_or_json_empty_args:
277 + assert args in ["{}", ""], f"Expected empty args, got {args}"
278 + else:
279 + assert args == "{}", f"Expected {{}}, got {args}"
280 +
281 + def test_surrounding_text(
282 + self,
283 + request: pytest.FixtureRequest,
284 + tool_parser: Any,
285 + test_config: ToolParserTestConfig,
286 + streaming: bool,
287 + ):
288 + """Verify parser extracts tools from mixed content."""
289 + # Apply xfail markers if configured
290 + test_name = "test_surrounding_text"
291 + self.apply_xfail_mark(request, test_config, test_name, streaming)
292 +
293 + content, tool_calls = run_tool_extraction(
294 + tool_parser, test_config.surrounding_text_output, streaming=streaming
295 + )
296 + assert len(tool_calls) >= 1, (
297 + f"Expected at least 1 tool call, got {len(tool_calls)}"
298 + )
299 +
300 + def test_escaped_strings(
301 + self,
302 + request: pytest.FixtureRequest,
303 + tool_parser: Any,
304 + test_config: ToolParserTestConfig,
305 + streaming: bool,
306 + ):
307 + """Verify parser handles escaped characters in arguments."""
308 + # Apply xfail markers if configured
309 + test_name = "test_escaped_strings"
310 + self.apply_xfail_mark(request, test_config, test_name, streaming)
311 +
312 + content, tool_calls = run_tool_extraction(
313 + tool_parser, test_config.escaped_strings_output, streaming=streaming
314 + )
315 + assert len(tool_calls) == 1, f"Expected 1 tool call, got {len(tool_calls)}"
316 +
317 + args = json.loads(tool_calls[0].function.arguments)
318 + # At minimum, verify we can parse and have expected fields
319 + # Exact escaping behavior varies by parser
320 + assert len(args) > 0, "Expected some arguments with escaped strings"
321 +
322 + def test_malformed_input(
323 + self,
324 + request: pytest.FixtureRequest,
325 + tool_parser: Any,
326 + test_config: ToolParserTestConfig,
327 + streaming: bool,
328 + ):
329 + """Verify parser gracefully handles invalid syntax."""
330 + # Apply xfail markers if configured
331 + test_name = "test_malformed_input"
332 + self.apply_xfail_mark(request, test_config, test_name, streaming)
333 +
334 + for malformed_input in test_config.malformed_input_outputs:
335 + # Should not raise exception
336 + content, tool_calls = run_tool_extraction(
337 + tool_parser, malformed_input, streaming=streaming
338 + )
339 + # Parser should handle gracefully (exact behavior varies)
340 +
341 + def test_streaming_reconstruction(
342 + self,
343 + request: pytest.FixtureRequest,
344 + tool_parser: Any,
345 + test_config: ToolParserTestConfig,
346 + ):
347 + """Verify streaming produces same result as non-streaming."""
348 + test_name = "test_streaming_reconstruction"
349 + self.apply_xfail_mark(request, test_config, test_name, True)
350 +
351 + test_output = test_config.single_tool_call_output
352 +
353 + # Non-streaming result
354 + content_non, tools_non = run_tool_extraction(
355 + tool_parser, test_output, streaming=False
356 + )
357 +
358 + # Streaming result
359 + content_stream, tools_stream = run_tool_extraction(
360 + tool_parser, test_output, streaming=True
361 + )
362 +
363 + # Compare results
364 + assert content_non == content_stream, "Content should match between modes"
365 + assert len(tools_non) == len(tools_stream), "Tool count should match"
366 + if len(tools_non) > 0:
367 + assert tools_non[0].function.name == tools_stream[0].function.name
368 + assert tools_non[0].function.arguments == tools_stream[0].function.arguments
369 +
370 + def apply_xfail_mark(self, request, test_config, test_name, streaming):
371 + reason = None
372 + if streaming and test_name in test_config.xfail_streaming:
373 + reason = test_config.xfail_streaming[test_name]
374 + elif not streaming and test_name in test_config.xfail_nonstreaming:
375 + reason = test_config.xfail_nonstreaming[test_name]
376 + if reason is not None:
377 + mark = pytest.mark.xfail(reason=reason, strict=True)
378 + request.node.add_marker(mark)
tests/tool_parsers/conftest.py new
+12
@@ -0,0 +1,12 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import pytest
5 +from transformers import AutoTokenizer
6 +
7 +from vllm.tokenizers import TokenizerLike
8 +
9 +
10 +@pytest.fixture(scope="module")
11 +def default_tokenizer() -> TokenizerLike:
12 + return AutoTokenizer.from_pretrained("gpt2")
tests/tool_parsers/test_deepseekv3_tool_parser.py new
+92
@@ -0,0 +1,92 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +
5 +import pytest
6 +
7 +from tests.tool_parsers.common_tests import (
8 + ToolParserTestConfig,
9 + ToolParserTests,
10 +)
11 +from vllm.tokenizers import TokenizerLike, get_tokenizer
12 +
13 +
14 +class TestDeepSeekV3ToolParser(ToolParserTests):
15 + @pytest.fixture(scope="class")
16 + def tokenizer(self) -> TokenizerLike:
17 + return get_tokenizer("deepseek-ai/DeepSeek-V3")
18 +
19 + @pytest.fixture
20 + def test_config(self) -> ToolParserTestConfig:
21 + return ToolParserTestConfig(
22 + parser_name="deepseek_v3",
23 + # Test data
24 + no_tool_calls_output=(
25 + "How can I help you today? I can check weather for you."
26 + ),
27 + single_tool_call_output="""<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_weather
28 +```json
29 +{"city": "Tokyo", "unit": "celsius"}
30 +```<｜tool▁call▁end｜><｜tool▁calls▁end｜>""",
31 + parallel_tool_calls_output="""<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_weather
32 +```json
33 +{"city": "Tokyo", "unit": "celsius"}
34 +```<｜tool▁call▁end｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>search_hotels
35 +```json
36 +{"location": "Tokyo", "check_in": "2025-01-15"}
37 +```<｜tool▁call▁end｜><｜tool▁calls▁end｜>""",
38 + various_data_types_output=(
39 + """<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>test_function
40 +```json
41 +"""
42 + """{"string_field": "hello", "int_field": 42, "float_field": 3.14, """
43 + """"bool_field": true, "null_field": null, """
44 + """"array_field": ["a", "b", "c"], """
45 + """"object_field": {"nested": "value"}, """
46 + """"empty_array": [], "empty_object": {}}
47 +```<｜tool▁call▁end｜><｜tool▁calls▁end｜>"""
48 + ),
49 + empty_arguments_output="""<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_current_time
50 +```json
51 +{}
52 +```<｜tool▁call▁end｜><｜tool▁calls▁end｜>""",
53 + surrounding_text_output=(
54 + """Let me check the weather for you."""
55 + """<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_weather
56 +```json
57 +{"city": "Paris"}
58 +```<｜tool▁call▁end｜><｜tool▁calls▁end｜>"""
59 + ),
60 + escaped_strings_output=(
61 + """<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>send_message
62 +```json
63 +"""
64 + """{"text": "He said \\"hello\\"", "path": "C:\\\\Users\\\\file", """
65 + """"newline": "line1\\nline2"}
66 +```<｜tool▁call▁end｜><｜tool▁calls▁end｜>"""
67 + ),
68 + malformed_input_outputs=[
69 + """<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_weather
70 +```json
71 +{"city": "Tokyo"
72 +```<｜tool▁call▁end｜><｜tool▁calls▁end｜>""",
73 + """<｜tool▁calls▁begin｜>function<｜tool▁sep｜>get_weather
74 +```json
75 +{"city": "Tokyo"}
76 +```<｜tool▁calls▁end｜>""",
77 + ],
78 + # Expected results
79 + single_tool_call_expected_name="get_weather",
80 + single_tool_call_expected_args={"city": "Tokyo", "unit": "celsius"},
81 + single_tool_call_expected_content=None,
82 + parallel_tool_calls_count=2,
83 + parallel_tool_calls_names=["get_weather", "search_hotels"],
84 + # xfail markers
85 + xfail_streaming={},
86 + xfail_nonstreaming={
87 + "test_malformed_input": (
88 + "Parser sets tools_called=True even when tool_calls is "
89 + "empty (detects start token but fails to parse)"
90 + ),
91 + },
92 + )
tests/tool_parsers/test_granite_20b_fc_tool_parser.py new
+76
@@ -0,0 +1,76 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import pytest
5 +
6 +from tests.tool_parsers.common_tests import (
7 + ToolParserTestConfig,
8 + ToolParserTests,
9 +)
10 +
11 +
12 +class TestGranite20bFcToolParser(ToolParserTests):
13 + @pytest.fixture
14 + def test_config(self) -> ToolParserTestConfig:
15 + return ToolParserTestConfig(
16 + parser_name="granite-20b-fc",
17 + # Test data
18 + no_tool_calls_output="This is a regular response without any tool calls.",
19 + single_tool_call_output=(
20 + '<function_call> {"name": "get_weather", '
21 + '"arguments": {"city": "Tokyo"}}'
22 + ),
23 + parallel_tool_calls_output=(
24 + '<function_call> {"name": "get_weather", '
25 + '"arguments": {"city": "Tokyo"}}\n'
26 + '<function_call> {"name": "get_time", '
27 + '"arguments": {"timezone": "Asia/Tokyo"}}'
28 + ),
29 + various_data_types_output="""<function_call> {
30 + "name": "test_function",
31 + "arguments": {
32 + "string_field": "hello",
33 + "int_field": 42,
34 + "float_field": 3.14,
35 + "bool_field": true,
36 + "null_field": null,
37 + "array_field": ["a", "b", "c"],
38 + "object_field": {"nested": "value"},
39 + "empty_array": [],
40 + "empty_object": {}
41 + }
42 +}""",
43 + empty_arguments_output=(
44 + '<function_call> {"name": "refresh", "arguments": {}}'
45 + ),
46 + surrounding_text_output="""Let me check the weather for you.
47 +<function_call> {"name": "get_weather", "arguments": {"city": "Tokyo"}}""",
48 + escaped_strings_output="""<function_call> {
49 + "name": "test_function",
50 + "arguments": {
51 + "quoted": "He said \\"hello\\"",
52 + "path": "C:\\\\Users\\\\file.txt",
53 + "newline": "line1\\nline2",
54 + "unicode": "emoji: 🎉"
55 + }
56 +}""",
57 + malformed_input_outputs=[
58 + '<function_call> {"name": "func", "arguments": {',
59 + '<function_call> [{"name": "func", "arguments": {}}]',
60 + '{"name": "func", "arguments": {}}',
61 + '<function_call> {"name": 123}',
62 + ],
63 + # Expected results
64 + single_tool_call_expected_name="get_weather",
65 + single_tool_call_expected_args={"city": "Tokyo"},
66 + single_tool_call_expected_content=None,
67 + parallel_tool_calls_count=2,
68 + parallel_tool_calls_names=["get_weather", "get_time"],
69 + # xfail markers
70 + xfail_streaming={
71 + "test_surrounding_text": (
72 + "Granite 20B FC streaming requires <function_call> at start"
73 + ),
74 + },
75 + xfail_nonstreaming={},
76 + )
tests/tool_parsers/test_granite_tool_parser.py new
+118
@@ -0,0 +1,118 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +
5 +import pytest
6 +
7 +from tests.tool_parsers.common_tests import (
8 + ToolParserTestConfig,
9 + ToolParserTests,
10 +)
11 +from tests.tool_parsers.utils import run_tool_extraction
12 +
13 +
14 +class TestGraniteToolParser(ToolParserTests):
15 + @pytest.fixture
16 + def test_config(self) -> ToolParserTestConfig:
17 + return ToolParserTestConfig(
18 + parser_name="granite",
19 + # Test data
20 + no_tool_calls_output="This is a regular response without any tool calls.",
21 + single_tool_call_output=(
22 + '<|tool_call|> [{"name": "get_weather", '
23 + '"arguments": {"city": "Tokyo"}}]'
24 + ),
25 + parallel_tool_calls_output="""<|tool_call|> [
26 + {"name": "get_weather", "arguments": {"city": "Tokyo"}},
27 + {"name": "get_time", "arguments": {"timezone": "Asia/Tokyo"}}
28 +]""",
29 + various_data_types_output="""<tool_call> [{
30 + "name": "test_function",
31 + "arguments": {
32 + "string_field": "hello",
33 + "int_field": 42,
34 + "float_field": 3.14,
35 + "bool_field": true,
36 + "null_field": null,
37 + "array_field": ["a", "b", "c"],
38 + "object_field": {"nested": "value"},
39 + "empty_array": [],
40 + "empty_object": {}
41 + }
42 +}]""",
43 + empty_arguments_output=(
44 + '<|tool_call|> [{"name": "refresh", "arguments": {}}]'
45 + ),
46 + surrounding_text_output="""Let me check the weather for you.
47 +<|tool_call|> [{"name": "get_weather", "arguments": {"city": "Tokyo"}}]
48 +I'll get that information.""",
49 + escaped_strings_output="""<tool_call> [{
50 + "name": "test_function",
51 + "arguments": {
52 + "quoted": "He said \\"hello\\"",
53 + "path": "C:\\\\Users\\\\file.txt",
54 + "newline": "line1\\nline2",
55 + "unicode": "emoji: 🎉"
56 + }
57 +}]""",
58 + malformed_input_outputs=[
59 + '<|tool_call|> [{"name": "func", "arguments": {',
60 + '<|tool_call|> {"name": "func", "arguments": {}}', # Not an array
61 + '[{"name": "func", "arguments": "not a dict"}]',
62 + 'Some text [{"name": "func"}]', # JSON but not tool call format
63 + ],
64 + # Expected results
65 + single_tool_call_expected_name="get_weather",
66 + single_tool_call_expected_args={"city": "Tokyo"},
67 + # Granite strips content when tool calls present
68 + single_tool_call_expected_content=None,
69 + parallel_tool_calls_count=2,
70 + parallel_tool_calls_names=["get_weather", "get_time"],
71 + # xfail markers
72 + xfail_streaming={
73 + "test_malformed_input": (
74 + "Streaming mode incorrectly creates tool call from malformed JSON"
75 + ),
76 + "test_surrounding_text": (
77 + "Parser doesn't handle surrounding text correctly in streaming"
78 + ),
79 + "test_streaming_reconstruction": (
80 + "Streaming mode doesn't strip <|tool_call|> marker from content"
81 + ),
82 + },
83 + xfail_nonstreaming={
84 + "test_surrounding_text": (
85 + "Parser doesn't handle surrounding text correctly in non-streaming"
86 + ),
87 + },
88 + )
89 +
90 + # Granite-Specific Tests
91 +
92 + @pytest.mark.parametrize("streaming", [True, False])
93 + def test_granite_token_prefix_format(self, tool_parser, streaming):
94 + """Verify parser handles Granite 3.0 <|tool_call|> token format."""
95 + single_tool_call_token = (
96 + '<|tool_call|> [{"name": "get_weather", "arguments": {"city": "Tokyo"}}]'
97 + )
98 + content, tool_calls = run_tool_extraction(
99 + tool_parser, single_tool_call_token, streaming=streaming
100 + )
101 + assert len(tool_calls) == 1, (
102 + f"Expected 1 tool call from token format, got {len(tool_calls)}"
103 + )
104 + assert tool_calls[0].function.name == "get_weather"
105 +
106 + @pytest.mark.parametrize("streaming", [True, False])
107 + def test_granite_string_prefix_format(self, tool_parser, streaming):
108 + """Verify parser handles Granite 3.1 <tool_call> string format."""
109 + single_tool_call_string = (
110 + '<tool_call> [{"name": "get_weather", "arguments": {"city": "Tokyo"}}]'
111 + )
112 + content, tool_calls = run_tool_extraction(
113 + tool_parser, single_tool_call_string, streaming=streaming
114 + )
115 + assert len(tool_calls) == 1, (
116 + f"Expected 1 tool call from string format, got {len(tool_calls)}"
117 + )
118 + assert tool_calls[0].function.name == "get_weather"
tests/tool_parsers/test_internlm2_tool_parser.py new
+122
@@ -0,0 +1,122 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from unittest.mock import MagicMock
5 +
6 +import pytest
7 +
8 +from tests.tool_parsers.common_tests import (
9 + ToolParserTestConfig,
10 + ToolParserTests,
11 +)
12 +from vllm.tokenizers import TokenizerLike
13 +
14 +
15 +class TestInternLM2ToolParser(ToolParserTests):
16 + @pytest.fixture
17 + def tokenizer(self, default_tokenizer: TokenizerLike) -> TokenizerLike:
18 + """Add some internlm2 specific tokens to the default vocab."""
19 +
20 + tokenizer_vocab = default_tokenizer.get_vocab()
21 + default_tokenizer.get_vocab = MagicMock()
22 + tokenizer_vocab.update(
23 + {
24 + "<|action_start|>": 92540,
25 + "<|plugin|>": 92541,
26 + "<|action_end|>": 92542,
27 + }
28 + )
29 + default_tokenizer.get_vocab.return_value = tokenizer_vocab
30 + return default_tokenizer
31 +
32 + @pytest.fixture
33 + def test_config(self) -> ToolParserTestConfig:
34 + return ToolParserTestConfig(
35 + parser_name="internlm",
36 + # Test data
37 + no_tool_calls_output="This is a regular response without any tool calls.",
38 + single_tool_call_output=(
39 + '<|action_start|><|plugin|>{"name": "get_weather", '
40 + '"parameters": {"city": "Tokyo"}}<|action_end|>'
41 + ),
42 + # InternLM2 doesn't support parallel calls
43 + parallel_tool_calls_output=(
44 + '<|action_start|><|plugin|>{"name": "get_weather", '
45 + '"parameters": {"city": "Tokyo"}}<|action_end|>'
46 + ),
47 + various_data_types_output="""<|action_start|><|plugin|>{
48 + "name": "test_function",
49 + "parameters": {
50 + "string_field": "hello",
51 + "int_field": 42,
52 + "float_field": 3.14,
53 + "bool_field": true,
54 + "null_field": null,
55 + "array_field": ["a", "b", "c"],
56 + "object_field": {"nested": "value"},
57 + "empty_array": [],
58 + "empty_object": {}
59 + }
60 +}<|action_end|>""",
61 + empty_arguments_output=(
62 + '<|action_start|><|plugin|>{"name": "refresh", '
63 + '"parameters": {}}<|action_end|>'
64 + ),
65 + surrounding_text_output=(
66 + "Let me check the weather for you. "
67 + '<|action_start|><|plugin|>{"name": "get_weather", '
68 + '"parameters": {"city": "Tokyo"}}<|action_end|>'
69 + ),
70 + escaped_strings_output="""<|action_start|><|plugin|>{
71 + "name": "test_function",
72 + "parameters": {
73 + "quoted": "He said \\"hello\\"",
74 + "path": "C:\\\\Users\\\\file.txt",
75 + "newline": "line1\\nline2",
76 + "unicode": "emoji: 🎉"
77 + }
78 +}<|action_end|>""",
79 + malformed_input_outputs=[
80 + '<|action_start|><|plugin|>{"name": "func", "parameters": {',
81 + (
82 + '<|action_start|><|plugin|>{"name": "func", '
83 + '"parameters": "not a dict"}<|action_end|>'
84 + ),
85 + "<|action_start|><|plugin|>not json<|action_end|>",
86 + "<|action_start|><|plugin|>",
87 + '<|action_start|>{"name": "func"}',
88 + ],
89 + # Expected results
90 + single_tool_call_expected_name="get_weather",
91 + single_tool_call_expected_args={"city": "Tokyo"},
92 + single_tool_call_expected_content=None,
93 + parallel_tool_calls_count=1, # InternLM2 only supports single tool calls
94 + parallel_tool_calls_names=["get_weather"],
95 + # Parser-specific settings
96 + allow_empty_or_json_empty_args=True,
97 + # xfail markers
98 + xfail_streaming={
99 + "test_single_tool_call_simple_args": (
100 + "InternLM2 streaming not fully implemented"
101 + ),
102 + "test_parallel_tool_calls": (
103 + "InternLM2 streaming not fully implemented"
104 + ),
105 + "test_various_data_types": (
106 + "InternLM2 streaming not fully implemented"
107 + ),
108 + "test_empty_arguments": ("InternLM2 streaming not fully implemented"),
109 + "test_surrounding_text": ("InternLM2 streaming not fully implemented"),
110 + "test_escaped_strings": ("InternLM2 streaming not fully implemented"),
111 + "test_streaming_reconstruction": (
112 + "InternLM2 streaming parser returns '<|action_start|' as "
113 + "content instead of None - streaming/non-streaming inconsistency"
114 + ),
115 + },
116 + xfail_nonstreaming={
117 + "test_malformed_input": (
118 + "InternLM2 parser raises JSONDecodeError on malformed JSON "
119 + "instead of gracefully handling it"
120 + ),
121 + },
122 + )
tests/tool_parsers/test_longcat_tool_parser.py new
+101
@@ -0,0 +1,101 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from unittest.mock import MagicMock
5 +
6 +import pytest
7 +
8 +from tests.tool_parsers.common_tests import (
9 + ToolParserTestConfig,
10 + ToolParserTests,
11 +)
12 +from vllm.tokenizers import TokenizerLike
13 +
14 +
15 +class TestLongCatToolParser(ToolParserTests):
16 + @pytest.fixture
17 + def tokenizer(self, default_tokenizer: TokenizerLike) -> TokenizerLike:
18 + """Add some longcat specific tokens to the default vocab."""
19 + tokenizer = default_tokenizer
20 + tokenizer_vocab = tokenizer.get_vocab()
21 + tokenizer.get_vocab = MagicMock()
22 + tokenizer_vocab.update(
23 + {
24 + "<longcat_tool_call>": 32000,
25 + "</longcat_tool_call>": 32001,
26 + }
27 + )
28 + tokenizer.get_vocab.return_value = tokenizer_vocab
29 + return tokenizer
30 +
31 + @pytest.fixture
32 + def test_config(self) -> ToolParserTestConfig:
33 + return ToolParserTestConfig(
34 + parser_name="longcat",
35 + # Test data
36 + no_tool_calls_output="This is a regular response without any tool calls.",
37 + single_tool_call_output=(
38 + '<longcat_tool_call>{"name": "get_weather", '
39 + '"arguments": {"city": "Tokyo"}}</longcat_tool_call>'
40 + ),
41 + parallel_tool_calls_output=(
42 + '<longcat_tool_call>{"name": "get_weather", '
43 + '"arguments": {"city": "Tokyo"}}</longcat_tool_call>\n'
44 + '<longcat_tool_call>{"name": "get_time", '
45 + '"arguments": {"timezone": "Asia/Tokyo"}}</longcat_tool_call>'
46 + ),
47 + various_data_types_output="""<longcat_tool_call>{
48 + "name": "test_function",
49 + "arguments": {
50 + "string_field": "hello",
51 + "int_field": 42,
52 + "float_field": 3.14,
53 + "bool_field": true,
54 + "null_field": null,
55 + "array_field": ["a", "b", "c"],
56 + "object_field": {"nested": "value"},
57 + "empty_array": [],
58 + "empty_object": {}
59 + }
60 +}</longcat_tool_call>""",
61 + empty_arguments_output=(
62 + '<longcat_tool_call>{"name": "refresh", "arguments": {}}'
63 + "</longcat_tool_call>"
64 + ),
65 + surrounding_text_output=(
66 + "Let me check the weather for you.\n"
67 + '<longcat_tool_call>{"name": "get_weather", '
68 + '"arguments": {"city": "Tokyo"}}</longcat_tool_call>\n'
69 + "Here is the result."
70 + ),
71 + escaped_strings_output="""<longcat_tool_call>{
72 + "name": "test_function",
73 + "arguments": {
74 + "quoted": "He said \\"hello\\"",
75 + "path": "C:\\\\Users\\\\file.txt",
76 + "newline": "line1\\nline2",
77 + "unicode": "emoji: 🎉"
78 + }
79 +}</longcat_tool_call>""",
80 + malformed_input_outputs=[
81 + '<longcat_tool_call>{"name": "func", "arguments": {',
82 + (
83 + '<longcat_tool_call>{"name": "func", '
84 + '"arguments": "not a dict"}</longcat_tool_call>'
85 + ),
86 + "Some text with <longcat_tool_call>invalid json",
87 + ],
88 + # Expected results
89 + single_tool_call_expected_name="get_weather",
90 + single_tool_call_expected_args={"city": "Tokyo"},
91 + single_tool_call_expected_content=None,
92 + parallel_tool_calls_count=2,
93 + parallel_tool_calls_names=["get_weather", "get_time"],
94 + # xfail markers
95 + xfail_streaming={
96 + "test_malformed_input": "Streaming has complex buffering behavior",
97 + },
98 + xfail_nonstreaming={},
99 + # Configuration
100 + allow_empty_or_json_empty_args=True,
101 + )
tests/tool_parsers/test_phi4mini_tool_parser.py new
+110
@@ -0,0 +1,110 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from unittest.mock import MagicMock
5 +
6 +import pytest
7 +
8 +from tests.tool_parsers.common_tests import (
9 + ToolParserTestConfig,
10 + ToolParserTests,
11 +)
12 +from vllm.tokenizers import TokenizerLike
13 +
14 +
15 +class TestPhi4MiniToolParser(ToolParserTests):
16 + @pytest.fixture
17 + def tokenizer(self, default_tokenizer: TokenizerLike) -> TokenizerLike:
18 + """Add some phi4mini specific tokens to the default vocab."""
19 +
20 + tokenizer = default_tokenizer
21 + tokenizer_vocab = tokenizer.get_vocab()
22 + tokenizer.get_vocab = MagicMock()
23 + tokenizer_vocab.update(
24 + {
25 + "functools": 32000,
26 + }
27 + )
28 + tokenizer.get_vocab.return_value = tokenizer_vocab
29 + return tokenizer
30 +
31 + @pytest.fixture
32 + def test_config(self) -> ToolParserTestConfig:
33 + return ToolParserTestConfig(
34 + parser_name="phi4_mini_json",
35 + # Test data
36 + no_tool_calls_output="This is a regular response without any tool calls.",
37 + single_tool_call_output=(
38 + 'functools[{"name": "get_weather", "arguments": {"city": "Tokyo"}}]'
39 + ),
40 + parallel_tool_calls_output="""functools[
41 + {"name": "get_weather", "arguments": {"city": "Tokyo"}},
42 + {"name": "get_time", "arguments": {"timezone": "Asia/Tokyo"}}
43 +]""",
44 + various_data_types_output="""functools[{
45 + "name": "test_function",
46 + "arguments": {
47 + "string_field": "hello",
48 + "int_field": 42,
49 + "float_field": 3.14,
50 + "bool_field": true,
51 + "null_field": null,
52 + "array_field": ["a", "b", "c"],
53 + "object_field": {"nested": "value"},
54 + "empty_array": [],
55 + "empty_object": {}
56 + }
57 +}]""",
58 + empty_arguments_output='functools[{"name": "refresh", "arguments": {}}]',
59 + surrounding_text_output="""Let me check the weather for you.
60 +functools[{"name": "get_weather", "arguments": {"city": "Tokyo"}}]
61 +Would you like to know more?""",
62 + escaped_strings_output="""functools[{
63 + "name": "test_function",
64 + "arguments": {
65 + "quoted": "He said \\"hello\\"",
66 + "path": "C:\\\\Users\\\\file.txt",
67 + "newline": "line1\\nline2",
68 + "unicode": "emoji: 🎉"
69 + }
70 +}]""",
71 + malformed_input_outputs=[
72 + 'functools[{"name": "func", "arguments": {',
73 + 'functools[{"name": "func", "arguments": "not a dict"}]',
74 + 'functools{"name": "func"}', # Missing brackets
75 + 'functools[{"name": "func"}]', # Missing arguments/parameters
76 + "functools[] This is just text", # Empty functools
77 + "functools[ This is just text ]", # functools with invalid JSON
78 + ],
79 + # Expected results
80 + single_tool_call_expected_name="get_weather",
81 + single_tool_call_expected_args={"city": "Tokyo"},
82 + # Phi-4 Mini strips content when tool calls present
83 + single_tool_call_expected_content=None,
84 + parallel_tool_calls_count=2,
85 + parallel_tool_calls_names=["get_weather", "get_time"],
86 + parallel_tool_calls_expected_content=None,
87 + # xfail markers
88 + xfail_streaming={
89 + "test_no_tool_calls": "Phi4 Mini streaming not implemented",
90 + "test_single_tool_call_simple_args": (
91 + "Phi4 Mini streaming not implemented"
92 + ),
93 + "test_parallel_tool_calls": "Phi4 Mini streaming not implemented",
94 + "test_various_data_types": "Phi4 Mini streaming not implemented",
95 + "test_empty_arguments": "Phi4 Mini streaming not implemented",
96 + "test_surrounding_text": "Phi4 Mini streaming not implemented",
97 + "test_escaped_strings": "Phi4 Mini streaming not implemented",
98 + "test_streaming_reconstruction": "Phi4 Mini streaming not implemented",
99 + },
100 + xfail_nonstreaming={
101 + "test_various_data_types": (
102 + "Phi4MiniJsonToolParser regex has nesting limitations "
103 + "with nested objects"
104 + ),
105 + "test_malformed_input": (
106 + "Phi4MiniJsonToolParser incorrectly sets "
107 + "tools_called=True on empty array"
108 + ),
109 + },
110 + )
tests/tool_parsers/test_qwen3xml_tool_parser.py new
+75
@@ -0,0 +1,75 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +
5 +import pytest
6 +
7 +from tests.tool_parsers.common_tests import (
8 + ToolParserTestConfig,
9 + ToolParserTests,
10 +)
11 +
12 +
13 +class TestQwen3xmlToolParser(ToolParserTests):
14 + @pytest.fixture
15 + def test_config(self) -> ToolParserTestConfig:
16 + return ToolParserTestConfig(
17 + parser_name="qwen3_xml",
18 + # Test data
19 + no_tool_calls_output="This is a regular response without any tool calls.",
20 + single_tool_call_output="<tool_call>\n<function=get_weather>\n<parameter=city>Tokyo</parameter>\n</function>\n</tool_call>",
21 + parallel_tool_calls_output="<tool_call>\n<function=get_weather>\n<parameter=city>Tokyo</parameter>\n</function>\n</tool_call><tool_call>\n<function=get_time>\n<parameter=timezone>Asia/Tokyo</parameter>\n</function>\n</tool_call>",
22 + various_data_types_output=(
23 + "<tool_call>\n<function=test_function>\n"
24 + "<parameter=string_field>hello</parameter>\n"
25 + "<parameter=int_field>42</parameter>\n"
26 + "<parameter=float_field>3.14</parameter>\n"
27 + "<parameter=bool_field>true</parameter>\n"
28 + "<parameter=null_field>null</parameter>\n"
29 + '<parameter=array_field>["a", "b", "c"]</parameter>\n'
30 + '<parameter=object_field>{"nested": "value"}</parameter>\n'
31 + "</function>\n</tool_call>"
32 + ),
33 + empty_arguments_output="<tool_call>\n<function=refresh>\n</function>\n</tool_call>",
34 + surrounding_text_output=(
35 + "Let me check the weather for you.\n\n"
36 + "<tool_call>\n<function=get_weather>\n"
37 + "<parameter=city>Tokyo</parameter>\n"
38 + "</function>\n</tool_call>\n\n"
39 + "I will get that information."
40 + ),
41 + escaped_strings_output=(
42 + "<tool_call>\n<function=test_function>\n"
43 + '<parameter=quoted>He said "hello"</parameter>\n'
44 + "<parameter=path>C:\\Users\\file.txt</parameter>\n"
45 + "<parameter=newline>line1\nline2</parameter>\n"
46 + "</function>\n</tool_call>"
47 + ),
48 + malformed_input_outputs=[
49 + "<tool_call><function=func>",
50 + "<tool_call><function=></function></tool_call>",
51 + ],
52 + # Expected results
53 + single_tool_call_expected_name="get_weather",
54 + single_tool_call_expected_args={"city": "Tokyo"},
55 + parallel_tool_calls_count=2,
56 + parallel_tool_calls_names=["get_weather", "get_time"],
57 + # xfail markers - Qwen3XML has systematic streaming issues
58 + xfail_streaming={
59 + "test_single_tool_call_simple_args": (
60 + "Qwen3XML streaming has systematic issues"
61 + ),
62 + "test_parallel_tool_calls": "Qwen3XML streaming has systematic issues",
63 + "test_various_data_types": "Qwen3XML streaming has systematic issues",
64 + "test_empty_arguments": "Qwen3XML streaming has systematic issues",
65 + "test_surrounding_text": "Qwen3XML streaming has systematic issues",
66 + "test_escaped_strings": "Qwen3XML streaming has systematic issues",
67 + "test_malformed_input": (
68 + "Qwen3XML parser is lenient with malformed input"
69 + ),
70 + "test_streaming_reconstruction": (
71 + "Qwen3XML streaming reconstruction has known issues"
72 + ),
73 + },
74 + supports_typed_arguments=False,
75 + )
tests/tool_parsers/test_step3_tool_parser.py new
+112
@@ -0,0 +1,112 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +
5 +import pytest
6 +
7 +from tests.tool_parsers.common_tests import (
8 + ToolParserTestConfig,
9 + ToolParserTests,
10 +)
11 +from vllm.tokenizers import TokenizerLike, get_tokenizer
12 +
13 +
14 +class TestStep3ToolParser(ToolParserTests):
15 + @pytest.fixture(scope="class")
16 + def tokenizer(self) -> TokenizerLike:
17 + return get_tokenizer("stepfun-ai/step3")
18 +
19 + @pytest.fixture
20 + def test_config(self) -> ToolParserTestConfig:
21 + return ToolParserTestConfig(
22 + parser_name="step3",
23 + # Test data
24 + no_tool_calls_output="This is a regular response without any tool calls.",
25 + single_tool_call_output=(
26 + "<｜tool_calls_begin｜><｜tool_call_begin｜>"
27 + '<steptml:invoke name="get_weather">'
28 + '<steptml:parameter name="city">Tokyo</steptml:parameter>'
29 + "</steptml:invoke><｜tool_call_end｜><｜tool_calls_end｜>"
30 + ),
31 + parallel_tool_calls_output=(
32 + "<｜tool_calls_begin｜><｜tool_call_begin｜>"
33 + '<steptml:invoke name="get_weather">'
34 + '<steptml:parameter name="city">Tokyo</steptml:parameter>'
35 + "</steptml:invoke><｜tool_call_end｜><｜tool_sep｜>"
36 + '<｜tool_call_begin｜><steptml:invoke name="get_time">'
37 + '<steptml:parameter name="timezone">Asia/Tokyo</steptml:parameter>'
38 + "</steptml:invoke><｜tool_call_end｜><｜tool_calls_end｜>"
39 + ),
40 + various_data_types_output=(
41 + "<｜tool_calls_begin｜><｜tool_call_begin｜>"
42 + '<steptml:invoke name="test_function">'
43 + '<steptml:parameter name="string_field">hello</steptml:parameter>'
44 + '<steptml:parameter name="int_field">42</steptml:parameter>'
45 + '<steptml:parameter name="float_field">3.14</steptml:parameter>'
46 + '<steptml:parameter name="bool_field">true</steptml:parameter>'
47 + '<steptml:parameter name="null_field">null</steptml:parameter>'
48 + '<steptml:parameter name="array_field">'
49 + '["a", "b", "c"]</steptml:parameter>'
50 + '<steptml:parameter name="object_field">'
51 + '{"nested": "value"}</steptml:parameter>'
52 + "</steptml:invoke><｜tool_call_end｜><｜tool_calls_end｜>"
53 + ),
54 + empty_arguments_output=(
55 + "<｜tool_calls_begin｜><｜tool_call_begin｜>"
56 + '<steptml:invoke name="refresh"></steptml:invoke>'
57 + "<｜tool_call_end｜><｜tool_calls_end｜>"
58 + ),
59 + surrounding_text_output=(
60 + "Let me check the weather for you.\n\n"
61 + "<｜tool_calls_begin｜><｜tool_call_begin｜>"
62 + '<steptml:invoke name="get_weather">'
63 + '<steptml:parameter name="city">Tokyo</steptml:parameter>'
64 + "</steptml:invoke><｜tool_call_end｜><｜tool_calls_end｜>\n\n"
65 + "I'll get that information."
66 + ),
67 + escaped_strings_output=(
68 + "<｜tool_calls_begin｜><｜tool_call_begin｜>"
69 + '<steptml:invoke name="test_function">'
70 + '<steptml:parameter name="quoted">He said "hello"</steptml:parameter>'
71 + '<steptml:parameter name="path">C:\\Users\\file.txt</steptml:parameter>'
72 + '<steptml:parameter name="newline">line1\nline2</steptml:parameter>'
73 + "</steptml:invoke><｜tool_call_end｜><｜tool_calls_end｜>"
74 + ),
75 + malformed_input_outputs=[
76 + (
77 + "<｜tool_calls_begin｜><｜tool_call_begin｜>"
78 + '<steptml:invoke name="func">'
79 + ),
80 + (
81 + '<｜tool_call_begin｜><steptml:invoke name="func">'
82 + "</steptml:invoke><｜tool_call_end｜>"
83 + ),
84 + ],
85 + # Expected results
86 + single_tool_call_expected_name="get_weather",
87 + single_tool_call_expected_args={"city": "Tokyo"},
88 + parallel_tool_calls_count=2,
89 + parallel_tool_calls_names=["get_weather", "get_time"],
90 + # xfail markers
91 + xfail_nonstreaming={
92 + "test_single_tool_call_simple_args": (
93 + "Step3 parser non-streaming has bugs"
94 + ),
95 + "test_parallel_tool_calls": ("Step3 parser non-streaming has bugs"),
96 + "test_various_data_types": "Step3 parser non-streaming has bugs",
97 + "test_empty_arguments": "Step3 parser non-streaming has bugs",
98 + "test_surrounding_text": "Step3 parser non-streaming has bugs",
99 + "test_escaped_strings": "Step3 parser non-streaming has bugs",
100 + },
101 + xfail_streaming={
102 + "test_parallel_tool_calls": (
103 + "Step3 parser has significant bugs in both streaming "
104 + "and non-streaming"
105 + ),
106 + "test_streaming_reconstruction": (
107 + "Step3 parser non-streaming has bugs, so streaming "
108 + "doesn't match non-streaming"
109 + ),
110 + },
111 + supports_typed_arguments=False,
112 + )
tests/tool_parsers/utils.py renamed