A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 16:00 UTC
[Bugfix] properly handle nested json with llama3 tool parser (#27701)
Signed-off-by: Aydin Abiar <aydin@anyscale.com> Signed-off-by: Aydin Abiar <62435714+Aydin-ab@users.noreply.github.com> Co-authored-by: Aydin Abiar <aydin@anyscale.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
Aydin Abiar committed
Nov 24, 2025 at 07:28 UTC
656516c3158ef932c6a19a6aab9fdf4df74b105f
2 files changed
+201
-39
tests/entrypoints/openai/tool_parsers/test_llama3_json_tool_parser.py
+128
@@ -1,6 +1,8 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+from unittest.mock import MagicMock, patch
5
+
6
import pytest
7
8
from vllm.entrypoints.openai.protocol import ExtractedToolCallInformation
@@ -132,3 +134,129 @@ def test_extract_tool_calls_multiple_json_with_surrounding_text(parser):
134
assert result.tool_calls[0].function.name == "searchTool"
135
assert result.tool_calls[1].function.name == "getOpenIncidentsTool"
136
assert result.tool_calls[2].function.name == "searchTool"
137
+
138
+
139
+def test_extract_tool_calls_deeply_nested_json(parser):
140
+ # Test with deeply nested JSON parameters (5 levels)
141
+ model_output = (
142
+ '{"name": "complexTool", '
143
+ '"parameters": {'
144
+ '"level1": {'
145
+ '"level2": {'
146
+ '"level3": {'
147
+ '"level4": {'
148
+ '"value": "deep"'
149
+ "}}}}}}"
150
+ )
151
+ result = parser.extract_tool_calls(model_output, None)
152
+
153
+ assert result.tools_called is True
154
+ assert len(result.tool_calls) == 1
155
+ assert result.tool_calls[0].function.name == "complexTool"
156
+ # Verify the nested structure is preserved in the arguments
157
+ import json
158
+
159
+ args = json.loads(result.tool_calls[0].function.arguments)
160
+ assert args["level1"]["level2"]["level3"]["level4"]["value"] == "deep"
161
+
162
+
163
+def test_extract_tool_calls_multiple_with_deep_nesting(parser):
164
+ # Test with multiple tool calls where some have deeply nested parameters
165
+ model_output = (
166
+ '{"name": "simpleTool", "parameters": {"value": "test"}}; '
167
+ '{"name": "complexTool", "parameters": '
168
+ '{"config": {"database": {"connection": {"pool": {"size": 10}}}}}}'
169
+ )
170
+ result = parser.extract_tool_calls(model_output, None)
171
+
172
+ assert result.tools_called is True
173
+ assert len(result.tool_calls) == 2
174
+
175
+ # Check first tool call
176
+ assert result.tool_calls[0].function.name == "simpleTool"
177
+ import json
178
+
179
+ args0 = json.loads(result.tool_calls[0].function.arguments)
180
+ assert args0["value"] == "test"
181
+
182
+ # Check second tool call with deep nesting
183
+ assert result.tool_calls[1].function.name == "complexTool"
184
+ args1 = json.loads(result.tool_calls[1].function.arguments)
185
+ assert args1["config"]["database"]["connection"]["pool"]["size"] == 10
186
+
187
+
188
+def test_extract_tool_calls_with_quotes_and_brackets_in_string(parser):
189
+ # Test with quotes and brackets inside quoted string values
190
+ model_output = (
191
+ '{"name": "searchTool", '
192
+ '"parameters": {'
193
+ '"query": "test {value} [complex]",'
194
+ '"nested": {"inner": "more {brackets}"}'
195
+ "}}"
196
+ )
197
+ result = parser.extract_tool_calls(model_output, None)
198
+
199
+ assert result.tools_called is True
200
+ assert len(result.tool_calls) == 1
201
+ assert result.tool_calls[0].function.name == "searchTool"
202
+ # Verify the string values are preserved including brackets and quotes
203
+ import json
204
+
205
+ args = json.loads(result.tool_calls[0].function.arguments)
206
+ assert args["query"] == "test {value} [complex]"
207
+ assert args["nested"]["inner"] == "more {brackets}"
208
+
209
+
210
+def test_extract_tool_calls_with_escaped_quotes_in_nested_json(parser):
211
+ # Test with escaped quotes in deeply nested JSON
212
+ model_output = (
213
+ '{"name": "parserTool", "parameters": {"text": "He said \\"Hello {world}\\""}}'
214
+ )
215
+ result = parser.extract_tool_calls(model_output, None)
216
+
217
+ assert result.tools_called is True
218
+ assert len(result.tool_calls) == 1
219
+ assert result.tool_calls[0].function.name == "parserTool"
220
+ # Verify escaped quotes are preserved
221
+ import json
222
+
223
+ args = json.loads(result.tool_calls[0].function.arguments)
224
+ assert args["text"] == 'He said "Hello {world}"'
225
+
226
+
227
+def test_extract_tool_calls_missing_name_key(parser):
228
+ # Test that missing "name" key returns content
229
+ model_output = '{"parameters": {}}'
230
+ result = parser.extract_tool_calls(model_output, None)
231
+
232
+ assert result.tools_called is False
233
+ assert len(result.tool_calls) == 0
234
+ assert result.content == model_output
235
+
236
+
237
+def test_extract_tool_calls_missing_parameters_and_arguments_key(parser):
238
+ # Test that missing both "parameters" and "arguments" keys returns content
239
+ model_output = '{"name": "toolWithoutParams"}'
240
+ result = parser.extract_tool_calls(model_output, None)
241
+
242
+ assert result.tools_called is False
243
+ assert len(result.tool_calls) == 0
244
+ assert result.content == model_output
245
+
246
+
247
+def test_regex_timeout_handling(parser):
248
+ """Test regex timeout is handled gracefully"""
249
+ fake_problematic_input = "{hello world[A(A=" + "\t)A(A=,\t" * 2
250
+
251
+ # create a mock regex that raises TimeoutError
252
+ mock_regex = MagicMock()
253
+ mock_regex.finditer.side_effect = TimeoutError("Regex timeout")
254
+
255
+ with patch.object(parser, "tool_call_start_regex", mock_regex):
256
+ result = parser.extract_tool_calls(fake_problematic_input, None)
257
+
258
+ # should treat as regular text when regex times out
259
+ assert result.content == fake_problematic_input
260
+ assert result.tools_called is False
261
+ assert len(result.tool_calls) == 0
262
+ mock_regex.finditer.assert_called_once()
vllm/entrypoints/openai/tool_parsers/llama_tool_parser.py
+73
-39
@@ -9,6 +9,7 @@ import regex as re
9
from partial_json_parser.core.options import Allow
10
from transformers import PreTrainedTokenizerBase
11
12
+import vllm.envs as envs
13
from vllm.entrypoints.chat_utils import make_tool_call_id
14
from vllm.entrypoints.openai.protocol import (
15
ChatCompletionRequest,
@@ -56,12 +57,10 @@ class Llama3JsonToolParser(ToolParser):
57
self.bot_token_id = tokenizer.encode(self.bot_token, add_special_tokens=False)[
58
0
59
]
59
- # Updated regex to match multiple JSONs separated by semicolons
60
- # This pattern is more robust and can handle nested JSON objects
61
- self.tool_call_regex = re.compile(
62
- r"{[^{}]*(?:{[^{}]*}[^{}]*)*}(?:\s*;\s*{[^{}]*(?:{[^{}]*}[^{}]*)*})*",
63
- re.DOTALL,
64
- )
60
+ # Simple regex to find opening braces - we'll use JSON decoder for parsing
61
+ # This handles arbitrary nesting depth correctly
62
+ self.tool_call_start_regex = re.compile(r"\{")
63
+ self.json_decoder = json.JSONDecoder()
64
65
def extract_tool_calls(
66
self, model_output: str, request: ChatCompletionRequest
@@ -77,50 +76,85 @@ class Llama3JsonToolParser(ToolParser):
76
tools_called=False, tool_calls=[], content=model_output
77
)
78
80
- # Find JSON object(s) in the text using regex
81
- match = self.tool_call_regex.search(model_output)
82
- if not match:
83
- return ExtractedToolCallInformation(
84
- tools_called=False, tool_calls=[], content=model_output
85
- )
79
+ # Keep track of the end index of the last parsed JSON object
80
+ # so we don't parse inner brackets
81
+ end_index = -1
82
+ tool_calls: list[ToolCall] = []
83
84
try:
88
- json_str = match.group(0)
89
- # Split by semicolon and strip whitespace
90
- json_objects = [obj.strip() for obj in json_str.split(";")]
91
-
92
- tool_calls: list[ToolCall] = []
93
- for json_obj in json_objects:
94
- if not json_obj: # Skip empty strings
85
+ for match in self.tool_call_start_regex.finditer(
86
+ model_output, timeout=envs.VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS
87
+ ):
88
+ start_index = match.start()
89
+ # Skip if this brace is inside a previously parsed JSON object
90
+ if start_index <= end_index:
91
continue
96
- obj = json.loads(json_obj)
97
- tool_calls.append(
98
- ToolCall(
99
- type="function",
100
- function=FunctionCall(
101
- name=obj["name"],
102
- # function call args are JSON but as a string
103
- arguments=json.dumps(
104
- obj["arguments"]
105
- if "arguments" in obj
106
- else obj["parameters"],
107
- ensure_ascii=False,
108
- ),
109
- ),
92
+
93
+ try:
94
+ obj, json_end_index = self.json_decoder.raw_decode(
95
+ model_output[start_index:]
96
+ )
97
+ end_index = start_index + json_end_index
98
+
99
+ # raise KeyError if missing
100
+ name = obj["name"]
101
+ arguments_or_params = (
102
+ obj["arguments"] if "arguments" in obj else obj["parameters"]
103
)
111
- )
104
105
+ tool_calls.append(
106
+ ToolCall(
107
+ type="function",
108
+ function=FunctionCall(
109
+ name=name,
110
+ # function call args are JSON but as a string
111
+ arguments=json.dumps(
112
+ arguments_or_params, ensure_ascii=False
113
+ ),
114
+ ),
115
+ )
116
+ )
117
+ except KeyError as e:
118
+ # Missing required key
119
+ missing_key = str(e).strip("'\"")
120
+ logger.exception(
121
+ "Couldn't extract tool call from JSON response. "
122
+ "Required key '%s' not present. "
123
+ "Returning output in content with empty tool calls.",
124
+ missing_key,
125
+ )
126
+ return ExtractedToolCallInformation(
127
+ tools_called=False, tool_calls=[], content=model_output
128
+ )
129
+ except Exception:
130
+ # Any other error during parsing
131
+ logger.exception(
132
+ "Error in extracting tool call from response. "
133
+ "Returning output in content with empty tool calls"
134
+ )
135
+ return ExtractedToolCallInformation(
136
+ tools_called=False, tool_calls=[], content=model_output
137
+ )
138
+ except TimeoutError:
139
+ logger.warning("Regex timeout occurred when matching tool call pattern.")
140
+ logger.debug(
141
+ "Regex timeout occurred when matching user input: %s", model_output
142
+ )
143
return ExtractedToolCallInformation(
114
- tools_called=True, tool_calls=tool_calls, content=None
144
+ tools_called=False, tool_calls=[], content=model_output
145
)
146
117
- except Exception:
118
- logger.exception("Error in extracting tool call from response.")
119
- # return information to just treat the tool call as regular JSON
147
+ # If we have valid tool calls, return them normally
148
+ if tool_calls:
149
return ExtractedToolCallInformation(
121
- tools_called=False, tool_calls=[], content=model_output
150
+ tools_called=True, tool_calls=tool_calls, content=None
151
)
152
153
+ # No valid tool calls found
154
+ return ExtractedToolCallInformation(
155
+ tools_called=False, tool_calls=[], content=model_output
156
+ )
157
+
158
def extract_tool_calls_streaming(
159
self,
160
previous_text: str,