A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 00:15 UTC
[Misc] Refactored 5 duplicate helper functions that were copied-pasted across multiple parsers (#36436)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
Taneem Ibrahim committed
Mar 9, 2026 at 14:14 UTC
8d6b3d5dda293231c7c2fc9301002113f270a534
4 files changed
+247
-452
vllm/tool_parsers/llama4_pythonic_tool_parser.py
+13
-147
@@ -1,9 +1,8 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
import ast
4
-import json
5
from collections.abc import Sequence
6
-from typing import Any
6
7
import regex as re
8
from transformers import PreTrainedTokenizerBase
@@ -13,25 +12,23 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
12
ChatCompletionRequest,
13
)
14
from vllm.entrypoints.openai.engine.protocol import (
16
- DeltaFunctionCall,
15
DeltaMessage,
18
- DeltaToolCall,
16
ExtractedToolCallInformation,
20
- FunctionCall,
21
- ToolCall,
17
)
18
from vllm.logger import init_logger
19
from vllm.tool_parsers.abstract_tool_parser import (
20
ToolParser,
21
)
22
+from vllm.tool_parsers.utils import (
23
+ UnexpectedAstError,
24
+ compute_tool_delta,
25
+ handle_single_tool,
26
+ make_valid_python,
27
+)
28
29
logger = init_logger(__name__)
30
31
31
-class _UnexpectedAstError(Exception):
32
- pass
33
-
34
-
32
class Llama4PythonicToolParser(ToolParser):
33
"""
34
Toolcall parser for Llama4 that produce tool calls in a pythonic style
@@ -103,15 +100,13 @@ class Llama4PythonicToolParser(ToolParser):
100
return ExtractedToolCallInformation(
101
tools_called=True,
102
tool_calls=[
106
- _handle_single_tool(e) # type: ignore
103
+ handle_single_tool(e) # type: ignore
104
for e in parsed.elts
105
],
106
content=None,
107
)
108
else:
112
- raise _UnexpectedAstError(
113
- "Tool output must be a list of function calls"
114
- )
109
+ raise UnexpectedAstError("Tool output must be a list of function calls")
110
except Exception:
111
logger.exception("Error in extracting tool call from response.")
112
# Treat as regular text
@@ -140,7 +135,7 @@ class Llama4PythonicToolParser(ToolParser):
135
current_text = current_text[len("<|python_start|>") :]
136
if current_text.endswith("<|python_end|>"):
137
current_text = current_text[: current_text.rfind("<|python_end|>")]
143
- valid_and_added_text = _make_valid_python(current_text)
138
+ valid_and_added_text = make_valid_python(current_text)
139
if valid_and_added_text is None:
140
return None
141
valid_text, added_text = valid_and_added_text
@@ -150,11 +145,9 @@ class Llama4PythonicToolParser(ToolParser):
145
if not isinstance(parsed, ast.List) or not all(
146
isinstance(e, ast.Call) for e in parsed.elts
147
):
153
- raise _UnexpectedAstError(
154
- "Tool output must be a list of function calls"
155
- )
148
+ raise UnexpectedAstError("Tool output must be a list of function calls")
149
tool_calls = [
157
- _handle_single_tool(e) # type: ignore
150
+ handle_single_tool(e) # type: ignore
151
for e in parsed.elts
152
]
153
@@ -180,7 +173,7 @@ class Llama4PythonicToolParser(ToolParser):
173
# Strings get single quotes in the model-produced string.
174
# JSON requires double quotes.
175
withheld_suffix = withheld_suffix.replace("'", '"')
183
- delta = _compute_tool_delta(
176
+ delta = compute_tool_delta(
177
self.streamed_args_for_tool[index], new_call, index, withheld_suffix
178
)
179
@@ -214,130 +207,3 @@ class Llama4PythonicToolParser(ToolParser):
207
"Skipping chunk as a result of tool streaming extraction error"
208
)
209
return None
217
-
218
-
219
-def _get_parameter_value(val: ast.expr) -> Any:
220
- if isinstance(val, ast.Constant):
221
- return val.value
222
- elif isinstance(val, ast.Dict):
223
- if not all(isinstance(k, ast.Constant) for k in val.keys):
224
- raise _UnexpectedAstError("Dict tool call arguments must have literal keys")
225
- return {
226
- k.value: _get_parameter_value(v) # type: ignore
227
- for k, v in zip(val.keys, val.values)
228
- }
229
- elif isinstance(val, ast.List):
230
- return [_get_parameter_value(v) for v in val.elts]
231
- else:
232
- raise _UnexpectedAstError("Tool call arguments must be literals")
233
-
234
-
235
-def _handle_single_tool(call: ast.Call) -> ToolCall:
236
- if not isinstance(call.func, ast.Name):
237
- raise _UnexpectedAstError("Invalid tool call name")
238
- function_name = call.func.id
239
- arguments = {}
240
- for keyword in call.keywords:
241
- arguments[keyword.arg] = _get_parameter_value(keyword.value)
242
- return ToolCall(
243
- type="function",
244
- function=FunctionCall(name=function_name, arguments=json.dumps(arguments)),
245
- )
246
-
247
-
248
-def _make_valid_python(text: str) -> tuple[str, str] | None:
249
- bracket_stack = []
250
- for index, char in enumerate(text):
251
- if char in {"[", "(", "{"}:
252
- bracket_stack.append(char)
253
- elif char == "]":
254
- if not bracket_stack or bracket_stack.pop() != "[":
255
- raise _UnexpectedAstError("Mismatched square brackets")
256
- elif char == ")":
257
- if not bracket_stack or bracket_stack.pop() != "(":
258
- raise _UnexpectedAstError("Mismatched parentheses")
259
- elif char == "}":
260
- if not bracket_stack or bracket_stack.pop() != "{":
261
- raise _UnexpectedAstError("Mismatched curly braces")
262
- elif char in {"'", '"'}:
263
- if bracket_stack and bracket_stack[-1] == char:
264
- if index > 0 and text[index - 1] == "\\":
265
- # Treat an escaped quote as a regular character
266
- pass
267
- else:
268
- bracket_stack.pop()
269
- elif bracket_stack and bracket_stack[-1] in {"'", '"'}:
270
- # Double quote within a single quote string or vice versa.
271
- pass
272
- else:
273
- bracket_stack.append(char)
274
-
275
- text = text.rstrip()
276
- if text.endswith("=") or text.endswith(":"):
277
- # Since we have no type information for this property/parameter value,
278
- # we can't fill in a valid value.
279
- return None
280
- if bracket_stack and bracket_stack[-1] == "{":
281
- trailing_dict_text = text[: text.rfind("{")]
282
- num_keys = trailing_dict_text.count(":")
283
- num_values = trailing_dict_text.count(",")
284
- if num_keys <= num_values:
285
- return None # Incomplete property name within parameter value
286
- if bracket_stack and bracket_stack[-1] == "(":
287
- trailing_params_text = text[: text.rfind("(")]
288
- num_full_param_names = trailing_params_text.count("=")
289
- num_full_param_values = trailing_params_text.count(",")
290
- if num_full_param_names <= num_full_param_values:
291
- return None # Incomplete parameter name
292
- if text.endswith(","):
293
- text = text[:-1]
294
- if (
295
- bracket_stack
296
- and bracket_stack[-1] == "["
297
- and not text.endswith("[")
298
- and not text.endswith(")")
299
- ):
300
- return None # Incomplete function name
301
-
302
- added_text = ""
303
- for char in reversed(bracket_stack):
304
- if char == "[":
305
- added_text += "]"
306
- elif char == "(":
307
- added_text += ")"
308
- elif char == "{":
309
- added_text += "}"
310
- elif char == "'":
311
- added_text += "'"
312
- elif char == '"':
313
- added_text += '"'
314
-
315
- return text + added_text, added_text
316
-
317
-
318
-def _compute_tool_delta(
319
- previously_sent_args: str, new_call: ToolCall, index: int, withheld_suffix: str
320
-) -> DeltaToolCall | None:
321
- new_call_args = new_call.function.arguments
322
- if withheld_suffix:
323
- assert new_call_args.endswith(withheld_suffix)
324
- new_call_args = new_call_args[: -len(withheld_suffix)]
325
- if not previously_sent_args:
326
- return DeltaToolCall(
327
- id=new_call.id,
328
- type="function",
329
- index=index,
330
- function=DeltaFunctionCall(
331
- name=new_call.function.name,
332
- arguments=new_call_args,
333
- ),
334
- )
335
-
336
- arg_diff = new_call_args[len(previously_sent_args) :]
337
- return (
338
- DeltaToolCall(
339
- id=None, index=index, function=DeltaFunctionCall(arguments=arg_diff)
340
- )
341
- if arg_diff
342
- else None
343
- )
vllm/tool_parsers/olmo3_tool_parser.py
+13
-156
@@ -1,9 +1,8 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
import ast
4
-import json
5
from collections.abc import Sequence
6
-from typing import Any
6
7
import regex as re
8
from transformers import PreTrainedTokenizerBase
@@ -13,25 +12,23 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
12
ChatCompletionRequest,
13
)
14
from vllm.entrypoints.openai.engine.protocol import (
16
- DeltaFunctionCall,
15
DeltaMessage,
18
- DeltaToolCall,
16
ExtractedToolCallInformation,
20
- FunctionCall,
21
- ToolCall,
17
)
18
from vllm.logger import init_logger
19
from vllm.tool_parsers.abstract_tool_parser import (
20
ToolParser,
21
)
22
+from vllm.tool_parsers.utils import (
23
+ UnexpectedAstError,
24
+ compute_tool_delta,
25
+ handle_single_tool,
26
+ make_valid_python,
27
+)
28
29
logger = init_logger(__name__)
30
31
31
-class _UnexpectedAstError(Exception):
32
- pass
33
-
34
-
32
class Olmo3PythonicToolParser(ToolParser):
33
"""
34
Tool call parser for Olmo 3 models that produce tool calls as
@@ -113,15 +110,13 @@ class Olmo3PythonicToolParser(ToolParser):
110
return ExtractedToolCallInformation(
111
tools_called=True,
112
tool_calls=[
116
- _handle_single_tool(e) # type: ignore
113
+ handle_single_tool(e) # type: ignore
114
for e in parsed.elts
115
],
116
content=None,
117
)
118
else:
122
- raise _UnexpectedAstError(
123
- "Tool output must be a list of function calls"
124
- )
119
+ raise UnexpectedAstError("Tool output must be a list of function calls")
120
except Exception:
121
logger.exception("Error in extracting tool call from response.")
122
# Treat as regular text
@@ -151,7 +146,7 @@ class Olmo3PythonicToolParser(ToolParser):
146
if current_text.endswith("</function_calls>"):
147
current_text = current_text[: -len("</function_calls>")]
148
154
- valid_and_added_text = _make_valid_python(current_text)
149
+ valid_and_added_text = make_valid_python(current_text)
150
if valid_and_added_text is None:
151
return None
152
valid_text, added_text = valid_and_added_text
@@ -166,11 +161,11 @@ class Olmo3PythonicToolParser(ToolParser):
161
if not isinstance(parsed, ast.List) or not all(
162
isinstance(e, ast.Call) for e in parsed.elts
163
):
169
- raise _UnexpectedAstError(
164
+ raise UnexpectedAstError(
165
"Tool output must be a sequence of newline-separated calls"
166
)
167
tool_calls = [
173
- _handle_single_tool(e) # type: ignore
168
+ handle_single_tool(e) # type: ignore
169
for e in parsed.elts
170
]
171
@@ -194,7 +189,7 @@ class Olmo3PythonicToolParser(ToolParser):
189
# Strings get single quotes in the model-produced string.
190
# JSON requires double quotes.
191
withheld_suffix = withheld_suffix.replace("'", '"')
197
- delta = _compute_tool_delta(
192
+ delta = compute_tool_delta(
193
self.streamed_args_for_tool[index], new_call, index, withheld_suffix
194
)
195
@@ -228,141 +223,3 @@ class Olmo3PythonicToolParser(ToolParser):
223
"Skipping chunk as a result of tool streaming extraction error"
224
)
225
return None
231
-
232
-
233
-def _get_parameter_value(val: ast.expr) -> Any:
234
- if isinstance(val, ast.Constant):
235
- return val.value
236
- elif isinstance(val, ast.Dict):
237
- if not all(isinstance(k, ast.Constant) for k in val.keys):
238
- raise _UnexpectedAstError("Dict tool call arguments must have literal keys")
239
- return {
240
- k.value: _get_parameter_value(v) # type: ignore
241
- for k, v in zip(val.keys, val.values)
242
- }
243
- elif isinstance(val, ast.List):
244
- return [_get_parameter_value(v) for v in val.elts]
245
- # The model may return function calls where the values are null/true/false
246
- # because the system prompt has API description in json.
247
- elif isinstance(val, ast.Name) and val.id in ["null", "true", "false"]:
248
- if val.id == "null":
249
- return None
250
- elif val.id == "true":
251
- return True
252
- elif val.id == "false":
253
- return False
254
- else:
255
- raise _UnexpectedAstError("Tool call arguments must be literals")
256
-
257
-
258
-def _handle_single_tool(call: ast.Call) -> ToolCall:
259
- if not isinstance(call.func, ast.Name):
260
- raise _UnexpectedAstError("Invalid tool call name")
261
- function_name = call.func.id
262
- arguments = {}
263
- for keyword in call.keywords:
264
- arguments[keyword.arg] = _get_parameter_value(keyword.value)
265
- return ToolCall(
266
- type="function",
267
- function=FunctionCall(
268
- name=function_name, arguments=json.dumps(arguments, ensure_ascii=False)
269
- ),
270
- )
271
-
272
-
273
-def _make_valid_python(text: str) -> tuple[str, str] | None:
274
- bracket_stack = []
275
- for index, char in enumerate(text):
276
- if char in {"[", "(", "{"}:
277
- bracket_stack.append(char)
278
- elif char == "]":
279
- if not bracket_stack or bracket_stack.pop() != "[":
280
- raise _UnexpectedAstError("Mismatched square brackets")
281
- elif char == ")":
282
- if not bracket_stack or bracket_stack.pop() != "(":
283
- raise _UnexpectedAstError("Mismatched parentheses")
284
- elif char == "}":
285
- if not bracket_stack or bracket_stack.pop() != "{":
286
- raise _UnexpectedAstError("Mismatched curly braces")
287
- elif char in {"'", '"'}:
288
- if bracket_stack and bracket_stack[-1] == char:
289
- if index > 0 and text[index - 1] == "\\":
290
- # Treat an escaped quote as a regular character
291
- pass
292
- else:
293
- bracket_stack.pop()
294
- elif bracket_stack and bracket_stack[-1] in {"'", '"'}:
295
- # Double quote within a single quote string or vice versa.
296
- pass
297
- else:
298
- bracket_stack.append(char)
299
-
300
- text = text.rstrip()
301
- if text.endswith("=") or text.endswith(":"):
302
- # Since we have no type information for this property/parameter value,
303
- # we can't fill in a valid value.
304
- return None
305
- if bracket_stack and bracket_stack[-1] == "{":
306
- trailing_dict_text = text[: text.rfind("{")]
307
- num_keys = trailing_dict_text.count(":")
308
- num_values = trailing_dict_text.count(",")
309
- if num_keys <= num_values:
310
- return None # Incomplete property name within parameter value
311
- if bracket_stack and bracket_stack[-1] == "(":
312
- trailing_params_text = text[: text.rfind("(")]
313
- num_full_param_names = trailing_params_text.count("=")
314
- num_full_param_values = trailing_params_text.count(",")
315
- if num_full_param_names <= num_full_param_values:
316
- return None # Incomplete parameter name
317
- if text.endswith(","):
318
- text = text[:-1]
319
- if (
320
- bracket_stack
321
- and bracket_stack[-1] == "["
322
- and not text.endswith("[")
323
- and not text.endswith(")")
324
- ):
325
- return None # Incomplete function name
326
-
327
- added_text = ""
328
- for char in reversed(bracket_stack):
329
- if char == "[":
330
- added_text += "]"
331
- elif char == "(":
332
- added_text += ")"
333
- elif char == "{":
334
- added_text += "}"
335
- elif char == "'":
336
- added_text += "'"
337
- elif char == '"':
338
- added_text += '"'
339
-
340
- return text + added_text, added_text
341
-
342
-
343
-def _compute_tool_delta(
344
- previously_sent_args: str, new_call: ToolCall, index: int, withheld_suffix: str
345
-) -> DeltaToolCall | None:
346
- new_call_args = new_call.function.arguments
347
- if withheld_suffix:
348
- assert new_call_args.endswith(withheld_suffix)
349
- new_call_args = new_call_args[: -len(withheld_suffix)]
350
- if not previously_sent_args:
351
- return DeltaToolCall(
352
- id=new_call.id,
353
- type="function",
354
- index=index,
355
- function=DeltaFunctionCall(
356
- name=new_call.function.name,
357
- arguments=new_call_args,
358
- ),
359
- )
360
-
361
- arg_diff = new_call_args[len(previously_sent_args) :]
362
- return (
363
- DeltaToolCall(
364
- id=None, index=index, function=DeltaFunctionCall(arguments=arg_diff)
365
- )
366
- if arg_diff
367
- else None
368
- )
vllm/tool_parsers/pythonic_tool_parser.py
+12
-149
@@ -2,9 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
import ast
5
-import json
5
from collections.abc import Sequence
7
-from typing import Any
6
7
import regex as re
8
from transformers import PreTrainedTokenizerBase
@@ -14,25 +12,23 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
12
ChatCompletionRequest,
13
)
14
from vllm.entrypoints.openai.engine.protocol import (
17
- DeltaFunctionCall,
15
DeltaMessage,
19
- DeltaToolCall,
16
ExtractedToolCallInformation,
21
- FunctionCall,
22
- ToolCall,
17
)
18
from vllm.logger import init_logger
19
from vllm.tool_parsers.abstract_tool_parser import (
20
ToolParser,
21
)
22
+from vllm.tool_parsers.utils import (
23
+ UnexpectedAstError,
24
+ compute_tool_delta,
25
+ handle_single_tool,
26
+ make_valid_python,
27
+)
28
29
logger = init_logger(__name__)
30
31
32
-class _UnexpectedAstError(Exception):
33
- pass
34
-
35
-
32
class PythonicToolParser(ToolParser):
33
"""
34
Tool call parser for models that produce tool calls in a pythonic style,
@@ -99,15 +95,13 @@ class PythonicToolParser(ToolParser):
95
return ExtractedToolCallInformation(
96
tools_called=True,
97
tool_calls=[
102
- _handle_single_tool(e) # type: ignore
98
+ handle_single_tool(e) # type: ignore
99
for e in parsed.elts
100
],
101
content=None,
102
)
103
else:
108
- raise _UnexpectedAstError(
109
- "Tool output must be a list of function calls"
110
- )
104
+ raise UnexpectedAstError("Tool output must be a list of function calls")
105
except Exception:
106
logger.exception("Error in extracting tool call from response.")
107
# Treat as regular text
@@ -129,7 +123,7 @@ class PythonicToolParser(ToolParser):
123
return DeltaMessage(content=delta_text)
124
125
try:
132
- valid_and_added_text = _make_valid_python(current_text)
126
+ valid_and_added_text = make_valid_python(current_text)
127
if valid_and_added_text is None:
128
return None
129
valid_text, added_text = valid_and_added_text
@@ -139,11 +133,9 @@ class PythonicToolParser(ToolParser):
133
if not isinstance(parsed, ast.List) or not all(
134
isinstance(e, ast.Call) for e in parsed.elts
135
):
142
- raise _UnexpectedAstError(
143
- "Tool output must be a list of function calls"
144
- )
136
+ raise UnexpectedAstError("Tool output must be a list of function calls")
137
tool_calls = [
146
- _handle_single_tool(e) # type: ignore
138
+ handle_single_tool(e) # type: ignore
139
for e in parsed.elts
140
]
141
@@ -169,7 +161,7 @@ class PythonicToolParser(ToolParser):
161
# Strings get single quotes in the model-produced string.
162
# JSON requires double quotes.
163
withheld_suffix = withheld_suffix.replace("'", '"')
172
- delta = _compute_tool_delta(
164
+ delta = compute_tool_delta(
165
self.streamed_args_for_tool[index], new_call, index, withheld_suffix
166
)
167
@@ -203,132 +195,3 @@ class PythonicToolParser(ToolParser):
195
"Skipping chunk as a result of tool streaming extraction error"
196
)
197
return None
206
-
207
-
208
-def _get_parameter_value(val: ast.expr) -> Any:
209
- if isinstance(val, ast.Constant):
210
- return val.value
211
- elif isinstance(val, ast.Dict):
212
- if not all(isinstance(k, ast.Constant) for k in val.keys):
213
- raise _UnexpectedAstError("Dict tool call arguments must have literal keys")
214
- return {
215
- k.value: _get_parameter_value(v) # type: ignore
216
- for k, v in zip(val.keys, val.values)
217
- }
218
- elif isinstance(val, ast.List):
219
- return [_get_parameter_value(v) for v in val.elts]
220
- else:
221
- raise _UnexpectedAstError("Tool call arguments must be literals")
222
-
223
-
224
-def _handle_single_tool(call: ast.Call) -> ToolCall:
225
- if not isinstance(call.func, ast.Name):
226
- raise _UnexpectedAstError("Invalid tool call name")
227
- function_name = call.func.id
228
- arguments = {}
229
- for keyword in call.keywords:
230
- arguments[keyword.arg] = _get_parameter_value(keyword.value)
231
- return ToolCall(
232
- type="function",
233
- function=FunctionCall(
234
- name=function_name, arguments=json.dumps(arguments, ensure_ascii=False)
235
- ),
236
- )
237
-
238
-
239
-def _make_valid_python(text: str) -> tuple[str, str] | None:
240
- bracket_stack = []
241
- for index, char in enumerate(text):
242
- if char in {"[", "(", "{"}:
243
- bracket_stack.append(char)
244
- elif char == "]":
245
- if not bracket_stack or bracket_stack.pop() != "[":
246
- raise _UnexpectedAstError("Mismatched square brackets")
247
- elif char == ")":
248
- if not bracket_stack or bracket_stack.pop() != "(":
249
- raise _UnexpectedAstError("Mismatched parentheses")
250
- elif char == "}":
251
- if not bracket_stack or bracket_stack.pop() != "{":
252
- raise _UnexpectedAstError("Mismatched curly braces")
253
- elif char in {"'", '"'}:
254
- if bracket_stack and bracket_stack[-1] == char:
255
- if index > 0 and text[index - 1] == "\\":
256
- # Treat an escaped quote as a regular character
257
- pass
258
- else:
259
- bracket_stack.pop()
260
- elif bracket_stack and bracket_stack[-1] in {"'", '"'}:
261
- # Double quote within a single quote string or vice versa.
262
- pass
263
- else:
264
- bracket_stack.append(char)
265
-
266
- text = text.rstrip()
267
- if text.endswith("=") or text.endswith(":"):
268
- # Since we have no type information for this property/parameter value,
269
- # we can't fill in a valid value.
270
- return None
271
- if bracket_stack and bracket_stack[-1] == "{":
272
- trailing_dict_text = text[: text.rfind("{")]
273
- num_keys = trailing_dict_text.count(":")
274
- num_values = trailing_dict_text.count(",")
275
- if num_keys <= num_values:
276
- return None # Incomplete property name within parameter value
277
- if bracket_stack and bracket_stack[-1] == "(":
278
- trailing_params_text = text[: text.rfind("(")]
279
- num_full_param_names = trailing_params_text.count("=")
280
- num_full_param_values = trailing_params_text.count(",")
281
- if num_full_param_names <= num_full_param_values:
282
- return None # Incomplete parameter name
283
- if text.endswith(","):
284
- text = text[:-1]
285
- if (
286
- bracket_stack
287
- and bracket_stack[-1] == "["
288
- and not text.endswith("[")
289
- and not text.endswith(")")
290
- ):
291
- return None # Incomplete function name
292
-
293
- added_text = ""
294
- for char in reversed(bracket_stack):
295
- if char == "[":
296
- added_text += "]"
297
- elif char == "(":
298
- added_text += ")"
299
- elif char == "{":
300
- added_text += "}"
301
- elif char == "'":
302
- added_text += "'"
303
- elif char == '"':
304
- added_text += '"'
305
-
306
- return text + added_text, added_text
307
-
308
-
309
-def _compute_tool_delta(
310
- previously_sent_args: str, new_call: ToolCall, index: int, withheld_suffix: str
311
-) -> DeltaToolCall | None:
312
- new_call_args = new_call.function.arguments
313
- if withheld_suffix:
314
- assert new_call_args.endswith(withheld_suffix)
315
- new_call_args = new_call_args[: -len(withheld_suffix)]
316
- if not previously_sent_args:
317
- return DeltaToolCall(
318
- id=new_call.id,
319
- type="function",
320
- index=index,
321
- function=DeltaFunctionCall(
322
- name=new_call.function.name,
323
- arguments=new_call_args,
324
- ),
325
- )
326
-
327
- arg_diff = new_call_args[len(previously_sent_args) :]
328
- return (
329
- DeltaToolCall(
330
- id=None, index=index, function=DeltaFunctionCall(arguments=arg_diff)
331
- )
332
- if arg_diff
333
- else None
334
- )
vllm/tool_parsers/utils.py
+209
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+import ast
5
import json
6
from json import JSONDecodeError, JSONDecoder
7
from typing import Any
@@ -17,6 +18,15 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
18
ChatCompletionNamedToolChoiceParam,
19
ChatCompletionToolsParam,
20
)
21
+from vllm.entrypoints.openai.engine.protocol import (
22
+ DeltaFunctionCall,
23
+ DeltaToolCall,
24
+ FunctionCall,
25
+ ToolCall,
26
+)
27
+from vllm.logger import init_logger
28
+
29
+logger = init_logger(__name__)
30
31
32
def find_common_prefix(s1: str, s2: str) -> str:
@@ -212,3 +222,202 @@ def get_json_schema_from_tools(
222
return _get_json_schema_from_tools(tools)
223
# tool_choice: "auto"
224
return None
225
+
226
+
227
+# ---------------------------------------------------------------------------
228
+# Shared utilities for pythonic-style tool call parsers
229
+# (PythonicToolParser, Llama4PythonicToolParser, Olmo3PythonicToolParser)
230
+# ---------------------------------------------------------------------------
231
+
232
+
233
+class UnexpectedAstError(Exception):
234
+ """Raised when the AST structure does not match the expected
235
+ pythonic tool call format."""
236
+
237
+ pass
238
+
239
+
240
+_JSON_NAME_LITERALS = {
241
+ "null": None,
242
+ "true": True,
243
+ "false": False,
244
+}
245
+
246
+
247
+def get_parameter_value(val: ast.expr) -> Any:
248
+ """Extract a Python literal value from an AST expression node.
249
+
250
+ Handles constants, dicts, lists, and JSON-style name literals
251
+ (null, true, false) that some models produce instead of Python
252
+ literals (None, True, False).
253
+
254
+ Raises:
255
+ UnexpectedAstError: If the AST node is not a supported literal type.
256
+ """
257
+ if isinstance(val, ast.Constant):
258
+ return val.value
259
+ elif isinstance(val, ast.Dict):
260
+ if not all(isinstance(k, ast.Constant) for k in val.keys):
261
+ logger.warning(
262
+ "Dict argument keys are not all literals: %s",
263
+ ast.dump(val),
264
+ )
265
+ raise UnexpectedAstError("Dict tool call arguments must have literal keys")
266
+ return {
267
+ k.value: get_parameter_value(v) # type: ignore
268
+ for k, v in zip(val.keys, val.values)
269
+ }
270
+ elif isinstance(val, ast.List):
271
+ return [get_parameter_value(v) for v in val.elts]
272
+ elif isinstance(val, ast.Name) and val.id in _JSON_NAME_LITERALS:
273
+ return _JSON_NAME_LITERALS[val.id]
274
+ else:
275
+ logger.warning(
276
+ "Unsupported AST node type in tool call arguments: %s",
277
+ ast.dump(val),
278
+ )
279
+ raise UnexpectedAstError("Tool call arguments must be literals")
280
+
281
+
282
+def handle_single_tool(call: ast.Call) -> ToolCall:
283
+ """Convert a single AST function call node into a ToolCall object.
284
+
285
+ Raises:
286
+ UnexpectedAstError: If the call node does not have a simple
287
+ function name (e.g. it's an attribute access or subscript).
288
+ """
289
+ if not isinstance(call.func, ast.Name):
290
+ logger.warning(
291
+ "Tool call has non-simple function name: %s",
292
+ ast.dump(call.func),
293
+ )
294
+ raise UnexpectedAstError("Invalid tool call name")
295
+ function_name = call.func.id
296
+ arguments = {}
297
+ for keyword in call.keywords:
298
+ arguments[keyword.arg] = get_parameter_value(keyword.value)
299
+ return ToolCall(
300
+ type="function",
301
+ function=FunctionCall(
302
+ name=function_name,
303
+ arguments=json.dumps(arguments, ensure_ascii=False),
304
+ ),
305
+ )
306
+
307
+
308
+def make_valid_python(text: str) -> tuple[str, str] | None:
309
+ """Attempt to close all open brackets/quotes to make partial Python valid.
310
+
311
+ Used during streaming to parse incomplete tool call expressions by
312
+ appending the necessary closing characters.
313
+
314
+ Returns:
315
+ A tuple of (completed_text, added_suffix) if the text can be
316
+ made valid, or None if the text is too incomplete to complete
317
+ meaningfully (e.g. mid-parameter-name or mid-dict-key).
318
+
319
+ Raises:
320
+ UnexpectedAstError: If mismatched brackets or parentheses
321
+ are detected.
322
+ """
323
+ bracket_stack: list[str] = []
324
+ for index, char in enumerate(text):
325
+ if char in {"[", "(", "{"}:
326
+ bracket_stack.append(char)
327
+ elif char == "]":
328
+ if not bracket_stack or bracket_stack.pop() != "[":
329
+ raise UnexpectedAstError("Mismatched square brackets")
330
+ elif char == ")":
331
+ if not bracket_stack or bracket_stack.pop() != "(":
332
+ raise UnexpectedAstError("Mismatched parentheses")
333
+ elif char == "}":
334
+ if not bracket_stack or bracket_stack.pop() != "{":
335
+ raise UnexpectedAstError("Mismatched curly braces")
336
+ elif char in {"'", '"'}:
337
+ if bracket_stack and bracket_stack[-1] == char:
338
+ if index > 0 and text[index - 1] == "\\":
339
+ pass
340
+ else:
341
+ bracket_stack.pop()
342
+ elif bracket_stack and bracket_stack[-1] in {"'", '"'}:
343
+ pass
344
+ else:
345
+ bracket_stack.append(char)
346
+
347
+ text = text.rstrip()
348
+ if text.endswith("=") or text.endswith(":"):
349
+ return None
350
+ if bracket_stack and bracket_stack[-1] == "{":
351
+ trailing_dict_text = text[: text.rfind("{")]
352
+ num_keys = trailing_dict_text.count(":")
353
+ num_values = trailing_dict_text.count(",")
354
+ if num_keys <= num_values:
355
+ return None
356
+ if bracket_stack and bracket_stack[-1] == "(":
357
+ trailing_params_text = text[: text.rfind("(")]
358
+ num_full_param_names = trailing_params_text.count("=")
359
+ num_full_param_values = trailing_params_text.count(",")
360
+ if num_full_param_names <= num_full_param_values:
361
+ return None
362
+ if text.endswith(","):
363
+ text = text[:-1]
364
+ if (
365
+ bracket_stack
366
+ and bracket_stack[-1] == "["
367
+ and not text.endswith("[")
368
+ and not text.endswith(")")
369
+ ):
370
+ return None
371
+
372
+ _CLOSING = {"[": "]", "(": ")", "{": "}", "'": "'", '"': '"'}
373
+ added_text = ""
374
+ for char in reversed(bracket_stack):
375
+ added_text += _CLOSING[char]
376
+
377
+ return text + added_text, added_text
378
+
379
+
380
+def compute_tool_delta(
381
+ previously_sent_args: str,
382
+ new_call: ToolCall,
383
+ index: int,
384
+ withheld_suffix: str,
385
+) -> DeltaToolCall | None:
386
+ """Compute the incremental delta between previously streamed arguments
387
+ and the current tool call state.
388
+
389
+ Returns:
390
+ A DeltaToolCall with only the new argument characters, or None
391
+ if there is no difference from what was previously sent.
392
+ """
393
+ new_call_args = new_call.function.arguments
394
+ if withheld_suffix:
395
+ if not new_call_args.endswith(withheld_suffix):
396
+ msg = (
397
+ f"Tool call arguments '{new_call_args}' do not end with "
398
+ f"expected withheld suffix '{withheld_suffix}'"
399
+ )
400
+ logger.error(msg)
401
+ raise ValueError(msg)
402
+ new_call_args = new_call_args[: -len(withheld_suffix)]
403
+ if not previously_sent_args:
404
+ return DeltaToolCall(
405
+ id=new_call.id,
406
+ type="function",
407
+ index=index,
408
+ function=DeltaFunctionCall(
409
+ name=new_call.function.name,
410
+ arguments=new_call_args,
411
+ ),
412
+ )
413
+
414
+ arg_diff = new_call_args[len(previously_sent_args) :]
415
+ return (
416
+ DeltaToolCall(
417
+ id=None,
418
+ index=index,
419
+ function=DeltaFunctionCall(arguments=arg_diff),
420
+ )
421
+ if arg_diff
422
+ else None
423
+ )