A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 09:45 UTC
[Bugfix] Consistent ascii handling in tool parsers (#17704)
Signed-off-by: Sebastian Schönnenbeck <sebastian.schoennenbeck@comma-soft.com>
Sebastian Schoennenbeck committed
May 21, 2025 at 22:41 UTC
1f079540db5f1080a2f61a730da50d3009934c5a
7 files changed
+53
-29
vllm/entrypoints/openai/tool_parsers/granite_20b_fc_tool_parser.py
+8
-4
@@ -80,7 +80,8 @@ class Granite20bFCToolParser(ToolParser):
80
function=FunctionCall(
81
name=function_call["name"],
82
# function call args are JSON but as a string
83
- arguments=json.dumps(function_call["arguments"]),
83
+ arguments=json.dumps(function_call["arguments"],
84
+ ensure_ascii=False),
85
),
86
) for function_call in raw_function_calls
87
]
@@ -166,7 +167,8 @@ class Granite20bFCToolParser(ToolParser):
167
if self.current_tool_id >= 0:
168
cur_arguments = current_tool_call.get("arguments")
169
if cur_arguments:
169
- cur_args_json = json.dumps(cur_arguments)
170
+ cur_args_json = json.dumps(cur_arguments,
171
+ ensure_ascii=False)
172
sent = len(
173
self.streamed_args_for_tool[self.current_tool_id])
174
argument_diff = cur_args_json[sent:]
@@ -218,7 +220,8 @@ class Granite20bFCToolParser(ToolParser):
220
if cur_arguments:
221
sent = len(
222
self.streamed_args_for_tool[self.current_tool_id])
221
- cur_args_json = json.dumps(cur_arguments)
223
+ cur_args_json = json.dumps(cur_arguments,
224
+ ensure_ascii=False)
225
prev_arguments = self.prev_tool_call_arr[
226
self.current_tool_id].get("arguments")
227
@@ -226,7 +229,8 @@ class Granite20bFCToolParser(ToolParser):
229
if is_complete[self.current_tool_id]:
230
argument_diff = cur_args_json[sent:]
231
elif prev_arguments:
229
- prev_args_json = json.dumps(prev_arguments)
232
+ prev_args_json = json.dumps(prev_arguments,
233
+ ensure_ascii=False)
234
if cur_args_json != prev_args_json:
235
236
prefix = find_common_prefix(
vllm/entrypoints/openai/tool_parsers/granite_tool_parser.py
+8
-4
@@ -67,7 +67,8 @@ class GraniteToolParser(ToolParser):
67
function=FunctionCall(
68
name=function_call["name"],
69
# function call args are JSON but as a string
70
- arguments=json.dumps(function_call["arguments"]),
70
+ arguments=json.dumps(function_call["arguments"],
71
+ ensure_ascii=False),
72
),
73
) for function_call in raw_function_calls
74
]
@@ -151,7 +152,8 @@ class GraniteToolParser(ToolParser):
152
if self.current_tool_id >= 0:
153
cur_arguments = current_tool_call.get("arguments")
154
if cur_arguments:
154
- cur_args_json = json.dumps(cur_arguments)
155
+ cur_args_json = json.dumps(cur_arguments,
156
+ ensure_ascii=False)
157
sent = len(
158
self.streamed_args_for_tool[self.current_tool_id])
159
argument_diff = cur_args_json[sent:]
@@ -197,7 +199,8 @@ class GraniteToolParser(ToolParser):
199
if cur_arguments:
200
sent = len(
201
self.streamed_args_for_tool[self.current_tool_id])
200
- cur_args_json = json.dumps(cur_arguments)
202
+ cur_args_json = json.dumps(cur_arguments,
203
+ ensure_ascii=False)
204
prev_arguments = self.prev_tool_call_arr[
205
self.current_tool_id].get("arguments")
206
@@ -205,7 +208,8 @@ class GraniteToolParser(ToolParser):
208
if is_complete[self.current_tool_id]:
209
argument_diff = cur_args_json[sent:]
210
elif prev_arguments:
208
- prev_args_json = json.dumps(prev_arguments)
211
+ prev_args_json = json.dumps(prev_arguments,
212
+ ensure_ascii=False)
213
if cur_args_json != prev_args_json:
214
prefix = find_common_prefix(
215
prev_args_json, cur_args_json)
vllm/entrypoints/openai/tool_parsers/internlm2_tool_parser.py
+8
-4
@@ -133,7 +133,8 @@ class Internlm2ToolParser(ToolParser):
133
delta = None
134
# first time to get parameters
135
elif cur_arguments and not prev_arguments:
136
- cur_arguments_json = json.dumps(cur_arguments)
136
+ cur_arguments_json = json.dumps(cur_arguments,
137
+ ensure_ascii=False)
138
139
arguments_delta = cur_arguments_json[:cur_arguments_json.
140
index(delta_text) +
@@ -148,8 +149,10 @@ class Internlm2ToolParser(ToolParser):
149
self.current_tool_id] += arguments_delta
150
# both prev and cur parameters, send the increase parameters
151
elif cur_arguments and prev_arguments:
151
- cur_args_json = json.dumps(cur_arguments)
152
- prev_args_json = json.dumps(prev_arguments)
152
+ cur_args_json = json.dumps(cur_arguments,
153
+ ensure_ascii=False)
154
+ prev_args_json = json.dumps(prev_arguments,
155
+ ensure_ascii=False)
156
157
argument_diff = extract_intermediate_diff(
158
cur_args_json, prev_args_json)
@@ -190,7 +193,8 @@ class Internlm2ToolParser(ToolParser):
193
action_dict = json.loads(action)
194
name, parameters = action_dict['name'], json.dumps(
195
action_dict.get('parameters', action_dict.get('arguments',
193
- {})))
196
+ {})),
197
+ ensure_ascii=False)
198
199
if not tools or name not in [t.function.name for t in tools]:
200
ExtractedToolCallInformation(tools_called=False,
vllm/entrypoints/openai/tool_parsers/jamba_tool_parser.py
+10
-6
@@ -96,8 +96,9 @@ class JambaToolParser(ToolParser):
96
function=FunctionCall(
97
name=function_call["name"],
98
# function call args are JSON but as a string
99
- arguments=json.dumps(function_call["arguments"])))
100
- for function_call in raw_function_calls
99
+ arguments=json.dumps(function_call["arguments"],
100
+ ensure_ascii=False),
101
+ )) for function_call in raw_function_calls
102
]
103
104
content = model_output[:model_output.
@@ -187,7 +188,7 @@ class JambaToolParser(ToolParser):
188
diff: Union[str, None] = current_tool_call.get("arguments")
189
190
if diff:
190
- diff = json.dumps(diff).replace(
191
+ diff = json.dumps(diff, ensure_ascii=False).replace(
192
self.streamed_args_for_tool[self.current_tool_id],
193
"")
194
delta = DeltaMessage(tool_calls=[
@@ -248,7 +249,8 @@ class JambaToolParser(ToolParser):
249
"mid-arguments")
250
delta = None
251
elif cur_arguments and not prev_arguments:
251
- cur_arguments_json = json.dumps(cur_arguments)
252
+ cur_arguments_json = json.dumps(cur_arguments,
253
+ ensure_ascii=False)
254
logger.debug("finding %s in %s", new_text,
255
cur_arguments_json)
256
@@ -267,8 +269,10 @@ class JambaToolParser(ToolParser):
269
self.current_tool_id] += arguments_delta
270
271
elif cur_arguments and prev_arguments:
270
- cur_args_json = json.dumps(cur_arguments)
271
- prev_args_json = json.dumps(prev_arguments)
272
+ cur_args_json = json.dumps(cur_arguments,
273
+ ensure_ascii=False)
274
+ prev_args_json = json.dumps(prev_arguments,
275
+ ensure_ascii=False)
276
logger.debug("Searching for diff between \n%s\n%s",
277
cur_args_json, prev_args_json)
278
vllm/entrypoints/openai/tool_parsers/llama_tool_parser.py
+8
-4
@@ -88,7 +88,8 @@ class Llama3JsonToolParser(ToolParser):
88
# function call args are JSON but as a string
89
arguments=json.dumps(raw_function_call["arguments"] \
90
if "arguments" in raw_function_call \
91
- else raw_function_call["parameters"])))
91
+ else raw_function_call["parameters"],
92
+ ensure_ascii=False)))
93
for raw_function_call in function_call_arr
94
]
95
@@ -174,7 +175,8 @@ class Llama3JsonToolParser(ToolParser):
175
if self.current_tool_id >= 0:
176
cur_arguments = current_tool_call.get("arguments")
177
if cur_arguments:
177
- cur_args_json = json.dumps(cur_arguments)
178
+ cur_args_json = json.dumps(cur_arguments,
179
+ ensure_ascii=False)
180
sent = len(
181
self.streamed_args_for_tool[self.current_tool_id])
182
argument_diff = cur_args_json[sent:]
@@ -226,7 +228,8 @@ class Llama3JsonToolParser(ToolParser):
228
if cur_arguments:
229
sent = len(
230
self.streamed_args_for_tool[self.current_tool_id])
229
- cur_args_json = json.dumps(cur_arguments)
231
+ cur_args_json = json.dumps(cur_arguments,
232
+ ensure_ascii=False)
233
prev_arguments = self.prev_tool_call_arr[
234
self.current_tool_id].get("arguments")
235
@@ -234,7 +237,8 @@ class Llama3JsonToolParser(ToolParser):
237
if is_complete[self.current_tool_id]:
238
argument_diff = cur_args_json[sent:]
239
elif prev_arguments:
237
- prev_args_json = json.dumps(prev_arguments)
240
+ prev_args_json = json.dumps(prev_arguments,
241
+ ensure_ascii=False)
242
if cur_args_json != prev_args_json:
243
244
prefix = find_common_prefix(
vllm/entrypoints/openai/tool_parsers/phi4mini_tool_parser.py
+5
-4
@@ -79,10 +79,11 @@ class Phi4MiniJsonToolParser(ToolParser):
79
name=raw_function_call["name"],
80
# function call args are JSON but as a string
81
arguments=json.dumps(
82
- raw_function_call["arguments"] if "arguments" in
83
- raw_function_call else
84
- raw_function_call["parameters"])))
85
- for raw_function_call in function_call_arr
82
+ raw_function_call["arguments"]
83
+ if "arguments" in raw_function_call else
84
+ raw_function_call["parameters"],
85
+ ensure_ascii=False),
86
+ )) for raw_function_call in function_call_arr
87
]
88
89
# get any content before the tool call
vllm/entrypoints/openai/tool_parsers/pythonic_tool_parser.py
+6
-3
@@ -200,9 +200,12 @@ def _handle_single_tool(call: ast.Call) -> ToolCall:
200
arguments = {}
201
for keyword in call.keywords:
202
arguments[keyword.arg] = _get_parameter_value(keyword.value)
203
- return ToolCall(type="function",
204
- function=FunctionCall(name=function_name,
205
- arguments=json.dumps(arguments)))
203
+ return ToolCall(
204
+ type="function",
205
+ function=FunctionCall(name=function_name,
206
+ arguments=json.dumps(arguments,
207
+ ensure_ascii=False)),
208
+ )
209
210
211
def _make_valid_python(text: str) -> Union[tuple[str, str], None]: