A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 00:00 UTC
Fix edge case Mistral tool parser (#30724)
Signed-off-by: Joachim Studnia <joachim@mistral.ai> Signed-off-by: Joachim Studnia <studniajoachim@gmail.com> Signed-off-by: juliendenize <julien.denize@mistral.ai> Signed-off-by: Julien Denize <40604584+juliendenize@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: juliendenize <julien.denize@mistral.ai> Co-authored-by: Julien Denize <40604584+juliendenize@users.noreply.github.com> Co-authored-by: Patrick von Platen <patrick.v.platen@gmail.com>
Joachim Studnia committed
Dec 23, 2025 at 15:19 UTC
38c361f99dffcf2b83725e81a1b5ed949eef43ab
2 files changed
+115
-56
tests/tool_parsers/test_mistral_tool_parser.py
+34
-2
@@ -281,6 +281,8 @@ def test_extract_tool_calls_pre_v11_tokenizer(
281
"single_tool_add",
282
"single_tool_weather",
283
"multiple_tool_calls",
284
+ "complex",
285
+ "wrong_json",
286
],
287
argnames=["model_output", "expected_tool_calls", "expected_content"],
288
argvalues=[
@@ -326,6 +328,36 @@ def test_extract_tool_calls_pre_v11_tokenizer(
328
],
329
None,
330
),
331
+ (
332
+ # Complex
333
+ """hi{hi[TOOL_CALLS]bash{"command": "print(\\"hello world!\\")\\nre.compile(r\'{}\')""", # noqa: E501
334
+ [
335
+ ToolCall(
336
+ function=FunctionCall(
337
+ name="bash",
338
+ arguments=json.dumps(
339
+ {"command": "print(\"hello world!\")\nre.compile(r'{}')"}
340
+ )[:-2],
341
+ )
342
+ )
343
+ ],
344
+ "hi{hi",
345
+ ),
346
+ (
347
+ # Wrong json
348
+ """hi{hi[TOOL_CALLS]bash{"command": "print(\\"hello world!\\")\\nre.compile(r\'{}\')"}""", # noqa: E501
349
+ [
350
+ ToolCall(
351
+ function=FunctionCall(
352
+ name="bash",
353
+ arguments=json.dumps(
354
+ {"command": "print(\"hello world!\")\nre.compile(r'{}')"}
355
+ ),
356
+ )
357
+ )
358
+ ],
359
+ "hi{hi",
360
+ ),
361
],
362
)
363
def test_extract_tool_calls(
@@ -673,7 +705,7 @@ def test_extract_tool_calls_streaming(
705
),
706
(
707
# Complex
676
- """[TOOL_CALLS]bash{"command": "print(\\"hello world!\\")\\nre.compile(r\'{}\')"}""", # noqa: E501
708
+ """hi{hi[TOOL_CALLS]bash{"command": "print(\\"hello world!\\")\\nre.compile(r\'{}\')"}""", # noqa: E501
709
[
710
ToolCall(
711
function=FunctionCall(
@@ -684,7 +716,7 @@ def test_extract_tool_calls_streaming(
716
)
717
)
718
],
687
- "",
719
+ "hi{hi",
720
),
721
],
722
)
vllm/tool_parsers/mistral_tool_parser.py
+81
-54
@@ -131,78 +131,105 @@ class MistralToolParser(ToolParser):
131
request: ChatCompletionRequest,
132
) -> ExtractedToolCallInformation:
133
"""
134
- Extract the tool calls from a complete model response. Requires
135
- find-and-replacing single quotes with double quotes for JSON parsing,
136
- make sure your tool call arguments don't ever include quotes!
134
+ Extract the tool calls from a complete model response.
135
+
136
+ Content and tool calls formatting depends on the Mistral's tokenizer version
137
+ used to train the model:
138
+
139
+ - < v11: `content[BOT] [{tool_call1},{tool_call2}]`
140
+ - >= v11: `content[BOT]tool_name1{args_call1}[BOT]tool_name2{args_call2}`
141
+
142
+ with [BOT] the tool call token.
143
+
144
+ Note:
145
+ For tokenizer versions >= v11, tool calls with arguments wrongly formatted
146
+ are still returned as tool calls. This is to allow the model to know it
147
+ tried to make a tool call. It reduces chance of another failure and
148
+ prevents that the context is filled with tool calls wrongly placed in
149
+ assistant message contents.
150
"""
151
139
- # case -- if a tool call token is not present, return a text response
152
+ # If the tool call token is not present, return a text response
153
if self.bot_token not in model_output:
154
return ExtractedToolCallInformation(
155
tools_called=False, tool_calls=[], content=model_output
156
)
157
145
- # first remove the BOT token
146
- tool_content = model_output.replace(self.bot_token, "").strip()
158
+ content_and_raw_tool_calls = model_output.split(self.bot_token)
159
+ content = content_and_raw_tool_calls[0]
160
+ raw_tool_calls = content_and_raw_tool_calls[1:]
161
+
162
+ # >= v11: content[BOT]tool_name1{args_call1}[BOT]tool_name2{args_call2}
163
+ if not self._is_pre_v11:
164
+ tool_calls = []
165
+ for raw_tool_call in raw_tool_calls:
166
+ if "{" not in raw_tool_call:
167
+ continue
168
+
169
+ end_name = raw_tool_call.find("{")
170
+ tool_name, args = (
171
+ raw_tool_call[:end_name],
172
+ raw_tool_call[end_name:],
173
+ )
174
148
- try:
175
+ tool_calls.append({"name": tool_name, "arguments": args})
176
+
177
+ # < v11: content[BOT] [{tool_call1},{tool_call2}]
178
+ else:
179
+ if len(raw_tool_calls) != 1:
180
+ raise ValueError(
181
+ "Only one BOT token should have been outputted, "
182
+ f"but got {model_output}."
183
+ )
184
+ stringified_tool_calls = raw_tool_calls[0].strip()
185
try:
150
- if not self._is_pre_v11:
151
- function_call_arr = []
152
- for single_tool_content in model_output.split(self.bot_token):
153
- if "{" not in single_tool_content:
154
- continue
155
-
156
- end_name = single_tool_content.find("{")
157
- fn_name, args = (
158
- single_tool_content[:end_name],
159
- single_tool_content[end_name:],
160
- )
161
-
162
- # fn_name is encoded outside serialized json dump
163
- # only arguments are serialized
164
- function_call_arr.append(
165
- {"name": fn_name, "arguments": json.loads(args)}
166
- )
167
- else:
168
- function_call_arr = json.loads(tool_content)
186
+ tool_calls = json.loads(stringified_tool_calls)
187
except json.JSONDecodeError:
188
# use a regex to find the part corresponding to the tool call.
189
# NOTE: This use case should not happen if the model is trained
190
# correctly. It's an easy possible fix so it's included, but
191
# can be brittle for very complex / highly nested tool calls
174
- raw_tool_call = self.tool_call_regex.findall(tool_content)[0]
175
- function_call_arr = json.loads(raw_tool_call)
176
-
177
- # Tool Call
178
- tool_calls: list[MistralToolCall] = [
179
- MistralToolCall(
180
- type="function",
181
- function=FunctionCall(
182
- name=raw_function_call["name"],
183
- # function call args are JSON but as a string
184
- arguments=json.dumps(
185
- raw_function_call["arguments"], ensure_ascii=False
192
+ try:
193
+ raw_tool_call = self.tool_call_regex.findall(
194
+ stringified_tool_calls
195
+ )[0]
196
+ tool_calls = json.loads(raw_tool_call)
197
+ except (IndexError, json.JSONDecodeError):
198
+ logger.exception("Error in extracting tool call from response: {e}")
199
+ # If raw decoding and decoding post regex rule fails, then just
200
+ # return content.
201
+ return ExtractedToolCallInformation(
202
+ tools_called=False,
203
+ tool_calls=[],
204
+ content=stringified_tool_calls,
205
+ )
206
+ else:
207
+ tool_calls = [
208
+ {
209
+ "name": tool_call["name"],
210
+ "arguments": json.dumps(
211
+ tool_call["arguments"], ensure_ascii=False
212
),
187
- ),
188
- )
189
- for raw_function_call in function_call_arr
190
- ]
213
+ }
214
+ for tool_call in tool_calls
215
+ ]
216
192
- # get any content before the tool call
193
- content = model_output.split(self.bot_token)[0]
194
- return ExtractedToolCallInformation(
195
- tools_called=True,
196
- tool_calls=tool_calls,
197
- content=content if len(content) > 0 else None,
217
+ mistral_tool_calls: list[MistralToolCall] = [
218
+ MistralToolCall(
219
+ type="function",
220
+ function=FunctionCall(
221
+ name=tool_call["name"],
222
+ arguments=tool_call["arguments"],
223
+ ),
224
)
225
+ for tool_call in tool_calls
226
+ ]
227
200
- except Exception:
201
- logger.exception("Error in extracting tool call from response.")
202
- # return information to just treat the tool call as regular JSON
203
- return ExtractedToolCallInformation(
204
- tools_called=False, tool_calls=[], content=tool_content
205
- )
228
+ return ExtractedToolCallInformation(
229
+ tools_called=True,
230
+ tool_calls=mistral_tool_calls,
231
+ content=content if len(content) > 0 else None,
232
+ )
233
234
def extract_tool_calls_streaming(
235
self,