A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 07:15 UTC
[mistral_common] Add v11 tokenizer (#19193)
Signed-off-by: Patrick von Platen <patrick.v.platen@gmail.com>
Patrick von Platen committed
Jun 5, 2025 at 17:27 UTC
f20f9f063b8a61a96f055470c03dd26de8a2c19e
2 files changed
+32
-4
vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py
+30
-4
@@ -44,11 +44,17 @@ class MistralToolCall(ToolCall):
44
return id.isalnum() and len(id) == 9
45
46
47
+def _is_fn_name_regex_support(model_tokenizer: AnyTokenizer) -> bool:
48
+ return isinstance(model_tokenizer, MistralTokenizer) \
49
+ and model_tokenizer.version >= 11
50
+
51
+
52
@ToolParserManager.register_module("mistral")
53
class MistralToolParser(ToolParser):
54
"""
50
- Tool call parser for Mistral 7B Instruct v0.3, intended for use with the
51
- examples/tool_chat_template_mistral.jinja template.
55
+ Tool call parser for Mistral 7B Instruct v0.3, intended for use with
56
+ - [`mistral_common`](https://github.com/mistralai/mistral-common/)
57
+ - the examples/tool_chat_template_mistral.jinja template.
58
59
Used when --enable-auto-tool-choice --tool-call-parser mistral are all set
60
"""
@@ -70,6 +76,12 @@ class MistralToolParser(ToolParser):
76
self.bot_token = "[TOOL_CALLS]"
77
self.bot_token_id = self.vocab.get(self.bot_token)
78
self.tool_call_regex = re.compile(r"\[{.*}\]", re.DOTALL)
79
+ if _is_fn_name_regex_support(self.model_tokenizer):
80
+ self.fn_name_regex = re.compile(r'([a-zA-Z0-9_-]+)(\{.*?\})',
81
+ re.DOTALL)
82
+ else:
83
+ self.fn_name_regex = None
84
+
85
if self.bot_token_id is None:
86
raise RuntimeError(
87
"Mistral Tool Parser could not locate the tool call token in "
@@ -109,11 +121,25 @@ class MistralToolParser(ToolParser):
121
tool_content = model_output.replace(self.bot_token, "").strip()
122
123
try:
112
-
124
# we first try to directly load the json as parsing very nested
125
# jsons is difficult
126
try:
116
- function_call_arr = json.loads(tool_content)
127
+ if self.fn_name_regex:
128
+ matches = self.fn_name_regex.findall(tool_content)
129
+
130
+ function_call_arr = []
131
+ for match in matches:
132
+ fn_name = match[0]
133
+ args = match[1]
134
+
135
+ # fn_name is encoded outside serialized json dump
136
+ # only arguments are serialized
137
+ function_call_arr.append({
138
+ "name": fn_name,
139
+ "arguments": json.loads(args)
140
+ })
141
+ else:
142
+ function_call_arr = json.loads(tool_content)
143
except json.JSONDecodeError:
144
# use a regex to find the part corresponding to the tool call.
145
# NOTE: This use case should not happen if the model is trained
vllm/transformers_utils/tokenizers/mistral.py
+2
@@ -187,6 +187,8 @@ class MistralTokenizer(TokenizerBase):
187
def __init__(self, tokenizer: "PublicMistralTokenizer") -> None:
188
self.mistral = tokenizer
189
self.instruct = tokenizer.instruct_tokenizer
190
+ _mistral_version_str = self.instruct.tokenizer.version.value
191
+ self.version: int = int(_mistral_version_str.split("v")[-1])
192
193
tokenizer_ = tokenizer.instruct_tokenizer.tokenizer
194
from mistral_common.tokens.tokenizers.tekken import (