A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 06:15 UTC
[Mistral Tokenizer] allow more leniency in apply_chat_template (#41658)
Signed-off-by: juliendenize <julien.denize@mistral.ai> Co-authored-by: Roger Wang <hey@rogerw.io>
Julien Denize committed
May 6, 2026 at 04:56 UTC
16e336491e96041946e2cd379a80034fcac439bd
10 files changed
+144
-397
requirements/common.txt
+1
-1
@@ -31,7 +31,7 @@ partial-json-parser # used for parsing partial JSON outputs
31
pyzmq >= 25.0.0
32
msgspec
33
gguf >= 0.17.0
34
-mistral_common[image] >= 1.11.0
34
+mistral_common[image] >= 1.11.2
35
opencv-python-headless >= 4.13.0 # required for video IO
36
pyyaml
37
six>=1.16.0; python_version > '3.11' # transitive dependency of pandas that needs to be the latest version for python 3.12
requirements/test/cuda.in
+1
-1
@@ -31,7 +31,7 @@ torchaudio==2.11.0
31
torchvision==0.26.0
32
transformers_stream_generator # required for qwen-vl test
33
matplotlib # required for qwen-vl test
34
-mistral_common[image,audio] >= 1.11.0 # required for voxtral test
34
+mistral_common[image,audio] >= 1.11.2 # required for voxtral test
35
num2words # required for smolvlm test
36
open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_common.py
37
opencv-python-headless >= 4.13.0 # required for video test
requirements/test/cuda.txt
+1
-1
@@ -412,7 +412,7 @@ mbstrdecoder==1.1.3
412
# typepy
413
mdurl==0.1.2
414
# via markdown-it-py
415
-mistral-common==1.11.0
415
+mistral-common==1.11.2
416
# via
417
# -c requirements/common.txt
418
# -r requirements/test/cuda.in
requirements/test/nightly-torch.txt
+1
-1
@@ -23,7 +23,7 @@ jiwer # required for audio tests
23
timm # required for internvl test
24
transformers_stream_generator # required for qwen-vl test
25
matplotlib # required for qwen-vl test
26
-mistral_common[image,audio] >= 1.11.0 # required for voxtral test
26
+mistral_common[image,audio] >= 1.11.2 # required for voxtral test
27
num2words # required for smolvlm test
28
opencv-python-headless >= 4.13.0 # required for video test
29
datamodel_code_generator # required for minicpm3 test
requirements/test/rocm.in
+1
-1
@@ -30,7 +30,7 @@ tblib # for pickling test exceptions
30
timm>=1.0.17 # required for internvl and gemma3n-mm test
31
transformers_stream_generator # required for qwen-vl test
32
matplotlib # required for qwen-vl test
33
-mistral_common[image,audio]>=1.11.0 # required for voxtral test
33
+mistral_common[image,audio]>=1.11.2 # required for voxtral test
34
num2words # required for smolvlm test
35
open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_common.py
36
opencv-python-headless>=4.13.0 # required for video test
requirements/test/rocm.txt
+1
-1
@@ -509,7 +509,7 @@ mcp==1.27.0
509
# via -r requirements/test/../common.txt
510
mdurl==0.1.2
511
# via markdown-it-py
512
-mistral-common==1.11.0
512
+mistral-common==1.11.2
513
# via
514
# -c requirements/common.txt
515
# -r requirements/test/../common.txt
requirements/test/xpu.txt
+1
-1
@@ -266,7 +266,7 @@ mbstrdecoder==1.1.4
266
# typepy
267
mdurl==0.1.2
268
# via markdown-it-py
269
-mistral-common==1.11.0
269
+mistral-common==1.11.2
270
# via
271
# -c requirements/common.txt
272
# -r requirements/test/xpu.in
tests/tokenizers_/test_mistral.py
+131
-332
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+import copy
5
from typing import Any
6
7
import llguidance
@@ -11,353 +12,34 @@ from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy
12
13
from vllm.tokenizers.mistral import (
14
MistralTokenizer,
14
- _prepare_apply_chat_template_tools_and_messages,
15
+ _validate_apply_chat_template_args,
16
)
17
18
18
-@pytest.mark.parametrize(
19
- "openai_request,expected_mistral_output",
20
- [
21
- (
22
- {
23
- "messages": [
24
- {
25
- "role": "user",
26
- "content": "What is the current local date and time?",
27
- }
28
- ],
29
- "tools": [
30
- {
31
- "type": "function",
32
- "function": {
33
- "description": "Fetch the current local date and time.",
34
- "name": "get_current_time",
35
- },
36
- }
37
- ],
38
- },
39
- (
40
- [
41
- {
42
- "role": "user",
43
- "content": "What is the current local date and time?",
44
- }
45
- ],
46
- [
47
- {
48
- "type": "function",
49
- "function": {
50
- "description": "Fetch the current local date and time.",
51
- "name": "get_current_time",
52
- "parameters": {},
53
- },
54
- }
55
- ],
56
- ),
57
- ),
58
- (
59
- {
60
- "messages": [
61
- {
62
- "role": "user",
63
- "content": "What is the current local date and time?",
64
- }
65
- ],
66
- "tools": [
67
- {
68
- "type": "function",
69
- "function": {
70
- "description": "Fetch the current local date and time.",
71
- "name": "get_current_time",
72
- "parameters": {},
73
- },
74
- }
75
- ],
76
- },
77
- (
78
- [
79
- {
80
- "role": "user",
81
- "content": "What is the current local date and time?",
82
- }
83
- ],
84
- [
85
- {
86
- "type": "function",
87
- "function": {
88
- "description": "Fetch the current local date and time.",
89
- "name": "get_current_time",
90
- "parameters": {},
91
- },
92
- }
93
- ],
94
- ),
95
- ),
96
- (
97
- {
98
- "messages": [
99
- {
100
- "role": "user",
101
- "content": "What is the current local date and time?",
102
- }
103
- ],
104
- "tools": [
105
- {
106
- "type": "function",
107
- "function": {
108
- "description": "Fetch the current local date and time.",
109
- "unsupported_field": False,
110
- "name": "get_current_time",
111
- "parameters": {},
112
- },
113
- },
114
- {
115
- "type": "function",
116
- "function": {
117
- "description": "Fetch the current local date and time.",
118
- "unsupported_field2": False,
119
- "name": "get_current_time",
120
- "parameters": {},
121
- },
122
- },
123
- ],
124
- },
125
- (
126
- [
127
- {
128
- "role": "user",
129
- "content": "What is the current local date and time?",
130
- }
131
- ],
132
- [
133
- {
134
- "type": "function",
135
- "function": {
136
- "description": "Fetch the current local date and time.",
137
- "name": "get_current_time",
138
- "parameters": {},
139
- },
140
- },
141
- {
142
- "type": "function",
143
- "function": {
144
- "description": "Fetch the current local date and time.",
145
- "name": "get_current_time",
146
- "parameters": {},
147
- },
148
- },
149
- ],
150
- ),
151
- ),
152
- (
153
- {
154
- "messages": [
155
- {
156
- "role": "user",
157
- "content": "What is the current local date and time?",
158
- }
159
- ],
160
- "tools": [
161
- {
162
- "type": "function",
163
- "unsupported_field": False,
164
- "function": {
165
- "description": "Fetch the current local date and time.",
166
- "name": "get_current_time",
167
- "parameters": {},
168
- },
169
- },
170
- {
171
- "type": "function",
172
- "unsupported_field2": False,
173
- "function": {
174
- "description": "Fetch the current local date and time 2.",
175
- "name": "get_current_time2",
176
- "parameters": {"a": "1"},
177
- },
178
- },
179
- ],
180
- },
181
- (
182
- [
183
- {
184
- "role": "user",
185
- "content": "What is the current local date and time?",
186
- }
187
- ],
188
- [
189
- {
190
- "type": "function",
191
- "function": {
192
- "description": "Fetch the current local date and time.",
193
- "name": "get_current_time",
194
- "parameters": {},
195
- },
196
- },
197
- {
198
- "type": "function",
199
- "function": {
200
- "description": "Fetch the current local date and time 2.",
201
- "name": "get_current_time2",
202
- "parameters": {"a": "1"},
203
- },
204
- },
205
- ],
206
- ),
207
- ),
208
- ],
209
-)
210
-def test_prepare_apply_chat_template_tools_and_messages(
211
- openai_request, expected_mistral_output
212
-):
213
- actual_request = _prepare_apply_chat_template_tools_and_messages(
214
- openai_request["messages"], openai_request["tools"]
215
- )
216
- assert actual_request == expected_mistral_output
217
-
218
-
219
-# Tool use with list content and reasoning
220
-@pytest.mark.parametrize(
221
- "openai_request,expected_mistral_output",
222
- [
223
- (
224
- {
225
- "messages": [
226
- {
227
- "role": "user",
228
- "content": "What's the weather in Paris?",
229
- },
230
- {
231
- "role": "assistant",
232
- "reasoning": None,
233
- "content": None,
234
- "tool_calls": [
235
- {
236
- "id": "call123",
237
- "type": "function",
238
- "function": {
239
- "name": "get_weather",
240
- "arguments": '{"city": "Paris"}',
241
- },
242
- }
243
- ],
244
- },
245
- {
246
- "role": "tool",
247
- "content": [{"type": "text", "text": "Rainy"}],
248
- "name": "get_weather",
249
- "tool_call_id": "call123",
250
- },
251
- ],
252
- "tools": [
253
- {
254
- "type": "function",
255
- "function": {
256
- "name": "get_weather",
257
- "description": "Gets the current weather in a city.",
258
- "parameters": {
259
- "type": "object",
260
- "properties": {
261
- "city": {
262
- "type": "string",
263
- "description": "The city name",
264
- }
265
- },
266
- "required": ["city"],
267
- },
268
- },
269
- }
270
- ],
271
- },
272
- (
273
- [
274
- {
275
- "role": "user",
276
- "content": "What's the weather in Paris?",
277
- },
278
- {
279
- "role": "assistant",
280
- "content": None,
281
- "tool_calls": [
282
- {
283
- "id": "call123",
284
- "type": "function",
285
- "function": {
286
- "name": "get_weather",
287
- "arguments": '{"city": "Paris"}',
288
- },
289
- }
290
- ],
291
- },
292
- {
293
- "role": "tool",
294
- "content": [{"type": "text", "text": "Rainy"}],
295
- "name": "get_weather",
296
- "tool_call_id": "call123",
297
- },
298
- ],
299
- [
300
- {
301
- "type": "function",
302
- "function": {
303
- "name": "get_weather",
304
- "description": "Gets the current weather in a city.",
305
- "parameters": {
306
- "type": "object",
307
- "properties": {
308
- "city": {
309
- "type": "string",
310
- "description": "The city name",
311
- }
312
- },
313
- "required": ["city"],
314
- },
315
- },
316
- }
317
- ],
318
- ),
319
- )
320
- ],
321
-)
322
-def test_prepare_apply_chat_template_tools_and_messages_list_content(
323
- openai_request, expected_mistral_output
324
-):
325
- actual_request = _prepare_apply_chat_template_tools_and_messages(
326
- openai_request["messages"], openai_request["tools"]
327
- )
328
- assert actual_request == expected_mistral_output
329
-
330
-
331
-def test_prepare_apply_chat_template_generation_prompt_and_continue():
19
+def test_validate_apply_chat_template_args():
20
+ # add_generation_prompt with assistant last message → error
21
messages = [{"role": "assistant", "content": "Hello"}]
333
- tools: list[dict[str, Any]] = []
22
with pytest.raises(ValueError):
335
- _prepare_apply_chat_template_tools_and_messages(
336
- messages, tools, add_generation_prompt=True
337
- )
23
+ _validate_apply_chat_template_args(messages, add_generation_prompt=True)
24
25
+ # add_generation_prompt with user last message → ok
26
messages = [{"role": "user", "content": "Hello"}]
340
- out_messages, _ = _prepare_apply_chat_template_tools_and_messages(
341
- messages, tools, add_generation_prompt=True
342
- )
343
- assert out_messages == [{"role": "user", "content": "Hello"}]
27
+ _validate_apply_chat_template_args(messages, add_generation_prompt=True)
28
29
+ # both add_generation_prompt and continue_final_message → error
30
with pytest.raises(ValueError):
346
- _prepare_apply_chat_template_tools_and_messages(
347
- messages, tools, add_generation_prompt=True, continue_final_message=True
31
+ _validate_apply_chat_template_args(
32
+ messages, add_generation_prompt=True, continue_final_message=True
33
)
34
35
+ # continue_final_message with assistant last message → ok
36
messages = [{"role": "assistant", "content": "Hello"}]
351
- out_messages, _ = _prepare_apply_chat_template_tools_and_messages(
352
- messages, tools, add_generation_prompt=False, continue_final_message=True
353
- )
354
- assert out_messages == [{"role": "assistant", "content": "Hello"}]
37
+ _validate_apply_chat_template_args(messages, continue_final_message=True)
38
39
+ # continue_final_message with user last message → error
40
messages = [{"role": "user", "content": "Hello"}]
41
with pytest.raises(ValueError):
358
- _prepare_apply_chat_template_tools_and_messages(
359
- messages, tools, add_generation_prompt=False, continue_final_message=True
360
- )
42
+ _validate_apply_chat_template_args(messages, continue_final_message=True)
43
44
45
@pytest.fixture(scope="module")
@@ -2435,3 +2117,120 @@ class TestMistralTokenizer:
2117
# Test caching
2118
llg_tokenizer_2 = mistral_tokenizer.llg_tokenizer
2119
assert llg_tokenizer is llg_tokenizer_2
2120
+
2121
+ @pytest.mark.parametrize(
2122
+ "messages,tools,tekken_expected_substrings,spm_expected_substrings",
2123
+ [
2124
+ pytest.param(
2125
+ [{"role": "user", "content": "Hello"}],
2126
+ [{"type": "function", "function": {"name": "do_nothing"}}],
2127
+ ["do_nothing", '"description": ""', '"parameters": {}'],
2128
+ ["do_nothing", '"description":▁""', '"parameters":▁{}'],
2129
+ id="tool_without_description_and_parameters",
2130
+ ),
2131
+ pytest.param(
2132
+ [
2133
+ {"role": "user", "content": "Do nothing"},
2134
+ {
2135
+ "role": "assistant",
2136
+ "content": "",
2137
+ "tool_calls": [
2138
+ {
2139
+ "id": "123456789",
2140
+ "type": "function",
2141
+ "function": {
2142
+ "name": "do_nothing",
2143
+ "arguments": None,
2144
+ },
2145
+ }
2146
+ ],
2147
+ },
2148
+ {
2149
+ "role": "tool",
2150
+ "tool_call_id": "123456789",
2151
+ "content": "done",
2152
+ },
2153
+ ],
2154
+ [{"type": "function", "function": {"name": "do_nothing"}}],
2155
+ ["do_nothing"],
2156
+ ["do_nothing"],
2157
+ id="tool_call_with_none_arguments",
2158
+ ),
2159
+ ],
2160
+ )
2161
+ def test_apply_chat_template_tool_optional_fields(
2162
+ self,
2163
+ mistral_tokenizer: MistralTokenizer,
2164
+ messages: list[dict[str, Any]],
2165
+ tools: list[dict[str, Any]],
2166
+ tekken_expected_substrings: list[str],
2167
+ spm_expected_substrings: list[str],
2168
+ ) -> None:
2169
+ output = mistral_tokenizer.apply_chat_template(
2170
+ messages, tools=tools, add_generation_prompt=True
2171
+ )
2172
+ decoded = mistral_tokenizer.tokenizer.decode(output, SpecialTokenPolicy.KEEP)
2173
+
2174
+ expected = (
2175
+ tekken_expected_substrings
2176
+ if mistral_tokenizer.is_tekken
2177
+ else spm_expected_substrings
2178
+ )
2179
+ for substring in expected:
2180
+ assert substring in decoded
2181
+
2182
+ def test_apply_chat_template_tools_not_mutated(
2183
+ self, mistral_tokenizer: MistralTokenizer
2184
+ ) -> None:
2185
+ messages: list[dict[str, Any]] = [
2186
+ {"role": "user", "content": "Hello"},
2187
+ ]
2188
+ tools: list[dict[str, Any]] = [
2189
+ {
2190
+ "type": "function",
2191
+ "function": {
2192
+ "name": "get_weather",
2193
+ "description": "Gets weather.",
2194
+ "parameters": {
2195
+ "type": "object",
2196
+ "properties": {
2197
+ "city": {"type": "string"},
2198
+ },
2199
+ },
2200
+ },
2201
+ },
2202
+ ]
2203
+ original_tools = copy.deepcopy(tools)
2204
+
2205
+ mistral_tokenizer.apply_chat_template(
2206
+ messages, tools=tools, add_generation_prompt=True
2207
+ )
2208
+
2209
+ assert tools == original_tools
2210
+
2211
+ @pytest.mark.parametrize(
2212
+ "reasoning_key",
2213
+ ["reasoning", "reasoning_content"],
2214
+ )
2215
+ def test_apply_chat_template_reasoning_assistant(
2216
+ self, mistral_tokenizer: MistralTokenizer, reasoning_key: str
2217
+ ) -> None:
2218
+ if not mistral_tokenizer.is_tekken:
2219
+ pytest.skip("Reasoning tokens only supported on tekken tokenizers")
2220
+
2221
+ messages: list[dict[str, Any]] = [
2222
+ {"role": "user", "content": "What is 2+2?"},
2223
+ {
2224
+ "role": "assistant",
2225
+ "content": "4",
2226
+ reasoning_key: "2+2 equals 4",
2227
+ },
2228
+ {"role": "user", "content": "Are you sure?"},
2229
+ ]
2230
+
2231
+ output = mistral_tokenizer.apply_chat_template(
2232
+ messages, add_generation_prompt=True
2233
+ )
2234
+ decoded = mistral_tokenizer.tokenizer.decode(output, SpecialTokenPolicy.KEEP)
2235
+
2236
+ assert "[THINK]2+2 equals 4[/THINK]" in decoded
vllm/tokenizers/mistral.py
+4
-51
@@ -13,7 +13,6 @@ from mistral_common.protocol.instruct.request import (
13
from mistral_common.protocol.instruct.request import (
14
ReasoningEffort,
15
)
16
-from mistral_common.protocol.instruct.tool_calls import Function, Tool
16
from mistral_common.protocol.instruct.validator import ValidationMode
17
from mistral_common.tokens.tokenizers.base import (
18
SpecialTokenPolicy,
@@ -68,36 +67,6 @@ def _pop_unallowed_keys_and_warn(
67
)
68
69
71
-# TODO(juliendenize): remove this once OpenAI API is better supported by
72
-# `mistral-common`.
73
-def adapt_inplace_to_mistral_tool(
74
- tool: dict[str, Any],
75
-) -> dict[str, Any]:
76
- tools_fields = set(Tool.model_fields.keys())
77
- function_fields = set(Function.model_fields.keys())
78
-
79
- # The Mistral client, in comparison to the OpenAI client, requires the
80
- # "parameters" dict and the "description" string to be present
81
- # even if they are empty.
82
- if function := tool.get("function"):
83
- if function.get("parameters") is None:
84
- function["parameters"] = {}
85
- if function.get("description") is None:
86
- function["description"] = ""
87
-
88
- _pop_unallowed_keys_and_warn(
89
- dictionary=function,
90
- allowed_keys=function_fields,
91
- err_dict_name="function",
92
- )
93
-
94
- _pop_unallowed_keys_and_warn(
95
- dictionary=tool, allowed_keys=tools_fields, err_dict_name="tools"
96
- )
97
-
98
- return tool
99
-
100
-
70
def maybe_serialize_tool_calls(request: "MistralChatCompletionRequest"):
71
# SEE: https://github.com/vllm-project/vllm/pull/9951
72
# Credits go to: @gcalmettes
@@ -167,12 +136,11 @@ def truncate_tool_call_ids(request: "MistralChatCompletionRequest"):
136
request.messages[i]["tool_call_id"] = tool_call_id
137
138
170
-def _prepare_apply_chat_template_tools_and_messages(
139
+def _validate_apply_chat_template_args(
140
messages: list["ChatCompletionMessageParam"],
172
- tools: list[dict[str, Any]] | None = None,
141
continue_final_message: bool = False,
142
add_generation_prompt: bool = False,
175
-) -> tuple[list["ChatCompletionMessageParam"], list[dict[str, Any]] | None]:
143
+) -> None:
144
if add_generation_prompt and continue_final_message:
145
raise ValueError(
146
"Cannot set both `add_generation_prompt` and "
@@ -196,21 +164,6 @@ def _prepare_apply_chat_template_tools_and_messages(
164
"the last message is not from the assistant."
165
)
166
199
- # mistral-common requires AssistantMessage content to be string [1].
200
- #
201
- # [1]: https://github.com/mistralai/mistral-common/blob/f4a06998b75ed78bbf5aaf569590b772ea26c9f6/src/mistral_common/protocol/instruct/messages.py#L80
202
- for message in messages:
203
- # Remove reasoning as unsupported by Mistral
204
- _ = message.pop("reasoning", None) # type: ignore
205
-
206
- tools = (
207
- [adapt_inplace_to_mistral_tool(tool=tool) for tool in tools]
208
- if tools is not None
209
- else None
210
- )
211
-
212
- return messages, tools
213
-
167
168
def validate_request_params(request: "ChatCompletionRequest"):
169
if request.chat_template is not None or request.chat_template_kwargs is not None:
@@ -449,8 +402,8 @@ class MistralTokenizer(TokenizerLike):
402
if self.version >= 15:
403
version_kwargs["reasoning_effort"] = kwargs.get("reasoning_effort")
404
452
- messages, tools = _prepare_apply_chat_template_tools_and_messages(
453
- messages, tools, continue_final_message, add_generation_prompt
405
+ _validate_apply_chat_template_args(
406
+ messages, continue_final_message, add_generation_prompt
407
)
408
409
return self.transformers_tokenizer.apply_chat_template(
vllm/tool_parsers/mistral_tool_parser.py
+2
-7
@@ -43,7 +43,7 @@ from vllm.logger import init_logger
43
from vllm.reasoning.mistral_reasoning_parser import MistralReasoningParser
44
from vllm.sampling_params import StructuredOutputsParams
45
from vllm.tokenizers import TokenizerLike
46
-from vllm.tokenizers.mistral import MistralTokenizer, adapt_inplace_to_mistral_tool
46
+from vllm.tokenizers.mistral import MistralTokenizer
47
from vllm.tool_parsers.abstract_tool_parser import (
48
Tool,
49
ToolParser,
@@ -241,12 +241,7 @@ class MistralToolParser(ToolParser):
241
)
242
243
mistral_tools = (
244
- [
245
- MistralTool.model_validate(
246
- adapt_inplace_to_mistral_tool(tool.model_dump())
247
- )
248
- for tool in request.tools
249
- ]
244
+ [MistralTool.from_openai(tool.model_dump()) for tool in request.tools]
245
if request.tools is not None
246
else None
247
)