@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c6187f55f7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

Refactor MistralTokenizer (#26358)

Signed-off-by: Julien Denize <julien.denize@mistral.ai>

Julien Denize committed Oct 10, 2025 at 00:48 UTC c6187f55f7c4844ed9ff5630d41114cbe6fccb6b
18 files changed +2349 -461
docs/features/tool_calling.md
+9 -2
@@ -145,7 +145,7 @@ Supported models:
145 Known issues:
146
147 1. Mistral 7B struggles to generate parallel tool calls correctly.
148 -2. Mistral's `tokenizer_config.json` chat template requires tool call IDs that are exactly 9 digits, which is
148 +2. **For Transformers tokenization backend only**: Mistral's `tokenizer_config.json` chat template requires tool call IDs that are exactly 9 digits, which is
149 much shorter than what vLLM generates. Since an exception is thrown when this condition
150 is not met, the following additional chat templates are provided:
151
@@ -154,7 +154,14 @@ Known issues:
154 * <gh-file:examples/tool_chat_template_mistral_parallel.jinja> - this is a "better" version that adds a tool-use system prompt
155 when tools are provided, that results in much better reliability when working with parallel tool calling.
156
157 -Recommended flags: `--tool-call-parser mistral --chat-template examples/tool_chat_template_mistral_parallel.jinja`
157 +Recommended flags:
158 +
159 +1. To use [mistral-common](https://github.com/mistralai/mistral-common) the official Mistral tokenization backend:
160 +
161 + `--tokenizer_mode mistral --config_format mistral --load_format mistral --tool-call-parser mistral`
162 +
163 +2. To use the default Transformers tokenization backend:
164 + `--tool-call-parser mistral --chat-template examples/tool_chat_template_mistral_parallel.jinja`
165
166 ### Llama Models (`llama3_json`)
167
examples/offline_inference/audio_language.py
+3 -1
@@ -45,10 +45,12 @@ class ModelRequestData(NamedTuple):
45 # Voxtral
46 def run_voxtral(question: str, audio_count: int) -> ModelRequestData:
47 from mistral_common.audio import Audio
48 - from mistral_common.protocol.instruct.messages import (
48 + from mistral_common.protocol.instruct.chunk import (
49 AudioChunk,
50 RawAudio,
51 TextChunk,
52 + )
53 + from mistral_common.protocol.instruct.messages import (
54 UserMessage,
55 )
56 from mistral_common.protocol.instruct.request import ChatCompletionRequest
requirements/common.txt
+1 -1
@@ -32,7 +32,7 @@ pyzmq >= 25.0.0
32 msgspec
33 gguf >= 0.13.0
34 importlib_metadata; python_version < '3.10'
35 -mistral_common[image,audio] >= 1.8.2
35 +mistral_common[image,audio] >= 1.8.5
36 opencv-python-headless >= 4.11.0 # required for video IO
37 pyyaml
38 six>=1.16.0; python_version > '3.11' # transitive dependency of pandas that needs to be the latest version for python 3.12
requirements/nightly_torch_test.txt
+1 -1
@@ -23,7 +23,7 @@ jiwer # required for audio tests
23 timm # required for internvl test
24 transformers_stream_generator # required for qwen-vl test
25 matplotlib # required for qwen-vl test
26 -mistral_common[image,audio] >= 1.8.2 # required for voxtral test
26 +mistral_common[image,audio] >= 1.8.5 # required for voxtral test
27 num2words # required for smolvlm test
28 opencv-python-headless >= 4.11.0 # required for video test
29 datamodel_code_generator # required for minicpm3 test
requirements/test.in
+1 -1
@@ -29,7 +29,7 @@ torchaudio==2.8.0
29 torchvision==0.23.0
30 transformers_stream_generator # required for qwen-vl test
31 matplotlib # required for qwen-vl test
32 -mistral_common[image,audio] >= 1.8.2 # required for voxtral test
32 +mistral_common[image,audio] >= 1.8.5 # required for voxtral test
33 num2words # required for smolvlm test
34 open_clip_torch==2.32.0 # Required for nemotron_vl test
35 opencv-python-headless >= 4.11.0 # required for video test
requirements/test.txt
+1 -3
@@ -474,7 +474,7 @@ mbstrdecoder==1.1.3
474 # typepy
475 mdurl==0.1.2
476 # via markdown-it-py
477 -mistral-common==1.8.2
477 +mistral-common==1.8.5
478 # via -r requirements/test.in
479 mlflow==2.22.0
480 # via terratorch
@@ -1012,8 +1012,6 @@ sentence-transformers==3.2.1
1012 # via
1013 # -r requirements/test.in
1014 # mteb
1015 -sentencepiece==0.2.0
1016 - # via mistral-common
1015 setuptools==77.0.3
1016 # via
1017 # lightning-utilities
tests/entrypoints/test_chat_utils.py
+2 -28
@@ -6,8 +6,7 @@ from collections.abc import Mapping
6 from typing import Literal, Optional
7
8 import pytest
9 -from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy, SpecialTokens
10 -from mistral_common.tokens.tokenizers.tekken import SpecialTokenInfo, Tekkenizer
9 +from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy
10
11 from vllm.assets.audio import AudioAsset
12 from vllm.assets.image import ImageAsset
@@ -2119,34 +2118,9 @@ def test_apply_mistral_chat_template_thinking_chunk():
2118 },
2119 {"role": "user", "content": "Thanks, what is 3+3?"},
2120 ]
2122 -
2123 - # TODO(Julien): upon model release change to a tokenizer already configured.
2124 - # =================================================================
2121 mistral_tokenizer = MistralTokenizer.from_pretrained(
2126 - "mistralai/Devstral-Small-2507"
2127 - )
2128 - assert isinstance(mistral_tokenizer.tokenizer, Tekkenizer)
2129 - # Add think special tokens to the tokenizer
2130 - mistral_tokenizer.tokenizer._all_special_tokens[35] = SpecialTokenInfo(
2131 - rank=35, is_control=True, token_str=SpecialTokens.begin_think.value
2122 + "mistralai/Magistral-Small-2509"
2123 )
2133 - mistral_tokenizer.tokenizer._all_special_tokens[36] = SpecialTokenInfo(
2134 - rank=36, is_control=True, token_str=SpecialTokens.end_think.value
2135 - )
2136 - mistral_tokenizer.tokenizer._special_tokens_reverse_vocab = {
2137 - k: v
2138 - for k, v in mistral_tokenizer.tokenizer._special_tokens_reverse_vocab.items()
2139 - if v not in {35, 36}
2140 - }
2141 - mistral_tokenizer.tokenizer._special_tokens_reverse_vocab[
2142 - SpecialTokens.begin_think.value
2143 - ] = 35
2144 - mistral_tokenizer.tokenizer._special_tokens_reverse_vocab[
2145 - SpecialTokens.end_think.value
2146 - ] = 36
2147 - mistral_tokenizer.instruct.BEGIN_THINK = 35
2148 - mistral_tokenizer.instruct.END_THINK = 36
2149 - # =================================================================
2124
2125 tokens_ids = apply_mistral_chat_template(
2126 mistral_tokenizer, messages, chat_template=None, tools=None
tests/models/multimodal/generation/test_pixtral.py
+1 -1
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Any, Optional
6
7 import pytest
8 from mistral_common.multimodal import download_image
9 -from mistral_common.protocol.instruct.messages import ImageURLChunk
9 +from mistral_common.protocol.instruct.chunk import ImageURLChunk
10 from mistral_common.protocol.instruct.request import ChatCompletionRequest
11 from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
12 from mistral_common.tokens.tokenizers.multimodal import image_from_chunk
tests/models/multimodal/generation/test_voxtral.py
+2 -6
@@ -6,12 +6,8 @@ import json
6 import pytest
7 import pytest_asyncio
8 from mistral_common.audio import Audio
9 -from mistral_common.protocol.instruct.messages import (
10 - AudioChunk,
11 - RawAudio,
12 - TextChunk,
13 - UserMessage,
14 -)
9 +from mistral_common.protocol.instruct.chunk import AudioChunk, RawAudio, TextChunk
10 +from mistral_common.protocol.instruct.messages import UserMessage
11
12 from vllm.transformers_utils.tokenizer import MistralTokenizer
13
tests/models/multimodal/processing/test_common.py
+2 -1
@@ -6,7 +6,8 @@ from typing import Optional, Union
6
7 import numpy as np
8 import pytest
9 -from mistral_common.protocol.instruct.messages import ImageChunk, TextChunk, UserMessage
9 +from mistral_common.protocol.instruct.chunk import ImageChunk, TextChunk
10 +from mistral_common.protocol.instruct.messages import UserMessage
11 from mistral_common.protocol.instruct.request import ChatCompletionRequest
12 from PIL import Image
13
tests/models/multimodal/processing/test_tensor_schema.py
+2 -1
@@ -9,7 +9,8 @@ from typing import Any, Union
9 import numpy as np
10 import pytest
11 import torch.nn as nn
12 -from mistral_common.protocol.instruct.messages import ImageChunk, TextChunk, UserMessage
12 +from mistral_common.protocol.instruct.chunk import ImageChunk, TextChunk
13 +from mistral_common.protocol.instruct.messages import UserMessage
14 from mistral_common.protocol.instruct.request import ChatCompletionRequest
15 from PIL import Image
16
tests/reasoning/test_mistral_reasoning_parser.py
+1 -27
@@ -2,8 +2,6 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 import pytest
5 -from mistral_common.tokens.tokenizers.base import SpecialTokens
6 -from mistral_common.tokens.tokenizers.tekken import SpecialTokenInfo, Tekkenizer
5
6 from tests.reasoning.utils import run_reasoning_extraction_mistral
7 from vllm.reasoning import ReasoningParser, ReasoningParserManager
@@ -14,33 +12,9 @@ parser_name = "mistral"
12
13 @pytest.fixture(scope="module")
14 def mistral_tokenizer():
17 - # TODO(Julien): upon model release change to a tokenizer already configured.
18 - # =================================================================
15 mistral_tokenizer = MistralTokenizer.from_pretrained(
20 - "mistralai/Devstral-Small-2507"
16 + "mistralai/Magistral-Small-2509"
17 )
22 - assert isinstance(mistral_tokenizer.tokenizer, Tekkenizer)
23 - # Add think special tokens to the tokenizer
24 - mistral_tokenizer.tokenizer._all_special_tokens[35] = SpecialTokenInfo(
25 - rank=35, is_control=True, token_str=SpecialTokens.begin_think.value
26 - )
27 - mistral_tokenizer.tokenizer._all_special_tokens[36] = SpecialTokenInfo(
28 - rank=36, is_control=True, token_str=SpecialTokens.end_think.value
29 - )
30 - mistral_tokenizer.tokenizer._special_tokens_reverse_vocab = {
31 - k: v
32 - for k, v in mistral_tokenizer.tokenizer._special_tokens_reverse_vocab.items()
33 - if v not in {35, 36}
34 - }
35 - mistral_tokenizer.tokenizer._special_tokens_reverse_vocab[
36 - SpecialTokens.begin_think.value
37 - ] = 35
38 - mistral_tokenizer.tokenizer._special_tokens_reverse_vocab[
39 - SpecialTokens.end_think.value
40 - ] = 36
41 - mistral_tokenizer.instruct.BEGIN_THINK = 35
42 - mistral_tokenizer.instruct.END_THINK = 36
43 - # =================================================================
18 return mistral_tokenizer
19
20
tests/tokenization/test_mistral_tokenizer.py
+2077 -75
@@ -1,27 +1,20 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 +from typing import Any
5 +
6 import pytest
5 -from mistral_common.protocol.instruct.messages import (
6 - AssistantMessage,
7 - ToolMessage,
8 - UserMessage,
9 -)
10 -from mistral_common.protocol.instruct.request import ChatCompletionRequest
11 -from mistral_common.protocol.instruct.tool_calls import (
12 - Function,
13 - FunctionCall,
14 - Tool,
15 - ToolCall,
16 -)
7 +from mistral_common.exceptions import InvalidMessageStructureException
8 +from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy
9
10 from vllm.transformers_utils.tokenizers.mistral import (
19 - make_mistral_chat_completion_request,
11 + MistralTokenizer,
12 + _prepare_apply_chat_template_tools_and_messages,
13 )
14
15
16 @pytest.mark.parametrize(
24 - "openai_request,expected_mistral_request",
17 + "openai_request,expected_mistral_output",
18 [
19 (
20 {
@@ -41,19 +34,22 @@ from vllm.transformers_utils.tokenizers.mistral import (
34 }
35 ],
36 },
44 - ChatCompletionRequest(
45 - messages=[
46 - UserMessage(content="What is the current local date and time?")
37 + (
38 + [
39 + {
40 + "role": "user",
41 + "content": "What is the current local date and time?",
42 + }
43 ],
48 - tools=[
49 - Tool(
50 - type="function",
51 - function=Function(
52 - name="get_current_time",
53 - description="Fetch the current local date and time.",
54 - parameters={},
55 - ),
56 - )
44 + [
45 + {
46 + "type": "function",
47 + "function": {
48 + "description": "Fetch the current local date and time.",
49 + "name": "get_current_time",
50 + "parameters": {},
51 + },
52 + }
53 ],
54 ),
55 ),
@@ -71,39 +67,44 @@ from vllm.transformers_utils.tokenizers.mistral import (
67 "function": {
68 "description": "Fetch the current local date and time.",
69 "name": "get_current_time",
74 - "parameters": None,
70 + "parameters": {},
71 },
72 }
73 ],
74 },
79 - ChatCompletionRequest(
80 - messages=[
81 - UserMessage(content="What is the current local date and time?")
75 + (
76 + [
77 + {
78 + "role": "user",
79 + "content": "What is the current local date and time?",
80 + }
81 ],
83 - tools=[
84 - Tool(
85 - type="function",
86 - function=Function(
87 - name="get_current_time",
88 - description="Fetch the current local date and time.",
89 - parameters={},
90 - ),
91 - )
82 + [
83 + {
84 + "type": "function",
85 + "function": {
86 + "description": "Fetch the current local date and time.",
87 + "name": "get_current_time",
88 + "parameters": {},
89 + },
90 + }
91 ],
92 ),
93 ),
94 ],
95 )
97 -def test_make_mistral_chat_completion_request(openai_request, expected_mistral_request):
98 - actual_request = make_mistral_chat_completion_request(
96 +def test_prepare_apply_chat_template_tools_and_messages(
97 + openai_request, expected_mistral_output
98 +):
99 + actual_request = _prepare_apply_chat_template_tools_and_messages(
100 openai_request["messages"], openai_request["tools"]
101 )
101 - assert actual_request == expected_mistral_request
102 + assert actual_request == expected_mistral_output
103
104
105 # Tool use with list content and reasoning_content
106 @pytest.mark.parametrize(
106 - "openai_request,expected_mistral_request",
107 + "openai_request,expected_mistral_output",
108 [
109 (
110 {
@@ -154,34 +155,40 @@ def test_make_mistral_chat_completion_request(openai_request, expected_mistral_r
155 }
156 ],
157 },
157 - ChatCompletionRequest(
158 - messages=[
159 - UserMessage(content="What's the weather in Paris?"),
160 - AssistantMessage(
161 - content=None,
162 - tool_calls=[
163 - ToolCall(
164 - id="call123",
165 - function=FunctionCall(
166 - name="get_weather",
167 - arguments='{"city": "Paris"}',
168 - ),
169 - )
158 + (
159 + [
160 + {
161 + "role": "user",
162 + "content": "What's the weather in Paris?",
163 + },
164 + {
165 + "role": "assistant",
166 + "content": None,
167 + "tool_calls": [
168 + {
169 + "id": "call123",
170 + "type": "function",
171 + "function": {
172 + "name": "get_weather",
173 + "arguments": '{"city": "Paris"}',
174 + },
175 + }
176 ],
171 - ),
172 - ToolMessage(
173 - content="Rainy",
174 - tool_call_id="call123",
175 - name="get_weather",
176 - ),
177 + },
178 + {
179 + "role": "tool",
180 + "content": [{"type": "text", "text": "Rainy"}],
181 + "name": "get_weather",
182 + "tool_call_id": "call123",
183 + },
184 ],
178 - tools=[
179 - Tool(
180 - type="function",
181 - function=Function(
182 - name="get_weather",
183 - description="Gets the current weather in a city.",
184 - parameters={
185 + [
186 + {
187 + "type": "function",
188 + "function": {
189 + "name": "get_weather",
190 + "description": "Gets the current weather in a city.",
191 + "parameters": {
192 "type": "object",
193 "properties": {
194 "city": {
@@ -191,17 +198,2012 @@ def test_make_mistral_chat_completion_request(openai_request, expected_mistral_r
198 },
199 "required": ["city"],
200 },
194 - ),
195 - )
201 + },
202 + }
203 ],
204 ),
205 )
206 ],
207 )
201 -def test_make_mistral_chat_completion_request_list_content(
202 - openai_request, expected_mistral_request
208 +def test_prepare_apply_chat_template_tools_and_messages_list_content(
209 + openai_request, expected_mistral_output
210 ):
204 - actual_request = make_mistral_chat_completion_request(
211 + actual_request = _prepare_apply_chat_template_tools_and_messages(
212 openai_request["messages"], openai_request["tools"]
213 )
207 - assert actual_request == expected_mistral_request
214 + assert actual_request == expected_mistral_output
215 +
216 +
217 +def test_prepare_apply_chat_template_generation_prompt_and_continue():
218 + messages = [{"role": "assistant", "content": "Hello"}]
219 + tools: list[dict[str, Any]] = []
220 + with pytest.raises(ValueError):
221 + _prepare_apply_chat_template_tools_and_messages(
222 + messages, tools, add_generation_prompt=True
223 + )
224 +
225 + messages = [{"role": "user", "content": "Hello"}]
226 + out_messages, _ = _prepare_apply_chat_template_tools_and_messages(
227 + messages, tools, add_generation_prompt=True
228 + )
229 + assert out_messages == [{"role": "user", "content": "Hello"}]
230 +
231 + with pytest.raises(ValueError):
232 + _prepare_apply_chat_template_tools_and_messages(
233 + messages, tools, add_generation_prompt=True, continue_final_message=True
234 + )
235 +
236 + messages = [{"role": "assistant", "content": "Hello"}]
237 + out_messages, _ = _prepare_apply_chat_template_tools_and_messages(
238 + messages, tools, add_generation_prompt=False, continue_final_message=True
239 + )
240 + assert out_messages == [{"role": "assistant", "content": "Hello"}]
241 +
242 + messages = [{"role": "user", "content": "Hello"}]
243 + with pytest.raises(ValueError):
244 + _prepare_apply_chat_template_tools_and_messages(
245 + messages, tools, add_generation_prompt=False, continue_final_message=True
246 + )
247 +
248 +
249 +@pytest.fixture(scope="module")
250 +def mistral_tokenizer(request) -> MistralTokenizer:
251 + return MistralTokenizer.from_pretrained(request.param)
252 +
253 +
254 +@pytest.mark.parametrize(
255 + "mistral_tokenizer",
256 + ["mistralai/Mistral-7B-Instruct-v0.3", "mistralai/Magistral-Small-2509"],
257 + indirect=True,
258 +)
259 +class TestMistralTokenizer:
260 + def test_all_special_tokens(self, mistral_tokenizer: MistralTokenizer):
261 + attributes = [
262 + mistral_tokenizer.all_special_tokens,
263 + mistral_tokenizer.all_special_tokens_extended,
264 + ]
265 +
266 + for attribute in attributes:
267 + if mistral_tokenizer.is_tekken:
268 + assert attribute == [
269 + "<unk>",
270 + "<s>",
271 + "</s>",
272 + "[INST]",
273 + "[/INST]",
274 + "[AVAILABLE_TOOLS]",
275 + "[/AVAILABLE_TOOLS]",
276 + "[TOOL_RESULTS]",
277 + "[/TOOL_RESULTS]",
278 + "[TOOL_CALLS]",
279 + "[IMG]",
280 + "<pad>",
281 + "[IMG_BREAK]",
282 + "[IMG_END]",
283 + "[PREFIX]",
284 + "[MIDDLE]",
285 + "[SUFFIX]",
286 + "[SYSTEM_PROMPT]",
287 + "[/SYSTEM_PROMPT]",
288 + "[TOOL_CONTENT]",
289 + ] + [f"<SPECIAL_{i}>" for i in range(20, 32)] + [
290 + "[ARGS]",
291 + "[CALL_ID]",
292 + "[THINK]",
293 + "[/THINK]",
294 + ] + [f"<SPECIAL_{i}>" for i in range(36, 1000)]
295 + else:
296 + assert attribute == [
297 + "<s>",
298 + "</s>",
299 + "[INST]",
300 + "[/INST]",
301 + "[TOOL_CALLS]",
302 + "[AVAILABLE_TOOLS]",
303 + "[/AVAILABLE_TOOLS]",
304 + "[TOOL_RESULTS]",
305 + "[/TOOL_RESULTS]",
306 + ] + [f"[control_{i}]" for i in range(8, 769)]
307 +
308 + def get_vocab(self, mistral_tokenizer: MistralTokenizer):
309 + assert (
310 + mistral_tokenizer.get_vocab()
311 + == mistral_tokenizer.transformers_tokenizer.get_vocab()
312 + )
313 +
314 + def test_get_added_vocab(self, mistral_tokenizer: MistralTokenizer):
315 + assert mistral_tokenizer.get_added_vocab() == {}
316 +
317 + def test_encode_one(self, mistral_tokenizer: MistralTokenizer):
318 + token_ids = (
319 + [22177, 4304, 2662] if mistral_tokenizer.is_tekken else [23325, 2294, 1686]
320 + )
321 +
322 + assert mistral_tokenizer.encode_one("Hello world !") == token_ids
323 + assert mistral_tokenizer.encode_one("Hello world !", max_length=1) == token_ids
324 + assert (
325 + mistral_tokenizer.encode_one("Hello world !", truncation=True, max_length=1)
326 + == token_ids[:-2]
327 + )
328 + assert (
329 + mistral_tokenizer.encode_one(
330 + "Hello world !", truncation=False, max_length=1
331 + )
332 + == token_ids
333 + )
334 +
335 + def test_encode(self, mistral_tokenizer: MistralTokenizer):
336 + token_ids = (
337 + [1, 22177, 4304, 2662, 2]
338 + if mistral_tokenizer.is_tekken
339 + else [1, 23325, 2294, 1686, 2]
340 + )
341 +
342 + assert mistral_tokenizer.encode("Hello world !") == token_ids[:-1]
343 + assert mistral_tokenizer.encode("Hello world !", max_length=3) == token_ids[:-2]
344 + assert (
345 + mistral_tokenizer.encode("Hello world !", truncation=True, max_length=3)
346 + == token_ids[:-2]
347 + )
348 + assert (
349 + mistral_tokenizer.encode("Hello world !", truncation=False, max_length=3)
350 + == token_ids[:-1]
351 + )
352 +
353 + assert (
354 + mistral_tokenizer.encode("Hello world !", add_special_tokens=True)
355 + == token_ids
356 + )
357 + assert (
358 + mistral_tokenizer.encode(
359 + "Hello world !", add_special_tokens=True, max_length=3
360 + )
361 + == token_ids[:-2]
362 + )
363 + assert (
364 + mistral_tokenizer.encode(
365 + "Hello world !", add_special_tokens=True, truncation=False, max_length=3
366 + )
367 + == token_ids
368 + )
369 + assert (
370 + mistral_tokenizer.encode("Hello world !", add_special_tokens=False)
371 + == token_ids[1:-1]
372 + )
373 +
374 + @pytest.mark.parametrize(
375 + "openai_request,add_generation_prompt,continue_final_message,expected_output,decoded_expected_output",
376 + [
377 + (
378 + {
379 + "messages": [
380 + {
381 + "role": "user",
382 + "content": "Hello world !",
383 + }
384 + ],
385 + },
386 + True,
387 + False,
388 + ([1, 3, 23325, 2294, 1686, 4], [1, 3, 22177, 4304, 2662, 4]),
389 + ("<s>[INST]▁Hello▁world▁![/INST]", ("<s>[INST]Hello world ![/INST]")),
390 + ),
391 + (
392 + {
393 + "messages": [
394 + {
395 + "role": "system",
396 + "content": "I am an AI",
397 + },
398 + {
399 + "role": "user",
400 + "content": "Hello world !",
401 + },
402 + ],
403 + },
404 + True,
405 + False,
406 + (
407 + [1, 3, 1083, 1605, 1164, 16875, 781, 781, 16998, 2294, 1686, 4],
408 + [1, 17, 1073, 1855, 1420, 26554, 18, 3, 22177, 4304, 2662, 4],
409 + ),
410 + (
411 + "<s>[INST]▁I▁am▁an▁AI<0x0A><0x0A>Hello▁world▁![/INST]",
412 + (
413 + "<s>[SYSTEM_PROMPT]I am an AI[/SYSTEM_PROMPT][INST]Hello world ![/INST]" # noqa: E501
414 + ),
415 + ),
416 + ),
417 + (
418 + {
419 + "messages": [
420 + {
421 + "role": "system",
422 + "content": "I am an AI",
423 + },
424 + {
425 + "role": "user",
426 + "content": "Hello world !",
427 + },
428 + ],
429 + "tools": [
430 + {
431 + "type": "function",
432 + "function": {
433 + "name": "get_weather",
434 + "description": "Gets the current weather in a city.",
435 + "parameters": {
436 + "type": "object",
437 + "properties": {
438 + "city": {
439 + "type": "string",
440 + "description": "The city name",
441 + }
442 + },
443 + "required": ["city"],
444 + },
445 + },
446 + }
447 + ],
448 + },
449 + True,
450 + False,
451 + (
452 + [
453 + 1,
454 + 6,
455 + 1501,
456 + 7567,
457 + 1891,
458 + 2032,
459 + 1113,
460 + 3396,
461 + 1316,
462 + 1113,
463 + 3396,
464 + 2032,
465 + 10598,
466 + 1629,
467 + 2032,
468 + 1113,
469 + 1295,
470 + 29498,
471 + 1537,
472 + 1991,
473 + 1316,
474 + 1113,
475 + 7286,
476 + 2032,
477 + 1113,
478 + 2226,
479 + 29481,
480 + 1040,
481 + 2636,
482 + 8854,
483 + 1065,
484 + 1032,
485 + 3758,
486 + 9959,
487 + 1113,
488 + 12206,
489 + 2032,
490 + 10598,
491 + 1891,
492 + 2032,
493 + 1113,
494 + 3582,
495 + 1316,
496 + 1113,
497 + 11491,
498 + 2032,
499 + 10598,
500 + 19141,
501 + 2032,
502 + 10598,
503 + 1891,
504 + 2032,
505 + 1113,
506 + 2195,
507 + 1316,
508 + 1113,
509 + 7286,
510 + 2032,
511 + 1113,
512 + 1782,
513 + 3758,
514 + 1909,
515 + 29507,
516 + 11549,
517 + 1113,
518 + 11661,
519 + 2032,
520 + 8135,
521 + 19141,
522 + 3010,
523 + 1743,
524 + 10925,
525 + 7,
526 + 3,
527 + 1083,
528 + 1605,
529 + 1164,
530 + 16875,
531 + 781,
532 + 781,
533 + 16998,
534 + 2294,
535 + 1686,
536 + 4,
537 + ],
538 + [
539 + 1,
540 + 17,
541 + 1073,
542 + 1855,
543 + 1420,
544 + 26554,
545 + 18,
546 + 5,
547 + 1091,
548 + 19227,
549 + 4994,
550 + 2811,
551 + 1429,
552 + 5165,
553 + 1897,
554 + 1429,
555 + 5165,
556 + 2811,
557 + 16753,
558 + 2391,
559 + 2811,
560 + 1429,
561 + 1689,
562 + 1095,
563 + 45629,
564 + 1897,
565 + 1429,
566 + 14653,
567 + 2811,
568 + 1429,
569 + 1071,
570 + 3083,
571 + 1278,
572 + 3519,
573 + 17253,
574 + 1294,
575 + 1261,
576 + 5970,
577 + 39249,
578 + 1429,
579 + 26204,
580 + 2811,
581 + 16753,
582 + 4994,
583 + 2811,
584 + 1429,
585 + 6371,
586 + 1897,
587 + 1429,
588 + 48649,
589 + 2811,
590 + 16753,
591 + 29363,
592 + 2811,
593 + 16753,
594 + 4994,
595 + 2811,
596 + 1429,
597 + 3607,
598 + 1897,
599 + 1429,
600 + 14653,
601 + 2811,
602 + 1429,
603 + 1784,
604 + 5970,
605 + 2564,
606 + 1034,
607 + 47579,
608 + 1429,
609 + 15760,
610 + 2811,
611 + 12161,
612 + 29363,
613 + 4964,
614 + 2821,
615 + 27028,
616 + 6,
617 + 3,
618 + 22177,
619 + 4304,
620 + 2662,
621 + 4,
622 + ],
623 + ),
624 + (
625 + '<s>[AVAILABLE_TOOLS]▁[{"type":▁"function",▁"function":▁{"name":▁"get_weather",▁"description":▁"Gets▁the▁current▁weather▁in▁a▁city.",▁"parameters":▁{"type":▁"object",▁"properties":▁{"city":▁{"type":▁"string",▁"description":▁"The▁city▁name"}},▁"required":▁["city"]}}}][/AVAILABLE_TOOLS][INST]▁I▁am▁an▁AI<0x0A><0x0A>Hello▁world▁![/INST]',
626 + (
627 + '<s>[SYSTEM_PROMPT]I am an AI[/SYSTEM_PROMPT][AVAILABLE_TOOLS][{"type": "function", "function": {"name": "get_weather", "description": "Gets the current weather in a city.", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city name"}}, "required": ["city"]}}}][/AVAILABLE_TOOLS][INST]Hello world ![/INST]' # noqa: E501
628 + ),
629 + ),
630 + ),
631 + (
632 + {
633 + "messages": [
634 + {
635 + "role": "system",
636 + "content": "I am an AI",
637 + },
638 + {
639 + "role": "user",
640 + "content": "Hello world !",
641 + },
642 + {
643 + "role": "assistant",
644 + "content": "",
645 + "tool_calls": [
646 + {
647 + "id": "123456789",
648 + "type": "function",
649 + "function": {
650 + "name": "get_weather",
651 + "arguments": '{"city": "Paris"}',
652 + },
653 + }
654 + ],
655 + },
656 + {
657 + "role": "tool",
658 + "tool_call_id": "123456789",
659 + "content": '{"temperature": 20, "unit": "celsius"}',
660 + },
661 + ],
662 + "tools": [
663 + {
664 + "type": "function",
665 + "function": {
666 + "name": "get_weather",
667 + "description": "Gets the current weather in a city.",
668 + "parameters": {
669 + "type": "object",
670 + "properties": {
671 + "city": {
672 + "type": "string",
673 + "description": "The city name",
674 + }
675 + },
676 + "required": ["city"],
677 + },
678 + },
679 + }
680 + ],
681 + },
682 + True,
683 + False,
684 + (
685 + [
686 + 1,
687 + 6,
688 + 1501,
689 + 7567,
690 + 1891,
691 + 2032,
692 + 1113,
693 + 3396,
694 + 1316,
695 + 1113,
696 + 3396,
697 + 2032,
698 + 10598,
699 + 1629,
700 + 2032,
701 + 1113,
702 + 1295,
703 + 29498,
704 + 1537,
705 + 1991,
706 + 1316,
707 + 1113,
708 + 7286,
709 + 2032,
710 + 1113,
711 + 2226,
712 + 29481,
713 + 1040,
714 + 2636,
715 + 8854,
716 + 1065,
717 + 1032,
718 + 3758,
719 + 9959,
720 + 1113,
721 + 12206,
722 + 2032,
723 + 10598,
724 + 1891,
725 + 2032,
726 + 1113,
727 + 3582,
728 + 1316,
729 + 1113,
730 + 11491,
731 + 2032,
732 + 10598,
733 + 19141,
734 + 2032,
735 + 10598,
736 + 1891,
737 + 2032,
738 + 1113,
739 + 2195,
740 + 1316,
741 + 1113,
742 + 7286,
743 + 2032,
744 + 1113,
745 + 1782,
746 + 3758,
747 + 1909,
748 + 29507,
749 + 11549,
750 + 1113,
751 + 11661,
752 + 2032,
753 + 8135,
754 + 19141,
755 + 3010,
756 + 1743,
757 + 10925,
758 + 7,
759 + 3,
760 + 1083,
761 + 1605,
762 + 1164,
763 + 16875,
764 + 781,
765 + 781,
766 + 16998,
767 + 2294,
768 + 1686,
769 + 4,
770 + 5,
771 + 1501,
772 + 7567,
773 + 1629,
774 + 2032,
775 + 1113,
776 + 1295,
777 + 29498,
778 + 1537,
779 + 1991,
780 + 1316,
781 + 1113,
782 + 17452,
783 + 2032,
784 + 10598,
785 + 19141,
786 + 2032,
787 + 1113,
788 + 4684,
789 + 1046,
790 + 8474,
791 + 1113,
792 + 1081,
793 + 2032,
794 + 1113,
795 + 29508,
796 + 29518,
797 + 29538,
798 + 29549,
799 + 29550,
800 + 29552,
801 + 29555,
802 + 29551,
803 + 29542,
804 + 29507,
805 + 10925,
806 + 2,
807 + 8,
808 + 10598,
809 + 4557,
810 + 2032,
811 + 10598,
812 + 29475,
813 + 17329,
814 + 2032,
815 + 29473,
816 + 29518,
817 + 29502,
818 + 29493,
819 + 1113,
820 + 6074,
821 + 2032,
822 + 1113,
823 + 29485,
824 + 1958,
825 + 3938,
826 + 8474,
827 + 1113,
828 + 3613,
829 + 29498,
830 + 1081,
831 + 2032,
832 + 1113,
833 + 29508,
834 + 29518,
835 + 29538,
836 + 29549,
837 + 29550,
838 + 29552,
839 + 29555,
840 + 29551,
841 + 29542,
842 + 18163,
843 + 9,
844 + ],
845 + [
846 + 1,
847 + 17,
848 + 1073,
849 + 1855,
850 + 1420,
851 + 26554,
852 + 18,
853 + 5,
854 + 1091,
855 + 19227,
856 + 4994,
857 + 2811,
858 + 1429,
859 + 5165,
860 + 1897,
861 + 1429,
862 + 5165,
863 + 2811,
864 + 16753,
865 + 2391,
866 + 2811,
867 + 1429,
868 + 1689,
869 + 1095,
870 + 45629,
871 + 1897,
872 + 1429,
873 + 14653,
874 + 2811,
875 + 1429,
876 + 1071,
877 + 3083,
878 + 1278,
879 + 3519,
880 + 17253,
881 + 1294,
882 + 1261,
883 + 5970,
884 + 39249,
885 + 1429,
886 + 26204,
887 + 2811,
888 + 16753,
889 + 4994,
890 + 2811,
891 + 1429,
892 + 6371,
893 + 1897,
894 + 1429,
895 + 48649,
896 + 2811,
897 + 16753,
898 + 29363,
899 + 2811,
900 + 16753,
901 + 4994,
902 + 2811,
903 + 1429,
904 + 3607,
905 + 1897,
906 + 1429,
907 + 14653,
908 + 2811,
909 + 1429,
910 + 1784,
911 + 5970,
912 + 2564,
913 + 1034,
914 + 47579,
915 + 1429,
916 + 15760,
917 + 2811,
918 + 12161,
919 + 29363,
920 + 4964,
921 + 2821,
922 + 27028,
923 + 6,
924 + 3,
925 + 22177,
926 + 4304,
927 + 2662,
928 + 4,
929 + 9,
930 + 1689,
931 + 1095,
932 + 45629,
933 + 32,
934 + 19227,
935 + 29363,
936 + 2811,
937 + 1429,
938 + 42572,
939 + 46005,
940 + 2,
941 + 7,
942 + 19227,
943 + 113824,
944 + 2811,
945 + 1032,
946 + 1050,
947 + 1048,
948 + 1044,
949 + 1429,
950 + 8979,
951 + 2811,
952 + 1429,
953 + 1099,
954 + 79092,
955 + 46005,
956 + 8,
957 + ],
958 + ),
959 + (
960 + '<s>[AVAILABLE_TOOLS]▁[{"type":▁"function",▁"function":▁{"name":▁"get_weather",▁"description":▁"Gets▁the▁current▁weather▁in▁a▁city.",▁"parameters":▁{"type":▁"object",▁"properties":▁{"city":▁{"type":▁"string",▁"description":▁"The▁city▁name"}},▁"required":▁["city"]}}}][/AVAILABLE_TOOLS][INST]▁I▁am▁an▁AI<0x0A><0x0A>Hello▁world▁![/INST][TOOL_CALLS]▁[{"name":▁"get_weather",▁"arguments":▁{"city":▁"Paris"},▁"id":▁"123456789"}]</s>[TOOL_RESULTS]▁{"content":▁{"temperature":▁20,▁"unit":▁"celsius"},▁"call_id":▁"123456789"}[/TOOL_RESULTS]',
961 + (
962 + '<s>[SYSTEM_PROMPT]I am an AI[/SYSTEM_PROMPT][AVAILABLE_TOOLS][{"type": "function", "function": {"name": "get_weather", "description": "Gets the current weather in a city.", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city name"}}, "required": ["city"]}}}][/AVAILABLE_TOOLS][INST]Hello world ![/INST][TOOL_CALLS]get_weather[ARGS]{"city": "Paris"}</s>[TOOL_RESULTS]{"temperature": 20, "unit": "celsius"}[/TOOL_RESULTS]' # noqa: E501
963 + ),
964 + ),
965 + ),
966 + (
967 + {
968 + "messages": [
969 + {
970 + "role": "user",
971 + "content": "Hello world !",
972 + },
973 + {
974 + "role": "assistant",
975 + "content": "Hello ",
976 + },
977 + ],
978 + },
979 + False,
980 + True,
981 + (
982 + [1, 3, 23325, 2294, 1686, 4, 23325],
983 + [1, 3, 22177, 4304, 2662, 4, 22177, 2],
984 + ),
985 + (
986 + "<s>[INST]▁Hello▁world▁![/INST]▁Hello",
987 + ("<s>[INST]Hello world ![/INST]Hello</s>"),
988 + ),
989 + ),
990 + ],
991 + )
992 + def test_apply_chat_template(
993 + self,
994 + mistral_tokenizer: MistralTokenizer,
995 + openai_request: dict[str, Any],
996 + add_generation_prompt: bool,
997 + continue_final_message: bool,
998 + expected_output: tuple[list[int], list[int]],
999 + decoded_expected_output: tuple[str, str],
1000 + ):
1001 + actual_output = mistral_tokenizer.apply_chat_template(
1002 + openai_request["messages"],
1003 + tools=openai_request.get("tools", []),
1004 + add_generation_prompt=add_generation_prompt,
1005 + continue_final_message=continue_final_message,
1006 + )
1007 + decoded_actual_output = mistral_tokenizer.tokenizer.decode(
1008 + actual_output, SpecialTokenPolicy.KEEP
1009 + )
1010 +
1011 + assert actual_output == expected_output[mistral_tokenizer.is_tekken]
1012 + assert (
1013 + decoded_actual_output
1014 + == decoded_expected_output[mistral_tokenizer.is_tekken]
1015 + )
1016 +
1017 + def test_apply_chat_template_error(self, mistral_tokenizer: MistralTokenizer):
1018 + messages = [{"role": "user", "content": "Hello world !"}]
1019 +
1020 + with pytest.raises(ValueError):
1021 + mistral_tokenizer.apply_chat_template(
1022 + messages,
1023 + tools=[],
1024 + add_generation_prompt=True,
1025 + continue_final_message=True,
1026 + )
1027 +
1028 + with pytest.raises(ValueError):
1029 + mistral_tokenizer.apply_chat_template(
1030 + messages,
1031 + tools=[],
1032 + add_generation_prompt=False,
1033 + continue_final_message=True,
1034 + )
1035 +
1036 + messages = [
1037 + {"role": "user", "content": "Hello world !"},
1038 + {"role": "assistant", "content": "Hello "},
1039 + ]
1040 + with pytest.raises(ValueError):
1041 + mistral_tokenizer.apply_chat_template(
1042 + messages,
1043 + tools=[],
1044 + add_generation_prompt=True,
1045 + continue_final_message=False,
1046 + )
1047 +
1048 + messages = [
1049 + {"role": "user", "content": "Hello world !"},
1050 + {"role": "assistant", "content": "Hello "},
1051 + ]
1052 + with pytest.raises(InvalidMessageStructureException):
1053 + mistral_tokenizer.apply_chat_template(
1054 + messages,
1055 + tools=[],
1056 + add_generation_prompt=False,
1057 + continue_final_message=False,
1058 + )
1059 +
1060 + @pytest.mark.parametrize(
1061 + "skip_special_tokens,expected_tokens",
1062 + (
1063 + (
1064 + False,
1065 + (
1066 + "<s>[INST]▁Hello▁world▁![/INST]▁Hello</s>",
1067 + "<s>[INST]Hello world ![/INST]Hello</s>",
1068 + ),
1069 + ),
1070 + (True, ("Hello world ! Hello", "Hello world !Hello")),
1071 + ),
1072 + )
1073 + def test_decode(
1074 + self,
1075 + mistral_tokenizer: MistralTokenizer,
1076 + skip_special_tokens: bool,
1077 + expected_tokens: tuple[str, str],
1078 + ):
1079 + ids = (
1080 + [1, 3, 23325, 2294, 1686, 4, 23325, 2],
1081 + [1, 3, 22177, 4304, 2662, 4, 22177, 2],
1082 + )
1083 + assert (
1084 + mistral_tokenizer.decode(
1085 + ids[mistral_tokenizer.is_tekken],
1086 + skip_special_tokens=skip_special_tokens,
1087 + )
1088 + == expected_tokens[mistral_tokenizer.is_tekken]
1089 + )
1090 +
1091 + def test_convert_tokens_to_string(self, mistral_tokenizer: MistralTokenizer):
1092 + tokens = (
1093 + [
1094 + "<s>",
1095 + "[AVAILABLE_TOOLS]",
1096 + "▁[",
1097 + '{"',
1098 + "type",
1099 + '":',
1100 + '▁"',
1101 + "function",
1102 + '",',
1103 + '▁"',
1104 + "function",
1105 + '":',
1106 + '▁{"',
1107 + "name",
1108 + '":',
1109 + '▁"',
1110 + "get",
1111 + "_",
1112 + "we",
1113 + "ather",
1114 + '",',
1115 + '▁"',
1116 + "description",
1117 + '":',
1118 + '▁"',
1119 + "Get",
1120 + "s",
1121 + "▁the",
1122 + "▁current",
1123 + "▁weather",
1124 + "▁in",
1125 + "▁a",
1126 + "▁city",
1127 + '.",',
1128 + '▁"',
1129 + "parameters",
1130 + '":',
1131 + '▁{"',
1132 + "type",
1133 + '":',
1134 + '▁"',
1135 + "object",
1136 + '",',
1137 + '▁"',
1138 + "properties",
1139 + '":',
1140 + '▁{"',
1141 + "city",
1142 + '":',
1143 + '▁{"',
1144 + "type",
1145 + '":',
1146 + '▁"',
1147 + "string",
1148 + '",',
1149 + '▁"',
1150 + "description",
1151 + '":',
1152 + '▁"',
1153 + "The",
1154 + "▁city",
1155 + "▁name",
1156 + '"',
1157 + "}},",
1158 + '▁"',
1159 + "required",
1160 + '":',
1161 + '▁["',
1162 + "city",
1163 + '"]',
1164 + "}}",
1165 + "}]",
1166 + "[/AVAILABLE_TOOLS]",
1167 + "[INST]",
1168 + "▁I",
1169 + "▁am",
1170 + "▁an",
1171 + "▁AI",
1172 + "<0x0A>",
1173 + "<0x0A>",
1174 + "Hello",
1175 + "▁world",
1176 + "▁!",
1177 + "[/INST]",
1178 + "[TOOL_CALLS]",
1179 + "▁[",
1180 + '{"',
1181 + "name",
1182 + '":',
1183 + '▁"',
1184 + "get",
1185 + "_",
1186 + "we",
1187 + "ather",
1188 + '",',
1189 + '▁"',
1190 + "arguments",
1191 + '":',
1192 + '▁{"',
1193 + "city",
1194 + '":',
1195 + '▁"',
1196 + "Par",
1197 + "is",
1198 + '"},',
1199 + '▁"',
1200 + "id",
1201 + '":',
1202 + '▁"',
1203 + "1",
1204 + "2",
1205 + "3",
1206 + "4",
1207 + "5",
1208 + "6",
1209 + "7",
1210 + "8",
1211 + "9",
1212 + '"',
1213 + "}]",
1214 + "</s>",
1215 + "[TOOL_RESULTS]",
1216 + '▁{"',
1217 + "content",
1218 + '":',
1219 + '▁{"',
1220 + "t",
1221 + "emperature",
1222 + '":',
1223 + "▁",
1224 + "2",
1225 + "0",
1226 + ",",
1227 + '▁"',
1228 + "unit",
1229 + '":',
1230 + '▁"',
1231 + "c",
1232 + "els",
1233 + "ius",
1234 + '"},',
1235 + '▁"',
1236 + "call",
1237 + "_",
1238 + "id",
1239 + '":',
1240 + '▁"',
1241 + "1",
1242 + "2",
1243 + "3",
1244 + "4",
1245 + "5",
1246 + "6",
1247 + "7",
1248 + "8",
1249 + "9",
1250 + '"}',
1251 + "[/TOOL_RESULTS]",
1252 + ],
1253 + [
1254 + "<s>",
1255 + "[SYSTEM_PROMPT]",
1256 + "I",
1257 + " am",
1258 + " an",
1259 + " AI",
1260 + "[/SYSTEM_PROMPT]",
1261 + "[AVAILABLE_TOOLS]",
1262 + "[",
1263 + '{"',
1264 + "type",
1265 + '":',
1266 + ' "',
1267 + "function",
1268 + '",',
1269 + ' "',
1270 + "function",
1271 + '":',
1272 + ' {"',
1273 + "name",
1274 + '":',
1275 + ' "',
1276 + "get",
1277 + "_",
1278 + "weather",
1279 + '",',
1280 + ' "',
1281 + "description",
1282 + '":',
1283 + ' "',
1284 + "G",
1285 + "ets",
1286 + " the",
1287 + " current",
1288 + " weather",
1289 + " in",
1290 + " a",
1291 + " city",
1292 + '.",',
1293 + ' "',
1294 + "parameters",
1295 + '":',
1296 + ' {"',
1297 + "type",
1298 + '":',
1299 + ' "',
1300 + "object",
1301 + '",',
1302 + ' "',
1303 + "properties",
1304 + '":',
1305 + ' {"',
1306 + "city",
1307 + '":',
1308 + ' {"',
1309 + "type",
1310 + '":',
1311 + ' "',
1312 + "string",
1313 + '",',
1314 + ' "',
1315 + "description",
1316 + '":',
1317 + ' "',
1318 + "The",
1319 + " city",
1320 + " name",
1321 + '"',
1322 + "}},",
1323 + ' "',
1324 + "required",
1325 + '":',
1326 + ' ["',
1327 + "city",
1328 + '"]',
1329 + "}}",
1330 + "}]",
1331 + "[/AVAILABLE_TOOLS]",
1332 + "[INST]",
1333 + "Hello",
1334 + " world",
1335 + " !",
1336 + "[/INST]",
1337 + "[TOOL_CALLS]",
1338 + "get",
1339 + "_",
1340 + "weather",
1341 + "[ARGS]",
1342 + '{"',
1343 + "city",
1344 + '":',
1345 + ' "',
1346 + "Paris",
1347 + '"}',
1348 + "</s>",
1349 + "[TOOL_RESULTS]",
1350 + '{"',
1351 + "temperature",
1352 + '":',
1353 + " ",
1354 + "2",
1355 + "0",
1356 + ",",
1357 + ' "',
1358 + "unit",
1359 + '":',
1360 + ' "',
1361 + "c",
1362 + "elsius",
1363 + '"}',
1364 + "[/TOOL_RESULTS]",
1365 + ],
1366 + )
1367 +
1368 + expected_strings = (
1369 + '[{"type": "function", "function": {"name": "get_weather", "description": "Gets the current weather in a city.", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city name"}}, "required": ["city"]}}}] I am an AI\n\nHello world ![TOOL_CALLS][{"name": "get_weather", "arguments": {"city": "Paris"}, "id": "123456789"}] {"content": {"temperature": 20, "unit": "celsius"}, "call_id": "123456789"}', # noqa: E501
1370 + 'I am an AI[{"type": "function", "function": {"name": "get_weather", "description": "Gets the current weather in a city.", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city name"}}, "required": ["city"]}}}]Hello world ![TOOL_CALLS]get_weather{"city": "Paris"}{"temperature": 20, "unit": "celsius"}', # noqa: E501
1371 + )
1372 +
1373 + assert (
1374 + mistral_tokenizer.convert_tokens_to_string(
1375 + tokens[mistral_tokenizer.is_tekken]
1376 + )
1377 + == expected_strings[mistral_tokenizer.is_tekken]
1378 + )
1379 +
1380 + @pytest.mark.parametrize(
1381 + "skip_special_tokens,tuple_expected_tokens",
1382 + (
1383 + (
1384 + True,
1385 + (
1386 + [
1387 + "▁[",
1388 + '{"',
1389 + "type",
1390 + '":',
1391 + '▁"',
1392 + "function",
1393 + '",',
1394 + '▁"',
1395 + "function",
1396 + '":',
1397 + '▁{"',
1398 + "name",
1399 + '":',
1400 + '▁"',
1401 + "get",
1402 + "_",
1403 + "we",
1404 + "ather",
1405 + '",',
1406 + '▁"',
1407 + "description",
1408 + '":',
1409 + '▁"',
1410 + "Get",
1411 + "s",
1412 + "▁the",
1413 + "▁current",
1414 + "▁weather",
1415 + "▁in",
1416 + "▁a",
1417 + "▁city",
1418 + '.",',
1419 + '▁"',
1420 + "parameters",
1421 + '":',
1422 + '▁{"',
1423 + "type",
1424 + '":',
1425 + '▁"',
1426 + "object",
1427 + '",',
1428 + '▁"',
1429 + "properties",
1430 + '":',
1431 + '▁{"',
1432 + "city",
1433 + '":',
1434 + '▁{"',
1435 + "type",
1436 + '":',
1437 + '▁"',
1438 + "string",
1439 + '",',
1440 + '▁"',
1441 + "description",
1442 + '":',
1443 + '▁"',
1444 + "The",
1445 + "▁city",
1446 + "▁name",
1447 + '"',
1448 + "}},",
1449 + '▁"',
1450 + "required",
1451 + '":',
1452 + '▁["',
1453 + "city",
1454 + '"]',
1455 + "}}",
1456 + "}]",
1457 + "▁I",
1458 + "▁am",
1459 + "▁an",
1460 + "▁AI",
1461 + "<0x0A>",
1462 + "<0x0A>",
1463 + "Hello",
1464 + "▁world",
1465 + "▁!",
1466 + "[TOOL_CALLS]",
1467 + "▁[",
1468 + '{"',
1469 + "name",
1470 + '":',
1471 + '▁"',
1472 + "get",
1473 + "_",
1474 + "we",
1475 + "ather",
1476 + '",',
1477 + '▁"',
1478 + "arguments",
1479 + '":',
1480 + '▁{"',
1481 + "city",
1482 + '":',
1483 + '▁"',
1484 + "Par",
1485 + "is",
1486 + '"},',
1487 + '▁"',
1488 + "id",
1489 + '":',
1490 + '▁"',
1491 + "1",
1492 + "2",
1493 + "3",
1494 + "4",
1495 + "5",
1496 + "6",
1497 + "7",
1498 + "8",
1499 + "9",
1500 + '"',
1501 + "}]",
1502 + '▁{"',
1503 + "content",
1504 + '":',
1505 + '▁{"',
1506 + "t",
1507 + "emperature",
1508 + '":',
1509 + "▁",
1510 + "2",
1511 + "0",
1512 + ",",
1513 + '▁"',
1514 + "unit",
1515 + '":',
1516 + '▁"',
1517 + "c",
1518 + "els",
1519 + "ius",
1520 + '"},',
1521 + '▁"',
1522 + "call",
1523 + "_",
1524 + "id",
1525 + '":',
1526 + '▁"',
1527 + "1",
1528 + "2",
1529 + "3",
1530 + "4",
1531 + "5",
1532 + "6",
1533 + "7",
1534 + "8",
1535 + "9",
1536 + '"}',
1537 + ],
1538 + [
1539 + "I",
1540 + " am",
1541 + " an",
1542 + " AI",
1543 + "[",
1544 + '{"',
1545 + "type",
1546 + '":',
1547 + ' "',
1548 + "function",
1549 + '",',
1550 + ' "',
1551 + "function",
1552 + '":',
1553 + ' {"',
1554 + "name",
1555 + '":',
1556 + ' "',
1557 + "get",
1558 + "_",
1559 + "weather",
1560 + '",',
1561 + ' "',
1562 + "description",
1563 + '":',
1564 + ' "',
1565 + "G",
1566 + "ets",
1567 + " the",
1568 + " current",
1569 + " weather",
1570 + " in",
1571 + " a",
1572 + " city",
1573 + '.",',
1574 + ' "',
1575 + "parameters",
1576 + '":',
1577 + ' {"',
1578 + "type",
1579 + '":',
1580 + ' "',
1581 + "object",
1582 + '",',
1583 + ' "',
1584 + "properties",
1585 + '":',
1586 + ' {"',
1587 + "city",
1588 + '":',
1589 + ' {"',
1590 + "type",
1591 + '":',
1592 + ' "',
1593 + "string",
1594 + '",',
1595 + ' "',
1596 + "description",
1597 + '":',
1598 + ' "',
1599 + "The",
1600 + " city",
1601 + " name",
1602 + '"',
1603 + "}},",
1604 + ' "',
1605 + "required",
1606 + '":',
1607 + ' ["',
1608 + "city",
1609 + '"]',
1610 + "}}",
1611 + "}]",
1612 + "Hello",
1613 + " world",
1614 + " !",
1615 + "[TOOL_CALLS]",
1616 + "get",
1617 + "_",
1618 + "weather",
1619 + '{"',
1620 + "city",
1621 + '":',
1622 + ' "',
1623 + "Paris",
1624 + '"}',
1625 + '{"',
1626 + "temperature",
1627 + '":',
1628 + " ",
1629 + "2",
1630 + "0",
1631 + ",",
1632 + ' "',
1633 + "unit",
1634 + '":',
1635 + ' "',
1636 + "c",
1637 + "elsius",
1638 + '"}',
1639 + ],
1640 + ),
1641 + ),
1642 + (
1643 + False,
1644 + (
1645 + [
1646 + "<s>",
1647 + "[AVAILABLE_TOOLS]",
1648 + "▁[",
1649 + '{"',
1650 + "type",
1651 + '":',
1652 + '▁"',
1653 + "function",
1654 + '",',
1655 + '▁"',
1656 + "function",
1657 + '":',
1658 + '▁{"',
1659 + "name",
1660 + '":',
1661 + '▁"',
1662 + "get",
1663 + "_",
1664 + "we",
1665 + "ather",
1666 + '",',
1667 + '▁"',
1668 + "description",
1669 + '":',
1670 + '▁"',
1671 + "Get",
1672 + "s",
1673 + "▁the",
1674 + "▁current",
1675 + "▁weather",
1676 + "▁in",
1677 + "▁a",
1678 + "▁city",
1679 + '.",',
1680 + '▁"',
1681 + "parameters",
1682 + '":',
1683 + '▁{"',
1684 + "type",
1685 + '":',
1686 + '▁"',
1687 + "object",
1688 + '",',
1689 + '▁"',
1690 + "properties",
1691 + '":',
1692 + '▁{"',
1693 + "city",
1694 + '":',
1695 + '▁{"',
1696 + "type",
1697 + '":',
1698 + '▁"',
1699 + "string",
1700 + '",',
1701 + '▁"',
1702 + "description",
1703 + '":',
1704 + '▁"',
1705 + "The",
1706 + "▁city",
1707 + "▁name",
1708 + '"',
1709 + "}},",
1710 + '▁"',
1711 + "required",
1712 + '":',
1713 + '▁["',
1714 + "city",
1715 + '"]',
1716 + "}}",
1717 + "}]",
1718 + "[/AVAILABLE_TOOLS]",
1719 + "[INST]",
1720 + "▁I",
1721 + "▁am",
1722 + "▁an",
1723 + "▁AI",
1724 + "<0x0A>",
1725 + "<0x0A>",
1726 + "Hello",
1727 + "▁world",
1728 + "▁!",
1729 + "[/INST]",
1730 + "[TOOL_CALLS]",
1731 + "▁[",
1732 + '{"',
1733 + "name",
1734 + '":',
1735 + '▁"',
1736 + "get",
1737 + "_",
1738 + "we",
1739 + "ather",
1740 + '",',
1741 + '▁"',
1742 + "arguments",
1743 + '":',
1744 + '▁{"',
1745 + "city",
1746 + '":',
1747 + '▁"',
1748 + "Par",
1749 + "is",
1750 + '"},',
1751 + '▁"',
1752 + "id",
1753 + '":',
1754 + '▁"',
1755 + "1",
1756 + "2",
1757 + "3",
1758 + "4",
1759 + "5",
1760 + "6",
1761 + "7",
1762 + "8",
1763 + "9",
1764 + '"',
1765 + "}]",
1766 + "</s>",
1767 + "[TOOL_RESULTS]",
1768 + '▁{"',
1769 + "content",
1770 + '":',
1771 + '▁{"',
1772 + "t",
1773 + "emperature",
1774 + '":',
1775 + "▁",
1776 + "2",
1777 + "0",
1778 + ",",
1779 + '▁"',
1780 + "unit",
1781 + '":',
1782 + '▁"',
1783 + "c",
1784 + "els",
1785 + "ius",
1786 + '"},',
1787 + '▁"',
1788 + "call",
1789 + "_",
1790 + "id",
1791 + '":',
1792 + '▁"',
1793 + "1",
1794 + "2",
1795 + "3",
1796 + "4",
1797 + "5",
1798 + "6",
1799 + "7",
1800 + "8",
1801 + "9",
1802 + '"}',
1803 + "[/TOOL_RESULTS]",
1804 + ],
1805 + [
1806 + "<s>",
1807 + "[SYSTEM_PROMPT]",
1808 + "I",
1809 + " am",
1810 + " an",
1811 + " AI",
1812 + "[/SYSTEM_PROMPT]",
1813 + "[AVAILABLE_TOOLS]",
1814 + "[",
1815 + '{"',
1816 + "type",
1817 + '":',
1818 + ' "',
1819 + "function",
1820 + '",',
1821 + ' "',
1822 + "function",
1823 + '":',
1824 + ' {"',
1825 + "name",
1826 + '":',
1827 + ' "',
1828 + "get",
1829 + "_",
1830 + "weather",
1831 + '",',
1832 + ' "',
1833 + "description",
1834 + '":',
1835 + ' "',
1836 + "G",
1837 + "ets",
1838 + " the",
1839 + " current",
1840 + " weather",
1841 + " in",
1842 + " a",
1843 + " city",
1844 + '.",',
1845 + ' "',
1846 + "parameters",
1847 + '":',
1848 + ' {"',
1849 + "type",
1850 + '":',
1851 + ' "',
1852 + "object",
1853 + '",',
1854 + ' "',
1855 + "properties",
1856 + '":',
1857 + ' {"',
1858 + "city",
1859 + '":',
1860 + ' {"',
1861 + "type",
1862 + '":',
1863 + ' "',
1864 + "string",
1865 + '",',
1866 + ' "',
1867 + "description",
1868 + '":',
1869 + ' "',
1870 + "The",
1871 + " city",
1872 + " name",
1873 + '"',
1874 + "}},",
1875 + ' "',
1876 + "required",
1877 + '":',
1878 + ' ["',
1879 + "city",
1880 + '"]',
1881 + "}}",
1882 + "}]",
1883 + "[/AVAILABLE_TOOLS]",
1884 + "[INST]",
1885 + "Hello",
1886 + " world",
1887 + " !",
1888 + "[/INST]",
1889 + "[TOOL_CALLS]",
1890 + "get",
1891 + "_",
1892 + "weather",
1893 + "[ARGS]",
1894 + '{"',
1895 + "city",
1896 + '":',
1897 + ' "',
1898 + "Paris",
1899 + '"}',
1900 + "</s>",
1901 + "[TOOL_RESULTS]",
1902 + '{"',
1903 + "temperature",
1904 + '":',
1905 + " ",
1906 + "2",
1907 + "0",
1908 + ",",
1909 + ' "',
1910 + "unit",
1911 + '":',
1912 + ' "',
1913 + "c",
1914 + "elsius",
1915 + '"}',
1916 + "[/TOOL_RESULTS]",
1917 + ],
1918 + ),
1919 + ),
1920 + ),
1921 + )
1922 + def test_convert_ids_to_tokens(
1923 + self,
1924 + mistral_tokenizer: MistralTokenizer,
1925 + skip_special_tokens: bool,
1926 + tuple_expected_tokens: tuple[list[str], list[str]],
1927 + ):
1928 + tuple_ids = (
1929 + [
1930 + 1,
1931 + 6,
1932 + 1501,
1933 + 7567,
1934 + 1891,
1935 + 2032,
1936 + 1113,
1937 + 3396,
1938 + 1316,
1939 + 1113,
1940 + 3396,
1941 + 2032,
1942 + 10598,
1943 + 1629,
1944 + 2032,
1945 + 1113,
1946 + 1295,
1947 + 29498,
1948 + 1537,
1949 + 1991,
1950 + 1316,
1951 + 1113,
1952 + 7286,
1953 + 2032,
1954 + 1113,
1955 + 2226,
1956 + 29481,
1957 + 1040,
1958 + 2636,
1959 + 8854,
1960 + 1065,
1961 + 1032,
1962 + 3758,
1963 + 9959,
1964 + 1113,
1965 + 12206,
1966 + 2032,
1967 + 10598,
1968 + 1891,
1969 + 2032,
1970 + 1113,
1971 + 3582,
1972 + 1316,
1973 + 1113,
1974 + 11491,
1975 + 2032,
1976 + 10598,
1977 + 19141,
1978 + 2032,
1979 + 10598,
1980 + 1891,
1981 + 2032,
1982 + 1113,
1983 + 2195,
1984 + 1316,
1985 + 1113,
1986 + 7286,
1987 + 2032,
1988 + 1113,
1989 + 1782,
1990 + 3758,
1991 + 1909,
1992 + 29507,
1993 + 11549,
1994 + 1113,
1995 + 11661,
1996 + 2032,
1997 + 8135,
1998 + 19141,
1999 + 3010,
2000 + 1743,
2001 + 10925,
2002 + 7,
2003 + 3,
2004 + 1083,
2005 + 1605,
2006 + 1164,
2007 + 16875,
2008 + 781,
2009 + 781,
2010 + 16998,
2011 + 2294,
2012 + 1686,
2013 + 4,
2014 + 5,
2015 + 1501,
2016 + 7567,
2017 + 1629,
2018 + 2032,
2019 + 1113,
2020 + 1295,
2021 + 29498,
2022 + 1537,
2023 + 1991,
2024 + 1316,
2025 + 1113,
2026 + 17452,
2027 + 2032,
2028 + 10598,
2029 + 19141,
2030 + 2032,
2031 + 1113,
2032 + 4684,
2033 + 1046,
2034 + 8474,
2035 + 1113,
2036 + 1081,
2037 + 2032,
2038 + 1113,
2039 + 29508,
2040 + 29518,
2041 + 29538,
2042 + 29549,
2043 + 29550,
2044 + 29552,
2045 + 29555,
2046 + 29551,
2047 + 29542,
2048 + 29507,
2049 + 10925,
2050 + 2,
2051 + 8,
2052 + 10598,
2053 + 4557,
2054 + 2032,
2055 + 10598,
2056 + 29475,
2057 + 17329,
2058 + 2032,
2059 + 29473,
2060 + 29518,
2061 + 29502,
2062 + 29493,
2063 + 1113,
2064 + 6074,
2065 + 2032,
2066 + 1113,
2067 + 29485,
2068 + 1958,
2069 + 3938,
2070 + 8474,
2071 + 1113,
2072 + 3613,
2073 + 29498,
2074 + 1081,
2075 + 2032,
2076 + 1113,
2077 + 29508,
2078 + 29518,
2079 + 29538,
2080 + 29549,
2081 + 29550,
2082 + 29552,
2083 + 29555,
2084 + 29551,
2085 + 29542,
2086 + 18163,
2087 + 9,
2088 + ],
2089 + [
2090 + 1,
2091 + 17,
2092 + 1073,
2093 + 1855,
2094 + 1420,
2095 + 26554,
2096 + 18,
2097 + 5,
2098 + 1091,
2099 + 19227,
2100 + 4994,
2101 + 2811,
2102 + 1429,
2103 + 5165,
2104 + 1897,
2105 + 1429,
2106 + 5165,
2107 + 2811,
2108 + 16753,
2109 + 2391,
2110 + 2811,
2111 + 1429,
2112 + 1689,
2113 + 1095,
2114 + 45629,
2115 + 1897,
2116 + 1429,
2117 + 14653,
2118 + 2811,
2119 + 1429,
2120 + 1071,
2121 + 3083,
2122 + 1278,
2123 + 3519,
2124 + 17253,
2125 + 1294,
2126 + 1261,
2127 + 5970,
2128 + 39249,
2129 + 1429,
2130 + 26204,
2131 + 2811,
2132 + 16753,
2133 + 4994,
2134 + 2811,
2135 + 1429,
2136 + 6371,
2137 + 1897,
2138 + 1429,
2139 + 48649,
2140 + 2811,
2141 + 16753,
2142 + 29363,
2143 + 2811,
2144 + 16753,
2145 + 4994,
2146 + 2811,
2147 + 1429,
2148 + 3607,
2149 + 1897,
2150 + 1429,
2151 + 14653,
2152 + 2811,
2153 + 1429,
2154 + 1784,
2155 + 5970,
2156 + 2564,
2157 + 1034,
2158 + 47579,
2159 + 1429,
2160 + 15760,
2161 + 2811,
2162 + 12161,
2163 + 29363,
2164 + 4964,
2165 + 2821,
2166 + 27028,
2167 + 6,
2168 + 3,
2169 + 22177,
2170 + 4304,
2171 + 2662,
2172 + 4,
2173 + 9,
2174 + 1689,
2175 + 1095,
2176 + 45629,
2177 + 32,
2178 + 19227,
2179 + 29363,
2180 + 2811,
2181 + 1429,
2182 + 42572,
2183 + 46005,
2184 + 2,
2185 + 7,
2186 + 19227,
2187 + 113824,
2188 + 2811,
2189 + 1032,
2190 + 1050,
2191 + 1048,
2192 + 1044,
2193 + 1429,
2194 + 8979,
2195 + 2811,
2196 + 1429,
2197 + 1099,
2198 + 79092,
2199 + 46005,
2200 + 8,
2201 + ],
2202 + )
2203 +
2204 + ids = tuple_ids[mistral_tokenizer.is_tekken]
2205 + expected_tokens = tuple_expected_tokens[mistral_tokenizer.is_tekken]
2206 + actual_tokens = mistral_tokenizer.convert_ids_to_tokens(
2207 + ids, skip_special_tokens=skip_special_tokens
2208 + )
2209 + assert actual_tokens == expected_tokens
vllm/entrypoints/chat_utils.py
+5 -13
@@ -403,20 +403,12 @@ def resolve_mistral_chat_template(
403 chat_template: Optional[str],
404 **kwargs: Any,
405 ) -> Optional[str]:
406 - if chat_template is not None:
407 - logger.warning_once(
408 - "'chat_template' cannot be overridden for mistral tokenizer."
409 - )
410 - if "add_generation_prompt" in kwargs:
411 - logger.warning_once(
412 - "'add_generation_prompt' is not supported for mistral tokenizer, "
413 - "so it will be ignored."
414 - )
415 - if "continue_final_message" in kwargs:
416 - logger.warning_once(
417 - "'continue_final_message' is not supported for mistral tokenizer, "
418 - "so it will be ignored."
406 + if chat_template is not None or kwargs.get("chat_template_kwargs") is not None:
407 + raise ValueError(
408 + "'chat_template' or 'chat_template_kwargs' cannot be overridden "
409 + "for mistral tokenizer."
410 )
411 +
412 return None
413
414
vllm/model_executor/models/pixtral.py
+2 -1
@@ -10,7 +10,8 @@ from typing import Annotated, Literal, Optional, Union
10 import torch
11 import torch.nn as nn
12 import torch.nn.functional as F
13 -from mistral_common.protocol.instruct.messages import ImageChunk, TextChunk, UserMessage
13 +from mistral_common.protocol.instruct.chunk import ImageChunk, TextChunk
14 +from mistral_common.protocol.instruct.messages import UserMessage
15 from mistral_common.protocol.instruct.request import ChatCompletionRequest
16 from mistral_common.tokens.tokenizers.multimodal import ImageEncoder
17 from PIL import Image
vllm/model_executor/models/voxtral.py
+2 -6
@@ -12,12 +12,8 @@ import regex as re
12 import torch
13 import torch.nn as nn
14 from mistral_common.audio import mel_filter_bank
15 -from mistral_common.protocol.instruct.messages import (
16 - AudioChunk,
17 - RawAudio,
18 - TextChunk,
19 - UserMessage,
20 -)
15 +from mistral_common.protocol.instruct.chunk import AudioChunk, RawAudio, TextChunk
16 +from mistral_common.protocol.instruct.messages import UserMessage
17 from mistral_common.protocol.instruct.request import ChatCompletionRequest
18 from mistral_common.protocol.transcription.request import TranscriptionRequest
19 from mistral_common.tokens.tokenizers.audio import Audio, AudioEncoder
vllm/transformers_utils/tokenizers/mistral.py
+231 -266
@@ -1,34 +1,27 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 -import os
5 -from pathlib import Path
4 from typing import TYPE_CHECKING, Any, Optional, Union, cast
5
8 -import huggingface_hub
9 -import regex as re
10 -from huggingface_hub import HfApi, hf_hub_download
11 -from transformers.tokenization_utils_base import BatchEncoding
12 -
6 from vllm.logger import init_logger
7 from vllm.transformers_utils.tokenizer_base import TokenizerBase
15 -from vllm.utils import is_list_of
8
9 if TYPE_CHECKING:
18 - # make sure `mistral_common` is lazy imported,
19 - # so that users who only use non-mistral models
20 - # will not be bothered by the dependency.
21 - from mistral_common.protocol.instruct.request import ChatCompletionRequest
22 - from mistral_common.tokens.tokenizers.mistral import (
23 - MistralTokenizer as PublicMistralTokenizer,
10 + from mistral_common.protocol.instruct.request import (
11 + ChatCompletionRequest as MistralChatCompletionRequest,
12 + )
13 + from mistral_common.tokens.tokenizers.tekken import Tekkenizer
14 + from transformers.tokenization_mistral_common import (
15 + MistralCommonTokenizer as TransformersMistralTokenizer,
16 )
17
18 from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
19 + from vllm.entrypoints.openai.protocol import ChatCompletionRequest
20
21 logger = init_logger(__name__)
22
23
31 -def maybe_serialize_tool_calls(request: "ChatCompletionRequest"):
24 +def maybe_serialize_tool_calls(request: "MistralChatCompletionRequest"):
25 # SEE: https://github.com/vllm-project/vllm/pull/9951
26 # Credits go to: @gcalmettes
27 # NOTE: There is currently a bug in pydantic where attributes
@@ -65,7 +58,7 @@ def maybe_serialize_tool_calls(request: "ChatCompletionRequest"):
58 request.messages[i]["tool_calls"] = validated_tool_calls
59
60
68 -def truncate_tool_call_ids(request: "ChatCompletionRequest"):
61 +def truncate_tool_call_ids(request: "MistralChatCompletionRequest"):
62 """Truncates tool call IDs for Mistral's ID requirements."""
63 for i, message in enumerate(request.messages):
64 if message.get("role") == "assistant":
@@ -95,85 +88,35 @@ def truncate_tool_call_ids(request: "ChatCompletionRequest"):
88 request.messages[i]["tool_call_id"] = tool_call_id
89
90
98 -def validate_request_params(request: "ChatCompletionRequest"):
99 - if request.skip_special_tokens is not None and not request.skip_special_tokens:
91 +def _prepare_apply_chat_template_tools_and_messages(
92 + messages: list["ChatCompletionMessageParam"],
93 + tools: Optional[list[dict[str, Any]]] = None,
94 + continue_final_message: bool = False,
95 + add_generation_prompt: bool = False,
96 +) -> tuple[list["ChatCompletionMessageParam"], Optional[list[dict[str, Any]]]]:
97 + if add_generation_prompt and continue_final_message:
98 raise ValueError(
101 - "skip_special_tokens=False is not supported for Mistral tokenizers."
99 + "Cannot set both `add_generation_prompt` and "
100 + "`continue_final_message` to True."
101 )
102
104 -
105 -def list_local_repo_files(repo_id: str, revision: Optional[str]) -> list[str]:
106 - repo_cache = os.path.join(
107 - huggingface_hub.constants.HF_HUB_CACHE,
108 - huggingface_hub.constants.REPO_ID_SEPARATOR.join(
109 - ["models", *repo_id.split("/")]
110 - ),
111 - )
112 -
113 - if revision is None:
114 - revision_file = os.path.join(repo_cache, "refs", "main")
115 - if os.path.isfile(revision_file):
116 - with open(revision_file) as file:
117 - revision = file.read()
118 -
119 - if revision:
120 - revision_dir = os.path.join(repo_cache, "snapshots", revision)
121 - if os.path.isdir(revision_dir):
122 - return os.listdir(revision_dir)
123 -
124 - return []
125 -
126 -
127 -def find_tokenizer_file(files: list[str]):
128 - # Accept both versioned (tokenizer.model.v3) and unversioned
129 - # (tokenizer.model) forms, plus tekken.json and tokenizer.mm.model
130 - # variants. Previous pattern only matched the versioned variants.
131 - file_pattern = re.compile(
132 - r"^tokenizer\.model(\.v.*)?|tekken\.json|tokenizer\.mm\.model(\.v.*)?$"
133 - )
134 -
135 - matched_files = [file for file in files if file_pattern.match(file)]
136 - if len(matched_files) > 1:
137 - logger.warning(
138 - "Multiple files matched pattern `%s`: %s. Using %s.",
139 - file_pattern.pattern,
140 - matched_files,
141 - matched_files[0],
103 + last_message = cast(dict[str, Any], messages[-1])
104 + # add_generation_prompt is directly handled by the tokenizer but we
105 + # check if the user is trying to use it with a final assistant message
106 + # which is probably not what they want.
107 + # If add_generation_prompt is False, we don't need to check anything.
108 + if add_generation_prompt and last_message["role"] == "assistant":
109 + raise ValueError(
110 + "Cannot set `add_generation_prompt` to True when "
111 + "the last message is from the assistant. Consider "
112 + "using `continue_final_message` instead."
113 )
143 - elif len(matched_files) == 0:
144 - raise OSError(
145 - f"Found {len(matched_files)} files matching the "
146 - f"pattern: `{file_pattern.pattern}`. Make sure that a Mistral "
147 - f"tokenizer is present in {files}."
114 + if continue_final_message and last_message["role"] != "assistant":
115 + raise ValueError(
116 + "Cannot set `continue_final_message` to True when "
117 + "the last message is not from the assistant."
118 )
119
150 - return matched_files[0]
151 -
152 -
153 -def _aggregate_content(content: list) -> list[dict[str, Any]]:
154 - aggregated_content: list[dict[str, Any]] = []
155 - for chunk in content:
156 - if (
157 - chunk.get("type") == "text"
158 - and aggregated_content
159 - and aggregated_content[-1].get("type") == "text"
160 - ):
161 - aggregated_content[-1]["text"] += "\n\n" + chunk.get("text")
162 - else:
163 - aggregated_content.append(chunk)
164 - if len(aggregated_content) == 1 and aggregated_content[0].get("type") == "text":
165 - content = aggregated_content[0]["text"]
166 - return content
167 -
168 -
169 -def make_mistral_chat_completion_request(
170 - messages: list["ChatCompletionMessageParam"],
171 - tools: Optional[list[dict[str, Any]]] = None,
172 -) -> "ChatCompletionRequest":
173 - last_message = cast(dict[str, Any], messages[-1])
174 - if last_message["role"] == "assistant":
175 - last_message["prefix"] = True
176 -
120 # mistral-common requires AssistantMessage content to be string [1].
121 #
122 # [1]: https://github.com/mistralai/mistral-common/blob/f4a06998b75ed78bbf5aaf569590b772ea26c9f6/src/mistral_common/protocol/instruct/messages.py#L80
@@ -181,13 +124,6 @@ def make_mistral_chat_completion_request(
124 # Remove reasoning_content as unsupported by Mistral
125 _ = message.pop("reasoning_content", None) # type: ignore
126
184 - # Convert list text content to string
185 - if message.get("role") in ("assistant", "tool"):
186 - content: Any = message.get("content")
187 - if isinstance(content, list):
188 - content = _aggregate_content(content)
189 - message["content"] = content
190 -
127 # The Mistral client, in comparison to the OpenAI client, requires the
128 # "parameters" dict and the "description" string to be present
129 # even if they are empty.
@@ -200,108 +136,113 @@ def make_mistral_chat_completion_request(
136 if function.get("description") is None:
137 function["description"] = ""
138
203 - from mistral_common.protocol.instruct.request import ChatCompletionRequest
139 + return messages, tools
140
205 - return ChatCompletionRequest(messages=messages, tools=tools) # type: ignore[type-var]
141
142 +def validate_request_params(request: "ChatCompletionRequest"):
143 + if request.chat_template is not None or request.chat_template_kwargs is not None:
144 + raise ValueError("chat_template is not supported for Mistral tokenizers.")
145
208 -class MistralTokenizer(TokenizerBase):
209 - def __init__(self, tokenizer: "PublicMistralTokenizer") -> None:
210 - self.mistral = tokenizer
211 - self.instruct = tokenizer.instruct_tokenizer
212 - _mistral_version_str = self.instruct.tokenizer.version.value
213 - self.version: int = int(_mistral_version_str.split("v")[-1])
146
215 - tokenizer_ = tokenizer.instruct_tokenizer.tokenizer
216 - from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy
217 - from mistral_common.tokens.tokenizers.tekken import Tekkenizer
147 +def _tekken_token_to_id(tokenizer: "Tekkenizer", t: Union[str, bytes]) -> int:
148 + from mistral_common.tokens.tokenizers.tekken import Tekkenizer
149 +
150 + assert isinstance(tokenizer, Tekkenizer), type(tokenizer)
151 +
152 + t_bytes = t.encode("utf-8") if not isinstance(t, bytes) else t
153 + shift = tokenizer.num_special_tokens
154 + try:
155 + return shift + tokenizer._tekken_token2id_nospecial[t_bytes]
156 + except KeyError:
157 + t_str = t_bytes.decode("utf-8")
158 + if t_str in tokenizer._special_tokens_reverse_vocab:
159 + return tokenizer._special_tokens_reverse_vocab[t_str]
160 + logger.warning(
161 + "Failed to convert token %s to id, replacing with <unk>", t_bytes
162 + )
163 + return tokenizer.unk_id
164 +
165
219 - self.is_tekken = isinstance(tokenizer_, Tekkenizer)
166 +class MistralTokenizer(TokenizerBase):
167 + def __init__(self, tokenizer: "TransformersMistralTokenizer") -> None:
168 from mistral_common.tokens.tokenizers.sentencepiece import (
169 SentencePieceTokenizer,
170 )
171 + from mistral_common.tokens.tokenizers.tekken import Tekkenizer
172
224 - self.is_spm = isinstance(tokenizer_, SentencePieceTokenizer)
225 - self._special_token_policy = (
226 - SpecialTokenPolicy.IGNORE if self.is_tekken else None
227 - )
173 + self.transformers_tokenizer = tokenizer
174 + self.mistral = tokenizer.tokenizer
175 + self.instruct = self.mistral.instruct_tokenizer
176 + self.tokenizer = self.instruct.tokenizer
177 +
178 + _mistral_version_str = str(self.tokenizer.version.value)
179 + self.version: int = int(_mistral_version_str.split("v")[-1])
180 +
181 + self.is_tekken = isinstance(self.tokenizer, Tekkenizer)
182 + self.is_spm = isinstance(self.tokenizer, SentencePieceTokenizer)
183 if not (self.is_tekken or self.is_spm):
229 - raise TypeError(f"Unsupported tokenizer: {type(tokenizer_)}")
230 -
231 - self._vocab = tokenizer_.vocab()
232 - # Convert to a dict[str, int] to match protocol, but this is a lossy
233 - # conversion. There may be multiple token ids that decode to the same
234 - # string due to partial UTF-8 byte sequences being converted to �
235 - self._vocab_dict = {token: idx for idx, token in enumerate(self._vocab)}
236 - self.tokenizer = tokenizer_
184 + raise TypeError(f"Unsupported tokenizer: {type(self.tokenizer)}")
185 +
186 + # Reverse order to ensure that the lowest token id is kept.
187 + self._vocab_dict = {
188 + self.convert_ids_to_tokens([i], skip_special_tokens=False)[0]: i
189 + for i in range(self.vocab_size - 1, -1, -1)
190 + }
191 + # Sort the dict for convenience
192 + self._vocab_dict = dict(sorted(self._vocab_dict.items(), key=lambda x: x[1]))
193 +
194 + # Vocab sorted by token id.
195 + self._vocab = self.tokenizer._vocab
196 self._max_token_id = self.vocab_size - 1
197
198 @classmethod
199 def from_pretrained(
200 cls, path_or_repo_id: str, *, revision: Optional[str] = None
201 ) -> "MistralTokenizer":
243 - if not Path(path_or_repo_id).exists():
244 - assert len(path_or_repo_id.split("/")) == 2, (
245 - "You have either provided a non-existent path: "
246 - "{path_or_repo_id} or an invalid HF Hub repo id."
247 - )
248 - tokenizer_file = cls._download_mistral_tokenizer_from_hf(
249 - path_or_repo_id, revision
250 - )
251 - elif Path(path_or_repo_id).is_dir():
252 - tokenizer_file_name = find_tokenizer_file(os.listdir(path_or_repo_id))
253 - tokenizer_file = str(Path(path_or_repo_id) / tokenizer_file_name)
254 - else:
255 - assert Path(path_or_repo_id).is_file(), f"Invalid path: {path_or_repo_id}"
256 - tokenizer_file = str(Path(path_or_repo_id))
257 -
258 - from mistral_common.tokens.tokenizers.mistral import (
259 - MistralTokenizer as PublicMistralTokenizer,
202 + from transformers.tokenization_mistral_common import (
203 + MistralCommonTokenizer as TransformersMistralTokenizer,
204 )
205
262 - mistral_tokenizer = PublicMistralTokenizer.from_file(tokenizer_file)
263 - return cls(mistral_tokenizer)
264 -
265 - @staticmethod
266 - def _download_mistral_tokenizer_from_hf(
267 - tokenizer_name: str, revision: Optional[str]
268 - ) -> str:
269 - try:
270 - hf_api = HfApi()
271 - files = hf_api.list_repo_files(repo_id=tokenizer_name, revision=revision)
272 - except ConnectionError as exc:
273 - files = list_local_repo_files(repo_id=tokenizer_name, revision=revision)
274 -
275 - if len(files) == 0:
276 - raise exc
277 -
278 - filename = find_tokenizer_file(files)
279 -
280 - tokenizer_file = hf_hub_download(
281 - tokenizer_name, filename=filename, revision=revision
206 + str_revision = "main" if revision is None else revision
207 + return cls(
208 + TransformersMistralTokenizer.from_pretrained(
209 + path_or_repo_id, revision=str_revision
210 + )
211 )
283 - return tokenizer_file
212
213 # the following attributes are set to fit vLLM's design and are used
214 # by the structured output backends.
215 @property
216 def all_special_tokens_extended(self) -> list[str]:
289 - from mistral_common.tokens.tokenizers.base import SpecialTokens
290 -
291 - # tekken defines its own extended special tokens list
292 - if hasattr(self.tokenizer, "SPECIAL_TOKENS"):
293 - special_tokens = self.tokenizer.SPECIAL_TOKENS
294 - else:
295 - special_tokens = list(SpecialTokens)
296 - return [s.value if isinstance(s, SpecialTokens) else s for s in special_tokens]
217 + return self.all_special_tokens
218
219 @property
220 def all_special_tokens(self) -> list[str]:
300 - return self.all_special_tokens_extended
221 + from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy
222 +
223 + return [
224 + self.tokenizer.decode([i], special_token_policy=SpecialTokenPolicy.KEEP)
225 + for i in self.all_special_ids
226 + ]
227
228 @property
229 def all_special_ids(self) -> list[int]:
304 - return [self.all_special_tokens.index(t) for t in self.all_special_tokens]
230 + from mistral_common.tokens.tokenizers.sentencepiece import (
231 + SentencePieceTokenizer,
232 + )
233 + from mistral_common.tokens.tokenizers.tekken import Tekkenizer
234 +
235 + if self.is_tekken:
236 + assert isinstance(self.tokenizer, Tekkenizer), type(self.tokenizer)
237 + special_ids = {t["rank"] for t in self.tokenizer._all_special_tokens}
238 + elif self.is_spm:
239 + assert isinstance(self.tokenizer, SentencePieceTokenizer), type(
240 + self.tokenizer
241 + )
242 + special_ids = self.tokenizer._control_tokens
243 + else:
244 + raise ValueError(f"Unknown tokenizer type: {type(self.tokenizer)}")
245 + return sorted(special_ids)
246
247 @property
248 def bos_token_id(self) -> int:
@@ -317,7 +258,7 @@ class MistralTokenizer(TokenizerBase):
258
259 @property
260 def pad_token(self) -> str:
320 - raise NotImplementedError()
261 + return self.transformers_tokenizer.pad_token
262
263 @property
264 def is_fast(self) -> bool:
@@ -325,7 +266,7 @@ class MistralTokenizer(TokenizerBase):
266
267 @property
268 def vocab_size(self) -> int:
328 - return len(self._vocab)
269 + return self.transformers_tokenizer.vocab_size
270
271 @property
272 def max_token_id(self) -> int:
@@ -335,6 +276,23 @@ class MistralTokenizer(TokenizerBase):
276 def truncation_side(self) -> str:
277 raise NotImplementedError()
278
279 + def _is_special_token_id(self, token_id: int) -> bool:
280 + from mistral_common.tokens.tokenizers.sentencepiece import (
281 + SentencePieceTokenizer,
282 + )
283 + from mistral_common.tokens.tokenizers.tekken import Tekkenizer
284 +
285 + if self.is_spm:
286 + assert isinstance(self.tokenizer, SentencePieceTokenizer), type(
287 + self.tokenizer
288 + )
289 + return token_id in self.tokenizer._control_tokens
290 + if self.is_tekken:
291 + assert isinstance(self.tokenizer, Tekkenizer), type(self.tokenizer)
292 + return token_id < self.tokenizer.num_special_tokens
293 + else:
294 + raise ValueError(f"Unknown tokenizer type: {type(self.tokenizer)}")
295 +
296 def __len__(self) -> int:
297 return self.vocab_size
298
@@ -346,25 +304,19 @@ class MistralTokenizer(TokenizerBase):
304 truncation: bool = False,
305 max_length: Optional[int] = None,
306 ):
349 - input_ids: Union[list[int], list[list[int]]]
350 - # For list[str], original prompt text
351 - if is_list_of(text, str):
352 - input_ids_: list[list[int]] = []
353 - for p in text:
354 - each_input_ids = self.encode_one(p, truncation, max_length)
355 - input_ids_.append(each_input_ids)
356 - input_ids = input_ids_
357 - # For list[int], apply chat template output, already tokens.
358 - elif is_list_of(text, int):
359 - input_ids = text
360 - # For str, single prompt text
361 - else:
362 - input_ids = self.encode_one(text, truncation, max_length)
363 - return BatchEncoding({"input_ids": input_ids})
307 + return self.transformers_tokenizer(
308 + text=text,
309 + text_pair=text_pair,
310 + add_special_tokens=add_special_tokens,
311 + truncation=truncation,
312 + max_length=max_length,
313 + )
314 +
315 + @property
316 + def vocab(self) -> list[str]:
317 + return self._vocab
318
319 def get_vocab(self) -> dict[str, int]:
366 - # NB: the dictionary form of the vocabulary collapses token ids that map
367 - # to the same string but have different bytes
320 return self._vocab_dict
321
322 def get_added_vocab(self) -> dict[str, int]:
@@ -378,11 +330,9 @@ class MistralTokenizer(TokenizerBase):
330 max_length: Optional[int] = None,
331 ) -> list[int]:
332 # Mistral Tokenizers should not add special tokens
381 - input_ids = self.encode(text)
382 -
383 - if truncation:
384 - input_ids = input_ids[:max_length]
385 - return input_ids
333 + return self.transformers_tokenizer.encode(
334 + text, add_special_tokens=False, truncation=truncation, max_length=max_length
335 + )
336
337 def encode(
338 self,
@@ -391,15 +341,20 @@ class MistralTokenizer(TokenizerBase):
341 max_length: Optional[int] = None,
342 add_special_tokens: Optional[bool] = None,
343 ) -> list[int]:
394 - # `encode` should only be used for prompt completion
395 - # it should never be used for chat_completion.
396 - # For chat completion use `apply_chat_template`
344 if add_special_tokens is not None:
398 - return self.tokenizer.encode(
399 - text, bos=add_special_tokens, eos=add_special_tokens
345 + return self.transformers_tokenizer.encode(
346 + text,
347 + truncation=truncation,
348 + max_length=max_length,
349 + add_special_tokens=add_special_tokens,
350 )
351 else:
402 - return self.tokenizer.encode(text, bos=True, eos=False)
352 + encoded = self.tokenizer.encode(text, bos=True, eos=False)
353 +
354 + if truncation is not False and max_length is not None:
355 + return encoded[:max_length]
356 + else:
357 + return encoded
358
359 def apply_chat_template(
360 self,
@@ -407,59 +362,79 @@ class MistralTokenizer(TokenizerBase):
362 tools: Optional[list[dict[str, Any]]] = None,
363 **kwargs,
364 ) -> list[int]:
410 - request = make_mistral_chat_completion_request(messages, tools)
411 - encoded = self.mistral.encode_chat_completion(request)
365 + add_generation_prompt = kwargs.pop("add_generation_prompt", False)
366 + continue_final_message = kwargs.get("continue_final_message", False)
367 + padding = kwargs.get("padding", False)
368 + truncation = kwargs.get("truncation", False)
369 + max_length = kwargs.get("max_length")
370 +
371 + messages, tools = _prepare_apply_chat_template_tools_and_messages(
372 + messages, tools, continue_final_message, add_generation_prompt
373 + )
374
413 - # encode-decode to get clean prompt
414 - return encoded.tokens
375 + return self.transformers_tokenizer.apply_chat_template(
376 + conversation=messages,
377 + tools=tools,
378 + continue_final_message=continue_final_message,
379 + tokenize=True,
380 + padding=padding,
381 + truncation=truncation,
382 + max_length=max_length,
383 + return_tensors=None,
384 + return_dict=False,
385 + )
386 +
387 + def decode(
388 + self, ids: Union[list[int], int], skip_special_tokens: bool = True
389 + ) -> str:
390 + return self.transformers_tokenizer.decode(
391 + ids, skip_special_tokens=skip_special_tokens
392 + )
393
394 def convert_tokens_to_string(self, tokens: list[str]) -> str:
417 - from mistral_common.tokens.tokenizers.base import SpecialTokens
395 + from mistral_common.tokens.tokenizers.base import (
396 + SpecialTokenPolicy,
397 + SpecialTokens,
398 + )
399 + from mistral_common.tokens.tokenizers.sentencepiece import (
400 + SentencePieceTokenizer,
401 + )
402 + from mistral_common.tokens.tokenizers.tekken import Tekkenizer
403
404 + to_decode_special_tokens = {SpecialTokens.tool_calls}
405 if self.is_tekken:
406 + assert isinstance(self.tokenizer, Tekkenizer), type(self.tokenizer)
407 tokens = [
408 t
409 for t in tokens
423 - if (
424 - t is SpecialTokens.tool_calls
425 - or t not in self.tokenizer._all_special_tokens
426 - )
410 + if (t in to_decode_special_tokens or t not in self.all_special_tokens)
411 ]
412
413 if any(isinstance(t, bytes) for t in tokens):
414 # we need to encode and decode all tokens again
431 - shift = self.tokenizer.num_special_tokens
432 -
433 - def _token_to_id(t: str):
434 - t_bytes = t.encode("utf-8") if not isinstance(t, bytes) else t
435 - try:
436 - return (
437 - shift + self.tokenizer._tekken_token2id_nospecial[t_bytes]
438 - )
439 - except KeyError:
440 - logger.warning(
441 - "Failed to convert token %s to id, replacing with <unk>",
442 - t_bytes,
443 - )
444 - return self.tokenizer.unk_id
445 -
446 - ids = [_token_to_id(t) for t in tokens]
447 - decoded = self.tokenizer.decode(ids, self._special_token_policy)
415 + ids = [_tekken_token_to_id(self.tokenizer, t) for t in tokens]
416 + # We filtered unwanted special tokens before
417 + # so we can decode the rest.
418 + decoded = self.tokenizer.decode(ids, SpecialTokenPolicy.KEEP)
419 else:
420 decoded = "".join(tokens)
421 else:
422 # make sure certain special tokens like Tool calls are
423 # not decoded
453 - special_tokens = {SpecialTokens.tool_calls}
424 + assert isinstance(self.tokenizer, SentencePieceTokenizer), type(
425 + self.tokenizer
426 + )
427 +
428 regular_tokens: list[str] = []
455 - decoded_list = []
429 + decoded_list: list[str] = []
430 + decoded = ""
431
432 for token in tokens:
458 - if token in special_tokens:
433 + if token in to_decode_special_tokens:
434 if regular_tokens:
435 decoded_list.append(
436 self.tokenizer.decode(
462 - regular_tokens, self._special_token_policy
437 + regular_tokens, SpecialTokenPolicy.IGNORE
438 )
439 )
440 regular_tokens = []
@@ -469,66 +444,56 @@ class MistralTokenizer(TokenizerBase):
444
445 if regular_tokens:
446 decoded_list.append(
472 - self.tokenizer.decode(regular_tokens, self._special_token_policy)
447 + self.tokenizer.decode(regular_tokens, SpecialTokenPolicy.IGNORE)
448 )
474 -
449 decoded = "".join(decoded_list)
450
451 return decoded
452
479 - def decode(
480 - self, ids: Union[list[int], int], skip_special_tokens: bool = True
481 - ) -> str:
482 - assert skip_special_tokens, (
483 - "skip_special_tokens=False is not supported for Mistral tokenizers."
484 - )
485 -
486 - if isinstance(ids, int):
487 - ids = [ids]
488 - return self.tokenizer.decode(ids, self._special_token_policy)
489 -
453 def convert_ids_to_tokens(
454 self,
455 ids: list[int],
456 skip_special_tokens: bool = True,
457 ) -> list[str]:
495 - from mistral_common.tokens.tokenizers.base import SpecialTokens
458 + from mistral_common.tokens.tokenizers.base import (
459 + SpecialTokenPolicy,
460 + SpecialTokens,
461 + )
462 from mistral_common.tokens.tokenizers.instruct import InstructTokenizerV13
463
498 - # TODO(Patrick) - potentially allow special tokens to not be skipped
499 - assert skip_special_tokens, (
500 - "skip_special_tokens=False is not supported for Mistral tokenizers."
501 - )
464 + if not skip_special_tokens:
465 + return [self.tokenizer.id_to_piece(token_id) for token_id in ids]
466
503 - assert self.is_tekken or self.is_spm, type(self.tokenizer)
467 + non_skip_special_tokens_ids = {
468 + self.tokenizer.get_control_token(SpecialTokens.tool_calls),
469 + }
470 + if isinstance(self.instruct, InstructTokenizerV13):
471 + if self.instruct.BEGIN_THINK:
472 + non_skip_special_tokens_ids.add(self.instruct.BEGIN_THINK)
473 + if self.instruct.END_THINK:
474 + non_skip_special_tokens_ids.add(self.instruct.END_THINK)
475
505 - if self.is_tekken:
506 - # skip special tokens except tool call and think tokens
507 - non_skip_special_tokens = {
508 - self.tokenizer.get_control_token(SpecialTokens.tool_calls)
509 - }
510 - if isinstance(self.instruct, InstructTokenizerV13):
511 - if self.instruct.BEGIN_THINK:
512 - non_skip_special_tokens.add(self.instruct.BEGIN_THINK)
513 - if self.instruct.END_THINK:
514 - non_skip_special_tokens.add(self.instruct.END_THINK)
515 - ids = [
516 - i
517 - for i in ids
518 - if i > self.tokenizer.num_special_tokens or i in non_skip_special_tokens
519 - ]
476 + ids_kept = [
477 + i
478 + for i in ids
479 + if i in non_skip_special_tokens_ids or not self._is_special_token_id(i)
480 + ]
481
521 - tokens = [self.tokenizer.id_to_piece(id) for id in ids]
482 + # We filtered unwanted special tokens so we can decode the rest.
483 + tokens = [self.tokenizer.id_to_piece(token_id) for token_id in ids_kept]
484
485 if any("�" in t for t in tokens) and self.is_tekken:
486 # if a decoded token contains the replacement character, then the
487 # token has an incomplete UTF-8 character so we must use bytes
488 # See: https://github.com/vllm-project/vllm/pull/8640
489 # https://github.com/vllm-project/vllm/pull/9625
528 - # if underlying tokenizeir is sentencepiece, we just add "�"
490 + # if underlying tokenizer is sentencepiece, we just add "�".
491 + # We filtered unwanted special tokens so we can decode the rest.
492 tokens = [
530 - self.tokenizer.id_to_byte_piece(id, self._special_token_policy)
531 - for id in ids
493 + self.tokenizer.id_to_byte_piece(token_id, SpecialTokenPolicy.KEEP)
494 + if token_id not in self.all_special_ids
495 + else self.tokenizer.decode([token_id], SpecialTokenPolicy.KEEP)
496 + for token_id in ids_kept
497 ]
498
499 return tokens
vllm/v1/structured_output/backend_xgrammar.py
+6 -27
@@ -43,34 +43,13 @@ class XgrammarBackend(StructuredOutputBackend):
43 if isinstance(self.tokenizer, MistralTokenizer):
44 # NOTE: ideally, xgrammar should handle this accordingly.
45 # refer to https://github.com/mlc-ai/xgrammar/blob/d77c0a0173ef14779c918e3be7966ba852f7910f/python/xgrammar/tokenizer_info.py#L98
46 - try:
47 - if self.tokenizer.is_tekken:
48 - encoded_vocab = self.tokenizer._vocab
49 - else:
50 - encoded_vocab = [
51 - token
52 - for token, _ in sorted(
53 - self.tokenizer.get_vocab().items(),
54 - key=lambda x: x[1],
55 - )
56 - ]
57 - stop_token_ids = None
58 - if (
59 - hasattr(
60 - self.tokenizer,
61 - "eos_token_id",
62 - )
63 - and self.tokenizer.eos_token_id is not None
64 - ):
65 - stop_token_ids = [self.tokenizer.eos_token_id]
66 - except AttributeError as e:
67 - raise ValueError(
68 - f"Cannot get the vocabulary of the tokenizer "
69 - f"{type(self.tokenizer)}. The tokenizer should have a "
70 - "get_vocab method."
71 - ) from e
46 + stop_token_ids = [self.tokenizer.eos_token_id]
47 +
48 + # not self.tokenizer.vocab_size as self.tokenizer.vocab
49 + # collapses all decoded errors into a single token.
50 + self.vocab_size = len(self.tokenizer.vocab)
51 tokenizer_info = xgr.TokenizerInfo( # type: ignore
73 - encoded_vocab=encoded_vocab,
52 + encoded_vocab=self.tokenizer.vocab,
53 # NOTE: https://github.com/mlc-ai/xgrammar/blob/5e141f6ff1ca02bc31f9e512e68b61f2a8ae88e5/tests/python/test_tokenizer_info.py#L43 # noqa: E501
54 vocab_type=xgr.VocabType.RAW
55 if self.tokenizer.is_tekken