@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6d8d0a24c0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 23:45 UTC

Add think chunk (#21333)

Signed-off-by: Julien Denize <julien.denize@mistral.ai>

Julien Denize committed Jul 24, 2025 at 06:51 UTC 6d8d0a24c02bfd84d46b3016b865a44f048ae84b
11 files changed +682 -13
requirements/common.txt
+1 -1
@@ -33,7 +33,7 @@ pyzmq >= 25.0.0
33 msgspec
34 gguf >= 0.13.0
35 importlib_metadata; python_version < '3.10'
36 -mistral_common[opencv] >= 1.8.0
36 +mistral_common[image,audio] >= 1.8.2
37 opencv-python-headless >= 4.11.0 # required for video IO
38 pyyaml
39 six>=1.16.0; python_version > '3.11' # transitive dependency of pandas that needs to be the latest version for python 3.12
requirements/nightly_torch_test.txt
+1 -1
@@ -23,7 +23,7 @@ jiwer # required for audio tests
23 timm # required for internvl test
24 transformers_stream_generator # required for qwen-vl test
25 matplotlib # required for qwen-vl test
26 -mistral_common[opencv] >= 1.8.0 # required for voxtral test
26 +mistral_common[image,audio] >= 1.8.2 # required for voxtral test
27 num2words # required for smolvlm test
28 opencv-python-headless >= 4.11.0 # required for video test
29 datamodel_code_generator # required for minicpm3 test
requirements/test.in
+1 -1
@@ -28,7 +28,7 @@ torchvision==0.22.1
28 transformers_stream_generator # required for qwen-vl test
29 mamba_ssm # required for plamo2 test
30 matplotlib # required for qwen-vl test
31 -mistral_common[opencv] >= 1.8.0 # required for voxtral test
31 +mistral_common[image,audio] >= 1.8.2 # required for voxtral test
32 num2words # required for smolvlm test
33 open_clip_torch==2.32.0 # Required for nemotron_vl test
34 opencv-python-headless >= 4.11.0 # required for video test
requirements/test.txt
+5 -2
@@ -447,7 +447,7 @@ mbstrdecoder==1.1.3
447 # typepy
448 mdurl==0.1.2
449 # via markdown-it-py
450 -mistral-common==1.8.0
450 +mistral-common==1.8.2
451 # via -r requirements/test.in
452 mlflow==2.22.0
453 # via terratorch
@@ -999,8 +999,11 @@ soundfile==0.12.1
999 # via
1000 # -r requirements/test.in
1001 # librosa
1002 + # mistral-common
1003 soxr==0.5.0.post1
1003 - # via librosa
1004 + # via
1005 + # librosa
1006 + # mistral-common
1007 sqlalchemy==2.0.41
1008 # via
1009 # alembic
tests/entrypoints/test_chat_utils.py
+167
@@ -6,6 +6,10 @@ from collections.abc import Mapping
6 from typing import Literal, Optional
7
8 import pytest
9 +from mistral_common.tokens.tokenizers.base import (SpecialTokenPolicy,
10 + SpecialTokens)
11 +from mistral_common.tokens.tokenizers.tekken import (SpecialTokenInfo,
12 + Tekkenizer)
13
14 from vllm.assets.audio import AudioAsset
15 from vllm.assets.image import ImageAsset
@@ -21,6 +25,7 @@ from vllm.multimodal import MultiModalDataDict
25 from vllm.multimodal.utils import (encode_audio_base64, encode_image_base64,
26 encode_video_base64)
27 from vllm.transformers_utils.tokenizer_group import TokenizerGroup
28 +from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
29
30 from ..models.registry import HF_EXAMPLE_MODELS
31 from ..utils import VLLM_PATH
@@ -1374,3 +1379,165 @@ def test_resolve_content_format_examples(template_path, expected_format):
1379 )
1380
1381 assert resolved_format == expected_format
1382 +
1383 +
1384 +def test_parse_chat_messages_include_thinking_chunk(mistral_model_config,
1385 + mistral_tokenizer):
1386 + messages = [{
1387 + "role":
1388 + "system",
1389 + "content": [{
1390 + "type": "text",
1391 + "text": "You are a helpful assistant."
1392 + }, {
1393 + "type":
1394 + "thinking",
1395 + "closed":
1396 + True,
1397 + "thinking":
1398 + "Only return the answer when you are confident."
1399 + }]
1400 + }, {
1401 + "role": "user",
1402 + "content": "What is 2+2?"
1403 + }, {
1404 + "role":
1405 + "assistant",
1406 + "content": [{
1407 + "type": "text",
1408 + "text": "Let me think about it."
1409 + }, {
1410 + "type": "thinking",
1411 + "closed": True,
1412 + "thinking": "2+2 = 4"
1413 + }, {
1414 + "type": "text",
1415 + "text": "The answer is 4.",
1416 + }],
1417 + }]
1418 +
1419 + conversation_with_thinking, _ = parse_chat_messages(
1420 + messages,
1421 + mistral_model_config,
1422 + mistral_tokenizer,
1423 + content_format="openai",
1424 + )
1425 +
1426 + expected_conversation = [{
1427 + "role":
1428 + "system",
1429 + "content": [{
1430 + "type": "text",
1431 + "text": "You are a helpful assistant."
1432 + }, {
1433 + "type": "text",
1434 + "text": "Only return the answer when you are confident."
1435 + }],
1436 + }, {
1437 + "role":
1438 + "user",
1439 + "content": [{
1440 + "type": "text",
1441 + "text": "What is 2+2?"
1442 + }],
1443 + }, {
1444 + "role":
1445 + "assistant",
1446 + "content": [
1447 + {
1448 + "type": "text",
1449 + "text": "Let me think about it."
1450 + },
1451 + {
1452 + "type": "text",
1453 + "text": "2+2 = 4"
1454 + },
1455 + {
1456 + "type": "text",
1457 + "text": "The answer is 4."
1458 + },
1459 + ]
1460 + }]
1461 +
1462 + assert conversation_with_thinking == expected_conversation
1463 +
1464 +
1465 +def test_apply_mistral_chat_template_thinking_chunk():
1466 + # Moved import here to avoid yapf and isort conflicts
1467 + from vllm.entrypoints.chat_utils import apply_mistral_chat_template
1468 + messages = [{
1469 + "role":
1470 + "system",
1471 + "content": [{
1472 + "type": "text",
1473 + "text": "You are a helpful assistant."
1474 + }, {
1475 + "type":
1476 + "thinking",
1477 + "closed":
1478 + True,
1479 + "thinking":
1480 + "Only return the answer when you are confident."
1481 + }]
1482 + }, {
1483 + "role": "user",
1484 + "content": "What is 2+2?"
1485 + }, {
1486 + "role":
1487 + "assistant",
1488 + "content": [{
1489 + "type": "text",
1490 + "text": "Let me think about it."
1491 + }, {
1492 + "type": "thinking",
1493 + "closed": True,
1494 + "thinking": "2+2 = 4"
1495 + }, {
1496 + "type": "text",
1497 + "text": "The answer is 4.",
1498 + }],
1499 + }, {
1500 + "role": "user",
1501 + "content": "Thanks, what is 3+3?"
1502 + }]
1503 +
1504 + # TODO(Julien): upon model release change to a tokenizer already configured.
1505 + # =================================================================
1506 + mistral_tokenizer = MistralTokenizer.from_pretrained(
1507 + "mistralai/Devstral-Small-2507")
1508 + assert isinstance(mistral_tokenizer.tokenizer, Tekkenizer)
1509 + # Add think special tokens to the tokenizer
1510 + mistral_tokenizer.tokenizer._all_special_tokens[35] = SpecialTokenInfo(
1511 + rank=35, is_control=True, token_str=SpecialTokens.begin_think.value)
1512 + mistral_tokenizer.tokenizer._all_special_tokens[36] = SpecialTokenInfo(
1513 + rank=36, is_control=True, token_str=SpecialTokens.end_think.value)
1514 + mistral_tokenizer.tokenizer._special_tokens_reverse_vocab = {
1515 + k: v
1516 + for k, v in
1517 + mistral_tokenizer.tokenizer._special_tokens_reverse_vocab.items()
1518 + if v not in {35, 36}
1519 + }
1520 + mistral_tokenizer.tokenizer._special_tokens_reverse_vocab[
1521 + SpecialTokens.begin_think.value] = 35
1522 + mistral_tokenizer.tokenizer._special_tokens_reverse_vocab[
1523 + SpecialTokens.end_think.value] = 36
1524 + mistral_tokenizer.instruct.BEGIN_THINK = 35
1525 + mistral_tokenizer.instruct.END_THINK = 36
1526 + # =================================================================
1527 +
1528 + tokens_ids = apply_mistral_chat_template(mistral_tokenizer,
1529 + messages,
1530 + chat_template=None,
1531 + tools=None)
1532 +
1533 + string_tokens = mistral_tokenizer.mistral.decode(
1534 + tokens_ids, special_token_policy=SpecialTokenPolicy.KEEP)
1535 +
1536 + expected_tokens = (
1537 + r"<s>[SYSTEM_PROMPT]You are a helpful assistant.[THINK]Only return the"
1538 + r" answer when you are confident.[/THINK][/SYSTEM_PROMPT]"
1539 + r"[INST]What is 2+2?[/INST]"
1540 + r"Let me think about it.[THINK]2+2 = 4[/THINK]The answer is 4.</s>"
1541 + r"[INST]Thanks, what is 3+3?[/INST]")
1542 +
1543 + assert string_tokens == expected_tokens
tests/reasoning/test_mistral_reasoning_parser.py new
+341
@@ -0,0 +1,341 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import pytest
5 +from mistral_common.tokens.tokenizers.base import SpecialTokens
6 +from mistral_common.tokens.tokenizers.tekken import (SpecialTokenInfo,
7 + Tekkenizer)
8 +
9 +from tests.reasoning.utils import run_reasoning_extraction_mistral
10 +from vllm.reasoning import ReasoningParser, ReasoningParserManager
11 +from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
12 +
13 +parser_name = "mistral"
14 +
15 +
16 +@pytest.fixture(scope="module")
17 +def mistral_tokenizer():
18 + # TODO(Julien): upon model release change to a tokenizer already configured.
19 + # =================================================================
20 + mistral_tokenizer = MistralTokenizer.from_pretrained(
21 + "mistralai/Devstral-Small-2507")
22 + assert isinstance(mistral_tokenizer.tokenizer, Tekkenizer)
23 + # Add think special tokens to the tokenizer
24 + mistral_tokenizer.tokenizer._all_special_tokens[35] = SpecialTokenInfo(
25 + rank=35, is_control=True, token_str=SpecialTokens.begin_think.value)
26 + mistral_tokenizer.tokenizer._all_special_tokens[36] = SpecialTokenInfo(
27 + rank=36, is_control=True, token_str=SpecialTokens.end_think.value)
28 + mistral_tokenizer.tokenizer._special_tokens_reverse_vocab = {
29 + k: v
30 + for k, v in
31 + mistral_tokenizer.tokenizer._special_tokens_reverse_vocab.items()
32 + if v not in {35, 36}
33 + }
34 + mistral_tokenizer.tokenizer._special_tokens_reverse_vocab[
35 + SpecialTokens.begin_think.value] = 35
36 + mistral_tokenizer.tokenizer._special_tokens_reverse_vocab[
37 + SpecialTokens.end_think.value] = 36
38 + mistral_tokenizer.instruct.BEGIN_THINK = 35
39 + mistral_tokenizer.instruct.END_THINK = 36
40 + # =================================================================
41 + return mistral_tokenizer
42 +
43 +
44 +SIMPLE_REASONING = {
45 + "output": "This is a reasoning section[/THINK]This is the rest",
46 + "reasoning_content": "This is a reasoning section",
47 + "content": "This is the rest",
48 + "is_reasoning_end": True,
49 +}
50 +COMPLETE_REASONING = {
51 + "output": "This is a reasoning section[/THINK]",
52 + "reasoning_content": "This is a reasoning section",
53 + "content": None,
54 + "is_reasoning_end": True,
55 +}
56 +NO_CONTENT = {
57 + "output": "This is content",
58 + "reasoning_content": "This is content",
59 + "content": None,
60 + "is_reasoning_end": False,
61 +}
62 +NO_REASONING_STREAMING = {
63 + "output": "This is a reasoning section",
64 + "reasoning_content": "This is a reasoning section",
65 + "content": None,
66 + "is_reasoning_end": False,
67 +}
68 +MULTIPLE_LINES = {
69 + "output": "This\nThat[/THINK]This is the rest\nThat",
70 + "reasoning_content": "This\nThat",
71 + "content": "This is the rest\nThat",
72 + "is_reasoning_end": True,
73 +}
74 +SHORTEST_REASONING_NO_STREAMING = {
75 + "output": "[/THINK]This is the rest",
76 + "reasoning_content": "",
77 + "content": "This is the rest",
78 + "is_reasoning_end": True,
79 +}
80 +SHORTEST_REASONING = {
81 + "output": "[/THINK]This is the rest",
82 + "reasoning_content": None,
83 + "content": "This is the rest",
84 + "is_reasoning_end": True,
85 +}
86 +REASONING_WITH_THINK = {
87 + "output": "[THINK]This is a reasoning section[/THINK]This is the rest",
88 + "reasoning_content": "This is a reasoning section",
89 + "content": "This is the rest",
90 + "is_reasoning_end": True,
91 +}
92 +COMPLETE_REASONING_WITH_THINK = {
93 + "output": "[THINK]This is a reasoning section[/THINK]",
94 + "reasoning_content": "This is a reasoning section",
95 + "content": None,
96 + "is_reasoning_end": True,
97 +}
98 +MULTIPLE_LINES_WITH_THINK = {
99 + "output": "[THINK]This\nThat[/THINK]This is the rest\nThat",
100 + "reasoning_content": "This\nThat",
101 + "content": "This is the rest\nThat",
102 + "is_reasoning_end": True,
103 +}
104 +SHORTEST_REASONING_NO_STREAMING_WITH_THINK = {
105 + "output": "[/THINK]This is the rest",
106 + "reasoning_content": "",
107 + "content": "This is the rest",
108 + "is_reasoning_end": True,
109 +}
110 +SHORTEST_REASONING_WITH_THINK = {
111 + "output": "[/THINK]This is the rest",
112 + "reasoning_content": None,
113 + "content": "This is the rest",
114 + "is_reasoning_end": True,
115 +}
116 +THINK_NO_END = {
117 + "output": "[THINK]This is a reasoning section",
118 + "reasoning_content": "This is a reasoning section",
119 + "content": None,
120 + "is_reasoning_end": False,
121 +}
122 +EMPTY = {
123 + "output": "",
124 + "reasoning_content": "",
125 + "content": None,
126 + "is_reasoning_end": False,
127 +}
128 +EMPTY_STREAMING = {
129 + "output": "",
130 + "reasoning_content": None,
131 + "content": None,
132 + "is_reasoning_end": False,
133 +}
134 +NEW_LINE = {
135 + "output": "\n[THINK]This is a reasoning section[/THINK]\nThis is the rest",
136 + "reasoning_content": "This is a reasoning section",
137 + "content": "\nThis is the rest",
138 + "is_reasoning_end": True,
139 +}
140 +# Streaming cannot handle new lines at the beginning of the output
141 +# because we need to support [THINK]...[/THINK] and [/THINK]...
142 +# We cannot know if the text before [THINK] is reasoning content
143 +# or not.
144 +NEW_LINE_STREAMING = {
145 + "output": "\n[THINK]This is a reasoning section[/THINK]\nThis is the rest",
146 + "reasoning_content": "\nThis is a reasoning section",
147 + "content": "\nThis is the rest",
148 + "is_reasoning_end": True,
149 +}
150 +
151 +TEST_CASES = [
152 + pytest.param(
153 + False,
154 + SIMPLE_REASONING,
155 + id="simple_reasoning",
156 + ),
157 + pytest.param(
158 + True,
159 + SIMPLE_REASONING,
160 + id="simple_reasoning_streaming",
161 + ),
162 + pytest.param(
163 + False,
164 + COMPLETE_REASONING,
165 + id="complete_reasoning",
166 + ),
167 + pytest.param(
168 + True,
169 + COMPLETE_REASONING,
170 + id="complete_reasoning_streaming",
171 + ),
172 + pytest.param(
173 + False,
174 + NO_CONTENT,
175 + id="no_content_token",
176 + ),
177 + pytest.param(
178 + True,
179 + NO_REASONING_STREAMING,
180 + id="no_reasoning_token_streaming",
181 + ),
182 + pytest.param(
183 + False,
184 + MULTIPLE_LINES,
185 + id="multiple_lines",
186 + ),
187 + pytest.param(
188 + True,
189 + MULTIPLE_LINES,
190 + id="multiple_lines_streaming",
191 + ),
192 + pytest.param(
193 + True,
194 + SHORTEST_REASONING,
195 + id="shortest",
196 + ),
197 + pytest.param(
198 + False,
199 + SHORTEST_REASONING_NO_STREAMING,
200 + id="shortest_streaming",
201 + ),
202 + pytest.param(
203 + False,
204 + REASONING_WITH_THINK,
205 + id="reasoning_with_think",
206 + ),
207 + pytest.param(
208 + True,
209 + REASONING_WITH_THINK,
210 + id="reasoning_with_think_streaming",
211 + ),
212 + pytest.param(
213 + False,
214 + COMPLETE_REASONING_WITH_THINK,
215 + id="complete_reasoning_with_think",
216 + ),
217 + pytest.param(
218 + True,
219 + COMPLETE_REASONING_WITH_THINK,
220 + id="complete_reasoning_with_think_streaming",
221 + ),
222 + pytest.param(
223 + False,
224 + MULTIPLE_LINES_WITH_THINK,
225 + id="multiple_lines_with_think",
226 + ),
227 + pytest.param(
228 + True,
229 + MULTIPLE_LINES_WITH_THINK,
230 + id="multiple_lines_with_think_streaming",
231 + ),
232 + pytest.param(
233 + False,
234 + SHORTEST_REASONING_NO_STREAMING_WITH_THINK,
235 + id="shortest_with_think",
236 + ),
237 + pytest.param(
238 + True,
239 + SHORTEST_REASONING_WITH_THINK,
240 + id="shortest_with_think_streaming",
241 + ),
242 + pytest.param(
243 + False,
244 + THINK_NO_END,
245 + id="think_no_end",
246 + ),
247 + pytest.param(
248 + True,
249 + THINK_NO_END,
250 + id="think_no_end_streaming",
251 + ),
252 + pytest.param(
253 + False,
254 + EMPTY,
255 + id="empty",
256 + ),
257 + pytest.param(
258 + True,
259 + EMPTY_STREAMING,
260 + id="empty_streaming",
261 + ),
262 + pytest.param(
263 + False,
264 + NEW_LINE,
265 + id="new_line",
266 + ),
267 + pytest.param(
268 + True,
269 + NEW_LINE_STREAMING,
270 + id="new_line_streaming",
271 + ),
272 +]
273 +
274 +
275 +@pytest.mark.parametrize("streaming, param_dict", TEST_CASES)
276 +def test_mistral_reasoning(
277 + streaming: bool,
278 + param_dict: dict,
279 + mistral_tokenizer: MistralTokenizer,
280 +):
281 + output = param_dict["output"]
282 +
283 + index_think = output.find("[THINK]")
284 + len_think = len("[THINK]")
285 + index_end_think = output.find("[/THINK]")
286 + len_end_think = len("[/THINK]")
287 +
288 + # encode everything to tokens ids
289 + output_tokens = []
290 + if index_think != -1:
291 + output_before_think = output[:index_think]
292 + output_tokens += mistral_tokenizer.tokenizer.encode(
293 + output_before_think, False, False)
294 + output_tokens += [mistral_tokenizer.instruct.BEGIN_THINK]
295 +
296 + if index_end_think != -1:
297 + output_middle = output[index_think + len_think:index_end_think]
298 + output_after_think = output[index_end_think + len_end_think:]
299 + output_tokens += mistral_tokenizer.tokenizer.encode(
300 + output_middle, False, False)
301 + output_tokens += [mistral_tokenizer.instruct.END_THINK]
302 + output_tokens += mistral_tokenizer.tokenizer.encode(
303 + output_after_think, False, False)
304 + else:
305 + output_middle = output[index_think + len_think:]
306 + output_tokens += mistral_tokenizer.tokenizer.encode(
307 + output_middle, False, False)
308 + elif index_end_think != -1:
309 + output_before_think = output[:index_end_think]
310 + output_after_think = output[index_end_think + len_end_think:]
311 + output_tokens += mistral_tokenizer.tokenizer.encode(
312 + output_before_think, False, False)
313 + output_tokens += [mistral_tokenizer.instruct.END_THINK]
314 + output_tokens += mistral_tokenizer.tokenizer.encode(
315 + output_after_think, False, False)
316 + else:
317 + output_tokens += mistral_tokenizer.tokenizer.encode(
318 + output, False, False)
319 +
320 + parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(
321 + parser_name)(mistral_tokenizer)
322 +
323 + reasoning, content = run_reasoning_extraction_mistral(parser,
324 + output_tokens,
325 + streaming=streaming)
326 +
327 + assert reasoning == param_dict["reasoning_content"]
328 + assert content == param_dict["content"]
329 +
330 + # Test is_reasoning_end
331 + is_reasoning_end = parser.is_reasoning_end(output_tokens)
332 + assert is_reasoning_end == param_dict["is_reasoning_end"]
333 +
334 + # Test extract_content
335 + if param_dict["content"] is not None:
336 + content = parser.extract_content_ids(output_tokens)
337 + assert content == mistral_tokenizer.tokenizer.encode(
338 + param_dict["content"], bos=False, eos=False)
339 + else:
340 + content = parser.extract_content_ids(output_tokens)
341 + assert content == []
tests/reasoning/utils.py
+59
@@ -6,6 +6,7 @@ from typing import Optional, Union
6 from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
7 DeltaMessage)
8 from vllm.reasoning import ReasoningParser
9 +from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
10
11
12 class StreamingReasoningReconstructor:
@@ -54,6 +55,32 @@ def run_reasoning_extraction(
55 return reasoning, content
56
57
58 +def run_reasoning_extraction_mistral(
59 + reasoning_parser: ReasoningParser,
60 + model_output: list[int],
61 + request: Union[ChatCompletionRequest, None] = None,
62 + streaming: bool = False,
63 +) -> tuple[Optional[str], Optional[str]]:
64 + assert isinstance(reasoning_parser.model_tokenizer,
65 + MistralTokenizer), type(reasoning_parser.model_tokenizer)
66 + if streaming:
67 + reconstructor = run_reasoning_extraction_streaming_mistral(
68 + reasoning_parser,
69 + model_output,
70 + request,
71 + )
72 + return (
73 + reconstructor.reasoning_content,
74 + reconstructor.other_content or None,
75 + )
76 + else:
77 + str_output = reasoning_parser.model_tokenizer.convert_ids_to_tokens(
78 + model_output)
79 + reasoning, content = run_reasoning_extraction_nonstreaming(
80 + reasoning_parser, str_output, request)
81 + return reasoning, content
82 +
83 +
84 def run_reasoning_extraction_nonstreaming(
85 reasoning_parser: ReasoningParser,
86 model_output: list[str],
@@ -94,3 +121,35 @@ def run_reasoning_extraction_streaming(
121 previous_text = current_text
122 previous_tokens = current_tokens
123 return reconstructor
124 +
125 +
126 +def run_reasoning_extraction_streaming_mistral(
127 + reasoning_parser: ReasoningParser,
128 + model_deltas: list[int],
129 + request: Union[ChatCompletionRequest, None] = None,
130 +) -> StreamingReasoningReconstructor:
131 + assert isinstance(reasoning_parser.model_tokenizer,
132 + MistralTokenizer), type(reasoning_parser.model_tokenizer)
133 + request = request or ChatCompletionRequest(messages=[], model="test-model")
134 + reconstructor = StreamingReasoningReconstructor()
135 + previous_text = ""
136 + previous_tokens: list[int] = []
137 + for model_delta in model_deltas:
138 + token_delta = [model_delta]
139 + delta = reasoning_parser.model_tokenizer.convert_ids_to_tokens(
140 + [model_delta])[0]
141 + current_text = previous_text + delta
142 + current_tokens = previous_tokens + token_delta
143 + delta_message = reasoning_parser.extract_reasoning_content_streaming(
144 + previous_text,
145 + current_text,
146 + delta,
147 + previous_tokens,
148 + current_tokens,
149 + token_delta,
150 + )
151 + if delta_message is not None:
152 + reconstructor.append_delta(delta_message)
153 + previous_text = current_text
154 + previous_tokens = current_tokens
155 + return reconstructor
vllm/entrypoints/chat_utils.py
+27 -2
@@ -151,6 +151,27 @@ class CustomChatCompletionContentSimpleVideoParam(TypedDict, total=False):
151 video_url: Required[str]
152
153
154 +class CustomThinkCompletionContentParam(TypedDict, total=False):
155 + """A Think Completion Content Param that accepts a plain text and a boolean.
156 +
157 + Example:
158 + {
159 + "thinking": "I am thinking about the answer",
160 + "closed": True,
161 + "type": "thinking"
162 + }
163 + """
164 +
165 + thinking: Required[str]
166 + """The thinking content."""
167 +
168 + closed: bool
169 + """Whether the thinking is closed."""
170 +
171 + type: Required[Literal["thinking"]]
172 + """The thinking type."""
173 +
174 +
175 ChatCompletionContentPartParam: TypeAlias = Union[
176 OpenAIChatCompletionContentPartParam, ChatCompletionContentPartAudioParam,
177 ChatCompletionContentPartInputAudioParam,
@@ -159,7 +180,8 @@ ChatCompletionContentPartParam: TypeAlias = Union[
180 CustomChatCompletionContentSimpleImageParam,
181 ChatCompletionContentPartImageEmbedsParam,
182 CustomChatCompletionContentSimpleAudioParam,
162 - CustomChatCompletionContentSimpleVideoParam, str]
183 + CustomChatCompletionContentSimpleVideoParam, str,
184 + CustomThinkCompletionContentParam]
185
186
187 class CustomChatCompletionMessageParam(TypedDict, total=False):
@@ -938,6 +960,7 @@ _ImageEmbedsParser = partial(cast, ChatCompletionContentPartImageEmbedsParam)
960 _InputAudioParser = partial(cast, ChatCompletionContentPartInputAudioParam)
961 _RefusalParser = partial(cast, ChatCompletionContentPartRefusalParam)
962 _PILImageParser = partial(cast, CustomChatCompletionContentPILImageParam)
963 +_ThinkParser = partial(cast, CustomThinkCompletionContentParam)
964 # Need to validate url objects
965 _ImageParser = TypeAdapter(ChatCompletionContentPartImageParam).validate_python
966 _AudioParser = TypeAdapter(ChatCompletionContentPartAudioParam).validate_python
@@ -954,6 +977,8 @@ MM_PARSER_MAP: dict[
977 ] = {
978 "text":
979 lambda part: _TextParser(part).get("text", None),
980 + "thinking":
981 + lambda part: _ThinkParser(part).get("thinking", None),
982 "input_text":
983 lambda part: _TextParser(part).get("text", None),
984 "input_image":
@@ -1100,7 +1125,7 @@ def _parse_chat_message_content_part(
1125 "with empty / unparsable content.", part, part_type)
1126 return None
1127
1103 - if part_type in ("text", "input_text", "refusal"):
1128 + if part_type in ("text", "input_text", "refusal", "thinking"):
1129 str_content = cast(str, content)
1130 if wrap_dicts:
1131 return {'type': 'text', 'text': str_content}
vllm/reasoning/__init__.py
+2
@@ -6,6 +6,7 @@ from .deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParser
6 from .glm4_moe_reasoning_parser import Glm4MoeModelReasoningParser
7 from .granite_reasoning_parser import GraniteReasoningParser
8 from .hunyuan_a13b_reasoning_parser import HunyuanA13BReasoningParser
9 +from .mistral_reasoning_parser import MistralReasoningParser
10 from .qwen3_reasoning_parser import Qwen3ReasoningParser
11
12 __all__ = [
@@ -16,4 +17,5 @@ __all__ = [
17 "HunyuanA13BReasoningParser",
18 "Qwen3ReasoningParser",
19 "Glm4MoeModelReasoningParser",
20 + "MistralReasoningParser",
21 ]
vllm/reasoning/mistral_reasoning_parser.py new
+47
@@ -0,0 +1,47 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from vllm.logger import init_logger
5 +from vllm.reasoning import ReasoningParser, ReasoningParserManager
6 +from vllm.reasoning.deepseek_r1_reasoning_parser import (
7 + DeepSeekR1ReasoningParser)
8 +from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
9 +
10 +logger = init_logger(__name__)
11 +
12 +
13 +@ReasoningParserManager.register_module("mistral")
14 +class MistralReasoningParser(DeepSeekR1ReasoningParser):
15 + """
16 + Reasoning parser for Mistral models.
17 +
18 + The Mistral models uses [THINK]...[/THINK] tokens to denote reasoning
19 + text. This parser extracts the reasoning content from the model output.
20 + """
21 +
22 + def __init__(self, tokenizer: MistralTokenizer):
23 + if not isinstance(tokenizer, MistralTokenizer):
24 + raise ValueError(
25 + "The tokenizer must be an instance of MistralTokenizer.")
26 +
27 + ReasoningParser.__init__(self, tokenizer)
28 +
29 + if not self.model_tokenizer:
30 + raise ValueError(
31 + "The model tokenizer must be passed to the ReasoningParser "
32 + "constructor during construction.")
33 +
34 + from mistral_common.tokens.tokenizers.base import SpecialTokens
35 +
36 + self.start_token = SpecialTokens.begin_think
37 + self.end_token = SpecialTokens.end_think
38 +
39 + self.start_token_id = tokenizer.tokenizer.get_control_token(
40 + self.start_token)
41 + self.end_token_id = tokenizer.tokenizer.get_control_token(
42 + self.end_token)
43 +
44 + if self.start_token_id is None or self.end_token_id is None:
45 + raise RuntimeError(
46 + "Mistral reasoning parser could not locate think start/end "
47 + "tokens in the tokenizer!")
vllm/transformers_utils/tokenizers/mistral.py
+31 -6
@@ -145,6 +145,21 @@ def find_tokenizer_file(files: list[str]):
145 return matched_files[0]
146
147
148 +def _aggregate_content(content: list) -> list[dict[str, Any]]:
149 + aggregated_content: list[dict[str, Any]] = []
150 + for chunk in content:
151 + if chunk.get("type"
152 + ) == "text" and aggregated_content and aggregated_content[
153 + -1].get("type") == "text":
154 + aggregated_content[-1]["text"] += "\n\n" + chunk.get("text")
155 + else:
156 + aggregated_content.append(chunk)
157 + if len(aggregated_content) == 1 and aggregated_content[0].get(
158 + "type") == "text":
159 + content = aggregated_content[0]["text"]
160 + return content
161 +
162 +
163 def make_mistral_chat_completion_request(
164 messages: list["ChatCompletionMessageParam"],
165 tools: Optional[list[dict[str,
@@ -162,10 +177,10 @@ def make_mistral_chat_completion_request(
177
178 # Convert list text content to string
179 if message.get("role") in ("assistant", "tool"):
165 - content = message.get("content")
180 + content: Any = message.get("content")
181 if isinstance(content, list):
167 - content = "\n".join(chunk.get("text") for chunk in content)
168 - message["content"] = content
182 + content = _aggregate_content(content)
183 + message["content"] = content
184
185 # The Mistral client, in comparison to the OpenAI client, requires the
186 # "parameters" dict to be present, even if it's empty.
@@ -465,6 +480,8 @@ class MistralTokenizer(TokenizerBase):
480 skip_special_tokens: bool = True,
481 ) -> list[str]:
482 from mistral_common.tokens.tokenizers.base import SpecialTokens
483 + from mistral_common.tokens.tokenizers.instruct import (
484 + InstructTokenizerV13)
485
486 # TODO(Patrick) - potentially allow special tokens to not be skipped
487 assert (
@@ -474,10 +491,18 @@ class MistralTokenizer(TokenizerBase):
491 assert self.is_tekken or self.is_spm, type(self.tokenizer)
492
493 if self.is_tekken:
477 - # skip special tokens except tool call
478 - ids = [
479 - i for i in ids if i > self.tokenizer.num_special_tokens or i ==
494 + # skip special tokens except tool call and think tokens
495 + non_skip_special_tokens = {
496 self.tokenizer.get_control_token(SpecialTokens.tool_calls)
497 + }
498 + if isinstance(self.instruct, InstructTokenizerV13):
499 + if self.instruct.BEGIN_THINK:
500 + non_skip_special_tokens.add(self.instruct.BEGIN_THINK)
501 + if self.instruct.END_THINK:
502 + non_skip_special_tokens.add(self.instruct.END_THINK)
503 + ids = [
504 + i for i in ids if i > self.tokenizer.num_special_tokens
505 + or i in non_skip_special_tokens
506 ]
507
508 tokens = [self.tokenizer.id_to_piece(id) for id in ids]