@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / aa3c250c48
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 11, 15:00 UTC

[IMPROVEMENT] Change MistralReasoningParser behavior (#30391)

Signed-off-by: juliendenize <julien.denize@mistral.ai> Signed-off-by: Julien Denize <40604584+juliendenize@users.noreply.github.com> Co-authored-by: Patrick von Platen <patrick.v.platen@gmail.com>

Julien Denize committed Dec 11, 2025 at 17:53 UTC aa3c250c487e843b229a58d9978b02707b71109c
2 files changed +186 -64
tests/reasoning/test_mistral_reasoning_parser.py
+84 -61
@@ -18,47 +18,53 @@ def mistral_tokenizer():
18 return mistral_tokenizer
19
20
21 -SIMPLE_REASONING = {
21 +INVALID_SIMPLE_REASONING = {
22 "output": "This is a reasoning section[/THINK]This is the rest",
23 - "reasoning": "This is a reasoning section",
24 - "content": "This is the rest",
25 - "is_reasoning_end": True,
23 + "reasoning": None,
24 + "content": "This is a reasoning sectionThis is the rest",
25 + "is_reasoning_end": False,
26 }
27 -COMPLETE_REASONING = {
27 +INVALID_COMPLETE_REASONING = {
28 "output": "This is a reasoning section[/THINK]",
29 - "reasoning": "This is a reasoning section",
30 - "content": None,
31 - "is_reasoning_end": True,
29 + "reasoning": None,
30 + "content": "This is a reasoning section",
31 + "is_reasoning_end": False,
32 }
33 NO_CONTENT = {
34 - "output": "This is content",
35 - "reasoning": "This is content",
34 + "output": "[THINK]This is reasoning",
35 + "reasoning": "This is reasoning",
36 "content": None,
37 "is_reasoning_end": False,
38 }
39 +NO_REASONING = {
40 + "output": "This is content",
41 + "reasoning": None,
42 + "content": "This is content",
43 + "is_reasoning_end": False,
44 +}
45 NO_REASONING_STREAMING = {
46 "output": "This is a reasoning section",
41 - "reasoning": "This is a reasoning section",
42 - "content": None,
47 + "reasoning": None,
48 + "content": "This is a reasoning section",
49 "is_reasoning_end": False,
50 }
45 -MULTIPLE_LINES = {
51 +INVALID_MULTIPLE_LINES = {
52 "output": "This\nThat[/THINK]This is the rest\nThat",
47 - "reasoning": "This\nThat",
48 - "content": "This is the rest\nThat",
49 - "is_reasoning_end": True,
53 + "reasoning": None,
54 + "content": "This\nThatThis is the rest\nThat",
55 + "is_reasoning_end": False,
56 }
51 -SHORTEST_REASONING_NO_STREAMING = {
57 +INVALID_SHORTEST_REASONING_NO_STREAMING = {
58 "output": "[/THINK]This is the rest",
53 - "reasoning": "",
59 + "reasoning": None,
60 "content": "This is the rest",
55 - "is_reasoning_end": True,
61 + "is_reasoning_end": False,
62 }
57 -SHORTEST_REASONING = {
63 +INVALID_SHORTEST_REASONING = {
64 "output": "[/THINK]This is the rest",
65 "reasoning": None,
66 "content": "This is the rest",
61 - "is_reasoning_end": True,
67 + "is_reasoning_end": False,
68 }
69 REASONING_WITH_THINK = {
70 "output": "[THINK]This is a reasoning section[/THINK]This is the rest",
@@ -78,17 +84,17 @@ MULTIPLE_LINES_WITH_THINK = {
84 "content": "This is the rest\nThat",
85 "is_reasoning_end": True,
86 }
81 -SHORTEST_REASONING_NO_STREAMING_WITH_THINK = {
87 +INVALID_SHORTEST_REASONING_NO_STREAMING_WITH_THINK = {
88 "output": "[/THINK]This is the rest",
83 - "reasoning": "",
89 + "reasoning": None,
90 "content": "This is the rest",
85 - "is_reasoning_end": True,
91 + "is_reasoning_end": False,
92 }
87 -SHORTEST_REASONING_WITH_THINK = {
93 +INVALID_SHORTEST_REASONING_WITH_THINK = {
94 "output": "[/THINK]This is the rest",
95 "reasoning": None,
96 "content": "This is the rest",
91 - "is_reasoning_end": True,
97 + "is_reasoning_end": False,
98 }
99 THINK_NO_END = {
100 "output": "[THINK]This is a reasoning section",
@@ -98,8 +104,8 @@ THINK_NO_END = {
104 }
105 EMPTY = {
106 "output": "",
101 - "reasoning": "",
102 - "content": None,
107 + "reasoning": None,
108 + "content": "",
109 "is_reasoning_end": False,
110 }
111 EMPTY_STREAMING = {
@@ -109,47 +115,48 @@ EMPTY_STREAMING = {
115 "is_reasoning_end": False,
116 }
117 NEW_LINE = {
112 - "output": "\n[THINK]This is a reasoning section[/THINK]\nThis is the rest",
118 + "output": "Before\n[THINK]This is a reasoning section[/THINK]\nThis is the rest",
119 "reasoning": "This is a reasoning section",
114 - "content": "\nThis is the rest",
120 + "content": "Before\n\nThis is the rest",
121 "is_reasoning_end": True,
122 }
117 -# Streaming cannot handle new lines at the beginning of the output
118 -# because we need to support [THINK]...[/THINK] and [/THINK]...
119 -# We cannot know if the text before [THINK] is reasoning content
120 -# or not.
123 NEW_LINE_STREAMING = {
122 - "output": "\n[THINK]This is a reasoning section[/THINK]\nThis is the rest",
123 - "reasoning": "\nThis is a reasoning section",
124 - "content": "\nThis is the rest",
124 + "output": "Before\n[THINK]This is a reasoning section[/THINK]\nThis is the rest",
125 + "reasoning": "This is a reasoning section",
126 + "content": "Before\n\nThis is the rest",
127 "is_reasoning_end": True,
128 }
129
130 TEST_CASES = [
131 pytest.param(
132 False,
131 - SIMPLE_REASONING,
132 - id="simple_reasoning",
133 + INVALID_SIMPLE_REASONING,
134 + id="invalid_simple_reasoning",
135 ),
136 pytest.param(
137 True,
136 - SIMPLE_REASONING,
137 - id="simple_reasoning_streaming",
138 + INVALID_SIMPLE_REASONING,
139 + id="invalid_simple_reasoning_streaming",
140 ),
141 pytest.param(
142 False,
141 - COMPLETE_REASONING,
142 - id="complete_reasoning",
143 + INVALID_COMPLETE_REASONING,
144 + id="invalid_complete_reasoning",
145 ),
146 pytest.param(
147 True,
146 - COMPLETE_REASONING,
147 - id="complete_reasoning_streaming",
148 + INVALID_COMPLETE_REASONING,
149 + id="invalid_complete_reasoning_streaming",
150 ),
151 pytest.param(
152 False,
153 NO_CONTENT,
152 - id="no_content_token",
154 + id="no_content",
155 + ),
156 + pytest.param(
157 + False,
158 + NO_REASONING,
159 + id="no_reasoning",
160 ),
161 pytest.param(
162 True,
@@ -158,23 +165,23 @@ TEST_CASES = [
165 ),
166 pytest.param(
167 False,
161 - MULTIPLE_LINES,
162 - id="multiple_lines",
168 + INVALID_MULTIPLE_LINES,
169 + id="invalid_multiple_lines",
170 ),
171 pytest.param(
172 True,
166 - MULTIPLE_LINES,
167 - id="multiple_lines_streaming",
173 + INVALID_MULTIPLE_LINES,
174 + id="invalid_multiple_lines_streaming",
175 ),
176 pytest.param(
177 True,
171 - SHORTEST_REASONING,
172 - id="shortest",
178 + INVALID_SHORTEST_REASONING,
179 + id="invalid_shortest",
180 ),
181 pytest.param(
182 False,
176 - SHORTEST_REASONING_NO_STREAMING,
177 - id="shortest_streaming",
183 + INVALID_SHORTEST_REASONING_NO_STREAMING,
184 + id="invalid_shortest_streaming",
185 ),
186 pytest.param(
187 False,
@@ -208,13 +215,13 @@ TEST_CASES = [
215 ),
216 pytest.param(
217 False,
211 - SHORTEST_REASONING_NO_STREAMING_WITH_THINK,
212 - id="shortest_with_think",
218 + INVALID_SHORTEST_REASONING_NO_STREAMING_WITH_THINK,
219 + id="invalid_shortest_with_think",
220 ),
221 pytest.param(
222 True,
216 - SHORTEST_REASONING_WITH_THINK,
217 - id="shortest_with_think_streaming",
223 + INVALID_SHORTEST_REASONING_WITH_THINK,
224 + id="invalid_shortest_with_think_streaming",
225 ),
226 pytest.param(
227 False,
@@ -316,10 +323,26 @@ def test_mistral_reasoning(
323
324 # Test extract_content
325 if param_dict["content"] is not None:
319 - content = parser.extract_content_ids(output_tokens)
320 - assert content == mistral_tokenizer.tokenizer.encode(
321 - param_dict["content"], bos=False, eos=False
326 + # Handle the case where there are tokens outputted before Thinking.
327 + # This should not occur if the model is well trained and prompted.
328 + if "[THINK]" in param_dict["output"] and not param_dict["output"].startswith(
329 + "[THINK]"
330 + ):
331 + before_content = param_dict["output"].split("[THINK]")[0]
332 + before_token_ids = mistral_tokenizer.tokenizer.encode(
333 + before_content, bos=False, eos=False
334 + )
335 + left_to_encode = param_dict["content"][len(before_content) :]
336 + # Normal situation.
337 + else:
338 + before_token_ids = []
339 + left_to_encode = param_dict["content"]
340 +
341 + content_tokens = parser.extract_content_ids(output_tokens)
342 + expected_token_ids = before_token_ids + mistral_tokenizer.tokenizer.encode(
343 + left_to_encode, bos=False, eos=False
344 )
345 + assert content_tokens == expected_token_ids
346 else:
347 content = parser.extract_content_ids(output_tokens)
348 assert content == []
vllm/reasoning/mistral_reasoning_parser.py
+102 -3
@@ -3,20 +3,29 @@
3
4 from functools import cached_property
5
6 +from vllm.entrypoints.openai.protocol import (
7 + ChatCompletionRequest,
8 + ResponsesRequest,
9 +)
10 from vllm.logger import init_logger
11 from vllm.reasoning import ReasoningParser
8 -from vllm.reasoning.deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParser
12 +from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
13 from vllm.tokenizers import MistralTokenizer
14
15 logger = init_logger(__name__)
16
17
14 -class MistralReasoningParser(DeepSeekR1ReasoningParser):
18 +class MistralReasoningParser(BaseThinkingReasoningParser):
19 """
20 Reasoning parser for Mistral models.
21
18 - The Mistral models uses [THINK]...[/THINK] tokens to denote reasoning
22 + The Mistral models uses `[THINK]`...`[/THINK]` tokens to denote reasoning
23 text. This parser extracts the reasoning content from the model output.
24 +
25 + A valid reasoning trace should always start with a `[THINK]` token and end with
26 + a `[/THINK]` token.
27 +
28 + If `[THINK]` token is not generated, then this parser only returns content.
29 """
30
31 def __init__(self, tokenizer: MistralTokenizer, *args, **kwargs):
@@ -53,3 +62,93 @@ class MistralReasoningParser(DeepSeekR1ReasoningParser):
62 from mistral_common.tokens.tokenizers.base import SpecialTokens
63
64 return SpecialTokens.end_think
65 +
66 + def is_reasoning_end(self, input_ids: list[int]) -> bool:
67 + has_eot_token = False
68 +
69 + for id in input_ids[::-1]:
70 + if id == self.start_token_id:
71 + # Reasoning ends only if a BOT token is found before a EOT token.
72 + return has_eot_token
73 + elif id == self.end_token_id:
74 + has_eot_token = True
75 + return False
76 +
77 + def extract_content_ids(self, input_ids: list[int]) -> list[int]:
78 + """
79 + Extract the content
80 + """
81 + has_bot_token = False
82 + has_eot_token = False
83 + bot_token_index = -1
84 + eot_token_index = -1
85 + # One for loop instead of multiple lookups
86 + for i, token_id in enumerate(input_ids):
87 + # We filter that we have multiple BOT tokens which should not
88 + # happen for a well prompted trained model
89 + if token_id == self.start_token_id and not has_bot_token:
90 + has_bot_token = True
91 + bot_token_index = i
92 + elif token_id == self.end_token_id:
93 + has_eot_token = True
94 + eot_token_index = i
95 + break
96 +
97 + # 1. Only BOT has been outputted
98 + if has_bot_token and not has_eot_token:
99 + # Should be = [] if model is well prompted and trained.
100 + return input_ids[:bot_token_index]
101 + # 2. Neither BOT or EOT have been outputted
102 + elif not has_bot_token and not has_eot_token:
103 + return input_ids
104 + # 3. Both BOT and EOT have been outputted.
105 + elif has_bot_token and has_eot_token:
106 + return input_ids[:bot_token_index] + input_ids[eot_token_index + 1 :]
107 + # 4. Only EOT has been outputted => this should not have occured for a model
108 + # well prompted and trained.
109 + else:
110 + return input_ids[:eot_token_index] + input_ids[eot_token_index + 1 :]
111 +
112 + def extract_reasoning(
113 + self, model_output: str, request: ChatCompletionRequest | ResponsesRequest
114 + ) -> tuple[str | None, str | None]:
115 + """
116 + Extract reasoning content from the model output.
117 + """
118 + if not model_output:
119 + return (None, "")
120 +
121 + # Check if the start token is present in the model output, remove it
122 + # if it is present.
123 + prev_bot_token, bot_token, post_bot_token = model_output.partition(
124 + self.start_token
125 + )
126 +
127 + has_bot_token = bool(bot_token)
128 + # Valid EOT tokens should follow BOT token
129 + has_valid_eot_token = has_bot_token and self.end_token in post_bot_token
130 +
131 + # 1. If there is BOT token followed by EOT token
132 + if has_bot_token and has_valid_eot_token:
133 + prev_eot_token, _, post_eot_token = post_bot_token.partition(self.end_token)
134 + # If model is well prompted and trained prev_bot_token should be ""
135 + content = prev_bot_token + post_eot_token
136 + return prev_eot_token, content if content else None
137 + # 2. Only BOT token
138 + elif has_bot_token:
139 + # If model is well prompted and trained prev_bot_token should be ""
140 + return post_bot_token, prev_bot_token if prev_bot_token else None
141 + # 3. EOT token has been outputted without BOT or neither has been outputted
142 + else:
143 + has_non_valid_eot_token = self.end_token in prev_bot_token
144 + # 3.a EOT token has been outputted without BOT
145 + # If model is well prompted and trained `has_non_valid_eot_token` should
146 + # be `False` and the parser outputs all tokens as 'content'
147 + if has_non_valid_eot_token:
148 + prev_eot_token, _, post_eot_token = prev_bot_token.partition(
149 + self.end_token
150 + )
151 + return None, prev_eot_token + post_eot_token
152 + # 3.b neither BOT or EOT have been outputted
153 + else:
154 + return None, prev_bot_token