A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 14:30 UTC
Fix some Mistral parser issues (#37209)
Signed-off-by: juliendenize <julien.denize@mistral.ai>
Julien Denize committed
Mar 17, 2026 at 01:08 UTC
5db91f0aaf3566b1d9f8b0720065eb9009296d98
3 files changed
+42
-34
vllm/entrypoints/openai/chat_completion/serving.py
+8
-5
@@ -310,11 +310,14 @@ class OpenAIServingChat(OpenAIServing):
310
trace_headers=trace_headers,
311
)
312
else:
313
- reasoning_ended = (
314
- reasoning_parser.is_reasoning_end(prompt_token_ids or [])
315
- if reasoning_parser
316
- else None
317
- )
313
+ if not request.include_reasoning:
314
+ reasoning_ended = True
315
+ elif reasoning_parser:
316
+ reasoning_ended = reasoning_parser.is_reasoning_end(
317
+ prompt_token_ids or []
318
+ )
319
+ else:
320
+ reasoning_ended = None
321
322
generator = self.engine_client.generate(
323
engine_prompt,
vllm/tokenizers/mistral.py
+27
-26
@@ -15,8 +15,15 @@ from mistral_common.protocol.instruct.validator import ValidationMode
15
from mistral_common.tokens.tokenizers.base import (
16
SpecialTokenPolicy,
17
SpecialTokens,
18
+ Tokenizer,
19
+)
20
+from mistral_common.tokens.tokenizers.instruct import (
21
+ InstructTokenizerBase,
22
+ InstructTokenizerV13,
23
+)
24
+from mistral_common.tokens.tokenizers.mistral import (
25
+ MistralTokenizer as MistralCommonTokenizer,
26
)
19
-from mistral_common.tokens.tokenizers.instruct import InstructTokenizerV13
27
from mistral_common.tokens.tokenizers.sentencepiece import (
28
SentencePieceTokenizer,
29
)
@@ -26,21 +33,20 @@ from pydantic import ValidationError
33
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
34
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
35
from vllm.logger import init_logger
36
+from vllm.tokenizers.protocol import TokenizerLike
37
30
-from .protocol import TokenizerLike
38
+try:
39
+ # Transformers v5
40
+ from transformers.tokenization_mistral_common import MistralCommonBackend
41
+except ImportError:
42
+ # Transformers v4
43
+ from transformers.tokenization_mistral_common import (
44
+ MistralCommonTokenizer as MistralCommonBackend,
45
+ )
46
47
if TYPE_CHECKING:
48
from transformers import BatchEncoding
49
35
- try:
36
- # Transformers v5
37
- from transformers.tokenization_mistral_common import MistralCommonBackend
38
- except ImportError:
39
- # Transformers v4
40
- from transformers.tokenization_mistral_common import (
41
- MistralCommonTokenizer as MistralCommonBackend,
42
- )
43
-
50
logger = init_logger(__name__)
51
52
@@ -235,15 +241,6 @@ class MistralTokenizer(TokenizerLike):
241
download_dir: str | None = None,
242
**kwargs,
243
) -> "MistralTokenizer":
238
- try:
239
- # Transformers v5
240
- from transformers.tokenization_mistral_common import MistralCommonBackend
241
- except ImportError:
242
- # Transformers v4
243
- from transformers.tokenization_mistral_common import (
244
- MistralCommonTokenizer as MistralCommonBackend,
245
- )
246
-
244
tokenizer = MistralCommonBackend.from_pretrained(
245
path_or_repo_id,
246
*args,
@@ -255,13 +252,13 @@ class MistralTokenizer(TokenizerLike):
252
253
return cls(tokenizer)
254
258
- def __init__(self, tokenizer: "MistralCommonBackend") -> None:
255
+ def __init__(self, tokenizer: MistralCommonBackend) -> None:
256
super().__init__()
257
261
- self.transformers_tokenizer = tokenizer
262
- self.mistral = tokenizer.tokenizer
263
- self.instruct = self.mistral.instruct_tokenizer
264
- self.tokenizer = self.instruct.tokenizer
258
+ self.transformers_tokenizer: MistralCommonBackend = tokenizer
259
+ self.mistral: MistralCommonTokenizer = tokenizer.tokenizer
260
+ self.instruct: InstructTokenizerBase = self.mistral.instruct_tokenizer
261
+ self.tokenizer: Tokenizer = self.instruct.tokenizer
262
263
mode = self.mistral._chat_completion_request_validator._mode
264
if mode != ValidationMode.test:
@@ -483,7 +480,11 @@ class MistralTokenizer(TokenizerLike):
480
return self.transformers_tokenizer.convert_tokens_to_ids(tokens)
481
482
def convert_tokens_to_string(self, tokens: list[str]) -> str:
486
- to_decode_special_tokens = {SpecialTokens.tool_calls}
483
+ to_decode_special_tokens = {
484
+ SpecialTokens.tool_calls,
485
+ SpecialTokens.begin_think,
486
+ SpecialTokens.end_think,
487
+ }
488
if self.is_tekken:
489
assert isinstance(self.tokenizer, Tekkenizer), type(self.tokenizer)
490
tokens = [
vllm/tool_parsers/mistral_tool_parser.py
+7
-3
@@ -241,7 +241,10 @@ class MistralToolParser(ToolParser):
241
delta_token_ids: Sequence[int],
242
request: ChatCompletionRequest,
243
) -> DeltaMessage | None:
244
- if self.bot_token_id not in current_token_ids:
244
+ has_bot_token = (
245
+ self.bot_token_id in current_token_ids or self.bot_token in current_text
246
+ )
247
+ if not has_bot_token:
248
# if the tool call token is not in the tokens generated so far,
249
# append output to contents since it's not a tool
250
return DeltaMessage(content=delta_text)
@@ -275,7 +278,8 @@ class MistralToolParser(ToolParser):
278
additional_content: str = ""
279
if self.streaming_state == StreamingState.WAITING_FOR_TOOL_START:
280
# this is the first tool call
278
- assert self.bot_token_id in delta_token_ids
281
+ if self.bot_token not in delta_text:
282
+ return DeltaMessage(content=delta_text)
283
if not delta_text.startswith(self.bot_token):
284
additional_content += delta_text.split(self.bot_token)[0]
285
delta_text = self.bot_token + "".join(
@@ -411,7 +415,7 @@ class MistralToolParser(ToolParser):
415
index=self.current_tool_id, type="function"
416
)
417
current_tool_call_modified = False
414
- if self.bot_token_id in delta_token_ids:
418
+ if self.bot_token_id in delta_token_ids or self.bot_token in delta_text:
419
# this is the first tool call
420
if not delta_text.startswith(self.bot_token):
421
content = delta_text.split(self.bot_token)[0]