A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
[Bugfix] Fix AssertionError: skip_special_tokens=False is not supported for Mistral tokenizers (#16964)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
Apr 23, 2025 at 15:24 UTC
8c87a9ad46dd8b972d4cd9c6cecb5b284c92f583
1 file changed
+8
-4
vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py
+8
-4
@@ -72,10 +72,14 @@ class MistralToolParser(ToolParser):
72
73
def adjust_request(
74
self, request: ChatCompletionRequest) -> ChatCompletionRequest:
75
- if request.tools and request.tool_choice != 'none':
76
- # do not skip special tokens because mistral uses the special
77
- # tokens to indicate the start and end of the tool calls
78
- # information.
75
+ if not isinstance(
76
+ self.model_tokenizer, MistralTokenizer
77
+ ) and request.tools and request.tool_choice != 'none':
78
+ # Do not skip special tokens when using chat template
79
+ # with Mistral parser as TOOL_CALL token is needed
80
+ # for tool detection.
81
+ # Note: we don't want skip_special_tokens=False
82
+ # with MistralTokenizer as it is incompatible
83
request.skip_special_tokens = False
84
return request
85