@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cb080f32e3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[Bugfix] Support missing tool parameters in mistral tokenizer (#12884)

Signed-off-by: Florian Greinacher <florian.greinacher@siemens.com>

Florian Greinacher committed Feb 11, 2025 at 04:33 UTC cb080f32e38e87beda897d0602bf6a0d0c79d00f
2 files changed +88 -19
tests/tokenization/test_mistral_tokenizer.py new
+50
@@ -0,0 +1,50 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +import pytest
4 +from mistral_common.protocol.instruct.messages import UserMessage
5 +from mistral_common.protocol.instruct.request import ChatCompletionRequest
6 +from mistral_common.protocol.instruct.tool_calls import Function, Tool
7 +
8 +from vllm.transformers_utils.tokenizers.mistral import (
9 + make_mistral_chat_completion_request)
10 +
11 +
12 +# yapf: enable
13 +@pytest.mark.parametrize(
14 + "openai_request,expected_mistral_request",
15 + [(
16 + {
17 + "messages": [{
18 + "role": "user",
19 + "content": "What is the current local date and time?",
20 + }],
21 + "tools": [{
22 + "type": "function",
23 + "function": {
24 + "description": "Fetch the current local date and time.",
25 + "name": "get_current_time",
26 + },
27 + }],
28 + },
29 + ChatCompletionRequest(
30 + messages=[
31 + UserMessage(content="What is the current local date and time?")
32 + ],
33 + tools=[
34 + Tool(
35 + type="function",
36 + function=Function(
37 + name="get_current_time",
38 + description="Fetch the current local date and time.",
39 + parameters={},
40 + ),
41 + )
42 + ],
43 + ),
44 + )],
45 +)
46 +def test_make_mistral_chat_completion_request(openai_request,
47 + expected_mistral_request):
48 + assert (make_mistral_chat_completion_request(
49 + openai_request["messages"],
50 + openai_request["tools"]) == expected_mistral_request)
vllm/transformers_utils/tokenizers/mistral.py
+38 -19
@@ -104,6 +104,42 @@ def find_tokenizer_file(files: List[str]):
104 return matched_files[0]
105
106
107 +def make_mistral_chat_completion_request(
108 + messages: List["ChatCompletionMessageParam"],
109 + tools: Optional[List[Dict[str,
110 + Any]]] = None) -> "ChatCompletionRequest":
111 + last_message = cast(Dict[str, Any], messages[-1])
112 + if last_message["role"] == "assistant":
113 + last_message["prefix"] = True
114 +
115 + last_message = cast(Dict[str, Any], messages[-1])
116 + if last_message["role"] == "assistant":
117 + last_message["prefix"] = True
118 +
119 + # mistral-common requires AssistantMessage content to be string [1].
120 + #
121 + # [1]: https://github.com/mistralai/mistral-common/blob/f4a06998b75ed78bbf5aaf569590b772ea26c9f6/src/mistral_common/protocol/instruct/messages.py#L80
122 + for message in messages:
123 + if message.get("role") == "assistant":
124 + content = message.get("content")
125 + if isinstance(content, list):
126 + content = "\n".join(chunk.get("text") for chunk in content)
127 + message["content"] = content
128 +
129 + # The Mistral client, in comparison to the OpenAI client, requires the
130 + # "parameters" dict to be present, even if it's empty.
131 + if tools:
132 + for function in [
133 + tool["function"] for tool in tools
134 + if tool["type"] == "function"
135 + ]:
136 + function.setdefault("parameters", {})
137 +
138 + from mistral_common.protocol.instruct.request import ChatCompletionRequest
139 + return ChatCompletionRequest(messages=messages,
140 + tools=tools) # type: ignore[type-var]
141 +
142 +
143 class MistralTokenizer:
144
145 def __init__(self, tokenizer: "PublicMistralTokenizer") -> None:
@@ -283,27 +319,10 @@ class MistralTokenizer:
319
320 def apply_chat_template(self,
321 messages: List["ChatCompletionMessageParam"],
286 - tools: Optional[Dict[str, Any]] = None,
322 + tools: Optional[List[Dict[str, Any]]] = None,
323 **kwargs) -> List[int]:
324
289 - last_message = cast(Dict[str, Any], messages[-1])
290 - if last_message["role"] == "assistant":
291 - last_message["prefix"] = True
292 -
293 - from mistral_common.protocol.instruct.request import (
294 - ChatCompletionRequest)
295 -
296 - # mistral-common requires AssistantMessage content to be string [1].
297 - #
298 - # [1]: https://github.com/mistralai/mistral-common/blob/f4a06998b75ed78bbf5aaf569590b772ea26c9f6/src/mistral_common/protocol/instruct/messages.py#L80
299 - for message in messages:
300 - if message.get("role") == "assistant":
301 - content = message.get("content")
302 - if isinstance(content, list):
303 - content = "\n".join(chunk.get("text") for chunk in content)
304 - message["content"] = content
305 - request = ChatCompletionRequest(messages=messages,
306 - tools=tools) # type: ignore[type-var]
325 + request = make_mistral_chat_completion_request(messages, tools)
326 encoded = self.mistral.encode_chat_completion(request)
327
328 # encode-decode to get clean prompt