A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Tool parsing] Improve / correct mistral tool parsing (#10333)
Patrick von Platen committed
Nov 15, 2024 at 01:42 UTC
11cd1ae6ad6fa7d35060fea35133e08c0a1c227c
5 files changed
+172
-59
tests/models/decoder_only/language/test_mistral.py
+82
-11
@@ -2,9 +2,13 @@
2
3
Run `pytest tests/models/test_mistral.py`.
4
"""
5
+import copy
6
+
7
import pytest
8
9
from vllm import SamplingParams
10
+from vllm.entrypoints.openai.tool_parsers.mistral_tool_parser import ( # noqa
11
+ MistralToolParser)
12
13
from ...utils import check_logprobs_close
14
@@ -58,17 +62,69 @@ TOOLS = [{
62
},
63
"required": ["city", "state", "unit"]
64
}
65
+ },
66
+}, {
67
+ "type": "function",
68
+ "function": {
69
+ "name": "rewrite",
70
+ "description": "Rewrites text",
71
+ "parameters": {
72
+ "type": "object",
73
+ "required": [],
74
+ "properties": {
75
+ "text": {
76
+ "type": "string",
77
+ "description": "The input text to rewrite."
78
+ }
79
+ }
80
+ }
81
}
82
}]
63
-MSGS = [{
64
- "role":
65
- "user",
66
- "content": ("Can you tell me what the temperate"
67
- " will be in Dallas, in fahrenheit?")
68
-}]
69
-EXPECTED_FUNC_CALL = (
70
- '[{"name": "get_current_weather", "arguments": '
71
- '{"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}]')
83
+MSGS = [
84
+ {
85
+ "role": "system",
86
+ "content": "You are an assistant."
87
+ },
88
+ {
89
+ "role":
90
+ "user",
91
+ "content":
92
+ "Could you please rewrite the below article? \n\n My English needs improvving, maybe I make errors." # noqa
93
+ },
94
+ {
95
+ "role":
96
+ "assistant",
97
+ "content":
98
+ "",
99
+ "tool_calls": [{
100
+ "id": "bbc5b7ede",
101
+ "type": "function",
102
+ "function": {
103
+ "name":
104
+ "rewrite",
105
+ "arguments":
106
+ '{\"text\":\"My English needs improvving, maybe I make errors.\"}' # noqa
107
+ }
108
+ }]
109
+ },
110
+ {
111
+ "role": "tool",
112
+ "content":
113
+ "{\"action\":\"rewrite\",\"outcome\":\"My English needs improving, maybe I make errors.\"}", # noqa
114
+ "tool_call_id": "bbc5b7ede",
115
+ "name": "rewrite"
116
+ },
117
+ {
118
+ "role": "assistant",
119
+ "content": "---\n\nMy English needs improving, maybe I make errors"
120
+ },
121
+ {
122
+ "role":
123
+ "user",
124
+ "content": ("Can you tell me what the temperate"
125
+ " will be in Dallas, in fahrenheit?")
126
+ }
127
+]
128
129
130
@pytest.mark.parametrize("model", MODELS)
@@ -175,8 +231,23 @@ def test_mistral_function_calling(
231
tokenizer_mode="mistral",
232
config_format="mistral",
233
load_format="mistral") as vllm_model:
178
- outputs = vllm_model.model.chat(MSGS,
234
+
235
+ msgs = copy.deepcopy(MSGS)
236
+ outputs = vllm_model.model.chat(msgs,
237
tools=TOOLS,
238
sampling_params=SAMPLING_PARAMS)
239
182
- assert outputs[0].outputs[0].text.strip() == EXPECTED_FUNC_CALL
240
+ tokenizer = vllm_model.model.get_tokenizer()
241
+ tool_parser = MistralToolParser(tokenizer)
242
+
243
+ model_output = outputs[0].outputs[0].text.strip()
244
+ assert model_output.startswith(tool_parser.bot_token), model_output
245
+ parsed_message = tool_parser.extract_tool_calls(model_output, None)
246
+
247
+ assert parsed_message.tools_called
248
+ assert parsed_message.tool_calls[0].id == "0UAqFzWsD"
249
+ assert parsed_message.tool_calls[
250
+ 0].function.name == "get_current_weather"
251
+ assert parsed_message.tool_calls[
252
+ 0].function.arguments == '{"city": "Dallas", "state": "TX", "unit": "fahrenheit"}' # noqa
253
+ assert parsed_message.content is None
vllm/entrypoints/openai/serving_chat.py
+5
-34
@@ -30,6 +30,7 @@ from vllm.outputs import CompletionOutput, RequestOutput
30
from vllm.sampling_params import BeamSearchParams, SamplingParams
31
from vllm.sequence import Logprob
32
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
33
+from vllm.transformers_utils.tokenizers import maybe_serialize_tool_calls
34
from vllm.utils import iterate_with_cancellation
35
36
logger = init_logger(__name__)
@@ -127,41 +128,11 @@ class OpenAIServingChat(OpenAIServing):
128
return self.create_error_response(
129
"tool_choice = \"required\" is not supported!")
130
130
- # NOTE: There is currently a bug in pydantic where attributes
131
- # declared as iterables are replaced in in the instances by
132
- # pydantic-core ValidatorIterator instance. In particular, this
133
- # affects tool_calls defined in ChatCompletionAssistantMessageParam
134
- # model:
135
- # see:
136
- # - https://github.com/pydantic/pydantic/issues/9467
137
- # As a result, tool_calls from assistant messages are never
138
- # deserialized in the request object if the tool_calls iterator is
139
- # not consumed. This affect messages passed to the MistralTokenizer
140
- # since no chat template is applied and therefore the tools_calls
141
- # iterator is not directly consumed.
142
- # Issue is tracked on Pydantic side, with resolution planned for
143
- # v2.11 release. In the meantime, the official workaround is to
144
- # consume the iterator so the tool_calls are correctly deserialized
145
- # in the OpenAI ChatCompletionAssistantMessageParam object
146
- # https://github.com/pydantic/pydantic/issues/9467#issuecomment-2442097291 # noqa: E501
147
- # Official Pydantic Issues:
148
- # - https://github.com/pydantic/pydantic/issues/9541
149
- # TODO: remove when pydantic v2.11 is released
131
+ # because of issues with pydantic we need to potentially
132
+ # re-serialize the tool_calls field of the request
133
+ # for more info: see comment in `maybe_serialize_tool_calls`
134
if isinstance(tokenizer, MistralTokenizer):
151
- for i, message in enumerate(request.messages):
152
- if message.get("role") == 'assistant':
153
- tool_calls_validator = message.get(
154
- "tool_calls", ().__iter__())
155
- validated_tool_calls = []
156
- while True:
157
- try:
158
- tool_call = next(
159
- tool_calls_validator) # type: ignore
160
- validated_tool_calls.append(tool_call)
161
- except StopIteration:
162
- break
163
- request.messages[i][
164
- "tool_calls"] = validated_tool_calls
135
+ maybe_serialize_tool_calls(request)
136
137
if (request.tool_choice == "auto" and
138
not (self.enable_auto_tools and tool_parser is not None)
vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py
+17
-8
@@ -62,7 +62,7 @@ class MistralToolParser(ToolParser):
62
] # map what has been streamed for each tool so far to a list
63
self.bot_token = "[TOOL_CALLS]"
64
self.bot_token_id = self.vocab.get(self.bot_token)
65
- self.tool_call_regex = re.compile(r"\[{.*?}\]", re.DOTALL)
65
+ self.tool_call_regex = re.compile(r"\[{.*}\]", re.DOTALL)
66
if self.bot_token_id is None:
67
raise RuntimeError(
68
"Mistral Tool Parser could not locate the tool call token in "
@@ -84,16 +84,25 @@ class MistralToolParser(ToolParser):
84
return ExtractedToolCallInformation(tools_called=False,
85
tool_calls=[],
86
content=model_output)
87
+
88
+ # first remove the BOT token
89
+ tool_content = model_output.replace(self.bot_token, "").strip()
90
+
91
try:
92
89
- # use a regex to find the tool call. remove the BOT token
90
- # and make sure to replace single quotes with double quotes
91
- raw_tool_call = self.tool_call_regex.findall(
92
- model_output.replace(self.bot_token, ""))[0]
93
+ # we first try to directly load the json as parsing very nested
94
+ # jsons is difficult
95
+ try:
96
+ function_call_arr = json.loads(tool_content)
97
+ except json.JSONDecodeError:
98
+ # use a regex to find the part corresponding to the tool call.
99
+ # NOTE: This use case should not happen if the model is trained
100
+ # correctly. It's a easy possible fix so it's included, but
101
+ # can be brittle for very complex / highly nested tool calls
102
+ raw_tool_call = self.tool_call_regex.findall(tool_content)[0]
103
+ function_call_arr = json.loads(raw_tool_call)
104
94
- # load the JSON, and then use it to build the Function and
105
# Tool Call
96
- function_call_arr = json.loads(raw_tool_call)
106
tool_calls: List[MistralToolCall] = [
107
MistralToolCall(
108
type="function",
@@ -116,7 +125,7 @@ class MistralToolParser(ToolParser):
125
# return information to just treat the tool call as regular JSON
126
return ExtractedToolCallInformation(tools_called=False,
127
tool_calls=[],
119
- content=model_output)
128
+ content=tool_content)
129
130
def extract_tool_calls_streaming(
131
self,
vllm/transformers_utils/tokenizers/__init__.py
+2
-2
@@ -1,3 +1,3 @@
1
-from .mistral import MistralTokenizer
1
+from .mistral import MistralTokenizer, maybe_serialize_tool_calls
2
3
-__all__ = ["MistralTokenizer"]
3
+__all__ = ["MistralTokenizer", "maybe_serialize_tool_calls"]
vllm/transformers_utils/tokenizers/mistral.py
+66
-4
@@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union, cast
7
import huggingface_hub
8
from huggingface_hub import HfApi, hf_hub_download
9
from mistral_common.protocol.instruct.request import ChatCompletionRequest
10
+from mistral_common.tokens.tokenizers.base import SpecialTokens
11
# yapf: disable
12
from mistral_common.tokens.tokenizers.mistral import (
13
MistralTokenizer as PublicMistralTokenizer)
@@ -29,6 +30,43 @@ class Encoding:
30
input_ids: List[int]
31
32
33
+def maybe_serialize_tool_calls(request: ChatCompletionRequest):
34
+ # SEE: https://github.com/vllm-project/vllm/pull/9951
35
+ # Credits go to: @gcalmettes
36
+ # NOTE: There is currently a bug in pydantic where attributes
37
+ # declared as iterables are replaced in in the instances by
38
+ # pydantic-core ValidatorIterator instance. In particular, this
39
+ # affects tool_calls defined in ChatCompletionAssistantMessageParam
40
+ # model:
41
+ # see:
42
+ # - https://github.com/pydantic/pydantic/issues/9467
43
+ # As a result, tool_calls from assistant messages are never
44
+ # deserialized in the request object if the tool_calls iterator is
45
+ # not consumed. This affect messages passed to the MistralTokenizer
46
+ # since no chat template is applied and therefore the tools_calls
47
+ # iterator is not directly consumed.
48
+ # Issue is tracked on Pydantic side, with resolution planned for
49
+ # v2.11 release. In the meantime, the official workaround is to
50
+ # consume the iterator so the tool_calls are correctly deserialized
51
+ # in the OpenAI ChatCompletionAssistantMessageParam object
52
+ # https://github.com/pydantic/pydantic/issues/9467#issuecomment-2442097291 # noqa: E501
53
+ # Official Pydantic Issues:
54
+ # - https://github.com/pydantic/pydantic/issues/9541
55
+ # TODO: remove when pydantic v2.11 is released
56
+ for i, message in enumerate(request.messages):
57
+ if message.get("role") == 'assistant':
58
+ tool_calls_validator = message.get("tool_calls", ().__iter__())
59
+ validated_tool_calls = []
60
+ while True:
61
+ try:
62
+ tool_call = next(tool_calls_validator) # type: ignore
63
+ validated_tool_calls.append(tool_call)
64
+ except StopIteration:
65
+ break
66
+
67
+ request.messages[i]["tool_calls"] = validated_tool_calls
68
+
69
+
70
def list_local_repo_files(repo_id: str, revision: Optional[str]) -> List[str]:
71
repo_cache = os.path.join(
72
huggingface_hub.constants.HF_HUB_CACHE,
@@ -222,7 +260,8 @@ class MistralTokenizer:
260
if self.is_tekken:
261
tokens = [
262
t for t in tokens
225
- if t not in self.tokenizer._all_special_tokens
263
+ if (t is SpecialTokens.tool_calls
264
+ or t not in self.tokenizer._all_special_tokens)
265
]
266
267
if any(isinstance(t, bytes) for t in tokens):
@@ -246,7 +285,27 @@ class MistralTokenizer:
285
else:
286
decoded = "".join(tokens)
287
else:
249
- decoded = self.tokenizer.decode(tokens) # type: ignore[arg-type]
288
+ # make sure certain special tokens like Tool calls are
289
+ # not decoded
290
+ special_tokens = {SpecialTokens.tool_calls}
291
+ regular_tokens: List[str] = []
292
+ decoded_list = []
293
+
294
+ for token in tokens:
295
+ if token in special_tokens:
296
+ if regular_tokens:
297
+ decoded_list.append(
298
+ self.tokenizer.decode(regular_tokens))
299
+ regular_tokens = []
300
+ decoded_list.append(token)
301
+ else:
302
+ regular_tokens.append(token)
303
+
304
+ if regular_tokens:
305
+ decoded_list.append(
306
+ self.decode(regular_tokens)) # type: ignore
307
+
308
+ decoded = ''.join(decoded_list)
309
310
return decoded
311
@@ -274,8 +333,11 @@ class MistralTokenizer:
333
assert self.is_tekken or self.is_spm, type(self.tokenizer)
334
335
if self.is_tekken:
277
- # skip special tokens
278
- ids = [i for i in ids if i > self.tokenizer.num_special_tokens]
336
+ # skip special tokens except tool call
337
+ ids = [
338
+ i for i in ids if i > self.tokenizer.num_special_tokens or i ==
339
+ self.tokenizer.get_control_token(SpecialTokens.tool_calls)
340
+ ]
341
342
tokens = [self.tokenizer.id_to_piece(id) for id in ids]
343