A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[BugFix]: properly deserialize `tool_calls` iterator before processing by mistral-common when MistralTokenizer is used (#9951)
Signed-off-by: Guillaume Calmettes <gcalmettes@scaleway.com>
Guillaume Calmettes committed
Nov 14, 2024 at 05:48 UTC
52b48c1ead683ec2afe6b0396ece32d73884cd21
1 file changed
+36
vllm/entrypoints/openai/serving_chat.py
+36
@@ -127,6 +127,42 @@ class OpenAIServingChat(OpenAIServing):
127
return self.create_error_response(
128
"tool_choice = \"required\" is not supported!")
129
130
+ # NOTE: There is currently a bug in pydantic where attributes
131
+ # declared as iterables are replaced in in the instances by
132
+ # pydantic-core ValidatorIterator instance. In particular, this
133
+ # affects tool_calls defined in ChatCompletionAssistantMessageParam
134
+ # model:
135
+ # see:
136
+ # - https://github.com/pydantic/pydantic/issues/9467
137
+ # As a result, tool_calls from assistant messages are never
138
+ # deserialized in the request object if the tool_calls iterator is
139
+ # not consumed. This affect messages passed to the MistralTokenizer
140
+ # since no chat template is applied and therefore the tools_calls
141
+ # iterator is not directly consumed.
142
+ # Issue is tracked on Pydantic side, with resolution planned for
143
+ # v2.11 release. In the meantime, the official workaround is to
144
+ # consume the iterator so the tool_calls are correctly deserialized
145
+ # in the OpenAI ChatCompletionAssistantMessageParam object
146
+ # https://github.com/pydantic/pydantic/issues/9467#issuecomment-2442097291 # noqa: E501
147
+ # Official Pydantic Issues:
148
+ # - https://github.com/pydantic/pydantic/issues/9541
149
+ # TODO: remove when pydantic v2.11 is released
150
+ if isinstance(tokenizer, MistralTokenizer):
151
+ for i, message in enumerate(request.messages):
152
+ if message.get("role") == 'assistant':
153
+ tool_calls_validator = message.get(
154
+ "tool_calls", ().__iter__())
155
+ validated_tool_calls = []
156
+ while True:
157
+ try:
158
+ tool_call = next(
159
+ tool_calls_validator) # type: ignore
160
+ validated_tool_calls.append(tool_call)
161
+ except StopIteration:
162
+ break
163
+ request.messages[i][
164
+ "tool_calls"] = validated_tool_calls
165
+
166
if (request.tool_choice == "auto" and
167
not (self.enable_auto_tools and tool_parser is not None)
168
and not isinstance(tokenizer, MistralTokenizer)):