A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Bugfix] Fix multi-round chat error when mistral tokenizer is used (#12859)
Signed-off-by: Zifei Tong <zifeitong@gmail.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
zifeitong committed
Feb 7, 2025 at 23:04 UTC
d01f66b0394e62a11429c8f0afd9a56b7b2b7f0c
1 file changed
+10
vllm/transformers_utils/tokenizers/mistral.py
+10
@@ -291,6 +291,16 @@ class MistralTokenizer:
291
292
from mistral_common.protocol.instruct.request import (
293
ChatCompletionRequest)
294
+
295
+ # mistral-common requires AssistantMessage content to be string [1].
296
+ #
297
+ # [1]: https://github.com/mistralai/mistral-common/blob/f4a06998b75ed78bbf5aaf569590b772ea26c9f6/src/mistral_common/protocol/instruct/messages.py#L80
298
+ for message in messages:
299
+ if message.get("role") == "assistant":
300
+ content = message.get("content")
301
+ if isinstance(content, list):
302
+ content = "\n".join(chunk.get("text") for chunk in content)
303
+ message["content"] = content
304
request = ChatCompletionRequest(messages=messages,
305
tools=tools) # type: ignore[type-var]
306
encoded = self.mistral.encode_chat_completion(request)