@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d01f66b039
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[Bugfix] Fix multi-round chat error when mistral tokenizer is used (#12859)

Signed-off-by: Zifei Tong <zifeitong@gmail.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

zifeitong committed Feb 7, 2025 at 23:04 UTC d01f66b0394e62a11429c8f0afd9a56b7b2b7f0c
1 file changed +10
vllm/transformers_utils/tokenizers/mistral.py
+10
@@ -291,6 +291,16 @@ class MistralTokenizer:
291
292 from mistral_common.protocol.instruct.request import (
293 ChatCompletionRequest)
294 +
295 + # mistral-common requires AssistantMessage content to be string [1].
296 + #
297 + # [1]: https://github.com/mistralai/mistral-common/blob/f4a06998b75ed78bbf5aaf569590b772ea26c9f6/src/mistral_common/protocol/instruct/messages.py#L80
298 + for message in messages:
299 + if message.get("role") == "assistant":
300 + content = message.get("content")
301 + if isinstance(content, list):
302 + content = "\n".join(chunk.get("text") for chunk in content)
303 + message["content"] = content
304 request = ChatCompletionRequest(messages=messages,
305 tools=tools) # type: ignore[type-var]
306 encoded = self.mistral.encode_chat_completion(request)