A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Bugfix] Allow prefill of assistant response when using `mistral_common` (#9446)
sasha0552 committed
Oct 17, 2024 at 15:06 UTC
5e443b594fab5c4e93b462a0206ddd24b2e40238
1 file changed
+4
vllm/transformers_utils/tokenizers/mistral.py
+4
@@ -166,6 +166,10 @@ class MistralTokenizer:
166
tools: Optional[Dict[str, Any]] = None,
167
**kwargs) -> List[int]:
168
169
+ last_message = messages[-1]
170
+ if last_message["role"] == "assistant":
171
+ last_message["prefix"] = True
172
+
173
request = ChatCompletionRequest(messages=messages,
174
tools=tools) # type: ignore[type-var]
175
encoded = self.mistral.encode_chat_completion(request)