@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 54cacf008f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Bugfix] Mistral tokenizer encode accept list of str (#12149)

Signed-off-by: Kunshang Ji <kunshang.ji@intel.com>

Kunshang Ji committed Jan 18, 2025 at 00:47 UTC 54cacf008f00d35d46273fed4d538cf5740d0965
1 file changed +30 -8
vllm/transformers_utils/tokenizers/mistral.py
+30 -8
@@ -18,6 +18,7 @@ from mistral_common.tokens.tokenizers.tekken import (SpecialTokenPolicy,
18 Tekkenizer)
19
20 from vllm.logger import init_logger
21 +from vllm.utils import is_list_of
22
23 if TYPE_CHECKING:
24 from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
@@ -27,7 +28,7 @@ logger = init_logger(__name__)
28
29 @dataclass
30 class Encoding:
30 - input_ids: List[int]
31 + input_ids: Union[List[int], List[List[int]]]
32
33
34 def maybe_serialize_tool_calls(request: ChatCompletionRequest):
@@ -223,17 +224,25 @@ class MistralTokenizer:
224
225 def __call__(
226 self,
226 - prompt: str,
227 + prompt: Union[str, List[str], List[int]],
228 add_special_tokens: bool = False,
229 truncation: bool = False,
230 max_length: Optional[int] = None,
231 ):
231 - # Mistral Tokenizers should not add special tokens
232 - input_ids = self.encode(prompt)
233 -
234 - if truncation:
235 - input_ids = input_ids[:max_length]
236 -
232 + input_ids: Union[List[int], List[List[int]]]
233 + # For List[str], original prompt text
234 + if is_list_of(prompt, str):
235 + input_ids_: List[List[int]] = []
236 + for p in prompt:
237 + each_input_ids = self.encode_one(p, truncation, max_length)
238 + input_ids_.append(each_input_ids)
239 + input_ids = input_ids_
240 + # For List[int], apply chat template output, already tokens.
241 + elif is_list_of(prompt, int):
242 + input_ids = prompt
243 + # For str, single prompt text
244 + else:
245 + input_ids = self.encode_one(prompt, truncation, max_length)
246 return Encoding(input_ids=input_ids)
247
248 def get_vocab(self) -> Dict[str, int]:
@@ -245,6 +254,19 @@ class MistralTokenizer:
254 # Mistral tokenizers have no added vocabulary
255 return {}
256
257 + def encode_one(
258 + self,
259 + prompt: str,
260 + truncation: bool = False,
261 + max_length: Optional[int] = None,
262 + ) -> List[int]:
263 + # Mistral Tokenizers should not add special tokens
264 + input_ids = self.encode(prompt)
265 +
266 + if truncation:
267 + input_ids = input_ids[:max_length]
268 + return input_ids
269 +
270 def encode(self, prompt: str) -> List[int]:
271 # `encode` should only be used for prompt completion
272 # it should never be used for chat_completion.