@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0535e5fe6c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

Fix edge case Mistral tokenizer (#10152)

Patrick von Platen committed Nov 8, 2024 at 16:42 UTC 0535e5fe6c38a25bad71d92bb7a396f04fd1aee5
1 file changed +9 -9
vllm/transformers_utils/tokenizers/mistral.py
+9 -9
@@ -72,11 +72,12 @@ class MistralTokenizer:
72 self.instruct = tokenizer.instruct_tokenizer
73
74 tokenizer_ = tokenizer.instruct_tokenizer.tokenizer
75 - if isinstance(tokenizer_, Tekkenizer):
75 + self.is_tekken = isinstance(tokenizer_, Tekkenizer)
76 + self.is_spm = isinstance(tokenizer_, SentencePieceTokenizer)
77 + if self.is_tekken:
78 # Make sure special tokens will not raise
79 tokenizer_.special_token_policy = SpecialTokenPolicy.IGNORE
78 -
79 - elif isinstance(tokenizer_, SentencePieceTokenizer):
80 + elif self.is_spm:
81 pass
82 else:
83 raise TypeError(f"Unsupported tokenizer: {type(tokenizer_)}")
@@ -218,7 +219,7 @@ class MistralTokenizer:
219 return encoded.tokens
220
221 def convert_tokens_to_string(self, tokens: List[str]) -> str:
221 - if isinstance(self.tokenizer, Tekkenizer):
222 + if self.is_tekken:
223 tokens = [
224 t for t in tokens
225 if t not in self.tokenizer._all_special_tokens
@@ -270,21 +271,20 @@ class MistralTokenizer:
271 skip_special_tokens
272 ), "skip_special_tokens=False is not supported for Mistral tokenizers."
273
273 - assert isinstance(self.tokenizer,
274 - (Tekkenizer, SentencePieceTokenizer)), type(
275 - self.tokenizer)
274 + assert self.is_tekken or self.is_spm, type(self.tokenizer)
275
277 - if isinstance(self.tokenizer, Tekkenizer):
276 + if self.is_tekken:
277 # skip special tokens
278 ids = [i for i in ids if i > self.tokenizer.num_special_tokens]
279
280 tokens = [self.tokenizer.id_to_piece(id) for id in ids]
281
283 - if any("�" in t for t in tokens):
282 + if any("�" in t for t in tokens) and self.is_tekken:
283 # if a decoded token contains the replacement character, then the
284 # token has an incomplete UTF-8 character so we must use bytes
285 # See: https://github.com/vllm-project/vllm/pull/8640
286 # https://github.com/vllm-project/vllm/pull/9625
287 + # if underlying tokenizeir is sentencepiece, we just add "�"
288 tokens = [self.tokenizer.id_to_byte_piece(id) for id in ids]
289
290 return tokens