A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
Fix edge case Mistral tokenizer (#10152)
Patrick von Platen committed
Nov 8, 2024 at 16:42 UTC
0535e5fe6c38a25bad71d92bb7a396f04fd1aee5
1 file changed
+9
-9
vllm/transformers_utils/tokenizers/mistral.py
+9
-9
@@ -72,11 +72,12 @@ class MistralTokenizer:
72
self.instruct = tokenizer.instruct_tokenizer
73
74
tokenizer_ = tokenizer.instruct_tokenizer.tokenizer
75
- if isinstance(tokenizer_, Tekkenizer):
75
+ self.is_tekken = isinstance(tokenizer_, Tekkenizer)
76
+ self.is_spm = isinstance(tokenizer_, SentencePieceTokenizer)
77
+ if self.is_tekken:
78
# Make sure special tokens will not raise
79
tokenizer_.special_token_policy = SpecialTokenPolicy.IGNORE
78
-
79
- elif isinstance(tokenizer_, SentencePieceTokenizer):
80
+ elif self.is_spm:
81
pass
82
else:
83
raise TypeError(f"Unsupported tokenizer: {type(tokenizer_)}")
@@ -218,7 +219,7 @@ class MistralTokenizer:
219
return encoded.tokens
220
221
def convert_tokens_to_string(self, tokens: List[str]) -> str:
221
- if isinstance(self.tokenizer, Tekkenizer):
222
+ if self.is_tekken:
223
tokens = [
224
t for t in tokens
225
if t not in self.tokenizer._all_special_tokens
@@ -270,21 +271,20 @@ class MistralTokenizer:
271
skip_special_tokens
272
), "skip_special_tokens=False is not supported for Mistral tokenizers."
273
273
- assert isinstance(self.tokenizer,
274
- (Tekkenizer, SentencePieceTokenizer)), type(
275
- self.tokenizer)
274
+ assert self.is_tekken or self.is_spm, type(self.tokenizer)
275
277
- if isinstance(self.tokenizer, Tekkenizer):
276
+ if self.is_tekken:
277
# skip special tokens
278
ids = [i for i in ids if i > self.tokenizer.num_special_tokens]
279
280
tokens = [self.tokenizer.id_to_piece(id) for id in ids]
281
283
- if any("�" in t for t in tokens):
282
+ if any("�" in t for t in tokens) and self.is_tekken:
283
# if a decoded token contains the replacement character, then the
284
# token has an incomplete UTF-8 character so we must use bytes
285
# See: https://github.com/vllm-project/vllm/pull/8640
286
# https://github.com/vllm-project/vllm/pull/9625
287
+ # if underlying tokenizeir is sentencepiece, we just add "�"
288
tokens = [self.tokenizer.id_to_byte_piece(id) for id in ids]
289
290
return tokens