A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Bugfix] Add missing attributes in mistral tokenizer (#8364)
Cyrus Leung committed
Sep 12, 2024 at 02:36 UTC
7015417fd4910a47263ea34c79c2cdb2ff314fdf
2 files changed
+63
-32
vllm/entrypoints/chat_utils.py
+5
-2
@@ -519,11 +519,14 @@ def apply_hf_chat_template(
519
def apply_mistral_chat_template(
520
tokenizer: MistralTokenizer,
521
messages: List[ChatCompletionMessageParam],
522
- chat_template: Optional[str],
522
+ chat_template: Optional[str] = None,
523
**kwargs: Any,
524
) -> List[int]:
525
+ if chat_template is not None:
526
+ logger.warning(
527
+ "'chat_template' cannot be overridden for mistral tokenizer.")
528
+
529
return tokenizer.apply_chat_template(
530
messages=messages,
527
- chat_template=chat_template,
531
**kwargs,
532
)
vllm/transformers_utils/tokenizers/mistral.py
+58
-30
@@ -45,26 +45,25 @@ class MistralTokenizer:
45
def __init__(self, tokenizer: PublicMistralTokenizer) -> None:
46
self.mistral = tokenizer
47
self.instruct = tokenizer.instruct_tokenizer
48
- self.tokenizer = tokenizer.instruct_tokenizer.tokenizer
48
50
- self.vocab_size = len(self.tokenizer.vocab())
51
-
52
- assert isinstance(self.tokenizer,
53
- (Tekkenizer, SentencePieceTokenizer)), type(
54
- self.tokenizer)
55
-
56
- if (is_tekken := isinstance(self.tokenizer, Tekkenizer)):
49
+ tokenizer_ = tokenizer.instruct_tokenizer.tokenizer
50
+ if isinstance(tokenizer_, Tekkenizer):
51
# Make sure special tokens will not raise
58
- self.tokenizer.special_token_policy = SpecialTokenPolicy.IGNORE
59
-
60
- self._is_tekken = is_tekken
52
+ tokenizer_.special_token_policy = SpecialTokenPolicy.IGNORE
53
+
54
+ self._vocab = {
55
+ token: idx
56
+ for idx, token in enumerate(tokenizer_.vocab())
57
+ }
58
+ elif isinstance(tokenizer_, SentencePieceTokenizer):
59
+ self._vocab = {
60
+ token: idx
61
+ for idx, token in enumerate(tokenizer_.vocab())
62
+ }
63
+ else:
64
+ raise TypeError(f"Unsupported tokenizer: {type(tokenizer_)}")
65
62
- # the following attributes are set to fit VLLM's design
63
- self.is_fast = True
64
- self.chat_template = True
65
- self.all_special_ids: List[Any] = []
66
- self.all_special_tokens: List[Any] = []
67
- self.all_special_tokens_extended: List[Any] = []
66
+ self.tokenizer = tokenizer_
67
68
@classmethod
69
def from_pretrained(cls,
@@ -102,6 +101,38 @@ class MistralTokenizer:
101
revision=revision)
102
return tokenizer_file
103
104
+ # the following attributes are set to fit VLLM's design
105
+ @property
106
+ def all_special_tokens_extended(self) -> List[str]:
107
+ return []
108
+
109
+ @property
110
+ def all_special_tokens(self) -> List[str]:
111
+ return []
112
+
113
+ @property
114
+ def all_special_ids(self) -> List[int]:
115
+ return []
116
+
117
+ @property
118
+ def bos_token_id(self) -> int:
119
+ return self.tokenizer.bos_id
120
+
121
+ @property
122
+ def eos_token_id(self) -> int:
123
+ return self.tokenizer.eos_id
124
+
125
+ @property
126
+ def is_fast(self) -> bool:
127
+ return True
128
+
129
+ @property
130
+ def vocab_size(self) -> int:
131
+ return len(self._vocab)
132
+
133
+ def __len__(self) -> int:
134
+ return self.vocab_size
135
+
136
def __call__(
137
self,
138
prompt: str,
@@ -117,9 +148,12 @@ class MistralTokenizer:
148
149
return Encoding(input_ids=input_ids)
150
120
- def get_added_vocab(self) -> List[str]:
151
+ def get_vocab(self) -> Dict[str, int]:
152
+ return self._vocab
153
+
154
+ def get_added_vocab(self) -> Dict[str, int]:
155
# Mistral tokenizers have no added vocabulary
122
- return []
156
+ return {}
157
158
def encode(self, prompt: str) -> List[int]:
159
# `encode` should only be used for prompt completion
@@ -141,7 +175,7 @@ class MistralTokenizer:
175
return encoded.tokens
176
177
def convert_tokens_to_string(self, tokens: List[str]) -> str:
144
- if self._is_tekken:
178
+ if isinstance(self.tokenizer, Tekkenizer):
179
return "".join(tokens)
180
else:
181
return self.tokenizer.decode(tokens) # type: ignore[arg-type]
@@ -151,14 +185,11 @@ class MistralTokenizer:
185
ids = [ids]
186
return self.tokenizer.decode(ids)
187
154
- @property
155
- def eos_token_id(self):
156
- return self.tokenizer.eos_id
157
-
188
def convert_ids_to_tokens(
159
- self,
160
- ids: List[int],
161
- skip_special_tokens: Optional[bool] = True) -> List[str]:
189
+ self,
190
+ ids: List[int],
191
+ skip_special_tokens: bool = True,
192
+ ) -> List[str]:
193
# TODO(Patrick) - potentially allow special tokens to not be skipped
194
assert (
195
skip_special_tokens
@@ -170,6 +201,3 @@ class MistralTokenizer:
201
202
tokens = [self.tokenizer.id_to_piece(id) for id in ids]
203
return tokens
173
-
174
- def __len__(self):
175
- return self.vocab_size