A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Fix Baichuan tokenizer error (#1874)
Woosuk Kwon committed
Nov 30, 2023 at 18:35 UTC
d06980dfa7e8dcc1738656beb46d3735c86faa21
3 files changed
+281
vllm/transformers_utils/tokenizer.py
+13
@@ -4,6 +4,7 @@ from transformers import (AutoTokenizer, PreTrainedTokenizer,
4
PreTrainedTokenizerFast)
5
6
from vllm.logger import init_logger
7
+from vllm.transformers_utils.tokenizers import *
8
9
logger = init_logger(__name__)
10
@@ -61,6 +62,18 @@ def get_tokenizer(
62
raise RuntimeError(err_msg) from e
63
else:
64
raise e
65
+ except AttributeError as e:
66
+ if "BaichuanTokenizer" in str(e):
67
+ # This is for the error "'BaichuanTokenizer' object has no
68
+ # attribute 'sp_model'".
69
+ tokenizer = BaichuanTokenizer.from_pretrained(
70
+ tokenizer_name,
71
+ *args,
72
+ trust_remote_code=trust_remote_code,
73
+ tokenizer_revision=tokenizer_revision,
74
+ **kwargs)
75
+ else:
76
+ raise e
77
78
if not isinstance(tokenizer, PreTrainedTokenizerFast):
79
logger.warning(
vllm/transformers_utils/tokenizers/__init__.py
new
+5
@@ -0,0 +1,5 @@
1
+from vllm.transformers_utils.tokenizers.baichuan import BaichuanTokenizer
2
+
3
+__all__ = [
4
+ "BaichuanTokenizer",
5
+]
vllm/transformers_utils/tokenizers/baichuan.py
new
+263
@@ -0,0 +1,263 @@
1
+# yapf: disable
2
+# Adapted from
3
+# https://huggingface.co/baichuan-inc/Baichuan2-13B-Chat/blob/8f6e343d545c503b91429582231d1d354dac2740/tokenization_baichuan.py
4
+# This includes a fix suggested in
5
+# https://github.com/vllm-project/vllm/issues/1403#issuecomment-1767503058
6
+# Copyright (c) 2023, Baichuan Intelligent Technology. All rights reserved.
7
+
8
+import os
9
+from shutil import copyfile
10
+from typing import Any, Dict, List, Optional, Tuple
11
+
12
+import sentencepiece as spm
13
+from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
14
+from transformers.utils import logging
15
+
16
+
17
+logger = logging.get_logger(__name__)
18
+
19
+VOCAB_FILES_NAMES = {"vocab_file": "tokenizer.model"}
20
+
21
+PRETRAINED_VOCAB_FILES_MAP = {
22
+ "vocab_file": {},
23
+ "tokenizer_file": {},
24
+}
25
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {}
26
+
27
+
28
+class BaichuanTokenizer(PreTrainedTokenizer):
29
+ """
30
+ Construct a Baichuan tokenizer. Based on byte-level Byte-Pair-Encoding.
31
+
32
+ Args:
33
+ vocab_file (`str`):
34
+ Path to the vocabulary file.
35
+ """
36
+
37
+ vocab_files_names = VOCAB_FILES_NAMES
38
+ pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
39
+ max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
40
+ model_input_names = ["input_ids", "attention_mask"]
41
+
42
+ def __init__(
43
+ self,
44
+ vocab_file,
45
+ unk_token="<unk>",
46
+ bos_token="<s>",
47
+ eos_token="</s>",
48
+ pad_token=None,
49
+ sp_model_kwargs: Optional[Dict[str, Any]] = None,
50
+ add_bos_token=True,
51
+ add_eos_token=False,
52
+ clean_up_tokenization_spaces=False,
53
+ **kwargs,
54
+ ):
55
+ self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
56
+ bos_token = (
57
+ AddedToken(bos_token, lstrip=False, rstrip=False)
58
+ if isinstance(bos_token, str)
59
+ else bos_token
60
+ )
61
+ eos_token = (
62
+ AddedToken(eos_token, lstrip=False, rstrip=False)
63
+ if isinstance(eos_token, str)
64
+ else eos_token
65
+ )
66
+ unk_token = (
67
+ AddedToken(unk_token, lstrip=False, rstrip=False)
68
+ if isinstance(unk_token, str)
69
+ else unk_token
70
+ )
71
+ pad_token = (
72
+ AddedToken(pad_token, lstrip=False, rstrip=False)
73
+ if isinstance(pad_token, str)
74
+ else pad_token
75
+ )
76
+ self.vocab_file = vocab_file
77
+ self.add_bos_token = add_bos_token
78
+ self.add_eos_token = add_eos_token
79
+ self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
80
+ self.sp_model.Load(vocab_file)
81
+ super().__init__(
82
+ bos_token=bos_token,
83
+ eos_token=eos_token,
84
+ unk_token=unk_token,
85
+ pad_token=pad_token,
86
+ add_bos_token=add_bos_token,
87
+ add_eos_token=add_eos_token,
88
+ sp_model_kwargs=self.sp_model_kwargs,
89
+ clean_up_tokenization_spaces=clean_up_tokenization_spaces,
90
+ **kwargs,
91
+ )
92
+
93
+ def __getstate__(self):
94
+ state = self.__dict__.copy()
95
+ state["sp_model"] = None
96
+ return state
97
+
98
+ def __setstate__(self, d):
99
+ self.__dict__ = d
100
+ self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
101
+ self.sp_model.Load(self.vocab_file)
102
+
103
+ @property
104
+ def vocab_size(self):
105
+ """Returns vocab size"""
106
+ return self.sp_model.get_piece_size()
107
+
108
+ def get_vocab(self):
109
+ """Returns vocab as a dict"""
110
+ vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
111
+ vocab.update(self.added_tokens_encoder)
112
+ return vocab
113
+
114
+ def _tokenize(self, text):
115
+ """Returns a tokenized string."""
116
+ return self.sp_model.encode(text, out_type=str)
117
+
118
+ def _convert_token_to_id(self, token):
119
+ """Converts a token (str) in an id using the vocab."""
120
+ return self.sp_model.piece_to_id(token)
121
+
122
+ def _convert_id_to_token(self, index):
123
+ """Converts an index (integer) in a token (str) using the vocab."""
124
+ token = self.sp_model.IdToPiece(index)
125
+ return token
126
+
127
+ def convert_tokens_to_string(self, tokens):
128
+ """Converts a sequence of tokens (string) in a single string."""
129
+ current_sub_tokens = []
130
+ out_string = ""
131
+ prev_is_special = False
132
+ for i, token in enumerate(tokens):
133
+ # make sure that special tokens are not decoded using sentencepiece model
134
+ if token in self.all_special_tokens:
135
+ if not prev_is_special and i != 0:
136
+ out_string += " "
137
+ out_string += self.sp_model.decode(current_sub_tokens) + token
138
+ prev_is_special = True
139
+ current_sub_tokens = []
140
+ else:
141
+ current_sub_tokens.append(token)
142
+ prev_is_special = False
143
+ out_string += self.sp_model.decode(current_sub_tokens)
144
+ return out_string
145
+
146
+ def save_vocabulary(
147
+ self, save_directory, filename_prefix: Optional[str] = None
148
+ ) -> Tuple[str]:
149
+ """
150
+ Save the vocabulary and special tokens file to a directory.
151
+
152
+ Args:
153
+ save_directory (`str`):
154
+ The directory in which to save the vocabulary.
155
+
156
+ Returns:
157
+ `Tuple(str)`: Paths to the files saved.
158
+ """
159
+ if not os.path.isdir(save_directory):
160
+ logger.error(f"Vocabulary path ({save_directory}) should be a directory")
161
+ return
162
+ out_vocab_file = os.path.join(
163
+ save_directory,
164
+ (filename_prefix + "-" if filename_prefix else "")
165
+ + VOCAB_FILES_NAMES["vocab_file"],
166
+ )
167
+
168
+ if os.path.abspath(self.vocab_file) != os.path.abspath(
169
+ out_vocab_file
170
+ ) and os.path.isfile(self.vocab_file):
171
+ copyfile(self.vocab_file, out_vocab_file)
172
+ elif not os.path.isfile(self.vocab_file):
173
+ with open(out_vocab_file, "wb") as fi:
174
+ content_spiece_model = self.sp_model.serialized_model_proto()
175
+ fi.write(content_spiece_model)
176
+
177
+ return (out_vocab_file,)
178
+
179
+ def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
180
+ bos_token_id = [self.bos_token_id] if self.add_bos_token else []
181
+ eos_token_id = [self.eos_token_id] if self.add_eos_token else []
182
+
183
+ output = bos_token_id + token_ids_0 + eos_token_id
184
+
185
+ if token_ids_1 is not None:
186
+ output = output + bos_token_id + token_ids_1 + eos_token_id
187
+
188
+ return output
189
+
190
+ def get_special_tokens_mask(
191
+ self,
192
+ token_ids_0: List[int],
193
+ token_ids_1: Optional[List[int]] = None,
194
+ already_has_special_tokens: bool = False,
195
+ ) -> List[int]:
196
+ """
197
+ Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
198
+ special tokens using the tokenizer `prepare_for_model` method.
199
+
200
+ Args:
201
+ token_ids_0 (`List[int]`):
202
+ List of IDs.
203
+ token_ids_1 (`List[int]`, *optional*):
204
+ Optional second list of IDs for sequence pairs.
205
+ already_has_special_tokens (`bool`, *optional*, defaults to `False`):
206
+ Whether or not the token list is already formatted with special tokens for the model.
207
+
208
+ Returns:
209
+ `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
210
+ """
211
+ if already_has_special_tokens:
212
+ return super().get_special_tokens_mask(
213
+ token_ids_0=token_ids_0,
214
+ token_ids_1=token_ids_1,
215
+ already_has_special_tokens=True,
216
+ )
217
+
218
+ bos_token_id = [1] if self.add_bos_token else []
219
+ eos_token_id = [1] if self.add_eos_token else []
220
+
221
+ if token_ids_1 is None:
222
+ return bos_token_id + ([0] * len(token_ids_0)) + eos_token_id
223
+ return (
224
+ bos_token_id
225
+ + ([0] * len(token_ids_0))
226
+ + eos_token_id
227
+ + bos_token_id
228
+ + ([0] * len(token_ids_1))
229
+ + eos_token_id
230
+ )
231
+
232
+ def create_token_type_ids_from_sequences(
233
+ self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
234
+ ) -> List[int]:
235
+ """
236
+ Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT
237
+ sequence pair mask has the following format:
238
+
239
+ ```
240
+ 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
241
+ | first sequence | second sequence |
242
+ ```
243
+
244
+ if token_ids_1 is None, only returns the first portion of the mask (0s).
245
+
246
+ Args:
247
+ token_ids_0 (`List[int]`):
248
+ List of ids.
249
+ token_ids_1 (`List[int]`, *optional*):
250
+ Optional second list of IDs for sequence pairs.
251
+
252
+ Returns:
253
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
254
+ """
255
+ bos_token_id = [self.bos_token_id] if self.add_bos_token else []
256
+ eos_token_id = [self.eos_token_id] if self.add_eos_token else []
257
+
258
+ output = [0] * len(bos_token_id + token_ids_0 + eos_token_id)
259
+
260
+ if token_ids_1 is not None:
261
+ output += [1] * len(bos_token_id + token_ids_1 + eos_token_id)
262
+
263
+ return output