@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d06980dfa7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Fix Baichuan tokenizer error (#1874)

Woosuk Kwon committed Nov 30, 2023 at 18:35 UTC d06980dfa7e8dcc1738656beb46d3735c86faa21
3 files changed +281
vllm/transformers_utils/tokenizer.py
+13
@@ -4,6 +4,7 @@ from transformers import (AutoTokenizer, PreTrainedTokenizer,
4 PreTrainedTokenizerFast)
5
6 from vllm.logger import init_logger
7 +from vllm.transformers_utils.tokenizers import *
8
9 logger = init_logger(__name__)
10
@@ -61,6 +62,18 @@ def get_tokenizer(
62 raise RuntimeError(err_msg) from e
63 else:
64 raise e
65 + except AttributeError as e:
66 + if "BaichuanTokenizer" in str(e):
67 + # This is for the error "'BaichuanTokenizer' object has no
68 + # attribute 'sp_model'".
69 + tokenizer = BaichuanTokenizer.from_pretrained(
70 + tokenizer_name,
71 + *args,
72 + trust_remote_code=trust_remote_code,
73 + tokenizer_revision=tokenizer_revision,
74 + **kwargs)
75 + else:
76 + raise e
77
78 if not isinstance(tokenizer, PreTrainedTokenizerFast):
79 logger.warning(
vllm/transformers_utils/tokenizers/__init__.py new
+5
@@ -0,0 +1,5 @@
1 +from vllm.transformers_utils.tokenizers.baichuan import BaichuanTokenizer
2 +
3 +__all__ = [
4 + "BaichuanTokenizer",
5 +]
vllm/transformers_utils/tokenizers/baichuan.py new
+263
@@ -0,0 +1,263 @@
1 +# yapf: disable
2 +# Adapted from
3 +# https://huggingface.co/baichuan-inc/Baichuan2-13B-Chat/blob/8f6e343d545c503b91429582231d1d354dac2740/tokenization_baichuan.py
4 +# This includes a fix suggested in
5 +# https://github.com/vllm-project/vllm/issues/1403#issuecomment-1767503058
6 +# Copyright (c) 2023, Baichuan Intelligent Technology. All rights reserved.
7 +
8 +import os
9 +from shutil import copyfile
10 +from typing import Any, Dict, List, Optional, Tuple
11 +
12 +import sentencepiece as spm
13 +from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
14 +from transformers.utils import logging
15 +
16 +
17 +logger = logging.get_logger(__name__)
18 +
19 +VOCAB_FILES_NAMES = {"vocab_file": "tokenizer.model"}
20 +
21 +PRETRAINED_VOCAB_FILES_MAP = {
22 + "vocab_file": {},
23 + "tokenizer_file": {},
24 +}
25 +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {}
26 +
27 +
28 +class BaichuanTokenizer(PreTrainedTokenizer):
29 + """
30 + Construct a Baichuan tokenizer. Based on byte-level Byte-Pair-Encoding.
31 +
32 + Args:
33 + vocab_file (`str`):
34 + Path to the vocabulary file.
35 + """
36 +
37 + vocab_files_names = VOCAB_FILES_NAMES
38 + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
39 + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
40 + model_input_names = ["input_ids", "attention_mask"]
41 +
42 + def __init__(
43 + self,
44 + vocab_file,
45 + unk_token="<unk>",
46 + bos_token="<s>",
47 + eos_token="</s>",
48 + pad_token=None,
49 + sp_model_kwargs: Optional[Dict[str, Any]] = None,
50 + add_bos_token=True,
51 + add_eos_token=False,
52 + clean_up_tokenization_spaces=False,
53 + **kwargs,
54 + ):
55 + self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
56 + bos_token = (
57 + AddedToken(bos_token, lstrip=False, rstrip=False)
58 + if isinstance(bos_token, str)
59 + else bos_token
60 + )
61 + eos_token = (
62 + AddedToken(eos_token, lstrip=False, rstrip=False)
63 + if isinstance(eos_token, str)
64 + else eos_token
65 + )
66 + unk_token = (
67 + AddedToken(unk_token, lstrip=False, rstrip=False)
68 + if isinstance(unk_token, str)
69 + else unk_token
70 + )
71 + pad_token = (
72 + AddedToken(pad_token, lstrip=False, rstrip=False)
73 + if isinstance(pad_token, str)
74 + else pad_token
75 + )
76 + self.vocab_file = vocab_file
77 + self.add_bos_token = add_bos_token
78 + self.add_eos_token = add_eos_token
79 + self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
80 + self.sp_model.Load(vocab_file)
81 + super().__init__(
82 + bos_token=bos_token,
83 + eos_token=eos_token,
84 + unk_token=unk_token,
85 + pad_token=pad_token,
86 + add_bos_token=add_bos_token,
87 + add_eos_token=add_eos_token,
88 + sp_model_kwargs=self.sp_model_kwargs,
89 + clean_up_tokenization_spaces=clean_up_tokenization_spaces,
90 + **kwargs,
91 + )
92 +
93 + def __getstate__(self):
94 + state = self.__dict__.copy()
95 + state["sp_model"] = None
96 + return state
97 +
98 + def __setstate__(self, d):
99 + self.__dict__ = d
100 + self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
101 + self.sp_model.Load(self.vocab_file)
102 +
103 + @property
104 + def vocab_size(self):
105 + """Returns vocab size"""
106 + return self.sp_model.get_piece_size()
107 +
108 + def get_vocab(self):
109 + """Returns vocab as a dict"""
110 + vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
111 + vocab.update(self.added_tokens_encoder)
112 + return vocab
113 +
114 + def _tokenize(self, text):
115 + """Returns a tokenized string."""
116 + return self.sp_model.encode(text, out_type=str)
117 +
118 + def _convert_token_to_id(self, token):
119 + """Converts a token (str) in an id using the vocab."""
120 + return self.sp_model.piece_to_id(token)
121 +
122 + def _convert_id_to_token(self, index):
123 + """Converts an index (integer) in a token (str) using the vocab."""
124 + token = self.sp_model.IdToPiece(index)
125 + return token
126 +
127 + def convert_tokens_to_string(self, tokens):
128 + """Converts a sequence of tokens (string) in a single string."""
129 + current_sub_tokens = []
130 + out_string = ""
131 + prev_is_special = False
132 + for i, token in enumerate(tokens):
133 + # make sure that special tokens are not decoded using sentencepiece model
134 + if token in self.all_special_tokens:
135 + if not prev_is_special and i != 0:
136 + out_string += " "
137 + out_string += self.sp_model.decode(current_sub_tokens) + token
138 + prev_is_special = True
139 + current_sub_tokens = []
140 + else:
141 + current_sub_tokens.append(token)
142 + prev_is_special = False
143 + out_string += self.sp_model.decode(current_sub_tokens)
144 + return out_string
145 +
146 + def save_vocabulary(
147 + self, save_directory, filename_prefix: Optional[str] = None
148 + ) -> Tuple[str]:
149 + """
150 + Save the vocabulary and special tokens file to a directory.
151 +
152 + Args:
153 + save_directory (`str`):
154 + The directory in which to save the vocabulary.
155 +
156 + Returns:
157 + `Tuple(str)`: Paths to the files saved.
158 + """
159 + if not os.path.isdir(save_directory):
160 + logger.error(f"Vocabulary path ({save_directory}) should be a directory")
161 + return
162 + out_vocab_file = os.path.join(
163 + save_directory,
164 + (filename_prefix + "-" if filename_prefix else "")
165 + + VOCAB_FILES_NAMES["vocab_file"],
166 + )
167 +
168 + if os.path.abspath(self.vocab_file) != os.path.abspath(
169 + out_vocab_file
170 + ) and os.path.isfile(self.vocab_file):
171 + copyfile(self.vocab_file, out_vocab_file)
172 + elif not os.path.isfile(self.vocab_file):
173 + with open(out_vocab_file, "wb") as fi:
174 + content_spiece_model = self.sp_model.serialized_model_proto()
175 + fi.write(content_spiece_model)
176 +
177 + return (out_vocab_file,)
178 +
179 + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
180 + bos_token_id = [self.bos_token_id] if self.add_bos_token else []
181 + eos_token_id = [self.eos_token_id] if self.add_eos_token else []
182 +
183 + output = bos_token_id + token_ids_0 + eos_token_id
184 +
185 + if token_ids_1 is not None:
186 + output = output + bos_token_id + token_ids_1 + eos_token_id
187 +
188 + return output
189 +
190 + def get_special_tokens_mask(
191 + self,
192 + token_ids_0: List[int],
193 + token_ids_1: Optional[List[int]] = None,
194 + already_has_special_tokens: bool = False,
195 + ) -> List[int]:
196 + """
197 + Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
198 + special tokens using the tokenizer `prepare_for_model` method.
199 +
200 + Args:
201 + token_ids_0 (`List[int]`):
202 + List of IDs.
203 + token_ids_1 (`List[int]`, *optional*):
204 + Optional second list of IDs for sequence pairs.
205 + already_has_special_tokens (`bool`, *optional*, defaults to `False`):
206 + Whether or not the token list is already formatted with special tokens for the model.
207 +
208 + Returns:
209 + `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
210 + """
211 + if already_has_special_tokens:
212 + return super().get_special_tokens_mask(
213 + token_ids_0=token_ids_0,
214 + token_ids_1=token_ids_1,
215 + already_has_special_tokens=True,
216 + )
217 +
218 + bos_token_id = [1] if self.add_bos_token else []
219 + eos_token_id = [1] if self.add_eos_token else []
220 +
221 + if token_ids_1 is None:
222 + return bos_token_id + ([0] * len(token_ids_0)) + eos_token_id
223 + return (
224 + bos_token_id
225 + + ([0] * len(token_ids_0))
226 + + eos_token_id
227 + + bos_token_id
228 + + ([0] * len(token_ids_1))
229 + + eos_token_id
230 + )
231 +
232 + def create_token_type_ids_from_sequences(
233 + self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
234 + ) -> List[int]:
235 + """
236 + Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT
237 + sequence pair mask has the following format:
238 +
239 + ```
240 + 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
241 + | first sequence | second sequence |
242 + ```
243 +
244 + if token_ids_1 is None, only returns the first portion of the mask (0s).
245 +
246 + Args:
247 + token_ids_0 (`List[int]`):
248 + List of ids.
249 + token_ids_1 (`List[int]`, *optional*):
250 + Optional second list of IDs for sequence pairs.
251 +
252 + Returns:
253 + `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
254 + """
255 + bos_token_id = [self.bos_token_id] if self.add_bos_token else []
256 + eos_token_id = [self.eos_token_id] if self.add_eos_token else []
257 +
258 + output = [0] * len(bos_token_id + token_ids_0 + eos_token_id)
259 +
260 + if token_ids_1 is not None:
261 + output += [1] * len(bos_token_id + token_ids_1 + eos_token_id)
262 +
263 + return output