A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[RFC][vllm-API] Support tokenizer registry for customized tokenizer in vLLM (#12518)
Signed-off-by: Keyun Tong <tongkeyun@gmail.com>
Keyun Tong committed
Feb 11, 2025 at 20:25 UTC
3ee696a63dd0c2acee44809a3bedec33ea27dfa0
11 files changed
+343
-41
benchmarks/benchmark_serving.py
+3
-2
@@ -1275,11 +1275,12 @@ if __name__ == "__main__":
1275
'--tokenizer-mode',
1276
type=str,
1277
default="auto",
1278
- choices=['auto', 'slow', 'mistral'],
1278
+ choices=['auto', 'slow', 'mistral', 'custom'],
1279
help='The tokenizer mode.\n\n* "auto" will use the '
1280
'fast tokenizer if available.\n* "slow" will '
1281
'always use the slow tokenizer. \n* '
1282
- '"mistral" will always use the `mistral_common` tokenizer.')
1282
+ '"mistral" will always use the `mistral_common` tokenizer. \n*'
1283
+ '"custom" will use --tokenizer to select the preregistered tokenizer.')
1284
1285
parser.add_argument("--served-model-name",
1286
type=str,
tests/tokenization/test_tokenizer_registry.py
new
+123
@@ -0,0 +1,123 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+
3
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
4
+
5
+from vllm.transformers_utils.tokenizer import get_tokenizer
6
+from vllm.transformers_utils.tokenizer_base import (TokenizerBase,
7
+ TokenizerRegistry)
8
+
9
+if TYPE_CHECKING:
10
+ from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
11
+
12
+
13
+class TestTokenizer(TokenizerBase):
14
+
15
+ @classmethod
16
+ def from_pretrained(cls, *args, **kwargs) -> "TestTokenizer":
17
+ return TestTokenizer()
18
+
19
+ @property
20
+ def all_special_tokens_extended(self) -> List[str]:
21
+ raise NotImplementedError()
22
+
23
+ @property
24
+ def all_special_tokens(self) -> List[str]:
25
+ raise NotImplementedError()
26
+
27
+ @property
28
+ def all_special_ids(self) -> List[int]:
29
+ raise NotImplementedError()
30
+
31
+ @property
32
+ def bos_token_id(self) -> int:
33
+ return 0
34
+
35
+ @property
36
+ def eos_token_id(self) -> int:
37
+ return 1
38
+
39
+ @property
40
+ def sep_token(self) -> str:
41
+ raise NotImplementedError()
42
+
43
+ @property
44
+ def pad_token(self) -> str:
45
+ raise NotImplementedError()
46
+
47
+ @property
48
+ def is_fast(self) -> bool:
49
+ raise NotImplementedError()
50
+
51
+ @property
52
+ def vocab_size(self) -> int:
53
+ raise NotImplementedError()
54
+
55
+ @property
56
+ def max_token_id(self) -> int:
57
+ raise NotImplementedError()
58
+
59
+ def __call__(
60
+ self,
61
+ text: Union[str, List[str], List[int]],
62
+ text_pair: Optional[str] = None,
63
+ add_special_tokens: bool = False,
64
+ truncation: bool = False,
65
+ max_length: Optional[int] = None,
66
+ ):
67
+ raise NotImplementedError()
68
+
69
+ def get_vocab(self) -> Dict[str, int]:
70
+ raise NotImplementedError()
71
+
72
+ def get_added_vocab(self) -> Dict[str, int]:
73
+ raise NotImplementedError()
74
+
75
+ def encode_one(
76
+ self,
77
+ text: str,
78
+ truncation: bool = False,
79
+ max_length: Optional[int] = None,
80
+ ) -> List[int]:
81
+ raise NotImplementedError()
82
+
83
+ def encode(self,
84
+ text: str,
85
+ add_special_tokens: Optional[bool] = None) -> List[int]:
86
+ raise NotImplementedError()
87
+
88
+ def apply_chat_template(self,
89
+ messages: List["ChatCompletionMessageParam"],
90
+ tools: Optional[List[Dict[str, Any]]] = None,
91
+ **kwargs) -> List[int]:
92
+ raise NotImplementedError()
93
+
94
+ def convert_tokens_to_string(self, tokens: List[str]) -> str:
95
+ raise NotImplementedError()
96
+
97
+ def decode(self,
98
+ ids: Union[List[int], int],
99
+ skip_special_tokens: bool = True) -> str:
100
+ raise NotImplementedError()
101
+
102
+ def convert_ids_to_tokens(
103
+ self,
104
+ ids: List[int],
105
+ skip_special_tokens: bool = True,
106
+ ) -> List[str]:
107
+ raise NotImplementedError()
108
+
109
+
110
+def test_customized_tokenizer():
111
+ TokenizerRegistry.register("test_tokenizer",
112
+ "tests.tokenization.test_tokenizer_registry",
113
+ "TestTokenizer")
114
+
115
+ tokenizer = TokenizerRegistry.get_tokenizer("test_tokenizer")
116
+ assert isinstance(tokenizer, TestTokenizer)
117
+ assert tokenizer.bos_token_id == 0
118
+ assert tokenizer.eos_token_id == 1
119
+
120
+ tokenizer = get_tokenizer("test_tokenizer", tokenizer_mode="custom")
121
+ assert isinstance(tokenizer, TestTokenizer)
122
+ assert tokenizer.bos_token_id == 0
123
+ assert tokenizer.eos_token_id == 1
vllm/config.py
+5
-4
@@ -102,8 +102,9 @@ class ModelConfig:
102
it; otherwise, you must specify explicitly which task to use.
103
tokenizer: Name or path of the huggingface tokenizer to use.
104
tokenizer_mode: Tokenizer mode. "auto" will use the fast tokenizer if
105
- available, "slow" will always use the slow tokenizer, and
106
- "mistral" will always use the tokenizer from `mistral_common`.
105
+ available, "slow" will always use the slow tokenizer,
106
+ "mistral" will always use the tokenizer from `mistral_common`, and
107
+ "custom" will use --tokenizer to select the preregistered tokenizer.
108
trust_remote_code: Trust remote code (e.g., from HuggingFace) when
109
downloading the model and tokenizer.
110
allowed_local_media_path: Allowing API requests to read local images or
@@ -467,10 +468,10 @@ class ModelConfig:
468
469
def _verify_tokenizer_mode(self) -> None:
470
tokenizer_mode = self.tokenizer_mode.lower()
470
- if tokenizer_mode not in ["auto", "slow", "mistral"]:
471
+ if tokenizer_mode not in ["auto", "slow", "mistral", "custom"]:
472
raise ValueError(
473
f"Unknown tokenizer mode: {self.tokenizer_mode}. Must be "
473
- "either 'auto', 'slow' or 'mistral'.")
474
+ "either 'auto', 'slow', 'mistral' or 'custom'.")
475
self.tokenizer_mode = tokenizer_mode
476
477
def _get_preferred_task(
vllm/engine/arg_utils.py
+4
-2
@@ -284,11 +284,13 @@ class EngineArgs:
284
'--tokenizer-mode',
285
type=str,
286
default=EngineArgs.tokenizer_mode,
287
- choices=['auto', 'slow', 'mistral'],
287
+ choices=['auto', 'slow', 'mistral', 'custom'],
288
help='The tokenizer mode.\n\n* "auto" will use the '
289
'fast tokenizer if available.\n* "slow" will '
290
'always use the slow tokenizer. \n* '
291
- '"mistral" will always use the `mistral_common` tokenizer.')
291
+ '"mistral" will always use the `mistral_common` tokenizer. \n* '
292
+ '"custom" will use --tokenizer to select the '
293
+ 'preregistered tokenizer.')
294
parser.add_argument('--trust-remote-code',
295
action='store_true',
296
help='Trust remote code from huggingface.')
vllm/entrypoints/llm.py
+19
-12
@@ -1051,9 +1051,9 @@ class LLM:
1051
1052
def _cross_encoding_score(
1053
self,
1054
- tokenizer: Union[AnyTokenizer],
1055
- text_1: List[Union[str, TextPrompt, TokensPrompt]],
1056
- text_2: List[Union[str, TextPrompt, TokensPrompt]],
1054
+ tokenizer: AnyTokenizer,
1055
+ text_1: List[str],
1056
+ text_2: List[str],
1057
truncate_prompt_tokens: Optional[int] = None,
1058
use_tqdm: bool = True,
1059
lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
@@ -1176,29 +1176,36 @@ class LLM:
1176
if isinstance(text_1, (str, dict)):
1177
# Convert a single prompt to a list.
1178
text_1 = [text_1]
1179
- text_1 = [ensure_str(t) for t in text_1]
1179
+ input_text_1: List[str] = [ensure_str(t) for t in text_1]
1180
1181
if isinstance(text_2, (str, dict)):
1182
# Convert a single prompt to a list.
1183
text_2 = [text_2]
1184
- text_2 = [ensure_str(t) for t in text_2]
1184
+ input_text_2: List[str] = [ensure_str(t) for t in text_2]
1185
1186
- if len(text_1) > 1 and len(text_1) != len(text_2):
1186
+ if len(input_text_1) > 1 and len(input_text_1) != len(input_text_2):
1187
raise ValueError("Input lengths must be either 1:1, 1:N or N:N")
1188
- if len(text_1) == 0:
1188
+ if len(input_text_1) == 0:
1189
raise ValueError("At least one text element must be given")
1190
- if len(text_2) == 0:
1190
+ if len(input_text_2) == 0:
1191
raise ValueError("At least one text_pair element must be given")
1192
1193
if self.llm_engine.model_config.is_cross_encoder:
1194
- return self._cross_encoding_score(tokenizer, text_1, text_2,
1194
+ return self._cross_encoding_score(tokenizer, input_text_1,
1195
+ input_text_2,
1196
truncate_prompt_tokens, use_tqdm,
1197
lora_request,
1198
prompt_adapter_request)
1199
else:
1199
- return self._embedding_score(tokenizer, text_1, text_2,
1200
- truncate_prompt_tokens, use_tqdm,
1201
- lora_request, prompt_adapter_request)
1200
+
1201
+ return self._embedding_score(
1202
+ tokenizer,
1203
+ input_text_1, # type: ignore[arg-type]
1204
+ input_text_2, # type: ignore[arg-type]
1205
+ truncate_prompt_tokens,
1206
+ use_tqdm,
1207
+ lora_request,
1208
+ prompt_adapter_request)
1209
1210
def start_profile(self) -> None:
1211
self.llm_engine.start_profile()
vllm/entrypoints/openai/serving_engine.py
+1
-2
@@ -400,8 +400,7 @@ class OpenAIServing:
400
_chat_template_kwargs.update(chat_template_kwargs or {})
401
402
request_prompt: Union[str, List[int]]
403
- is_mistral_tokenizer = isinstance(tokenizer, MistralTokenizer)
404
- if is_mistral_tokenizer:
403
+ if isinstance(tokenizer, MistralTokenizer):
404
request_prompt = apply_mistral_chat_template(
405
tokenizer,
406
messages=messages,
vllm/entrypoints/openai/serving_score.py
+1
-1
@@ -121,7 +121,7 @@ class OpenAIServingScores(OpenAIServing):
121
122
tokenize_async = make_async(tokenizer.__call__,
123
executor=self._tokenizer_executor)
124
- prompt_inputs = await tokenize_async(text=q,
124
+ prompt_inputs = await tokenize_async(q,
125
text_pair=t,
126
**tokenization_kwargs)
127
vllm/logits_process.py
+1
-1
@@ -31,7 +31,7 @@ def get_bad_words_logits_processors(
31
32
if isinstance(tokenizer, MistralTokenizer):
33
# Mistral tokenizers should not add special tokens
34
- prompt_token_ids = tokenizer.encode(prompt=prompt)
34
+ prompt_token_ids = tokenizer.encode(text=prompt)
35
else:
36
prompt_token_ids = tokenizer.encode(text=prompt,
37
add_special_tokens=False)
vllm/transformers_utils/tokenizer.py
+12
-6
@@ -14,6 +14,8 @@ from transformers import (AutoTokenizer, PreTrainedTokenizer,
14
from vllm.envs import VLLM_USE_MODELSCOPE
15
from vllm.logger import init_logger
16
from vllm.lora.request import LoRARequest
17
+from vllm.transformers_utils.tokenizer_base import (TokenizerBase,
18
+ TokenizerRegistry)
19
from vllm.transformers_utils.tokenizers import MistralTokenizer
20
from vllm.transformers_utils.utils import check_gguf_file
21
from vllm.utils import make_async
@@ -21,7 +23,7 @@ from vllm.utils import make_async
23
logger = init_logger(__name__)
24
25
AnyTokenizer = Union[PreTrainedTokenizer, PreTrainedTokenizerFast,
24
- MistralTokenizer]
26
+ TokenizerBase]
27
28
29
def decode_tokens(
@@ -47,11 +49,7 @@ def encode_tokens(
49
Backend-agnostic equivalent of HF's
50
:code:`tokenizer.encode(text, add_special_tokens=...)`.
51
"""
50
- if isinstance(tokenizer, MistralTokenizer):
51
- return tokenizer.tokenizer.encode(text,
52
- bos=add_special_tokens,
53
- eos=add_special_tokens)
54
- elif add_special_tokens is not None:
52
+ if add_special_tokens is not None:
53
return tokenizer.encode(text, add_special_tokens=add_special_tokens)
54
return tokenizer.encode(text)
55
@@ -183,9 +181,17 @@ def get_tokenizer(
181
'encoding and decoding.',
182
FutureWarning,
183
stacklevel=2)
184
+
185
+ tokenizer: AnyTokenizer
186
if tokenizer_mode == "mistral":
187
tokenizer = MistralTokenizer.from_pretrained(str(tokenizer_name),
188
revision=revision)
189
+ elif tokenizer_mode == "custom":
190
+ tokenizer = TokenizerRegistry.get_tokenizer(str(tokenizer_name),
191
+ *args,
192
+ revision=revision,
193
+ download_dir=download_dir,
194
+ **kwargs)
195
else:
196
try:
197
tokenizer = AutoTokenizer.from_pretrained(
vllm/transformers_utils/tokenizer_base.py
new
+146
@@ -0,0 +1,146 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+
3
+import importlib
4
+from abc import ABC, abstractmethod
5
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
6
+
7
+if TYPE_CHECKING:
8
+ from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
9
+
10
+
11
+class TokenizerBase(ABC):
12
+
13
+ @property
14
+ @abstractmethod
15
+ def all_special_tokens_extended(self) -> List[str]:
16
+ raise NotImplementedError()
17
+
18
+ @property
19
+ @abstractmethod
20
+ def all_special_tokens(self) -> List[str]:
21
+ raise NotImplementedError()
22
+
23
+ @property
24
+ @abstractmethod
25
+ def all_special_ids(self) -> List[int]:
26
+ raise NotImplementedError()
27
+
28
+ @property
29
+ @abstractmethod
30
+ def bos_token_id(self) -> int:
31
+ raise NotImplementedError()
32
+
33
+ @property
34
+ @abstractmethod
35
+ def eos_token_id(self) -> int:
36
+ raise NotImplementedError()
37
+
38
+ @property
39
+ @abstractmethod
40
+ def sep_token(self) -> str:
41
+ raise NotImplementedError()
42
+
43
+ @property
44
+ @abstractmethod
45
+ def pad_token(self) -> str:
46
+ raise NotImplementedError()
47
+
48
+ @property
49
+ @abstractmethod
50
+ def is_fast(self) -> bool:
51
+ raise NotImplementedError()
52
+
53
+ @property
54
+ @abstractmethod
55
+ def vocab_size(self) -> int:
56
+ raise NotImplementedError()
57
+
58
+ @property
59
+ @abstractmethod
60
+ def max_token_id(self) -> int:
61
+ raise NotImplementedError()
62
+
63
+ def __len__(self) -> int:
64
+ return self.vocab_size
65
+
66
+ @abstractmethod
67
+ def __call__(
68
+ self,
69
+ text: Union[str, List[str], List[int]],
70
+ text_pair: Optional[str] = None,
71
+ add_special_tokens: bool = False,
72
+ truncation: bool = False,
73
+ max_length: Optional[int] = None,
74
+ ):
75
+ raise NotImplementedError()
76
+
77
+ @abstractmethod
78
+ def get_vocab(self) -> Dict[str, int]:
79
+ raise NotImplementedError()
80
+
81
+ @abstractmethod
82
+ def get_added_vocab(self) -> Dict[str, int]:
83
+ raise NotImplementedError()
84
+
85
+ @abstractmethod
86
+ def encode_one(
87
+ self,
88
+ text: str,
89
+ truncation: bool = False,
90
+ max_length: Optional[int] = None,
91
+ ) -> List[int]:
92
+ raise NotImplementedError()
93
+
94
+ @abstractmethod
95
+ def encode(self,
96
+ text: str,
97
+ add_special_tokens: Optional[bool] = None) -> List[int]:
98
+ raise NotImplementedError()
99
+
100
+ @abstractmethod
101
+ def apply_chat_template(self,
102
+ messages: List["ChatCompletionMessageParam"],
103
+ tools: Optional[List[Dict[str, Any]]] = None,
104
+ **kwargs) -> List[int]:
105
+ raise NotImplementedError()
106
+
107
+ @abstractmethod
108
+ def convert_tokens_to_string(self, tokens: List[str]) -> str:
109
+ raise NotImplementedError()
110
+
111
+ @abstractmethod
112
+ def decode(self,
113
+ ids: Union[List[int], int],
114
+ skip_special_tokens: bool = True) -> str:
115
+ raise NotImplementedError()
116
+
117
+ @abstractmethod
118
+ def convert_ids_to_tokens(
119
+ self,
120
+ ids: List[int],
121
+ skip_special_tokens: bool = True,
122
+ ) -> List[str]:
123
+ raise NotImplementedError()
124
+
125
+
126
+class TokenizerRegistry:
127
+ # Tokenizer name -> (tokenizer module, tokenizer class)
128
+ REGISTRY: Dict[str, Tuple[str, str]] = {}
129
+
130
+ @staticmethod
131
+ def register(name: str, module: str, class_name: str) -> None:
132
+ TokenizerRegistry.REGISTRY[name] = (module, class_name)
133
+
134
+ @staticmethod
135
+ def get_tokenizer(
136
+ tokenizer_name: str,
137
+ *args,
138
+ **kwargs,
139
+ ) -> TokenizerBase:
140
+ tokenizer_cls = TokenizerRegistry.REGISTRY.get(tokenizer_name)
141
+ if tokenizer_cls is None:
142
+ raise ValueError(f"Tokenizer {tokenizer_name} not found.")
143
+
144
+ tokenizer_module = importlib.import_module(tokenizer_cls[0])
145
+ class_ = getattr(tokenizer_module, tokenizer_cls[1])
146
+ return class_.from_pretrained(*args, **kwargs)
vllm/transformers_utils/tokenizers/mistral.py
+28
-11
@@ -10,6 +10,7 @@ import huggingface_hub
10
from huggingface_hub import HfApi, hf_hub_download
11
12
from vllm.logger import init_logger
13
+from vllm.transformers_utils.tokenizer_base import TokenizerBase
14
from vllm.utils import is_list_of
15
16
if TYPE_CHECKING:
@@ -140,7 +141,7 @@ def make_mistral_chat_completion_request(
141
tools=tools) # type: ignore[type-var]
142
143
143
-class MistralTokenizer:
144
+class MistralTokenizer(TokenizerBase):
145
146
def __init__(self, tokenizer: "PublicMistralTokenizer") -> None:
147
self.mistral = tokenizer
@@ -251,6 +252,14 @@ class MistralTokenizer:
252
def eos_token_id(self) -> int:
253
return self.tokenizer.eos_id
254
255
+ @property
256
+ def sep_token(self) -> str:
257
+ raise NotImplementedError()
258
+
259
+ @property
260
+ def pad_token(self) -> str:
261
+ raise NotImplementedError()
262
+
263
@property
264
def is_fast(self) -> bool:
265
return True
@@ -268,25 +277,26 @@ class MistralTokenizer:
277
278
def __call__(
279
self,
271
- prompt: Union[str, List[str], List[int]],
280
+ text: Union[str, List[str], List[int]],
281
+ text_pair: Optional[str] = None,
282
add_special_tokens: bool = False,
283
truncation: bool = False,
284
max_length: Optional[int] = None,
285
):
286
input_ids: Union[List[int], List[List[int]]]
287
# For List[str], original prompt text
278
- if is_list_of(prompt, str):
288
+ if is_list_of(text, str):
289
input_ids_: List[List[int]] = []
280
- for p in prompt:
290
+ for p in text:
291
each_input_ids = self.encode_one(p, truncation, max_length)
292
input_ids_.append(each_input_ids)
293
input_ids = input_ids_
294
# For List[int], apply chat template output, already tokens.
285
- elif is_list_of(prompt, int):
286
- input_ids = prompt
295
+ elif is_list_of(text, int):
296
+ input_ids = text
297
# For str, single prompt text
298
else:
289
- input_ids = self.encode_one(prompt, truncation, max_length)
299
+ input_ids = self.encode_one(text, truncation, max_length)
300
return Encoding(input_ids=input_ids)
301
302
def get_vocab(self) -> Dict[str, int]:
@@ -300,22 +310,29 @@ class MistralTokenizer:
310
311
def encode_one(
312
self,
303
- prompt: str,
313
+ text: str,
314
truncation: bool = False,
315
max_length: Optional[int] = None,
316
) -> List[int]:
317
# Mistral Tokenizers should not add special tokens
308
- input_ids = self.encode(prompt)
318
+ input_ids = self.encode(text)
319
320
if truncation:
321
input_ids = input_ids[:max_length]
322
return input_ids
323
314
- def encode(self, prompt: str) -> List[int]:
324
+ def encode(self,
325
+ text: str,
326
+ add_special_tokens: Optional[bool] = None) -> List[int]:
327
# `encode` should only be used for prompt completion
328
# it should never be used for chat_completion.
329
# For chat completion use `apply_chat_template`
318
- return self.tokenizer.encode(prompt, bos=True, eos=False)
330
+ if add_special_tokens is not None:
331
+ return self.tokenizer.encode(text,
332
+ bos=add_special_tokens,
333
+ eos=add_special_tokens)
334
+ else:
335
+ return self.tokenizer.encode(text, bos=True, eos=False)
336
337
def apply_chat_template(self,
338
messages: List["ChatCompletionMessageParam"],