@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3ee696a63d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[RFC][vllm-API] Support tokenizer registry for customized tokenizer in vLLM (#12518)

Signed-off-by: Keyun Tong <tongkeyun@gmail.com>

Keyun Tong committed Feb 11, 2025 at 20:25 UTC 3ee696a63dd0c2acee44809a3bedec33ea27dfa0
11 files changed +343 -41
benchmarks/benchmark_serving.py
+3 -2
@@ -1275,11 +1275,12 @@ if __name__ == "__main__":
1275 '--tokenizer-mode',
1276 type=str,
1277 default="auto",
1278 - choices=['auto', 'slow', 'mistral'],
1278 + choices=['auto', 'slow', 'mistral', 'custom'],
1279 help='The tokenizer mode.\n\n* "auto" will use the '
1280 'fast tokenizer if available.\n* "slow" will '
1281 'always use the slow tokenizer. \n* '
1282 - '"mistral" will always use the `mistral_common` tokenizer.')
1282 + '"mistral" will always use the `mistral_common` tokenizer. \n*'
1283 + '"custom" will use --tokenizer to select the preregistered tokenizer.')
1284
1285 parser.add_argument("--served-model-name",
1286 type=str,
tests/tokenization/test_tokenizer_registry.py new
+123
@@ -0,0 +1,123 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
4 +
5 +from vllm.transformers_utils.tokenizer import get_tokenizer
6 +from vllm.transformers_utils.tokenizer_base import (TokenizerBase,
7 + TokenizerRegistry)
8 +
9 +if TYPE_CHECKING:
10 + from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
11 +
12 +
13 +class TestTokenizer(TokenizerBase):
14 +
15 + @classmethod
16 + def from_pretrained(cls, *args, **kwargs) -> "TestTokenizer":
17 + return TestTokenizer()
18 +
19 + @property
20 + def all_special_tokens_extended(self) -> List[str]:
21 + raise NotImplementedError()
22 +
23 + @property
24 + def all_special_tokens(self) -> List[str]:
25 + raise NotImplementedError()
26 +
27 + @property
28 + def all_special_ids(self) -> List[int]:
29 + raise NotImplementedError()
30 +
31 + @property
32 + def bos_token_id(self) -> int:
33 + return 0
34 +
35 + @property
36 + def eos_token_id(self) -> int:
37 + return 1
38 +
39 + @property
40 + def sep_token(self) -> str:
41 + raise NotImplementedError()
42 +
43 + @property
44 + def pad_token(self) -> str:
45 + raise NotImplementedError()
46 +
47 + @property
48 + def is_fast(self) -> bool:
49 + raise NotImplementedError()
50 +
51 + @property
52 + def vocab_size(self) -> int:
53 + raise NotImplementedError()
54 +
55 + @property
56 + def max_token_id(self) -> int:
57 + raise NotImplementedError()
58 +
59 + def __call__(
60 + self,
61 + text: Union[str, List[str], List[int]],
62 + text_pair: Optional[str] = None,
63 + add_special_tokens: bool = False,
64 + truncation: bool = False,
65 + max_length: Optional[int] = None,
66 + ):
67 + raise NotImplementedError()
68 +
69 + def get_vocab(self) -> Dict[str, int]:
70 + raise NotImplementedError()
71 +
72 + def get_added_vocab(self) -> Dict[str, int]:
73 + raise NotImplementedError()
74 +
75 + def encode_one(
76 + self,
77 + text: str,
78 + truncation: bool = False,
79 + max_length: Optional[int] = None,
80 + ) -> List[int]:
81 + raise NotImplementedError()
82 +
83 + def encode(self,
84 + text: str,
85 + add_special_tokens: Optional[bool] = None) -> List[int]:
86 + raise NotImplementedError()
87 +
88 + def apply_chat_template(self,
89 + messages: List["ChatCompletionMessageParam"],
90 + tools: Optional[List[Dict[str, Any]]] = None,
91 + **kwargs) -> List[int]:
92 + raise NotImplementedError()
93 +
94 + def convert_tokens_to_string(self, tokens: List[str]) -> str:
95 + raise NotImplementedError()
96 +
97 + def decode(self,
98 + ids: Union[List[int], int],
99 + skip_special_tokens: bool = True) -> str:
100 + raise NotImplementedError()
101 +
102 + def convert_ids_to_tokens(
103 + self,
104 + ids: List[int],
105 + skip_special_tokens: bool = True,
106 + ) -> List[str]:
107 + raise NotImplementedError()
108 +
109 +
110 +def test_customized_tokenizer():
111 + TokenizerRegistry.register("test_tokenizer",
112 + "tests.tokenization.test_tokenizer_registry",
113 + "TestTokenizer")
114 +
115 + tokenizer = TokenizerRegistry.get_tokenizer("test_tokenizer")
116 + assert isinstance(tokenizer, TestTokenizer)
117 + assert tokenizer.bos_token_id == 0
118 + assert tokenizer.eos_token_id == 1
119 +
120 + tokenizer = get_tokenizer("test_tokenizer", tokenizer_mode="custom")
121 + assert isinstance(tokenizer, TestTokenizer)
122 + assert tokenizer.bos_token_id == 0
123 + assert tokenizer.eos_token_id == 1
vllm/config.py
+5 -4
@@ -102,8 +102,9 @@ class ModelConfig:
102 it; otherwise, you must specify explicitly which task to use.
103 tokenizer: Name or path of the huggingface tokenizer to use.
104 tokenizer_mode: Tokenizer mode. "auto" will use the fast tokenizer if
105 - available, "slow" will always use the slow tokenizer, and
106 - "mistral" will always use the tokenizer from `mistral_common`.
105 + available, "slow" will always use the slow tokenizer,
106 + "mistral" will always use the tokenizer from `mistral_common`, and
107 + "custom" will use --tokenizer to select the preregistered tokenizer.
108 trust_remote_code: Trust remote code (e.g., from HuggingFace) when
109 downloading the model and tokenizer.
110 allowed_local_media_path: Allowing API requests to read local images or
@@ -467,10 +468,10 @@ class ModelConfig:
468
469 def _verify_tokenizer_mode(self) -> None:
470 tokenizer_mode = self.tokenizer_mode.lower()
470 - if tokenizer_mode not in ["auto", "slow", "mistral"]:
471 + if tokenizer_mode not in ["auto", "slow", "mistral", "custom"]:
472 raise ValueError(
473 f"Unknown tokenizer mode: {self.tokenizer_mode}. Must be "
473 - "either 'auto', 'slow' or 'mistral'.")
474 + "either 'auto', 'slow', 'mistral' or 'custom'.")
475 self.tokenizer_mode = tokenizer_mode
476
477 def _get_preferred_task(
vllm/engine/arg_utils.py
+4 -2
@@ -284,11 +284,13 @@ class EngineArgs:
284 '--tokenizer-mode',
285 type=str,
286 default=EngineArgs.tokenizer_mode,
287 - choices=['auto', 'slow', 'mistral'],
287 + choices=['auto', 'slow', 'mistral', 'custom'],
288 help='The tokenizer mode.\n\n* "auto" will use the '
289 'fast tokenizer if available.\n* "slow" will '
290 'always use the slow tokenizer. \n* '
291 - '"mistral" will always use the `mistral_common` tokenizer.')
291 + '"mistral" will always use the `mistral_common` tokenizer. \n* '
292 + '"custom" will use --tokenizer to select the '
293 + 'preregistered tokenizer.')
294 parser.add_argument('--trust-remote-code',
295 action='store_true',
296 help='Trust remote code from huggingface.')
vllm/entrypoints/llm.py
+19 -12
@@ -1051,9 +1051,9 @@ class LLM:
1051
1052 def _cross_encoding_score(
1053 self,
1054 - tokenizer: Union[AnyTokenizer],
1055 - text_1: List[Union[str, TextPrompt, TokensPrompt]],
1056 - text_2: List[Union[str, TextPrompt, TokensPrompt]],
1054 + tokenizer: AnyTokenizer,
1055 + text_1: List[str],
1056 + text_2: List[str],
1057 truncate_prompt_tokens: Optional[int] = None,
1058 use_tqdm: bool = True,
1059 lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
@@ -1176,29 +1176,36 @@ class LLM:
1176 if isinstance(text_1, (str, dict)):
1177 # Convert a single prompt to a list.
1178 text_1 = [text_1]
1179 - text_1 = [ensure_str(t) for t in text_1]
1179 + input_text_1: List[str] = [ensure_str(t) for t in text_1]
1180
1181 if isinstance(text_2, (str, dict)):
1182 # Convert a single prompt to a list.
1183 text_2 = [text_2]
1184 - text_2 = [ensure_str(t) for t in text_2]
1184 + input_text_2: List[str] = [ensure_str(t) for t in text_2]
1185
1186 - if len(text_1) > 1 and len(text_1) != len(text_2):
1186 + if len(input_text_1) > 1 and len(input_text_1) != len(input_text_2):
1187 raise ValueError("Input lengths must be either 1:1, 1:N or N:N")
1188 - if len(text_1) == 0:
1188 + if len(input_text_1) == 0:
1189 raise ValueError("At least one text element must be given")
1190 - if len(text_2) == 0:
1190 + if len(input_text_2) == 0:
1191 raise ValueError("At least one text_pair element must be given")
1192
1193 if self.llm_engine.model_config.is_cross_encoder:
1194 - return self._cross_encoding_score(tokenizer, text_1, text_2,
1194 + return self._cross_encoding_score(tokenizer, input_text_1,
1195 + input_text_2,
1196 truncate_prompt_tokens, use_tqdm,
1197 lora_request,
1198 prompt_adapter_request)
1199 else:
1199 - return self._embedding_score(tokenizer, text_1, text_2,
1200 - truncate_prompt_tokens, use_tqdm,
1201 - lora_request, prompt_adapter_request)
1200 +
1201 + return self._embedding_score(
1202 + tokenizer,
1203 + input_text_1, # type: ignore[arg-type]
1204 + input_text_2, # type: ignore[arg-type]
1205 + truncate_prompt_tokens,
1206 + use_tqdm,
1207 + lora_request,
1208 + prompt_adapter_request)
1209
1210 def start_profile(self) -> None:
1211 self.llm_engine.start_profile()
vllm/entrypoints/openai/serving_engine.py
+1 -2
@@ -400,8 +400,7 @@ class OpenAIServing:
400 _chat_template_kwargs.update(chat_template_kwargs or {})
401
402 request_prompt: Union[str, List[int]]
403 - is_mistral_tokenizer = isinstance(tokenizer, MistralTokenizer)
404 - if is_mistral_tokenizer:
403 + if isinstance(tokenizer, MistralTokenizer):
404 request_prompt = apply_mistral_chat_template(
405 tokenizer,
406 messages=messages,
vllm/entrypoints/openai/serving_score.py
+1 -1
@@ -121,7 +121,7 @@ class OpenAIServingScores(OpenAIServing):
121
122 tokenize_async = make_async(tokenizer.__call__,
123 executor=self._tokenizer_executor)
124 - prompt_inputs = await tokenize_async(text=q,
124 + prompt_inputs = await tokenize_async(q,
125 text_pair=t,
126 **tokenization_kwargs)
127
vllm/logits_process.py
+1 -1
@@ -31,7 +31,7 @@ def get_bad_words_logits_processors(
31
32 if isinstance(tokenizer, MistralTokenizer):
33 # Mistral tokenizers should not add special tokens
34 - prompt_token_ids = tokenizer.encode(prompt=prompt)
34 + prompt_token_ids = tokenizer.encode(text=prompt)
35 else:
36 prompt_token_ids = tokenizer.encode(text=prompt,
37 add_special_tokens=False)
vllm/transformers_utils/tokenizer.py
+12 -6
@@ -14,6 +14,8 @@ from transformers import (AutoTokenizer, PreTrainedTokenizer,
14 from vllm.envs import VLLM_USE_MODELSCOPE
15 from vllm.logger import init_logger
16 from vllm.lora.request import LoRARequest
17 +from vllm.transformers_utils.tokenizer_base import (TokenizerBase,
18 + TokenizerRegistry)
19 from vllm.transformers_utils.tokenizers import MistralTokenizer
20 from vllm.transformers_utils.utils import check_gguf_file
21 from vllm.utils import make_async
@@ -21,7 +23,7 @@ from vllm.utils import make_async
23 logger = init_logger(__name__)
24
25 AnyTokenizer = Union[PreTrainedTokenizer, PreTrainedTokenizerFast,
24 - MistralTokenizer]
26 + TokenizerBase]
27
28
29 def decode_tokens(
@@ -47,11 +49,7 @@ def encode_tokens(
49 Backend-agnostic equivalent of HF's
50 :code:`tokenizer.encode(text, add_special_tokens=...)`.
51 """
50 - if isinstance(tokenizer, MistralTokenizer):
51 - return tokenizer.tokenizer.encode(text,
52 - bos=add_special_tokens,
53 - eos=add_special_tokens)
54 - elif add_special_tokens is not None:
52 + if add_special_tokens is not None:
53 return tokenizer.encode(text, add_special_tokens=add_special_tokens)
54 return tokenizer.encode(text)
55
@@ -183,9 +181,17 @@ def get_tokenizer(
181 'encoding and decoding.',
182 FutureWarning,
183 stacklevel=2)
184 +
185 + tokenizer: AnyTokenizer
186 if tokenizer_mode == "mistral":
187 tokenizer = MistralTokenizer.from_pretrained(str(tokenizer_name),
188 revision=revision)
189 + elif tokenizer_mode == "custom":
190 + tokenizer = TokenizerRegistry.get_tokenizer(str(tokenizer_name),
191 + *args,
192 + revision=revision,
193 + download_dir=download_dir,
194 + **kwargs)
195 else:
196 try:
197 tokenizer = AutoTokenizer.from_pretrained(
vllm/transformers_utils/tokenizer_base.py new
+146
@@ -0,0 +1,146 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +import importlib
4 +from abc import ABC, abstractmethod
5 +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
6 +
7 +if TYPE_CHECKING:
8 + from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
9 +
10 +
11 +class TokenizerBase(ABC):
12 +
13 + @property
14 + @abstractmethod
15 + def all_special_tokens_extended(self) -> List[str]:
16 + raise NotImplementedError()
17 +
18 + @property
19 + @abstractmethod
20 + def all_special_tokens(self) -> List[str]:
21 + raise NotImplementedError()
22 +
23 + @property
24 + @abstractmethod
25 + def all_special_ids(self) -> List[int]:
26 + raise NotImplementedError()
27 +
28 + @property
29 + @abstractmethod
30 + def bos_token_id(self) -> int:
31 + raise NotImplementedError()
32 +
33 + @property
34 + @abstractmethod
35 + def eos_token_id(self) -> int:
36 + raise NotImplementedError()
37 +
38 + @property
39 + @abstractmethod
40 + def sep_token(self) -> str:
41 + raise NotImplementedError()
42 +
43 + @property
44 + @abstractmethod
45 + def pad_token(self) -> str:
46 + raise NotImplementedError()
47 +
48 + @property
49 + @abstractmethod
50 + def is_fast(self) -> bool:
51 + raise NotImplementedError()
52 +
53 + @property
54 + @abstractmethod
55 + def vocab_size(self) -> int:
56 + raise NotImplementedError()
57 +
58 + @property
59 + @abstractmethod
60 + def max_token_id(self) -> int:
61 + raise NotImplementedError()
62 +
63 + def __len__(self) -> int:
64 + return self.vocab_size
65 +
66 + @abstractmethod
67 + def __call__(
68 + self,
69 + text: Union[str, List[str], List[int]],
70 + text_pair: Optional[str] = None,
71 + add_special_tokens: bool = False,
72 + truncation: bool = False,
73 + max_length: Optional[int] = None,
74 + ):
75 + raise NotImplementedError()
76 +
77 + @abstractmethod
78 + def get_vocab(self) -> Dict[str, int]:
79 + raise NotImplementedError()
80 +
81 + @abstractmethod
82 + def get_added_vocab(self) -> Dict[str, int]:
83 + raise NotImplementedError()
84 +
85 + @abstractmethod
86 + def encode_one(
87 + self,
88 + text: str,
89 + truncation: bool = False,
90 + max_length: Optional[int] = None,
91 + ) -> List[int]:
92 + raise NotImplementedError()
93 +
94 + @abstractmethod
95 + def encode(self,
96 + text: str,
97 + add_special_tokens: Optional[bool] = None) -> List[int]:
98 + raise NotImplementedError()
99 +
100 + @abstractmethod
101 + def apply_chat_template(self,
102 + messages: List["ChatCompletionMessageParam"],
103 + tools: Optional[List[Dict[str, Any]]] = None,
104 + **kwargs) -> List[int]:
105 + raise NotImplementedError()
106 +
107 + @abstractmethod
108 + def convert_tokens_to_string(self, tokens: List[str]) -> str:
109 + raise NotImplementedError()
110 +
111 + @abstractmethod
112 + def decode(self,
113 + ids: Union[List[int], int],
114 + skip_special_tokens: bool = True) -> str:
115 + raise NotImplementedError()
116 +
117 + @abstractmethod
118 + def convert_ids_to_tokens(
119 + self,
120 + ids: List[int],
121 + skip_special_tokens: bool = True,
122 + ) -> List[str]:
123 + raise NotImplementedError()
124 +
125 +
126 +class TokenizerRegistry:
127 + # Tokenizer name -> (tokenizer module, tokenizer class)
128 + REGISTRY: Dict[str, Tuple[str, str]] = {}
129 +
130 + @staticmethod
131 + def register(name: str, module: str, class_name: str) -> None:
132 + TokenizerRegistry.REGISTRY[name] = (module, class_name)
133 +
134 + @staticmethod
135 + def get_tokenizer(
136 + tokenizer_name: str,
137 + *args,
138 + **kwargs,
139 + ) -> TokenizerBase:
140 + tokenizer_cls = TokenizerRegistry.REGISTRY.get(tokenizer_name)
141 + if tokenizer_cls is None:
142 + raise ValueError(f"Tokenizer {tokenizer_name} not found.")
143 +
144 + tokenizer_module = importlib.import_module(tokenizer_cls[0])
145 + class_ = getattr(tokenizer_module, tokenizer_cls[1])
146 + return class_.from_pretrained(*args, **kwargs)
vllm/transformers_utils/tokenizers/mistral.py
+28 -11
@@ -10,6 +10,7 @@ import huggingface_hub
10 from huggingface_hub import HfApi, hf_hub_download
11
12 from vllm.logger import init_logger
13 +from vllm.transformers_utils.tokenizer_base import TokenizerBase
14 from vllm.utils import is_list_of
15
16 if TYPE_CHECKING:
@@ -140,7 +141,7 @@ def make_mistral_chat_completion_request(
141 tools=tools) # type: ignore[type-var]
142
143
143 -class MistralTokenizer:
144 +class MistralTokenizer(TokenizerBase):
145
146 def __init__(self, tokenizer: "PublicMistralTokenizer") -> None:
147 self.mistral = tokenizer
@@ -251,6 +252,14 @@ class MistralTokenizer:
252 def eos_token_id(self) -> int:
253 return self.tokenizer.eos_id
254
255 + @property
256 + def sep_token(self) -> str:
257 + raise NotImplementedError()
258 +
259 + @property
260 + def pad_token(self) -> str:
261 + raise NotImplementedError()
262 +
263 @property
264 def is_fast(self) -> bool:
265 return True
@@ -268,25 +277,26 @@ class MistralTokenizer:
277
278 def __call__(
279 self,
271 - prompt: Union[str, List[str], List[int]],
280 + text: Union[str, List[str], List[int]],
281 + text_pair: Optional[str] = None,
282 add_special_tokens: bool = False,
283 truncation: bool = False,
284 max_length: Optional[int] = None,
285 ):
286 input_ids: Union[List[int], List[List[int]]]
287 # For List[str], original prompt text
278 - if is_list_of(prompt, str):
288 + if is_list_of(text, str):
289 input_ids_: List[List[int]] = []
280 - for p in prompt:
290 + for p in text:
291 each_input_ids = self.encode_one(p, truncation, max_length)
292 input_ids_.append(each_input_ids)
293 input_ids = input_ids_
294 # For List[int], apply chat template output, already tokens.
285 - elif is_list_of(prompt, int):
286 - input_ids = prompt
295 + elif is_list_of(text, int):
296 + input_ids = text
297 # For str, single prompt text
298 else:
289 - input_ids = self.encode_one(prompt, truncation, max_length)
299 + input_ids = self.encode_one(text, truncation, max_length)
300 return Encoding(input_ids=input_ids)
301
302 def get_vocab(self) -> Dict[str, int]:
@@ -300,22 +310,29 @@ class MistralTokenizer:
310
311 def encode_one(
312 self,
303 - prompt: str,
313 + text: str,
314 truncation: bool = False,
315 max_length: Optional[int] = None,
316 ) -> List[int]:
317 # Mistral Tokenizers should not add special tokens
308 - input_ids = self.encode(prompt)
318 + input_ids = self.encode(text)
319
320 if truncation:
321 input_ids = input_ids[:max_length]
322 return input_ids
323
314 - def encode(self, prompt: str) -> List[int]:
324 + def encode(self,
325 + text: str,
326 + add_special_tokens: Optional[bool] = None) -> List[int]:
327 # `encode` should only be used for prompt completion
328 # it should never be used for chat_completion.
329 # For chat completion use `apply_chat_template`
318 - return self.tokenizer.encode(prompt, bos=True, eos=False)
330 + if add_special_tokens is not None:
331 + return self.tokenizer.encode(text,
332 + bos=add_special_tokens,
333 + eos=add_special_tokens)
334 + else:
335 + return self.tokenizer.encode(text, bos=True, eos=False)
336
337 def apply_chat_template(self,
338 messages: List["ChatCompletionMessageParam"],