@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 39cefbdf17
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 11, 12:30 UTC

[Refactor] `TokenizerRegistry` only uses lazy imports (#30609)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Dec 13, 2025 at 23:16 UTC 39cefbdf17e2e906e0eae3e82bd601f66137deb4
14 files changed +202 -176
tests/test_inputs.py
+2 -2
@@ -7,7 +7,7 @@ from vllm.config import ModelConfig
7 from vllm.inputs import zip_enc_dec_prompts
8 from vllm.inputs.parse import parse_raw_prompts
9 from vllm.inputs.preprocess import InputPreprocessor
10 -from vllm.tokenizers import init_tokenizer_from_config
10 +from vllm.tokenizers import cached_tokenizer_from_config
11
12 pytestmark = pytest.mark.cpu_test
13
@@ -108,7 +108,7 @@ def test_zip_enc_dec_prompts(mm_processor_kwargs, expected_mm_kwargs):
108 )
109 def test_preprocessor_always_mm_code_path(model_id, prompt):
110 model_config = ModelConfig(model=model_id)
111 - tokenizer = init_tokenizer_from_config(model_config)
111 + tokenizer = cached_tokenizer_from_config(model_config)
112 input_preprocessor = InputPreprocessor(model_config, tokenizer)
113
114 # HF processor adds sep token
tests/tokenizers_/test_basic.py
+24 -23
@@ -3,38 +3,39 @@
3 from typing import _get_protocol_attrs # type: ignore
4
5 import pytest
6 -from transformers import PreTrainedTokenizerBase
6 +from transformers import (
7 + PreTrainedTokenizer,
8 + PreTrainedTokenizerBase,
9 + PreTrainedTokenizerFast,
10 +)
11
12 from vllm.tokenizers import TokenizerLike, get_tokenizer
13 +from vllm.tokenizers.mistral import MistralTokenizer
14
15
16 def _get_missing_attrs(obj: object, target: type):
17 return [k for k in _get_protocol_attrs(target) if not hasattr(obj, k)]
18
19
20 +def _assert_tokenizer_like(tokenizer: object):
21 + missing_attrs = _get_missing_attrs(tokenizer, TokenizerLike)
22 + assert not missing_attrs, f"Missing attrs: {missing_attrs}"
23 +
24 +
25 def test_tokenizer_like_protocol():
16 - assert not (
17 - missing_attrs := _get_missing_attrs(
18 - get_tokenizer("gpt2", use_fast=False),
19 - TokenizerLike,
20 - )
21 - ), f"Missing attrs: {missing_attrs}"
22 -
23 - assert not (
24 - missing_attrs := _get_missing_attrs(
25 - get_tokenizer("gpt2", use_fast=True),
26 - TokenizerLike,
27 - )
28 - ), f"Missing attrs: {missing_attrs}"
29 -
30 - assert not (
31 - missing_attrs := _get_missing_attrs(
32 - get_tokenizer(
33 - "mistralai/Mistral-7B-Instruct-v0.3", tokenizer_mode="mistral"
34 - ),
35 - TokenizerLike,
36 - )
37 - ), f"Missing attrs: {missing_attrs}"
26 + tokenizer = get_tokenizer("gpt2", use_fast=False)
27 + assert isinstance(tokenizer, PreTrainedTokenizer)
28 + _assert_tokenizer_like(tokenizer)
29 +
30 + tokenizer = get_tokenizer("gpt2", use_fast=True)
31 + assert isinstance(tokenizer, PreTrainedTokenizerFast)
32 + _assert_tokenizer_like(tokenizer)
33 +
34 + tokenizer = get_tokenizer(
35 + "mistralai/Mistral-7B-Instruct-v0.3", tokenizer_mode="mistral"
36 + )
37 + assert isinstance(tokenizer, MistralTokenizer)
38 + _assert_tokenizer_like(tokenizer)
39
40
41 @pytest.mark.parametrize("tokenizer_name", ["facebook/opt-125m", "gpt2"])
tests/tokenizers_/test_registry.py
+21 -2
@@ -2,7 +2,14 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 from pathlib import Path
4
5 -from vllm.tokenizers import TokenizerLike, TokenizerRegistry, get_tokenizer
5 +import pytest
6 +
7 +from vllm.tokenizers import TokenizerLike
8 +from vllm.tokenizers.registry import (
9 + TokenizerRegistry,
10 + get_tokenizer,
11 + resolve_tokenizer_args,
12 +)
13
14
15 class TestTokenizer(TokenizerLike):
@@ -40,10 +47,22 @@ class TestTokenizer(TokenizerLike):
47 return True
48
49
50 +@pytest.mark.parametrize("runner_type", ["generate", "pooling"])
51 +def test_resolve_tokenizer_args_idempotent(runner_type):
52 + tokenizer_mode, tokenizer_name, args, kwargs = resolve_tokenizer_args(
53 + "facebook/opt-125m",
54 + runner_type=runner_type,
55 + )
56 +
57 + assert (tokenizer_mode, tokenizer_name, args, kwargs) == resolve_tokenizer_args(
58 + tokenizer_name, *args, **kwargs
59 + )
60 +
61 +
62 def test_customized_tokenizer():
63 TokenizerRegistry.register("test_tokenizer", __name__, TestTokenizer.__name__)
64
46 - tokenizer = TokenizerRegistry.get_tokenizer("test_tokenizer", "abc")
65 + tokenizer = TokenizerRegistry.load_tokenizer("test_tokenizer", "abc")
66 assert isinstance(tokenizer, TestTokenizer)
67 assert tokenizer.path_or_repo_id == "abc"
68 assert tokenizer.bos_token_id == 0
vllm/entrypoints/chat_utils.py
+3 -2
@@ -50,7 +50,6 @@ from vllm.model_executor.models import SupportsMultiModal
50 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict, MultiModalUUIDDict
51 from vllm.multimodal.utils import MEDIA_CONNECTOR_REGISTRY, MediaConnector
52 from vllm.tokenizers import TokenizerLike
53 -from vllm.tokenizers.mistral import MistralTokenizer
53 from vllm.transformers_utils.chat_templates import get_chat_template_fallback_path
54 from vllm.transformers_utils.processor import cached_get_processor
55 from vllm.utils import random_uuid
@@ -60,6 +59,8 @@ from vllm.utils.import_utils import LazyLoader
59
60 if TYPE_CHECKING:
61 import torch
62 +
63 + from vllm.tokenizers.mistral import MistralTokenizer
64 else:
65 torch = LazyLoader("torch", globals(), "torch")
66
@@ -1832,7 +1833,7 @@ def apply_hf_chat_template(
1833
1834
1835 def apply_mistral_chat_template(
1835 - tokenizer: MistralTokenizer,
1836 + tokenizer: "MistralTokenizer",
1837 messages: list[ChatCompletionMessageParam],
1838 chat_template: str | None,
1839 tools: list[dict[str, Any]] | None,
vllm/tokenizers/__init__.py
-6
@@ -1,9 +1,6 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 -from .deepseekv32 import DeepseekV32Tokenizer
5 -from .hf import HfTokenizer
6 -from .mistral import MistralTokenizer
4 from .protocol import TokenizerLike
5 from .registry import (
6 TokenizerRegistry,
@@ -15,12 +12,9 @@ from .registry import (
12
13 __all__ = [
14 "TokenizerLike",
18 - "HfTokenizer",
19 - "MistralTokenizer",
15 "TokenizerRegistry",
16 "cached_get_tokenizer",
17 "get_tokenizer",
18 "cached_tokenizer_from_config",
19 "init_tokenizer_from_config",
25 - "DeepseekV32Tokenizer",
20 ]
vllm/tokenizers/deepseekv32.py
+33 -14
@@ -2,24 +2,18 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 from pathlib import Path
5 +from typing import Any
6
7 from transformers import BatchEncoding
8
8 -from .deepseek_v32_encoding import encode_messages
9 -from .hf import HfTokenizer, TokenizerLike
10 -from .registry import TokenizerRegistry
9 +from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
10
11 +from .deepseek_v32_encoding import encode_messages
12 +from .hf import CachedHfTokenizer
13 +from .protocol import TokenizerLike
14
13 -@TokenizerRegistry.register("deepseek_v32")
14 -class DeepseekV32Tokenizer(HfTokenizer):
15 - def __init__(self, tokenizer: TokenizerLike):
16 - self.tokenizer = tokenizer
17 - self.name_or_path = (
18 - tokenizer.name_or_path if hasattr(tokenizer, "name_or_path") else ""
19 - )
20 - self._added_vocab = self.tokenizer.get_added_vocab()
21 - self._added_vocab_size = len(self._added_vocab)
15
16 +class DeepseekV32Tokenizer(CachedHfTokenizer):
17 @classmethod
18 def from_pretrained(
19 cls,
@@ -40,7 +34,21 @@ class DeepseekV32Tokenizer(HfTokenizer):
34 )
35 return DeepseekV32Tokenizer(tokenizer)
36
43 - def apply_chat_template(self, messages, tools=None, **kwargs):
37 + def __init__(self, tokenizer: TokenizerLike) -> None:
38 + super().__init__()
39 +
40 + self.tokenizer = tokenizer
41 + self.name_or_path = getattr(tokenizer, "name_or_path", "")
42 +
43 + self._added_vocab = self.tokenizer.get_added_vocab()
44 + self._added_vocab_size = len(self._added_vocab)
45 +
46 + def apply_chat_template(
47 + self,
48 + messages: list["ChatCompletionMessageParam"],
49 + tools: list[dict[str, Any]] | None = None,
50 + **kwargs,
51 + ) -> str | list[int]:
52 thinking = kwargs.get("thinking", False)
53 thinking_mode = "thinking"
54 if not thinking:
@@ -49,13 +57,24 @@ class DeepseekV32Tokenizer(HfTokenizer):
57 messages = conversation.copy()
58 if tools is not None and len(tools) > 0:
59 messages.insert(0, {"role": "system"})
52 - messages[0]["tools"] = tools
60 + messages[0]["tools"] = tools # type: ignore[typeddict-unknown-key]
61
62 # Historical reasoning content is dropped when a new user message is introduced
63 drop_thinking = messages[-1]["role"] == "user"
64
65 encode_config = dict(thinking_mode=thinking_mode, drop_thinking=drop_thinking)
66 prompt_str = encode_messages(messages, **encode_config) # type: ignore
67 +
68 + if kwargs.get("tokenize", True):
69 + tokenizer_kwargs = {
70 + k: kwargs[k] for k in ("truncation", "max_length") if k in kwargs
71 + }
72 + return self.encode(
73 + prompt_str,
74 + add_special_tokens=False,
75 + **tokenizer_kwargs,
76 + )
77 +
78 return prompt_str
79
80 def num_special_tokens_to_add(self) -> int:
vllm/tokenizers/hf.py
+7 -12
@@ -3,22 +3,18 @@
3 import contextlib
4 import copy
5 from pathlib import Path
6 -from typing import TYPE_CHECKING
6 +from typing import TypeAlias
7
8 -from transformers import AutoTokenizer
8 +from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast
9
10 from vllm.transformers_utils.config import get_sentence_transformer_tokenizer_config
11
12 from .protocol import TokenizerLike
13 -from .registry import TokenizerRegistry
13
15 -if TYPE_CHECKING:
16 - from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
14 +HfTokenizer: TypeAlias = PreTrainedTokenizer | PreTrainedTokenizerFast
15
16
19 -def get_cached_tokenizer(
20 - tokenizer: "PreTrainedTokenizer | PreTrainedTokenizerFast",
21 -) -> TokenizerLike:
17 +def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
18 """
19 By default, transformers will recompute multiple tokenizer properties
20 each time they are called, leading to a significant slowdown.
@@ -65,11 +61,10 @@ def get_cached_tokenizer(
61 CachedTokenizer.__name__ = f"Cached{tokenizer.__class__.__name__}"
62
63 cached_tokenizer.__class__ = CachedTokenizer
68 - return cached_tokenizer # type: ignore
64 + return cached_tokenizer
65
66
71 -@TokenizerRegistry.register("hf")
72 -class HfTokenizer(TokenizerLike):
67 +class CachedHfTokenizer(TokenizerLike):
68 @classmethod
69 def from_pretrained(
70 cls,
@@ -79,7 +74,7 @@ class HfTokenizer(TokenizerLike):
74 revision: str | None = None,
75 download_dir: str | None = None,
76 **kwargs,
82 - ) -> "TokenizerLike":
77 + ) -> HfTokenizer:
78 try:
79 tokenizer = AutoTokenizer.from_pretrained(
80 path_or_repo_id,
vllm/tokenizers/mistral.py
+2 -5
@@ -3,10 +3,11 @@
3 from pathlib import Path
4 from typing import TYPE_CHECKING, Any, cast
5
6 +from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
7 +from vllm.entrypoints.openai.protocol import ChatCompletionRequest
8 from vllm.logger import init_logger
9
10 from .protocol import TokenizerLike
9 -from .registry import TokenizerRegistry
11
12 if TYPE_CHECKING:
13 from mistral_common.protocol.instruct.request import (
@@ -15,9 +16,6 @@ if TYPE_CHECKING:
16 from mistral_common.tokens.tokenizers.tekken import Tekkenizer
17 from transformers import BatchEncoding
18
18 - from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
19 - from vllm.entrypoints.openai.protocol import ChatCompletionRequest
20 -
19 try:
20 # Transformers v5
21 from transformers.tokenization_mistral_common import MistralCommonBackend
@@ -201,7 +199,6 @@ def _tekken_token_to_id(tokenizer: "Tekkenizer", t: str | bytes) -> int:
199 return tokenizer.unk_id
200
201
204 -@TokenizerRegistry.register("mistral")
202 class MistralTokenizer(TokenizerLike):
203 @classmethod
204 def from_pretrained(
vllm/tokenizers/protocol.py
+1 -1
@@ -97,7 +97,7 @@ class TokenizerLike(Protocol):
97 messages: list["ChatCompletionMessageParam"],
98 tools: list[dict[str, Any]] | None = None,
99 **kwargs,
100 - ) -> list[int]:
100 + ) -> str | list[int]:
101 raise NotImplementedError
102
103 def convert_tokens_to_string(self, tokens: list[str]) -> str:
vllm/tokenizers/registry.py
+100 -100
@@ -1,13 +1,13 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 import importlib.util
4 -from collections.abc import Callable
4 +from dataclasses import dataclass, field
5 from functools import lru_cache
6 from pathlib import Path
7 -from typing import TYPE_CHECKING, TypeVar, overload
7 +from typing import TYPE_CHECKING
8
9 import huggingface_hub
10 -from typing_extensions import assert_never
10 +from typing_extensions import TypeVar, assert_never, deprecated
11
12 import vllm.envs as envs
13 from vllm.logger import init_logger
@@ -24,46 +24,25 @@ from vllm.utils.import_utils import resolve_obj_by_qualname
24 from .protocol import TokenizerLike
25
26 if TYPE_CHECKING:
27 - from vllm.config import ModelConfig
27 + from vllm.config.model import ModelConfig, RunnerType
28
29 logger = init_logger(__name__)
30
31 -_T = TypeVar("_T", bound=type[TokenizerLike])
31
32 +_VLLM_TOKENIZERS = {
33 + "deepseekv32": ("deepseekv32", "DeepseekV32Tokenizer"),
34 + "hf": ("hf", "CachedHfTokenizer"),
35 + "mistral": ("mistral", "MistralTokenizer"),
36 +}
37
34 -class TokenizerRegistry:
35 - # Tokenizer name -> tokenizer_cls or (tokenizer module, tokenizer class)
36 - REGISTRY: dict[str, type[TokenizerLike] | tuple[str, str]] = {}
38
38 - # In-tree tokenizers
39 - @staticmethod
40 - @overload
41 - def register(tokenizer_mode: str) -> Callable[[_T], _T]: ...
39 +@dataclass
40 +class _TokenizerRegistry:
41 + # Tokenizer mode -> (tokenizer module, tokenizer class)
42 + tokenizers: dict[str, tuple[str, str]] = field(default_factory=dict)
43
43 - # OOT tokenizers
44 - @staticmethod
45 - @overload
46 - def register(tokenizer_mode: str, module: str, class_name: str) -> None: ...
47 -
48 - @staticmethod
49 - def register(
50 - tokenizer_mode: str,
51 - module: str | None = None,
52 - class_name: str | None = None,
53 - ) -> Callable[[_T], _T] | None:
54 - # In-tree tokenizers
55 - if module is None or class_name is None:
56 -
57 - def wrapper(tokenizer_cls: _T) -> _T:
58 - assert tokenizer_mode not in TokenizerRegistry.REGISTRY
59 - TokenizerRegistry.REGISTRY[tokenizer_mode] = tokenizer_cls
60 -
61 - return tokenizer_cls
62 -
63 - return wrapper
64 -
65 - # OOT tokenizers
66 - if tokenizer_mode in TokenizerRegistry.REGISTRY:
44 + def register(self, tokenizer_mode: str, module: str, class_name: str) -> None:
45 + if tokenizer_mode in self.tokenizers:
46 logger.warning(
47 "%s.%s is already registered for tokenizer_mode=%r. "
48 "It is overwritten by the new one.",
@@ -72,36 +51,42 @@ class TokenizerRegistry:
51 tokenizer_mode,
52 )
53
75 - TokenizerRegistry.REGISTRY[tokenizer_mode] = (module, class_name)
54 + self.tokenizers[tokenizer_mode] = (module, class_name)
55
56 return None
57
79 - @staticmethod
80 - def get_tokenizer(tokenizer_mode: str, *args, **kwargs) -> "TokenizerLike":
81 - if tokenizer_mode not in TokenizerRegistry.REGISTRY:
58 + def load_tokenizer_cls(self, tokenizer_mode: str) -> type[TokenizerLike]:
59 + if tokenizer_mode not in self.tokenizers:
60 raise ValueError(f"No tokenizer registered for {tokenizer_mode=!r}.")
61
84 - item = TokenizerRegistry.REGISTRY[tokenizer_mode]
85 - if isinstance(item, type):
86 - return item.from_pretrained(*args, **kwargs)
87 -
88 - module, class_name = item
62 + module, class_name = self.tokenizers[tokenizer_mode]
63 logger.debug_once(f"Loading {class_name} for {tokenizer_mode=!r}")
64
91 - class_ = resolve_obj_by_qualname(f"{module}.{class_name}")
92 - return class_.from_pretrained(*args, **kwargs)
65 + return resolve_obj_by_qualname(f"{module}.{class_name}")
66
67 + def load_tokenizer(self, tokenizer_mode: str, *args, **kwargs) -> TokenizerLike:
68 + tokenizer_cls = self.load_tokenizer_cls(tokenizer_mode)
69 + return tokenizer_cls.from_pretrained(*args, **kwargs)
70
95 -def get_tokenizer(
71 +
72 +TokenizerRegistry = _TokenizerRegistry(
73 + {
74 + mode: (f"vllm.tokenizers.{mod_relname}", cls_name)
75 + for mode, (mod_relname, cls_name) in _VLLM_TOKENIZERS.items()
76 + }
77 +)
78 +
79 +
80 +def resolve_tokenizer_args(
81 tokenizer_name: str | Path,
82 *args,
83 + runner_type: "RunnerType" = "generate",
84 tokenizer_mode: str = "auto",
99 - trust_remote_code: bool = False,
100 - revision: str | None = None,
101 - download_dir: str | None = None,
85 **kwargs,
103 -) -> TokenizerLike:
104 - """Gets a tokenizer for the given model name via HuggingFace or ModelScope."""
86 +):
87 + revision: str | None = kwargs.get("revision")
88 + download_dir: str | None = kwargs.get("download_dir")
89 +
90 if envs.VLLM_USE_MODELSCOPE:
91 # download model from ModelScope hub,
92 # lazy import so that modelscope is not required for normal use.
@@ -125,16 +110,6 @@ def get_tokenizer(
110 )
111 tokenizer_name = tokenizer_path
112
128 - if tokenizer_mode == "slow":
129 - if kwargs.get("use_fast", False):
130 - raise ValueError("Cannot use the fast tokenizer in slow tokenizer mode.")
131 -
132 - tokenizer_mode = "hf"
133 - kwargs["use_fast"] = False
134 -
135 - if "truncation_side" not in kwargs:
136 - kwargs["truncation_side"] = "left"
137 -
113 # Separate model folder from file path for GGUF models
114 if is_gguf(tokenizer_name):
115 if check_gguf_file(tokenizer_name):
@@ -150,6 +125,21 @@ def get_tokenizer(
125 )
126 kwargs["gguf_file"] = gguf_file
127
128 + if "truncation_side" not in kwargs:
129 + if runner_type == "generate" or runner_type == "draft":
130 + kwargs["truncation_side"] = "left"
131 + elif runner_type == "pooling":
132 + kwargs["truncation_side"] = "right"
133 + else:
134 + assert_never(runner_type)
135 +
136 + if tokenizer_mode == "slow":
137 + if kwargs.get("use_fast", False):
138 + raise ValueError("Cannot use the fast tokenizer in slow tokenizer mode.")
139 +
140 + tokenizer_mode = "hf"
141 + kwargs["use_fast"] = False
142 +
143 # Try to use official Mistral tokenizer if possible
144 if tokenizer_mode == "auto" and importlib.util.find_spec("mistral_common"):
145 allow_patterns = ["tekken.json", "tokenizer.model.v*"]
@@ -165,49 +155,70 @@ def get_tokenizer(
155 if tokenizer_mode == "auto":
156 tokenizer_mode = "hf"
157
168 - tokenizer_args = (tokenizer_name, *args)
169 - tokenizer_kwargs = dict(
158 + return tokenizer_mode, tokenizer_name, args, kwargs
159 +
160 +
161 +cached_resolve_tokenizer_args = lru_cache(resolve_tokenizer_args)
162 +
163 +
164 +def tokenizer_args_from_config(config: "ModelConfig", **kwargs):
165 + return cached_resolve_tokenizer_args(
166 + config.tokenizer,
167 + runner_type=config.runner_type,
168 + tokenizer_mode=config.tokenizer_mode,
169 + revision=config.tokenizer_revision,
170 + trust_remote_code=config.trust_remote_code,
171 + **kwargs,
172 + )
173 +
174 +
175 +_T = TypeVar("_T", bound=TokenizerLike, default=TokenizerLike)
176 +
177 +
178 +def get_tokenizer(
179 + tokenizer_name: str | Path,
180 + *args,
181 + tokenizer_cls: type[_T] = TokenizerLike, # type: ignore[assignment]
182 + trust_remote_code: bool = False,
183 + revision: str | None = None,
184 + download_dir: str | None = None,
185 + **kwargs,
186 +) -> _T:
187 + """Gets a tokenizer for the given model name via HuggingFace or ModelScope."""
188 + tokenizer_mode, tokenizer_name, args, kwargs = cached_resolve_tokenizer_args(
189 + tokenizer_name,
190 + *args,
191 trust_remote_code=trust_remote_code,
192 revision=revision,
193 download_dir=download_dir,
194 **kwargs,
195 )
196
176 - if tokenizer_mode == "custom":
177 - logger.warning_once(
178 - "TokenizerRegistry now uses `tokenizer_mode` as the registry key "
179 - "instead of `tokenizer_name`. "
180 - "Please update the definition of `.from_pretrained` in "
181 - "your custom tokenizer to accept `args=%s`, `kwargs=%s`. "
182 - "Then, you can pass `tokenizer_mode=%r` instead of "
183 - "`tokenizer_mode='custom'` when initializing vLLM.",
184 - tokenizer_args,
185 - str(tokenizer_kwargs),
186 - tokenizer_name,
187 - )
188 -
189 - tokenizer_mode = str(tokenizer_name)
197 + if tokenizer_cls == TokenizerLike:
198 + tokenizer_cls_ = TokenizerRegistry.load_tokenizer_cls(tokenizer_mode)
199 + else:
200 + tokenizer_cls_ = tokenizer_cls
201
191 - tokenizer = TokenizerRegistry.get_tokenizer(
192 - tokenizer_mode,
193 - *tokenizer_args,
194 - **tokenizer_kwargs,
195 - )
202 + tokenizer = tokenizer_cls_.from_pretrained(tokenizer_name, *args, **kwargs)
203 if not tokenizer.is_fast:
204 logger.warning(
205 "Using a slow tokenizer. This might cause a significant "
206 "slowdown. Consider using a fast tokenizer instead."
207 )
208
202 - return tokenizer
209 + return tokenizer # type: ignore
210
211
212 cached_get_tokenizer = lru_cache(get_tokenizer)
213
214
215 def cached_tokenizer_from_config(model_config: "ModelConfig", **kwargs):
216 + if model_config.skip_tokenizer_init:
217 + return None
218 +
219 return cached_get_tokenizer(
220 model_config.tokenizer,
221 + runner_type=model_config.runner_type,
222 tokenizer_mode=model_config.tokenizer_mode,
223 revision=model_config.tokenizer_revision,
224 trust_remote_code=model_config.trust_remote_code,
@@ -215,19 +226,8 @@ def cached_tokenizer_from_config(model_config: "ModelConfig", **kwargs):
226 )
227
228
229 +@deprecated(
230 + "Renamed to `cached_tokenizer_from_config`. The old name will be removed in v0.14."
231 +)
232 def init_tokenizer_from_config(model_config: "ModelConfig"):
219 - runner_type = model_config.runner_type
220 - if runner_type == "generate" or runner_type == "draft":
221 - truncation_side = "left"
222 - elif runner_type == "pooling":
223 - truncation_side = "right"
224 - else:
225 - assert_never(runner_type)
226 -
227 - return get_tokenizer(
228 - model_config.tokenizer,
229 - tokenizer_mode=model_config.tokenizer_mode,
230 - trust_remote_code=model_config.trust_remote_code,
231 - revision=model_config.tokenizer_revision,
232 - truncation_side=truncation_side,
233 - )
233 + return cached_tokenizer_from_config(model_config)
vllm/transformers_utils/tokenizer.py
+3 -3
@@ -60,17 +60,17 @@ def __getattr__(name: str):
60
61 return cached_tokenizer_from_config
62 if name == "init_tokenizer_from_configs":
63 - from vllm.tokenizers import init_tokenizer_from_config
63 + from vllm.tokenizers import cached_tokenizer_from_config
64
65 warnings.warn(
66 "`vllm.transformers_utils.tokenizer.init_tokenizer_from_configs` "
67 - "has been moved to `vllm.tokenizers.init_tokenizer_from_config`. "
67 + "has been moved to `vllm.tokenizers.cached_tokenizer_from_config`. "
68 "The old name will be removed in v0.14.",
69 DeprecationWarning,
70 stacklevel=2,
71 )
72
73 - return init_tokenizer_from_config
73 + return cached_tokenizer_from_config
74
75 raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
76
vllm/v1/engine/async_llm.py
+2 -2
@@ -26,7 +26,7 @@ from vllm.plugins.io_processors import get_io_processor
26 from vllm.pooling_params import PoolingParams
27 from vllm.sampling_params import SamplingParams
28 from vllm.tasks import SupportedTask
29 -from vllm.tokenizers import TokenizerLike, init_tokenizer_from_config
29 +from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
30 from vllm.tracing import init_tracer
31 from vllm.transformers_utils.config import maybe_register_config_serialize_by_value
32 from vllm.usage.usage_lib import UsageContext
@@ -111,7 +111,7 @@ class AsyncLLM(EngineClient):
111 if self.model_config.skip_tokenizer_init:
112 tokenizer = None
113 else:
114 - tokenizer = init_tokenizer_from_config(self.model_config)
114 + tokenizer = cached_tokenizer_from_config(self.model_config)
115
116 self.input_processor = InputProcessor(self.vllm_config, tokenizer)
117 self.io_processor = get_io_processor(
vllm/v1/engine/llm_engine.py
+2 -2
@@ -23,7 +23,7 @@ from vllm.plugins.io_processors import get_io_processor
23 from vllm.pooling_params import PoolingParams
24 from vllm.sampling_params import SamplingParams
25 from vllm.tasks import SupportedTask
26 -from vllm.tokenizers import TokenizerLike, init_tokenizer_from_config
26 +from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
27 from vllm.tracing import init_tracer
28 from vllm.usage.usage_lib import UsageContext
29 from vllm.v1.engine import EngineCoreRequest
@@ -86,7 +86,7 @@ class LLMEngine:
86 if self.model_config.skip_tokenizer_init:
87 tokenizer = None
88 else:
89 - tokenizer = init_tokenizer_from_config(self.model_config)
89 + tokenizer = cached_tokenizer_from_config(self.model_config)
90
91 self.input_processor = InputProcessor(self.vllm_config, tokenizer)
92 self.io_processor = get_io_processor(
vllm/v1/structured_output/__init__.py
+2 -2
@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING
7 from vllm.config import VllmConfig
8 from vllm.logger import init_logger
9 from vllm.reasoning import ReasoningParserManager
10 -from vllm.tokenizers import init_tokenizer_from_config
10 +from vllm.tokenizers import cached_tokenizer_from_config
11 from vllm.utils.import_utils import LazyLoader
12 from vllm.v1.structured_output.backend_guidance import GuidanceBackend
13 from vllm.v1.structured_output.backend_types import (
@@ -71,7 +71,7 @@ class StructuredOutputManager:
71 # of CPUs.
72 max_workers = max(1, (multiprocessing.cpu_count() + 1) // 2)
73 self.executor = ThreadPoolExecutor(max_workers=max_workers)
74 - self.tokenizer = init_tokenizer_from_config(
74 + self.tokenizer = cached_tokenizer_from_config(
75 model_config=self.vllm_config.model_config
76 )
77 reasoning_parser = (