A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 07:15 UTC
[Misc] Unify tokenizer registration (#29767)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Dec 1, 2025 at 19:34 UTC
f0a28bf6610602a4f064ac59e1449f5acdf04c7a
14 files changed
+237
-183
tests/entrypoints/openai/test_tokenization.py
+5
-5
@@ -53,7 +53,7 @@ async def test_tokenize_completions(
53
model_name: str,
54
tokenizer_name: str,
55
):
56
- tokenizer = get_tokenizer(tokenizer_name=tokenizer_name, tokenizer_mode="fast")
56
+ tokenizer = get_tokenizer(tokenizer_name=tokenizer_name)
57
58
for add_special in [False, True]:
59
prompt = "vllm1 This is a test prompt."
@@ -87,7 +87,7 @@ async def test_tokenize_chat(
87
model_name: str,
88
tokenizer_name: str,
89
):
90
- tokenizer = get_tokenizer(tokenizer_name=tokenizer_name, tokenizer_mode="fast")
90
+ tokenizer = get_tokenizer(tokenizer_name=tokenizer_name)
91
92
for add_generation in [False, True]:
93
for add_special in [False, True]:
@@ -140,7 +140,7 @@ async def test_tokenize_chat_with_tools(
140
model_name: str,
141
tokenizer_name: str,
142
):
143
- tokenizer = get_tokenizer(tokenizer_name=tokenizer_name, tokenizer_mode="fast")
143
+ tokenizer = get_tokenizer(tokenizer_name=tokenizer_name)
144
145
for add_generation in [False, True]:
146
for add_special in [False, True]:
@@ -210,7 +210,7 @@ async def test_tokenize_with_return_token_strs(
210
model_name: str,
211
tokenizer_name: str,
212
):
213
- tokenizer = get_tokenizer(tokenizer_name=tokenizer_name, tokenizer_mode="fast")
213
+ tokenizer = get_tokenizer(tokenizer_name=tokenizer_name)
214
215
prompt = "This is a token_strs test prompt! vllm1"
216
response = requests.post(
@@ -240,7 +240,7 @@ async def test_detokenize(
240
model_name: str,
241
tokenizer_name: str,
242
):
243
- tokenizer = get_tokenizer(tokenizer_name=tokenizer_name, tokenizer_mode="fast")
243
+ tokenizer = get_tokenizer(tokenizer_name=tokenizer_name)
244
245
prompt = "This is a test prompt. vllm1"
246
tokens = tokenizer.encode(prompt, add_special_tokens=False)
tests/entrypoints/pooling/embed/test_online.py
+1
-1
@@ -197,7 +197,7 @@ async def test_conversation_embedding(
197
chat_response.raise_for_status()
198
chat_embeddings = EmbeddingResponse.model_validate(chat_response.json())
199
200
- tokenizer = get_tokenizer(tokenizer_name=model_name, tokenizer_mode="fast")
200
+ tokenizer = get_tokenizer(tokenizer_name=model_name)
201
prompt = tokenizer.apply_chat_template(
202
messages,
203
chat_template=DUMMY_CHAT_TEMPLATE,
tests/entrypoints/pooling/pooling/test_online.py
+1
-5
@@ -158,11 +158,7 @@ async def test_conversation_pooling(server: RemoteOpenAIServer, model_name: str)
158
chat_response.raise_for_status()
159
chat_poolings = PoolingResponse.model_validate(chat_response.json())
160
161
- tokenizer = get_tokenizer(
162
- tokenizer_name=model_name,
163
- tokenizer_mode="fast",
164
- trust_remote_code=True,
165
- )
161
+ tokenizer = get_tokenizer(tokenizer_name=model_name, trust_remote_code=True)
162
prompt = tokenizer.apply_chat_template(
163
messages,
164
chat_template=DUMMY_CHAT_TEMPLATE,
tests/models/registry.py
+1
-1
@@ -23,7 +23,7 @@ class _HfExamplesInfo:
23
tokenizer: str | None = None
24
"""Set the tokenizer to load for this architecture."""
25
26
- tokenizer_mode: TokenizerMode = "auto"
26
+ tokenizer_mode: TokenizerMode | str = "auto"
27
"""Set the tokenizer type for this architecture."""
28
29
speculative_model: str | None = None
tests/tokenizers_/test_registry.py
+21
-4
@@ -1,13 +1,28 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from pathlib import Path
4
+
5
from vllm.tokenizers import TokenizerLike, TokenizerRegistry
6
from vllm.transformers_utils.tokenizer import get_tokenizer
7
8
9
class TestTokenizer(TokenizerLike):
10
@classmethod
9
- def from_pretrained(cls, *args, **kwargs) -> "TestTokenizer":
10
- return TestTokenizer() # type: ignore
11
+ def from_pretrained(
12
+ cls,
13
+ path_or_repo_id: str | Path,
14
+ *args,
15
+ trust_remote_code: bool = False,
16
+ revision: str | None = None,
17
+ download_dir: str | None = None,
18
+ **kwargs,
19
+ ) -> "TestTokenizer":
20
+ return TestTokenizer(path_or_repo_id) # type: ignore
21
+
22
+ def __init__(self, path_or_repo_id: str | Path) -> None:
23
+ super().__init__()
24
+
25
+ self.path_or_repo_id = path_or_repo_id
26
27
@property
28
def bos_token_id(self) -> int:
@@ -29,14 +44,16 @@ class TestTokenizer(TokenizerLike):
44
def test_customized_tokenizer():
45
TokenizerRegistry.register("test_tokenizer", __name__, TestTokenizer.__name__)
46
32
- tokenizer = TokenizerRegistry.get_tokenizer("test_tokenizer")
47
+ tokenizer = TokenizerRegistry.get_tokenizer("test_tokenizer", "abc")
48
assert isinstance(tokenizer, TestTokenizer)
49
+ assert tokenizer.path_or_repo_id == "abc"
50
assert tokenizer.bos_token_id == 0
51
assert tokenizer.eos_token_id == 1
52
assert tokenizer.pad_token_id == 2
53
38
- tokenizer = get_tokenizer("test_tokenizer", tokenizer_mode="custom")
54
+ tokenizer = get_tokenizer("abc", tokenizer_mode="test_tokenizer")
55
assert isinstance(tokenizer, TestTokenizer)
56
+ assert tokenizer.path_or_repo_id == "abc"
57
assert tokenizer.bos_token_id == 0
58
assert tokenizer.eos_token_id == 1
59
assert tokenizer.pad_token_id == 2
tests/v1/entrypoints/llm/test_struct_output_generate.py
+2
-7
@@ -4,7 +4,7 @@
4
5
import json
6
from enum import Enum
7
-from typing import TYPE_CHECKING, Any
7
+from typing import Any
8
9
import jsonschema
10
import pytest
@@ -24,11 +24,6 @@ from vllm.sampling_params import (
24
StructuredOutputsParams,
25
)
26
27
-if TYPE_CHECKING:
28
- from vllm.config.model import TokenizerMode
29
-else:
30
- TokenizerMode = str
31
-
27
NGRAM_SPEC_CONFIG = {
28
"model": "[ngram]",
29
"num_speculative_tokens": 5,
@@ -627,7 +622,7 @@ Make the response as short as possible.
622
)
623
def test_structured_output_with_reasoning_matrices(
624
backend: str,
630
- tokenizer_mode: TokenizerMode,
625
+ tokenizer_mode: str,
626
reasoning_parser: str,
627
model_name: str,
628
speculative_config: dict[str, Any] | None,
vllm/config/model.py
+7
-15
@@ -86,7 +86,7 @@ TaskOption = Literal[
86
"transcription",
87
"draft",
88
]
89
-TokenizerMode = Literal["auto", "hf", "slow", "mistral", "custom"]
89
+TokenizerMode = Literal["auto", "hf", "slow", "mistral"]
90
ModelDType = Literal["auto", "half", "float16", "bfloat16", "float", "float32"]
91
LogprobsMode = Literal[
92
"raw_logits", "raw_logprobs", "processed_logits", "processed_logprobs"
@@ -137,13 +137,13 @@ class ModelConfig:
137
tokenizer: SkipValidation[str] = None # type: ignore
138
"""Name or path of the Hugging Face tokenizer to use. If unspecified, model
139
name or path will be used."""
140
- tokenizer_mode: TokenizerMode = "auto"
140
+ tokenizer_mode: TokenizerMode | str = "auto"
141
"""Tokenizer mode:\n
142
- "auto" will use "hf" tokenizer if Mistral's tokenizer is not available.\n
143
- "hf" will use the fast tokenizer if available.\n
144
- "slow" will always use the slow tokenizer.\n
145
- "mistral" will always use the tokenizer from `mistral_common`.\n
146
- - "custom" will use --tokenizer to select the preregistered tokenizer."""
146
+ - Other custom values can be supported via plugins."""
147
trust_remote_code: bool = False
148
"""Trust remote code (e.g., from HuggingFace) when downloading the model
149
and tokenizer."""
@@ -708,9 +708,6 @@ class ModelConfig:
708
# can be correctly capped to sliding window size
709
self.hf_text_config.sliding_window = None
710
711
- if not self.skip_tokenizer_init:
712
- self._verify_tokenizer_mode()
713
-
711
# Avoid running try_verify_and_update_config multiple times
712
self.config_updated = False
713
@@ -718,6 +715,10 @@ class ModelConfig:
715
self._verify_cuda_graph()
716
self._verify_bnb_config()
717
718
+ @field_validator("tokenizer_mode", mode="after")
719
+ def _lowercase_tokenizer_mode(cls, tokenizer_mode: str) -> str:
720
+ return tokenizer_mode.lower()
721
+
722
@field_validator("quantization", mode="before")
723
@classmethod
724
def validate_quantization_before(cls, value: Any) -> Any:
@@ -829,15 +830,6 @@ class ModelConfig:
830
model, _ = split_remote_gguf(model)
831
return get_sentence_transformer_tokenizer_config(model, self.revision)
832
832
- def _verify_tokenizer_mode(self) -> None:
833
- tokenizer_mode = cast(TokenizerMode, self.tokenizer_mode.lower())
834
- if tokenizer_mode not in get_args(TokenizerMode):
835
- raise ValueError(
836
- f"Unknown tokenizer mode: {self.tokenizer_mode}. Must be "
837
- f"one of {get_args(TokenizerMode)}."
838
- )
839
- self.tokenizer_mode = tokenizer_mode
840
-
833
def _get_default_runner_type(
834
self,
835
architectures: list[str],
vllm/engine/arg_utils.py
+1
-1
@@ -360,7 +360,7 @@ class EngineArgs:
360
task: TaskOption | None = ModelConfig.task
361
skip_tokenizer_init: bool = ModelConfig.skip_tokenizer_init
362
enable_prompt_embeds: bool = ModelConfig.enable_prompt_embeds
363
- tokenizer_mode: TokenizerMode = ModelConfig.tokenizer_mode
363
+ tokenizer_mode: TokenizerMode | str = ModelConfig.tokenizer_mode
364
trust_remote_code: bool = ModelConfig.trust_remote_code
365
allowed_local_media_path: str = ModelConfig.allowed_local_media_path
366
allowed_media_domains: list[str] | None = ModelConfig.allowed_media_domains
vllm/entrypoints/llm.py
+1
-1
@@ -188,7 +188,7 @@ class LLM:
188
runner: RunnerOption = "auto",
189
convert: ConvertOption = "auto",
190
tokenizer: str | None = None,
191
- tokenizer_mode: TokenizerMode = "auto",
191
+ tokenizer_mode: TokenizerMode | str = "auto",
192
skip_tokenizer_init: bool = False,
193
trust_remote_code: bool = False,
194
allowed_local_media_path: str = "",
vllm/tokenizers/__init__.py
+8
-2
@@ -4,6 +4,12 @@
4
from .hf import HfTokenizer
5
from .mistral import MistralTokenizer
6
from .protocol import TokenizerLike
7
-from .registry import TokenizerRegistry
7
+from .registry import TokenizerRegistry, get_tokenizer
8
9
-__all__ = ["TokenizerLike", "HfTokenizer", "MistralTokenizer", "TokenizerRegistry"]
9
+__all__ = [
10
+ "TokenizerLike",
11
+ "HfTokenizer",
12
+ "MistralTokenizer",
13
+ "TokenizerRegistry",
14
+ "get_tokenizer",
15
+]
vllm/tokenizers/hf.py
+2
@@ -10,6 +10,7 @@ from transformers import AutoTokenizer
10
from vllm.transformers_utils.config import get_sentence_transformer_tokenizer_config
11
12
from .protocol import TokenizerLike
13
+from .registry import TokenizerRegistry
14
15
if TYPE_CHECKING:
16
from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
@@ -67,6 +68,7 @@ def get_cached_tokenizer(
68
return cached_tokenizer # type: ignore
69
70
71
+@TokenizerRegistry.register("hf")
72
class HfTokenizer(TokenizerLike):
73
@classmethod
74
def from_pretrained(
vllm/tokenizers/mistral.py
+2
@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Any, cast
6
from vllm.logger import init_logger
7
8
from .protocol import TokenizerLike
9
+from .registry import TokenizerRegistry
10
11
if TYPE_CHECKING:
12
from mistral_common.protocol.instruct.request import (
@@ -165,6 +166,7 @@ def _tekken_token_to_id(tokenizer: "Tekkenizer", t: str | bytes) -> int:
166
return tokenizer.unk_id
167
168
169
+@TokenizerRegistry.register("mistral")
170
class MistralTokenizer(TokenizerLike):
171
@classmethod
172
def from_pretrained(
vllm/tokenizers/registry.py
+184
-15
@@ -1,28 +1,197 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-import importlib
3
+import importlib.util
4
+from collections.abc import Callable
5
+from pathlib import Path
6
+from typing import TypeVar, overload
7
+
8
+import huggingface_hub
9
+
10
+import vllm.envs as envs
11
+from vllm.logger import init_logger
12
+from vllm.transformers_utils.gguf_utils import get_gguf_file_path_from_hf
13
+from vllm.transformers_utils.repo_utils import list_filtered_repo_files
14
+from vllm.transformers_utils.utils import (
15
+ check_gguf_file,
16
+ is_gguf,
17
+ is_remote_gguf,
18
+ split_remote_gguf,
19
+)
20
+from vllm.utils.import_utils import resolve_obj_by_qualname
21
22
from .protocol import TokenizerLike
23
24
+logger = init_logger(__name__)
25
+
26
+_T = TypeVar("_T", bound=type[TokenizerLike])
27
+
28
29
class TokenizerRegistry:
9
- # Tokenizer name -> (tokenizer module, tokenizer class)
10
- REGISTRY: dict[str, tuple[str, str]] = {}
30
+ # Tokenizer name -> tokenizer_cls or (tokenizer module, tokenizer class)
31
+ REGISTRY: dict[str, type[TokenizerLike] | tuple[str, str]] = {}
32
33
+ # In-tree tokenizers
34
@staticmethod
13
- def register(name: str, module: str, class_name: str) -> None:
14
- TokenizerRegistry.REGISTRY[name] = (module, class_name)
35
+ @overload
36
+ def register(tokenizer_mode: str) -> Callable[[_T], _T]: ...
37
38
+ # OOT tokenizers
39
@staticmethod
17
- def get_tokenizer(
18
- tokenizer_name: str,
19
- *args,
20
- **kwargs,
21
- ) -> "TokenizerLike":
22
- tokenizer_cls = TokenizerRegistry.REGISTRY.get(tokenizer_name)
23
- if tokenizer_cls is None:
24
- raise ValueError(f"Tokenizer {tokenizer_name} not found.")
40
+ @overload
41
+ def register(tokenizer_mode: str, module: str, class_name: str) -> None: ...
42
+
43
+ @staticmethod
44
+ def register(
45
+ tokenizer_mode: str,
46
+ module: str | None = None,
47
+ class_name: str | None = None,
48
+ ) -> Callable[[_T], _T] | None:
49
+ # In-tree tokenizers
50
+ if module is None or class_name is None:
51
+
52
+ def wrapper(tokenizer_cls: _T) -> _T:
53
+ assert tokenizer_mode not in TokenizerRegistry.REGISTRY
54
+ TokenizerRegistry.REGISTRY[tokenizer_mode] = tokenizer_cls
55
+
56
+ return tokenizer_cls
57
+
58
+ return wrapper
59
+
60
+ # OOT tokenizers
61
+ if tokenizer_mode in TokenizerRegistry.REGISTRY:
62
+ logger.warning(
63
+ "%s.%s is already registered for tokenizer_mode=%r. "
64
+ "It is overwritten by the new one.",
65
+ module,
66
+ class_name,
67
+ tokenizer_mode,
68
+ )
69
+
70
+ TokenizerRegistry.REGISTRY[tokenizer_mode] = (module, class_name)
71
+
72
+ return None
73
+
74
+ @staticmethod
75
+ def get_tokenizer(tokenizer_mode: str, *args, **kwargs) -> "TokenizerLike":
76
+ if tokenizer_mode not in TokenizerRegistry.REGISTRY:
77
+ raise ValueError(f"No tokenizer registered for {tokenizer_mode=!r}.")
78
+
79
+ item = TokenizerRegistry.REGISTRY[tokenizer_mode]
80
+ if isinstance(item, type):
81
+ return item.from_pretrained(*args, **kwargs)
82
+
83
+ module, class_name = item
84
+ logger.debug_once(f"Loading {class_name} for {tokenizer_mode=!r}")
85
26
- tokenizer_module = importlib.import_module(tokenizer_cls[0])
27
- class_ = getattr(tokenizer_module, tokenizer_cls[1])
86
+ class_ = resolve_obj_by_qualname(f"{module}.{class_name}")
87
return class_.from_pretrained(*args, **kwargs)
88
+
89
+
90
+def get_tokenizer(
91
+ tokenizer_name: str | Path,
92
+ *args,
93
+ tokenizer_mode: str = "auto",
94
+ trust_remote_code: bool = False,
95
+ revision: str | None = None,
96
+ download_dir: str | None = None,
97
+ **kwargs,
98
+) -> TokenizerLike:
99
+ """Gets a tokenizer for the given model name via HuggingFace or ModelScope."""
100
+ if envs.VLLM_USE_MODELSCOPE:
101
+ # download model from ModelScope hub,
102
+ # lazy import so that modelscope is not required for normal use.
103
+ from modelscope.hub.snapshot_download import snapshot_download
104
+
105
+ # avoid circular import
106
+ from vllm.model_executor.model_loader.weight_utils import get_lock
107
+
108
+ # Only set the tokenizer here, model will be downloaded on the workers.
109
+ if not Path(tokenizer_name).exists():
110
+ # Use file lock to prevent multiple processes from
111
+ # downloading the same file at the same time.
112
+ with get_lock(tokenizer_name, download_dir):
113
+ tokenizer_path = snapshot_download(
114
+ model_id=str(tokenizer_name),
115
+ cache_dir=download_dir,
116
+ revision=revision,
117
+ local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
118
+ # Ignore weights - we only need the tokenizer.
119
+ ignore_file_pattern=[".*.pt", ".*.safetensors", ".*.bin"],
120
+ )
121
+ tokenizer_name = tokenizer_path
122
+
123
+ if tokenizer_mode == "slow":
124
+ if kwargs.get("use_fast", False):
125
+ raise ValueError("Cannot use the fast tokenizer in slow tokenizer mode.")
126
+
127
+ tokenizer_mode = "hf"
128
+ kwargs["use_fast"] = False
129
+
130
+ if "truncation_side" not in kwargs:
131
+ kwargs["truncation_side"] = "left"
132
+
133
+ # Separate model folder from file path for GGUF models
134
+ if is_gguf(tokenizer_name):
135
+ if check_gguf_file(tokenizer_name):
136
+ kwargs["gguf_file"] = Path(tokenizer_name).name
137
+ tokenizer_name = Path(tokenizer_name).parent
138
+ elif is_remote_gguf(tokenizer_name):
139
+ tokenizer_name, quant_type = split_remote_gguf(tokenizer_name)
140
+ # Get the HuggingFace Hub path for the GGUF file
141
+ gguf_file = get_gguf_file_path_from_hf(
142
+ tokenizer_name,
143
+ quant_type,
144
+ revision=revision,
145
+ )
146
+ kwargs["gguf_file"] = gguf_file
147
+
148
+ # Try to use official Mistral tokenizer if possible
149
+ if tokenizer_mode == "auto" and importlib.util.find_spec("mistral_common"):
150
+ allow_patterns = ["tekken.json", "tokenizer.model.v*"]
151
+ files_list = list_filtered_repo_files(
152
+ model_name_or_path=str(tokenizer_name),
153
+ allow_patterns=allow_patterns,
154
+ revision=revision,
155
+ )
156
+ if len(files_list) > 0:
157
+ tokenizer_mode = "mistral"
158
+
159
+ # Fallback to HF tokenizer
160
+ if tokenizer_mode == "auto":
161
+ tokenizer_mode = "hf"
162
+
163
+ tokenizer_args = (tokenizer_name, *args)
164
+ tokenizer_kwargs = dict(
165
+ trust_remote_code=trust_remote_code,
166
+ revision=revision,
167
+ download_dir=download_dir,
168
+ **kwargs,
169
+ )
170
+
171
+ if tokenizer_mode == "custom":
172
+ logger.warning_once(
173
+ "TokenizerRegistry now uses `tokenizer_mode` as the registry key "
174
+ "instead of `tokenizer_name`. "
175
+ "Please update the definition of `.from_pretrained` in "
176
+ "your custom tokenizer to accept `args=%s`, `kwargs=%s`. "
177
+ "Then, you can pass `tokenizer_mode=%r` instead of "
178
+ "`tokenizer_mode='custom'` when initializing vLLM.",
179
+ tokenizer_args,
180
+ str(tokenizer_kwargs),
181
+ tokenizer_mode,
182
+ )
183
+
184
+ tokenizer_mode = str(tokenizer_name)
185
+
186
+ tokenizer = TokenizerRegistry.get_tokenizer(
187
+ tokenizer_mode,
188
+ *tokenizer_args,
189
+ **tokenizer_kwargs,
190
+ )
191
+ if not tokenizer.is_fast:
192
+ logger.warning(
193
+ "Using a slow tokenizer. This might cause a significant "
194
+ "slowdown. Consider using a fast tokenizer instead."
195
+ )
196
+
197
+ return tokenizer
vllm/transformers_utils/tokenizer.py
+1
-126
@@ -1,28 +1,14 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-import importlib.util
5
-import os
4
import warnings
5
from functools import lru_cache
8
-from pathlib import Path
6
from typing import TYPE_CHECKING, Any
7
11
-import huggingface_hub
8
from typing_extensions import assert_never
9
14
-from vllm import envs
10
from vllm.logger import init_logger
16
-from vllm.tokenizers import (
17
- HfTokenizer,
18
- MistralTokenizer,
19
- TokenizerLike,
20
- TokenizerRegistry,
21
-)
22
-
23
-from .gguf_utils import get_gguf_file_path_from_hf
24
-from .repo_utils import list_filtered_repo_files
25
-from .utils import check_gguf_file, is_gguf, is_remote_gguf, split_remote_gguf
11
+from vllm.tokenizers import TokenizerLike, get_tokenizer
12
13
if TYPE_CHECKING:
14
from vllm.config import ModelConfig
@@ -108,117 +94,6 @@ def encode_tokens(
94
return tokenizer.encode(text, **kw_args)
95
96
111
-def get_tokenizer(
112
- tokenizer_name: str | Path,
113
- *args,
114
- tokenizer_mode: str = "auto",
115
- trust_remote_code: bool = False,
116
- revision: str | None = None,
117
- download_dir: str | None = None,
118
- **kwargs,
119
-) -> TokenizerLike:
120
- """Gets a tokenizer for the given model name via HuggingFace or ModelScope."""
121
- if envs.VLLM_USE_MODELSCOPE:
122
- # download model from ModelScope hub,
123
- # lazy import so that modelscope is not required for normal use.
124
- # pylint: disable=C.
125
- from modelscope.hub.snapshot_download import snapshot_download
126
-
127
- # avoid circuit import
128
- from vllm.model_executor.model_loader.weight_utils import get_lock
129
-
130
- # Only set the tokenizer here, model will be downloaded on the workers.
131
- if not os.path.exists(tokenizer_name):
132
- # Use file lock to prevent multiple processes from
133
- # downloading the same file at the same time.
134
- with get_lock(tokenizer_name, download_dir):
135
- tokenizer_path = snapshot_download(
136
- model_id=tokenizer_name,
137
- cache_dir=download_dir,
138
- revision=revision,
139
- local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
140
- # Ignore weights - we only need the tokenizer.
141
- ignore_file_pattern=[".*.pt", ".*.safetensors", ".*.bin"],
142
- )
143
- tokenizer_name = tokenizer_path
144
-
145
- if tokenizer_mode == "slow":
146
- if kwargs.get("use_fast", False):
147
- raise ValueError("Cannot use the fast tokenizer in slow tokenizer mode.")
148
- kwargs["use_fast"] = False
149
-
150
- if "truncation_side" not in kwargs:
151
- kwargs["truncation_side"] = "left"
152
-
153
- # Separate model folder from file path for GGUF models
154
- if is_gguf(tokenizer_name):
155
- if check_gguf_file(tokenizer_name):
156
- kwargs["gguf_file"] = Path(tokenizer_name).name
157
- tokenizer_name = Path(tokenizer_name).parent
158
- elif is_remote_gguf(tokenizer_name):
159
- tokenizer_name, quant_type = split_remote_gguf(tokenizer_name)
160
- # Get the HuggingFace Hub path for the GGUF file
161
- gguf_file = get_gguf_file_path_from_hf(
162
- tokenizer_name,
163
- quant_type,
164
- revision=revision,
165
- )
166
- kwargs["gguf_file"] = gguf_file
167
-
168
- # if `tokenizer_mode` == "auto", check if tokenizer can be loaded via Mistral format
169
- # first to use official Mistral tokenizer if possible.
170
- mistral_common_installed = importlib.util.find_spec("mistral_common") is not None
171
- if tokenizer_mode == "auto" and mistral_common_installed:
172
- allow_patterns = ["tekken.json", "tokenizer.model.v*"]
173
- files_list = list_filtered_repo_files(
174
- model_name_or_path=str(tokenizer_name),
175
- allow_patterns=allow_patterns,
176
- revision=revision,
177
- )
178
- if len(files_list) > 0:
179
- tokenizer_mode = "mistral"
180
-
181
- tokenizer: TokenizerLike
182
- if tokenizer_mode == "mistral":
183
- logger.debug_once(f"Loading MistralTokenizer from {tokenizer_name}")
184
- tokenizer = MistralTokenizer.from_pretrained(
185
- tokenizer_name,
186
- *args,
187
- trust_remote_code=trust_remote_code,
188
- revision=revision,
189
- download_dir=download_dir,
190
- **kwargs,
191
- )
192
- elif tokenizer_mode == "custom":
193
- logger.debug_once(f"Loading CustomTokenizer from {tokenizer_name}")
194
- tokenizer = TokenizerRegistry.get_tokenizer(
195
- str(tokenizer_name),
196
- *args,
197
- trust_remote_code=trust_remote_code,
198
- revision=revision,
199
- download_dir=download_dir,
200
- **kwargs,
201
- )
202
- else:
203
- logger.debug_once(f"Loading HfTokenizer from {tokenizer_name}")
204
- tokenizer = HfTokenizer.from_pretrained(
205
- tokenizer_name,
206
- *args,
207
- trust_remote_code=trust_remote_code,
208
- revision=revision,
209
- download_dir=download_dir,
210
- **kwargs,
211
- )
212
-
213
- if not tokenizer.is_fast:
214
- logger.warning(
215
- "Using a slow tokenizer. This might cause a significant "
216
- "slowdown. Consider using a fast tokenizer instead."
217
- )
218
-
219
- return tokenizer
220
-
221
-
97
cached_get_tokenizer = lru_cache(get_tokenizer)
98
99