A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 07:15 UTC
[Misc] Update `TokenizerLike` interface and move `get_cached_tokenizer` (#29730)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Nov 30, 2025 at 14:59 UTC
2afcec4decbbd64cf8da20d49281f828dd136c27
15 files changed
+260
-174
.buildkite/test-amd.yaml
+6
-7
@@ -61,8 +61,8 @@ steps:
61
- pytest -v -s -m 'not cpu_test' multimodal
62
- pytest -v -s utils_
63
64
-- label: Async Engine, Inputs, Utils, Worker, Config Test (CPU) # 4 mins
65
- timeout_in_minutes: 10
64
+- label: Async Engine, Inputs, Utils, Worker, Config Test (CPU) # 15min
65
+ timeout_in_minutes: 20
66
mirror_hardwares: [amdexperimental, amdproduction]
67
agent_pool: mi325_1
68
# grade: Blocking
@@ -72,6 +72,7 @@ steps:
72
- tests/test_outputs.py
73
- tests/multimodal
74
- tests/standalone_tests/lazy_imports.py
75
+ - tests/tokenizers_
76
- tests/transformers_utils
77
- tests/config
78
no_gpu: true
@@ -80,6 +81,7 @@ steps:
81
- pytest -v -s test_inputs.py
82
- pytest -v -s test_outputs.py
83
- pytest -v -s -m 'cpu_test' multimodal
84
+ - pytest -v -s tokenizers_
85
- pytest -v -s transformers_utils
86
- pytest -v -s config
87
@@ -308,23 +310,20 @@ steps:
310
- pytest -v -s test_regression.py
311
working_dir: "/vllm-workspace/tests" # optional
312
311
-- label: Engine Test # 25min
312
- timeout_in_minutes: 40
313
+- label: Engine Test # 9min
314
+ timeout_in_minutes: 15
315
mirror_hardwares: [amdexperimental, amdproduction]
316
agent_pool: mi325_1
317
# grade: Blocking
318
source_file_dependencies:
319
- vllm/
320
- tests/engine
319
- - tests/tokenizers_
321
- tests/test_sequence
322
- tests/test_config
323
- tests/test_logger
324
- tests/test_vllm_port
325
commands:
326
- pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py
326
- # OOM in the CI unless we run this separately
327
- - pytest -v -s tokenizers_
327
328
- label: V1 Test e2e + engine # 30min
329
timeout_in_minutes: 45
.buildkite/test-pipeline.yaml
+6
-7
@@ -57,14 +57,15 @@ steps:
57
- pytest -v -s -m 'not cpu_test' multimodal
58
- pytest -v -s utils_
59
60
-- label: Async Engine, Inputs, Utils, Worker, Config Test (CPU) # 4 mins
61
- timeout_in_minutes: 10
60
+- label: Async Engine, Inputs, Utils, Worker, Config Test (CPU) # 15min
61
+ timeout_in_minutes: 20
62
source_file_dependencies:
63
- vllm/
64
- tests/test_inputs.py
65
- tests/test_outputs.py
66
- tests/multimodal
67
- tests/standalone_tests/lazy_imports.py
68
+ - tests/tokenizers_
69
- tests/transformers_utils
70
- tests/config
71
no_gpu: true
@@ -73,6 +74,7 @@ steps:
74
- pytest -v -s test_inputs.py
75
- pytest -v -s test_outputs.py
76
- pytest -v -s -m 'cpu_test' multimodal
77
+ - pytest -v -s tokenizers_
78
- pytest -v -s transformers_utils
79
- pytest -v -s config
80
@@ -276,21 +278,18 @@ steps:
278
- pytest -v -s test_regression.py
279
working_dir: "/vllm-workspace/tests" # optional
280
279
-- label: Engine Test # 25min
280
- timeout_in_minutes: 40
281
+- label: Engine Test # 9min
282
+ timeout_in_minutes: 15
283
mirror_hardwares: [amdexperimental]
284
source_file_dependencies:
285
- vllm/
286
- tests/engine
285
- - tests/tokenizers_
287
- tests/test_sequence
288
- tests/test_config
289
- tests/test_logger
290
- tests/test_vllm_port
291
commands:
292
- pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py
292
- # OOM in the CI unless we run this separately
293
- - pytest -v -s tokenizers_
293
294
- label: V1 Test e2e + engine # 30min
295
timeout_in_minutes: 45
docs/design/huggingface_integration.md
+1
-1
@@ -21,7 +21,7 @@ Let's say we want to serve the popular Qwen model by running `vllm serve Qwen/Qw
21
22
Beyond that, there are two more things vLLM depends on Hugging Face for.
23
24
-1. **Tokenizer**: vLLM uses the tokenizer from Hugging Face to tokenize the input text. The tokenizer is loaded using [AutoTokenizer.from_pretrained](https://huggingface.co/docs/transformers/en/model_doc/auto#transformers.AutoTokenizer.from_pretrained) with the `model` argument as the model name and the `--revision` argument as the revision. It is also possible to use a tokenizer from another model by specifying the `--tokenizer` argument in the `vllm serve` command. Other relevant arguments are `--tokenizer-revision` and `--tokenizer-mode`. Please check Hugging Face's documentation for the meaning of these arguments. This part of the logic can be found in the [get_tokenizer](https://github.com/vllm-project/vllm/blob/127c07480ecea15e4c2990820c457807ff78a057/vllm/transformers_utils/tokenizer.py#L87) function. After obtaining the tokenizer, notably, vLLM will cache some expensive attributes of the tokenizer in [get_cached_tokenizer](https://github.com/vllm-project/vllm/blob/127c07480ecea15e4c2990820c457807ff78a057/vllm/transformers_utils/tokenizer.py#L24).
24
+1. **Tokenizer**: vLLM uses the tokenizer from Hugging Face to tokenize the input text. The tokenizer is loaded using [AutoTokenizer.from_pretrained](https://huggingface.co/docs/transformers/en/model_doc/auto#transformers.AutoTokenizer.from_pretrained) with the `model` argument as the model name and the `--revision` argument as the revision. It is also possible to use a tokenizer from another model by specifying the `--tokenizer` argument in the `vllm serve` command. Other relevant arguments are `--tokenizer-revision` and `--tokenizer-mode`. Please check Hugging Face's documentation for the meaning of these arguments. This part of the logic can be found in the [get_tokenizer](https://github.com/vllm-project/vllm/blob/127c07480ecea15e4c2990820c457807ff78a057/vllm/transformers_utils/tokenizer.py#L87) function. After obtaining the tokenizer, notably, vLLM will cache some expensive attributes of the tokenizer in [vllm.tokenizers.hf.get_cached_tokenizer][].
25
26
2. **Model weight**: vLLM downloads the model weight from the Hugging Face model hub using the `model` argument as the model name and the `--revision` argument as the revision. vLLM provides the argument `--load-format` to control what files to download from the model hub. By default, it will try to load the weights in the safetensors format and fall back to the PyTorch bin format if the safetensors format is not available. We can also pass `--load-format dummy` to skip downloading the weights.
27
- It is recommended to use the safetensors format, as it is efficient for loading in distributed inference and also safe from arbitrary code execution. See the [documentation](https://huggingface.co/docs/safetensors/en/index) for more information on the safetensors format. This part of the logic can be found [here](https://github.com/vllm-project/vllm/blob/10b67d865d92e376956345becafc249d4c3c0ab7/vllm/model_executor/model_loader/loader.py#L385). Please note that:
tests/tokenizers_/test_hf.py
renamed
+1
-1
@@ -7,7 +7,7 @@ import pytest
7
from transformers import AutoTokenizer
8
9
from vllm.tokenizers import TokenizerLike
10
-from vllm.transformers_utils.tokenizer import get_cached_tokenizer
10
+from vllm.tokenizers.hf import get_cached_tokenizer
11
12
13
@pytest.mark.parametrize("model_id", ["gpt2", "zai-org/chatglm3-6b"])
tests/tokenizers_/test_mistral.py
+3
-3
@@ -356,8 +356,8 @@ class TestMistralTokenizer:
356
)
357
attn_mask = [1 for _ in range(len(token_ids))]
358
359
- # Test 1: default
360
- assert mistral_tokenizer("Hello world !") == {
359
+ # Test 1: no special tokens
360
+ assert mistral_tokenizer("Hello world !", add_special_tokens=False) == {
361
"attention_mask": attn_mask[1:],
362
"input_ids": token_ids[1:],
363
}
@@ -381,7 +381,7 @@ class TestMistralTokenizer:
381
"input_ids": token_ids,
382
}
383
# Test 5: empty string
384
- assert mistral_tokenizer("") == {
384
+ assert mistral_tokenizer("", add_special_tokens=False) == {
385
"attention_mask": [],
386
"input_ids": [],
387
}
tests/tokenizers_/test_registry.py
+11
-5
@@ -17,20 +17,26 @@ class TestTokenizer(TokenizerLike):
17
def eos_token_id(self) -> int:
18
return 1
19
20
+ @property
21
+ def pad_token_id(self) -> int:
22
+ return 2
23
+
24
+ @property
25
+ def is_fast(self) -> bool:
26
+ return True
27
+
28
29
def test_customized_tokenizer():
22
- TokenizerRegistry.register(
23
- "test_tokenizer",
24
- __name__,
25
- TestTokenizer.__name__,
26
- )
30
+ TokenizerRegistry.register("test_tokenizer", __name__, TestTokenizer.__name__)
31
32
tokenizer = TokenizerRegistry.get_tokenizer("test_tokenizer")
33
assert isinstance(tokenizer, TestTokenizer)
34
assert tokenizer.bos_token_id == 0
35
assert tokenizer.eos_token_id == 1
36
+ assert tokenizer.pad_token_id == 2
37
38
tokenizer = get_tokenizer("test_tokenizer", tokenizer_mode="custom")
39
assert isinstance(tokenizer, TestTokenizer)
40
assert tokenizer.bos_token_id == 0
41
assert tokenizer.eos_token_id == 1
42
+ assert tokenizer.pad_token_id == 2
tools/pre_commit/check_pickle_imports.py
+1
-1
@@ -27,7 +27,7 @@ ALLOWED_FILES = {
27
"vllm/distributed/device_communicators/shm_broadcast.py",
28
"vllm/distributed/device_communicators/shm_object_storage.py",
29
"vllm/utils/hashing.py",
30
- "tests/tokenizers_/test_cached_tokenizer.py",
30
+ "tests/tokenizers_/test_hf.py",
31
"tests/utils_/test_hashing.py",
32
"benchmarks/kernels/graph_machete_bench.py",
33
"benchmarks/kernels/benchmark_lora.py",
vllm/entrypoints/llm.py
+1
-1
@@ -72,7 +72,7 @@ from vllm.pooling_params import PoolingParams
72
from vllm.sampling_params import BeamSearchParams, RequestOutputKind, SamplingParams
73
from vllm.tasks import PoolingTask
74
from vllm.tokenizers import MistralTokenizer, TokenizerLike
75
-from vllm.transformers_utils.tokenizer import get_cached_tokenizer
75
+from vllm.tokenizers.hf import get_cached_tokenizer
76
from vllm.usage.usage_lib import UsageContext
77
from vllm.utils.collection_utils import as_iter, is_list_of
78
from vllm.utils.counter import Counter
vllm/entrypoints/score_utils.py
+2
-2
@@ -51,8 +51,8 @@ def _cosine_similarity(
51
for emb_1, emb_2 in zip(embed_1, embed_2):
52
pair_score = scorer(emb_1.outputs.data, emb_2.outputs.data)
53
54
- padding = []
55
- if (pad_token_id := getattr(tokenizer, "pad_token_id", None)) is not None:
54
+ padding: list[int] = []
55
+ if (pad_token_id := tokenizer.pad_token_id) is not None:
56
padding = [pad_token_id]
57
58
tokens = emb_1.prompt_token_ids + padding + emb_2.prompt_token_ids
vllm/tokenizers/__init__.py
+2
-1
@@ -1,8 +1,9 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+from .hf import HfTokenizer
5
from .mistral import MistralTokenizer
6
from .protocol import TokenizerLike
7
from .registry import TokenizerRegistry
8
8
-__all__ = ["TokenizerLike", "MistralTokenizer", "TokenizerRegistry"]
9
+__all__ = ["TokenizerLike", "HfTokenizer", "MistralTokenizer", "TokenizerRegistry"]
vllm/tokenizers/hf.py
new
+122
@@ -0,0 +1,122 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import contextlib
4
+import copy
5
+from pathlib import Path
6
+from typing import TYPE_CHECKING
7
+
8
+from transformers import AutoTokenizer
9
+
10
+from vllm.transformers_utils.config import get_sentence_transformer_tokenizer_config
11
+
12
+from .protocol import TokenizerLike
13
+
14
+if TYPE_CHECKING:
15
+ from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
16
+
17
+
18
+def get_cached_tokenizer(
19
+ tokenizer: "PreTrainedTokenizer | PreTrainedTokenizerFast",
20
+) -> TokenizerLike:
21
+ """
22
+ By default, transformers will recompute multiple tokenizer properties
23
+ each time they are called, leading to a significant slowdown.
24
+ This proxy caches these properties for faster access.
25
+ """
26
+ cached_tokenizer = copy.copy(tokenizer)
27
+
28
+ tokenizer_all_special_ids = tokenizer.all_special_ids
29
+ tokenizer_all_special_tokens = tokenizer.all_special_tokens
30
+ tokenizer_vocab = tokenizer.get_vocab()
31
+ tokenizer_len = len(tokenizer)
32
+
33
+ max_token_id = max(tokenizer_vocab.values())
34
+ # Some tokenizers (e.g., QwenTokenizer) have special tokens that
35
+ # are added and included in the implementation of the vocab_size
36
+ # property, but not in get_vocab(); if there is an implementation
37
+ # of vocab size, we should take the greater value.
38
+ if hasattr(tokenizer, "vocab_size"):
39
+ with contextlib.suppress(NotImplementedError):
40
+ max_token_id = max(max_token_id, tokenizer.vocab_size)
41
+
42
+ class CachedTokenizer(tokenizer.__class__): # type: ignore
43
+ @property
44
+ def all_special_ids(self) -> list[int]:
45
+ return tokenizer_all_special_ids
46
+
47
+ @property
48
+ def all_special_tokens(self) -> list[str]:
49
+ return tokenizer_all_special_tokens
50
+
51
+ @property
52
+ def max_token_id(self) -> int:
53
+ return max_token_id
54
+
55
+ def get_vocab(self) -> dict[str, int]:
56
+ return tokenizer_vocab
57
+
58
+ def __len__(self) -> int:
59
+ return tokenizer_len
60
+
61
+ def __reduce__(self):
62
+ return get_cached_tokenizer, (tokenizer,)
63
+
64
+ CachedTokenizer.__name__ = f"Cached{tokenizer.__class__.__name__}"
65
+
66
+ cached_tokenizer.__class__ = CachedTokenizer
67
+ return cached_tokenizer # type: ignore
68
+
69
+
70
+class HfTokenizer(TokenizerLike):
71
+ @classmethod
72
+ def from_pretrained(
73
+ cls,
74
+ path_or_repo_id: str | Path,
75
+ *args,
76
+ trust_remote_code: bool = False,
77
+ revision: str | None = None,
78
+ download_dir: str | None = None,
79
+ **kwargs,
80
+ ) -> "TokenizerLike":
81
+ try:
82
+ tokenizer = AutoTokenizer.from_pretrained(
83
+ path_or_repo_id,
84
+ *args,
85
+ trust_remote_code=trust_remote_code,
86
+ revision=revision,
87
+ cache_dir=download_dir,
88
+ **kwargs,
89
+ )
90
+ except ValueError as e:
91
+ # If the error pertains to the tokenizer class not existing or not
92
+ # currently being imported,
93
+ # suggest using the --trust-remote-code flag.
94
+ if not trust_remote_code and (
95
+ "does not exist or is not currently imported." in str(e)
96
+ or "requires you to execute the tokenizer file" in str(e)
97
+ ):
98
+ err_msg = (
99
+ "Failed to load the tokenizer. If the tokenizer "
100
+ "is a custom tokenizer not yet available in the "
101
+ "HuggingFace transformers library, consider "
102
+ "setting `trust_remote_code=True` in LLM or using "
103
+ "the `--trust-remote-code` flag in the CLI."
104
+ )
105
+ raise RuntimeError(err_msg) from e
106
+ else:
107
+ raise e
108
+
109
+ # The special_tokens in tokenizer should also be
110
+ # controlled by do_lower_case in encoder_config
111
+ encoder_config = get_sentence_transformer_tokenizer_config(
112
+ path_or_repo_id, revision
113
+ )
114
+ if isinstance(encoder_config, dict) and encoder_config.get(
115
+ "do_lower_case", False
116
+ ):
117
+ special_tokens_map = {
118
+ k: v.lower() for k, v in tokenizer.special_tokens_map.items()
119
+ }
120
+ tokenizer.add_special_tokens(special_tokens_map)
121
+
122
+ return get_cached_tokenizer(tokenizer)
vllm/tokenizers/mistral.py
+41
-26
@@ -1,6 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
3
+from pathlib import Path
4
from typing import TYPE_CHECKING, Any, cast
5
6
from vllm.logger import init_logger
@@ -12,6 +12,7 @@ if TYPE_CHECKING:
12
ChatCompletionRequest as MistralChatCompletionRequest,
13
)
14
from mistral_common.tokens.tokenizers.tekken import Tekkenizer
15
+ from transformers import BatchEncoding
16
from transformers.tokenization_mistral_common import (
17
MistralCommonTokenizer as TransformersMistralTokenizer,
18
)
@@ -165,7 +166,35 @@ def _tekken_token_to_id(tokenizer: "Tekkenizer", t: str | bytes) -> int:
166
167
168
class MistralTokenizer(TokenizerLike):
169
+ @classmethod
170
+ def from_pretrained(
171
+ cls,
172
+ path_or_repo_id: str | Path,
173
+ *args,
174
+ trust_remote_code: bool = False,
175
+ revision: str | None = None,
176
+ download_dir: str | None = None,
177
+ **kwargs,
178
+ ) -> "MistralTokenizer":
179
+ from mistral_common.protocol.instruct.validator import ValidationMode
180
+ from transformers.tokenization_mistral_common import (
181
+ MistralCommonTokenizer as TransformersMistralTokenizer,
182
+ )
183
+
184
+ tokenizer = TransformersMistralTokenizer.from_pretrained(
185
+ path_or_repo_id,
186
+ *args,
187
+ mode=ValidationMode.test,
188
+ cache_dir=download_dir,
189
+ revision="main" if revision is None else revision,
190
+ **kwargs,
191
+ )
192
+
193
+ return cls(tokenizer)
194
+
195
def __init__(self, tokenizer: "TransformersMistralTokenizer") -> None:
196
+ super().__init__()
197
+
198
from mistral_common.protocol.instruct.validator import ValidationMode
199
from mistral_common.tokens.tokenizers.sentencepiece import (
200
SentencePieceTokenizer,
@@ -211,22 +240,6 @@ class MistralTokenizer(TokenizerLike):
240
self._vocab = self.tokenizer._vocab
241
self._max_token_id = self.vocab_size - 1
242
214
- @classmethod
215
- def from_pretrained(
216
- cls, path_or_repo_id: str, *, revision: str | None = None
217
- ) -> "MistralTokenizer":
218
- from mistral_common.protocol.instruct.validator import ValidationMode
219
- from transformers.tokenization_mistral_common import (
220
- MistralCommonTokenizer as TransformersMistralTokenizer,
221
- )
222
-
223
- str_revision = "main" if revision is None else revision
224
- return cls(
225
- TransformersMistralTokenizer.from_pretrained(
226
- path_or_repo_id, revision=str_revision, mode=ValidationMode.test
227
- )
228
- )
229
-
243
def _get_special_token_ids(self) -> list[int]:
244
from mistral_common.tokens.tokenizers.sentencepiece import (
245
SentencePieceTokenizer,
@@ -271,6 +284,10 @@ class MistralTokenizer(TokenizerLike):
284
def eos_token_id(self) -> int:
285
return self.tokenizer.eos_id
286
287
+ @property
288
+ def pad_token_id(self) -> int:
289
+ return self.tokenizer.pad_id
290
+
291
@property
292
def is_fast(self) -> bool:
293
return True
@@ -298,12 +315,12 @@ class MistralTokenizer(TokenizerLike):
315
316
def __call__(
317
self,
301
- text: str | list[str] | list[int],
318
+ text: str | list[str],
319
text_pair: str | None = None,
303
- add_special_tokens: bool = False,
320
+ add_special_tokens: bool = True,
321
truncation: bool = False,
322
max_length: int | None = None,
306
- ):
323
+ ) -> "BatchEncoding":
324
if text_pair is not None:
325
raise ValueError(
326
"`text_pair` is not supported by `MistralTokenizer.__call__`."
@@ -342,13 +359,11 @@ class MistralTokenizer(TokenizerLike):
359
text: str,
360
truncation: bool | None = None,
361
max_length: int | None = None,
345
- add_special_tokens: bool | None = None,
362
+ add_special_tokens: bool = True,
363
) -> list[int]:
364
# TODO(juliendenize): once https://github.com/huggingface/transformers/pull/41962
365
# is in, directly call self.transformers_tokenizer.encode(...).
349
- encoded = self.tokenizer.encode(
350
- text, bos=add_special_tokens is not False, eos=False
351
- )
366
+ encoded = self.tokenizer.encode(text, bos=add_special_tokens, eos=False)
367
368
if truncation is not False and max_length is not None:
369
return encoded[:max_length]
@@ -383,7 +398,7 @@ class MistralTokenizer(TokenizerLike):
398
return_dict=False,
399
)
400
386
- def decode(self, ids: list[int] | int, skip_special_tokens: bool = True) -> str:
401
+ def decode(self, ids: list[int] | int, skip_special_tokens: bool = False) -> str:
402
# TODO(juliendenize): once https://github.com/huggingface/transformers/pull/41962
403
# is in, directly call self.transformers_tokenizer.decode(...).
404
if isinstance(ids, int):
@@ -455,7 +470,7 @@ class MistralTokenizer(TokenizerLike):
470
def convert_ids_to_tokens(
471
self,
472
ids: list[int],
458
- skip_special_tokens: bool = True,
473
+ skip_special_tokens: bool = False,
474
) -> list[str]:
475
from mistral_common.tokens.tokenizers.base import (
476
SpecialTokenPolicy,
vllm/tokenizers/protocol.py
+19
-13
@@ -1,11 +1,11 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
3
+from pathlib import Path
4
from typing import TYPE_CHECKING, Any, Protocol
5
6
-from typing_extensions import Self
7
-
6
if TYPE_CHECKING:
7
+ from transformers import BatchEncoding
8
+
9
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
10
11
@@ -13,11 +13,13 @@ class TokenizerLike(Protocol):
13
@classmethod
14
def from_pretrained(
15
cls,
16
- pretrained_model_name_or_path: str,
17
- /,
18
- *,
16
+ path_or_repo_id: str | Path,
17
+ *args,
18
+ trust_remote_code: bool = False,
19
revision: str | None = None,
20
- ) -> Self:
20
+ download_dir: str | None = None,
21
+ **kwargs,
22
+ ) -> "TokenizerLike":
23
raise NotImplementedError
24
25
@property
@@ -36,6 +38,10 @@ class TokenizerLike(Protocol):
38
def eos_token_id(self) -> int:
39
raise NotImplementedError
40
41
+ @property
42
+ def pad_token_id(self) -> int:
43
+ raise NotImplementedError
44
+
45
@property
46
def is_fast(self) -> bool:
47
raise NotImplementedError
@@ -60,12 +66,12 @@ class TokenizerLike(Protocol):
66
67
def __call__(
68
self,
63
- text: str | list[str] | list[int],
69
+ text: str | list[str],
70
text_pair: str | None = None,
65
- add_special_tokens: bool = False,
71
+ add_special_tokens: bool = True,
72
truncation: bool = False,
73
max_length: int | None = None,
68
- ):
74
+ ) -> "BatchEncoding":
75
raise NotImplementedError
76
77
def get_vocab(self) -> dict[str, int]:
@@ -79,7 +85,7 @@ class TokenizerLike(Protocol):
85
text: str,
86
truncation: bool | None = None,
87
max_length: int | None = None,
82
- add_special_tokens: bool | None = None,
88
+ add_special_tokens: bool = True,
89
) -> list[int]:
90
raise NotImplementedError
91
@@ -94,12 +100,12 @@ class TokenizerLike(Protocol):
100
def convert_tokens_to_string(self, tokens: list[str]) -> str:
101
raise NotImplementedError
102
97
- def decode(self, ids: list[int] | int, skip_special_tokens: bool = True) -> str:
103
+ def decode(self, ids: list[int] | int, skip_special_tokens: bool = False) -> str:
104
raise NotImplementedError
105
106
def convert_ids_to_tokens(
107
self,
108
ids: list[int],
103
- skip_special_tokens: bool = True,
109
+ skip_special_tokens: bool = False,
110
) -> list[str]:
111
raise NotImplementedError
vllm/transformers_utils/tokenizer.py
+43
-105
@@ -1,8 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-import contextlib
5
-import copy
4
import importlib.util
5
import os
6
import warnings
@@ -11,14 +9,17 @@ from pathlib import Path
9
from typing import TYPE_CHECKING, Any
10
11
import huggingface_hub
14
-from transformers import AutoTokenizer, PreTrainedTokenizerBase
12
from typing_extensions import assert_never
13
14
from vllm import envs
15
from vllm.logger import init_logger
19
-from vllm.tokenizers import MistralTokenizer, TokenizerLike, TokenizerRegistry
16
+from vllm.tokenizers import (
17
+ HfTokenizer,
18
+ MistralTokenizer,
19
+ TokenizerLike,
20
+ TokenizerRegistry,
21
+)
22
21
-from .config import get_sentence_transformer_tokenizer_config
23
from .gguf_utils import get_gguf_file_path_from_hf
24
from .repo_utils import list_filtered_repo_files
25
from .utils import check_gguf_file, is_gguf, is_remote_gguf, split_remote_gguf
@@ -41,6 +42,18 @@ def __getattr__(name: str):
42
)
43
44
return TokenizerLike
45
+ if name == "get_cached_tokenizer":
46
+ from vllm.tokenizers.hf import get_cached_tokenizer
47
+
48
+ warnings.warn(
49
+ "`vllm.transformers_utils.tokenizer.get_cached_tokenizer` "
50
+ "has been moved to `vllm.tokenizers.hf.get_cached_tokenizer`. "
51
+ "The old name will be removed in v0.13.",
52
+ DeprecationWarning,
53
+ stacklevel=2,
54
+ )
55
+
56
+ return get_cached_tokenizer
57
58
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
59
@@ -58,10 +71,12 @@ def decode_tokens(
71
`skip_special_tokens=None` means to use the backend's default
72
settings.
73
"""
74
+ kw_args: dict[str, Any] = {}
75
+
76
if skip_special_tokens is not None:
62
- return tokenizer.decode(token_ids, skip_special_tokens=skip_special_tokens)
77
+ kw_args["skip_special_tokens"] = skip_special_tokens
78
64
- return tokenizer.decode(token_ids)
79
+ return tokenizer.decode(token_ids, **kw_args)
80
81
82
def encode_tokens(
@@ -93,56 +108,6 @@ def encode_tokens(
108
return tokenizer.encode(text, **kw_args)
109
110
96
-def get_cached_tokenizer(tokenizer: TokenizerLike) -> TokenizerLike:
97
- """
98
- By default, transformers will recompute multiple tokenizer properties
99
- each time they are called, leading to a significant slowdown.
100
- This proxy caches these properties for faster access.
101
- """
102
- cached_tokenizer = copy.copy(tokenizer)
103
-
104
- tokenizer_all_special_ids = tokenizer.all_special_ids
105
- tokenizer_all_special_tokens = tokenizer.all_special_tokens
106
- tokenizer_vocab = tokenizer.get_vocab()
107
- tokenizer_len = len(tokenizer)
108
-
109
- max_token_id = max(tokenizer_vocab.values())
110
- # Some tokenizers (e.g., QwenTokenizer) have special tokens that
111
- # are added and included in the implementation of the vocab_size
112
- # property, but not in get_vocab(); if there is an implementation
113
- # of vocab size, we should take the greater value.
114
- if hasattr(tokenizer, "vocab_size"):
115
- with contextlib.suppress(NotImplementedError):
116
- max_token_id = max(max_token_id, tokenizer.vocab_size)
117
-
118
- class CachedTokenizer(tokenizer.__class__): # type: ignore
119
- @property
120
- def all_special_ids(self) -> list[int]:
121
- return tokenizer_all_special_ids
122
-
123
- @property
124
- def all_special_tokens(self) -> list[str]:
125
- return tokenizer_all_special_tokens
126
-
127
- @property
128
- def max_token_id(self) -> int:
129
- return max_token_id
130
-
131
- def get_vocab(self) -> dict[str, int]:
132
- return tokenizer_vocab
133
-
134
- def __len__(self) -> int:
135
- return tokenizer_len
136
-
137
- def __reduce__(self):
138
- return get_cached_tokenizer, (tokenizer,)
139
-
140
- CachedTokenizer.__name__ = f"Cached{tokenizer.__class__.__name__}"
141
-
142
- cached_tokenizer.__class__ = CachedTokenizer
143
- return cached_tokenizer
144
-
145
-
111
def get_tokenizer(
112
tokenizer_name: str | Path,
113
*args,
@@ -217,66 +182,39 @@ def get_tokenizer(
182
if tokenizer_mode == "mistral":
183
logger.debug_once(f"Loading MistralTokenizer from {tokenizer_name}")
184
tokenizer = MistralTokenizer.from_pretrained(
220
- str(tokenizer_name), revision=revision
185
+ tokenizer_name,
186
+ *args,
187
+ trust_remote_code=trust_remote_code,
188
+ revision=revision,
189
+ download_dir=download_dir,
190
+ **kwargs,
191
)
192
elif tokenizer_mode == "custom":
193
logger.debug_once(f"Loading CustomTokenizer from {tokenizer_name}")
194
tokenizer = TokenizerRegistry.get_tokenizer(
195
str(tokenizer_name),
196
*args,
197
+ trust_remote_code=trust_remote_code,
198
revision=revision,
199
download_dir=download_dir,
200
**kwargs,
201
)
202
else:
232
- try:
233
- logger.debug_once(f"Loading AutoTokenizer from {tokenizer_name}")
234
- tokenizer = AutoTokenizer.from_pretrained(
235
- tokenizer_name,
236
- *args,
237
- trust_remote_code=trust_remote_code,
238
- revision=revision,
239
- **kwargs,
240
- )
241
- except ValueError as e:
242
- # If the error pertains to the tokenizer class not existing or not
243
- # currently being imported,
244
- # suggest using the --trust-remote-code flag.
245
- if not trust_remote_code and (
246
- "does not exist or is not currently imported." in str(e)
247
- or "requires you to execute the tokenizer file" in str(e)
248
- ):
249
- err_msg = (
250
- "Failed to load the tokenizer. If the tokenizer "
251
- "is a custom tokenizer not yet available in the "
252
- "HuggingFace transformers library, consider "
253
- "setting `trust_remote_code=True` in LLM or using "
254
- "the `--trust-remote-code` flag in the CLI."
255
- )
256
- raise RuntimeError(err_msg) from e
257
- else:
258
- raise e
259
-
260
- # The special_tokens in tokenizer should also be
261
- # controlled by do_lower_case in encoder_config
262
- encoder_config = get_sentence_transformer_tokenizer_config(
263
- tokenizer_name, revision
203
+ logger.debug_once(f"Loading HfTokenizer from {tokenizer_name}")
204
+ tokenizer = HfTokenizer.from_pretrained(
205
+ tokenizer_name,
206
+ *args,
207
+ trust_remote_code=trust_remote_code,
208
+ revision=revision,
209
+ download_dir=download_dir,
210
+ **kwargs,
211
+ )
212
+
213
+ if not tokenizer.is_fast:
214
+ logger.warning(
215
+ "Using a slow tokenizer. This might cause a significant "
216
+ "slowdown. Consider using a fast tokenizer instead."
217
)
265
- if isinstance(encoder_config, dict) and encoder_config.get(
266
- "do_lower_case", False
267
- ):
268
- assert isinstance(tokenizer, PreTrainedTokenizerBase)
269
- special_tokens_map = {
270
- k: v.lower() for k, v in tokenizer.special_tokens_map.items()
271
- }
272
- tokenizer.add_special_tokens(special_tokens_map)
273
-
274
- if not tokenizer.is_fast:
275
- logger.warning(
276
- "Using a slow tokenizer. This might cause a significant "
277
- "slowdown. Consider using a fast tokenizer instead."
278
- )
279
- tokenizer = get_cached_tokenizer(tokenizer)
218
219
return tokenizer
220
vllm/v1/engine/detokenizer.py
+1
-1
@@ -9,8 +9,8 @@ from tokenizers.decoders import DecodeStream
9
from transformers import PreTrainedTokenizerFast
10
11
from vllm.logger import init_logger
12
+from vllm.tokenizers import TokenizerLike
13
from vllm.tokenizers.detokenizer_utils import (
13
- TokenizerLike,
14
convert_prompt_ids_to_tokens,
15
detokenize_incrementally,
16
)