A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 01:00 UTC
[Bugfix] The special_tokens in tokenizer should also be controlled by do_lower_case in encoder_config. (#20750)
Signed-off-by: wang.yuqi <noooop@126.com>
wang.yuqi committed
Jul 18, 2025 at 17:10 UTC
5895afd78047614a037cac1fc4634825c749fd59
2 files changed
+32
tests/tokenization/test_do_lower_case.py
new
+18
@@ -0,0 +1,18 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import pytest
5
+
6
+from vllm.transformers_utils.tokenizer import get_tokenizer
7
+
8
+TOKENIZER_NAMES = ["BAAI/bge-base-en"]
9
+
10
+
11
+@pytest.mark.parametrize("tokenizer_name", TOKENIZER_NAMES)
12
+@pytest.mark.parametrize("n_tokens", [510])
13
+def test_special_tokens(tokenizer_name: str, n_tokens: int):
14
+ tokenizer = get_tokenizer(tokenizer_name, revision="main")
15
+
16
+ prompts = '[UNK]' * n_tokens
17
+ prompt_token_ids = tokenizer.encode(prompts)
18
+ assert len(prompt_token_ids) == n_tokens + 2
vllm/transformers_utils/tokenizer.py
+14
@@ -16,6 +16,8 @@ from transformers import (AutoTokenizer, PreTrainedTokenizer,
16
17
from vllm import envs
18
from vllm.logger import init_logger
19
+from vllm.transformers_utils.config import (
20
+ get_sentence_transformer_tokenizer_config)
21
from vllm.transformers_utils.tokenizers import MistralTokenizer
22
from vllm.transformers_utils.utils import check_gguf_file
23
from vllm.utils import make_async
@@ -256,6 +258,18 @@ def get_tokenizer(
258
else:
259
raise e
260
261
+ # The special_tokens in tokenizer should also be
262
+ # controlled by do_lower_case in encoder_config
263
+ encoder_config = get_sentence_transformer_tokenizer_config(
264
+ tokenizer_name, revision)
265
+ if isinstance(encoder_config, dict) and encoder_config.get(
266
+ "do_lower_case", False):
267
+ special_tokens_map = {
268
+ k: v.lower()
269
+ for k, v in tokenizer.special_tokens_map.items()
270
+ }
271
+ tokenizer.add_special_tokens(special_tokens_map)
272
+
273
# NOTE: We can remove this after https://github.com/THUDM/ChatGLM3/issues/1324
274
if type(tokenizer).__name__ in ("ChatGLMTokenizer",
275
"ChatGLM4Tokenizer"):