A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 06:00 UTC
[Chore] Move tokenizer initialization methods (#29793)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Dec 2, 2025 at 13:33 UTC
653591d5e73b34ffd9186c61e964474bcc4b7c80
51 files changed
+150
-129
benchmarks/benchmark_prefix_caching.py
+1
-1
@@ -40,7 +40,7 @@ from vllm.engine.arg_utils import EngineArgs
40
from vllm.utils.argparse_utils import FlexibleArgumentParser
41
42
try:
43
- from vllm.transformers_utils.tokenizer import get_tokenizer
43
+ from vllm.tokenizers import get_tokenizer
44
except ImportError:
45
from backend_request_func import get_tokenizer
46
benchmarks/benchmark_serving_structured_output.py
+1
-1
@@ -46,7 +46,7 @@ from tqdm.asyncio import tqdm
46
from transformers import PreTrainedTokenizerBase
47
48
try:
49
- from vllm.transformers_utils.tokenizer import get_tokenizer
49
+ from vllm.tokenizers import get_tokenizer
50
except ImportError:
51
from backend_request_func import get_tokenizer
52
tests/compile/test_dynamic_shapes_compilation.py
+1
-1
@@ -8,7 +8,7 @@ import torch
8
9
from vllm import LLM, SamplingParams
10
from vllm.config.compilation import CompilationMode, DynamicShapesType
11
-from vllm.transformers_utils.tokenizer import get_tokenizer
11
+from vllm.tokenizers import get_tokenizer
12
from vllm.utils.torch_utils import is_torch_equal_or_newer
13
14
tests/entrypoints/openai/test_chat_template.py
+1
-1
@@ -6,7 +6,7 @@ import pytest
6
from vllm.config import ModelConfig
7
from vllm.entrypoints.chat_utils import apply_hf_chat_template, load_chat_template
8
from vllm.entrypoints.openai.protocol import ChatCompletionRequest
9
-from vllm.transformers_utils.tokenizer import get_tokenizer
9
+from vllm.tokenizers import get_tokenizer
10
11
from ...models.registry import HF_EXAMPLE_MODELS
12
from ...utils import VLLM_PATH
tests/entrypoints/openai/test_lora_resolvers.py
+1
-1
@@ -14,7 +14,7 @@ from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
14
from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
15
from vllm.lora.request import LoRARequest
16
from vllm.lora.resolver import LoRAResolver, LoRAResolverRegistry
17
-from vllm.transformers_utils.tokenizer import get_tokenizer
17
+from vllm.tokenizers import get_tokenizer
18
from vllm.v1.engine.async_llm import AsyncLLM
19
20
MODEL_NAME = "openai-community/gpt2"
tests/entrypoints/openai/test_return_token_ids.py
+1
-1
@@ -3,7 +3,7 @@
3
4
import pytest
5
6
-from vllm.transformers_utils.tokenizer import get_tokenizer
6
+from vllm.tokenizers import get_tokenizer
7
8
from ...utils import RemoteOpenAIServer
9
tests/entrypoints/openai/test_return_tokens_as_ids.py
+1
-1
@@ -7,7 +7,7 @@
7
8
import pytest
9
10
-from vllm.transformers_utils.tokenizer import get_tokenizer
10
+from vllm.tokenizers import get_tokenizer
11
12
from ...utils import RemoteOpenAIServer
13
tests/entrypoints/openai/test_serving_chat.py
+1
-1
@@ -14,7 +14,7 @@ from vllm.config.multimodal import MultiModalConfig
14
from vllm.entrypoints.openai.protocol import ChatCompletionRequest
15
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
16
from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
17
-from vllm.transformers_utils.tokenizer import get_tokenizer
17
+from vllm.tokenizers import get_tokenizer
18
from vllm.v1.engine.async_llm import AsyncLLM
19
20
from ...utils import RemoteOpenAIServer
tests/entrypoints/openai/test_token_in_token_out.py
+1
-1
@@ -7,7 +7,7 @@ import tempfile
7
import pytest
8
9
from vllm.model_executor.model_loader.weight_utils import download_weights_from_hf
10
-from vllm.transformers_utils.tokenizer import get_tokenizer
10
+from vllm.tokenizers import get_tokenizer
11
12
from ...utils import RemoteOpenAIServer
13
tests/entrypoints/openai/test_tokenization.py
+1
-1
@@ -5,7 +5,7 @@ import pytest
5
import pytest_asyncio
6
import requests
7
8
-from vllm.transformers_utils.tokenizer import get_tokenizer
8
+from vllm.tokenizers import get_tokenizer
9
10
from ...utils import RemoteOpenAIServer
11
tests/entrypoints/openai/tool_parsers/test_hermes_tool_parser.py
+1
-1
@@ -271,7 +271,7 @@ async def test_streaming_product_tool_call():
271
272
@pytest.fixture
273
def qwen_tokenizer() -> TokenizerLike:
274
- from vllm.transformers_utils.tokenizer import get_tokenizer
274
+ from vllm.tokenizers import get_tokenizer
275
276
return get_tokenizer("Qwen/Qwen3-32B")
277
tests/entrypoints/pooling/embed/test_online.py
+1
-1
@@ -18,7 +18,7 @@ from tests.utils import RemoteOpenAIServer
18
from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse
19
from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse
20
from vllm.platforms import current_platform
21
-from vllm.transformers_utils.tokenizer import get_tokenizer
21
+from vllm.tokenizers import get_tokenizer
22
from vllm.utils.serial_utils import (
23
EMBED_DTYPE_TO_TORCH_DTYPE,
24
ENDIANNESS,
tests/entrypoints/pooling/pooling/test_online.py
+1
-1
@@ -12,7 +12,7 @@ import torch
12
from tests.models.utils import check_embeddings_close
13
from tests.utils import RemoteOpenAIServer
14
from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse
15
-from vllm.transformers_utils.tokenizer import get_tokenizer
15
+from vllm.tokenizers import get_tokenizer
16
from vllm.utils.serial_utils import (
17
EMBED_DTYPE_TO_TORCH_DTYPE,
18
ENDIANNESS,
tests/entrypoints/test_chat_utils.py
+1
-2
@@ -28,8 +28,7 @@ from vllm.multimodal.utils import (
28
encode_image_base64,
29
encode_video_base64,
30
)
31
-from vllm.tokenizers import MistralTokenizer
32
-from vllm.transformers_utils.tokenizer import get_tokenizer
31
+from vllm.tokenizers import MistralTokenizer, get_tokenizer
32
33
from ..models.registry import HF_EXAMPLE_MODELS
34
from ..utils import VLLM_PATH
tests/models/multimodal/processing/test_common.py
+2
-5
@@ -22,11 +22,8 @@ from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict
22
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
23
from vllm.multimodal.inputs import MultiModalInputs
24
from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext
25
-from vllm.tokenizers import MistralTokenizer
26
-from vllm.transformers_utils.tokenizer import (
27
- cached_tokenizer_from_config,
28
- encode_tokens,
29
-)
25
+from vllm.tokenizers import MistralTokenizer, cached_tokenizer_from_config
26
+from vllm.transformers_utils.tokenizer import encode_tokens
27
28
from ....multimodal.utils import random_audio, random_image, random_video
29
from ...registry import (
tests/models/multimodal/processing/test_tensor_schema.py
+1
-1
@@ -31,7 +31,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY, BatchedTensorInputs
31
from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext
32
from vllm.multimodal.utils import group_mm_kwargs_by_modality
33
from vllm.platforms import current_platform
34
-from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
34
+from vllm.tokenizers import cached_tokenizer_from_config
35
from vllm.utils.collection_utils import is_list_of
36
from vllm.utils.torch_utils import set_default_torch_dtype
37
tests/models/utils.py
+1
-1
@@ -13,7 +13,7 @@ from transformers import PretrainedConfig
13
from vllm.config.model import ModelConfig, ModelDType, RunnerOption
14
from vllm.logprobs import Logprob, PromptLogprobs, SampleLogprobs
15
from vllm.multimodal.processing import InputProcessingContext
16
-from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
16
+from vllm.tokenizers import cached_tokenizer_from_config
17
18
from .. import ci_envs
19
from .registry import HF_EXAMPLE_MODELS
tests/test_inputs.py
+2
-2
@@ -7,7 +7,7 @@ from vllm.config import ModelConfig
7
from vllm.inputs import zip_enc_dec_prompts
8
from vllm.inputs.parse import parse_raw_prompts
9
from vllm.inputs.preprocess import InputPreprocessor
10
-from vllm.transformers_utils.tokenizer import init_tokenizer_from_configs
10
+from vllm.tokenizers import init_tokenizer_from_config
11
12
pytestmark = pytest.mark.cpu_test
13
@@ -108,7 +108,7 @@ def test_zip_enc_dec_prompts(mm_processor_kwargs, expected_mm_kwargs):
108
)
109
def test_preprocessor_always_mm_code_path(model_id, prompt):
110
model_config = ModelConfig(model=model_id)
111
- tokenizer = init_tokenizer_from_configs(model_config)
111
+ tokenizer = init_tokenizer_from_config(model_config)
112
input_preprocessor = InputPreprocessor(model_config, tokenizer)
113
114
# HF processor adds sep token
tests/tokenizers_/test_basic.py
+1
-2
@@ -5,8 +5,7 @@ from typing import _get_protocol_attrs # type: ignore
5
import pytest
6
from transformers import PreTrainedTokenizerBase
7
8
-from vllm.tokenizers import TokenizerLike
9
-from vllm.transformers_utils.tokenizer import get_tokenizer
8
+from vllm.tokenizers import TokenizerLike, get_tokenizer
9
10
11
def _get_missing_attrs(obj: object, target: type):
tests/tokenizers_/test_registry.py
+1
-2
@@ -2,8 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from pathlib import Path
4
5
-from vllm.tokenizers import TokenizerLike, TokenizerRegistry
6
-from vllm.transformers_utils.tokenizer import get_tokenizer
5
+from vllm.tokenizers import TokenizerLike, TokenizerRegistry, get_tokenizer
6
7
8
class TestTokenizer(TokenizerLike):
tests/tool_use/test_deepseekv31_tool_parser.py
+1
-1
@@ -6,7 +6,7 @@ import pytest
6
from vllm.entrypoints.openai.tool_parsers.deepseekv31_tool_parser import (
7
DeepSeekV31ToolParser,
8
)
9
-from vllm.transformers_utils.tokenizer import get_tokenizer
9
+from vllm.tokenizers import get_tokenizer
10
11
MODEL = "deepseek-ai/DeepSeek-V3.1"
12
tests/tool_use/test_ernie45_moe_tool_parser.py
+1
-2
@@ -14,9 +14,8 @@ from vllm.entrypoints.openai.protocol import (
14
ToolCall,
15
)
16
from vllm.entrypoints.openai.tool_parsers.ernie45_tool_parser import Ernie45ToolParser
17
-from vllm.tokenizers import TokenizerLike
17
+from vllm.tokenizers import TokenizerLike, get_tokenizer
18
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
19
-from vllm.transformers_utils.tokenizer import get_tokenizer
19
20
# Use a common model that is likely to be available
21
MODEL = "baidu/ERNIE-4.5-21B-A3B-Thinking"
tests/tool_use/test_glm4_moe_tool_parser.py
+1
-1
@@ -10,7 +10,7 @@ from vllm.entrypoints.openai.protocol import FunctionCall, ToolCall
10
from vllm.entrypoints.openai.tool_parsers.glm4_moe_tool_parser import (
11
Glm4MoeModelToolParser,
12
)
13
-from vllm.transformers_utils.tokenizer import get_tokenizer
13
+from vllm.tokenizers import get_tokenizer
14
15
pytestmark = pytest.mark.cpu_test
16
tests/tool_use/test_jamba_tool_parser.py
+1
-2
@@ -10,9 +10,8 @@ from partial_json_parser.core.options import Allow
10
11
from vllm.entrypoints.openai.protocol import DeltaMessage, FunctionCall, ToolCall
12
from vllm.entrypoints.openai.tool_parsers.jamba_tool_parser import JambaToolParser
13
-from vllm.tokenizers import TokenizerLike
13
+from vllm.tokenizers import TokenizerLike, get_tokenizer
14
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
15
-from vllm.transformers_utils.tokenizer import get_tokenizer
15
16
pytestmark = pytest.mark.cpu_test
17
tests/tool_use/test_kimi_k2_tool_parser.py
+1
-1
@@ -8,7 +8,7 @@ import pytest
8
9
from vllm.entrypoints.openai.protocol import FunctionCall, ToolCall
10
from vllm.entrypoints.openai.tool_parsers.kimi_k2_tool_parser import KimiK2ToolParser
11
-from vllm.transformers_utils.tokenizer import get_tokenizer
11
+from vllm.tokenizers import get_tokenizer
12
13
pytestmark = pytest.mark.cpu_test
14
tests/tool_use/test_minimax_tool_parser.py
+1
-1
@@ -13,7 +13,7 @@ from vllm.entrypoints.openai.protocol import (
13
ToolCall,
14
)
15
from vllm.entrypoints.openai.tool_parsers.minimax_tool_parser import MinimaxToolParser
16
-from vllm.transformers_utils.tokenizer import get_tokenizer
16
+from vllm.tokenizers import get_tokenizer
17
18
pytestmark = pytest.mark.cpu_test
19
tests/tool_use/test_openai_tool_parser.py
+1
-1
@@ -16,7 +16,7 @@ from openai_harmony import (
16
17
from vllm.entrypoints.openai.protocol import FunctionCall, ToolCall
18
from vllm.entrypoints.openai.tool_parsers.openai_tool_parser import OpenAIToolParser
19
-from vllm.transformers_utils.tokenizer import get_tokenizer
19
+from vllm.tokenizers import get_tokenizer
20
21
MODEL = "gpt2"
22
tests/tool_use/test_qwen3coder_tool_parser.py
+1
-2
@@ -17,9 +17,8 @@ from vllm.entrypoints.openai.tool_parsers.qwen3coder_tool_parser import (
17
Qwen3CoderToolParser,
18
)
19
from vllm.entrypoints.openai.tool_parsers.qwen3xml_tool_parser import Qwen3XMLToolParser
20
-from vllm.tokenizers import TokenizerLike
20
+from vllm.tokenizers import TokenizerLike, get_tokenizer
21
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
22
-from vllm.transformers_utils.tokenizer import get_tokenizer
22
23
pytestmark = pytest.mark.cpu_test
24
tests/tool_use/test_seed_oss_tool_parser.py
+1
-2
@@ -15,9 +15,8 @@ from vllm.entrypoints.openai.protocol import (
15
ToolCall,
16
)
17
from vllm.entrypoints.openai.tool_parsers.seed_oss_tool_parser import SeedOssToolParser
18
-from vllm.tokenizers import TokenizerLike
18
+from vllm.tokenizers import TokenizerLike, get_tokenizer
19
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
20
-from vllm.transformers_utils.tokenizer import get_tokenizer
20
21
pytestmark = pytest.mark.cpu_test
22
tests/tool_use/test_xlam_tool_parser.py
+1
-2
@@ -13,9 +13,8 @@ from vllm.entrypoints.openai.protocol import (
13
ToolCall,
14
)
15
from vllm.entrypoints.openai.tool_parsers.xlam_tool_parser import xLAMToolParser
16
-from vllm.tokenizers import TokenizerLike
16
+from vllm.tokenizers import TokenizerLike, get_tokenizer
17
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
18
-from vllm.transformers_utils.tokenizer import get_tokenizer
18
19
pytestmark = pytest.mark.cpu_test
20
tests/transformers_utils/test_config.py
+1
-1
@@ -6,8 +6,8 @@ only get the `eos_token_id` from the tokenizer as defined by
6
`vllm.LLMEngine._get_eos_token_id`.
7
"""
8
9
+from vllm.tokenizers import get_tokenizer
10
from vllm.transformers_utils.config import try_get_generation_config
10
-from vllm.transformers_utils.tokenizer import get_tokenizer
11
12
13
def test_get_llama3_eos_token():
tests/utils.py
+1
-1
@@ -44,7 +44,7 @@ from vllm.engine.arg_utils import AsyncEngineArgs
44
from vllm.entrypoints.cli.serve import ServeSubcommand
45
from vllm.model_executor.model_loader import get_model_loader
46
from vllm.platforms import current_platform
47
-from vllm.transformers_utils.tokenizer import get_tokenizer
47
+from vllm.tokenizers import get_tokenizer
48
from vllm.utils.argparse_utils import FlexibleArgumentParser
49
from vllm.utils.mem_constants import GB_bytes
50
from vllm.utils.network_utils import get_open_port
tests/v1/entrypoints/openai/test_completion.py
+1
-1
@@ -9,7 +9,7 @@ import regex as re
9
from openai import BadRequestError
10
11
from tests.utils import RemoteOpenAIServer
12
-from vllm.transformers_utils.tokenizer import get_tokenizer
12
+from vllm.tokenizers import get_tokenizer
13
14
# any model with a chat template should work here
15
MODEL_NAME = "facebook/opt-125m"
tests/v1/tpu/test_perf.py
+1
-1
@@ -14,7 +14,7 @@ import pytest
14
15
from vllm.platforms import current_platform
16
from vllm.sampling_params import SamplingParams
17
-from vllm.transformers_utils.tokenizer import get_tokenizer
17
+from vllm.tokenizers import get_tokenizer
18
19
if TYPE_CHECKING:
20
from tests.conftest import VllmRunner
vllm/benchmarks/serve.py
+1
-1
@@ -47,7 +47,7 @@ from vllm.benchmarks.lib.endpoint_request_func import (
47
)
48
from vllm.benchmarks.lib.ready_checker import wait_for_endpoint
49
from vllm.benchmarks.lib.utils import convert_to_pytorch_benchmark_format, write_to_json
50
-from vllm.transformers_utils.tokenizer import get_tokenizer
50
+from vllm.tokenizers import get_tokenizer
51
from vllm.utils.gc_utils import freeze_gc_heap
52
from vllm.utils.network_utils import join_host_port
53
vllm/model_executor/models/adapters.py
+2
-2
@@ -444,7 +444,7 @@ def load_weights_using_from_2_way_softmax(
444
)
445
loaded_weights = pooling_model_cls.load_weights(model, weights, load_lm_head=True)
446
447
- from vllm.transformers_utils.tokenizer import get_tokenizer
447
+ from vllm.tokenizers import get_tokenizer
448
449
tokenizer = get_tokenizer(
450
model_config.tokenizer,
@@ -498,7 +498,7 @@ def load_weights_no_post_processing(model, weights: Iterable[tuple[str, torch.Te
498
# Skip ModelForSequenceClassification in MRO to avoid infinite recursion
499
loaded_weights = type(model).__mro__[1].load_weights(model, weights)
500
501
- from vllm.transformers_utils.tokenizer import get_tokenizer
501
+ from vllm.tokenizers import get_tokenizer
502
503
tokenizer = get_tokenizer(
504
model_config.tokenizer,
vllm/model_executor/models/deepseek_ocr.py
+1
-1
@@ -45,6 +45,7 @@ from vllm.multimodal.processing import (
45
from vllm.multimodal.profiling import BaseDummyInputsBuilder
46
from vllm.sampling_params import SamplingParams
47
from vllm.sequence import IntermediateTensors
48
+from vllm.tokenizers import cached_tokenizer_from_config
49
from vllm.transformers_utils.configs.deepseek_vl2 import DeepseekVLV2Config
50
from vllm.transformers_utils.processors.deepseek_ocr import (
51
BASE_SIZE,
@@ -53,7 +54,6 @@ from vllm.transformers_utils.processors.deepseek_ocr import (
54
DeepseekOCRProcessor,
55
count_tiles,
56
)
56
-from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
57
from vllm.utils.tensor_schema import TensorSchema, TensorShape
58
from vllm.v1.sample.logits_processor import (
59
AdapterLogitsProcessor,
vllm/model_executor/models/deepseek_vl2.py
+1
-1
@@ -41,13 +41,13 @@ from vllm.multimodal.processing import (
41
)
42
from vllm.multimodal.profiling import BaseDummyInputsBuilder
43
from vllm.sequence import IntermediateTensors
44
+from vllm.tokenizers import cached_tokenizer_from_config
45
from vllm.transformers_utils.configs.deepseek_vl2 import (
46
DeepseekVLV2Config,
47
MlpProjectorConfig,
48
VisionEncoderConfig,
49
)
50
from vllm.transformers_utils.processors.deepseek_vl2 import DeepseekVLV2Processor
50
-from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
51
from vllm.utils.tensor_schema import TensorSchema, TensorShape
52
from vllm.utils.torch_utils import set_default_torch_dtype
53
vllm/model_executor/models/granite_speech.py
+4
-4
@@ -59,8 +59,8 @@ from vllm.multimodal.processing import (
59
)
60
from vllm.multimodal.profiling import BaseDummyInputsBuilder
61
from vllm.sequence import IntermediateTensors
62
-from vllm.transformers_utils.processor import cached_get_processor
63
-from vllm.transformers_utils.tokenizer import cached_get_tokenizer
62
+from vllm.tokenizers import cached_tokenizer_from_config
63
+from vllm.transformers_utils.processor import cached_processor_from_config
64
from vllm.utils.tensor_schema import TensorSchema, TensorShape
65
66
from .blip2 import Blip2QFormerModel
@@ -862,7 +862,7 @@ class GraniteSpeechForConditionalGeneration(
862
else:
863
raise ValueError(f"Unsupported task type {task_type}")
864
865
- tokenizer = cached_get_tokenizer(model_config.model)
865
+ tokenizer = cached_tokenizer_from_config(model_config)
866
chat = [dict(role="user", content=user_prompt)]
867
prompt = tokenizer.apply_chat_template(
868
chat,
@@ -886,7 +886,7 @@ class GraniteSpeechForConditionalGeneration(
886
model_config: ModelConfig,
887
) -> int | None:
888
"""Get the number of audio tokens for an audio duration in sec."""
889
- processor = cached_get_processor(model_config.model)
889
+ processor = cached_processor_from_config(model_config)
890
hop_length = processor.audio_processor.melspec_kwargs["hop_length"]
891
proj_win_size = processor.audio_processor.projector_window_size
892
ds_rate = processor.audio_processor.projector_downsample_rate
vllm/model_executor/models/gritlm.py
+1
-1
@@ -19,7 +19,7 @@ from vllm.model_executor.layers.pooler import (
19
)
20
from vllm.model_executor.models.llama import LlamaForCausalLM
21
from vllm.tasks import PoolingTask
22
-from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
22
+from vllm.tokenizers import cached_tokenizer_from_config
23
from vllm.v1.outputs import PoolerOutput
24
from vllm.v1.pool.metadata import PoolingMetadata
25
vllm/model_executor/models/nano_nemotron_vl.py
+2
-5
@@ -73,12 +73,9 @@ from vllm.multimodal.processing import (
73
)
74
from vllm.multimodal.profiling import BaseDummyInputsBuilder
75
from vllm.sequence import IntermediateTensors
76
-from vllm.tokenizers import TokenizerLike
76
+from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
77
from vllm.transformers_utils.configs.radio import RadioConfig
78
-from vllm.transformers_utils.tokenizer import (
79
- cached_tokenizer_from_config,
80
- encode_tokens,
81
-)
78
+from vllm.transformers_utils.tokenizer import encode_tokens
79
from vllm.utils.tensor_schema import TensorSchema, TensorShape
80
81
from .utils import _merge_multimodal_embeddings
vllm/model_executor/models/pixtral.py
+1
-2
@@ -59,8 +59,7 @@ from vllm.multimodal.processing import (
59
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
60
from vllm.platforms import current_platform
61
from vllm.sequence import IntermediateTensors
62
-from vllm.tokenizers import MistralTokenizer
63
-from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
62
+from vllm.tokenizers import MistralTokenizer, cached_tokenizer_from_config
63
from vllm.utils.tensor_schema import TensorSchema, TensorShape
64
65
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
vllm/model_executor/models/voxtral.py
+1
-2
@@ -51,8 +51,7 @@ from vllm.multimodal.processing import (
51
)
52
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
53
from vllm.sequence import IntermediateTensors
54
-from vllm.tokenizers import MistralTokenizer
55
-from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
54
+from vllm.tokenizers import MistralTokenizer, cached_tokenizer_from_config
55
56
from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsTranscription
57
from .utils import init_vllm_registered_model, maybe_prefix
vllm/model_executor/models/whisper.py
+3
-3
@@ -48,7 +48,7 @@ from vllm.multimodal.processing import (
48
PromptUpdate,
49
)
50
from vllm.multimodal.profiling import BaseDummyInputsBuilder
51
-from vllm.transformers_utils.processor import cached_get_processor
51
+from vllm.transformers_utils.processor import cached_processor_from_config
52
from vllm.utils.jsontree import json_map_leaves
53
from vllm.utils.tensor_schema import TensorSchema, TensorShape
54
from vllm.utils.torch_utils import set_default_torch_dtype
@@ -850,7 +850,7 @@ class WhisperForConditionalGeneration(
850
def get_speech_to_text_config(
851
cls, model_config: ModelConfig, task_type: str
852
) -> SpeechToTextConfig:
853
- processor = cached_get_processor(model_config.model)
853
+ processor = cached_processor_from_config(model_config)
854
855
return SpeechToTextConfig(
856
max_audio_clip_s=processor.feature_extractor.chunk_length,
@@ -864,7 +864,7 @@ class WhisperForConditionalGeneration(
864
stt_config: SpeechToTextConfig,
865
model_config: ModelConfig,
866
) -> int | None:
867
- processor = cached_get_processor(model_config.model)
867
+ processor = cached_processor_from_config(model_config)
868
hop_length = processor.feature_extractor.hop_length
869
assert hop_length is not None
870
# NOTE(NickLucche) user can't pass encoder
vllm/multimodal/registry.py
+1
-2
@@ -6,8 +6,7 @@ from typing import TYPE_CHECKING, Generic, Protocol, TypeVar, cast
6
7
from vllm.config.multimodal import BaseDummyOptions
8
from vllm.logger import init_logger
9
-from vllm.tokenizers import TokenizerLike
10
-from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
9
+from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
10
11
from .cache import BaseMultiModalProcessorCache
12
from .processing import (
vllm/tokenizers/__init__.py
+10
-1
@@ -4,12 +4,21 @@
4
from .hf import HfTokenizer
5
from .mistral import MistralTokenizer
6
from .protocol import TokenizerLike
7
-from .registry import TokenizerRegistry, get_tokenizer
7
+from .registry import (
8
+ TokenizerRegistry,
9
+ cached_get_tokenizer,
10
+ cached_tokenizer_from_config,
11
+ get_tokenizer,
12
+ init_tokenizer_from_config,
13
+)
14
15
__all__ = [
16
"TokenizerLike",
17
"HfTokenizer",
18
"MistralTokenizer",
19
"TokenizerRegistry",
20
+ "cached_get_tokenizer",
21
"get_tokenizer",
22
+ "cached_tokenizer_from_config",
23
+ "init_tokenizer_from_config",
24
]
vllm/tokenizers/registry.py
+37
-1
@@ -2,10 +2,12 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import importlib.util
4
from collections.abc import Callable
5
+from functools import lru_cache
6
from pathlib import Path
6
-from typing import TypeVar, overload
7
+from typing import TYPE_CHECKING, TypeVar, overload
8
9
import huggingface_hub
10
+from typing_extensions import assert_never
11
12
import vllm.envs as envs
13
from vllm.logger import init_logger
@@ -21,6 +23,9 @@ from vllm.utils.import_utils import resolve_obj_by_qualname
23
24
from .protocol import TokenizerLike
25
26
+if TYPE_CHECKING:
27
+ from vllm.config import ModelConfig
28
+
29
logger = init_logger(__name__)
30
31
_T = TypeVar("_T", bound=type[TokenizerLike])
@@ -195,3 +200,34 @@ def get_tokenizer(
200
)
201
202
return tokenizer
203
+
204
+
205
+cached_get_tokenizer = lru_cache(get_tokenizer)
206
+
207
+
208
+def cached_tokenizer_from_config(model_config: "ModelConfig", **kwargs):
209
+ return cached_get_tokenizer(
210
+ model_config.tokenizer,
211
+ tokenizer_mode=model_config.tokenizer_mode,
212
+ revision=model_config.tokenizer_revision,
213
+ trust_remote_code=model_config.trust_remote_code,
214
+ **kwargs,
215
+ )
216
+
217
+
218
+def init_tokenizer_from_config(model_config: "ModelConfig"):
219
+ runner_type = model_config.runner_type
220
+ if runner_type == "generate" or runner_type == "draft":
221
+ truncation_side = "left"
222
+ elif runner_type == "pooling":
223
+ truncation_side = "right"
224
+ else:
225
+ assert_never(runner_type)
226
+
227
+ return get_tokenizer(
228
+ model_config.tokenizer,
229
+ tokenizer_mode=model_config.tokenizer_mode,
230
+ trust_remote_code=model_config.trust_remote_code,
231
+ revision=model_config.tokenizer_revision,
232
+ truncation_side=truncation_side,
233
+ )
vllm/transformers_utils/tokenizer.py
+43
-48
@@ -2,17 +2,10 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
import warnings
5
-from functools import lru_cache
6
-from typing import TYPE_CHECKING, Any
7
-
8
-from typing_extensions import assert_never
5
+from typing import Any
6
7
from vllm.logger import init_logger
11
-from vllm.tokenizers import TokenizerLike, get_tokenizer
12
-
13
-if TYPE_CHECKING:
14
- from vllm.config import ModelConfig
15
-
8
+from vllm.tokenizers import TokenizerLike
9
10
logger = init_logger(__name__)
11
@@ -28,18 +21,54 @@ def __getattr__(name: str):
21
)
22
23
return TokenizerLike
31
- if name == "get_cached_tokenizer":
32
- from vllm.tokenizers.hf import get_cached_tokenizer
24
+ if name == "get_tokenizer":
25
+ from vllm.tokenizers import get_tokenizer
26
+
27
+ warnings.warn(
28
+ "`vllm.transformers_utils.tokenizer.get_tokenizer` "
29
+ "has been moved to `vllm.tokenizers.get_tokenizer`. "
30
+ "The old name will be removed in v0.13.",
31
+ DeprecationWarning,
32
+ stacklevel=2,
33
+ )
34
+
35
+ return get_tokenizer
36
+ if name == "cached_get_tokenizer":
37
+ from vllm.tokenizers import cached_get_tokenizer
38
+
39
+ warnings.warn(
40
+ "`vllm.transformers_utils.tokenizer.cached_get_tokenizer` "
41
+ "has been moved to `vllm.tokenizers.cached_get_tokenizer`. "
42
+ "The old name will be removed in v0.13.",
43
+ DeprecationWarning,
44
+ stacklevel=2,
45
+ )
46
+
47
+ return cached_get_tokenizer
48
+ if name == "cached_tokenizer_from_config":
49
+ from vllm.tokenizers import cached_tokenizer_from_config
50
51
warnings.warn(
35
- "`vllm.transformers_utils.tokenizer.get_cached_tokenizer` "
36
- "has been moved to `vllm.tokenizers.hf.get_cached_tokenizer`. "
52
+ "`vllm.transformers_utils.tokenizer.cached_tokenizer_from_config` "
53
+ "has been moved to `vllm.tokenizers.cached_tokenizer_from_config`. "
54
"The old name will be removed in v0.13.",
55
DeprecationWarning,
56
stacklevel=2,
57
)
58
42
- return get_cached_tokenizer
59
+ return cached_tokenizer_from_config
60
+ if name == "init_tokenizer_from_configs":
61
+ from vllm.tokenizers import init_tokenizer_from_config
62
+
63
+ warnings.warn(
64
+ "`vllm.transformers_utils.tokenizer.init_tokenizer_from_configs` "
65
+ "has been moved to `vllm.tokenizers.init_tokenizer_from_config`. "
66
+ "The old name will be removed in v0.13.",
67
+ DeprecationWarning,
68
+ stacklevel=2,
69
+ )
70
+
71
+ return init_tokenizer_from_config
72
73
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
74
@@ -92,37 +121,3 @@ def encode_tokens(
121
kw_args["add_special_tokens"] = add_special_tokens
122
123
return tokenizer.encode(text, **kw_args)
95
-
96
-
97
-cached_get_tokenizer = lru_cache(get_tokenizer)
98
-
99
-
100
-def cached_tokenizer_from_config(
101
- model_config: "ModelConfig",
102
- **kwargs: Any,
103
-):
104
- return cached_get_tokenizer(
105
- model_config.tokenizer,
106
- tokenizer_mode=model_config.tokenizer_mode,
107
- revision=model_config.tokenizer_revision,
108
- trust_remote_code=model_config.trust_remote_code,
109
- **kwargs,
110
- )
111
-
112
-
113
-def init_tokenizer_from_configs(model_config: "ModelConfig"):
114
- runner_type = model_config.runner_type
115
- if runner_type == "generate" or runner_type == "draft":
116
- truncation_side = "left"
117
- elif runner_type == "pooling":
118
- truncation_side = "right"
119
- else:
120
- assert_never(runner_type)
121
-
122
- return get_tokenizer(
123
- model_config.tokenizer,
124
- tokenizer_mode=model_config.tokenizer_mode,
125
- trust_remote_code=model_config.trust_remote_code,
126
- revision=model_config.tokenizer_revision,
127
- truncation_side=truncation_side,
128
- )
vllm/v1/engine/async_llm.py
+2
-3
@@ -26,10 +26,9 @@ from vllm.plugins.io_processors import get_io_processor
26
from vllm.pooling_params import PoolingParams
27
from vllm.sampling_params import SamplingParams
28
from vllm.tasks import SupportedTask
29
-from vllm.tokenizers import TokenizerLike
29
+from vllm.tokenizers import TokenizerLike, init_tokenizer_from_config
30
from vllm.tracing import init_tracer
31
from vllm.transformers_utils.config import maybe_register_config_serialize_by_value
32
-from vllm.transformers_utils.tokenizer import init_tokenizer_from_configs
32
from vllm.usage.usage_lib import UsageContext
33
from vllm.utils.async_utils import cancel_task_threadsafe
34
from vllm.utils.collection_utils import as_list
@@ -112,7 +111,7 @@ class AsyncLLM(EngineClient):
111
if self.model_config.skip_tokenizer_init:
112
tokenizer = None
113
else:
115
- tokenizer = init_tokenizer_from_configs(self.model_config)
114
+ tokenizer = init_tokenizer_from_config(self.model_config)
115
116
self.input_processor = InputProcessor(self.vllm_config, tokenizer)
117
self.io_processor = get_io_processor(
vllm/v1/engine/llm_engine.py
+2
-3
@@ -23,9 +23,8 @@ from vllm.plugins.io_processors import get_io_processor
23
from vllm.pooling_params import PoolingParams
24
from vllm.sampling_params import SamplingParams
25
from vllm.tasks import SupportedTask
26
-from vllm.tokenizers import TokenizerLike
26
+from vllm.tokenizers import TokenizerLike, init_tokenizer_from_config
27
from vllm.tracing import init_tracer
28
-from vllm.transformers_utils.tokenizer import init_tokenizer_from_configs
28
from vllm.usage.usage_lib import UsageContext
29
from vllm.v1.engine import EngineCoreRequest
30
from vllm.v1.engine.core_client import EngineCoreClient
@@ -87,7 +86,7 @@ class LLMEngine:
86
if self.model_config.skip_tokenizer_init:
87
tokenizer = None
88
else:
90
- tokenizer = init_tokenizer_from_configs(self.model_config)
89
+ tokenizer = init_tokenizer_from_config(self.model_config)
90
91
self.input_processor = InputProcessor(self.vllm_config, tokenizer)
92
self.io_processor = get_io_processor(
vllm/v1/structured_output/__init__.py
+2
-2
@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING
7
from vllm.config import VllmConfig
8
from vllm.logger import init_logger
9
from vllm.reasoning import ReasoningParserManager
10
-from vllm.transformers_utils.tokenizer import init_tokenizer_from_configs
10
+from vllm.tokenizers import init_tokenizer_from_config
11
from vllm.utils.import_utils import LazyLoader
12
from vllm.v1.structured_output.backend_guidance import GuidanceBackend
13
from vllm.v1.structured_output.backend_types import (
@@ -61,7 +61,7 @@ class StructuredOutputManager:
61
# of CPUs.
62
max_workers = max(1, (multiprocessing.cpu_count() + 1) // 2)
63
self.executor = ThreadPoolExecutor(max_workers=max_workers)
64
- self.tokenizer = init_tokenizer_from_configs(
64
+ self.tokenizer = init_tokenizer_from_config(
65
model_config=self.vllm_config.model_config
66
)
67
reasoning_parser = (