A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Frontend] Consolidate tokenizer init code (#26276)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Oct 6, 2025 at 17:34 UTC
391612e78b8fb6dc1322fc0db5ab3a5b3732463e
8 files changed
+46
-70
tests/test_inputs.py
+3
-5
@@ -7,7 +7,6 @@ from vllm.config import ModelConfig
7
from vllm.inputs import zip_enc_dec_prompts
8
from vllm.inputs.parse import parse_raw_prompts
9
from vllm.inputs.preprocess import InputPreprocessor
10
-from vllm.transformers_utils.tokenizer import init_tokenizer_from_configs
10
11
pytestmark = pytest.mark.cpu_test
12
@@ -107,8 +106,7 @@ def test_zip_enc_dec_prompts(mm_processor_kwargs, expected_mm_kwargs):
106
)
107
def test_preprocessor_text_no_mm_inputs(model_id, prompt):
108
model_config = ModelConfig(model=model_id)
110
- tokenizer = init_tokenizer_from_configs(model_config)
111
- input_preprocessor = InputPreprocessor(model_config, tokenizer)
109
+ input_preprocessor = InputPreprocessor(model_config)
110
111
with pytest.raises(ValueError, match="does not support multimodal inputs"):
112
input_preprocessor.preprocess(prompt)
@@ -129,8 +127,8 @@ def test_preprocessor_text_no_mm_inputs(model_id, prompt):
127
)
128
def test_preprocessor_always_mm_code_path(model_id, prompt):
129
model_config = ModelConfig(model=model_id)
132
- tokenizer = init_tokenizer_from_configs(model_config)
133
- input_preprocessor = InputPreprocessor(model_config, tokenizer)
130
+ input_preprocessor = InputPreprocessor(model_config)
131
+ tokenizer = input_preprocessor.tokenizer
132
133
# HF processor adds sep token
134
sep_token_id = tokenizer.vocab[tokenizer.sep_token]
tests/v1/engine/test_processor_multi_modal_uuids.py
+1
-3
@@ -65,9 +65,7 @@ def _mk_processor(
65
device_config=DeviceConfig(device="cpu"),
66
)
67
68
- # Pass tokenizer=None; InputPreprocessor handles None when
69
- # skip_tokenizer_init is True.
70
- return Processor(vllm_config, tokenizer=None) # type: ignore[arg-type]
68
+ return Processor(vllm_config)
69
70
71
def test_multi_modal_uuids_length_mismatch_raises(monkeypatch):
vllm/entrypoints/llm.py
+2
-6
@@ -74,7 +74,6 @@ from vllm.transformers_utils.tokenizer import (
74
AnyTokenizer,
75
MistralTokenizer,
76
get_cached_tokenizer,
77
- init_tokenizer_from_configs,
77
)
78
from vllm.usage.usage_lib import UsageContext
79
from vllm.utils import Counter, Device, as_iter, is_list_of
@@ -367,11 +366,8 @@ class LLM:
366
def _get_processor(self) -> Processor:
367
if not hasattr(self, "_processor"):
368
vllm_config = self.llm_engine.vllm_config
370
- if self.model_config.skip_tokenizer_init:
371
- tokenizer = None
372
- else:
373
- tokenizer = init_tokenizer_from_configs(self.model_config)
374
- self._processor = Processor(vllm_config, tokenizer)
369
+ self._processor = Processor(vllm_config)
370
+
371
return self._processor
372
373
def get_default_sampling_params(self) -> SamplingParams:
vllm/entrypoints/openai/serving_engine.py
+2
-6
@@ -16,7 +16,6 @@ from starlette.datastructures import Headers
16
from typing_extensions import TypeIs
17
18
from vllm.entrypoints.utils import _validate_truncation_size
19
-from vllm.transformers_utils.tokenizer import init_tokenizer_from_configs
19
from vllm.v1.engine import EngineCoreRequest
20
from vllm.v1.engine.processor import Processor
21
@@ -272,11 +271,8 @@ class OpenAIServing:
271
async def _get_processor(self) -> Processor:
272
if not hasattr(self, "_processor"):
273
vllm_config = await self.engine_client.get_vllm_config()
275
- if self.model_config.skip_tokenizer_init:
276
- tokenizer = None
277
- else:
278
- tokenizer = init_tokenizer_from_configs(self.model_config)
279
- self._processor = Processor(vllm_config, tokenizer)
274
+ self._processor = Processor(vllm_config)
275
+
276
return self._processor
277
278
def _get_renderer(self, tokenizer: Optional[AnyTokenizer]) -> BaseRenderer:
vllm/inputs/preprocess.py
+11
-16
@@ -17,7 +17,8 @@ from vllm.multimodal.inputs import (
17
MultiModalUUIDDict,
18
)
19
from vllm.multimodal.processing import BaseMultiModalProcessor
20
-from vllm.transformers_utils.tokenizer import AnyTokenizer
20
+from vllm.transformers_utils.tokenizer import AnyTokenizer, init_tokenizer_from_configs
21
+from vllm.utils.jsontree import json_iter_leaves
22
23
from .data import (
24
DecoderOnlyInputs,
@@ -44,17 +45,20 @@ class InputPreprocessor:
45
def __init__(
46
self,
47
model_config: ModelConfig,
47
- tokenizer: Optional[AnyTokenizer],
48
mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
49
mm_processor_cache: Optional[BaseMultiModalProcessorCache] = None,
50
) -> None:
51
super().__init__()
52
53
self.model_config = model_config
54
- self.tokenizer = tokenizer
54
self.mm_registry = mm_registry
55
self.mm_processor_cache = mm_processor_cache
56
57
+ if model_config.skip_tokenizer_init:
58
+ self.tokenizer = None
59
+ else:
60
+ self.tokenizer = init_tokenizer_from_configs(model_config)
61
+
62
def get_tokenizer(self) -> AnyTokenizer:
63
if self.tokenizer is None:
64
raise ValueError(
@@ -273,7 +277,10 @@ class InputPreprocessor:
277
mm_hashes = mm_input["mm_hashes"]
278
279
# Validate that all mm items have a string as their hash
276
- if not contains_only_strings(mm_hashes):
280
+ contains_only_strings = all(
281
+ isinstance(leaf, str) for leaf in json_iter_leaves(mm_hashes)
282
+ )
283
+ if not contains_only_strings:
284
raise ValueError(
285
f"mm_hashes must contain only strings, got: {mm_hashes}. "
286
"This is likely due to an incorrect custom implementation of "
@@ -693,15 +700,3 @@ class InputPreprocessor:
700
def clear_cache(self) -> None:
701
if self.mm_processor_cache is not None:
702
self.mm_processor_cache.clear_cache()
696
-
697
-
698
-# Helper function to validate that a nested dictionary contains
699
-# only strings or list of strings as the leaf values.
700
-def contains_only_strings(obj: object):
701
- if isinstance(obj, str):
702
- return True
703
- if isinstance(obj, list):
704
- return all(isinstance(x, str) for x in obj)
705
- if isinstance(obj, dict):
706
- return all(contains_only_strings(v) for v in obj.values())
707
- return False
vllm/v1/engine/async_llm.py
+6
-14
@@ -28,7 +28,7 @@ from vllm.sampling_params import SamplingParams
28
from vllm.tasks import SupportedTask
29
from vllm.tracing import init_tracer
30
from vllm.transformers_utils.config import maybe_register_config_serialize_by_value
31
-from vllm.transformers_utils.tokenizer import AnyTokenizer, init_tokenizer_from_configs
31
+from vllm.transformers_utils.tokenizer import AnyTokenizer
32
from vllm.usage.usage_lib import UsageContext
33
from vllm.utils import Device, as_list, cancel_task_threadsafe, cdiv, deprecate_kwargs
34
from vllm.v1.engine import EngineCoreRequest
@@ -104,20 +104,8 @@ class AsyncLLM(EngineClient):
104
"logger list; enabling logging without default stat loggers"
105
)
106
107
- if self.model_config.skip_tokenizer_init:
108
- self.tokenizer = None
109
- else:
110
- # Tokenizer (+ ensure liveness if running in another process).
111
- self.tokenizer = init_tokenizer_from_configs(
112
- model_config=vllm_config.model_config
113
- )
114
-
107
# Processor (converts Inputs --> EngineCoreRequests).
116
- self.processor = Processor(
117
- vllm_config=vllm_config,
118
- tokenizer=self.tokenizer,
119
- mm_registry=mm_registry,
120
- )
108
+ self.processor = Processor(vllm_config, mm_registry=mm_registry)
109
110
# OutputProcessor (converts EngineCoreOutputs --> RequestOutput).
111
self.output_processor = OutputProcessor(
@@ -257,6 +245,10 @@ class AsyncLLM(EngineClient):
245
246
cancel_task_threadsafe(getattr(self, "output_handler", None))
247
248
+ @property
249
+ def tokenizer(self) -> Optional[AnyTokenizer]:
250
+ return self.processor.tokenizer
251
+
252
async def get_supported_tasks(self) -> tuple[SupportedTask, ...]:
253
return await self.engine_core.get_supported_tasks_async()
254
vllm/v1/engine/llm_engine.py
+10
-12
@@ -23,7 +23,7 @@ from vllm.pooling_params import PoolingParams
23
from vllm.sampling_params import SamplingParams
24
from vllm.tasks import SupportedTask
25
from vllm.tracing import init_tracer
26
-from vllm.transformers_utils.tokenizer import AnyTokenizer, init_tokenizer_from_configs
26
+from vllm.transformers_utils.tokenizer import AnyTokenizer
27
from vllm.usage.usage_lib import UsageContext
28
from vllm.utils import Device
29
from vllm.v1.engine import EngineCoreRequest
@@ -95,18 +95,8 @@ class LLMEngine:
95
self.dp_group = None
96
self.should_execute_dummy_batch = False
97
98
- if self.model_config.skip_tokenizer_init:
99
- self.tokenizer = None
100
- else:
101
- # Tokenizer (+ ensure liveness if running in another process).
102
- self.tokenizer = init_tokenizer_from_configs(
103
- model_config=vllm_config.model_config
104
- )
105
-
98
# Processor (convert Inputs --> EngineCoreRequests)
107
- self.processor = Processor(
108
- vllm_config=vllm_config, tokenizer=self.tokenizer, mm_registry=mm_registry
109
- )
99
+ self.processor = Processor(vllm_config, mm_registry=mm_registry)
100
101
# OutputProcessor (convert EngineCoreOutputs --> RequestOutput).
102
self.output_processor = OutputProcessor(
@@ -214,6 +204,14 @@ class LLMEngine:
204
def validate_outputs(cls, outputs, output_type):
205
return outputs
206
207
+ @property
208
+ def tokenizer(self) -> Optional[AnyTokenizer]:
209
+ return self.processor.tokenizer
210
+
211
+ @tokenizer.setter
212
+ def tokenizer(self, tokenizer: Optional[AnyTokenizer]) -> None:
213
+ self.processor.tokenizer = tokenizer
214
+
215
def get_supported_tasks(self) -> tuple[SupportedTask, ...]:
216
return self.engine_core.get_supported_tasks()
217
vllm/v1/engine/processor.py
+11
-8
@@ -37,15 +37,13 @@ class Processor:
37
def __init__(
38
self,
39
vllm_config: VllmConfig,
40
- tokenizer: AnyTokenizer,
40
mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
42
- ):
41
+ ) -> None:
42
self.vllm_config = vllm_config
43
self.model_config = vllm_config.model_config
44
self.cache_config = vllm_config.cache_config
45
self.lora_config = vllm_config.lora_config
46
self.structured_outputs_config = vllm_config.structured_outputs_config
48
- self.tokenizer = tokenizer
47
48
self.generation_config_fields = self.model_config.try_get_generation_config()
49
@@ -54,11 +52,18 @@ class Processor:
52
53
self.input_preprocessor = InputPreprocessor(
54
self.model_config,
57
- self.tokenizer,
55
mm_registry,
56
mm_processor_cache=self.mm_processor_cache,
57
)
58
59
+ @property
60
+ def tokenizer(self) -> Optional[AnyTokenizer]:
61
+ return self.input_preprocessor.tokenizer
62
+
63
+ @tokenizer.setter
64
+ def tokenizer(self, tokenizer: Optional[AnyTokenizer]) -> None:
65
+ self.input_preprocessor.tokenizer = tokenizer
66
+
67
def _validate_logprobs(
68
self,
69
params: SamplingParams,
@@ -511,10 +516,8 @@ class Processor:
516
else:
517
raise ValueError(f"The {prompt_type} prompt cannot be empty")
518
514
- if self.model_config.skip_tokenizer_init:
515
- tokenizer = None
516
- else:
517
- tokenizer = self.tokenizer
519
+ tokenizer = self.tokenizer
520
+ if tokenizer is not None:
521
max_input_id = max(prompt_ids or [], default=0)
522
523
# NOTE: tokenizer.max_token_id is the tokenizer’s vocab size while