@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fe3398fab2

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 12, 08:30 UTC

[Chore] Enable passing `tokenizer=None` into MM processor (#29724)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Nov 29, 2025 at 22:25 UTC fe3398fab2b11f92b4ada209a85558710c446a36

8 files changed +68 -91

tests/multimodal/test_processing.py

+8 -33

@@ -3,7 +3,6 @@

4 import time

5 from contextlib import nullcontext

6 -from typing import cast

7 import numpy as np

8 import pytest

@@ -24,7 +23,6 @@ from vllm.multimodal.processing import (

23 replace_token_matches,

24 )

25 from vllm.multimodal.profiling import MultiModalProfiler

27 -from vllm.tokenizers import TokenizerLike

27 from .utils import random_image

@@ -238,15 +236,12 @@ def test_find_token_matches(

236 expected_by_key,

237 update_type,

238 ):

241 - # Should not be used since there is nothing to convert to token IDs

242 - mock_tokenizer = cast(TokenizerLike, object())

243 -

239 prompt_updates = {

240 key: update_type(key, target, []).resolve(0)

241 for key, target in target_by_key.items()

242 }

243 result = {

249 - key: list(update.iter_token_matches(prompt, mock_tokenizer))

244 + key: list(update.iter_token_matches(prompt, tokenizer=None))

245 for key, update in prompt_updates.items()

246 }

247

@@ -385,15 +380,12 @@ def test_find_text_matches(

380 expected_by_key,

381 update_type,

382 ):

388 - # Should not be used since there is nothing to convert to text

389 - mock_tokenizer = cast(TokenizerLike, object())

390 -

383 prompt_updates = {

384 key: update_type(key, target, []).resolve(0)

385 for key, target in target_by_key.items()

386 }

387 result = {

396 - key: list(update.iter_text_matches(prompt, mock_tokenizer))

388 + key: list(update.iter_text_matches(prompt, tokenizer=None))

389 for key, update in prompt_updates.items()

390 }

391

@@ -545,9 +537,6 @@ def test_find_update_text(

537 repl_by_key,

538 expected_by_update_type_mm_count,

539 ):

548 - # Should not be used since there is nothing to convert to text

549 - mock_tokenizer = cast(TokenizerLike, object())

550 -

540 for (

541 update_type,

542 expected_by_mm_count,

@@ -564,7 +553,7 @@ def test_find_update_text(

553 new_prompt, result = apply_text_matches(

554 prompt,

555 mm_prompt_updates,

567 - mock_tokenizer,

556 + tokenizer=None,

557 )

558

559 # Only displayed on error

@@ -750,9 +739,6 @@ def test_find_update_tokens(

739 repl_by_key,

740 expected_by_update_type_mm_count,

741 ):

753 - # Should not be used since there is nothing to convert to tokens

754 - mock_tokenizer = cast(TokenizerLike, object())

755 -

742 for (

743 update_type,

744 expected_by_mm_count,

@@ -769,7 +755,7 @@ def test_find_update_tokens(

755 new_prompt, result = apply_token_matches(

756 prompt,

757 mm_prompt_updates,

772 - mock_tokenizer,

758 + tokenizer=None,

759 )

760

761 # Only displayed on error

@@ -900,15 +886,12 @@ def test_find_mm_placeholders(

886 expected,

887 update_type,

888 ):

903 - # Should not be used since there is nothing to convert to tokens

904 - mock_tokenizer = cast(TokenizerLike, object())

905 -

889 mm_prompt_updates = {

890 key: [[update_type(key, [], repl).resolve(i)] for i in range(3)]

891 for key, repl in repl_by_key.items()

892 }

893

911 - result = find_mm_placeholders(prompt, mm_prompt_updates, mock_tokenizer)

894 + result = find_mm_placeholders(prompt, mm_prompt_updates, tokenizer=None)

895

896 # Only displayed on error

897 print("result:", result)

@@ -1029,12 +1012,9 @@ def test_hf_processor_init_kwargs(

1012 inference_kwargs,

1013 expected_kwargs,

1014 ):

1032 - # Should not be used since there is nothing to convert to tokens

1033 - mock_tokenizer = cast(TokenizerLike, object())

1034 -

1015 ctx = InputProcessingContext(

1016 model_config=ModelConfig(model_id, mm_processor_kwargs=config_kwargs),

1037 - tokenizer=mock_tokenizer,

1017 + tokenizer=None,

1018 )

1019

1020 processor = ctx.get_hf_processor(

@@ -1065,12 +1045,9 @@ def test_hf_processor_call_kwargs(

1045 inference_kwargs,

1046 expected_kwargs,

1047 ):

1068 - # Should not be used since there is nothing to convert to tokens

1069 - mock_tokenizer = cast(TokenizerLike, object())

1070 -

1048 ctx = InputProcessingContext(

1049 model_config=ModelConfig(model_id, mm_processor_kwargs=config_kwargs),

1073 - tokenizer=mock_tokenizer,

1050 + tokenizer=None,

1051 )

1052

1053 processor = ctx.get_hf_processor(DummyProcessor) # type: ignore[arg-type]

@@ -1089,8 +1066,6 @@ def test_apply_matches_no_match_exits_quickly():

1066

1067 With the fix, it should exit immediately when no match is found.

1068 """

1092 - mock_tokenizer = cast(TokenizerLike, object())

1093 -

1069 # Create a long prompt with no placeholder

1070 long_prompt = "x" * 10000

1071

@@ -1103,7 +1078,7 @@ def test_apply_matches_no_match_exits_quickly():

1078 result, _ = _apply_matches(

1079 long_prompt,

1080 mm_prompt_updates,

1106 - mock_tokenizer,

1081 + tokenizer=None,

1082 )

1083 elapsed = time.perf_counter() - start

1084

vllm/entrypoints/openai/serving_engine.py

+1 -1

@@ -337,7 +337,7 @@ class OpenAIServing:

337 tokenizer = input_processor.tokenizer

338 if tokenizer is None:

339 raise ValueError(

340 - "You cannot use beam search when `skip_tokenizer_init` is True"

340 + "You cannot use beam search when `skip_tokenizer_init=True`"

341 )

342

343 eos_token_id: int = tokenizer.eos_token_id # type: ignore

vllm/inputs/preprocess.py

+2 -13

@@ -62,7 +62,7 @@ class InputPreprocessor:

62 def get_tokenizer(self) -> TokenizerLike:

63 if self.tokenizer is None:

64 raise ValueError(

65 - "You cannot pass text prompts when `skip_tokenizer_init` is True"

65 + "You cannot pass text prompts when `skip_tokenizer_init=True`"

66 )

68 return self.tokenizer

@@ -228,22 +228,11 @@ class InputPreprocessor:

228

229 return tokenizer.encode(prompt, **tokenization_kwargs)

230

231 - def _get_mm_tokenizer(self) -> TokenizerLike:

232 - # PrithviGeoSpatialMAE needs to be initialized without a tokenizer

233 - # while using also multi-modal input

234 - if not self.tokenizer:

235 - return cast(TokenizerLike, object()) # Dummy

236 -

237 - tokenizer = self.get_tokenizer()

238 - return tokenizer

239 -

231 def _get_mm_processor(self) -> BaseMultiModalProcessor:

232 if not hasattr(self, "_mm_processor"):

242 - tokenizer = self._get_mm_tokenizer()

243 -

233 self._mm_processor = self.mm_registry.create_processor(

234 self.model_config,

246 - tokenizer=tokenizer,

235 + tokenizer=self.tokenizer,

236 cache=self.mm_processor_cache,

237 )

238

vllm/model_executor/models/glm4_1v.py

-6

@@ -866,12 +866,6 @@ class Glm4vVisionTransformer(nn.Module):

866

867

868 class Glm4vProcessingInfo(BaseProcessingInfo):

869 - def get_hf_config(self):

870 - return self.ctx.get_hf_config()

871 -

872 - def get_tokenizer(self):

873 - return self.ctx.tokenizer

874 -

869 def get_supported_mm_limits(self) -> Mapping[str, int | None]:

870 return {"image": None, "video": 1}

871

vllm/model_executor/models/qwen3_vl.py

-3

@@ -615,9 +615,6 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):

615 **kwargs,

616 )

617

618 - def get_tokenizer(self):

619 - return self.ctx.tokenizer

620 -

618 def get_image_processor(self, **kwargs: object) -> Qwen2VLImageProcessorFast:

619 return self.get_hf_processor(**kwargs).image_processor

620

vllm/model_executor/models/qwen_vl.py

+1 -1

@@ -555,7 +555,7 @@ class QwenVLProcessor:

555

556 class QwenVLProcessingInfo(BaseProcessingInfo):

557 def get_tokenizer(self) -> PreTrainedTokenizer:

558 - tokenizer = self.ctx.tokenizer

558 + tokenizer = self.ctx.get_tokenizer()

559 assert isinstance(tokenizer, PreTrainedTokenizer)

560

561 return _get_tokenizer_without_image_pad(tokenizer)

vllm/multimodal/processing.py

+55 -31

@@ -97,15 +97,37 @@ def _cached_decode(

97 )

100 -def _seq2text(tokenizer: TokenizerLike, seq: PromptSeq) -> str:

100 +def _seq2text(

101 + tokenizer: TokenizerLike | None,

102 + seq: PromptSeq,

103 + *,

104 + use_cache: bool = True,

105 +) -> str:

106 if isinstance(seq, str):

107 return seq

108

109 + if tokenizer is None:

110 + raise ValueError("You cannot decode tokens when `skip_tokenizer_init=True`")

111 +

112 + if not use_cache:

113 + return decode_tokens(tokenizer, seq)

114 +

115 return _cached_decode(tokenizer, tuple(seq))

116

117

107 -def _seq2tokens(tokenizer: TokenizerLike, seq: PromptSeq) -> list[int]:

118 +def _seq2tokens(

119 + tokenizer: TokenizerLike | None,

120 + seq: PromptSeq,

121 + *,

122 + use_cache: bool = True,

123 +) -> list[int]:

124 if isinstance(seq, str):

125 + if tokenizer is None:

126 + raise ValueError("You cannot encode text when `skip_tokenizer_init=True`")

127 +

128 + if not use_cache:

129 + return encode_tokens(tokenizer, seq, add_special_tokens=False)

130 +

131 return _cached_encode(tokenizer, seq, add_special_tokens=False)

132

133 return seq

@@ -114,7 +136,7 @@ def _seq2tokens(tokenizer: TokenizerLike, seq: PromptSeq) -> list[int]:

136 class _GetMatchIndex(Protocol):

137 def __call__(

138 self,

117 - tokenizer: TokenizerLike,

139 + tokenizer: TokenizerLike | None,

140 prompt: PromptSeq,

141 start_idx: int = 0,

142 ) -> int | None: ...

@@ -144,7 +166,7 @@ class PromptIndexTargets:

166 """

167

168 def get_match_index(

147 - tokenizer: TokenizerLike,

169 + tokenizer: TokenizerLike | None,

170 prompt: PromptSeq,

171 start_idx: int = 0,

172 ) -> int | None:

@@ -154,13 +176,11 @@ class PromptIndexTargets:

176 prefix = seq

177

178 if isinstance(prompt, str):

157 - if not isinstance(prefix, str):

158 - # Make both `str`

159 - prefix = decode_tokens(tokenizer, prefix)

179 + # Make both `str`

180 + prefix = _seq2text(tokenizer, prefix, use_cache=False)

181 else:

161 - if isinstance(prefix, str):

162 - # Make both `list[int]`

163 - prefix = encode_tokens(tokenizer, prefix, add_special_tokens=False)

182 + # Make both `list[int]`

183 + prefix = _seq2tokens(tokenizer, prefix, use_cache=False)

184

185 match_idx = len(prefix)

186 return match_idx if prompt[:match_idx] == prefix else None

@@ -200,7 +220,7 @@ class PromptUpdateDetails(Generic[_S]):

220 full: _S

221 """The full content."""

222

203 - is_embed: Callable[[TokenizerLike, PromptSeq], torch.Tensor] | None = None

223 + is_embed: Callable[[TokenizerLike | None, PromptSeq], torch.Tensor] | None = None

224 """

225 Given [`full`][vllm.multimodal.processing.PromptUpdateDetails.full],

226 return a boolean mask of shape `(len(full),)` indicating which positions

@@ -221,8 +241,8 @@ class PromptUpdateDetails(Generic[_S]):

241 seq: _S,

242 embed_text: str,

243 ) -> "PromptUpdateDetails[_S]":

224 - def is_embed(tokenizer: TokenizerLike, full: PromptSeq) -> torch.Tensor:

225 - embed_token_ids = encode_tokens(tokenizer, embed_text)

244 + def is_embed(tokenizer: TokenizerLike | None, full: PromptSeq) -> torch.Tensor:

245 + embed_token_ids = _seq2tokens(tokenizer, embed_text, use_cache=False)

246 token_ids = _seq2tokens(tokenizer, full)

247

248 return torch.isin(

@@ -237,7 +257,7 @@ class PromptUpdateDetails(Generic[_S]):

257 seq: _S,

258 embed_token_id: int,

259 ) -> "PromptUpdateDetails[_S]":

240 - def is_embed(tokenizer: TokenizerLike, full: PromptSeq) -> torch.Tensor:

260 + def is_embed(tokenizer: TokenizerLike | None, full: PromptSeq) -> torch.Tensor:

261 token_ids = _seq2tokens(tokenizer, full)

262

263 return torch.tensor(token_ids) == embed_token_id

@@ -523,7 +543,7 @@ class ResolvedPromptUpdate:

543 def iter_token_matches(

544 self,

545 prompt: list[int],

526 - tokenizer: TokenizerLike,

546 + tokenizer: TokenizerLike | None,

547 *,

548 start_idx: int = 0,

549 ) -> Generator[PromptTargetMatch]:

@@ -545,7 +565,7 @@ class ResolvedPromptUpdate:

565 def iter_text_matches(

566 self,

567 prompt: str,

548 - tokenizer: TokenizerLike,

568 + tokenizer: TokenizerLike | None,

569 *,

570 start_idx: int = 0,

571 ) -> Generator[PromptTargetMatch]:

@@ -567,7 +587,7 @@ class ResolvedPromptUpdate:

587 def iter_matches(

588 self,

589 prompt: list[int] | str,

570 - tokenizer: TokenizerLike,

590 + tokenizer: TokenizerLike | None,

591 *,

592 start_idx: int = 0,

593 ) -> Generator[PromptTargetMatch]:

@@ -676,7 +696,7 @@ _MatchToApply = tuple[tuple[str, int], tuple[PromptTargetMatch, int]]

696 def _find_matches(

697 prompt: _S,

698 mm_prompt_updates: "MultiModalPromptUpdates",

679 - tokenizer: TokenizerLike,

699 + tokenizer: TokenizerLike | None,

700 *,

701 prev_end_idx: int = 0,

702 current_result: "MultiModalPromptUpdatesApplyResult",

@@ -741,7 +761,7 @@ def _all_items_found(

761 def _apply_matches(

762 prompt: _S,

763 mm_prompt_updates: "MultiModalPromptUpdates",

744 - tokenizer: TokenizerLike,

764 + tokenizer: TokenizerLike | None,

765 ) -> tuple[list[_S], "MultiModalPromptUpdatesApplyResult"]:

766 mm_item_counts = {m: len(items) for m, items in mm_prompt_updates.items()}

767

@@ -807,7 +827,7 @@ def _apply_matches(

827 def apply_token_matches(

828 prompt: list[int],

829 mm_prompt_updates: "MultiModalPromptUpdates",

810 - tokenizer: TokenizerLike,

830 + tokenizer: TokenizerLike | None,

831 ) -> tuple[list[int], "MultiModalPromptUpdatesApplyResult"]:

832 """

833 Apply the updates in `mm_prompt_updates` to `prompt`.

@@ -824,7 +844,7 @@ def apply_token_matches(

844 def apply_text_matches(

845 prompt: str,

846 mm_prompt_updates: "MultiModalPromptUpdates",

827 - tokenizer: TokenizerLike,

847 + tokenizer: TokenizerLike | None,

848 ) -> tuple[str, "MultiModalPromptUpdatesApplyResult"]:

849 """

850 Apply the updates in `mm_prompt_updates` to `prompt`.

@@ -841,7 +861,7 @@ def apply_text_matches(

861 def _iter_placeholders(

862 prompt: list[int],

863 mm_prompt_updates: "MultiModalPromptUpdates",

844 - tokenizer: TokenizerLike,

864 + tokenizer: TokenizerLike | None,

865 ) -> Iterable[PlaceholderFeaturesInfo]:

866 """

867 Yield each set of placeholder tokens found in `prompt`.

@@ -910,7 +930,7 @@ def _iter_placeholders(

930 def find_mm_placeholders(

931 prompt: list[int],

932 mm_prompt_updates: "MultiModalPromptUpdates",

913 - tokenizer: TokenizerLike,

933 + tokenizer: TokenizerLike | None,

934 ) -> Mapping[str, list[PlaceholderFeaturesInfo]]:

935 it = _iter_placeholders(prompt, mm_prompt_updates, tokenizer)

936 return dict(full_groupby_modality(it))

@@ -931,9 +951,17 @@ class InputProcessingContext:

951 model_config: ModelConfig

952 """The configuration of the model."""

953

934 - tokenizer: TokenizerLike

954 + tokenizer: TokenizerLike | None

955 """The tokenizer used to tokenize the inputs."""

956

957 + def get_tokenizer(self) -> TokenizerLike:

958 + if self.tokenizer is None:

959 + raise ValueError(

960 + "You cannot pass text prompts when `skip_tokenizer_init=True`"

961 + )

962 +

963 + return self.tokenizer

964 +

965 @overload

966 def get_hf_config(self, /) -> PretrainedConfig: ...

967

@@ -1148,7 +1176,7 @@ class BaseProcessingInfo:

1176 return self.ctx.model_config.model

1177

1178 def get_tokenizer(self) -> TokenizerLike:

1151 - return self.ctx.tokenizer

1179 + return self.ctx.get_tokenizer()

1180

1181 def get_hf_config(self) -> PretrainedConfig:

1182 return self.ctx.get_hf_config()

@@ -1960,15 +1988,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1988 for update_idxs in match_result.values()

1989 ):

1990 new_text, match_result = self._apply_text_matches(

1963 - decode_tokens(tokenizer, token_ids),

1991 + _seq2text(tokenizer, token_ids, use_cache=False),

1992 mm_prompt_updates,

1993 )

1994

1967 - new_token_ids = encode_tokens(

1968 - tokenizer,

1969 - new_text,

1970 - add_special_tokens=False,

1971 - )

1995 + new_token_ids = _seq2tokens(tokenizer, new_text, use_cache=False)

1996

1997 matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list)

1998 for modality, update_idxs in match_result.items():

vllm/multimodal/registry.py

+1 -3

@@ -234,9 +234,7 @@ class MultiModalRegistry:

234 model_config: "ModelConfig",

235 tokenizer: TokenizerLike | None = None,

236 ) -> InputProcessingContext:

237 - if model_config.skip_tokenizer_init:

238 - tokenizer = cast(TokenizerLike, object())

239 - elif tokenizer is None:

237 + if tokenizer is None and not model_config.skip_tokenizer_init:

238 tokenizer = cached_tokenizer_from_config(model_config)

239

240 return InputProcessingContext(model_config, tokenizer)