@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 68ffbca7e4

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 12, 04:45 UTC

[Chore] Use `tokenizer.encode` and `tokenizer.decode` directly (#29851)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Dec 2, 2025 at 20:30 UTC 68ffbca7e462cfa6a32b46dabc9a604c7c1b918d

9 files changed +36 -40

tests/models/multimodal/processing/test_common.py

+8 -6

@@ -22,8 +22,11 @@ from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict

22 from vllm.multimodal.cache import MultiModalProcessorOnlyCache

23 from vllm.multimodal.inputs import MultiModalInputs

24 from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext

25 -from vllm.tokenizers import MistralTokenizer, cached_tokenizer_from_config

26 -from vllm.transformers_utils.tokenizer import encode_tokens

25 +from vllm.tokenizers import (

26 + MistralTokenizer,

27 + TokenizerLike,

28 + cached_tokenizer_from_config,

29 +)

31 from ....multimodal.utils import random_audio, random_image, random_video

32 from ...registry import (

@@ -151,7 +154,7 @@ def get_text_token_prompts(

154 mm_data: MultiModalDataDict,

155 ):

156 dummy_inputs = processor.dummy_inputs

154 - tokenizer = processor.info.get_tokenizer()

157 + tokenizer: TokenizerLike = processor.info.get_tokenizer()

158 model_config = processor.info.ctx.model_config

159

160 model_type = model_config.hf_config.model_type

@@ -188,10 +191,9 @@ def get_text_token_prompts(

191 assert isinstance(inputs.prompt, str)

192

193 text_prompt = inputs.prompt

191 - token_prompt = encode_tokens(

192 - tokenizer,

194 + token_prompt = tokenizer.encode(

195 text_prompt,

194 - add_special_tokens=_ADD_SPECIAL_TOKENS_OVERRIDES.get(model_type),

196 + add_special_tokens=_ADD_SPECIAL_TOKENS_OVERRIDES.get(model_type, True),

197 )

198

199 return text_prompt, token_prompt

tests/models/multimodal/processing/test_llama4.py

+1 -2

@@ -5,7 +5,6 @@

5 import pytest

7 from vllm.multimodal import MULTIMODAL_REGISTRY

8 -from vllm.transformers_utils.tokenizer import encode_tokens

9 from ....conftest import ImageTestAssets

10 from ...utils import build_model_context

@@ -48,7 +47,7 @@ def test_processor_override(

47 ]

48 }

49 if tokenized_prompt:

51 - prompt = encode_tokens(tokenizer, prompt)

50 + prompt = tokenizer.encode(prompt)

52 processed_inputs = processor.apply(prompt, mm_data, mm_processor_kwargs)

53 mm_data = processed_inputs["mm_kwargs"].get_data()

vllm/entrypoints/openai/speech_to_text.py

+1 -1

@@ -37,7 +37,7 @@ from vllm.inputs.data import PromptType

37 from vllm.logger import init_logger

38 from vllm.model_executor.models import SupportsTranscription

39 from vllm.outputs import RequestOutput

40 -from vllm.transformers_utils.tokenizer import get_tokenizer

40 +from vllm.tokenizers import get_tokenizer

41 from vllm.utils.import_utils import PlaceholderModule

43 try:

vllm/entrypoints/renderer.py

+2 -2

@@ -33,7 +33,7 @@ class RenderConfig:

33 `0` yields an empty list (and skips embeds).

34 `-1` maps to `model_config.max_model_len`."""

36 - add_special_tokens: bool | None = True

36 + add_special_tokens: bool = True

37 """Whether to add model-specific special tokens during tokenization."""

39 cache_salt: str | None = None

@@ -315,7 +315,7 @@ class CompletionRenderer(BaseRenderer):

315 text: str,

316 max_length: int | None,

317 truncate_prompt_tokens: int | None,

318 - add_special_tokens: bool | None,

318 + add_special_tokens: bool,

319 cache_salt: str | None,

320 ) -> EngineTokensPrompt:

321 """Tokenize text input asynchronously."""

vllm/entrypoints/score_utils.py

+1 -1

@@ -19,7 +19,7 @@ from vllm.inputs import TokensPrompt

19 from vllm.model_executor.models.interfaces import supports_score_template

20 from vllm.multimodal.inputs import MultiModalDataDict

21 from vllm.outputs import PoolingRequestOutput

22 -from vllm.transformers_utils.tokenizer import TokenizerLike

22 +from vllm.tokenizers import TokenizerLike

24 ScoreContentPartParam: TypeAlias = (

25 ChatCompletionContentPartImageParam | ChatCompletionContentPartImageEmbedsParam

vllm/model_executor/models/nano_nemotron_vl.py

+10 -15

@@ -75,7 +75,6 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder

75 from vllm.sequence import IntermediateTensors

76 from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config

77 from vllm.transformers_utils.configs.radio import RadioConfig

78 -from vllm.transformers_utils.tokenizer import encode_tokens

78 from vllm.utils.tensor_schema import TensorSchema, TensorShape

80 from .utils import _merge_multimodal_embeddings

@@ -454,14 +453,12 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):

453

454 # Pre-tokenize special tokens for video processing

455 # to avoid repeated tokenization

457 - self._img_start_token_ids = encode_tokens(

458 - tokenizer, IMG_START, add_special_tokens=False

456 + self._img_start_token_ids = tokenizer.encode(

457 + IMG_START, add_special_tokens=False

458 )

460 - self._img_end_token_ids = encode_tokens(

461 - tokenizer, IMG_END, add_special_tokens=False

462 - )

463 - self._img_context_token_ids = encode_tokens(

464 - tokenizer, IMG_CONTEXT, add_special_tokens=False

459 + self._img_end_token_ids = tokenizer.encode(IMG_END, add_special_tokens=False)

460 + self._img_context_token_ids = tokenizer.encode(

461 + IMG_CONTEXT, add_special_tokens=False

462 )

463

464 @property

@@ -1179,14 +1176,12 @@ class NemotronH_Nano_VL_V2(

1176 # Pre-tokenize special tokens for video processing

1177 # to avoid repeated tokenization

1178 tokenizer = cached_tokenizer_from_config(vllm_config.model_config)

1182 - self._img_start_token_ids = encode_tokens(

1183 - tokenizer, IMG_START, add_special_tokens=False

1184 - )

1185 - self._img_end_token_ids = encode_tokens(

1186 - tokenizer, IMG_END, add_special_tokens=False

1179 + self._img_start_token_ids = tokenizer.encode(

1180 + IMG_START, add_special_tokens=False

1181 )

1188 - self._img_context_token_ids = encode_tokens(

1189 - tokenizer, IMG_CONTEXT, add_special_tokens=False

1182 + self._img_end_token_ids = tokenizer.encode(IMG_END, add_special_tokens=False)

1183 + self._img_context_token_ids = tokenizer.encode(

1184 + IMG_CONTEXT, add_special_tokens=False

1185 )

1186

1187 def pixel_shuffle(self, x, scale_factor=0.5):

vllm/model_executor/models/qwen2_5_omni_thinker.py

+1 -2

@@ -88,7 +88,6 @@ from vllm.multimodal.processing import (

88 )

89 from vllm.multimodal.profiling import BaseDummyInputsBuilder

90 from vllm.sequence import IntermediateTensors

91 -from vllm.transformers_utils.tokenizer import encode_tokens

91 from vllm.utils.tensor_schema import TensorSchema, TensorShape

93 from .interfaces import (

@@ -591,7 +590,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(

590 tokenization_kwargs=tokenization_kwargs,

591 )

592 tokenizer = self.info.get_tokenizer()

594 - prompt_ids = encode_tokens(tokenizer, prompt)

593 + prompt_ids = tokenizer.encode(prompt)

594 else:

595 prompt_ids = self._apply_hf_processor_tokens_only(prompt)

596

vllm/multimodal/processing.py

+8 -11

@@ -25,7 +25,6 @@ from typing_extensions import TypeVar, assert_never

25 from vllm.logger import init_logger

26 from vllm.tokenizers import TokenizerLike

27 from vllm.transformers_utils.processor import cached_processor_from_config

28 -from vllm.transformers_utils.tokenizer import decode_tokens, encode_tokens

28 from vllm.utils.collection_utils import flatten_2d_lists, full_groupby

29 from vllm.utils.func_utils import get_allowed_kwarg_only_overrides

30 from vllm.utils.jsontree import JSONTree, json_map_leaves

@@ -80,9 +79,9 @@ def _cached_encode(

79 tokenizer: TokenizerLike,

80 text: str,

81 *,

83 - add_special_tokens: bool | None = None,

82 + add_special_tokens: bool = True,

83 ) -> list[int]:

85 - return encode_tokens(tokenizer, text, add_special_tokens=add_special_tokens)

84 + return tokenizer.encode(text, add_special_tokens=add_special_tokens)

87 @lru_cache(maxsize=2048)

@@ -90,11 +89,9 @@ def _cached_decode(

89 tokenizer: TokenizerLike,

90 token_ids: tuple[int, ...],

91 *,

93 - skip_special_tokens: bool | None = None,

92 + skip_special_tokens: bool = False,

93 ) -> str:

95 - return decode_tokens(

96 - tokenizer, list(token_ids), skip_special_tokens=skip_special_tokens

97 - )

94 + return tokenizer.decode(list(token_ids), skip_special_tokens=skip_special_tokens)

97 def _seq2text(

@@ -110,7 +107,7 @@ def _seq2text(

107 raise ValueError("You cannot decode tokens when `skip_tokenizer_init=True`")

108

109 if not use_cache:

113 - return decode_tokens(tokenizer, seq)

110 + return tokenizer.decode(seq)

111

112 return _cached_decode(tokenizer, tuple(seq))

113

@@ -126,7 +123,7 @@ def _seq2tokens(

123 raise ValueError("You cannot encode text when `skip_tokenizer_init=True`")

124

125 if not use_cache:

129 - return encode_tokens(tokenizer, seq, add_special_tokens=False)

126 + return tokenizer.encode(seq, add_special_tokens=False)

127

128 return _cached_encode(tokenizer, seq, add_special_tokens=False)

129

@@ -2198,8 +2195,8 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):

2195 tokenizer = self.info.get_tokenizer()

2196 decoder_prompt_raw = self.create_decoder_prompt(prompt, mm_data)

2197 if isinstance(decoder_prompt_raw, str):

2201 - decoder_prompt_ids = encode_tokens(

2202 - tokenizer, decoder_prompt_raw, add_special_tokens=False

2198 + decoder_prompt_ids = tokenizer.encode(

2199 + decoder_prompt_raw, add_special_tokens=False

2200 )

2201 else:

2202 decoder_prompt_ids = decoder_prompt_raw

vllm/transformers_utils/tokenizer.py

@@ -4,6 +4,8 @@

4 import warnings

5 from typing import Any

7 +from typing_extensions import deprecated

8 +

9 from vllm.logger import init_logger

10 from vllm.tokenizers import TokenizerLike

@@ -73,6 +75,7 @@ def __getattr__(name: str):

75 raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

78 +@deprecated("Will be removed in v0.13. Please use `tokenizer.decode()` instead.")

79 def decode_tokens(

80 tokenizer: TokenizerLike,

81 token_ids: list[int],

@@ -94,6 +97,7 @@ def decode_tokens(

97 return tokenizer.decode(token_ids, **kw_args)

100 +@deprecated("Will be removed in v0.13. Please use `tokenizer.encode()` instead.")

101 def encode_tokens(

102 tokenizer: TokenizerLike,

103 text: str,