ecfa7bb373 · tduncan/Nvidia-Optimization-Stack-VLLM

tests/multimodal/test_processing.py

+2 -6

@@ -1082,7 +1082,7 @@ class _FakeTokenizer:

1082 _MERGES = {"food": (1000,), "foo": (101, 111, 111)}

1083 _INVERSE = {ids: text for text, ids in _MERGES.items()}

1084

1085 - def encode(self, text: str, add_special_tokens: bool = True) -> list[int]:

1085 + def encode(self, text: str, **kwargs) -> list[int]:

1086 token_ids = list[int]()

1087 pos = 0

1088 while pos < len(text):

@@ -1097,11 +1097,7 @@ class _FakeTokenizer:

1097 pos += 1

1098 return token_ids

1099

1100 - def decode(

1101 - self,

1102 - token_ids: list[int],

1103 - skip_special_tokens: bool = False,

1104 - ) -> str:

1100 + def decode(self, token_ids: list[int], **kwargs) -> str:

1101 chars = list[str]()

1102 pos = 0

1103 while pos < len(token_ids):

vllm/model_executor/models/llava_onevision2.py

+2 -1

@@ -98,6 +98,7 @@ from vllm.multimodal.processing import (

98 PromptReplacement,

99 PromptUpdate,

100 PromptUpdateDetails,

101 + cached_encode,

102 )

103 from vllm.multimodal.processing.dummy_inputs import BaseDummyInputsBuilder

104 from vllm.multimodal.video import (

@@ -1894,7 +1895,7 @@ class LlavaOnevision2MultiModalProcessor(

1895 video_pad_id = vocab["<|video_pad|>"]

1896 vision_start_id = vocab["<|vision_start|>"]

1897 vision_end_id = vocab["<|vision_end|>"]

1897 - newline_ids = tokenizer.encode("\n", add_special_tokens=False)

1898 + newline_ids = cached_encode(tokenizer, "\n", add_special_tokens=False)

1899 merge_length = image_processor.merge_size**2

1900 decimals = int(hf_processor_mm_kwargs.get("timestamp_decimals", 1))

1901

vllm/model_executor/models/minicpmv.py

+1 -1

@@ -1050,7 +1050,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

1050 tokenizer = self.info.get_tokenizer()

1051 for modality, pattern in placeholders:

1052 sub_pattern = tokenizer.decode(

1053 - tokenizer.encode(pattern, add_special_tokens=False)

1053 + cached_encode(tokenizer, pattern, add_special_tokens=False)

1054 )

1055 if sub_pattern != pattern:

1056 additional_placeholders.append((modality, sub_pattern))

vllm/model_executor/models/minicpmv4_6.py

+1 -1

@@ -328,7 +328,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):

328 additional_placeholders = []

329 for modality, pattern in placeholders:

330 sub_pattern = tokenizer.decode(

331 - tokenizer.encode(pattern, add_special_tokens=False)

331 + cached_encode(tokenizer, pattern, add_special_tokens=False)

332 )

333 if sub_pattern != pattern:

334 additional_placeholders.append((modality, sub_pattern))

vllm/model_executor/models/moondream3.py

+3 -1

@@ -57,6 +57,7 @@ from vllm.multimodal.processing import (

57 PromptReplacement,

58 PromptUpdate,

59 PromptUpdateDetails,

60 + cached_encode,

61 )

62 from vllm.sequence import IntermediateTensors

63 from vllm.transformers_utils.configs.moondream3 import (

@@ -945,7 +946,8 @@ class Moondream3MultiModalProcessor(BaseMultiModalProcessor[Moondream3Processing

946 @cached_property

947 def bos_image_placeholder_tokens(self) -> list[int]:

948 tokenizer = self.info.get_tokenizer()

948 - token_ids = tokenizer.encode(

949 + token_ids = cached_encode(

950 + tokenizer,

951 self.bos_image_placeholder,

952 add_special_tokens=False,

953 )

vllm/model_executor/models/moss_audio.py

+7 -2

@@ -53,6 +53,7 @@ from vllm.multimodal.processing import (

53 PromptReplacement,

54 PromptUpdate,

55 PromptUpdateDetails,

56 + cached_encode,

57 )

58 from vllm.sequence import IntermediateTensors

59 from vllm.transformers_utils.repo_utils import get_hf_file_to_dict

@@ -1376,6 +1377,8 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn

1377 out_mm_kwargs: MultiModalKwargsItems,

1378 ) -> Sequence[PromptUpdate]:

1379 processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)

1380 + tokenizer = processor.tokenizer

1381 +

1382 out_mm_data = out_mm_kwargs.get_data()

1383 audio_data_seqlens = out_mm_data.get("audio_data_seqlens")

1384 if audio_data_seqlens is None:

@@ -1431,11 +1434,13 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn

1434 )

1435 ]

1436 for suffix in ("", "\n"):

1434 - tokenizer_target = processor.tokenizer.encode(

1437 + tokenizer_target = cached_encode(

1438 + tokenizer,

1439 MOSS_AUDIO_PLACEHOLDER + suffix,

1440 add_special_tokens=False,

1441 )

1438 - suffix_token_ids = processor.tokenizer.encode(

1442 + suffix_token_ids = cached_encode(

1443 + tokenizer,

1444 suffix,

1445 add_special_tokens=False,

1446 )

vllm/model_executor/models/moss_transcribe_diarize.py

+4 -2

@@ -449,10 +449,12 @@ class MossTranscribeDiarizeDummyInputsBuilder(

449 dummy_mm_items = self.info.parse_mm_data(dummy_mm_data)

450 num_audios = mm_counts.get("audio", 0)

451 tokenizer = self.info.get_tokenizer()

452 - prompt = tokenizer.encode(

452 + prompt = cached_encode(

453 + tokenizer,

454 AUDIO_PLACEHOLDER * num_audios,

455 add_special_tokens=False,

455 - ) or tokenizer.encode(

456 + ) or cached_encode(

457 + tokenizer,

458 "\n",

459 add_special_tokens=False,

460 )

vllm/model_executor/models/paligemma.py

+3 -2

@@ -216,8 +216,9 @@ class PaliGemmaMultiModalProcessor(BaseMultiModalProcessor[PaliGemmaProcessingIn

216 prompt_token_ids = mm_inputs["prompt_token_ids"]

217

218 tokenizer = self.info.get_tokenizer()

219 - newline_prompt = "\n"

220 - newline_token_id = tokenizer.encode(newline_prompt)[-1] # 108

219 + vocab = tokenizer.get_vocab()

220 + newline_token_id = vocab["\n"]

221 +

222 # Force to add newline at the end of prompt for paligemma's format

223 # This step can NOT be replacemented by current PromptUpdate methods

224 if len(prompt_token_ids) and prompt_token_ids[-1] != newline_token_id:

vllm/model_executor/models/phi3v.py

+2 -1

@@ -535,7 +535,8 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):

535

536 # Keep the behavior in line with HF processor

537 if len(mm_prompt_updates) and (

538 - token_ids[:2] == tokenizer.encode("<s> <|image|>", add_special_tokens=False)

538 + token_ids[:2]

539 + == cached_encode(tokenizer, "<s> <|image|>", add_special_tokens=False)

540 ):

541 token_ids = [token_ids[0], *token_ids[2:]]

542 placeholders = {

vllm/model_executor/models/qwen3_asr.py

+2 -1

@@ -81,6 +81,7 @@ from vllm.multimodal.processing import (

81 BaseProcessingInfo,

82 PromptReplacement,

83 PromptUpdate,

84 + cached_encode,

85 )

86 from vllm.sequence import IntermediateTensors

87 from vllm.tokenizers import cached_tokenizer_from_config

@@ -691,7 +692,7 @@ class Qwen3ASRForConditionalGeneration(

692 full_lang_name = cls.supported_languages.get(lang_code, lang_code)

693 prompt += f"language {full_lang_name}{_ASR_TEXT_TAG}"

694

694 - prompt_token_ids = tokenizer.encode(prompt)

695 + prompt_token_ids = cached_encode(tokenizer, prompt)

696

697 return TokensPrompt(

698 prompt_token_ids=prompt_token_ids,

vllm/model_executor/models/qwen3_asr_realtime.py

+3 -2

@@ -39,10 +39,11 @@ from vllm.multimodal import MULTIMODAL_REGISTRY

39 from vllm.multimodal.cache import _I, BaseMultiModalProcessorCache

40 from vllm.multimodal.inputs import MultiModalKwargsOptionalItems

41 from vllm.multimodal.parse import MultiModalDataItems

42 -from vllm.multimodal.processing import BaseDummyInputsBuilder

42 from vllm.multimodal.processing.processor import (

43 + BaseDummyInputsBuilder,

44 MultiModalPromptUpdates,

45 PlaceholderFeaturesInfo,

46 + cached_encode,

47 )

48 from vllm.tokenizers import cached_tokenizer_from_config

49 from vllm.transformers_utils.processor import cached_processor_from_config

@@ -205,7 +206,7 @@ class Qwen3ASRRealtimeGeneration(Qwen3ASRForConditionalGeneration, SupportsRealt

206 f"<|im_start|>user\n{audio_placeholder}<|im_end|>\n<|im_start|>assistant\n"

207 )

208

208 - prompt_token_ids = tokenizer.encode(prompt_template)

209 + prompt_token_ids = cached_encode(tokenizer, prompt_template)

210

211 async for audio_chunk in audio_stream:

212 buffer.write_audio(audio_chunk)

vllm/model_executor/models/transformers/multimodal.py

+1 -1

@@ -841,7 +841,7 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):

841 replacements = defaultdict[str, list[list[int]]](list)

842 for entry in offsets[0]:

843 replacements[entry["type"]].append(

844 - tokenizer.encode(entry["replacement"], add_special_tokens=False)

844 + cached_encode(tokenizer, entry["replacement"], add_special_tokens=False)

845 )

846

847 for modality, seqs in replacements.items():

vllm/multimodal/processing/dummy_inputs.py

+3 -1

@@ -91,7 +91,9 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):

91 # prompt tokens.

92 dummy_prompt = []

93 else:

94 - dummy_prompt = tokenizer.encode(dummy_text, truncation=False)

94 + from .processor import cached_encode

95 +

96 + dummy_prompt = cached_encode(tokenizer, dummy_text, truncation=False)

97

98 return ProcessorInputs(

99 prompt=dummy_prompt,

vllm/multimodal/processing/processor.py

+6 -1

@@ -57,10 +57,15 @@ def cached_encode(

57 tokenizer: TokenizerLike,

58 text: str,

59 *,

60 + truncation: bool | None = None,

61 add_special_tokens: bool = True,

62 ) -> list[int]:

63 """Encode text while caching repeated tokenizer calls."""

63 - return tokenizer.encode(text, add_special_tokens=add_special_tokens)

64 + return tokenizer.encode(

65 + text,

66 + truncation=truncation,

67 + add_special_tokens=add_special_tokens,

68 + )

69

70

71 class _GetMatchIndex(Protocol):

[MM] Cache common token sequences (#53560)