@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cd7b6f0857
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[VLM] Avoid unnecessary tokenization (#12310)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 22, 2025 at 19:08 UTC cd7b6f0857c6622d6b98eae6d6526dfe8dfdec70
9 files changed +71 -40
vllm/model_executor/models/blip2.py
+10 -2
@@ -475,15 +475,23 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):
475 hf_processor_mm_kwargs: Mapping[str, object],
476 out_mm_kwargs: MultiModalKwargs,
477 ) -> list[PromptReplacement]:
478 + tokenizer = self.info.get_tokenizer()
479 + vocab = tokenizer.get_vocab()
480 +
481 + bos_token_id = tokenizer.bos_token_id
482 + assert isinstance(bos_token_id, int)
483 +
484 + image_token_id = vocab["image"]
485 num_image_tokens = self.info.get_num_image_tokens()
486 + image_tokens = [image_token_id] * num_image_tokens
487
488 return [
489 PromptReplacement(
490 modality="image",
491 target="</s>",
492 replacement=PromptReplacementDetails(
485 - full="<image>" * num_image_tokens + "</s>",
486 - features="<image>" * num_image_tokens,
493 + full=image_tokens + [bos_token_id],
494 + features=image_tokens,
495 ),
496 )
497 ]
vllm/model_executor/models/chameleon.py
+14 -9
@@ -122,8 +122,9 @@ class ChameleonMultiModalProcessor(
122 ) -> list[int]:
123 # HF processor adds sep token for chat mode
124 tokenizer = self.info.get_tokenizer()
125 - sep_token_id: int = \
126 - tokenizer.vocab[tokenizer.sep_token] # type: ignore
125 + vocab = tokenizer.get_vocab()
126 +
127 + sep_token_id = vocab[tokenizer.sep_token] # type: ignore
128
129 return prompt_tokens + [sep_token_id]
130
@@ -141,18 +142,22 @@ class ChameleonMultiModalProcessor(
142 out_mm_kwargs: MultiModalKwargs,
143 ) -> list[PromptReplacement]:
144 processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
144 - image_tokens = processor.image_token * self.info.get_num_image_tokens()
145 + tokenizer = self.info.get_tokenizer()
146 + vocab = tokenizer.get_vocab()
147 +
148 + image_start_id = vocab[processor.image_start_token]
149 + image_token_id = vocab[processor.image_token]
150 + image_end_id = vocab[processor.image_end_token]
151 +
152 + num_image_tokens = self.info.get_num_image_tokens()
153 + image_tokens = [image_token_id] * num_image_tokens
154
155 return [
156 PromptReplacement(
157 modality="image",
149 - target="<image>",
158 + target=[image_token_id],
159 replacement=PromptReplacementDetails(
151 - full="".join([
152 - processor.image_start_token,
153 - image_tokens,
154 - processor.image_end_token,
155 - ]),
160 + full=([image_start_id] + image_tokens + [image_end_id]),
161 features=image_tokens,
162 ),
163 )
vllm/model_executor/models/deepseek_vl2.py
+4 -2
@@ -249,8 +249,10 @@ class DeepseekVL2MultiModalProcessor(
249 hf_processor_mm_kwargs: Mapping[str, object],
250 out_mm_kwargs: MultiModalKwargs,
251 ) -> list[PromptReplacement]:
252 - hf_processor = self.info.get_hf_processor()
253 - image_token_id: int = hf_processor.image_token_id
252 + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
253 +
254 + image_token_id = hf_processor.image_token_id
255 + assert isinstance(image_token_id, int)
256
257 def get_replacement_deepseek_vl2(item_idx: int):
258 images = mm_items.get_items(
vllm/model_executor/models/fuyu.py
+4 -1
@@ -183,7 +183,9 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]):
183 ) -> list[int]:
184 # HF processor adds boa_token_id
185 tokenizer = self.info.get_tokenizer()
186 - boa_token_id: int = tokenizer.vocab["<0x04>"] # type: ignore
186 + vocab = tokenizer.get_vocab()
187 +
188 + boa_token_id = vocab["<0x04>"]
189
190 return prompt_tokens + [boa_token_id]
191
@@ -202,6 +204,7 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]):
204 ) -> list[PromptReplacement]:
205 hf_config = self.info.get_hf_config()
206 bos_token_id = hf_config.bos_token_id
207 + assert isinstance(bos_token_id, int)
208
209 tokenizer = self.info.get_tokenizer()
210 eot_token_id = tokenizer.bos_token_id
vllm/model_executor/models/llava.py
+9 -8
@@ -315,13 +315,14 @@ class PixtralHFMultiModalProcessor(
315 hf_processor_mm_kwargs: Mapping[str, object],
316 out_mm_kwargs: MultiModalKwargs,
317 ) -> list[PromptReplacement]:
318 + processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
319 hf_config = self.info.get_hf_config()
319 - image_token_id = hf_config.image_token_index
320 + tokenizer = self.info.get_tokenizer()
321 + vocab = tokenizer.get_vocab()
322
321 - processor = self.info.get_hf_processor()
322 - image_token = processor.image_token
323 - image_break_token = processor.image_break_token
324 - image_end_token = processor.image_end_token
323 + image_break_id = vocab[processor.image_break_token]
324 + image_token_id = hf_config.image_token_index
325 + image_end_id = vocab[processor.image_end_token]
326
327 vision_config = hf_config.vision_config
328 assert isinstance(vision_config, PixtralVisionConfig)
@@ -336,10 +337,10 @@ class PixtralHFMultiModalProcessor(
337 image_height=image_size.height,
338 )
339
339 - tokens = ([image_token] * ncols + [image_break_token]) * nrows
340 - tokens[-1] = image_end_token
340 + tokens = ([image_token_id] * ncols + [image_break_id]) * nrows
341 + tokens[-1] = image_end_id
342
342 - return "".join(tokens)
343 + return tokens
344
345 return [
346 PromptReplacement(
vllm/model_executor/models/qwen2_audio.py
+11 -9
@@ -188,7 +188,9 @@ class Qwen2AudioMultiModalProcessor(
188 hf_processor_mm_kwargs: Mapping[str, object],
189 out_mm_kwargs: MultiModalKwargs,
190 ) -> list[PromptReplacement]:
191 - processor = self.info.get_hf_processor()
191 + processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
192 + tokenizer = self.info.get_tokenizer()
193 + vocab = tokenizer.get_vocab()
194
195 # Use getattr with default to be compatible with transformers<4.48
196 audio_token = getattr(processor, "audio_token", "<|AUDIO|>")
@@ -197,6 +199,10 @@ class Qwen2AudioMultiModalProcessor(
199 audio_eos_token = getattr(processor, "audio_eos_token",
200 "<|audio_eos|>")
201
202 + audio_token_id = vocab[audio_token]
203 + audio_bos_id = vocab[audio_bos_token]
204 + audio_eos_id = vocab[audio_eos_token]
205 +
206 feature_attention_mask = out_mm_kwargs.get("feature_attention_mask")
207 if feature_attention_mask is None:
208 audio_output_lengths = []
@@ -208,22 +214,18 @@ class Qwen2AudioMultiModalProcessor(
214 audio_output_lengths = audio_output_lens.tolist()
215
216 def get_replacement_qwen2_audio(item_idx: int):
211 - num_placeholders = audio_output_lengths[item_idx]
212 - if num_placeholders == 0:
217 + num_features = audio_output_lengths[item_idx]
218 + if num_features == 0:
219 audios = mm_items.get_items("audio", AudioProcessorItems)
220 audio = audios.get(item_idx)
221 raise ValueError(
222 f"The audio {audio} (len={len(audio)}) is too short "
223 "to be represented inside the model")
224
219 - audio_tokens = audio_token * num_placeholders
225 + audio_tokens = [audio_token_id] * num_features
226
227 return PromptReplacementDetails(
222 - full="".join([
223 - audio_bos_token,
224 - audio_tokens,
225 - audio_eos_token,
226 - ]),
228 + full=[audio_bos_id] + audio_tokens + [audio_eos_id],
229 features=audio_tokens,
230 )
231
vllm/model_executor/models/qwen2_vl.py
+7 -5
@@ -953,12 +953,14 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo]
953 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
954 image_processor = self.info.get_image_processor(
955 **hf_processor_mm_kwargs)
956 + tokenizer = self.info.get_tokenizer()
957 + vocab = tokenizer.get_vocab()
958
959 # NOTE: Only Qwen2VLProcessor in transformers 4.47.0 has
960 # image_token and video_token registered
961 placeholder = {
960 - "image": hf_processor.image_token,
961 - "video": hf_processor.video_token,
962 + "image": vocab[hf_processor.image_token],
963 + "video": vocab[hf_processor.video_token],
964 }
965
966 merge_length = image_processor.merge_size**2
@@ -967,13 +969,13 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo]
969 grid_thw = out_mm_kwargs[f"{modality}_grid_thw"][item_idx]
970 assert isinstance(grid_thw, torch.Tensor)
971
970 - num_tokens = grid_thw.prod().item() // merge_length
971 - return placeholder[modality] * num_tokens
972 + num_tokens = int(grid_thw.prod()) // merge_length
973 + return [placeholder[modality]] * num_tokens
974
975 return [
976 PromptReplacement(
977 modality=modality,
976 - target=placeholder[modality],
978 + target=[placeholder[modality]],
979 replacement=partial(get_replacement_qwen2vl,
980 modality=modality),
981 ) for modality in ("image", "video")
vllm/model_executor/models/ultravox.py
+7 -3
@@ -205,16 +205,20 @@ class UltravoxMultiModalProcessor(
205 out_mm_kwargs: MultiModalKwargs,
206 ) -> list[PromptReplacement]:
207 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
208 - placeholder = hf_processor.audio_token_replacement # type: ignore
208 + tokenizer = self.info.get_tokenizer()
209 + vocab = tokenizer.get_vocab()
210 +
211 + replacement_id = vocab[
212 + hf_processor.audio_token_replacement] # type: ignore
213
214 def get_replacement_ultravox(item_idx: int):
215 audio_token_len = out_mm_kwargs["audio_token_len"][item_idx]
212 - return placeholder * audio_token_len
216 + return [replacement_id] * int(audio_token_len) # type: ignore
217
218 return [
219 PromptReplacement(
220 modality="audio",
217 - target="<|audio|>",
221 + target='<|audio|>',
222 replacement=get_replacement_ultravox,
223 )
224 ]
vllm/transformers_utils/tokenizer.py
+5 -1
@@ -67,9 +67,10 @@ def get_cached_tokenizer(tokenizer: AnyTokenizer) -> AnyTokenizer:
67 tokenizer_all_special_tokens_extended = (
68 tokenizer.all_special_tokens_extended)
69 tokenizer_all_special_tokens = set(tokenizer.all_special_tokens)
70 + tokenizer_vocab = tokenizer.get_vocab()
71 tokenizer_len = len(tokenizer)
72
72 - max_token_id = max(tokenizer.get_vocab().values())
73 + max_token_id = max(tokenizer_vocab.values())
74 # Some tokenizers (e.g., QwenTokenizer) have special tokens that
75 # are added and included in the implementation of the vocab_size
76 # property, but not in get_vocab(); if there is an implementation
@@ -96,6 +97,9 @@ def get_cached_tokenizer(tokenizer: AnyTokenizer) -> AnyTokenizer:
97 def max_token_id(self):
98 return max_token_id
99
100 + def get_vocab(self):
101 + return tokenizer_vocab
102 +
103 def __len__(self):
104 return tokenizer_len
105