A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[VLM] Avoid unnecessary tokenization (#12310)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 22, 2025 at 19:08 UTC
cd7b6f0857c6622d6b98eae6d6526dfe8dfdec70
9 files changed
+71
-40
vllm/model_executor/models/blip2.py
+10
-2
@@ -475,15 +475,23 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):
475
hf_processor_mm_kwargs: Mapping[str, object],
476
out_mm_kwargs: MultiModalKwargs,
477
) -> list[PromptReplacement]:
478
+ tokenizer = self.info.get_tokenizer()
479
+ vocab = tokenizer.get_vocab()
480
+
481
+ bos_token_id = tokenizer.bos_token_id
482
+ assert isinstance(bos_token_id, int)
483
+
484
+ image_token_id = vocab["image"]
485
num_image_tokens = self.info.get_num_image_tokens()
486
+ image_tokens = [image_token_id] * num_image_tokens
487
488
return [
489
PromptReplacement(
490
modality="image",
491
target="</s>",
492
replacement=PromptReplacementDetails(
485
- full="<image>" * num_image_tokens + "</s>",
486
- features="<image>" * num_image_tokens,
493
+ full=image_tokens + [bos_token_id],
494
+ features=image_tokens,
495
),
496
)
497
]
vllm/model_executor/models/chameleon.py
+14
-9
@@ -122,8 +122,9 @@ class ChameleonMultiModalProcessor(
122
) -> list[int]:
123
# HF processor adds sep token for chat mode
124
tokenizer = self.info.get_tokenizer()
125
- sep_token_id: int = \
126
- tokenizer.vocab[tokenizer.sep_token] # type: ignore
125
+ vocab = tokenizer.get_vocab()
126
+
127
+ sep_token_id = vocab[tokenizer.sep_token] # type: ignore
128
129
return prompt_tokens + [sep_token_id]
130
@@ -141,18 +142,22 @@ class ChameleonMultiModalProcessor(
142
out_mm_kwargs: MultiModalKwargs,
143
) -> list[PromptReplacement]:
144
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
144
- image_tokens = processor.image_token * self.info.get_num_image_tokens()
145
+ tokenizer = self.info.get_tokenizer()
146
+ vocab = tokenizer.get_vocab()
147
+
148
+ image_start_id = vocab[processor.image_start_token]
149
+ image_token_id = vocab[processor.image_token]
150
+ image_end_id = vocab[processor.image_end_token]
151
+
152
+ num_image_tokens = self.info.get_num_image_tokens()
153
+ image_tokens = [image_token_id] * num_image_tokens
154
155
return [
156
PromptReplacement(
157
modality="image",
149
- target="<image>",
158
+ target=[image_token_id],
159
replacement=PromptReplacementDetails(
151
- full="".join([
152
- processor.image_start_token,
153
- image_tokens,
154
- processor.image_end_token,
155
- ]),
160
+ full=([image_start_id] + image_tokens + [image_end_id]),
161
features=image_tokens,
162
),
163
)
vllm/model_executor/models/deepseek_vl2.py
+4
-2
@@ -249,8 +249,10 @@ class DeepseekVL2MultiModalProcessor(
249
hf_processor_mm_kwargs: Mapping[str, object],
250
out_mm_kwargs: MultiModalKwargs,
251
) -> list[PromptReplacement]:
252
- hf_processor = self.info.get_hf_processor()
253
- image_token_id: int = hf_processor.image_token_id
252
+ hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
253
+
254
+ image_token_id = hf_processor.image_token_id
255
+ assert isinstance(image_token_id, int)
256
257
def get_replacement_deepseek_vl2(item_idx: int):
258
images = mm_items.get_items(
vllm/model_executor/models/fuyu.py
+4
-1
@@ -183,7 +183,9 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]):
183
) -> list[int]:
184
# HF processor adds boa_token_id
185
tokenizer = self.info.get_tokenizer()
186
- boa_token_id: int = tokenizer.vocab["<0x04>"] # type: ignore
186
+ vocab = tokenizer.get_vocab()
187
+
188
+ boa_token_id = vocab["<0x04>"]
189
190
return prompt_tokens + [boa_token_id]
191
@@ -202,6 +204,7 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]):
204
) -> list[PromptReplacement]:
205
hf_config = self.info.get_hf_config()
206
bos_token_id = hf_config.bos_token_id
207
+ assert isinstance(bos_token_id, int)
208
209
tokenizer = self.info.get_tokenizer()
210
eot_token_id = tokenizer.bos_token_id
vllm/model_executor/models/llava.py
+9
-8
@@ -315,13 +315,14 @@ class PixtralHFMultiModalProcessor(
315
hf_processor_mm_kwargs: Mapping[str, object],
316
out_mm_kwargs: MultiModalKwargs,
317
) -> list[PromptReplacement]:
318
+ processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
319
hf_config = self.info.get_hf_config()
319
- image_token_id = hf_config.image_token_index
320
+ tokenizer = self.info.get_tokenizer()
321
+ vocab = tokenizer.get_vocab()
322
321
- processor = self.info.get_hf_processor()
322
- image_token = processor.image_token
323
- image_break_token = processor.image_break_token
324
- image_end_token = processor.image_end_token
323
+ image_break_id = vocab[processor.image_break_token]
324
+ image_token_id = hf_config.image_token_index
325
+ image_end_id = vocab[processor.image_end_token]
326
327
vision_config = hf_config.vision_config
328
assert isinstance(vision_config, PixtralVisionConfig)
@@ -336,10 +337,10 @@ class PixtralHFMultiModalProcessor(
337
image_height=image_size.height,
338
)
339
339
- tokens = ([image_token] * ncols + [image_break_token]) * nrows
340
- tokens[-1] = image_end_token
340
+ tokens = ([image_token_id] * ncols + [image_break_id]) * nrows
341
+ tokens[-1] = image_end_id
342
342
- return "".join(tokens)
343
+ return tokens
344
345
return [
346
PromptReplacement(
vllm/model_executor/models/qwen2_audio.py
+11
-9
@@ -188,7 +188,9 @@ class Qwen2AudioMultiModalProcessor(
188
hf_processor_mm_kwargs: Mapping[str, object],
189
out_mm_kwargs: MultiModalKwargs,
190
) -> list[PromptReplacement]:
191
- processor = self.info.get_hf_processor()
191
+ processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
192
+ tokenizer = self.info.get_tokenizer()
193
+ vocab = tokenizer.get_vocab()
194
195
# Use getattr with default to be compatible with transformers<4.48
196
audio_token = getattr(processor, "audio_token", "<|AUDIO|>")
@@ -197,6 +199,10 @@ class Qwen2AudioMultiModalProcessor(
199
audio_eos_token = getattr(processor, "audio_eos_token",
200
"<|audio_eos|>")
201
202
+ audio_token_id = vocab[audio_token]
203
+ audio_bos_id = vocab[audio_bos_token]
204
+ audio_eos_id = vocab[audio_eos_token]
205
+
206
feature_attention_mask = out_mm_kwargs.get("feature_attention_mask")
207
if feature_attention_mask is None:
208
audio_output_lengths = []
@@ -208,22 +214,18 @@ class Qwen2AudioMultiModalProcessor(
214
audio_output_lengths = audio_output_lens.tolist()
215
216
def get_replacement_qwen2_audio(item_idx: int):
211
- num_placeholders = audio_output_lengths[item_idx]
212
- if num_placeholders == 0:
217
+ num_features = audio_output_lengths[item_idx]
218
+ if num_features == 0:
219
audios = mm_items.get_items("audio", AudioProcessorItems)
220
audio = audios.get(item_idx)
221
raise ValueError(
222
f"The audio {audio} (len={len(audio)}) is too short "
223
"to be represented inside the model")
224
219
- audio_tokens = audio_token * num_placeholders
225
+ audio_tokens = [audio_token_id] * num_features
226
227
return PromptReplacementDetails(
222
- full="".join([
223
- audio_bos_token,
224
- audio_tokens,
225
- audio_eos_token,
226
- ]),
228
+ full=[audio_bos_id] + audio_tokens + [audio_eos_id],
229
features=audio_tokens,
230
)
231
vllm/model_executor/models/qwen2_vl.py
+7
-5
@@ -953,12 +953,14 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo]
953
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
954
image_processor = self.info.get_image_processor(
955
**hf_processor_mm_kwargs)
956
+ tokenizer = self.info.get_tokenizer()
957
+ vocab = tokenizer.get_vocab()
958
959
# NOTE: Only Qwen2VLProcessor in transformers 4.47.0 has
960
# image_token and video_token registered
961
placeholder = {
960
- "image": hf_processor.image_token,
961
- "video": hf_processor.video_token,
962
+ "image": vocab[hf_processor.image_token],
963
+ "video": vocab[hf_processor.video_token],
964
}
965
966
merge_length = image_processor.merge_size**2
@@ -967,13 +969,13 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo]
969
grid_thw = out_mm_kwargs[f"{modality}_grid_thw"][item_idx]
970
assert isinstance(grid_thw, torch.Tensor)
971
970
- num_tokens = grid_thw.prod().item() // merge_length
971
- return placeholder[modality] * num_tokens
972
+ num_tokens = int(grid_thw.prod()) // merge_length
973
+ return [placeholder[modality]] * num_tokens
974
975
return [
976
PromptReplacement(
977
modality=modality,
976
- target=placeholder[modality],
978
+ target=[placeholder[modality]],
979
replacement=partial(get_replacement_qwen2vl,
980
modality=modality),
981
) for modality in ("image", "video")
vllm/model_executor/models/ultravox.py
+7
-3
@@ -205,16 +205,20 @@ class UltravoxMultiModalProcessor(
205
out_mm_kwargs: MultiModalKwargs,
206
) -> list[PromptReplacement]:
207
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
208
- placeholder = hf_processor.audio_token_replacement # type: ignore
208
+ tokenizer = self.info.get_tokenizer()
209
+ vocab = tokenizer.get_vocab()
210
+
211
+ replacement_id = vocab[
212
+ hf_processor.audio_token_replacement] # type: ignore
213
214
def get_replacement_ultravox(item_idx: int):
215
audio_token_len = out_mm_kwargs["audio_token_len"][item_idx]
212
- return placeholder * audio_token_len
216
+ return [replacement_id] * int(audio_token_len) # type: ignore
217
218
return [
219
PromptReplacement(
220
modality="audio",
217
- target="<|audio|>",
221
+ target='<|audio|>',
222
replacement=get_replacement_ultravox,
223
)
224
]
vllm/transformers_utils/tokenizer.py
+5
-1
@@ -67,9 +67,10 @@ def get_cached_tokenizer(tokenizer: AnyTokenizer) -> AnyTokenizer:
67
tokenizer_all_special_tokens_extended = (
68
tokenizer.all_special_tokens_extended)
69
tokenizer_all_special_tokens = set(tokenizer.all_special_tokens)
70
+ tokenizer_vocab = tokenizer.get_vocab()
71
tokenizer_len = len(tokenizer)
72
72
- max_token_id = max(tokenizer.get_vocab().values())
73
+ max_token_id = max(tokenizer_vocab.values())
74
# Some tokenizers (e.g., QwenTokenizer) have special tokens that
75
# are added and included in the implementation of the vocab_size
76
# property, but not in get_vocab(); if there is an implementation
@@ -96,6 +97,9 @@ def get_cached_tokenizer(tokenizer: AnyTokenizer) -> AnyTokenizer:
97
def max_token_id(self):
98
return max_token_id
99
100
+ def get_vocab(self):
101
+ return tokenizer_vocab
102
+
103
def __len__(self):
104
return tokenizer_len
105