A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[VLM] Implement merged multimodal processor for Mllama (#11427)
Isotr0py committed
Feb 13, 2025 at 12:26 UTC
bc55d13070a9f3f2b8524901817d406cb8a37a3b
8 files changed
+456
-233
tests/models/encoder_decoder/vision_language/test_mllama.py
+67
-4
@@ -7,11 +7,11 @@ import torch
7
from transformers import (AutoConfig, AutoModelForVision2Seq, AutoTokenizer,
8
BatchEncoding)
9
10
+from vllm import LLM, SamplingParams
11
from vllm.attention.backends.flash_attn import FlashAttentionMetadata
12
from vllm.attention.selector import (_Backend, _cached_get_attn_backend,
13
global_force_attn_backend_context_manager)
13
-from vllm.model_executor.models.mllama import (MLLAMA_IMAGE_TOKEN_ID,
14
- MllamaForConditionalGeneration)
14
+from vllm.model_executor.models.mllama import MllamaForConditionalGeneration
15
from vllm.multimodal.image import rescale_image_size
16
from vllm.sequence import SampleLogprobs
17
@@ -21,6 +21,7 @@ from ....utils import large_gpu_test
21
from ...utils import check_logprobs_close
22
23
_LIMIT_IMAGE_PER_PROMPT = 3
24
+MLLAMA_IMAGE_TOKEN_ID = 128256
25
26
LIST_ENC_DEC_SUPPORTED_BACKENDS = [_Backend.XFORMERS, _Backend.FLASH_ATTN]
27
@@ -396,6 +397,64 @@ def test_models_interleaved_images(hf_runner, vllm_runner, image_assets, model,
397
)
398
399
400
+@large_gpu_test(min_gb=48)
401
+@pytest.mark.core_model
402
+@pytest.mark.parametrize("model", models)
403
+@pytest.mark.parametrize("dtype", ["bfloat16"])
404
+@pytest.mark.parametrize("max_tokens", [32])
405
+def test_explicit_implicit_prompt(
406
+ image_assets: _ImageAssets,
407
+ model: str,
408
+ dtype: str,
409
+ max_tokens: int,
410
+):
411
+ stop_sign = image_assets[0].pil_image
412
+ # yapf: disable
413
+ prompts = [
414
+ # explicit prompt
415
+ {
416
+ "encoder_prompt": {
417
+ "prompt": "<|image|>",
418
+ "multi_modal_data": {"image": stop_sign},
419
+ },
420
+ "decoder_prompt": {
421
+ "prompt_token_ids": [128000, 791, 2262, 315, 279, 2217, 220, 128256, 374], # noqa: E501
422
+ }
423
+ },
424
+ {
425
+ "encoder_prompt": "Not <|image|>",
426
+ "decoder_prompt": "The color of the sky is blue but sometimes it can also be", # noqa: E501
427
+ },
428
+ # implicit prompt
429
+ {
430
+ "prompt": "<|begin_of_text|>The content of the image <|image|> is", # noqa: E501
431
+ "multi_modal_data": {"image": stop_sign},
432
+ },
433
+ {
434
+ "prompt": "The color of the sky is blue but sometimes it can also be", # noqa: E501
435
+ },
436
+ ]
437
+ # yapf: enable
438
+ llm = LLM(
439
+ model=model,
440
+ dtype=dtype,
441
+ max_model_len=4096,
442
+ max_num_seqs=2,
443
+ tensor_parallel_size=1,
444
+ enforce_eager=True,
445
+ )
446
+ sampling_params = SamplingParams(
447
+ temperature=0,
448
+ max_tokens=max_tokens,
449
+ )
450
+ outputs = llm.generate(prompts, sampling_params)
451
+ n_prompts = len(prompts)
452
+ explicit_outputs = outputs[:n_prompts // 2]
453
+ implicit_outputs = outputs[n_prompts // 2:]
454
+ for exp_output, imp_output in zip(explicit_outputs, implicit_outputs):
455
+ assert exp_output.outputs[0].text == imp_output.outputs[0].text
456
+
457
+
458
@large_gpu_test(min_gb=48)
459
@pytest.mark.core_model
460
@pytest.mark.parametrize("model", models)
@@ -458,6 +517,10 @@ def test_regression(vllm_runner, image_assets, model, dtype, max_tokens,
517
images=images)
518
519
520
+class DummyModel:
521
+ image_token_id = MLLAMA_IMAGE_TOKEN_ID
522
+
523
+
524
@pytest.mark.core_model
525
@pytest.mark.parametrize(
526
"input_indices_and_output",
@@ -499,7 +562,7 @@ def test_get_cross_attention_mask(input_indices_and_output) -> None:
562
use_cuda_graph=False,
563
)
564
502
- dummy: dict[str, str] = {}
565
+ dummy = DummyModel()
566
567
cross_attention_mask, kv_range_for_decode = MllamaForConditionalGeneration\
568
.get_cross_attention_mask(dummy,
@@ -556,7 +619,7 @@ def test_get_full_text_row_masked_out_mask(input_indices) -> None:
619
use_cuda_graph=False,
620
)
621
559
- dummy: dict[str, str] = {}
622
+ dummy = DummyModel()
623
624
full_text_row_masked_out_mask = MllamaForConditionalGeneration\
625
.get_full_text_row_masked_out_mask(dummy,
tests/models/multimodal/processing/test_common.py
+11
-2
@@ -85,6 +85,14 @@ def _test_processing_correctness(
85
partial(random_audio, rng, min_len=512, max_len=1024, sr=16000),
86
}
87
88
+ tokenizer_encode_kwargs = {}
89
+ if model_config.hf_config.model_type == "mllama":
90
+ # For Mllama, tokenizer will always add bos_token at the beginning of
91
+ # prompt by default, causing hf_processor outputs incorrect token ids.
92
+ # So we need use `add_special_tokens=False` here to leave bos_token
93
+ # to be added by the processor.
94
+ tokenizer_encode_kwargs = {"add_special_tokens": False}
95
+
96
for batch_idx in range(num_batches):
97
mm_data = {
98
k:
@@ -122,7 +130,7 @@ def _test_processing_correctness(
130
f"Failed ({batch_idx=}, {prompt=}, {mm_data=})")
131
132
baseline_tokenized_result = baseline_processor.apply(
125
- tokenizer.encode(prompt),
133
+ tokenizer.encode(prompt, **tokenizer_encode_kwargs),
134
mm_data=mm_data,
135
hf_processor_mm_kwargs={},
136
)
@@ -131,7 +139,7 @@ def _test_processing_correctness(
139
f"Failed ({batch_idx=}, {prompt=}, {mm_data=})")
140
141
cached_tokenized_result = cached_processor.apply(
134
- tokenizer.encode(prompt),
142
+ tokenizer.encode(prompt, **tokenizer_encode_kwargs),
143
mm_data=mm_data,
144
hf_processor_mm_kwargs={},
145
)
@@ -155,6 +163,7 @@ def _test_processing_correctness(
163
"llava-hf/llava-v1.6-mistral-7b-hf",
164
"llava-hf/LLaVA-NeXT-Video-7B-hf",
165
"llava-hf/llava-onevision-qwen2-0.5b-ov-hf",
166
+ "meta-llama/Llama-3.2-11B-Vision-Instruct",
167
"TIGER-Lab/Mantis-8B-siglip-llama3",
168
"mistral-community/pixtral-12b",
169
"openbmb/MiniCPM-o-2_6",
vllm/inputs/preprocess.py
+83
-7
@@ -1,7 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
import asyncio
4
-from typing import List, Mapping, Optional, Union
4
+from typing import List, Mapping, Optional, Tuple, Union, cast
5
6
from typing_extensions import assert_never
7
@@ -9,7 +9,8 @@ from vllm.config import ModelConfig
9
from vllm.logger import init_logger
10
from vllm.lora.request import LoRARequest
11
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
12
-from vllm.multimodal.inputs import MultiModalDataDict, MultiModalInputs
12
+from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalEncDecInputs,
13
+ MultiModalInputs)
14
from vllm.prompt_adapter.request import PromptAdapterRequest
15
from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
16
@@ -495,6 +496,51 @@ class InputPreprocessor:
496
decoder=decoder_inputs,
497
)
498
499
+ def _separate_enc_dec_inputs_from_mm_processor_outputs(
500
+ self,
501
+ inputs: SingletonInputs,
502
+ decoder_inputs_to_override: Optional[SingletonInputs] = None,
503
+ ) -> Tuple[SingletonInputs, SingletonInputs]:
504
+ """
505
+ For encoder/decoder models only:
506
+ Separate Encoder/Decoder inputs from a MultiModalEncDecInputs
507
+ """
508
+ encoder_inputs: SingletonInputs
509
+ decoder_inputs: SingletonInputs
510
+ if inputs["type"] == "multimodal":
511
+ # Multimodal data inputs
512
+ assert ("encoder_prompt" in inputs
513
+ and "encoder_prompt_token_ids" in inputs)
514
+ inputs = cast(MultiModalEncDecInputs, inputs)
515
+ encoder_inputs = token_inputs(
516
+ prompt=inputs["encoder_prompt"],
517
+ prompt_token_ids=inputs["encoder_prompt_token_ids"],
518
+ )
519
+ if decoder_inputs_to_override is not None:
520
+ decoder_inputs = MultiModalInputs(
521
+ type="multimodal",
522
+ prompt=decoder_inputs_to_override.get("prompt", ""),
523
+ prompt_token_ids=decoder_inputs_to_override[
524
+ "prompt_token_ids"],
525
+ mm_kwargs=inputs["mm_kwargs"],
526
+ mm_placeholders=inputs["mm_placeholders"],
527
+ )
528
+ else:
529
+ decoder_inputs = MultiModalInputs(
530
+ type="multimodal",
531
+ prompt=inputs["prompt"],
532
+ prompt_token_ids=inputs["prompt_token_ids"],
533
+ mm_kwargs=inputs["mm_kwargs"],
534
+ mm_placeholders=inputs["mm_placeholders"],
535
+ )
536
+ elif inputs["type"] == "token":
537
+ # Text-only inputs
538
+ encoder_inputs = token_inputs(prompt="", prompt_token_ids=[])
539
+ decoder_inputs = decoder_inputs_to_override or inputs
540
+ else:
541
+ assert_never(inputs) # type: ignore[arg-type]
542
+ return encoder_inputs, decoder_inputs
543
+
544
def _process_encoder_decoder_prompt(
545
self,
546
prompt: PromptType,
@@ -539,7 +585,6 @@ class InputPreprocessor:
585
prompt["encoder_prompt"],
586
request_id=request_id,
587
)
542
-
588
if (decoder_input := prompt["decoder_prompt"]) is None:
589
decoder_inputs = None
590
else:
@@ -547,13 +592,28 @@ class InputPreprocessor:
592
decoder_input,
593
request_id=request_id,
594
)
595
+ # For multimodal model, override decoder prompt from processor
596
+ # with explicit decoder prompt.
597
+ if self.model_config.is_multimodal_model and (
598
+ self._can_process_multimodal()):
599
+ encoder_inputs, decoder_inputs = (
600
+ self._separate_enc_dec_inputs_from_mm_processor_outputs(
601
+ encoder_inputs, decoder_inputs))
602
else:
551
- encoder_inputs = self._prompt_to_llm_inputs(
603
+ inputs = self._prompt_to_llm_inputs(
604
prompt,
605
request_id=request_id,
606
)
607
+ if self.model_config.is_multimodal_model and (
608
+ self._can_process_multimodal()):
609
+ # Encoder-Decoder Multimodal model
610
+ encoder_inputs, decoder_inputs = (
611
+ self._separate_enc_dec_inputs_from_mm_processor_outputs(
612
+ inputs))
613
+ else:
614
+ encoder_inputs = inputs
615
556
- decoder_inputs = None
616
+ decoder_inputs = None
617
618
return self._build_enc_dec_llm_inputs(encoder_inputs, decoder_inputs)
619
@@ -583,13 +643,29 @@ class InputPreprocessor:
643
644
encoder_inputs, decoder_inputs = await asyncio.gather(
645
encoder_task, decoder_task)
646
+
647
+ # For multimodal model, override decoder prompt from processor
648
+ # with explicit decoder prompt.
649
+ if self.model_config.is_multimodal_model and (
650
+ self._can_process_multimodal()):
651
+ encoder_inputs, decoder_inputs = (
652
+ self._separate_enc_dec_inputs_from_mm_processor_outputs(
653
+ encoder_inputs, decoder_inputs))
654
else:
587
- encoder_inputs = await self._prompt_to_llm_inputs_async(
655
+ inputs = await self._prompt_to_llm_inputs_async(
656
prompt,
657
request_id=request_id,
658
)
659
+ if self.model_config.is_multimodal_model and (
660
+ self._can_process_multimodal()):
661
+ # Encoder-Decoder Multimodal model
662
+ encoder_inputs, decoder_inputs = (
663
+ self._separate_enc_dec_inputs_from_mm_processor_outputs(
664
+ inputs))
665
+ else:
666
+ encoder_inputs = inputs
667
592
- decoder_inputs = None
668
+ decoder_inputs = None
669
670
return self._build_enc_dec_llm_inputs(encoder_inputs, decoder_inputs)
671
vllm/inputs/registry.py
+2
-1
@@ -350,7 +350,8 @@ class InputRegistry:
350
)
351
processor = mm_registry.create_processor(model_config, tokenizer)
352
profiler = MultiModalProfiler(processor)
353
- dummy_data = profiler.get_dummy_data(seq_len)
353
+ dummy_data = profiler.get_dummy_data(
354
+ seq_len, is_encoder_data=is_encoder_data)
355
else:
356
model_cls, _ = get_model_architecture(model_config)
357
if is_encoder_data:
vllm/model_executor/models/mllama.py
+203
-205
@@ -23,14 +23,15 @@ import torch
23
import torch.nn.functional as F
24
import torch.utils.checkpoint
25
import transformers.models.mllama.configuration_mllama as config_mllama
26
-from PIL import Image
26
+from PIL.Image import Image
27
from torch import nn
28
+from transformers import BatchFeature, MllamaConfig
29
from transformers.modeling_outputs import (BaseModelOutput,
30
CausalLMOutputWithPast)
31
from transformers.models.mllama.image_processing_mllama import (
32
get_optimal_tiled_canvas)
33
from transformers.models.mllama.processing_mllama import (
33
- get_cross_attention_token_mask)
34
+ MllamaProcessor, get_cross_attention_token_mask)
35
36
import vllm.distributed.parallel_state as ps
37
from vllm.attention import Attention, AttentionMetadata, AttentionType
@@ -38,8 +39,6 @@ from vllm.attention.ops.paged_attn import PagedAttention
39
from vllm.attention.selector import _Backend
40
from vllm.config import VllmConfig
41
from vllm.distributed import get_tensor_model_parallel_world_size
41
-from vllm.inputs import (INPUT_REGISTRY, DummyData, EncoderDecoderInputs,
42
- InputContext, TokenInputs, token_inputs)
42
from vllm.logger import init_logger
43
from vllm.model_executor.layers.layernorm import RMSNorm
44
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -54,8 +53,13 @@ from vllm.model_executor.model_loader.weight_utils import (
53
default_weight_loader, maybe_remap_kv_scale_name)
54
from vllm.model_executor.sampling_metadata import SamplingMetadata
55
from vllm.multimodal import MULTIMODAL_REGISTRY
57
-from vllm.sequence import SequenceData
58
-from vllm.utils import is_list_of
56
+from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs
57
+from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
58
+ MultiModalDataDict, MultiModalDataItems)
59
+from vllm.multimodal.processing import (BaseProcessingInfo,
60
+ EncDecMultiModalProcessor,
61
+ PromptReplacement)
62
+from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
63
64
from .clip import CLIPMLP
65
from .interfaces import SupportsMultiModal
@@ -63,8 +67,6 @@ from .llama import LlamaDecoderLayer, LlamaMLP
67
from .utils import maybe_prefix
68
69
logger = init_logger(__name__)
66
-MLLAMA_IMAGE_TOKEN_ID = 128256
67
-MLLAMA_IMAGE_TOKEN = "<|image|>"
70
71
72
class MllamaImagePixelInputs(TypedDict):
@@ -81,158 +83,191 @@ class MllamaImagePixelInputs(TypedDict):
83
# TODO: support LlamaImageEmbeddingInputs
84
85
84
-def _get_num_image_in_last_group(prompt_token_ids: List[int]) -> int:
85
- num_images = 0
86
- for token_id in prompt_token_ids[::-1]:
87
- if token_id == MLLAMA_IMAGE_TOKEN_ID:
88
- num_images += 1
89
- elif num_images > 0:
90
- break
91
- return num_images
92
-
93
-
94
-def input_processor_for_mllama(
95
- ctx: InputContext,
96
- inputs: EncoderDecoderInputs,
97
-) -> EncoderDecoderInputs:
98
- # Example input to processor:
99
- # {
100
- # 'encoder': {
101
- # 'type': 'token',
102
- # 'prompt_token_ids': [128000, 128256, 128000, 3923, 374, 279, 2262, 315, 420, 2217, 30], # noqa: E501
103
- # 'prompt': '<|image|><|begin_of_text|>What is the content of this image?', # noqa: E501
104
- # 'multi_modal_data': {'image': <PIL.Image.Image image mode=RGB size=1770x1180 at 0x7FDE2C624880>}, # noqa: E501
105
- # },
106
- # 'decoder': {
107
- # 'type': 'token',
108
- # 'prompt_token_ids': [128000],
109
- # },
110
- # }
111
-
112
- # move encoder prompt to decoder
113
- dec_inputs = TokenInputs(**inputs["encoder"])
114
-
115
- multi_modal_data = dec_inputs.get("multi_modal_data")
116
- if multi_modal_data is None or "image" not in multi_modal_data:
117
- # text-only
118
- return EncoderDecoderInputs(
119
- encoder=token_inputs([]),
120
- decoder=dec_inputs,
86
+def calc_token_per_chunk(image_size: int) -> int:
87
+ assert image_size % 14 == 0, "chunk size should be multiple of 14"
88
+ token_per_chunk = (image_size // 14)**2 + 1
89
+ return token_per_chunk
90
+
91
+
92
+class MllamaProcessingInfo(BaseProcessingInfo):
93
+
94
+ def get_hf_config(self) -> MllamaConfig:
95
+ return self.ctx.get_hf_config(MllamaConfig)
96
+
97
+ def get_hf_processor(self) -> MllamaProcessor:
98
+ return self.ctx.get_hf_processor(MllamaProcessor)
99
+
100
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
101
+ return {"image": None}
102
+
103
+ def get_token_per_chunk_from_config(self) -> int:
104
+ image_size = self.get_hf_config().vision_config.image_size
105
+ return calc_token_per_chunk(image_size)
106
+
107
+ def get_mm_max_tokens_per_item(
108
+ self,
109
+ seq_len: int,
110
+ mm_counts: Mapping[str, int],
111
+ ) -> Mapping[str, int]:
112
+ vision_config = self.get_hf_config().vision_config
113
+ token_per_chunk = self.get_token_per_chunk_from_config()
114
+ mm_max_tokens = vision_config.max_num_tiles * token_per_chunk
115
+ return {"image": mm_max_tokens}
116
+
117
+ def get_num_tiles_per_image(self, image_height: int,
118
+ image_width: int) -> int:
119
+ vision_config = self.get_hf_config().vision_config
120
+ max_num_tiles = vision_config.max_num_tiles
121
+ image_size = vision_config.image_size
122
+ tiled_height, tiled_width = get_optimal_tiled_canvas(
123
+ image_height,
124
+ image_width,
125
+ max_num_tiles,
126
+ tile_size=image_size,
127
+ )
128
+ num_tiles_height = tiled_height // image_size
129
+ num_tiles_width = tiled_width // image_size
130
+ return num_tiles_height * num_tiles_width
131
+
132
+ def get_image_size_with_most_features(self) -> ImageSize:
133
+ vision_config = self.get_hf_config().vision_config
134
+ image_size = vision_config.image_size
135
+ max_num_tiles = vision_config.max_num_tiles
136
+ # Result in the max possible feature size (h:w = 16:1)
137
+ return ImageSize(height=max_num_tiles * image_size, width=image_size)
138
+
139
+
140
+class MllamaDummyInputsBuilder(BaseDummyInputsBuilder[MllamaProcessingInfo]):
141
+
142
+ def get_dummy_processor_inputs(
143
+ self,
144
+ seq_len: int,
145
+ mm_counts: Mapping[str, int],
146
+ ) -> ProcessorInputs:
147
+ num_images = mm_counts.get("image", 0)
148
+
149
+ target_width, target_height = \
150
+ self.info.get_image_size_with_most_features()
151
+
152
+ mm_data = {
153
+ "image":
154
+ self._get_dummy_images(width=target_width,
155
+ height=target_height,
156
+ num_images=num_images)
157
+ }
158
+
159
+ hf_processor = self.info.get_hf_processor()
160
+ image_token: str = hf_processor.image_token
161
+
162
+ return ProcessorInputs(
163
+ prompt_text=image_token * num_images,
164
+ mm_data=mm_data,
165
)
166
123
- image_data = multi_modal_data["image"]
124
- if isinstance(image_data, Image.Image):
125
- image_data = [image_data]
126
-
127
- assert is_list_of(image_data, Image.Image)
128
-
129
- num_image_tokens = dec_inputs['prompt_token_ids'].count(
130
- MLLAMA_IMAGE_TOKEN_ID)
131
- if num_image_tokens != len(image_data):
132
- raise ValueError(
133
- f"The number of image tokens ({num_image_tokens}) must be"
134
- f" the same as the number of images ({len(image_data)})")
135
-
136
- # Since only the last group of consecutive images
137
- # are attended by the decoded tokens, we only need to
138
- # get the number of tiles for those images.
139
- num_decode_images = _get_num_image_in_last_group(
140
- dec_inputs["prompt_token_ids"])
141
-
142
- hf_config = ctx.model_config.hf_config
143
- vision_config = hf_config.vision_config
144
-
145
- num_tiles = 0
146
- for image in image_data[::-1]:
147
- width, height = image.size
148
- tile_size = vision_config.image_size
149
- canvas_height, canvas_width = get_optimal_tiled_canvas(
150
- image_height=height,
151
- image_width=width,
152
- max_image_tiles=vision_config.max_num_tiles,
153
- tile_size=tile_size,
167
+
168
+class MllamaMultiModalProcessor(EncDecMultiModalProcessor[MllamaProcessingInfo]
169
+ ):
170
+
171
+ def _call_hf_processor(
172
+ self,
173
+ prompt: str,
174
+ mm_data: Mapping[str, object],
175
+ mm_kwargs: Mapping[str, object],
176
+ ) -> BatchFeature:
177
+ tokenizer = self.info.get_tokenizer()
178
+ if mm_data:
179
+ num_tiles = [
180
+ self.info.get_num_tiles_per_image(img.height, img.width)
181
+ for img in mm_data["images"]
182
+ ]
183
+ processed_outputs = super()._call_hf_processor(
184
+ prompt, mm_data, mm_kwargs)
185
+ processed_outputs["num_tiles"] = torch.tensor(num_tiles)
186
+ for k in ('pixel_values', 'aspect_ratio_ids', "aspect_ratio_mask"):
187
+ processed_outputs[k] = processed_outputs[k].squeeze(0)
188
+ # Example input to encoder and decoder:
189
+ # {
190
+ # 'encoder': {
191
+ # 'type': 'token',
192
+ # 'prompt_token_ids': [128256, 128000, 3923, 374, 279, 2262, 315, 420, 2217, 30], # noqa: E501
193
+ # 'prompt': '<|image|><|begin_of_text|>What is the content of this image?', # noqa: E501
194
+ # 'multi_modal_data': {'image': <PIL.Image.Image image mode=RGB size=1770x1180 at 0x7FDE2C624880>}, # noqa: E501
195
+ # },
196
+ # 'decoder': {
197
+ # 'type': 'token',
198
+ # 'prompt_token_ids': [128000],
199
+ # },
200
+ # }
201
+ processed_token_ids = processed_outputs.pop("input_ids")
202
+ start_idx, end_idx = 0, processed_token_ids.size(1)
203
+ processed_prompt_text = tokenizer.decode(processed_token_ids[0])
204
+
205
+ hf_processor = self.info.get_hf_processor()
206
+ bos_token = hf_processor.bos_token
207
+ # Remove the bos_token from the start of prompt,
208
+ # because we all know there would be image_token.
209
+ if processed_prompt_text.startswith(bos_token):
210
+ start_idx += 1
211
+ # Remove the bos_token from the end of prompt,
212
+ # because text is empty in this case.
213
+ if processed_prompt_text.endswith(bos_token):
214
+ end_idx -= 1
215
+ processed_outputs[
216
+ "input_ids"] = processed_token_ids[:, start_idx:end_idx]
217
+ else:
218
+ processed_outputs = tokenizer(prompt,
219
+ add_special_tokens=False,
220
+ return_tensors="pt")
221
+ return processed_outputs
222
+
223
+ def _get_mm_fields_config(
224
+ self,
225
+ hf_inputs: BatchFeature,
226
+ hf_processor_mm_kwargs: Mapping[str, object],
227
+ ) -> Mapping[str, MultiModalFieldConfig]:
228
+ return dict(
229
+ pixel_values=MultiModalFieldConfig.batched("image"),
230
+ aspect_ratio_ids=MultiModalFieldConfig.batched("image"),
231
+ aspect_ratio_mask=MultiModalFieldConfig.batched("image"),
232
+ num_tiles=MultiModalFieldConfig.batched("image"),
233
)
155
- num_tiles_height = canvas_height // tile_size
156
- num_tiles_width = canvas_width // tile_size
157
- num_tiles += num_tiles_height * num_tiles_width
158
- num_decode_images -= 1
159
- if num_decode_images == 0:
160
- break
161
-
162
- # Set encoder prompt length based on the number of tiles.
163
- # This tells the block manager to allocate correct number
164
- # of slots for encoder tokens.
165
- assert vision_config.image_size % 14 == 0, \
166
- "chunk size should be multiple of 14"
167
- token_per_chunk = (vision_config.image_size // 14)**2 + 1
168
- num_tokens = num_tiles * token_per_chunk
169
-
170
- # Example output from processor:
171
- # {
172
- # 'encoder': {
173
- # 'type': 'token',
174
- # 'prompt_token_ids': [128256, 128256, ..., 128256],
175
- # 'prompt': '<|image|><|image|>...<|image|>',
176
- # 'multi_modal_data': {'image': <PIL.Image.Image image mode=RGB size=1770x1180 at 0x7FDE2C624880>}, # noqa: E501
177
- # },
178
- # 'decoder': {
179
- # 'type': 'token',
180
- # 'prompt_token_ids': [128000, 128256, 128000, 3923, 374, 279, 2262, 315, 420, 2217, 30], # noqa: E501
181
- # 'prompt': '<|image|><|begin_of_text|>What is the content of this image?', # noqa: E501
182
- # 'multi_modal_data': {'image': <PIL.Image.Image image mode=RGB size=1770x1180 at 0x7FDE2C624880>}, # noqa: E501
183
- # },
184
- # }
185
- return EncoderDecoderInputs(
186
- encoder=token_inputs(
187
- prompt_token_ids=[MLLAMA_IMAGE_TOKEN_ID] * num_tokens,
188
- prompt=MLLAMA_IMAGE_TOKEN * num_tokens,
189
- multi_modal_data=multi_modal_data,
190
- ),
191
- decoder=dec_inputs,
192
- )
193
-
194
-
195
-def get_max_mllama_image_tokens(ctx: InputContext) -> int:
196
- hf_config = ctx.model_config.hf_config
197
- token_per_chunk = (hf_config.vision_config.image_size // 14)**2 + 1
198
- return hf_config.vision_config.max_num_tiles * token_per_chunk
199
-
200
-
201
-def dummy_decoder_seq_data(seq_len: int, num_images: int):
202
- # <|image|> * num_images + 0 * (seq_len - num_images)
203
- assert seq_len >= num_images, \
204
- "seq_len should be greater than or equal to num_images"
205
-
206
- return SequenceData.from_prompt_token_counts(
207
- (MLLAMA_IMAGE_TOKEN_ID, num_images),
208
- (0, seq_len - num_images),
209
- )
210
-
211
-
212
-def dummy_encoder_seq_data(ctx: InputContext, num_images: int):
213
- num_tokens = get_max_mllama_image_tokens(ctx) * num_images
214
-
215
- return SequenceData.from_prompt_token_counts(
216
- (MLLAMA_IMAGE_TOKEN_ID, num_tokens))
217
-
218
-
219
-def dummy_image(num_images: int, ):
220
- width = height = 1024
221
- image = Image.new("RGB", (width, height), color=0)
222
- return {"image": image if num_images == 1 else [image] * num_images}
223
-
224
-
225
-def dummy_decoder_data_for_mllama(ctx: InputContext, seq_len: int,
226
- mm_counts: Mapping[str, int]):
227
- num_images = mm_counts["image"]
228
- return DummyData(dummy_decoder_seq_data(seq_len, num_images))
229
-
230
-
231
-def dummy_encoder_data_for_mllama(ctx: InputContext, seq_len: int,
232
- mm_counts: Mapping[str, int]):
233
- num_images = mm_counts["image"]
234
- return DummyData(dummy_encoder_seq_data(ctx, num_images),
235
- dummy_image(num_images))
234
+
235
+ def create_encoder_prompt(
236
+ self,
237
+ prompt: Union[str, list[int]],
238
+ mm_data: MultiModalDataDict,
239
+ ) -> Union[str, list[int]]:
240
+ data = mm_data.get("image", [])
241
+ num_images = 1 if isinstance(data, Image) else len(data)
242
+ image_token_id = self.info.get_hf_config().image_token_index
243
+ return [image_token_id] * num_images
244
+
245
+ def _get_prompt_replacements(
246
+ self,
247
+ mm_items: MultiModalDataItems,
248
+ hf_processor_mm_kwargs: Mapping[str, object],
249
+ out_mm_kwargs: MultiModalKwargs,
250
+ ) -> list[PromptReplacement]:
251
+ token_per_chunk = self.info.get_token_per_chunk_from_config()
252
+ image_token_id = self.info.get_hf_config().image_token_index
253
+
254
+ def get_replacement_mllama(item_idx):
255
+ images = mm_items.get_items("image", ImageProcessorItems)
256
+ image_size = images.get_image_size(item_idx)
257
+ num_tile = self.info.get_num_tiles_per_image(
258
+ image_height=image_size.height,
259
+ image_width=image_size.width,
260
+ )
261
+ num_tokens = num_tile * token_per_chunk
262
+ return [image_token_id] * num_tokens
263
+
264
+ return [
265
+ PromptReplacement(
266
+ modality="image",
267
+ target=[image_token_id],
268
+ replacement=get_replacement_mllama,
269
+ )
270
+ ]
271
272
273
def _prepare_aspect_ratio_attention_mask(
@@ -1107,11 +1142,9 @@ class MllamaForCausalLM(nn.Module):
1142
return hidden_states
1143
1144
1110
-@MULTIMODAL_REGISTRY.register_image_input_mapper()
1111
-@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_mllama_image_tokens)
1112
-@INPUT_REGISTRY.register_dummy_data(dummy_decoder_data_for_mllama)
1113
-@INPUT_REGISTRY.register_dummy_encoder_data(dummy_encoder_data_for_mllama)
1114
-@INPUT_REGISTRY.register_input_processor(input_processor_for_mllama)
1145
+@MULTIMODAL_REGISTRY.register_processor(MllamaMultiModalProcessor,
1146
+ info=MllamaProcessingInfo,
1147
+ dummy_inputs=MllamaDummyInputsBuilder)
1148
class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1149
packed_modules_mapping = {
1150
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
@@ -1120,7 +1153,7 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1153
1154
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1155
super().__init__()
1123
- config = vllm_config.model_config.hf_config
1156
+ config: MllamaConfig = vllm_config.model_config.hf_config
1157
quant_config = vllm_config.quant_config
1158
self.quant_config = quant_config
1159
self.vocab_size = config.text_config.vocab_size
@@ -1130,6 +1163,7 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1163
self.pad_token_id = \
1164
config.pad_token_id if config.pad_token_id is not None else -1
1165
self.image_size = config.vision_config.image_size
1166
+ self.image_token_id = config.image_token_index
1167
1168
self.vision_model = MllamaVisionModel(config.vision_config,
1169
quant_config,
@@ -1204,48 +1238,12 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1238
if pixel_values is not None:
1239
assert aspect_ratio_ids is not None
1240
assert aspect_ratio_mask is not None
1207
- max_num_images = max([len(x[0]) for x in pixel_values])
1208
- if max_num_images == 0:
1209
- raise ValueError("No images provided.")
1210
- max_num_tiles = max(
1211
- max([len(x) for x in y[0]]) for y in pixel_values)
1212
- device = next(self.multi_modal_projector.parameters()).device
1213
- bsz = len(pixel_values)
1214
- out_num_tiles = []
1215
- out_images = torch.zeros(
1216
- bsz,
1217
- max_num_images,
1218
- max_num_tiles,
1219
- 3,
1220
- self.image_size,
1221
- self.image_size,
1222
- dtype=torch.float32,
1223
- device=device,
1224
- )
1225
- out_ar_ids = torch.ones(bsz,
1226
- max_num_images,
1227
- dtype=torch.int64,
1228
- device=device)
1229
- out_ar_mask = torch.zeros(bsz,
1230
- max_num_images,
1231
- max_num_tiles,
1232
- dtype=torch.int64,
1233
- device=device)
1234
- for b in range(len(pixel_values)):
1235
- _num_tiles = []
1236
- for i in range(len(pixel_values[b][0])):
1237
- img = pixel_values[b][0][i]
1238
- out_images[b, i, :img.shape[0]] = img
1239
- out_ar_ids[b, i] = aspect_ratio_ids[b][0][i]
1240
- out_ar_mask[b, i] = aspect_ratio_mask[b][0][i]
1241
- _num_tiles.append(img.shape[0])
1242
- out_num_tiles.append(_num_tiles)
1241
1242
return MllamaImagePixelInputs(
1243
type="pixel_values",
1246
- data=out_images,
1247
- aspect_ratio_ids=out_ar_ids,
1248
- aspect_ratio_mask=out_ar_mask,
1244
+ data=pixel_values,
1245
+ aspect_ratio_ids=aspect_ratio_ids,
1246
+ aspect_ratio_mask=aspect_ratio_mask,
1247
)
1248
1249
if image_embeds is not None:
@@ -1312,7 +1310,7 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1310
batch_token_ids.append(token_ids[start:start + seq_len])
1311
start += seq_len
1312
sparse_mask = [
1315
- get_cross_attention_token_mask(t, MLLAMA_IMAGE_TOKEN_ID)
1313
+ get_cross_attention_token_mask(t, self.image_token_id)
1314
for t in batch_token_ids
1315
]
1316
@@ -1384,8 +1382,8 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1382
# block manager to allocate blocks for those images only.
1383
# See input_processor_for_mllama() for more details.
1384
num_tiles_tensor = kwargs.pop("num_tiles")
1387
- num_tiles = [t[0].tolist() for t in num_tiles_tensor]
1388
- num_tokens_per_tile = (self.image_size // 14)**2 + 1
1385
+ num_tiles = [t.tolist() for t in num_tiles_tensor]
1386
+ num_tokens_per_tile = calc_token_per_chunk(self.image_size)
1387
actual_encoder_seq_lens = [
1388
sum(num_tile) * num_tokens_per_tile for num_tile in num_tiles
1389
]
vllm/multimodal/inputs.py
+16
@@ -739,3 +739,19 @@ class MultiModalInputs(TypedDict):
739
For each modality, information about the placeholder tokens in
740
:code:`prompt_token_ids`.
741
"""
742
+
743
+
744
+class MultiModalEncDecInputs(MultiModalInputs):
745
+ """
746
+ Represents the outputs of :class:`vllm.multimodal.EncDecMultiModalProcessor`
747
+ ready to be passed to vLLM internals.
748
+ """
749
+
750
+ encoder_prompt: str
751
+ """The processed encoder prompt text."""
752
+
753
+ encoder_prompt_token_ids: list[int]
754
+ """The processed token IDs of the encoder prompt."""
755
+
756
+ encoder_token_type_ids: NotRequired[list[int]]
757
+ """The token type IDs of the encoder prompt."""
vllm/multimodal/processing.py
+57
-3
@@ -20,9 +20,9 @@ from vllm.transformers_utils.tokenizer import (AnyTokenizer, decode_tokens,
20
from vllm.utils import LRUCache, flatten_2d_lists, full_groupby
21
22
from .hasher import MultiModalHasher
23
-from .inputs import (MultiModalDataDict, MultiModalFieldConfig,
24
- MultiModalInputs, MultiModalKwargs, MultiModalKwargsItem,
25
- PlaceholderRange)
23
+from .inputs import (MultiModalDataDict, MultiModalEncDecInputs,
24
+ MultiModalFieldConfig, MultiModalInputs, MultiModalKwargs,
25
+ MultiModalKwargsItem, PlaceholderRange)
26
from .parse import MultiModalDataItems, MultiModalDataParser
27
28
if TYPE_CHECKING:
@@ -1293,3 +1293,57 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1293
mm_hashes=mm_hashes,
1294
mm_placeholders=mm_placeholder_ranges,
1295
)
1296
+
1297
+
1298
+class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
1299
+
1300
+ @abstractmethod
1301
+ def create_encoder_prompt(
1302
+ self,
1303
+ prompt: Union[str, list[int]],
1304
+ mm_data: MultiModalDataDict,
1305
+ ) -> Union[str, list[int]]:
1306
+ """Create input prompt for the encoder."""
1307
+ raise NotImplementedError
1308
+
1309
+ def apply(
1310
+ self,
1311
+ prompt: Union[str, list[int]],
1312
+ mm_data: MultiModalDataDict,
1313
+ hf_processor_mm_kwargs: Mapping[str, object],
1314
+ ) -> MultiModalEncDecInputs:
1315
+ """
1316
+ Process multi-modal inputs to be used in vLLM.
1317
+ The main processing steps are modified to fit encoder-decoder model:
1318
+ 1. Create encoder prompt from input prompt text.
1319
+ 2. Apply the HF processor on encoder prompt.
1320
+ 3. Copy the input prompt text as decoder prompt inputs.
1321
+ """
1322
+ encoder_prompt = self.create_encoder_prompt(prompt, mm_data)
1323
+ encoder_inputs = super().apply(
1324
+ encoder_prompt,
1325
+ mm_data,
1326
+ hf_processor_mm_kwargs,
1327
+ )
1328
+
1329
+ # We assumed the decoder prompt text is copied from
1330
+ # the original encoder prompt without extra process
1331
+ tokenizer = self.info.get_tokenizer()
1332
+ if isinstance(prompt, str):
1333
+ decoder_prompt = prompt
1334
+ decoder_prompt_ids = encode_tokens(tokenizer,
1335
+ prompt,
1336
+ add_special_tokens=False)
1337
+ else:
1338
+ decoder_prompt = decode_tokens(tokenizer, prompt)
1339
+ decoder_prompt_ids = prompt
1340
+
1341
+ mm_inputs = MultiModalEncDecInputs(
1342
+ encoder_prompt=encoder_inputs["prompt"],
1343
+ encoder_prompt_token_ids=encoder_inputs["prompt_token_ids"],
1344
+ **encoder_inputs)
1345
+ mm_inputs.update({
1346
+ "prompt": decoder_prompt,
1347
+ "prompt_token_ids": decoder_prompt_ids
1348
+ })
1349
+ return mm_inputs
vllm/multimodal/profiling.py
+17
-11
@@ -144,7 +144,11 @@ class MultiModalProfiler(Generic[_I]):
144
hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
145
)
146
147
- def get_dummy_data(self, seq_len: int) -> DummyData:
147
+ def get_dummy_data(
148
+ self,
149
+ seq_len: int,
150
+ is_encoder_data: bool = False,
151
+ ) -> DummyData:
152
# Avoid circular import
153
from vllm.sequence import SequenceData
154
@@ -183,16 +187,18 @@ class MultiModalProfiler(Generic[_I]):
187
total_len = len(prompt_token_ids)
188
189
# V0 does not support chunked prefill.
186
- if total_len > seq_len and not envs.VLLM_USE_V1:
187
- logger.warning(
188
- "The context length (%d) of the model is too short "
189
- "to hold the multi-modal embeddings in the worst case "
190
- "(%d tokens in total, out of which %s are reserved for "
191
- "multi-modal embeddings). This may cause certain multi-modal "
192
- "inputs to fail during inference, even when the input text is "
193
- "short. To avoid this, you should increase `max_model_len`, "
194
- "reduce `max_num_seqs`, and/or reduce `mm_counts`.", seq_len,
195
- total_len, total_placeholders_by_modality)
190
+ if (total_len > seq_len and not envs.VLLM_USE_V1) or is_encoder_data:
191
+ if total_len > seq_len:
192
+ logger.warning(
193
+ "The context length (%d) of the model is too short "
194
+ "to hold the multi-modal embeddings in the worst case "
195
+ "(%d tokens in total, out of which %s are reserved for "
196
+ "multi-modal embeddings). This may cause certain "
197
+ "multi-modal inputs to fail during inference, even when "
198
+ "the input text is short. To avoid this, you should "
199
+ "increase `max_model_len`, reduce `max_num_seqs`, "
200
+ "and/or reduce `mm_counts`.", seq_len, total_len,
201
+ total_placeholders_by_modality)
202
203
return DummyData(
204
seq_data=SequenceData.from_prompt_token_counts((0, seq_len)),