@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bc55d13070
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[VLM] Implement merged multimodal processor for Mllama (#11427)

Isotr0py committed Feb 13, 2025 at 12:26 UTC bc55d13070a9f3f2b8524901817d406cb8a37a3b
8 files changed +456 -233
tests/models/encoder_decoder/vision_language/test_mllama.py
+67 -4
@@ -7,11 +7,11 @@ import torch
7 from transformers import (AutoConfig, AutoModelForVision2Seq, AutoTokenizer,
8 BatchEncoding)
9
10 +from vllm import LLM, SamplingParams
11 from vllm.attention.backends.flash_attn import FlashAttentionMetadata
12 from vllm.attention.selector import (_Backend, _cached_get_attn_backend,
13 global_force_attn_backend_context_manager)
13 -from vllm.model_executor.models.mllama import (MLLAMA_IMAGE_TOKEN_ID,
14 - MllamaForConditionalGeneration)
14 +from vllm.model_executor.models.mllama import MllamaForConditionalGeneration
15 from vllm.multimodal.image import rescale_image_size
16 from vllm.sequence import SampleLogprobs
17
@@ -21,6 +21,7 @@ from ....utils import large_gpu_test
21 from ...utils import check_logprobs_close
22
23 _LIMIT_IMAGE_PER_PROMPT = 3
24 +MLLAMA_IMAGE_TOKEN_ID = 128256
25
26 LIST_ENC_DEC_SUPPORTED_BACKENDS = [_Backend.XFORMERS, _Backend.FLASH_ATTN]
27
@@ -396,6 +397,64 @@ def test_models_interleaved_images(hf_runner, vllm_runner, image_assets, model,
397 )
398
399
400 +@large_gpu_test(min_gb=48)
401 +@pytest.mark.core_model
402 +@pytest.mark.parametrize("model", models)
403 +@pytest.mark.parametrize("dtype", ["bfloat16"])
404 +@pytest.mark.parametrize("max_tokens", [32])
405 +def test_explicit_implicit_prompt(
406 + image_assets: _ImageAssets,
407 + model: str,
408 + dtype: str,
409 + max_tokens: int,
410 +):
411 + stop_sign = image_assets[0].pil_image
412 + # yapf: disable
413 + prompts = [
414 + # explicit prompt
415 + {
416 + "encoder_prompt": {
417 + "prompt": "<|image|>",
418 + "multi_modal_data": {"image": stop_sign},
419 + },
420 + "decoder_prompt": {
421 + "prompt_token_ids": [128000, 791, 2262, 315, 279, 2217, 220, 128256, 374], # noqa: E501
422 + }
423 + },
424 + {
425 + "encoder_prompt": "Not <|image|>",
426 + "decoder_prompt": "The color of the sky is blue but sometimes it can also be", # noqa: E501
427 + },
428 + # implicit prompt
429 + {
430 + "prompt": "<|begin_of_text|>The content of the image <|image|> is", # noqa: E501
431 + "multi_modal_data": {"image": stop_sign},
432 + },
433 + {
434 + "prompt": "The color of the sky is blue but sometimes it can also be", # noqa: E501
435 + },
436 + ]
437 + # yapf: enable
438 + llm = LLM(
439 + model=model,
440 + dtype=dtype,
441 + max_model_len=4096,
442 + max_num_seqs=2,
443 + tensor_parallel_size=1,
444 + enforce_eager=True,
445 + )
446 + sampling_params = SamplingParams(
447 + temperature=0,
448 + max_tokens=max_tokens,
449 + )
450 + outputs = llm.generate(prompts, sampling_params)
451 + n_prompts = len(prompts)
452 + explicit_outputs = outputs[:n_prompts // 2]
453 + implicit_outputs = outputs[n_prompts // 2:]
454 + for exp_output, imp_output in zip(explicit_outputs, implicit_outputs):
455 + assert exp_output.outputs[0].text == imp_output.outputs[0].text
456 +
457 +
458 @large_gpu_test(min_gb=48)
459 @pytest.mark.core_model
460 @pytest.mark.parametrize("model", models)
@@ -458,6 +517,10 @@ def test_regression(vllm_runner, image_assets, model, dtype, max_tokens,
517 images=images)
518
519
520 +class DummyModel:
521 + image_token_id = MLLAMA_IMAGE_TOKEN_ID
522 +
523 +
524 @pytest.mark.core_model
525 @pytest.mark.parametrize(
526 "input_indices_and_output",
@@ -499,7 +562,7 @@ def test_get_cross_attention_mask(input_indices_and_output) -> None:
562 use_cuda_graph=False,
563 )
564
502 - dummy: dict[str, str] = {}
565 + dummy = DummyModel()
566
567 cross_attention_mask, kv_range_for_decode = MllamaForConditionalGeneration\
568 .get_cross_attention_mask(dummy,
@@ -556,7 +619,7 @@ def test_get_full_text_row_masked_out_mask(input_indices) -> None:
619 use_cuda_graph=False,
620 )
621
559 - dummy: dict[str, str] = {}
622 + dummy = DummyModel()
623
624 full_text_row_masked_out_mask = MllamaForConditionalGeneration\
625 .get_full_text_row_masked_out_mask(dummy,
tests/models/multimodal/processing/test_common.py
+11 -2
@@ -85,6 +85,14 @@ def _test_processing_correctness(
85 partial(random_audio, rng, min_len=512, max_len=1024, sr=16000),
86 }
87
88 + tokenizer_encode_kwargs = {}
89 + if model_config.hf_config.model_type == "mllama":
90 + # For Mllama, tokenizer will always add bos_token at the beginning of
91 + # prompt by default, causing hf_processor outputs incorrect token ids.
92 + # So we need use `add_special_tokens=False` here to leave bos_token
93 + # to be added by the processor.
94 + tokenizer_encode_kwargs = {"add_special_tokens": False}
95 +
96 for batch_idx in range(num_batches):
97 mm_data = {
98 k:
@@ -122,7 +130,7 @@ def _test_processing_correctness(
130 f"Failed ({batch_idx=}, {prompt=}, {mm_data=})")
131
132 baseline_tokenized_result = baseline_processor.apply(
125 - tokenizer.encode(prompt),
133 + tokenizer.encode(prompt, **tokenizer_encode_kwargs),
134 mm_data=mm_data,
135 hf_processor_mm_kwargs={},
136 )
@@ -131,7 +139,7 @@ def _test_processing_correctness(
139 f"Failed ({batch_idx=}, {prompt=}, {mm_data=})")
140
141 cached_tokenized_result = cached_processor.apply(
134 - tokenizer.encode(prompt),
142 + tokenizer.encode(prompt, **tokenizer_encode_kwargs),
143 mm_data=mm_data,
144 hf_processor_mm_kwargs={},
145 )
@@ -155,6 +163,7 @@ def _test_processing_correctness(
163 "llava-hf/llava-v1.6-mistral-7b-hf",
164 "llava-hf/LLaVA-NeXT-Video-7B-hf",
165 "llava-hf/llava-onevision-qwen2-0.5b-ov-hf",
166 + "meta-llama/Llama-3.2-11B-Vision-Instruct",
167 "TIGER-Lab/Mantis-8B-siglip-llama3",
168 "mistral-community/pixtral-12b",
169 "openbmb/MiniCPM-o-2_6",
vllm/inputs/preprocess.py
+83 -7
@@ -1,7 +1,7 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 import asyncio
4 -from typing import List, Mapping, Optional, Union
4 +from typing import List, Mapping, Optional, Tuple, Union, cast
5
6 from typing_extensions import assert_never
7
@@ -9,7 +9,8 @@ from vllm.config import ModelConfig
9 from vllm.logger import init_logger
10 from vllm.lora.request import LoRARequest
11 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
12 -from vllm.multimodal.inputs import MultiModalDataDict, MultiModalInputs
12 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalEncDecInputs,
13 + MultiModalInputs)
14 from vllm.prompt_adapter.request import PromptAdapterRequest
15 from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
16
@@ -495,6 +496,51 @@ class InputPreprocessor:
496 decoder=decoder_inputs,
497 )
498
499 + def _separate_enc_dec_inputs_from_mm_processor_outputs(
500 + self,
501 + inputs: SingletonInputs,
502 + decoder_inputs_to_override: Optional[SingletonInputs] = None,
503 + ) -> Tuple[SingletonInputs, SingletonInputs]:
504 + """
505 + For encoder/decoder models only:
506 + Separate Encoder/Decoder inputs from a MultiModalEncDecInputs
507 + """
508 + encoder_inputs: SingletonInputs
509 + decoder_inputs: SingletonInputs
510 + if inputs["type"] == "multimodal":
511 + # Multimodal data inputs
512 + assert ("encoder_prompt" in inputs
513 + and "encoder_prompt_token_ids" in inputs)
514 + inputs = cast(MultiModalEncDecInputs, inputs)
515 + encoder_inputs = token_inputs(
516 + prompt=inputs["encoder_prompt"],
517 + prompt_token_ids=inputs["encoder_prompt_token_ids"],
518 + )
519 + if decoder_inputs_to_override is not None:
520 + decoder_inputs = MultiModalInputs(
521 + type="multimodal",
522 + prompt=decoder_inputs_to_override.get("prompt", ""),
523 + prompt_token_ids=decoder_inputs_to_override[
524 + "prompt_token_ids"],
525 + mm_kwargs=inputs["mm_kwargs"],
526 + mm_placeholders=inputs["mm_placeholders"],
527 + )
528 + else:
529 + decoder_inputs = MultiModalInputs(
530 + type="multimodal",
531 + prompt=inputs["prompt"],
532 + prompt_token_ids=inputs["prompt_token_ids"],
533 + mm_kwargs=inputs["mm_kwargs"],
534 + mm_placeholders=inputs["mm_placeholders"],
535 + )
536 + elif inputs["type"] == "token":
537 + # Text-only inputs
538 + encoder_inputs = token_inputs(prompt="", prompt_token_ids=[])
539 + decoder_inputs = decoder_inputs_to_override or inputs
540 + else:
541 + assert_never(inputs) # type: ignore[arg-type]
542 + return encoder_inputs, decoder_inputs
543 +
544 def _process_encoder_decoder_prompt(
545 self,
546 prompt: PromptType,
@@ -539,7 +585,6 @@ class InputPreprocessor:
585 prompt["encoder_prompt"],
586 request_id=request_id,
587 )
542 -
588 if (decoder_input := prompt["decoder_prompt"]) is None:
589 decoder_inputs = None
590 else:
@@ -547,13 +592,28 @@ class InputPreprocessor:
592 decoder_input,
593 request_id=request_id,
594 )
595 + # For multimodal model, override decoder prompt from processor
596 + # with explicit decoder prompt.
597 + if self.model_config.is_multimodal_model and (
598 + self._can_process_multimodal()):
599 + encoder_inputs, decoder_inputs = (
600 + self._separate_enc_dec_inputs_from_mm_processor_outputs(
601 + encoder_inputs, decoder_inputs))
602 else:
551 - encoder_inputs = self._prompt_to_llm_inputs(
603 + inputs = self._prompt_to_llm_inputs(
604 prompt,
605 request_id=request_id,
606 )
607 + if self.model_config.is_multimodal_model and (
608 + self._can_process_multimodal()):
609 + # Encoder-Decoder Multimodal model
610 + encoder_inputs, decoder_inputs = (
611 + self._separate_enc_dec_inputs_from_mm_processor_outputs(
612 + inputs))
613 + else:
614 + encoder_inputs = inputs
615
556 - decoder_inputs = None
616 + decoder_inputs = None
617
618 return self._build_enc_dec_llm_inputs(encoder_inputs, decoder_inputs)
619
@@ -583,13 +643,29 @@ class InputPreprocessor:
643
644 encoder_inputs, decoder_inputs = await asyncio.gather(
645 encoder_task, decoder_task)
646 +
647 + # For multimodal model, override decoder prompt from processor
648 + # with explicit decoder prompt.
649 + if self.model_config.is_multimodal_model and (
650 + self._can_process_multimodal()):
651 + encoder_inputs, decoder_inputs = (
652 + self._separate_enc_dec_inputs_from_mm_processor_outputs(
653 + encoder_inputs, decoder_inputs))
654 else:
587 - encoder_inputs = await self._prompt_to_llm_inputs_async(
655 + inputs = await self._prompt_to_llm_inputs_async(
656 prompt,
657 request_id=request_id,
658 )
659 + if self.model_config.is_multimodal_model and (
660 + self._can_process_multimodal()):
661 + # Encoder-Decoder Multimodal model
662 + encoder_inputs, decoder_inputs = (
663 + self._separate_enc_dec_inputs_from_mm_processor_outputs(
664 + inputs))
665 + else:
666 + encoder_inputs = inputs
667
592 - decoder_inputs = None
668 + decoder_inputs = None
669
670 return self._build_enc_dec_llm_inputs(encoder_inputs, decoder_inputs)
671
vllm/inputs/registry.py
+2 -1
@@ -350,7 +350,8 @@ class InputRegistry:
350 )
351 processor = mm_registry.create_processor(model_config, tokenizer)
352 profiler = MultiModalProfiler(processor)
353 - dummy_data = profiler.get_dummy_data(seq_len)
353 + dummy_data = profiler.get_dummy_data(
354 + seq_len, is_encoder_data=is_encoder_data)
355 else:
356 model_cls, _ = get_model_architecture(model_config)
357 if is_encoder_data:
vllm/model_executor/models/mllama.py
+203 -205
@@ -23,14 +23,15 @@ import torch
23 import torch.nn.functional as F
24 import torch.utils.checkpoint
25 import transformers.models.mllama.configuration_mllama as config_mllama
26 -from PIL import Image
26 +from PIL.Image import Image
27 from torch import nn
28 +from transformers import BatchFeature, MllamaConfig
29 from transformers.modeling_outputs import (BaseModelOutput,
30 CausalLMOutputWithPast)
31 from transformers.models.mllama.image_processing_mllama import (
32 get_optimal_tiled_canvas)
33 from transformers.models.mllama.processing_mllama import (
33 - get_cross_attention_token_mask)
34 + MllamaProcessor, get_cross_attention_token_mask)
35
36 import vllm.distributed.parallel_state as ps
37 from vllm.attention import Attention, AttentionMetadata, AttentionType
@@ -38,8 +39,6 @@ from vllm.attention.ops.paged_attn import PagedAttention
39 from vllm.attention.selector import _Backend
40 from vllm.config import VllmConfig
41 from vllm.distributed import get_tensor_model_parallel_world_size
41 -from vllm.inputs import (INPUT_REGISTRY, DummyData, EncoderDecoderInputs,
42 - InputContext, TokenInputs, token_inputs)
42 from vllm.logger import init_logger
43 from vllm.model_executor.layers.layernorm import RMSNorm
44 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -54,8 +53,13 @@ from vllm.model_executor.model_loader.weight_utils import (
53 default_weight_loader, maybe_remap_kv_scale_name)
54 from vllm.model_executor.sampling_metadata import SamplingMetadata
55 from vllm.multimodal import MULTIMODAL_REGISTRY
57 -from vllm.sequence import SequenceData
58 -from vllm.utils import is_list_of
56 +from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs
57 +from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
58 + MultiModalDataDict, MultiModalDataItems)
59 +from vllm.multimodal.processing import (BaseProcessingInfo,
60 + EncDecMultiModalProcessor,
61 + PromptReplacement)
62 +from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
63
64 from .clip import CLIPMLP
65 from .interfaces import SupportsMultiModal
@@ -63,8 +67,6 @@ from .llama import LlamaDecoderLayer, LlamaMLP
67 from .utils import maybe_prefix
68
69 logger = init_logger(__name__)
66 -MLLAMA_IMAGE_TOKEN_ID = 128256
67 -MLLAMA_IMAGE_TOKEN = "<|image|>"
70
71
72 class MllamaImagePixelInputs(TypedDict):
@@ -81,158 +83,191 @@ class MllamaImagePixelInputs(TypedDict):
83 # TODO: support LlamaImageEmbeddingInputs
84
85
84 -def _get_num_image_in_last_group(prompt_token_ids: List[int]) -> int:
85 - num_images = 0
86 - for token_id in prompt_token_ids[::-1]:
87 - if token_id == MLLAMA_IMAGE_TOKEN_ID:
88 - num_images += 1
89 - elif num_images > 0:
90 - break
91 - return num_images
92 -
93 -
94 -def input_processor_for_mllama(
95 - ctx: InputContext,
96 - inputs: EncoderDecoderInputs,
97 -) -> EncoderDecoderInputs:
98 - # Example input to processor:
99 - # {
100 - # 'encoder': {
101 - # 'type': 'token',
102 - # 'prompt_token_ids': [128000, 128256, 128000, 3923, 374, 279, 2262, 315, 420, 2217, 30], # noqa: E501
103 - # 'prompt': '<|image|><|begin_of_text|>What is the content of this image?', # noqa: E501
104 - # 'multi_modal_data': {'image': <PIL.Image.Image image mode=RGB size=1770x1180 at 0x7FDE2C624880>}, # noqa: E501
105 - # },
106 - # 'decoder': {
107 - # 'type': 'token',
108 - # 'prompt_token_ids': [128000],
109 - # },
110 - # }
111 -
112 - # move encoder prompt to decoder
113 - dec_inputs = TokenInputs(**inputs["encoder"])
114 -
115 - multi_modal_data = dec_inputs.get("multi_modal_data")
116 - if multi_modal_data is None or "image" not in multi_modal_data:
117 - # text-only
118 - return EncoderDecoderInputs(
119 - encoder=token_inputs([]),
120 - decoder=dec_inputs,
86 +def calc_token_per_chunk(image_size: int) -> int:
87 + assert image_size % 14 == 0, "chunk size should be multiple of 14"
88 + token_per_chunk = (image_size // 14)**2 + 1
89 + return token_per_chunk
90 +
91 +
92 +class MllamaProcessingInfo(BaseProcessingInfo):
93 +
94 + def get_hf_config(self) -> MllamaConfig:
95 + return self.ctx.get_hf_config(MllamaConfig)
96 +
97 + def get_hf_processor(self) -> MllamaProcessor:
98 + return self.ctx.get_hf_processor(MllamaProcessor)
99 +
100 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
101 + return {"image": None}
102 +
103 + def get_token_per_chunk_from_config(self) -> int:
104 + image_size = self.get_hf_config().vision_config.image_size
105 + return calc_token_per_chunk(image_size)
106 +
107 + def get_mm_max_tokens_per_item(
108 + self,
109 + seq_len: int,
110 + mm_counts: Mapping[str, int],
111 + ) -> Mapping[str, int]:
112 + vision_config = self.get_hf_config().vision_config
113 + token_per_chunk = self.get_token_per_chunk_from_config()
114 + mm_max_tokens = vision_config.max_num_tiles * token_per_chunk
115 + return {"image": mm_max_tokens}
116 +
117 + def get_num_tiles_per_image(self, image_height: int,
118 + image_width: int) -> int:
119 + vision_config = self.get_hf_config().vision_config
120 + max_num_tiles = vision_config.max_num_tiles
121 + image_size = vision_config.image_size
122 + tiled_height, tiled_width = get_optimal_tiled_canvas(
123 + image_height,
124 + image_width,
125 + max_num_tiles,
126 + tile_size=image_size,
127 + )
128 + num_tiles_height = tiled_height // image_size
129 + num_tiles_width = tiled_width // image_size
130 + return num_tiles_height * num_tiles_width
131 +
132 + def get_image_size_with_most_features(self) -> ImageSize:
133 + vision_config = self.get_hf_config().vision_config
134 + image_size = vision_config.image_size
135 + max_num_tiles = vision_config.max_num_tiles
136 + # Result in the max possible feature size (h:w = 16:1)
137 + return ImageSize(height=max_num_tiles * image_size, width=image_size)
138 +
139 +
140 +class MllamaDummyInputsBuilder(BaseDummyInputsBuilder[MllamaProcessingInfo]):
141 +
142 + def get_dummy_processor_inputs(
143 + self,
144 + seq_len: int,
145 + mm_counts: Mapping[str, int],
146 + ) -> ProcessorInputs:
147 + num_images = mm_counts.get("image", 0)
148 +
149 + target_width, target_height = \
150 + self.info.get_image_size_with_most_features()
151 +
152 + mm_data = {
153 + "image":
154 + self._get_dummy_images(width=target_width,
155 + height=target_height,
156 + num_images=num_images)
157 + }
158 +
159 + hf_processor = self.info.get_hf_processor()
160 + image_token: str = hf_processor.image_token
161 +
162 + return ProcessorInputs(
163 + prompt_text=image_token * num_images,
164 + mm_data=mm_data,
165 )
166
123 - image_data = multi_modal_data["image"]
124 - if isinstance(image_data, Image.Image):
125 - image_data = [image_data]
126 -
127 - assert is_list_of(image_data, Image.Image)
128 -
129 - num_image_tokens = dec_inputs['prompt_token_ids'].count(
130 - MLLAMA_IMAGE_TOKEN_ID)
131 - if num_image_tokens != len(image_data):
132 - raise ValueError(
133 - f"The number of image tokens ({num_image_tokens}) must be"
134 - f" the same as the number of images ({len(image_data)})")
135 -
136 - # Since only the last group of consecutive images
137 - # are attended by the decoded tokens, we only need to
138 - # get the number of tiles for those images.
139 - num_decode_images = _get_num_image_in_last_group(
140 - dec_inputs["prompt_token_ids"])
141 -
142 - hf_config = ctx.model_config.hf_config
143 - vision_config = hf_config.vision_config
144 -
145 - num_tiles = 0
146 - for image in image_data[::-1]:
147 - width, height = image.size
148 - tile_size = vision_config.image_size
149 - canvas_height, canvas_width = get_optimal_tiled_canvas(
150 - image_height=height,
151 - image_width=width,
152 - max_image_tiles=vision_config.max_num_tiles,
153 - tile_size=tile_size,
167 +
168 +class MllamaMultiModalProcessor(EncDecMultiModalProcessor[MllamaProcessingInfo]
169 + ):
170 +
171 + def _call_hf_processor(
172 + self,
173 + prompt: str,
174 + mm_data: Mapping[str, object],
175 + mm_kwargs: Mapping[str, object],
176 + ) -> BatchFeature:
177 + tokenizer = self.info.get_tokenizer()
178 + if mm_data:
179 + num_tiles = [
180 + self.info.get_num_tiles_per_image(img.height, img.width)
181 + for img in mm_data["images"]
182 + ]
183 + processed_outputs = super()._call_hf_processor(
184 + prompt, mm_data, mm_kwargs)
185 + processed_outputs["num_tiles"] = torch.tensor(num_tiles)
186 + for k in ('pixel_values', 'aspect_ratio_ids', "aspect_ratio_mask"):
187 + processed_outputs[k] = processed_outputs[k].squeeze(0)
188 + # Example input to encoder and decoder:
189 + # {
190 + # 'encoder': {
191 + # 'type': 'token',
192 + # 'prompt_token_ids': [128256, 128000, 3923, 374, 279, 2262, 315, 420, 2217, 30], # noqa: E501
193 + # 'prompt': '<|image|><|begin_of_text|>What is the content of this image?', # noqa: E501
194 + # 'multi_modal_data': {'image': <PIL.Image.Image image mode=RGB size=1770x1180 at 0x7FDE2C624880>}, # noqa: E501
195 + # },
196 + # 'decoder': {
197 + # 'type': 'token',
198 + # 'prompt_token_ids': [128000],
199 + # },
200 + # }
201 + processed_token_ids = processed_outputs.pop("input_ids")
202 + start_idx, end_idx = 0, processed_token_ids.size(1)
203 + processed_prompt_text = tokenizer.decode(processed_token_ids[0])
204 +
205 + hf_processor = self.info.get_hf_processor()
206 + bos_token = hf_processor.bos_token
207 + # Remove the bos_token from the start of prompt,
208 + # because we all know there would be image_token.
209 + if processed_prompt_text.startswith(bos_token):
210 + start_idx += 1
211 + # Remove the bos_token from the end of prompt,
212 + # because text is empty in this case.
213 + if processed_prompt_text.endswith(bos_token):
214 + end_idx -= 1
215 + processed_outputs[
216 + "input_ids"] = processed_token_ids[:, start_idx:end_idx]
217 + else:
218 + processed_outputs = tokenizer(prompt,
219 + add_special_tokens=False,
220 + return_tensors="pt")
221 + return processed_outputs
222 +
223 + def _get_mm_fields_config(
224 + self,
225 + hf_inputs: BatchFeature,
226 + hf_processor_mm_kwargs: Mapping[str, object],
227 + ) -> Mapping[str, MultiModalFieldConfig]:
228 + return dict(
229 + pixel_values=MultiModalFieldConfig.batched("image"),
230 + aspect_ratio_ids=MultiModalFieldConfig.batched("image"),
231 + aspect_ratio_mask=MultiModalFieldConfig.batched("image"),
232 + num_tiles=MultiModalFieldConfig.batched("image"),
233 )
155 - num_tiles_height = canvas_height // tile_size
156 - num_tiles_width = canvas_width // tile_size
157 - num_tiles += num_tiles_height * num_tiles_width
158 - num_decode_images -= 1
159 - if num_decode_images == 0:
160 - break
161 -
162 - # Set encoder prompt length based on the number of tiles.
163 - # This tells the block manager to allocate correct number
164 - # of slots for encoder tokens.
165 - assert vision_config.image_size % 14 == 0, \
166 - "chunk size should be multiple of 14"
167 - token_per_chunk = (vision_config.image_size // 14)**2 + 1
168 - num_tokens = num_tiles * token_per_chunk
169 -
170 - # Example output from processor:
171 - # {
172 - # 'encoder': {
173 - # 'type': 'token',
174 - # 'prompt_token_ids': [128256, 128256, ..., 128256],
175 - # 'prompt': '<|image|><|image|>...<|image|>',
176 - # 'multi_modal_data': {'image': <PIL.Image.Image image mode=RGB size=1770x1180 at 0x7FDE2C624880>}, # noqa: E501
177 - # },
178 - # 'decoder': {
179 - # 'type': 'token',
180 - # 'prompt_token_ids': [128000, 128256, 128000, 3923, 374, 279, 2262, 315, 420, 2217, 30], # noqa: E501
181 - # 'prompt': '<|image|><|begin_of_text|>What is the content of this image?', # noqa: E501
182 - # 'multi_modal_data': {'image': <PIL.Image.Image image mode=RGB size=1770x1180 at 0x7FDE2C624880>}, # noqa: E501
183 - # },
184 - # }
185 - return EncoderDecoderInputs(
186 - encoder=token_inputs(
187 - prompt_token_ids=[MLLAMA_IMAGE_TOKEN_ID] * num_tokens,
188 - prompt=MLLAMA_IMAGE_TOKEN * num_tokens,
189 - multi_modal_data=multi_modal_data,
190 - ),
191 - decoder=dec_inputs,
192 - )
193 -
194 -
195 -def get_max_mllama_image_tokens(ctx: InputContext) -> int:
196 - hf_config = ctx.model_config.hf_config
197 - token_per_chunk = (hf_config.vision_config.image_size // 14)**2 + 1
198 - return hf_config.vision_config.max_num_tiles * token_per_chunk
199 -
200 -
201 -def dummy_decoder_seq_data(seq_len: int, num_images: int):
202 - # <|image|> * num_images + 0 * (seq_len - num_images)
203 - assert seq_len >= num_images, \
204 - "seq_len should be greater than or equal to num_images"
205 -
206 - return SequenceData.from_prompt_token_counts(
207 - (MLLAMA_IMAGE_TOKEN_ID, num_images),
208 - (0, seq_len - num_images),
209 - )
210 -
211 -
212 -def dummy_encoder_seq_data(ctx: InputContext, num_images: int):
213 - num_tokens = get_max_mllama_image_tokens(ctx) * num_images
214 -
215 - return SequenceData.from_prompt_token_counts(
216 - (MLLAMA_IMAGE_TOKEN_ID, num_tokens))
217 -
218 -
219 -def dummy_image(num_images: int, ):
220 - width = height = 1024
221 - image = Image.new("RGB", (width, height), color=0)
222 - return {"image": image if num_images == 1 else [image] * num_images}
223 -
224 -
225 -def dummy_decoder_data_for_mllama(ctx: InputContext, seq_len: int,
226 - mm_counts: Mapping[str, int]):
227 - num_images = mm_counts["image"]
228 - return DummyData(dummy_decoder_seq_data(seq_len, num_images))
229 -
230 -
231 -def dummy_encoder_data_for_mllama(ctx: InputContext, seq_len: int,
232 - mm_counts: Mapping[str, int]):
233 - num_images = mm_counts["image"]
234 - return DummyData(dummy_encoder_seq_data(ctx, num_images),
235 - dummy_image(num_images))
234 +
235 + def create_encoder_prompt(
236 + self,
237 + prompt: Union[str, list[int]],
238 + mm_data: MultiModalDataDict,
239 + ) -> Union[str, list[int]]:
240 + data = mm_data.get("image", [])
241 + num_images = 1 if isinstance(data, Image) else len(data)
242 + image_token_id = self.info.get_hf_config().image_token_index
243 + return [image_token_id] * num_images
244 +
245 + def _get_prompt_replacements(
246 + self,
247 + mm_items: MultiModalDataItems,
248 + hf_processor_mm_kwargs: Mapping[str, object],
249 + out_mm_kwargs: MultiModalKwargs,
250 + ) -> list[PromptReplacement]:
251 + token_per_chunk = self.info.get_token_per_chunk_from_config()
252 + image_token_id = self.info.get_hf_config().image_token_index
253 +
254 + def get_replacement_mllama(item_idx):
255 + images = mm_items.get_items("image", ImageProcessorItems)
256 + image_size = images.get_image_size(item_idx)
257 + num_tile = self.info.get_num_tiles_per_image(
258 + image_height=image_size.height,
259 + image_width=image_size.width,
260 + )
261 + num_tokens = num_tile * token_per_chunk
262 + return [image_token_id] * num_tokens
263 +
264 + return [
265 + PromptReplacement(
266 + modality="image",
267 + target=[image_token_id],
268 + replacement=get_replacement_mllama,
269 + )
270 + ]
271
272
273 def _prepare_aspect_ratio_attention_mask(
@@ -1107,11 +1142,9 @@ class MllamaForCausalLM(nn.Module):
1142 return hidden_states
1143
1144
1110 -@MULTIMODAL_REGISTRY.register_image_input_mapper()
1111 -@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_mllama_image_tokens)
1112 -@INPUT_REGISTRY.register_dummy_data(dummy_decoder_data_for_mllama)
1113 -@INPUT_REGISTRY.register_dummy_encoder_data(dummy_encoder_data_for_mllama)
1114 -@INPUT_REGISTRY.register_input_processor(input_processor_for_mllama)
1145 +@MULTIMODAL_REGISTRY.register_processor(MllamaMultiModalProcessor,
1146 + info=MllamaProcessingInfo,
1147 + dummy_inputs=MllamaDummyInputsBuilder)
1148 class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1149 packed_modules_mapping = {
1150 "qkv_proj": ["q_proj", "k_proj", "v_proj"],
@@ -1120,7 +1153,7 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1153
1154 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1155 super().__init__()
1123 - config = vllm_config.model_config.hf_config
1156 + config: MllamaConfig = vllm_config.model_config.hf_config
1157 quant_config = vllm_config.quant_config
1158 self.quant_config = quant_config
1159 self.vocab_size = config.text_config.vocab_size
@@ -1130,6 +1163,7 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1163 self.pad_token_id = \
1164 config.pad_token_id if config.pad_token_id is not None else -1
1165 self.image_size = config.vision_config.image_size
1166 + self.image_token_id = config.image_token_index
1167
1168 self.vision_model = MllamaVisionModel(config.vision_config,
1169 quant_config,
@@ -1204,48 +1238,12 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1238 if pixel_values is not None:
1239 assert aspect_ratio_ids is not None
1240 assert aspect_ratio_mask is not None
1207 - max_num_images = max([len(x[0]) for x in pixel_values])
1208 - if max_num_images == 0:
1209 - raise ValueError("No images provided.")
1210 - max_num_tiles = max(
1211 - max([len(x) for x in y[0]]) for y in pixel_values)
1212 - device = next(self.multi_modal_projector.parameters()).device
1213 - bsz = len(pixel_values)
1214 - out_num_tiles = []
1215 - out_images = torch.zeros(
1216 - bsz,
1217 - max_num_images,
1218 - max_num_tiles,
1219 - 3,
1220 - self.image_size,
1221 - self.image_size,
1222 - dtype=torch.float32,
1223 - device=device,
1224 - )
1225 - out_ar_ids = torch.ones(bsz,
1226 - max_num_images,
1227 - dtype=torch.int64,
1228 - device=device)
1229 - out_ar_mask = torch.zeros(bsz,
1230 - max_num_images,
1231 - max_num_tiles,
1232 - dtype=torch.int64,
1233 - device=device)
1234 - for b in range(len(pixel_values)):
1235 - _num_tiles = []
1236 - for i in range(len(pixel_values[b][0])):
1237 - img = pixel_values[b][0][i]
1238 - out_images[b, i, :img.shape[0]] = img
1239 - out_ar_ids[b, i] = aspect_ratio_ids[b][0][i]
1240 - out_ar_mask[b, i] = aspect_ratio_mask[b][0][i]
1241 - _num_tiles.append(img.shape[0])
1242 - out_num_tiles.append(_num_tiles)
1241
1242 return MllamaImagePixelInputs(
1243 type="pixel_values",
1246 - data=out_images,
1247 - aspect_ratio_ids=out_ar_ids,
1248 - aspect_ratio_mask=out_ar_mask,
1244 + data=pixel_values,
1245 + aspect_ratio_ids=aspect_ratio_ids,
1246 + aspect_ratio_mask=aspect_ratio_mask,
1247 )
1248
1249 if image_embeds is not None:
@@ -1312,7 +1310,7 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1310 batch_token_ids.append(token_ids[start:start + seq_len])
1311 start += seq_len
1312 sparse_mask = [
1315 - get_cross_attention_token_mask(t, MLLAMA_IMAGE_TOKEN_ID)
1313 + get_cross_attention_token_mask(t, self.image_token_id)
1314 for t in batch_token_ids
1315 ]
1316
@@ -1384,8 +1382,8 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1382 # block manager to allocate blocks for those images only.
1383 # See input_processor_for_mllama() for more details.
1384 num_tiles_tensor = kwargs.pop("num_tiles")
1387 - num_tiles = [t[0].tolist() for t in num_tiles_tensor]
1388 - num_tokens_per_tile = (self.image_size // 14)**2 + 1
1385 + num_tiles = [t.tolist() for t in num_tiles_tensor]
1386 + num_tokens_per_tile = calc_token_per_chunk(self.image_size)
1387 actual_encoder_seq_lens = [
1388 sum(num_tile) * num_tokens_per_tile for num_tile in num_tiles
1389 ]
vllm/multimodal/inputs.py
+16
@@ -739,3 +739,19 @@ class MultiModalInputs(TypedDict):
739 For each modality, information about the placeholder tokens in
740 :code:`prompt_token_ids`.
741 """
742 +
743 +
744 +class MultiModalEncDecInputs(MultiModalInputs):
745 + """
746 + Represents the outputs of :class:`vllm.multimodal.EncDecMultiModalProcessor`
747 + ready to be passed to vLLM internals.
748 + """
749 +
750 + encoder_prompt: str
751 + """The processed encoder prompt text."""
752 +
753 + encoder_prompt_token_ids: list[int]
754 + """The processed token IDs of the encoder prompt."""
755 +
756 + encoder_token_type_ids: NotRequired[list[int]]
757 + """The token type IDs of the encoder prompt."""
vllm/multimodal/processing.py
+57 -3
@@ -20,9 +20,9 @@ from vllm.transformers_utils.tokenizer import (AnyTokenizer, decode_tokens,
20 from vllm.utils import LRUCache, flatten_2d_lists, full_groupby
21
22 from .hasher import MultiModalHasher
23 -from .inputs import (MultiModalDataDict, MultiModalFieldConfig,
24 - MultiModalInputs, MultiModalKwargs, MultiModalKwargsItem,
25 - PlaceholderRange)
23 +from .inputs import (MultiModalDataDict, MultiModalEncDecInputs,
24 + MultiModalFieldConfig, MultiModalInputs, MultiModalKwargs,
25 + MultiModalKwargsItem, PlaceholderRange)
26 from .parse import MultiModalDataItems, MultiModalDataParser
27
28 if TYPE_CHECKING:
@@ -1293,3 +1293,57 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1293 mm_hashes=mm_hashes,
1294 mm_placeholders=mm_placeholder_ranges,
1295 )
1296 +
1297 +
1298 +class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
1299 +
1300 + @abstractmethod
1301 + def create_encoder_prompt(
1302 + self,
1303 + prompt: Union[str, list[int]],
1304 + mm_data: MultiModalDataDict,
1305 + ) -> Union[str, list[int]]:
1306 + """Create input prompt for the encoder."""
1307 + raise NotImplementedError
1308 +
1309 + def apply(
1310 + self,
1311 + prompt: Union[str, list[int]],
1312 + mm_data: MultiModalDataDict,
1313 + hf_processor_mm_kwargs: Mapping[str, object],
1314 + ) -> MultiModalEncDecInputs:
1315 + """
1316 + Process multi-modal inputs to be used in vLLM.
1317 + The main processing steps are modified to fit encoder-decoder model:
1318 + 1. Create encoder prompt from input prompt text.
1319 + 2. Apply the HF processor on encoder prompt.
1320 + 3. Copy the input prompt text as decoder prompt inputs.
1321 + """
1322 + encoder_prompt = self.create_encoder_prompt(prompt, mm_data)
1323 + encoder_inputs = super().apply(
1324 + encoder_prompt,
1325 + mm_data,
1326 + hf_processor_mm_kwargs,
1327 + )
1328 +
1329 + # We assumed the decoder prompt text is copied from
1330 + # the original encoder prompt without extra process
1331 + tokenizer = self.info.get_tokenizer()
1332 + if isinstance(prompt, str):
1333 + decoder_prompt = prompt
1334 + decoder_prompt_ids = encode_tokens(tokenizer,
1335 + prompt,
1336 + add_special_tokens=False)
1337 + else:
1338 + decoder_prompt = decode_tokens(tokenizer, prompt)
1339 + decoder_prompt_ids = prompt
1340 +
1341 + mm_inputs = MultiModalEncDecInputs(
1342 + encoder_prompt=encoder_inputs["prompt"],
1343 + encoder_prompt_token_ids=encoder_inputs["prompt_token_ids"],
1344 + **encoder_inputs)
1345 + mm_inputs.update({
1346 + "prompt": decoder_prompt,
1347 + "prompt_token_ids": decoder_prompt_ids
1348 + })
1349 + return mm_inputs
vllm/multimodal/profiling.py
+17 -11
@@ -144,7 +144,11 @@ class MultiModalProfiler(Generic[_I]):
144 hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
145 )
146
147 - def get_dummy_data(self, seq_len: int) -> DummyData:
147 + def get_dummy_data(
148 + self,
149 + seq_len: int,
150 + is_encoder_data: bool = False,
151 + ) -> DummyData:
152 # Avoid circular import
153 from vllm.sequence import SequenceData
154
@@ -183,16 +187,18 @@ class MultiModalProfiler(Generic[_I]):
187 total_len = len(prompt_token_ids)
188
189 # V0 does not support chunked prefill.
186 - if total_len > seq_len and not envs.VLLM_USE_V1:
187 - logger.warning(
188 - "The context length (%d) of the model is too short "
189 - "to hold the multi-modal embeddings in the worst case "
190 - "(%d tokens in total, out of which %s are reserved for "
191 - "multi-modal embeddings). This may cause certain multi-modal "
192 - "inputs to fail during inference, even when the input text is "
193 - "short. To avoid this, you should increase `max_model_len`, "
194 - "reduce `max_num_seqs`, and/or reduce `mm_counts`.", seq_len,
195 - total_len, total_placeholders_by_modality)
190 + if (total_len > seq_len and not envs.VLLM_USE_V1) or is_encoder_data:
191 + if total_len > seq_len:
192 + logger.warning(
193 + "The context length (%d) of the model is too short "
194 + "to hold the multi-modal embeddings in the worst case "
195 + "(%d tokens in total, out of which %s are reserved for "
196 + "multi-modal embeddings). This may cause certain "
197 + "multi-modal inputs to fail during inference, even when "
198 + "the input text is short. To avoid this, you should "
199 + "increase `max_model_len`, reduce `max_num_seqs`, "
200 + "and/or reduce `mm_counts`.", seq_len, total_len,
201 + total_placeholders_by_modality)
202
203 return DummyData(
204 seq_data=SequenceData.from_prompt_token_counts((0, seq_len)),