@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e24113a8fe
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Model] Refactor Qwen2-VL to use merged multimodal processor (#11258)

Signed-off-by: Isotr0py <2037008807@qq.com> Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>

Isotr0py committed Dec 20, 2024 at 00:28 UTC e24113a8fe5de5b96459d1f8509d1b48fd7ceebe
5 files changed +277 -527
examples/offline_inference_vision_language.py
+6 -2
@@ -447,7 +447,6 @@ def run_qwen_vl(question: str, modality: str):
447
448 # Qwen2-VL
449 def run_qwen2_vl(question: str, modality: str):
450 - assert modality == "image"
450
451 model_name = "Qwen/Qwen2-VL-7B-Instruct"
452
@@ -463,8 +462,13 @@ def run_qwen2_vl(question: str, modality: str):
462 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
463 )
464
465 + if modality == "image":
466 + placeholder = "<|image_pad|>"
467 + elif modality == "video":
468 + placeholder = "<|video_pad|>"
469 +
470 prompt = ("<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
467 - "<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>"
471 + f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"
472 f"{question}<|im_end|>\n"
473 "<|im_start|>assistant\n")
474 stop_token_ids = None
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_qwen2_vl.py
+65 -127
@@ -1,12 +1,9 @@
1 from typing import Any, Dict, Tuple
2
3 import pytest
4 -import torch
5 -from PIL.Image import Image
4 from transformers import AutoTokenizer
5
8 -from vllm.inputs import InputContext, token_inputs
9 -from vllm.multimodal import MultiModalRegistry
6 +from vllm.inputs import InputContext, InputProcessingContext
7
8 from .....conftest import _ImageAssets
9 from ....utils import build_model_context
@@ -20,22 +17,9 @@ MAX_PIXELS = "max_pixels"
17 # NOTE: Qwen2VL supports multiple input modalities, so it registers multiple
18 # input mappers.
19 @pytest.fixture()
23 -def image_input_mapper_for_qwen2_vl():
24 - from vllm.model_executor.models.qwen2_vl import (
25 - image_input_mapper_for_qwen2_vl)
26 - return image_input_mapper_for_qwen2_vl
27 -
28 -
29 -@pytest.fixture()
30 -def input_processor_for_qwen2_vl():
31 - from vllm.model_executor.models.qwen2_vl import (
32 - input_processor_for_qwen2_vl)
33 - return input_processor_for_qwen2_vl
34 -
35 -
36 -@pytest.fixture()
37 -def qwen2_vl_context() -> InputContext:
38 - return build_model_context(model_name=MODEL)
20 +def processor_for_qwen2_vl():
21 + from vllm.model_executor.models.qwen2_vl import Qwen2VLMultiModalProcessor
22 + return Qwen2VLMultiModalProcessor
23
24
25 @pytest.fixture()
@@ -45,12 +29,6 @@ def get_max_qwen2_vl_image_tokens():
29 return get_max_qwen2_vl_image_tokens
30
31
48 -@pytest.fixture()
49 -def dummy_data_for_qwen2_vl():
50 - from vllm.model_executor.models.qwen2_vl import dummy_data_for_qwen2_vl
51 - return dummy_data_for_qwen2_vl
52 -
53 -
32 @pytest.mark.parametrize("mm_processor_kwargs,expected_max_tokens", [
33 ({}, 1225),
34 ({
@@ -58,110 +36,70 @@ def dummy_data_for_qwen2_vl():
36 MAX_PIXELS: 512**2
37 }, 324),
38 ])
61 -def test_qwen2_vl_max_image_tokens(get_max_qwen2_vl_image_tokens,
62 - qwen2_vl_context: InputContext,
63 - mm_processor_kwargs: Dict[str, Any],
64 - expected_max_tokens: int):
39 +@pytest.mark.parametrize("model", [MODEL])
40 +def test_qwen2_vl_max_image_tokens(
41 + get_max_qwen2_vl_image_tokens,
42 + model: str,
43 + mm_processor_kwargs: Dict[str, Any],
44 + expected_max_tokens: int,
45 +):
46 """Ensure that the max token calc handles min/max pixels properly."""
66 - actual_max_tokens = get_max_qwen2_vl_image_tokens(qwen2_vl_context,
67 - **mm_processor_kwargs)
68 - assert actual_max_tokens == expected_max_tokens
69 -
70 -
71 -@pytest.mark.parametrize("mm_processor_kwargs,token_count,img_size", [
72 - [{}, 1225, (980, 980)],
73 - [{
74 - MIN_PIXELS: 64**2,
75 - MAX_PIXELS: 512**2
76 - }, 324, (504, 504)],
77 -])
78 -def test_qwen2_vl_dummy_data(dummy_data_for_qwen2_vl,
79 - qwen2_vl_context: InputContext,
80 - mm_processor_kwargs: Dict[str, Any],
81 - token_count: int, img_size: Tuple[int, int]):
82 - """Ensure that the dummy data handles min/max pixels properly."""
83 - seq_len = 3000
84 - hf_config = qwen2_vl_context.get_hf_config()
85 - image_token_id = hf_config.image_token_id
86 -
87 - # NOTE: video value is required, but isn't actually used
88 - # when making the dummy data except for error handling currently
89 - dummy_data = dummy_data_for_qwen2_vl(
90 - ctx=qwen2_vl_context,
91 - seq_len=seq_len,
92 - mm_counts={
93 - "image": 1,
94 - "video": 0
95 - },
96 - **mm_processor_kwargs,
47 + ctx = build_model_context(
48 + model_name=model,
49 + tokenizer_name=model,
50 + mm_processor_kwargs=None,
51 )
98 - seq_data = dummy_data.seq_data
99 - mm_data = dummy_data.multi_modal_data
100 -
101 - # Ensure we have the right number of placeholders for min/max pixel values
102 - assert seq_data.get_token_ids().count(image_token_id) == token_count
52
104 - # Ensure the images were resized correctly
105 - image = mm_data["image"]
106 - assert isinstance(image, Image)
107 - assert image.size == img_size
53 + actual_max_tokens = get_max_qwen2_vl_image_tokens(
54 + InputContext(ctx.model_config), **mm_processor_kwargs)
55 + assert actual_max_tokens == expected_max_tokens
56
57
110 -@pytest.mark.parametrize("mm_processor_kwargs,num_placeholders", [
111 - ({}, 1426),
112 - ({
113 - MIN_PIXELS: 64**2,
114 - MAX_PIXELS: 512**2
115 - }, 330),
116 -])
117 -def test_input_processor(input_processor_for_qwen2_vl,
118 - qwen2_vl_context: InputContext,
119 - image_assets: _ImageAssets, num_placeholders: int,
120 - mm_processor_kwargs: Dict[str, Any]):
121 - """Ensure that the image processor handles min/max pixels properly."""
122 - tokenizer = AutoTokenizer.from_pretrained(MODEL)
123 - prompt = "<|vision_start|><|image_pad|><|vision_end|>"
124 -
125 - image = image_assets[0].pil_image
126 - hf_config = qwen2_vl_context.get_hf_config()
127 - image_token_id = hf_config.image_token_id
128 -
129 - inputs = token_inputs(prompt_token_ids=tokenizer.encode(prompt),
130 - prompt=prompt,
131 - multi_modal_data={"image": [image]})
132 -
133 - processed_inputs = input_processor_for_qwen2_vl(qwen2_vl_context, inputs,
134 - **mm_processor_kwargs)
135 - assert processed_inputs["prompt_token_ids"].count(
136 - image_token_id) == num_placeholders
137 - assert len(processed_inputs["multi_modal_data"]["image"]) == 1
138 -
139 -
140 -@pytest.mark.parametrize("mm_processor_kwargs,pixels_shape", [
141 - ({}, [5704, 1176]),
142 - ({
143 - MIN_PIXELS: 64**2,
144 - MAX_PIXELS: 512**2
145 - }, [1320, 1176]),
146 -])
147 -def test_image_mapper_override(qwen2_vl_context: InputContext,
148 - image_assets: _ImageAssets,
149 - mm_processor_kwargs: Dict[str, Any],
150 - pixels_shape: Tuple[int, int]):
151 - """Ensure that the image mapper handles min/max pixels properly."""
152 - mm_registry = MultiModalRegistry()
153 - mm_registry.init_mm_limits_per_prompt(qwen2_vl_context.model_config)
154 -
155 - image = image_assets[0].pil_image
156 -
157 - mapped_output = mm_registry.map_input(
158 - qwen2_vl_context.model_config,
159 - {"image": image},
160 - mm_processor_kwargs=mm_processor_kwargs,
58 +@pytest.mark.parametrize(
59 + "mm_processor_kwargs, expected_toks_per_img, expected_pixels_shape", [
60 + ({}, 1426, (5704, 1176)),
61 + ({
62 + MIN_PIXELS: 64**2,
63 + MAX_PIXELS: 512**2
64 + }, 330, (1320, 1176)),
65 + ])
66 +@pytest.mark.parametrize("model", [MODEL])
67 +@pytest.mark.parametrize("num_imgs", [1, 2])
68 +def test_processor_override(
69 + processor_for_qwen2_vl,
70 + image_assets: _ImageAssets,
71 + model: str,
72 + mm_processor_kwargs: Dict[str, Any],
73 + expected_toks_per_img: int,
74 + expected_pixels_shape: Tuple[int, int],
75 + num_imgs: int,
76 +):
77 + """Ensure Qwen2VLMultiModalProcessor handles min/max pixels properly."""
78 + # Same as the previous test - don't initialize mm_processor_kwargs
79 + # in this test and assume that the kwargs will be correctly expanded by
80 + # the partial when calling the custom input processor.
81 + ctx = build_model_context(
82 + model_name=model,
83 + tokenizer_name=model,
84 + mm_processor_kwargs=None,
85 )
162 -
163 - # Dimension 0 of pixel values should match the product of image_grid_thw
164 - actual_pixels_shape = mapped_output["pixel_values"].shape
165 - assert list(actual_pixels_shape) == pixels_shape
166 - assert actual_pixels_shape[0] == torch.prod(
167 - mapped_output["image_grid_thw"])
86 + tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True)
87 + ctx = InputProcessingContext(ctx.model_config, tokenizer)
88 + # Build the image str / prompt based on the number of images we pass
89 + prompt = "<|vision_start|><|image_pad|><|vision_end|>" * num_imgs
90 + images = [image_assets[0].pil_image] * num_imgs
91 +
92 + mm_data = {"image": images}
93 +
94 + processor = processor_for_qwen2_vl(ctx)
95 + processed_inputs = processor.apply(prompt, mm_data, mm_processor_kwargs)
96 +
97 + # Ensure we have the right number of placeholders per num_crops size
98 + hf_processor = processor._get_hf_processor(**mm_processor_kwargs)
99 + image_token_id = tokenizer.convert_tokens_to_ids(hf_processor.image_token)
100 + img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
101 + pixel_shape = processed_inputs["mm_kwargs"]["pixel_values"].shape
102 +
103 + assert img_tok_count == expected_toks_per_img * num_imgs
104 + assert pixel_shape[0] == expected_pixels_shape[0] * num_imgs
105 + assert pixel_shape[1] == expected_pixels_shape[1]
vllm/model_executor/models/qwen2_audio.py
+3 -1
@@ -164,7 +164,9 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
164 self,
165 mm_counts: Mapping[str, int],
166 ) -> ProcessorInputs:
167 - audio_len = get_max_qwen2_audio_audio_tokens(self.ctx)
167 + feature_extractor = self._get_feature_extractor()
168 + sampling_rate = feature_extractor.sampling_rate
169 + audio_len = feature_extractor.chunk_length * sampling_rate
170
171 audio_count = mm_counts["audio"]
172 audio = np.zeros(audio_len)
vllm/model_executor/models/qwen2_vl.py
+187 -393
@@ -22,28 +22,26 @@
22 # limitations under the License.
23 """Inference-only Qwen2-VL model compatible with HuggingFace weights."""
24 from functools import cached_property, partial
25 -from typing import (Any, Callable, Dict, Iterable, List, Literal, Mapping,
26 - Optional, Set, Tuple, Type, TypedDict, Union)
25 +from typing import (Any, Iterable, List, Literal, Mapping, Optional, Set,
26 + Tuple, Type, TypedDict, Union)
27
28 import torch
29 import torch.nn as nn
30 import torch.nn.functional as F
31 from einops import rearrange, repeat
32 from PIL import Image
33 -from transformers.image_utils import (get_image_size,
34 - infer_channel_dimension_format,
35 - to_numpy_array)
33 +from transformers import BatchFeature
34 +from transformers.models.qwen2_vl import (Qwen2VLImageProcessor,
35 + Qwen2VLProcessor)
36 from transformers.models.qwen2_vl.configuration_qwen2_vl import (
37 Qwen2VLConfig, Qwen2VLVisionConfig)
38 -from transformers.models.qwen2_vl.image_processing_qwen2_vl import (
39 - make_batched_images, make_batched_videos, smart_resize)
38 +from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize
39
40 from vllm.attention import AttentionMetadata
41 from vllm.config import VllmConfig
42 from vllm.distributed import parallel_state
43 from vllm.distributed import utils as dist_utils
45 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
46 - InputContext, token_inputs)
44 +from vllm.inputs import InputContext
45 from vllm.logger import init_logger
46 from vllm.model_executor import SamplingMetadata
47 from vllm.model_executor.layers.activation import QuickGELU
@@ -56,14 +54,14 @@ from vllm.model_executor.layers.quantization.gptq_marlin import (
54 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
55 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
56 from vllm.multimodal import MULTIMODAL_REGISTRY
59 -from vllm.multimodal.image import cached_get_image_processor
60 -from vllm.multimodal.inputs import (MultiModalData, MultiModalDataDict,
61 - MultiModalKwargs, NestedTensors)
62 -from vllm.multimodal.utils import cached_get_tokenizer
57 +from vllm.multimodal.inputs import MultiModalDataDict, NestedTensors
58 +from vllm.multimodal.processing import (BaseMultiModalProcessor,
59 + MultiModalDataItems, ProcessorInputs,
60 + PromptReplacement)
61 from vllm.platforms import _Backend
64 -from vllm.sequence import IntermediateTensors, SequenceData
62 +from vllm.sequence import IntermediateTensors
63 from vllm.transformers_utils.config import uses_mrope
66 -from vllm.transformers_utils.processor import cached_get_processor
64 +from vllm.utils import is_list_of
65
66 from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
67 from .utils import (AutoWeightsLoader, WeightsMapper, get_vit_attn_backend,
@@ -159,7 +157,7 @@ class Qwen2VisionMLP(nn.Module):
157 def __init__(
158 self,
159 in_features: int,
162 - hidden_features: int = None,
160 + hidden_features: int,
161 act_layer: Type[nn.Module] = QuickGELU,
162 quant_config: Optional[QuantizationConfig] = None,
163 prefix: str = "",
@@ -644,78 +642,8 @@ class Qwen2VisionTransformer(nn.Module):
642 # === Vision input helpers === #
643
644
647 -def get_mm_processor_kwargs(
648 - min_pixels: Optional[int] = None,
649 - max_pixels: Optional[int] = None) -> Dict[str, int]:
650 - mm_processor_kwargs = {}
651 - if min_pixels:
652 - mm_processor_kwargs["min_pixels"] = min_pixels
653 - if max_pixels:
654 - mm_processor_kwargs["max_pixels"] = max_pixels
655 - return mm_processor_kwargs
656 -
657 -
658 -def mm_input_mapper_for_qwen2_vl(
659 - ctx: InputContext,
660 - data: MultiModalData[object],
661 - data_type_key: str,
662 - *,
663 - min_pixels: Optional[int] = None,
664 - max_pixels: Optional[int] = None,
665 -) -> MultiModalKwargs:
666 - """Input mapper for Qwen2-VL."""
667 - if data_type_key == "image" and isinstance(data, dict):
668 - return MultiModalKwargs({
669 - "image_embeds": data.get("image_embeds"),
670 - "image_grid_thw": data.get("image_grid_thw"),
671 - })
672 - if data_type_key == "video" and isinstance(data, dict):
673 - return MultiModalKwargs({
674 - "video_embeds": data.get("video_embeds"),
675 - "video_grid_thw": data.get("video_grid_thw"),
676 - })
677 -
678 - model_config = ctx.model_config
679 - # Handle mm processor kwargs; we pass these at creation time
680 - # because preprocess() in transformers doesn't expose them
681 - mm_processor_kwargs = get_mm_processor_kwargs(min_pixels=min_pixels,
682 - max_pixels=max_pixels)
683 - image_processor = cached_get_image_processor(
684 - model_config.model,
685 - trust_remote_code=model_config.trust_remote_code,
686 - **mm_processor_kwargs,
687 - )
688 - if image_processor is None:
689 - raise RuntimeError("No HuggingFace processor is available "
690 - "to process the image object")
691 -
692 - images = None
693 - videos = None
694 - if data_type_key == "image":
695 - images = data
696 - else:
697 - assert data_type_key == "video"
698 - videos = data
699 -
700 - try:
701 - batch_data = image_processor \
702 - .preprocess(images=images, videos=videos, return_tensors="pt") \
703 - .data
704 - except Exception:
705 - logger.error("Failed to process image (%s)", data)
706 - raise
707 -
708 - return MultiModalKwargs(batch_data)
709 -
710 -
711 -image_input_mapper_for_qwen2_vl = partial(mm_input_mapper_for_qwen2_vl,
712 - data_type_key="image")
713 -video_input_mapper_for_qwen2_vl = partial(mm_input_mapper_for_qwen2_vl,
714 - data_type_key="video")
715 -
716 -
645 def _get_vision_info(
718 - image_processor,
646 + vision_config: Qwen2VLVisionConfig,
647 height: int,
648 width: int,
649 min_pixels: int,
@@ -726,12 +654,15 @@ def _get_vision_info(
654 ):
655 """Get information (resized height / width and number of vision tokens)
656 of input image / video frame."""
657 + patch_size = vision_config.patch_size
658 + merge_size = vision_config.spatial_merge_size
659 + temporal_patch_size = vision_config.temporal_patch_size
660
661 if do_resize:
662 resized_height, resized_width = smart_resize(
663 height=height,
664 width=width,
734 - factor=image_processor.patch_size * image_processor.merge_size,
665 + factor=patch_size * merge_size,
666 min_pixels=min_pixels,
667 max_pixels=max_pixels,
668 )
@@ -742,54 +673,41 @@ def _get_vision_info(
673 grid_t = mm_count
674 else:
675 assert data_type_key == "video"
745 - grid_t = max(mm_count // image_processor.temporal_patch_size, 1)
676 + grid_t = max(mm_count // temporal_patch_size, 1)
677
747 - grid_h = resized_height // image_processor.patch_size
748 - grid_w = resized_width // image_processor.patch_size
678 + grid_h = resized_height // patch_size
679 + grid_w = resized_width // patch_size
680 vision_tokens = grid_t * grid_h * grid_w
750 - llm_num_vision_tokens = (vision_tokens // image_processor.merge_size //
751 - image_processor.merge_size)
681 + llm_num_vision_tokens = vision_tokens // (merge_size**2)
682
683 return resized_height, resized_width, llm_num_vision_tokens
684
685
756 -def _get_max_image_info(
757 - image_processor,
758 - data_type_key: str = "image",
759 - mm_count: int = 1,
760 - min_pixels: Optional[int] = None,
761 - max_pixels: Optional[int] = None,
762 -):
763 - # Limit min / max pixels unless they're explicitly provided
764 - if min_pixels is None:
765 - min_pixels = max(image_processor.min_pixels, 28 * 28)
766 - if max_pixels is None:
767 - max_pixels = min(image_processor.max_pixels, 1280 * 28 * 28)
768 -
769 - return _get_vision_info(
770 - image_processor,
771 - height=9999999,
772 - width=9999999,
773 - min_pixels=min_pixels,
774 - max_pixels=max_pixels,
775 - data_type_key=data_type_key,
776 - mm_count=mm_count,
777 - )
686 +def _get_image_processor(hf_processor: Qwen2VLProcessor):
687 + image_processor = hf_processor.image_processor # type: ignore
688 + assert isinstance(image_processor, Qwen2VLImageProcessor)
689 + return image_processor
690
691
692 def get_max_qwen2_vl_mm_tokens(ctx: InputContext,
693 data_type_key: str,
694 *,
783 - min_pixels=None,
784 - max_pixels=None) -> int:
785 - mm_processor_kwargs = get_mm_processor_kwargs(min_pixels=min_pixels,
786 - max_pixels=max_pixels)
787 - image_processor = cached_get_image_processor(ctx.model_config.model,
788 - **mm_processor_kwargs)
789 - max_resized_height, max_resized_width, max_llm_image_tokens = \
790 - _get_max_image_info(image_processor, data_type_key=data_type_key,
791 - mm_count=1, min_pixels=min_pixels,
792 - max_pixels=max_pixels)
695 + min_pixels: Optional[int] = None,
696 + max_pixels: Optional[int] = None) -> int:
697 + hf_config = ctx.get_hf_config(Qwen2VLConfig)
698 + vision_config = hf_config.vision_config
699 +
700 + hf_processor = ctx.get_hf_processor(Qwen2VLProcessor)
701 + image_processor = _get_image_processor(hf_processor)
702 +
703 + _, _, max_llm_image_tokens = _get_vision_info(
704 + vision_config,
705 + height=9999999,
706 + width=9999999,
707 + min_pixels=min_pixels or image_processor.min_pixels,
708 + max_pixels=max_pixels or image_processor.max_pixels,
709 + data_type_key=data_type_key,
710 + )
711 return max_llm_image_tokens
712
713
@@ -799,290 +717,166 @@ get_max_qwen2_vl_video_tokens = partial(get_max_qwen2_vl_mm_tokens,
717 data_type_key="video")
718
719
802 -def dummy_data_for_qwen2_vl(
803 - ctx: InputContext,
804 - seq_len: int,
805 - mm_counts: Mapping[str, int],
806 - *,
807 - min_pixels: Optional[int] = None,
808 - max_pixels: Optional[int] = None
809 -) -> Tuple[SequenceData, Optional[MultiModalDataDict]]:
810 - mm_processor_kwargs = get_mm_processor_kwargs(min_pixels=min_pixels,
811 - max_pixels=max_pixels)
812 - image_processor = cached_get_image_processor(ctx.model_config.model,
813 - **mm_processor_kwargs)
814 -
815 - num_images = mm_counts["image"]
816 - max_resized_height, max_resized_width, max_llm_image_tokens = \
817 - _get_max_image_info(image_processor, data_type_key="image",
818 - mm_count=num_images, min_pixels=min_pixels,
819 - max_pixels=max_pixels)
820 - if seq_len - max_llm_image_tokens - 2 < 0:
821 - raise RuntimeError(
822 - f"Qwen2-VL cannot process {num_images} images in a prompt, "
823 - "please increase max_model_len or reduce image limit by "
824 - "--limit-mm-per-prompt.")
825 -
826 - # Check video counts.
827 - num_videos = mm_counts["video"]
828 - max_resized_height, max_resized_width, max_llm_video_tokens = \
829 - _get_max_image_info(image_processor, data_type_key="video",
830 - mm_count=num_videos, min_pixels=min_pixels,
831 - max_pixels=max_pixels)
832 - if seq_len - max_llm_video_tokens - 2 < 0:
833 - raise RuntimeError(
834 - f"Qwen2-VL cannot process {num_videos} videos in a prompt, "
835 - "please increase max_model_len or reduce video limit by "
836 - "--limit-mm-per-prompt.")
837 -
838 - hf_config = ctx.get_hf_config(Qwen2VLConfig)
839 -
840 - dummy_seqdata = SequenceData.from_prompt_token_counts(
841 - (hf_config.vision_start_token_id, 1),
842 - (hf_config.image_token_id, max_llm_image_tokens),
843 - (hf_config.vision_end_token_id, 1),
844 - (0, seq_len - max_llm_image_tokens - 2),
845 - )
846 -
847 - dummy_image = Image.new("RGB", (max_resized_width, max_resized_height),
848 - color=0)
720 +class Qwen2VLMultiModalDataItems(MultiModalDataItems):
721
850 - return DummyData(dummy_seqdata, {
851 - "image":
852 - dummy_image if num_images == 1 else [dummy_image] * num_images
853 - })
722 + @staticmethod
723 + def from_dict(data: MultiModalDataDict) -> "MultiModalDataItems":
724 + """
725 + Normalize :class:`MultiModalDataDict` to :class:`MultiModalDataItems`.
726 + """
727 + multi_data = Qwen2VLMultiModalDataItems()
728 +
729 + for k, v in data.items():
730 + # TODO: Make a separate modality for embedding inputs
731 + # to avoid confusion
732 + # yapf: disable
733 + if k == "video":
734 + # Special case since even a single item can be a list
735 + multi_data[k] = ( # type: ignore[index]
736 + v if (isinstance(v, (dict, torch.Tensor)) # type: ignore[assignment]
737 + or is_list_of(v, list)) else [v]
738 + )
739 + elif k in ("image", "audio"):
740 + multi_data[k] = ( # type: ignore[index]
741 + v if isinstance(v, (dict, torch.Tensor, list)) else [v]
742 + )
743 + else:
744 + multi_data[k] = v if isinstance(v, list) else [v] # type: ignore[index]
745 + # yapf: enable
746
747 + return multi_data
748
856 -def _get_llm_num_vision_tokens(
857 - mm_inputs: list,
858 - data_type_key: str,
859 - image_processor,
860 - min_pixels: int,
861 - max_pixels: int,
862 -):
863 - """Get number of vision tokens of multimodal inputs.
749 + def get_item_counts(self) -> Mapping[str, int]:
750 + return {
751 + m: (
752 + len(items[f"{m}_grid_thw"]) # type: ignore
753 + if isinstance(items, dict) else len(items))
754 + for m, items in self.items()
755 + }
756
865 - This method is derived from `transformers.models.qwen2_vl.
866 - image_processing_qwen2_vl.Qwen2VLImageProcessor._preprocess`.
867 - """
868 - image = to_numpy_array(mm_inputs[0])
869 - input_data_format = infer_channel_dimension_format(image)
870 - height, width = get_image_size(image, channel_dim=input_data_format)
871 -
872 - _, _, llm_num_vision_tokens = _get_vision_info(
873 - image_processor,
874 - height=height,
875 - width=width,
876 - min_pixels=min_pixels,
877 - max_pixels=max_pixels,
878 - do_resize=image_processor.do_resize,
879 - data_type_key=data_type_key,
880 - mm_count=len(mm_inputs),
881 - )
882 - return llm_num_vision_tokens
757
758 +class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
759
885 -def _expand_pad_tokens(inputs: list, token_id: int, make_batched_fn: Callable,
886 - data_type_key: str, image_processor: Any,
887 - prompt_token_ids: List[int], min_pixels: Optional[int],
888 - max_pixels: Optional[int]) -> List[int]:
889 - """
890 - Expand pad tokens for multi-modal inputs (e.g., images or videos).
891 -
892 - Args:
893 - inputs (list): The multi-modal inputs (e.g., images or videos).
894 - token_id (int): The token ID used to represent the multi-modal input.
895 - make_batched_fn (Callable): A function to batch the inputs.
896 - data_type_key (str): The type of the multi-modal input.
897 - image_processor (Any): The image processor used to process the inputs.
898 - prompt_token_ids (List[int]): The list of token IDs in the prompt.
899 - min_pixels (int): min pixels to used for img processing
900 - max_pixels (int): max pixels to be used for img processing
901 -
902 - Returns:
903 - List[int]: The list of token IDs for the multi-modal inputs.
904 - """
905 - indices = [
906 - idx for idx, token in enumerate(prompt_token_ids) if token == token_id
907 - ]
908 - inputs = make_batched_fn(inputs)
909 - assert len(indices) == len(inputs)
910 -
911 - prompt_token_ids_with_data = []
912 - for cnt, data in enumerate(inputs):
913 - num_tokens = _get_llm_num_vision_tokens(
914 - [data] if data_type_key == "image" else data,
915 - data_type_key=data_type_key,
916 - image_processor=image_processor,
917 - min_pixels=min_pixels,
918 - max_pixels=max_pixels,
919 - )
920 - if cnt == 0:
921 - end_idx = indices[cnt]
922 - non_data_tokens = prompt_token_ids[:end_idx]
923 - else:
924 - non_data_tokens = prompt_token_ids[indices[cnt - 1] +
925 - 1:indices[cnt]]
926 - prompt_token_ids_with_data.extend(non_data_tokens)
927 - prompt_token_ids_with_data.extend(token_id for _ in range(num_tokens))
928 - prompt_token_ids_with_data.extend(prompt_token_ids[indices[-1] + 1:])
929 - return prompt_token_ids_with_data
930 -
931 -
932 -def input_processor_for_qwen2_vl(
933 - ctx: InputContext,
934 - inputs: DecoderOnlyInputs,
935 - *,
936 - min_pixels: Optional[int] = None,
937 - max_pixels: Optional[int] = None,
938 -) -> DecoderOnlyInputs:
939 - multi_modal_data = inputs.get("multi_modal_data")
940 - if multi_modal_data is None:
941 - return inputs
942 -
943 - image_inputs = multi_modal_data.get("image", None)
944 - video_inputs = multi_modal_data.get("video", None)
945 -
946 - processor = cached_get_processor(ctx.model_config.model)
947 - image_processor = processor.image_processor
948 - # Apply processor kwarg overrides for image processor options
949 - min_pixels = min_pixels if min_pixels else image_processor.min_pixels
950 - max_pixels = max_pixels if max_pixels else image_processor.max_pixels
951 -
952 - model_config = ctx.model_config
953 - hf_config = ctx.get_hf_config(Qwen2VLConfig)
760 + def _get_mm_items(
761 + self,
762 + mm_data: MultiModalDataDict,
763 + ) -> MultiModalDataItems:
764 + return Qwen2VLMultiModalDataItems.from_dict(mm_data)
765
955 - # To avoid redundant processing of vision objects (resize, rescale, etc.),
956 - # we extract code of calculating number of vision tokens from
957 - # `transformers.models.qwen2_vl.processing_qwen2_vl.Qwen2VLProcessor`.
958 - #
959 - # The following code is equivalent to:
960 - # prompt = inputs["prompt"]
961 - # inputs = processor(text=[prompt],
962 - # images=image_inputs,
963 - # videos=video_inputs,
964 - # padding=True,
965 - # return_tensors="pt")
966 - # prompt_token_ids = inputs["input_ids"][0].tolist()
967 -
968 - tokenizer = cached_get_tokenizer(
969 - model_config.tokenizer,
970 - trust_remote_code=model_config.trust_remote_code)
971 -
972 - prompt_token_ids = inputs["prompt_token_ids"]
973 -
974 - # Expand image pad tokens.
975 -
976 - if image_inputs is not None:
977 - if isinstance(image_inputs, dict):
978 - prompt_token_ids_with_image = []
979 - image_indices = [
980 - idx for idx, token in enumerate(prompt_token_ids)
981 - if token == hf_config.image_token_id
982 - ]
983 -
984 - # ensure all image tokens have grid_thw
985 - assert \
986 - len(image_indices) == image_inputs["image_grid_thw"].size(0), \
987 - "image token num does not match image_grid_thw.shape"
988 -
989 - image_counter = 0
990 - pad_token_counter = 0
991 - for idx, token in enumerate(prompt_token_ids):
992 - if idx in image_indices:
993 - grid_thw = image_inputs["image_grid_thw"][image_counter]
994 - grid_t, grid_h, grid_w = grid_thw
995 - num_pad_tokens = (grid_t * grid_h * grid_w //
996 - image_processor.merge_size //
997 - image_processor.merge_size)
998 - prompt_token_ids_with_image.extend([token] *
999 - num_pad_tokens)
1000 - image_counter += 1
1001 - pad_token_counter += num_pad_tokens
766 + def _get_hf_processor(
767 + self,
768 + *,
769 + min_pixels: Optional[int] = None,
770 + max_pixels: Optional[int] = None,
771 + ) -> Qwen2VLProcessor:
772 + hf_processor = self.ctx.get_hf_processor(Qwen2VLProcessor)
773 + image_processor = _get_image_processor(hf_processor)
774 +
775 + if min_pixels:
776 + image_processor.min_pixels = min_pixels
777 + if max_pixels:
778 + image_processor.max_pixels = max_pixels
779 + if max_pixels or min_pixels:
780 + image_processor.size = {
781 + "min_pixels": image_processor.min_pixels,
782 + "max_pixels": image_processor.max_pixels,
783 + }
784 +
785 + return hf_processor
786 +
787 + def _get_processor_data(
788 + self,
789 + mm_items: MultiModalDataItems,
790 + ) -> tuple[dict[str, Any], dict[str, Any]]:
791 + processor_data = dict[str, Any]()
792 + passthrough_data = dict[str, Any]()
793 +
794 + for k, v in mm_items.items():
795 + # TODO: Make a separate modality for embedding inputs
796 + # to avoid confusion
797 + if k in ("image", "video", "audio"):
798 + if isinstance(v, dict):
799 + # Pass through embedding inputs (dict)
800 + passthrough_data.update(v)
801 + elif isinstance(v, torch.Tensor) and v.ndim == 3:
802 + # Pass through embedding inputs (single)
803 + passthrough_data[f"{k}_embeds"] = [v]
804 + elif (is_list_of(v, torch.Tensor) and len(v) > 0
805 + and v[0].ndim == 2):
806 + # Pass through embedding inputs (multi)
807 + passthrough_data[f"{k}_embeds"] = v
808 else:
1003 - prompt_token_ids_with_image.append(token)
809 + # Map keys to plural form, e.g.: image -> images
810 + processor_data[f"{k}s"] = v
811 + else:
812 + processor_data[k] = v
813
1005 - # ensure all embeddings are used
1006 - assert \
1007 - pad_token_counter == image_inputs["image_embeds"].size(0), \
1008 - "image_embeds.shape does not match image_grid_thw"
814 + return processor_data, passthrough_data
815
1010 - prompt_token_ids = prompt_token_ids_with_image
1011 - else:
1012 - prompt_token_ids = _expand_pad_tokens(image_inputs,
1013 - hf_config.image_token_id,
1014 - make_batched_images,
1015 - "image",
1016 - image_processor,
1017 - prompt_token_ids,
1018 - min_pixels=min_pixels,
1019 - max_pixels=max_pixels)
1020 -
1021 - if video_inputs is not None:
1022 - if isinstance(video_inputs, dict):
1023 - prompt_token_ids_with_video = []
1024 - video_indices = [
1025 - idx for idx, token in enumerate(prompt_token_ids)
1026 - if token == hf_config.video_token_id
1027 - ]
1028 -
1029 - # ensure all video tokens have grid_thw
1030 - assert \
1031 - len(video_indices) == video_inputs["video_grid_thw"].size(0), \
1032 - "video token num does not match video_grid_thw.shape"
1033 -
1034 - video_counter = 0
1035 - pad_token_counter = 0
1036 - for idx, token in enumerate(prompt_token_ids):
1037 - if idx in video_indices:
1038 - grid_thw = video_inputs["video_grid_thw"][video_counter]
1039 - grid_t, grid_h, grid_w = grid_thw
1040 - num_pad_tokens = (grid_t * grid_h * grid_w //
1041 - image_processor.merge_size //
1042 - image_processor.merge_size)
1043 - prompt_token_ids_with_video.extend([token] *
1044 - num_pad_tokens)
1045 - video_counter += 1
1046 - pad_token_counter += num_pad_tokens
1047 - else:
1048 - prompt_token_ids_with_video.append(token)
816 + def _get_prompt_replacements(
817 + self,
818 + mm_items: MultiModalDataItems,
819 + hf_inputs: BatchFeature,
820 + mm_processor_kwargs: Mapping[str, object],
821 + ) -> list[PromptReplacement]:
822 + hf_processor = self._get_hf_processor()
823 + image_processor = _get_image_processor(hf_processor)
824 +
825 + # NOTE: Only Qwen2VLProcessor in transformers 4.47.0 has
826 + # image_token and video_token registered
827 + placeholder = {
828 + "image": hf_processor.image_token,
829 + "video": hf_processor.video_token,
830 + }
831 + merge_length = image_processor.merge_size**2
832 +
833 + def get_replacement_qwen2vl(item_idx: int, modality: str):
834 + grid_thw = hf_inputs[f"{modality}_grid_thw"][item_idx]
835 + num_tokens = grid_thw.prod() // merge_length
836 + return placeholder[modality] * num_tokens
837 +
838 + return [
839 + PromptReplacement(
840 + modality=modality,
841 + target=placeholder[modality],
842 + replacement=partial(get_replacement_qwen2vl,
843 + modality=modality),
844 + ) for modality in ("image", "video")
845 + ]
846
1050 - # ensure all embeddings are used
1051 - assert \
1052 - pad_token_counter == video_inputs["video_embeds"].size(0), \
1053 - "video_embeds.shape does not match video_grid_thw"
847 + def _get_dummy_mm_inputs(
848 + self,
849 + mm_counts: Mapping[str, int],
850 + ) -> ProcessorInputs:
851 + num_images = mm_counts["image"]
852 + hf_processor = self._get_hf_processor()
853 + image_token: str = hf_processor.image_token
854 + image_processor = _get_image_processor(hf_processor)
855 +
856 + data = {}
857 + resized_height, resized_width = smart_resize(
858 + height=9999999,
859 + width=9999999,
860 + factor=image_processor.patch_size * image_processor.merge_size,
861 + min_pixels=image_processor.min_pixels,
862 + max_pixels=image_processor.max_pixels,
863 + )
864
1055 - prompt_token_ids = prompt_token_ids_with_video
1056 - else:
1057 - prompt_token_ids = _expand_pad_tokens(video_inputs,
1058 - hf_config.video_token_id,
1059 - make_batched_videos,
1060 - "video",
1061 - image_processor,
1062 - prompt_token_ids,
1063 - min_pixels=min_pixels,
1064 - max_pixels=max_pixels)
1065 -
1066 - prompt = inputs.get("prompt")
1067 - if prompt is None:
1068 - prompt = tokenizer.decode(prompt_token_ids)
1069 -
1070 - return token_inputs(
1071 - prompt_token_ids=prompt_token_ids,
1072 - prompt=prompt,
1073 - multi_modal_data=multi_modal_data,
1074 - )
865 + dummy_image = Image.new("RGB", (resized_width, resized_height),
866 + color=0)
867 + data["image"] = [dummy_image] * num_images
868 +
869 + return ProcessorInputs(
870 + prompt_text=image_token * num_images,
871 + mm_data=data,
872 + mm_processor_kwargs={},
873 + )
874
875
1077 -@MULTIMODAL_REGISTRY.register_image_input_mapper(
1078 - image_input_mapper_for_qwen2_vl)
1079 -@MULTIMODAL_REGISTRY.register_input_mapper("video",
1080 - video_input_mapper_for_qwen2_vl)
876 @MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_qwen2_vl_image_tokens)
877 @MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
878 "video", get_max_qwen2_vl_video_tokens)
1084 -@INPUT_REGISTRY.register_dummy_data(dummy_data_for_qwen2_vl)
1085 -@INPUT_REGISTRY.register_input_processor(input_processor_for_qwen2_vl)
879 +@MULTIMODAL_REGISTRY.register_processor(Qwen2VLMultiModalProcessor)
880 class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
881 SupportsLoRA, SupportsPP):
882 packed_modules_mapping = {
@@ -1110,7 +904,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
904
905 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
906 super().__init__()
1113 - config = vllm_config.model_config.hf_config
907 + config: Qwen2VLConfig = vllm_config.model_config.hf_config
908 cache_config = vllm_config.cache_config
909 quant_config = vllm_config.quant_config
910 multimodal_config = vllm_config.model_config.multimodal_config
vllm/multimodal/processing.py
+16 -4
@@ -220,15 +220,18 @@ class MultiModalDataItems(UserDict[str, list[Any]]):
220 multi_data = MultiModalDataItems()
221
222 for k, v in data.items():
223 + # TODO: Make a separate modality for embedding inputs
224 + # to avoid confusion
225 # yapf: disable
226 if k == "video":
227 # Special case since even a single item can be a list
228 multi_data[k] = ( # type: ignore[index]
227 - v if is_list_of(v, (list, torch.Tensor)) else [v]
229 + v if (isinstance(v, torch.Tensor)
230 + or is_list_of(v, list)) else [v]
231 )
232 elif k in ("image", "audio"):
233 multi_data[k] = ( # type: ignore[index]
231 - v if isinstance(v, (list, torch.Tensor)) else [v]
234 + v if isinstance(v, (torch.Tensor, list)) else [v]
235 )
236 else:
237 multi_data[k] = v if isinstance(v, list) else [v] # type: ignore[index]
@@ -252,6 +255,9 @@ class MultiModalDataItems(UserDict[str, list[Any]]):
255 def audios(self) -> Sequence[AudioItem]:
256 return self.get("audio", [])
257
258 + def get_item_counts(self) -> Mapping[str, int]:
259 + return {m: len(items) for m, items in self.items()}
260 +
261 def get_image_size(self, item_idx: int) -> ImageSize:
262 image = self.images[item_idx]
263
@@ -612,6 +618,12 @@ class BaseMultiModalProcessor(ABC):
618 def _get_tokenizer(self) -> AnyTokenizer:
619 return self.ctx.tokenizer
620
621 + def _get_mm_items(
622 + self,
623 + mm_data: MultiModalDataDict,
624 + ) -> MultiModalDataItems:
625 + return MultiModalDataItems.from_dict(mm_data)
626 +
627 @abstractmethod
628 def _get_prompt_replacements(
629 self,
@@ -778,7 +790,7 @@ class BaseMultiModalProcessor(ABC):
790 3. Extract information about the placeholder tokens from the
791 processed token IDs.
792 """
781 - mm_items = MultiModalDataItems.from_dict(mm_data)
793 + mm_items = self._get_mm_items(mm_data)
794
795 hf_inputs = self._apply_hf_processor(prompt_text, mm_items,
796 mm_processor_kwargs)
@@ -791,7 +803,7 @@ class BaseMultiModalProcessor(ABC):
803
804 # If HF processor already inserts placeholder tokens,
805 # there is no need for us to insert them
794 - mm_item_counts = {m: len(items) for m, items in mm_items.items()}
806 + mm_item_counts = mm_items.get_item_counts()
807 all_placeholders = self._find_placeholders(all_prompt_repls,
808 prompt_ids, mm_item_counts)
809