A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Model] Refactor Qwen2-VL to use merged multimodal processor (#11258)
Signed-off-by: Isotr0py <2037008807@qq.com> Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
Isotr0py committed
Dec 20, 2024 at 00:28 UTC
e24113a8fe5de5b96459d1f8509d1b48fd7ceebe
5 files changed
+277
-527
examples/offline_inference_vision_language.py
+6
-2
@@ -447,7 +447,6 @@ def run_qwen_vl(question: str, modality: str):
447
448
# Qwen2-VL
449
def run_qwen2_vl(question: str, modality: str):
450
- assert modality == "image"
450
451
model_name = "Qwen/Qwen2-VL-7B-Instruct"
452
@@ -463,8 +462,13 @@ def run_qwen2_vl(question: str, modality: str):
462
disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
463
)
464
465
+ if modality == "image":
466
+ placeholder = "<|image_pad|>"
467
+ elif modality == "video":
468
+ placeholder = "<|video_pad|>"
469
+
470
prompt = ("<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
467
- "<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>"
471
+ f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"
472
f"{question}<|im_end|>\n"
473
"<|im_start|>assistant\n")
474
stop_token_ids = None
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_qwen2_vl.py
+65
-127
@@ -1,12 +1,9 @@
1
from typing import Any, Dict, Tuple
2
3
import pytest
4
-import torch
5
-from PIL.Image import Image
4
from transformers import AutoTokenizer
5
8
-from vllm.inputs import InputContext, token_inputs
9
-from vllm.multimodal import MultiModalRegistry
6
+from vllm.inputs import InputContext, InputProcessingContext
7
8
from .....conftest import _ImageAssets
9
from ....utils import build_model_context
@@ -20,22 +17,9 @@ MAX_PIXELS = "max_pixels"
17
# NOTE: Qwen2VL supports multiple input modalities, so it registers multiple
18
# input mappers.
19
@pytest.fixture()
23
-def image_input_mapper_for_qwen2_vl():
24
- from vllm.model_executor.models.qwen2_vl import (
25
- image_input_mapper_for_qwen2_vl)
26
- return image_input_mapper_for_qwen2_vl
27
-
28
-
29
-@pytest.fixture()
30
-def input_processor_for_qwen2_vl():
31
- from vllm.model_executor.models.qwen2_vl import (
32
- input_processor_for_qwen2_vl)
33
- return input_processor_for_qwen2_vl
34
-
35
-
36
-@pytest.fixture()
37
-def qwen2_vl_context() -> InputContext:
38
- return build_model_context(model_name=MODEL)
20
+def processor_for_qwen2_vl():
21
+ from vllm.model_executor.models.qwen2_vl import Qwen2VLMultiModalProcessor
22
+ return Qwen2VLMultiModalProcessor
23
24
25
@pytest.fixture()
@@ -45,12 +29,6 @@ def get_max_qwen2_vl_image_tokens():
29
return get_max_qwen2_vl_image_tokens
30
31
48
-@pytest.fixture()
49
-def dummy_data_for_qwen2_vl():
50
- from vllm.model_executor.models.qwen2_vl import dummy_data_for_qwen2_vl
51
- return dummy_data_for_qwen2_vl
52
-
53
-
32
@pytest.mark.parametrize("mm_processor_kwargs,expected_max_tokens", [
33
({}, 1225),
34
({
@@ -58,110 +36,70 @@ def dummy_data_for_qwen2_vl():
36
MAX_PIXELS: 512**2
37
}, 324),
38
])
61
-def test_qwen2_vl_max_image_tokens(get_max_qwen2_vl_image_tokens,
62
- qwen2_vl_context: InputContext,
63
- mm_processor_kwargs: Dict[str, Any],
64
- expected_max_tokens: int):
39
+@pytest.mark.parametrize("model", [MODEL])
40
+def test_qwen2_vl_max_image_tokens(
41
+ get_max_qwen2_vl_image_tokens,
42
+ model: str,
43
+ mm_processor_kwargs: Dict[str, Any],
44
+ expected_max_tokens: int,
45
+):
46
"""Ensure that the max token calc handles min/max pixels properly."""
66
- actual_max_tokens = get_max_qwen2_vl_image_tokens(qwen2_vl_context,
67
- **mm_processor_kwargs)
68
- assert actual_max_tokens == expected_max_tokens
69
-
70
-
71
-@pytest.mark.parametrize("mm_processor_kwargs,token_count,img_size", [
72
- [{}, 1225, (980, 980)],
73
- [{
74
- MIN_PIXELS: 64**2,
75
- MAX_PIXELS: 512**2
76
- }, 324, (504, 504)],
77
-])
78
-def test_qwen2_vl_dummy_data(dummy_data_for_qwen2_vl,
79
- qwen2_vl_context: InputContext,
80
- mm_processor_kwargs: Dict[str, Any],
81
- token_count: int, img_size: Tuple[int, int]):
82
- """Ensure that the dummy data handles min/max pixels properly."""
83
- seq_len = 3000
84
- hf_config = qwen2_vl_context.get_hf_config()
85
- image_token_id = hf_config.image_token_id
86
-
87
- # NOTE: video value is required, but isn't actually used
88
- # when making the dummy data except for error handling currently
89
- dummy_data = dummy_data_for_qwen2_vl(
90
- ctx=qwen2_vl_context,
91
- seq_len=seq_len,
92
- mm_counts={
93
- "image": 1,
94
- "video": 0
95
- },
96
- **mm_processor_kwargs,
47
+ ctx = build_model_context(
48
+ model_name=model,
49
+ tokenizer_name=model,
50
+ mm_processor_kwargs=None,
51
)
98
- seq_data = dummy_data.seq_data
99
- mm_data = dummy_data.multi_modal_data
100
-
101
- # Ensure we have the right number of placeholders for min/max pixel values
102
- assert seq_data.get_token_ids().count(image_token_id) == token_count
52
104
- # Ensure the images were resized correctly
105
- image = mm_data["image"]
106
- assert isinstance(image, Image)
107
- assert image.size == img_size
53
+ actual_max_tokens = get_max_qwen2_vl_image_tokens(
54
+ InputContext(ctx.model_config), **mm_processor_kwargs)
55
+ assert actual_max_tokens == expected_max_tokens
56
57
110
-@pytest.mark.parametrize("mm_processor_kwargs,num_placeholders", [
111
- ({}, 1426),
112
- ({
113
- MIN_PIXELS: 64**2,
114
- MAX_PIXELS: 512**2
115
- }, 330),
116
-])
117
-def test_input_processor(input_processor_for_qwen2_vl,
118
- qwen2_vl_context: InputContext,
119
- image_assets: _ImageAssets, num_placeholders: int,
120
- mm_processor_kwargs: Dict[str, Any]):
121
- """Ensure that the image processor handles min/max pixels properly."""
122
- tokenizer = AutoTokenizer.from_pretrained(MODEL)
123
- prompt = "<|vision_start|><|image_pad|><|vision_end|>"
124
-
125
- image = image_assets[0].pil_image
126
- hf_config = qwen2_vl_context.get_hf_config()
127
- image_token_id = hf_config.image_token_id
128
-
129
- inputs = token_inputs(prompt_token_ids=tokenizer.encode(prompt),
130
- prompt=prompt,
131
- multi_modal_data={"image": [image]})
132
-
133
- processed_inputs = input_processor_for_qwen2_vl(qwen2_vl_context, inputs,
134
- **mm_processor_kwargs)
135
- assert processed_inputs["prompt_token_ids"].count(
136
- image_token_id) == num_placeholders
137
- assert len(processed_inputs["multi_modal_data"]["image"]) == 1
138
-
139
-
140
-@pytest.mark.parametrize("mm_processor_kwargs,pixels_shape", [
141
- ({}, [5704, 1176]),
142
- ({
143
- MIN_PIXELS: 64**2,
144
- MAX_PIXELS: 512**2
145
- }, [1320, 1176]),
146
-])
147
-def test_image_mapper_override(qwen2_vl_context: InputContext,
148
- image_assets: _ImageAssets,
149
- mm_processor_kwargs: Dict[str, Any],
150
- pixels_shape: Tuple[int, int]):
151
- """Ensure that the image mapper handles min/max pixels properly."""
152
- mm_registry = MultiModalRegistry()
153
- mm_registry.init_mm_limits_per_prompt(qwen2_vl_context.model_config)
154
-
155
- image = image_assets[0].pil_image
156
-
157
- mapped_output = mm_registry.map_input(
158
- qwen2_vl_context.model_config,
159
- {"image": image},
160
- mm_processor_kwargs=mm_processor_kwargs,
58
+@pytest.mark.parametrize(
59
+ "mm_processor_kwargs, expected_toks_per_img, expected_pixels_shape", [
60
+ ({}, 1426, (5704, 1176)),
61
+ ({
62
+ MIN_PIXELS: 64**2,
63
+ MAX_PIXELS: 512**2
64
+ }, 330, (1320, 1176)),
65
+ ])
66
+@pytest.mark.parametrize("model", [MODEL])
67
+@pytest.mark.parametrize("num_imgs", [1, 2])
68
+def test_processor_override(
69
+ processor_for_qwen2_vl,
70
+ image_assets: _ImageAssets,
71
+ model: str,
72
+ mm_processor_kwargs: Dict[str, Any],
73
+ expected_toks_per_img: int,
74
+ expected_pixels_shape: Tuple[int, int],
75
+ num_imgs: int,
76
+):
77
+ """Ensure Qwen2VLMultiModalProcessor handles min/max pixels properly."""
78
+ # Same as the previous test - don't initialize mm_processor_kwargs
79
+ # in this test and assume that the kwargs will be correctly expanded by
80
+ # the partial when calling the custom input processor.
81
+ ctx = build_model_context(
82
+ model_name=model,
83
+ tokenizer_name=model,
84
+ mm_processor_kwargs=None,
85
)
162
-
163
- # Dimension 0 of pixel values should match the product of image_grid_thw
164
- actual_pixels_shape = mapped_output["pixel_values"].shape
165
- assert list(actual_pixels_shape) == pixels_shape
166
- assert actual_pixels_shape[0] == torch.prod(
167
- mapped_output["image_grid_thw"])
86
+ tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True)
87
+ ctx = InputProcessingContext(ctx.model_config, tokenizer)
88
+ # Build the image str / prompt based on the number of images we pass
89
+ prompt = "<|vision_start|><|image_pad|><|vision_end|>" * num_imgs
90
+ images = [image_assets[0].pil_image] * num_imgs
91
+
92
+ mm_data = {"image": images}
93
+
94
+ processor = processor_for_qwen2_vl(ctx)
95
+ processed_inputs = processor.apply(prompt, mm_data, mm_processor_kwargs)
96
+
97
+ # Ensure we have the right number of placeholders per num_crops size
98
+ hf_processor = processor._get_hf_processor(**mm_processor_kwargs)
99
+ image_token_id = tokenizer.convert_tokens_to_ids(hf_processor.image_token)
100
+ img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
101
+ pixel_shape = processed_inputs["mm_kwargs"]["pixel_values"].shape
102
+
103
+ assert img_tok_count == expected_toks_per_img * num_imgs
104
+ assert pixel_shape[0] == expected_pixels_shape[0] * num_imgs
105
+ assert pixel_shape[1] == expected_pixels_shape[1]
vllm/model_executor/models/qwen2_audio.py
+3
-1
@@ -164,7 +164,9 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
164
self,
165
mm_counts: Mapping[str, int],
166
) -> ProcessorInputs:
167
- audio_len = get_max_qwen2_audio_audio_tokens(self.ctx)
167
+ feature_extractor = self._get_feature_extractor()
168
+ sampling_rate = feature_extractor.sampling_rate
169
+ audio_len = feature_extractor.chunk_length * sampling_rate
170
171
audio_count = mm_counts["audio"]
172
audio = np.zeros(audio_len)
vllm/model_executor/models/qwen2_vl.py
+187
-393
@@ -22,28 +22,26 @@
22
# limitations under the License.
23
"""Inference-only Qwen2-VL model compatible with HuggingFace weights."""
24
from functools import cached_property, partial
25
-from typing import (Any, Callable, Dict, Iterable, List, Literal, Mapping,
26
- Optional, Set, Tuple, Type, TypedDict, Union)
25
+from typing import (Any, Iterable, List, Literal, Mapping, Optional, Set,
26
+ Tuple, Type, TypedDict, Union)
27
28
import torch
29
import torch.nn as nn
30
import torch.nn.functional as F
31
from einops import rearrange, repeat
32
from PIL import Image
33
-from transformers.image_utils import (get_image_size,
34
- infer_channel_dimension_format,
35
- to_numpy_array)
33
+from transformers import BatchFeature
34
+from transformers.models.qwen2_vl import (Qwen2VLImageProcessor,
35
+ Qwen2VLProcessor)
36
from transformers.models.qwen2_vl.configuration_qwen2_vl import (
37
Qwen2VLConfig, Qwen2VLVisionConfig)
38
-from transformers.models.qwen2_vl.image_processing_qwen2_vl import (
39
- make_batched_images, make_batched_videos, smart_resize)
38
+from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize
39
40
from vllm.attention import AttentionMetadata
41
from vllm.config import VllmConfig
42
from vllm.distributed import parallel_state
43
from vllm.distributed import utils as dist_utils
45
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
46
- InputContext, token_inputs)
44
+from vllm.inputs import InputContext
45
from vllm.logger import init_logger
46
from vllm.model_executor import SamplingMetadata
47
from vllm.model_executor.layers.activation import QuickGELU
@@ -56,14 +54,14 @@ from vllm.model_executor.layers.quantization.gptq_marlin import (
54
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
55
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
56
from vllm.multimodal import MULTIMODAL_REGISTRY
59
-from vllm.multimodal.image import cached_get_image_processor
60
-from vllm.multimodal.inputs import (MultiModalData, MultiModalDataDict,
61
- MultiModalKwargs, NestedTensors)
62
-from vllm.multimodal.utils import cached_get_tokenizer
57
+from vllm.multimodal.inputs import MultiModalDataDict, NestedTensors
58
+from vllm.multimodal.processing import (BaseMultiModalProcessor,
59
+ MultiModalDataItems, ProcessorInputs,
60
+ PromptReplacement)
61
from vllm.platforms import _Backend
64
-from vllm.sequence import IntermediateTensors, SequenceData
62
+from vllm.sequence import IntermediateTensors
63
from vllm.transformers_utils.config import uses_mrope
66
-from vllm.transformers_utils.processor import cached_get_processor
64
+from vllm.utils import is_list_of
65
66
from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
67
from .utils import (AutoWeightsLoader, WeightsMapper, get_vit_attn_backend,
@@ -159,7 +157,7 @@ class Qwen2VisionMLP(nn.Module):
157
def __init__(
158
self,
159
in_features: int,
162
- hidden_features: int = None,
160
+ hidden_features: int,
161
act_layer: Type[nn.Module] = QuickGELU,
162
quant_config: Optional[QuantizationConfig] = None,
163
prefix: str = "",
@@ -644,78 +642,8 @@ class Qwen2VisionTransformer(nn.Module):
642
# === Vision input helpers === #
643
644
647
-def get_mm_processor_kwargs(
648
- min_pixels: Optional[int] = None,
649
- max_pixels: Optional[int] = None) -> Dict[str, int]:
650
- mm_processor_kwargs = {}
651
- if min_pixels:
652
- mm_processor_kwargs["min_pixels"] = min_pixels
653
- if max_pixels:
654
- mm_processor_kwargs["max_pixels"] = max_pixels
655
- return mm_processor_kwargs
656
-
657
-
658
-def mm_input_mapper_for_qwen2_vl(
659
- ctx: InputContext,
660
- data: MultiModalData[object],
661
- data_type_key: str,
662
- *,
663
- min_pixels: Optional[int] = None,
664
- max_pixels: Optional[int] = None,
665
-) -> MultiModalKwargs:
666
- """Input mapper for Qwen2-VL."""
667
- if data_type_key == "image" and isinstance(data, dict):
668
- return MultiModalKwargs({
669
- "image_embeds": data.get("image_embeds"),
670
- "image_grid_thw": data.get("image_grid_thw"),
671
- })
672
- if data_type_key == "video" and isinstance(data, dict):
673
- return MultiModalKwargs({
674
- "video_embeds": data.get("video_embeds"),
675
- "video_grid_thw": data.get("video_grid_thw"),
676
- })
677
-
678
- model_config = ctx.model_config
679
- # Handle mm processor kwargs; we pass these at creation time
680
- # because preprocess() in transformers doesn't expose them
681
- mm_processor_kwargs = get_mm_processor_kwargs(min_pixels=min_pixels,
682
- max_pixels=max_pixels)
683
- image_processor = cached_get_image_processor(
684
- model_config.model,
685
- trust_remote_code=model_config.trust_remote_code,
686
- **mm_processor_kwargs,
687
- )
688
- if image_processor is None:
689
- raise RuntimeError("No HuggingFace processor is available "
690
- "to process the image object")
691
-
692
- images = None
693
- videos = None
694
- if data_type_key == "image":
695
- images = data
696
- else:
697
- assert data_type_key == "video"
698
- videos = data
699
-
700
- try:
701
- batch_data = image_processor \
702
- .preprocess(images=images, videos=videos, return_tensors="pt") \
703
- .data
704
- except Exception:
705
- logger.error("Failed to process image (%s)", data)
706
- raise
707
-
708
- return MultiModalKwargs(batch_data)
709
-
710
-
711
-image_input_mapper_for_qwen2_vl = partial(mm_input_mapper_for_qwen2_vl,
712
- data_type_key="image")
713
-video_input_mapper_for_qwen2_vl = partial(mm_input_mapper_for_qwen2_vl,
714
- data_type_key="video")
715
-
716
-
645
def _get_vision_info(
718
- image_processor,
646
+ vision_config: Qwen2VLVisionConfig,
647
height: int,
648
width: int,
649
min_pixels: int,
@@ -726,12 +654,15 @@ def _get_vision_info(
654
):
655
"""Get information (resized height / width and number of vision tokens)
656
of input image / video frame."""
657
+ patch_size = vision_config.patch_size
658
+ merge_size = vision_config.spatial_merge_size
659
+ temporal_patch_size = vision_config.temporal_patch_size
660
661
if do_resize:
662
resized_height, resized_width = smart_resize(
663
height=height,
664
width=width,
734
- factor=image_processor.patch_size * image_processor.merge_size,
665
+ factor=patch_size * merge_size,
666
min_pixels=min_pixels,
667
max_pixels=max_pixels,
668
)
@@ -742,54 +673,41 @@ def _get_vision_info(
673
grid_t = mm_count
674
else:
675
assert data_type_key == "video"
745
- grid_t = max(mm_count // image_processor.temporal_patch_size, 1)
676
+ grid_t = max(mm_count // temporal_patch_size, 1)
677
747
- grid_h = resized_height // image_processor.patch_size
748
- grid_w = resized_width // image_processor.patch_size
678
+ grid_h = resized_height // patch_size
679
+ grid_w = resized_width // patch_size
680
vision_tokens = grid_t * grid_h * grid_w
750
- llm_num_vision_tokens = (vision_tokens // image_processor.merge_size //
751
- image_processor.merge_size)
681
+ llm_num_vision_tokens = vision_tokens // (merge_size**2)
682
683
return resized_height, resized_width, llm_num_vision_tokens
684
685
756
-def _get_max_image_info(
757
- image_processor,
758
- data_type_key: str = "image",
759
- mm_count: int = 1,
760
- min_pixels: Optional[int] = None,
761
- max_pixels: Optional[int] = None,
762
-):
763
- # Limit min / max pixels unless they're explicitly provided
764
- if min_pixels is None:
765
- min_pixels = max(image_processor.min_pixels, 28 * 28)
766
- if max_pixels is None:
767
- max_pixels = min(image_processor.max_pixels, 1280 * 28 * 28)
768
-
769
- return _get_vision_info(
770
- image_processor,
771
- height=9999999,
772
- width=9999999,
773
- min_pixels=min_pixels,
774
- max_pixels=max_pixels,
775
- data_type_key=data_type_key,
776
- mm_count=mm_count,
777
- )
686
+def _get_image_processor(hf_processor: Qwen2VLProcessor):
687
+ image_processor = hf_processor.image_processor # type: ignore
688
+ assert isinstance(image_processor, Qwen2VLImageProcessor)
689
+ return image_processor
690
691
692
def get_max_qwen2_vl_mm_tokens(ctx: InputContext,
693
data_type_key: str,
694
*,
783
- min_pixels=None,
784
- max_pixels=None) -> int:
785
- mm_processor_kwargs = get_mm_processor_kwargs(min_pixels=min_pixels,
786
- max_pixels=max_pixels)
787
- image_processor = cached_get_image_processor(ctx.model_config.model,
788
- **mm_processor_kwargs)
789
- max_resized_height, max_resized_width, max_llm_image_tokens = \
790
- _get_max_image_info(image_processor, data_type_key=data_type_key,
791
- mm_count=1, min_pixels=min_pixels,
792
- max_pixels=max_pixels)
695
+ min_pixels: Optional[int] = None,
696
+ max_pixels: Optional[int] = None) -> int:
697
+ hf_config = ctx.get_hf_config(Qwen2VLConfig)
698
+ vision_config = hf_config.vision_config
699
+
700
+ hf_processor = ctx.get_hf_processor(Qwen2VLProcessor)
701
+ image_processor = _get_image_processor(hf_processor)
702
+
703
+ _, _, max_llm_image_tokens = _get_vision_info(
704
+ vision_config,
705
+ height=9999999,
706
+ width=9999999,
707
+ min_pixels=min_pixels or image_processor.min_pixels,
708
+ max_pixels=max_pixels or image_processor.max_pixels,
709
+ data_type_key=data_type_key,
710
+ )
711
return max_llm_image_tokens
712
713
@@ -799,290 +717,166 @@ get_max_qwen2_vl_video_tokens = partial(get_max_qwen2_vl_mm_tokens,
717
data_type_key="video")
718
719
802
-def dummy_data_for_qwen2_vl(
803
- ctx: InputContext,
804
- seq_len: int,
805
- mm_counts: Mapping[str, int],
806
- *,
807
- min_pixels: Optional[int] = None,
808
- max_pixels: Optional[int] = None
809
-) -> Tuple[SequenceData, Optional[MultiModalDataDict]]:
810
- mm_processor_kwargs = get_mm_processor_kwargs(min_pixels=min_pixels,
811
- max_pixels=max_pixels)
812
- image_processor = cached_get_image_processor(ctx.model_config.model,
813
- **mm_processor_kwargs)
814
-
815
- num_images = mm_counts["image"]
816
- max_resized_height, max_resized_width, max_llm_image_tokens = \
817
- _get_max_image_info(image_processor, data_type_key="image",
818
- mm_count=num_images, min_pixels=min_pixels,
819
- max_pixels=max_pixels)
820
- if seq_len - max_llm_image_tokens - 2 < 0:
821
- raise RuntimeError(
822
- f"Qwen2-VL cannot process {num_images} images in a prompt, "
823
- "please increase max_model_len or reduce image limit by "
824
- "--limit-mm-per-prompt.")
825
-
826
- # Check video counts.
827
- num_videos = mm_counts["video"]
828
- max_resized_height, max_resized_width, max_llm_video_tokens = \
829
- _get_max_image_info(image_processor, data_type_key="video",
830
- mm_count=num_videos, min_pixels=min_pixels,
831
- max_pixels=max_pixels)
832
- if seq_len - max_llm_video_tokens - 2 < 0:
833
- raise RuntimeError(
834
- f"Qwen2-VL cannot process {num_videos} videos in a prompt, "
835
- "please increase max_model_len or reduce video limit by "
836
- "--limit-mm-per-prompt.")
837
-
838
- hf_config = ctx.get_hf_config(Qwen2VLConfig)
839
-
840
- dummy_seqdata = SequenceData.from_prompt_token_counts(
841
- (hf_config.vision_start_token_id, 1),
842
- (hf_config.image_token_id, max_llm_image_tokens),
843
- (hf_config.vision_end_token_id, 1),
844
- (0, seq_len - max_llm_image_tokens - 2),
845
- )
846
-
847
- dummy_image = Image.new("RGB", (max_resized_width, max_resized_height),
848
- color=0)
720
+class Qwen2VLMultiModalDataItems(MultiModalDataItems):
721
850
- return DummyData(dummy_seqdata, {
851
- "image":
852
- dummy_image if num_images == 1 else [dummy_image] * num_images
853
- })
722
+ @staticmethod
723
+ def from_dict(data: MultiModalDataDict) -> "MultiModalDataItems":
724
+ """
725
+ Normalize :class:`MultiModalDataDict` to :class:`MultiModalDataItems`.
726
+ """
727
+ multi_data = Qwen2VLMultiModalDataItems()
728
+
729
+ for k, v in data.items():
730
+ # TODO: Make a separate modality for embedding inputs
731
+ # to avoid confusion
732
+ # yapf: disable
733
+ if k == "video":
734
+ # Special case since even a single item can be a list
735
+ multi_data[k] = ( # type: ignore[index]
736
+ v if (isinstance(v, (dict, torch.Tensor)) # type: ignore[assignment]
737
+ or is_list_of(v, list)) else [v]
738
+ )
739
+ elif k in ("image", "audio"):
740
+ multi_data[k] = ( # type: ignore[index]
741
+ v if isinstance(v, (dict, torch.Tensor, list)) else [v]
742
+ )
743
+ else:
744
+ multi_data[k] = v if isinstance(v, list) else [v] # type: ignore[index]
745
+ # yapf: enable
746
747
+ return multi_data
748
856
-def _get_llm_num_vision_tokens(
857
- mm_inputs: list,
858
- data_type_key: str,
859
- image_processor,
860
- min_pixels: int,
861
- max_pixels: int,
862
-):
863
- """Get number of vision tokens of multimodal inputs.
749
+ def get_item_counts(self) -> Mapping[str, int]:
750
+ return {
751
+ m: (
752
+ len(items[f"{m}_grid_thw"]) # type: ignore
753
+ if isinstance(items, dict) else len(items))
754
+ for m, items in self.items()
755
+ }
756
865
- This method is derived from `transformers.models.qwen2_vl.
866
- image_processing_qwen2_vl.Qwen2VLImageProcessor._preprocess`.
867
- """
868
- image = to_numpy_array(mm_inputs[0])
869
- input_data_format = infer_channel_dimension_format(image)
870
- height, width = get_image_size(image, channel_dim=input_data_format)
871
-
872
- _, _, llm_num_vision_tokens = _get_vision_info(
873
- image_processor,
874
- height=height,
875
- width=width,
876
- min_pixels=min_pixels,
877
- max_pixels=max_pixels,
878
- do_resize=image_processor.do_resize,
879
- data_type_key=data_type_key,
880
- mm_count=len(mm_inputs),
881
- )
882
- return llm_num_vision_tokens
757
758
+class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
759
885
-def _expand_pad_tokens(inputs: list, token_id: int, make_batched_fn: Callable,
886
- data_type_key: str, image_processor: Any,
887
- prompt_token_ids: List[int], min_pixels: Optional[int],
888
- max_pixels: Optional[int]) -> List[int]:
889
- """
890
- Expand pad tokens for multi-modal inputs (e.g., images or videos).
891
-
892
- Args:
893
- inputs (list): The multi-modal inputs (e.g., images or videos).
894
- token_id (int): The token ID used to represent the multi-modal input.
895
- make_batched_fn (Callable): A function to batch the inputs.
896
- data_type_key (str): The type of the multi-modal input.
897
- image_processor (Any): The image processor used to process the inputs.
898
- prompt_token_ids (List[int]): The list of token IDs in the prompt.
899
- min_pixels (int): min pixels to used for img processing
900
- max_pixels (int): max pixels to be used for img processing
901
-
902
- Returns:
903
- List[int]: The list of token IDs for the multi-modal inputs.
904
- """
905
- indices = [
906
- idx for idx, token in enumerate(prompt_token_ids) if token == token_id
907
- ]
908
- inputs = make_batched_fn(inputs)
909
- assert len(indices) == len(inputs)
910
-
911
- prompt_token_ids_with_data = []
912
- for cnt, data in enumerate(inputs):
913
- num_tokens = _get_llm_num_vision_tokens(
914
- [data] if data_type_key == "image" else data,
915
- data_type_key=data_type_key,
916
- image_processor=image_processor,
917
- min_pixels=min_pixels,
918
- max_pixels=max_pixels,
919
- )
920
- if cnt == 0:
921
- end_idx = indices[cnt]
922
- non_data_tokens = prompt_token_ids[:end_idx]
923
- else:
924
- non_data_tokens = prompt_token_ids[indices[cnt - 1] +
925
- 1:indices[cnt]]
926
- prompt_token_ids_with_data.extend(non_data_tokens)
927
- prompt_token_ids_with_data.extend(token_id for _ in range(num_tokens))
928
- prompt_token_ids_with_data.extend(prompt_token_ids[indices[-1] + 1:])
929
- return prompt_token_ids_with_data
930
-
931
-
932
-def input_processor_for_qwen2_vl(
933
- ctx: InputContext,
934
- inputs: DecoderOnlyInputs,
935
- *,
936
- min_pixels: Optional[int] = None,
937
- max_pixels: Optional[int] = None,
938
-) -> DecoderOnlyInputs:
939
- multi_modal_data = inputs.get("multi_modal_data")
940
- if multi_modal_data is None:
941
- return inputs
942
-
943
- image_inputs = multi_modal_data.get("image", None)
944
- video_inputs = multi_modal_data.get("video", None)
945
-
946
- processor = cached_get_processor(ctx.model_config.model)
947
- image_processor = processor.image_processor
948
- # Apply processor kwarg overrides for image processor options
949
- min_pixels = min_pixels if min_pixels else image_processor.min_pixels
950
- max_pixels = max_pixels if max_pixels else image_processor.max_pixels
951
-
952
- model_config = ctx.model_config
953
- hf_config = ctx.get_hf_config(Qwen2VLConfig)
760
+ def _get_mm_items(
761
+ self,
762
+ mm_data: MultiModalDataDict,
763
+ ) -> MultiModalDataItems:
764
+ return Qwen2VLMultiModalDataItems.from_dict(mm_data)
765
955
- # To avoid redundant processing of vision objects (resize, rescale, etc.),
956
- # we extract code of calculating number of vision tokens from
957
- # `transformers.models.qwen2_vl.processing_qwen2_vl.Qwen2VLProcessor`.
958
- #
959
- # The following code is equivalent to:
960
- # prompt = inputs["prompt"]
961
- # inputs = processor(text=[prompt],
962
- # images=image_inputs,
963
- # videos=video_inputs,
964
- # padding=True,
965
- # return_tensors="pt")
966
- # prompt_token_ids = inputs["input_ids"][0].tolist()
967
-
968
- tokenizer = cached_get_tokenizer(
969
- model_config.tokenizer,
970
- trust_remote_code=model_config.trust_remote_code)
971
-
972
- prompt_token_ids = inputs["prompt_token_ids"]
973
-
974
- # Expand image pad tokens.
975
-
976
- if image_inputs is not None:
977
- if isinstance(image_inputs, dict):
978
- prompt_token_ids_with_image = []
979
- image_indices = [
980
- idx for idx, token in enumerate(prompt_token_ids)
981
- if token == hf_config.image_token_id
982
- ]
983
-
984
- # ensure all image tokens have grid_thw
985
- assert \
986
- len(image_indices) == image_inputs["image_grid_thw"].size(0), \
987
- "image token num does not match image_grid_thw.shape"
988
-
989
- image_counter = 0
990
- pad_token_counter = 0
991
- for idx, token in enumerate(prompt_token_ids):
992
- if idx in image_indices:
993
- grid_thw = image_inputs["image_grid_thw"][image_counter]
994
- grid_t, grid_h, grid_w = grid_thw
995
- num_pad_tokens = (grid_t * grid_h * grid_w //
996
- image_processor.merge_size //
997
- image_processor.merge_size)
998
- prompt_token_ids_with_image.extend([token] *
999
- num_pad_tokens)
1000
- image_counter += 1
1001
- pad_token_counter += num_pad_tokens
766
+ def _get_hf_processor(
767
+ self,
768
+ *,
769
+ min_pixels: Optional[int] = None,
770
+ max_pixels: Optional[int] = None,
771
+ ) -> Qwen2VLProcessor:
772
+ hf_processor = self.ctx.get_hf_processor(Qwen2VLProcessor)
773
+ image_processor = _get_image_processor(hf_processor)
774
+
775
+ if min_pixels:
776
+ image_processor.min_pixels = min_pixels
777
+ if max_pixels:
778
+ image_processor.max_pixels = max_pixels
779
+ if max_pixels or min_pixels:
780
+ image_processor.size = {
781
+ "min_pixels": image_processor.min_pixels,
782
+ "max_pixels": image_processor.max_pixels,
783
+ }
784
+
785
+ return hf_processor
786
+
787
+ def _get_processor_data(
788
+ self,
789
+ mm_items: MultiModalDataItems,
790
+ ) -> tuple[dict[str, Any], dict[str, Any]]:
791
+ processor_data = dict[str, Any]()
792
+ passthrough_data = dict[str, Any]()
793
+
794
+ for k, v in mm_items.items():
795
+ # TODO: Make a separate modality for embedding inputs
796
+ # to avoid confusion
797
+ if k in ("image", "video", "audio"):
798
+ if isinstance(v, dict):
799
+ # Pass through embedding inputs (dict)
800
+ passthrough_data.update(v)
801
+ elif isinstance(v, torch.Tensor) and v.ndim == 3:
802
+ # Pass through embedding inputs (single)
803
+ passthrough_data[f"{k}_embeds"] = [v]
804
+ elif (is_list_of(v, torch.Tensor) and len(v) > 0
805
+ and v[0].ndim == 2):
806
+ # Pass through embedding inputs (multi)
807
+ passthrough_data[f"{k}_embeds"] = v
808
else:
1003
- prompt_token_ids_with_image.append(token)
809
+ # Map keys to plural form, e.g.: image -> images
810
+ processor_data[f"{k}s"] = v
811
+ else:
812
+ processor_data[k] = v
813
1005
- # ensure all embeddings are used
1006
- assert \
1007
- pad_token_counter == image_inputs["image_embeds"].size(0), \
1008
- "image_embeds.shape does not match image_grid_thw"
814
+ return processor_data, passthrough_data
815
1010
- prompt_token_ids = prompt_token_ids_with_image
1011
- else:
1012
- prompt_token_ids = _expand_pad_tokens(image_inputs,
1013
- hf_config.image_token_id,
1014
- make_batched_images,
1015
- "image",
1016
- image_processor,
1017
- prompt_token_ids,
1018
- min_pixels=min_pixels,
1019
- max_pixels=max_pixels)
1020
-
1021
- if video_inputs is not None:
1022
- if isinstance(video_inputs, dict):
1023
- prompt_token_ids_with_video = []
1024
- video_indices = [
1025
- idx for idx, token in enumerate(prompt_token_ids)
1026
- if token == hf_config.video_token_id
1027
- ]
1028
-
1029
- # ensure all video tokens have grid_thw
1030
- assert \
1031
- len(video_indices) == video_inputs["video_grid_thw"].size(0), \
1032
- "video token num does not match video_grid_thw.shape"
1033
-
1034
- video_counter = 0
1035
- pad_token_counter = 0
1036
- for idx, token in enumerate(prompt_token_ids):
1037
- if idx in video_indices:
1038
- grid_thw = video_inputs["video_grid_thw"][video_counter]
1039
- grid_t, grid_h, grid_w = grid_thw
1040
- num_pad_tokens = (grid_t * grid_h * grid_w //
1041
- image_processor.merge_size //
1042
- image_processor.merge_size)
1043
- prompt_token_ids_with_video.extend([token] *
1044
- num_pad_tokens)
1045
- video_counter += 1
1046
- pad_token_counter += num_pad_tokens
1047
- else:
1048
- prompt_token_ids_with_video.append(token)
816
+ def _get_prompt_replacements(
817
+ self,
818
+ mm_items: MultiModalDataItems,
819
+ hf_inputs: BatchFeature,
820
+ mm_processor_kwargs: Mapping[str, object],
821
+ ) -> list[PromptReplacement]:
822
+ hf_processor = self._get_hf_processor()
823
+ image_processor = _get_image_processor(hf_processor)
824
+
825
+ # NOTE: Only Qwen2VLProcessor in transformers 4.47.0 has
826
+ # image_token and video_token registered
827
+ placeholder = {
828
+ "image": hf_processor.image_token,
829
+ "video": hf_processor.video_token,
830
+ }
831
+ merge_length = image_processor.merge_size**2
832
+
833
+ def get_replacement_qwen2vl(item_idx: int, modality: str):
834
+ grid_thw = hf_inputs[f"{modality}_grid_thw"][item_idx]
835
+ num_tokens = grid_thw.prod() // merge_length
836
+ return placeholder[modality] * num_tokens
837
+
838
+ return [
839
+ PromptReplacement(
840
+ modality=modality,
841
+ target=placeholder[modality],
842
+ replacement=partial(get_replacement_qwen2vl,
843
+ modality=modality),
844
+ ) for modality in ("image", "video")
845
+ ]
846
1050
- # ensure all embeddings are used
1051
- assert \
1052
- pad_token_counter == video_inputs["video_embeds"].size(0), \
1053
- "video_embeds.shape does not match video_grid_thw"
847
+ def _get_dummy_mm_inputs(
848
+ self,
849
+ mm_counts: Mapping[str, int],
850
+ ) -> ProcessorInputs:
851
+ num_images = mm_counts["image"]
852
+ hf_processor = self._get_hf_processor()
853
+ image_token: str = hf_processor.image_token
854
+ image_processor = _get_image_processor(hf_processor)
855
+
856
+ data = {}
857
+ resized_height, resized_width = smart_resize(
858
+ height=9999999,
859
+ width=9999999,
860
+ factor=image_processor.patch_size * image_processor.merge_size,
861
+ min_pixels=image_processor.min_pixels,
862
+ max_pixels=image_processor.max_pixels,
863
+ )
864
1055
- prompt_token_ids = prompt_token_ids_with_video
1056
- else:
1057
- prompt_token_ids = _expand_pad_tokens(video_inputs,
1058
- hf_config.video_token_id,
1059
- make_batched_videos,
1060
- "video",
1061
- image_processor,
1062
- prompt_token_ids,
1063
- min_pixels=min_pixels,
1064
- max_pixels=max_pixels)
1065
-
1066
- prompt = inputs.get("prompt")
1067
- if prompt is None:
1068
- prompt = tokenizer.decode(prompt_token_ids)
1069
-
1070
- return token_inputs(
1071
- prompt_token_ids=prompt_token_ids,
1072
- prompt=prompt,
1073
- multi_modal_data=multi_modal_data,
1074
- )
865
+ dummy_image = Image.new("RGB", (resized_width, resized_height),
866
+ color=0)
867
+ data["image"] = [dummy_image] * num_images
868
+
869
+ return ProcessorInputs(
870
+ prompt_text=image_token * num_images,
871
+ mm_data=data,
872
+ mm_processor_kwargs={},
873
+ )
874
875
1077
-@MULTIMODAL_REGISTRY.register_image_input_mapper(
1078
- image_input_mapper_for_qwen2_vl)
1079
-@MULTIMODAL_REGISTRY.register_input_mapper("video",
1080
- video_input_mapper_for_qwen2_vl)
876
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_qwen2_vl_image_tokens)
877
@MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
878
"video", get_max_qwen2_vl_video_tokens)
1084
-@INPUT_REGISTRY.register_dummy_data(dummy_data_for_qwen2_vl)
1085
-@INPUT_REGISTRY.register_input_processor(input_processor_for_qwen2_vl)
879
+@MULTIMODAL_REGISTRY.register_processor(Qwen2VLMultiModalProcessor)
880
class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
881
SupportsLoRA, SupportsPP):
882
packed_modules_mapping = {
@@ -1110,7 +904,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
904
905
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
906
super().__init__()
1113
- config = vllm_config.model_config.hf_config
907
+ config: Qwen2VLConfig = vllm_config.model_config.hf_config
908
cache_config = vllm_config.cache_config
909
quant_config = vllm_config.quant_config
910
multimodal_config = vllm_config.model_config.multimodal_config
vllm/multimodal/processing.py
+16
-4
@@ -220,15 +220,18 @@ class MultiModalDataItems(UserDict[str, list[Any]]):
220
multi_data = MultiModalDataItems()
221
222
for k, v in data.items():
223
+ # TODO: Make a separate modality for embedding inputs
224
+ # to avoid confusion
225
# yapf: disable
226
if k == "video":
227
# Special case since even a single item can be a list
228
multi_data[k] = ( # type: ignore[index]
227
- v if is_list_of(v, (list, torch.Tensor)) else [v]
229
+ v if (isinstance(v, torch.Tensor)
230
+ or is_list_of(v, list)) else [v]
231
)
232
elif k in ("image", "audio"):
233
multi_data[k] = ( # type: ignore[index]
231
- v if isinstance(v, (list, torch.Tensor)) else [v]
234
+ v if isinstance(v, (torch.Tensor, list)) else [v]
235
)
236
else:
237
multi_data[k] = v if isinstance(v, list) else [v] # type: ignore[index]
@@ -252,6 +255,9 @@ class MultiModalDataItems(UserDict[str, list[Any]]):
255
def audios(self) -> Sequence[AudioItem]:
256
return self.get("audio", [])
257
258
+ def get_item_counts(self) -> Mapping[str, int]:
259
+ return {m: len(items) for m, items in self.items()}
260
+
261
def get_image_size(self, item_idx: int) -> ImageSize:
262
image = self.images[item_idx]
263
@@ -612,6 +618,12 @@ class BaseMultiModalProcessor(ABC):
618
def _get_tokenizer(self) -> AnyTokenizer:
619
return self.ctx.tokenizer
620
621
+ def _get_mm_items(
622
+ self,
623
+ mm_data: MultiModalDataDict,
624
+ ) -> MultiModalDataItems:
625
+ return MultiModalDataItems.from_dict(mm_data)
626
+
627
@abstractmethod
628
def _get_prompt_replacements(
629
self,
@@ -778,7 +790,7 @@ class BaseMultiModalProcessor(ABC):
790
3. Extract information about the placeholder tokens from the
791
processed token IDs.
792
"""
781
- mm_items = MultiModalDataItems.from_dict(mm_data)
793
+ mm_items = self._get_mm_items(mm_data)
794
795
hf_inputs = self._apply_hf_processor(prompt_text, mm_items,
796
mm_processor_kwargs)
@@ -791,7 +803,7 @@ class BaseMultiModalProcessor(ABC):
803
804
# If HF processor already inserts placeholder tokens,
805
# there is no need for us to insert them
794
- mm_item_counts = {m: len(items) for m, items in mm_items.items()}
806
+ mm_item_counts = mm_items.get_item_counts()
807
all_placeholders = self._find_placeholders(all_prompt_repls,
808
prompt_ids, mm_item_counts)
809