A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[VLM] Merged multi-modal processor for LLaVA-NeXT (#11682)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 3, 2025 at 00:39 UTC
8c38ee7007c50ac5aef9ed43ae91c6f031799c40
14 files changed
+605
-551
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_llava_next.py
deleted
-70
@@ -1,70 +0,0 @@
1
-import pytest
2
-
3
-from vllm.inputs import InputContext
4
-
5
-from ....utils import build_model_context
6
-
7
-
8
-@pytest.fixture()
9
-def get_max_llava_next_image_tokens():
10
- from vllm.model_executor.models.llava_next import (
11
- get_max_llava_next_image_tokens)
12
- return get_max_llava_next_image_tokens
13
-
14
-
15
-@pytest.fixture()
16
-def dummy_data_for_llava_next():
17
- from vllm.model_executor.models.llava_next import dummy_data_for_llava_next
18
- return dummy_data_for_llava_next
19
-
20
-
21
-@pytest.mark.parametrize("gridpoints,expected_max_tokens", [
22
- ([[336, 336]], 1176),
23
- ([[336, 672], [672, 336], [672, 672], [1008, 336], [336, 1008]], 2928),
24
-])
25
-def test_get_max_llava_next_image_tokens(gridpoints, expected_max_tokens,
26
- get_max_llava_next_image_tokens):
27
- ctx = build_model_context(model_name="llava-hf/llava-v1.6-mistral-7b-hf")
28
-
29
- # Update the config image_grid_pinpoints
30
- # and calculate the resulting max tokens
31
- ctx.model_config.hf_config.image_grid_pinpoints = gridpoints
32
-
33
- actual_max_tokens = get_max_llava_next_image_tokens(
34
- InputContext(ctx.model_config))
35
-
36
- assert expected_max_tokens == actual_max_tokens
37
-
38
-
39
-@pytest.mark.parametrize(
40
- "gridpoints,expected_size",
41
- [
42
- # One point; it has to be the largest
43
- ([[336, 336]], (336, 336)),
44
- # Default for most llava next models; the 2x2 tile is the largest
45
- ([[336, 672], [672, 336], [672, 672], [1008, 336], [336, 1008]],
46
- (672, 672)),
47
- # If two rectangular gridpoints are the same, the more vertical
48
- # one has the higher feature count due to newline features
49
- ([[336, 672], [672, 336]], (672, 336))
50
- ])
51
-def test_dummy_data_for_llava_next_feature_size(dummy_data_for_llava_next,
52
- gridpoints, expected_size):
53
- ctx = build_model_context(model_name="llava-hf/llava-v1.6-mistral-7b-hf")
54
-
55
- # Update the config image_grid_pinpoints
56
- ctx.model_config.hf_config.image_grid_pinpoints = gridpoints
57
- seq_len = 5000 # bigger than the max feature size for any image
58
-
59
- dummy_data = dummy_data_for_llava_next(
60
- ctx,
61
- seq_len=seq_len,
62
- mm_counts={"image": 1},
63
- )
64
- seq_data = dummy_data.seq_data
65
- mm_data = dummy_data.multi_modal_data
66
-
67
- # The dummy data dims should match the gridpoint with the biggest feat size
68
- assert mm_data["image"].height == expected_size[0]
69
- assert mm_data["image"].width == expected_size[1]
70
- assert len(seq_data.get_token_ids()) >= seq_len
tests/multimodal/test_mapper.py
deleted
-118
@@ -1,118 +0,0 @@
1
-from contextlib import nullcontext
2
-
3
-import numpy as np
4
-import pytest
5
-from transformers import LlavaNextImageProcessor
6
-
7
-from vllm.config import ModelConfig
8
-from vllm.multimodal import MultiModalRegistry
9
-from vllm.multimodal.image import rescale_image_size
10
-
11
-
12
-@pytest.fixture
13
-def mm_registry():
14
- return MultiModalRegistry()
15
-
16
-
17
-@pytest.mark.parametrize("dtype", ["half", "float"])
18
-@pytest.mark.parametrize("size_factor", [0.25, 0.5, 1.0])
19
-def test_llava_next_image_processor(image_assets, mm_registry, dtype,
20
- size_factor):
21
- MODEL_NAME = "llava-hf/llava-v1.6-vicuna-7b-hf"
22
-
23
- hf_processor = LlavaNextImageProcessor.from_pretrained(MODEL_NAME)
24
- assert isinstance(hf_processor, LlavaNextImageProcessor)
25
-
26
- model_config = ModelConfig(
27
- model=MODEL_NAME,
28
- task="auto",
29
- tokenizer=MODEL_NAME,
30
- tokenizer_mode="auto",
31
- trust_remote_code=False,
32
- seed=0,
33
- dtype=dtype,
34
- revision=None,
35
- limit_mm_per_prompt={"image": 1},
36
- )
37
-
38
- mm_registry.init_mm_limits_per_prompt(model_config)
39
-
40
- for asset in image_assets:
41
- image = rescale_image_size(asset.pil_image, size_factor)
42
-
43
- hf_result = hf_processor.preprocess(
44
- image,
45
- return_tensors="pt",
46
- )
47
- vllm_result = mm_registry.map_input(
48
- model_config,
49
- {"image": image},
50
- )
51
-
52
- assert hf_result.keys() == vllm_result.keys()
53
- for key, hf_tensor in hf_result.items():
54
- hf_arr: np.ndarray = hf_tensor.numpy()
55
- vllm_arr: np.ndarray = vllm_result[key].numpy()
56
-
57
- assert hf_arr.shape == vllm_arr.shape, f"Failed for key={key}"
58
- assert np.allclose(hf_arr, vllm_arr), f"Failed for key={key}"
59
-
60
-
61
-@pytest.mark.parametrize(
62
- ("num_images", "limit", "is_valid"),
63
- [(0, 0, True), (0, 1, True), (1, 0, False), (1, 1, True), (1, 2, True),
64
- (2, 1, False), (2, 2, True)],
65
-)
66
-def test_mm_limits(image_assets, mm_registry, num_images, limit, is_valid):
67
- MODEL_NAME = "llava-hf/llava-v1.6-mistral-7b-hf"
68
-
69
- model_config = ModelConfig(
70
- model=MODEL_NAME,
71
- task="auto",
72
- tokenizer=MODEL_NAME,
73
- tokenizer_mode="auto",
74
- trust_remote_code=False,
75
- seed=0,
76
- dtype="half",
77
- revision=None,
78
- limit_mm_per_prompt={"image": limit},
79
- )
80
-
81
- mm_registry.init_mm_limits_per_prompt(model_config)
82
-
83
- image = image_assets[0].pil_image
84
- if num_images == 0:
85
- mm_inputs = {}
86
- elif num_images == 1:
87
- mm_inputs = {"image": image}
88
- else:
89
- mm_inputs = {"image": [image] * num_images}
90
-
91
- with nullcontext() if is_valid else pytest.raises(ValueError):
92
- mm_registry.map_input(model_config, mm_inputs)
93
-
94
-
95
-# NOTE: We don't test zero images since the HF processor doesn't support it
96
-@pytest.mark.parametrize("num_images", [1, 2])
97
-def test_image_mapper_multi(image_assets, mm_registry, num_images):
98
- MODEL_NAME = "llava-hf/llava-v1.6-mistral-7b-hf"
99
-
100
- model_config = ModelConfig(
101
- model=MODEL_NAME,
102
- task="auto",
103
- tokenizer=MODEL_NAME,
104
- tokenizer_mode="auto",
105
- trust_remote_code=False,
106
- seed=0,
107
- dtype="half",
108
- revision=None,
109
- limit_mm_per_prompt={"image": num_images},
110
- )
111
-
112
- mm_registry.init_mm_limits_per_prompt(model_config)
113
-
114
- image = image_assets[0].pil_image
115
- mm_inputs = {"image": [image] * num_images}
116
-
117
- mapped_inputs = mm_registry.map_input(model_config, mm_inputs)
118
- assert len(mapped_inputs["pixel_values"]) == num_images
tests/multimodal/test_processing.py
+97
@@ -1,5 +1,7 @@
1
+from contextlib import nullcontext
2
from functools import partial
3
from typing import cast
4
+from unittest.mock import MagicMock
5
6
import numpy as np
7
import pytest
@@ -526,6 +528,100 @@ def _rand_audio(
528
return rng.rand(audio_len), sr
529
530
531
+@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
532
+@pytest.mark.parametrize(
533
+ ("limit", "num_supported", "is_valid"),
534
+ [(0, 0, True), (0, 1, True), (1, 0, False), (1, 1, True), (1, 2, True),
535
+ (2, 1, False), (2, 2, True)],
536
+)
537
+def test_limit_mm_per_prompt_dummy(model_id, limit, num_supported, is_valid):
538
+ limit_mm_per_prompt = {"image": limit}
539
+
540
+ model_config = ModelConfig(
541
+ model=model_id,
542
+ task="auto",
543
+ tokenizer=model_id,
544
+ tokenizer_mode="auto",
545
+ trust_remote_code=False,
546
+ seed=0,
547
+ dtype="half",
548
+ revision=None,
549
+ limit_mm_per_prompt=limit_mm_per_prompt,
550
+ )
551
+ model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
552
+
553
+ processor_factory = MULTIMODAL_REGISTRY._processor_factories[model_cls]
554
+ ctx = InputProcessingContext(
555
+ model_config,
556
+ tokenizer=cached_get_tokenizer(model_config.tokenizer),
557
+ )
558
+
559
+ processor = processor_factory(ctx, cache=None)
560
+
561
+ mock_supported_mm_limits = MagicMock(return_value={"image": num_supported})
562
+ processor.get_supported_mm_limits = mock_supported_mm_limits
563
+
564
+ if is_valid:
565
+ exc_ctx = nullcontext()
566
+ else:
567
+ exc_ctx = pytest.raises(ValueError, match="this model only supports")
568
+
569
+ with exc_ctx:
570
+ processor._get_and_validate_dummy_mm_counts()
571
+
572
+
573
+@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
574
+@pytest.mark.parametrize(
575
+ ("num_images", "limit", "is_valid"),
576
+ [(0, 0, True), (0, 1, True), (1, 0, False), (1, 1, True), (1, 2, True),
577
+ (2, 1, False), (2, 2, True)],
578
+)
579
+def test_limit_mm_per_prompt_apply(model_id, num_images, limit, is_valid):
580
+ limit_mm_per_prompt = {"image": limit}
581
+
582
+ model_config = ModelConfig(
583
+ model=model_id,
584
+ task="auto",
585
+ tokenizer=model_id,
586
+ tokenizer_mode="auto",
587
+ trust_remote_code=False,
588
+ seed=0,
589
+ dtype="half",
590
+ revision=None,
591
+ limit_mm_per_prompt=limit_mm_per_prompt,
592
+ )
593
+ model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
594
+
595
+ processor_factory = MULTIMODAL_REGISTRY._processor_factories[model_cls]
596
+ ctx = InputProcessingContext(
597
+ model_config,
598
+ tokenizer=cached_get_tokenizer(model_config.tokenizer),
599
+ )
600
+
601
+ processor = processor_factory(ctx, cache=None)
602
+
603
+ rng = np.random.RandomState(0)
604
+ image = _rand_img(rng, min_wh=128, max_wh=256)
605
+ if num_images == 0:
606
+ mm_data = {}
607
+ elif num_images == 1:
608
+ mm_data = {"image": image}
609
+ else:
610
+ mm_data = {"image": [image] * num_images}
611
+
612
+ if is_valid:
613
+ exc_ctx = nullcontext()
614
+ else:
615
+ exc_ctx = pytest.raises(ValueError, match=f"passed {num_images} image")
616
+
617
+ with exc_ctx:
618
+ processor.apply(
619
+ "<image>" * num_images,
620
+ mm_data=mm_data,
621
+ hf_processor_mm_kwargs={},
622
+ )
623
+
624
+
625
def _test_processing_cache_correctness(
626
model_id: str,
627
modalities: dict[str, bool],
@@ -631,6 +727,7 @@ def _test_processing_cache_correctness(
727
("facebook/chameleon-7b", {"image": False}),
728
("adept/fuyu-8b", {"image": False}),
729
("llava-hf/llava-1.5-7b-hf", {"image": True}),
730
+ ("llava-hf/llava-v1.6-mistral-7b-hf", {"image": True}),
731
("TIGER-Lab/Mantis-8B-siglip-llama3", {"image": True}),
732
("mistral-community/pixtral-12b", {"image": True}),
733
("Qwen/Qwen2-VL-2B-Instruct", {"image": True, "video": True}),
tests/plugins/vllm_add_dummy_model/vllm_add_dummy_model/my_llava.py
+1
-3
@@ -3,13 +3,11 @@ from typing import Optional
3
import torch
4
5
from vllm.model_executor.models.llava import (LlavaForConditionalGeneration,
6
- LlavaMultiModalProcessor,
7
- get_max_llava_image_tokens)
6
+ LlavaMultiModalProcessor)
7
from vllm.model_executor.sampling_metadata import SamplingMetadata
8
from vllm.multimodal import MULTIMODAL_REGISTRY
9
10
12
-@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_image_tokens)
11
@MULTIMODAL_REGISTRY.register_processor(LlavaMultiModalProcessor)
12
class MyLlava(LlavaForConditionalGeneration):
13
vllm/model_executor/models/clip.py
+25
@@ -24,6 +24,8 @@ from vllm.multimodal.utils import (cached_get_tokenizer,
24
resolve_visual_encoder_outputs)
25
from vllm.sequence import SequenceData
26
27
+from .vision import VisionEncoderInfo
28
+
29
30
def get_clip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
31
assert image_size % patch_size == 0
@@ -149,6 +151,29 @@ def input_processor_for_clip(
151
multi_modal_placeholders={"image": ranges})
152
153
154
+class CLIPEncoderInfo(VisionEncoderInfo[CLIPVisionConfig]):
155
+
156
+ def get_num_image_tokens(
157
+ self,
158
+ *,
159
+ image_width: int,
160
+ image_height: int,
161
+ ) -> int:
162
+ return get_clip_image_feature_size(self.vision_config)
163
+
164
+ def get_max_image_tokens(self) -> int:
165
+ return get_max_clip_image_tokens(self.vision_config)
166
+
167
+ def get_num_patches(self) -> int:
168
+ return get_clip_patch_grid_length(
169
+ image_size=self.vision_config.image_size,
170
+ patch_size=self.vision_config.patch_size,
171
+ )
172
+
173
+ def get_image_size(self) -> int:
174
+ return self.vision_config.image_size
175
+
176
+
177
# Adapted from https://github.com/huggingface/transformers/blob/v4.39.0/src/transformers/models/clip/modeling_clip.py#L164 # noqa
178
class CLIPVisionEmbeddings(nn.Module):
179
vllm/model_executor/models/fuyu.py
+3
-3
@@ -76,7 +76,7 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor):
76
return ImageSize(width=target_size["width"],
77
height=target_size["height"])
78
79
- def _get_image_grid_size(
79
+ def _get_image_feature_grid_size(
80
self,
81
*,
82
image_width: int,
@@ -99,7 +99,7 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor):
99
def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
100
target_width, target_height = self._get_image_target_size()
101
102
- max_ncols, max_nrows = self._get_image_grid_size(
102
+ max_ncols, max_nrows = self._get_image_feature_grid_size(
103
image_width=target_width,
104
image_height=target_height,
105
)
@@ -172,7 +172,7 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor):
172
images = mm_items.get_items("image", ImageProcessorItems)
173
image_size = images.get_image_size(item_idx)
174
175
- ncols, nrows = self._get_image_grid_size(
175
+ ncols, nrows = self._get_image_feature_grid_size(
176
image_width=image_size.width,
177
image_height=image_size.height,
178
)
vllm/model_executor/models/llava.py
+215
-119
@@ -1,6 +1,7 @@
1
+from abc import abstractmethod
2
from functools import cached_property
2
-from typing import (Iterable, List, Literal, Mapping, Optional, Protocol, Set,
3
- Tuple, TypedDict, Union)
3
+from typing import (Final, Iterable, List, Literal, Mapping, Optional,
4
+ Protocol, Set, Tuple, TypedDict, Union)
5
6
import torch
7
import torch.nn as nn
@@ -12,7 +13,6 @@ from transformers.models.pixtral import PixtralProcessor
13
14
from vllm.attention import AttentionMetadata
15
from vllm.config import VllmConfig
15
-from vllm.inputs import InputContext
16
from vllm.model_executor.layers.activation import get_act_fn
17
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
18
RowParallelLinear)
@@ -23,23 +23,23 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
23
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
24
MultiModalInputsV2, MultiModalKwargs,
25
NestedTensors)
26
-from vllm.multimodal.parse import ImageProcessorItems
26
+from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
27
+ ImageSize)
28
from vllm.multimodal.processing import (BaseMultiModalProcessor,
28
- MultiModalDataItems, ProcessorInputs,
29
- PromptReplacement,
29
+ InputProcessingContext,
30
+ MultiModalDataItems, ProcessingCache,
31
+ ProcessorInputs, PromptReplacement,
32
full_groupby_modality)
33
from vllm.sequence import IntermediateTensors
34
33
-from .clip import (CLIPVisionModel, dummy_image_for_clip,
34
- get_max_clip_image_tokens)
35
+from .clip import CLIPVisionModel
36
from .interfaces import SupportsMultiModal, SupportsPP
36
-from .pixtral import (PixtralHFVisionModel, dummy_image_for_pixtral_hf,
37
- get_max_pixtral_hf_image_tokens,
38
- get_pixtral_hf_image_feature_size)
39
-from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
40
- get_max_siglip_image_tokens)
37
+from .pixtral import (PixtralHFVisionModel,
38
+ get_pixtral_hf_image_feature_grid_size)
39
+from .siglip import SiglipVisionModel
40
from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
41
maybe_prefix, merge_multimodal_embeddings)
42
+from .vision import vision_encoder_info
43
44
45
class LlavaImagePixelInputs(TypedDict):
@@ -94,39 +94,167 @@ class LlavaMultiModalProjector(nn.Module):
94
return hidden_states
95
96
97
-def get_max_llava_image_tokens(ctx: InputContext):
98
- hf_config = ctx.get_hf_config(LlavaConfig)
99
- vision_config = hf_config.vision_config
97
+class LlavaLikeConfig(Protocol):
98
+ vision_config: Final[PretrainedConfig]
99
+ vision_feature_select_strategy: Final[str]
100
+ vision_feature_layer: Final[Union[int, List[int]]]
101
101
- if isinstance(vision_config, CLIPVisionConfig):
102
- num_image_tokens = get_max_clip_image_tokens(vision_config)
103
- elif isinstance(vision_config, SiglipVisionConfig):
104
- num_image_tokens = get_max_siglip_image_tokens(vision_config)
105
- elif isinstance(vision_config, PixtralVisionConfig):
106
- num_image_tokens = get_max_pixtral_hf_image_tokens(vision_config)
107
- else:
108
- msg = f"Unsupported vision config: {type(vision_config)}"
109
- raise NotImplementedError(msg)
102
111
- strategy = hf_config.vision_feature_select_strategy
112
- if strategy == "default":
113
- return num_image_tokens - 1
114
- elif strategy == "full":
115
- return num_image_tokens
116
- else:
117
- raise ValueError(f"Unexpected select feature strategy: {strategy}")
103
+class BaseLlavaMultiModalProcessor(BaseMultiModalProcessor):
104
105
+ def __init__(self,
106
+ ctx: InputProcessingContext,
107
+ *,
108
+ cache: Optional[ProcessingCache] = None,
109
+ enable_sanity_checks: bool = True) -> None:
110
+ super().__init__(ctx,
111
+ cache=cache,
112
+ enable_sanity_checks=enable_sanity_checks)
113
+
114
+ vision_config = self._get_hf_config().vision_config
115
+ self._vision_encoder_info = vision_encoder_info(vision_config)
116
120
-class LlavaMultiModalProcessor(BaseMultiModalProcessor):
117
+ @abstractmethod
118
+ def _get_hf_config(self) -> LlavaLikeConfig:
119
+ raise NotImplementedError
120
121
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
122
return {"image": None}
123
124
+ def _apply_feature_select_strategy(
125
+ self,
126
+ strategy: str,
127
+ encoder_num_image_tokens: int,
128
+ ) -> int:
129
+ if strategy == "default":
130
+ return encoder_num_image_tokens - 1
131
+ if strategy == "full":
132
+ return encoder_num_image_tokens
133
+
134
+ msg = f"Unexpected feature select strategy: {strategy!r}"
135
+ raise NotImplementedError(msg)
136
+
137
+ def _get_max_image_tokens(self) -> int:
138
+ hf_config = self._get_hf_config()
139
+
140
+ return self._apply_feature_select_strategy(
141
+ hf_config.vision_feature_select_strategy,
142
+ self._vision_encoder_info.get_max_image_tokens(),
143
+ )
144
+
145
def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
126
- return {"image": get_max_llava_image_tokens(self.ctx)}
146
+ return {"image": self._get_max_image_tokens()}
147
+
148
+ def _get_mm_fields_config(
149
+ self,
150
+ hf_inputs: BatchFeature,
151
+ hf_processor_mm_kwargs: Mapping[str, object],
152
+ ) -> Mapping[str, MultiModalFieldConfig]:
153
+ return dict(
154
+ pixel_values=MultiModalFieldConfig.batched("image"),
155
+ image_embeds=MultiModalFieldConfig.batched("image"),
156
+ )
157
+
158
+ def _get_dummy_image_size(self) -> ImageSize:
159
+ image_size = self._vision_encoder_info.get_image_size()
160
+ return ImageSize(image_size, image_size)
161
+
162
+ @abstractmethod
163
+ def _get_image_token(self) -> str:
164
+ raise NotImplementedError
165
+
166
+ def _get_dummy_mm_inputs(
167
+ self,
168
+ mm_counts: Mapping[str, int],
169
+ ) -> ProcessorInputs:
170
+ num_images = mm_counts.get("image", 0)
171
+
172
+ image_token = self._get_image_token()
173
+ target_width, target_height = self._get_dummy_image_size()
174
+
175
+ mm_data = {
176
+ "image":
177
+ self._get_dummy_images(width=target_width,
178
+ height=target_height,
179
+ num_images=num_images)
180
+ }
181
+
182
+ return ProcessorInputs(
183
+ prompt_text=image_token * num_images,
184
+ mm_data=mm_data,
185
+ )
186
+
187
+
188
+class LlavaMultiModalProcessor(BaseLlavaMultiModalProcessor):
189
+
190
+ def _get_hf_config(self) -> LlavaConfig:
191
+ return self.ctx.get_hf_config(LlavaConfig)
192
+
193
+ def _get_hf_processor(self) -> LlavaProcessor:
194
+ return self.ctx.get_hf_processor(LlavaProcessor)
195
+
196
+ def _get_image_token(self) -> str:
197
+ return self._get_hf_processor().image_token
198
+
199
+ def _get_num_image_tokens(
200
+ self,
201
+ *,
202
+ image_width: int,
203
+ image_height: int,
204
+ ) -> int:
205
+ hf_config = self._get_hf_config()
206
+
207
+ return self._apply_feature_select_strategy(
208
+ hf_config.vision_feature_select_strategy,
209
+ self._vision_encoder_info.get_num_image_tokens(
210
+ image_width=image_width,
211
+ image_height=image_height,
212
+ ),
213
+ )
214
+
215
+ def _get_prompt_replacements(
216
+ self,
217
+ mm_items: MultiModalDataItems,
218
+ hf_processor_mm_kwargs: Mapping[str, object],
219
+ out_mm_kwargs: MultiModalKwargs,
220
+ ) -> list[PromptReplacement]:
221
+ hf_config = self._get_hf_config()
222
+ image_token_id = hf_config.image_token_index
223
128
- def _get_hf_processor(self) -> Union[LlavaProcessor, PixtralProcessor]:
129
- return self.ctx.get_hf_processor((LlavaProcessor, PixtralProcessor))
224
+ def get_replacement(item_idx: int):
225
+ images = mm_items.get_items(
226
+ "image", (ImageEmbeddingItems, ImageProcessorItems))
227
+
228
+ if isinstance(images, ImageEmbeddingItems):
229
+ num_image_tokens = images.get_feature_size(item_idx)
230
+ else:
231
+ image_size = images.get_image_size(item_idx)
232
+ num_image_tokens = self._get_num_image_tokens(
233
+ image_width=image_size.width,
234
+ image_height=image_size.height,
235
+ )
236
+
237
+ return [image_token_id] * num_image_tokens
238
+
239
+ return [
240
+ PromptReplacement(
241
+ modality="image",
242
+ target=[image_token_id],
243
+ replacement=get_replacement,
244
+ ),
245
+ ]
246
+
247
+
248
+class PixtralHFMultiModalProcessor(BaseLlavaMultiModalProcessor):
249
+
250
+ def _get_hf_config(self) -> LlavaConfig:
251
+ return self.ctx.get_hf_config(LlavaConfig)
252
+
253
+ def _get_hf_processor(self) -> PixtralProcessor:
254
+ return self.ctx.get_hf_processor(PixtralProcessor)
255
+
256
+ def _get_image_token(self) -> str:
257
+ return self._get_hf_processor().image_token
258
259
def _call_hf_processor(
260
self,
@@ -140,119 +268,82 @@ class LlavaMultiModalProcessor(BaseMultiModalProcessor):
268
mm_kwargs=mm_kwargs,
269
)
270
143
- # NOTE: pixel_values=None for MLlavaProcessor
271
pixel_values = processed_outputs.get("pixel_values")
272
if pixel_values is not None:
273
images = mm_data["images"]
274
assert isinstance(images, list)
275
149
- if isinstance(self._get_hf_processor(), PixtralProcessor):
150
- # Original output: (1, num_images, C, H, W)
151
- # New output: (num_images, C, H, W)
152
- assert (isinstance(pixel_values, list)
153
- and len(pixel_values) == 1)
154
- assert (isinstance(pixel_values[0], list)
155
- and len(pixel_values[0]) == len(images))
276
+ # Original output: (1, num_images, C, H, W)
277
+ # New output: (num_images, C, H, W)
278
+ assert (isinstance(pixel_values, list) and len(pixel_values) == 1)
279
+ assert (isinstance(pixel_values[0], list)
280
+ and len(pixel_values[0]) == len(images))
281
157
- processed_outputs["pixel_values"] = pixel_values[0]
282
+ processed_outputs["pixel_values"] = pixel_values[0]
283
284
return processed_outputs
285
161
- def _get_mm_fields_config(
162
- self,
163
- hf_inputs: BatchFeature,
164
- hf_processor_mm_kwargs: Mapping[str, object],
165
- ) -> Mapping[str, MultiModalFieldConfig]:
166
- return dict(
167
- pixel_values=MultiModalFieldConfig.batched("image"),
168
- image_embeds=MultiModalFieldConfig.batched("image"),
169
- )
170
-
286
def _get_prompt_replacements(
287
self,
288
mm_items: MultiModalDataItems,
289
hf_processor_mm_kwargs: Mapping[str, object],
290
out_mm_kwargs: MultiModalKwargs,
291
) -> list[PromptReplacement]:
177
- hf_config = self.ctx.get_hf_config(LlavaConfig)
292
+ hf_config = self._get_hf_config()
293
image_token_id = hf_config.image_token_index
294
295
processor = self._get_hf_processor()
181
- if isinstance(processor, PixtralProcessor):
182
- image_token = processor.image_token
183
- image_break_token = processor.image_break_token
184
- image_end_token = processor.image_end_token
185
-
186
- vision_config = hf_config.vision_config
187
- assert isinstance(vision_config, PixtralVisionConfig)
296
+ image_token = processor.image_token
297
+ image_break_token = processor.image_break_token
298
+ image_end_token = processor.image_end_token
299
189
- def get_replacement_pixtral(item_idx: int):
190
- images = mm_items.get_items("image", ImageProcessorItems)
191
- image_size = images.get_image_size(item_idx)
192
-
193
- (
194
- num_width_tokens,
195
- num_height_tokens,
196
- ) = get_pixtral_hf_image_feature_size(
197
- vision_config,
198
- image_width=image_size.width,
199
- image_height=image_size.height,
200
- )
300
+ vision_config = hf_config.vision_config
301
+ assert isinstance(vision_config, PixtralVisionConfig)
302
202
- tokens = ([image_token] * num_width_tokens +
203
- [image_break_token]) * num_height_tokens
204
- tokens[-1] = image_end_token
303
+ def get_replacement(item_idx: int):
304
+ images = mm_items.get_items("image", ImageProcessorItems)
305
+ image_size = images.get_image_size(item_idx)
306
206
- return "".join(tokens)
307
+ ncols, nrows = get_pixtral_hf_image_feature_grid_size(
308
+ vision_config,
309
+ image_width=image_size.width,
310
+ image_height=image_size.height,
311
+ )
312
208
- return [
209
- PromptReplacement(
210
- modality="image",
211
- target=[image_token_id],
212
- replacement=get_replacement_pixtral,
213
- ),
214
- ]
313
+ tokens = ([image_token] * ncols + [image_break_token]) * nrows
314
+ tokens[-1] = image_end_token
315
216
- max_image_tokens = get_max_llava_image_tokens(self.ctx)
316
+ return "".join(tokens)
317
318
return [
319
PromptReplacement(
320
modality="image",
321
target=[image_token_id],
222
- replacement=[image_token_id] * max_image_tokens,
223
- )
322
+ replacement=get_replacement,
323
+ ),
324
]
325
226
- def _get_dummy_mm_inputs(
227
- self,
228
- mm_counts: Mapping[str, int],
229
- ) -> ProcessorInputs:
230
- hf_config = self.ctx.get_hf_config(LlavaConfig)
231
- vision_config = hf_config.vision_config
232
- num_images = mm_counts.get("image", 0)
233
-
234
- if isinstance(vision_config, CLIPVisionConfig):
235
- data = dummy_image_for_clip(vision_config, num_images)
236
- elif isinstance(vision_config, SiglipVisionConfig):
237
- data = dummy_image_for_siglip(vision_config, num_images)
238
- elif isinstance(vision_config, PixtralVisionConfig):
239
- data = dummy_image_for_pixtral_hf(vision_config, num_images)
240
- else:
241
- msg = f"Unsupported vision config: {type(vision_config)}"
242
- raise NotImplementedError(msg)
326
244
- hf_processor = self._get_hf_processor()
245
- image_token = hf_processor.image_token
327
+def _build_llava_or_pixtral_hf_processor(
328
+ ctx: InputProcessingContext,
329
+ *,
330
+ cache: Optional[ProcessingCache] = None,
331
+ enable_sanity_checks: bool = True,
332
+) -> BaseLlavaMultiModalProcessor:
333
+ hf_config = ctx.get_hf_config(LlavaConfig)
334
247
- return ProcessorInputs(
248
- prompt_text=image_token * num_images,
249
- mm_data=data,
335
+ if isinstance(hf_config.vision_config, PixtralVisionConfig):
336
+ return PixtralHFMultiModalProcessor(
337
+ ctx,
338
+ cache=cache,
339
+ enable_sanity_checks=enable_sanity_checks,
340
)
341
252
-
253
-class LlavaLikeConfig(Protocol):
254
- vision_config: PretrainedConfig
255
- vision_feature_layer: Union[int, List[int]]
342
+ return LlavaMultiModalProcessor(
343
+ ctx,
344
+ cache=cache,
345
+ enable_sanity_checks=enable_sanity_checks,
346
+ )
347
348
349
def _get_num_hidden_layers(hf_config: LlavaLikeConfig) -> int:
@@ -330,7 +421,7 @@ def init_vision_tower_for_llava(
421
raise NotImplementedError(msg)
422
423
333
-@MULTIMODAL_REGISTRY.register_processor(LlavaMultiModalProcessor)
424
+@MULTIMODAL_REGISTRY.register_processor(_build_llava_or_pixtral_hf_processor)
425
class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
426
# BitandBytes specific attributes
427
bitsandbytes_stacked_params_mapping = {
@@ -596,7 +687,12 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
687
) -> MultiModalInputsV2:
688
hf_config = self.ctx.get_hf_config(LlavaConfig)
689
image_token_id = hf_config.image_token_index
599
- max_image_tokens = get_max_llava_image_tokens(self.ctx)
690
+
691
+ # Assume that it doesn't depend on the image size
692
+ num_image_tokens = self._get_num_image_tokens(
693
+ image_width=-1,
694
+ image_height=-1,
695
+ )
696
697
result = super().apply(prompt_text, mm_data, hf_processor_mm_kwargs)
698
@@ -609,14 +705,14 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
705
def get_replacement_mantis(item_idx: int):
706
return "".join([
707
f"(image {item_idx+1}: <Image>", # 7 tokens
612
- "<image>" * max_image_tokens,
708
+ "<image>" * num_image_tokens,
709
"</Image>)", # 3 tokens
710
])
711
712
mantis_repls = self._bind_prompt_replacements([
713
PromptReplacement(
714
modality="image",
619
- target=[image_token_id] * max_image_tokens,
715
+ target=[image_token_id] * num_image_tokens,
716
replacement=get_replacement_mantis,
717
)
718
])
vllm/model_executor/models/llava_next.py
+112
-209
@@ -4,31 +4,25 @@ from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
4
5
import torch
6
import torch.nn as nn
7
-from PIL import Image
8
-from transformers import CLIPVisionConfig, LlavaNextConfig, SiglipVisionConfig
7
+from transformers import BatchFeature, LlavaNextConfig, LlavaNextProcessor
8
from transformers.models.llava_next.modeling_llava_next import (
9
get_anyres_image_grid_shape, unpad_image)
10
from typing_extensions import NotRequired
11
12
from vllm.attention import AttentionMetadata
13
from vllm.config import VllmConfig
15
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
16
- InputContext)
14
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
15
from vllm.model_executor.sampling_metadata import SamplingMetadata
16
from vllm.multimodal import MULTIMODAL_REGISTRY
20
-from vllm.multimodal.inputs import NestedTensors
17
+from vllm.multimodal.inputs import MultiModalFieldConfig, NestedTensors
18
+from vllm.multimodal.parse import ImageSize
19
from vllm.sequence import IntermediateTensors
22
-from vllm.utils import is_list_of
20
24
-from .clip import (CLIPVisionModel, dummy_image_for_clip,
25
- dummy_seq_data_for_clip, get_clip_image_feature_size,
26
- get_clip_patch_grid_length, input_processor_for_clip)
21
+from .clip import CLIPVisionModel
22
from .interfaces import SupportsMultiModal, SupportsPP
28
-from .llava import LlavaMultiModalProjector, init_vision_tower_for_llava
29
-from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
30
- dummy_seq_data_for_siglip, get_siglip_image_feature_size,
31
- get_siglip_patch_grid_length, input_processor_for_siglip)
23
+from .llava import (LlavaMultiModalProcessor, LlavaMultiModalProjector,
24
+ init_vision_tower_for_llava)
25
+from .siglip import SiglipVisionModel
26
from .utils import (AutoWeightsLoader, embed_multimodal, flatten_bn,
27
init_vllm_registered_model, maybe_prefix)
28
@@ -65,218 +59,127 @@ LlavaNextImageInputs = Union[LlavaNextImagePixelInputs,
59
LlavaNextImageEmbeddingInputs]
60
61
68
-# Based on: https://github.com/huggingface/text-generation-inference/blob/v2.2.0/server/text_generation_server/models/vlm_causal_lm.py#L79
69
-def _get_llava_next_num_unpadded_features(
70
- original_height: int,
71
- original_width: int,
72
- npatches: int,
73
- num_patch_height: int,
74
- num_patch_width: int,
75
-) -> Tuple[int, int]:
76
- current_height = npatches * num_patch_height
77
- current_width = npatches * num_patch_width
78
-
79
- original_aspect_ratio = original_width / original_height
80
- current_aspect_ratio = current_width / current_height
81
-
82
- if original_aspect_ratio > current_aspect_ratio:
83
- scale_factor = current_width / original_width
84
- new_height = int(original_height * scale_factor)
85
- padding = (current_height - new_height) // 2
86
- current_height -= 2 * padding
87
- else:
88
- scale_factor = current_height / original_height
89
- new_width = int(original_width * scale_factor)
90
- padding = (current_width - new_width) // 2
91
- current_width -= 2 * padding
92
-
93
- unpadded_features = current_height * current_width
94
- newline_features = current_height
95
- return (unpadded_features, newline_features)
96
-
97
-
98
-# Based on: https://github.com/huggingface/text-generation-inference/blob/v2.2.0/server/text_generation_server/models/vlm_causal_lm.py#L106
99
-def get_llava_next_image_feature_size(
100
- hf_config: LlavaNextConfig,
101
- *,
102
- input_height: int,
103
- input_width: int,
104
-) -> int:
105
- vision_config = hf_config.vision_config
106
-
107
- if isinstance(vision_config, CLIPVisionConfig):
108
- num_patches = get_clip_patch_grid_length(
109
- image_size=vision_config.image_size,
110
- patch_size=vision_config.patch_size,
111
- )
112
- base_feature_size = get_clip_image_feature_size(vision_config)
113
- elif isinstance(vision_config, SiglipVisionConfig):
114
- num_patches = get_siglip_patch_grid_length(
115
- image_size=vision_config.image_size,
116
- patch_size=vision_config.patch_size,
117
- )
118
- base_feature_size = get_siglip_image_feature_size(vision_config)
119
- else:
120
- msg = f"Unsupported vision config: {type(vision_config)}"
121
- raise NotImplementedError(msg)
122
-
123
- strategy = hf_config.vision_feature_select_strategy
124
- if strategy == "default":
125
- base_feature_size -= 1
126
- elif strategy == "full":
127
- pass
128
- else:
129
- raise ValueError(f"Unexpected select feature strategy: {strategy}")
62
+class LlavaNextMultiModalProcessor(LlavaMultiModalProcessor):
63
131
- num_patch_height, num_patch_width = get_anyres_image_grid_shape(
132
- image_size=(input_height, input_width),
133
- grid_pinpoints=hf_config.image_grid_pinpoints,
134
- patch_size=vision_config.image_size,
135
- )
136
-
137
- (
138
- unpadded_feature_size,
139
- newline_feature_size,
140
- ) = _get_llava_next_num_unpadded_features(input_height, input_width,
141
- num_patches, num_patch_height,
142
- num_patch_width)
143
-
144
- return unpadded_feature_size + newline_feature_size + base_feature_size
145
-
146
-
147
-def get_max_llava_next_image_tokens(ctx: InputContext):
148
- """Compute the max feature size for all possible image grid pinpoints."""
149
- return _get_pinpoint_with_largest_features(ctx)[0]
150
-
151
-
152
-def _get_pinpoint_with_largest_features(
153
- ctx: InputContext) -> Tuple[int, Tuple[int, int]]:
154
- """Get the grid pinpoint with the largest features & its feature size."""
155
- hf_config = ctx.get_hf_config(LlavaNextConfig)
156
- largest_feature_size = 0
157
- largest_feature_pinpoint = None
158
- for (height, width) in hf_config.image_grid_pinpoints:
159
- feat_size = get_llava_next_image_feature_size(
160
- hf_config,
161
- input_height=height,
162
- input_width=width,
163
- )
164
- if feat_size > largest_feature_size:
165
- largest_feature_size = feat_size
166
- largest_feature_pinpoint = (height, width)
167
- if not largest_feature_size or largest_feature_pinpoint is None:
168
- raise ValueError("Cannot have a largest feature size of 0!")
169
- return largest_feature_size, largest_feature_pinpoint
170
-
171
-
172
-def dummy_data_for_llava_next(ctx: InputContext, seq_len: int,
173
- mm_counts: Mapping[str, int]):
174
- hf_config = ctx.get_hf_config(LlavaNextConfig)
175
- vision_config = hf_config.vision_config
176
- num_images = mm_counts["image"]
177
-
178
- image_feature_size, pinpoint = _get_pinpoint_with_largest_features(ctx)
179
- max_feat_height, max_feat_width = pinpoint
180
-
181
- if isinstance(vision_config, CLIPVisionConfig):
182
- seq_data, ranges = dummy_seq_data_for_clip(
183
- vision_config,
184
- seq_len,
185
- num_images,
186
- image_token_id=hf_config.image_token_index,
187
- image_feature_size_override=image_feature_size,
188
- )
64
+ def _get_hf_config(self) -> LlavaNextConfig:
65
+ return self.ctx.get_hf_config(LlavaNextConfig)
66
+
67
+ def _get_hf_processor(self) -> LlavaNextProcessor:
68
+ return self.ctx.get_hf_processor(LlavaNextProcessor)
69
190
- mm_data = dummy_image_for_clip(
191
- vision_config,
192
- num_images,
193
- image_width_override=max_feat_width,
194
- image_height_override=max_feat_height,
70
+ def _get_image_token(self) -> str:
71
+ return self._get_hf_processor().image_token
72
+
73
+ def _get_mm_fields_config(
74
+ self,
75
+ hf_inputs: BatchFeature,
76
+ hf_processor_mm_kwargs: Mapping[str, object],
77
+ ) -> Mapping[str, MultiModalFieldConfig]:
78
+ return dict(
79
+ pixel_values=MultiModalFieldConfig.batched("image"),
80
+ image_sizes=MultiModalFieldConfig.batched("image"),
81
+ image_embeds=MultiModalFieldConfig.batched("image"),
82
)
83
197
- return DummyData(seq_data, mm_data, ranges)
198
- elif isinstance(vision_config, SiglipVisionConfig):
199
- seq_data, ranges = dummy_seq_data_for_siglip(
200
- vision_config,
201
- seq_len,
202
- num_images,
203
- image_token_id=hf_config.image_token_index,
204
- image_feature_size_override=image_feature_size,
84
+ def _get_max_image_tokens(self) -> int:
85
+ largest_feature_size, _ = self._get_pinpoint_with_most_features()
86
+ return largest_feature_size
87
+
88
+ def _get_dummy_image_size(self) -> ImageSize:
89
+ _, pinpoint = self._get_pinpoint_with_most_features()
90
+ return pinpoint
91
+
92
+ # Based on: https://github.com/huggingface/text-generation-inference/blob/v2.2.0/server/text_generation_server/models/vlm_causal_lm.py#L106
93
+ def _get_num_image_tokens(
94
+ self,
95
+ *,
96
+ image_width: int,
97
+ image_height: int,
98
+ ) -> int:
99
+ hf_config = self._get_hf_config()
100
+
101
+ base_feature_size = self._apply_feature_select_strategy(
102
+ hf_config.vision_feature_select_strategy,
103
+ self._vision_encoder_info.get_num_image_tokens(
104
+ image_width=image_width,
105
+ image_height=image_height,
106
+ ),
107
)
108
+ num_patches = self._vision_encoder_info.get_num_patches()
109
207
- mm_data = dummy_image_for_siglip(
208
- vision_config,
209
- num_images,
210
- image_width_override=max_feat_width,
211
- image_height_override=max_feat_height,
110
+ num_patch_height, num_patch_width = get_anyres_image_grid_shape(
111
+ image_size=(image_height, image_width),
112
+ grid_pinpoints=hf_config.image_grid_pinpoints,
113
+ patch_size=self._vision_encoder_info.get_image_size(),
114
)
115
214
- return DummyData(seq_data, mm_data, ranges)
116
+ (
117
+ unpadded_feature_size,
118
+ newline_feature_size,
119
+ ) = self._get_num_unpadded_features(
120
+ original_height=image_height,
121
+ original_width=image_width,
122
+ npatches=num_patches,
123
+ num_patch_height=num_patch_height,
124
+ num_patch_width=num_patch_width,
125
+ )
126
216
- msg = f"Unsupported vision config: {type(vision_config)}"
217
- raise NotImplementedError(msg)
127
+ return unpadded_feature_size + newline_feature_size + base_feature_size
128
129
+ # Based on: https://github.com/huggingface/text-generation-inference/blob/v2.2.0/server/text_generation_server/models/vlm_causal_lm.py#L79
130
+ def _get_num_unpadded_features(
131
+ self,
132
+ *,
133
+ original_height: int,
134
+ original_width: int,
135
+ npatches: int,
136
+ num_patch_height: int,
137
+ num_patch_width: int,
138
+ ) -> tuple[int, int]:
139
+ current_height = npatches * num_patch_height
140
+ current_width = npatches * num_patch_width
141
+
142
+ original_aspect_ratio = original_width / original_height
143
+ current_aspect_ratio = current_width / current_height
144
+
145
+ if original_aspect_ratio > current_aspect_ratio:
146
+ scale_factor = current_width / original_width
147
+ new_height = int(original_height * scale_factor)
148
+ padding = (current_height - new_height) // 2
149
+ current_height -= 2 * padding
150
+ else:
151
+ scale_factor = current_height / original_height
152
+ new_width = int(original_width * scale_factor)
153
+ padding = (current_width - new_width) // 2
154
+ current_width -= 2 * padding
155
220
-def input_processor_for_llava_next(ctx: InputContext,
221
- inputs: DecoderOnlyInputs):
222
- multi_modal_data = inputs.get("multi_modal_data")
223
- if multi_modal_data is None or "image" not in multi_modal_data:
224
- return inputs
156
+ unpadded_features = current_height * current_width
157
+ newline_features = current_height
158
+ return (unpadded_features, newline_features)
159
226
- model_config = ctx.model_config
227
- hf_config = ctx.get_hf_config(LlavaNextConfig)
228
- vision_config = hf_config.vision_config
160
+ def _get_pinpoint_with_most_features(self) -> tuple[int, ImageSize]:
161
+ """
162
+ Get the grid pinpoint with the most features and
163
+ the corresponding feature size.
164
+ """
165
+ hf_config = self._get_hf_config()
166
230
- image_data = multi_modal_data["image"]
231
- if isinstance(image_data, Image.Image):
232
- width, height = image_data.size
167
+ largest_feature_size, largest_feature_pinpoint = 0, None
168
+ for (height, width) in hf_config.image_grid_pinpoints:
169
+ feat_size = self._get_num_image_tokens(image_width=width,
170
+ image_height=height)
171
+ if feat_size > largest_feature_size:
172
+ largest_feature_size = feat_size
173
+ largest_feature_pinpoint = ImageSize(width=width,
174
+ height=height)
175
234
- image_feature_size = get_llava_next_image_feature_size(
235
- hf_config,
236
- input_height=height,
237
- input_width=width,
238
- )
239
- elif is_list_of(image_data, Image.Image):
240
- image_feature_size = [
241
- get_llava_next_image_feature_size(hf_config,
242
- input_height=img.height,
243
- input_width=img.width)
244
- for img in image_data
245
- ]
246
- elif isinstance(image_data, torch.Tensor):
247
- num_images, image_feature_size, hidden_size = image_data.shape
248
- elif is_list_of(image_data, torch.Tensor):
249
- image_feature_size = [item.shape[1] for item in image_data]
250
- else:
251
- raise TypeError(f"Invalid image type: {type(image_data)}")
252
-
253
- vision_config = hf_config.vision_config
254
-
255
- if isinstance(vision_config, CLIPVisionConfig):
256
- return input_processor_for_clip(
257
- model_config,
258
- vision_config,
259
- inputs,
260
- image_token_id=hf_config.image_token_index,
261
- image_feature_size_override=image_feature_size,
262
- )
263
- elif isinstance(vision_config, SiglipVisionConfig):
264
- return input_processor_for_siglip(
265
- model_config,
266
- vision_config,
267
- inputs,
268
- image_token_id=hf_config.image_token_index,
269
- image_feature_size_override=image_feature_size,
270
- )
176
+ if largest_feature_size == 0 or largest_feature_pinpoint is None:
177
+ raise ValueError("Cannot have a largest feature size of 0!")
178
272
- msg = f"Unsupported vision config: {type(vision_config)}"
273
- raise NotImplementedError(msg)
179
+ return largest_feature_size, largest_feature_pinpoint
180
181
276
-@MULTIMODAL_REGISTRY.register_image_input_mapper()
277
-@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_next_image_tokens)
278
-@INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava_next)
279
-@INPUT_REGISTRY.register_input_processor(input_processor_for_llava_next)
182
+@MULTIMODAL_REGISTRY.register_processor(LlavaNextMultiModalProcessor)
183
class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
184
SupportsPP):
185
@@ -507,7 +410,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
410
def _process_image_pixels(
411
self,
412
inputs: LlavaNextImagePixelInputs,
510
- ) -> Union[torch.Tensor, List[torch.Tensor]]:
413
+ ) -> Union[torch.Tensor, tuple[torch.Tensor, ...]]:
414
assert self.vision_tower is not None
415
416
pixel_values = inputs["data"]
vllm/model_executor/models/phi3v.py
+14
-10
@@ -34,7 +34,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
34
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
35
MultiModalInputsV2, MultiModalKwargs,
36
NestedTensors, PlaceholderRange)
37
-from vllm.multimodal.parse import ImageProcessorItems
37
+from vllm.multimodal.parse import ImageEmbeddingItems, ImageProcessorItems
38
from vllm.multimodal.processing import (BaseMultiModalProcessor,
39
MultiModalDataItems, ProcessorInputs,
40
PromptReplacement,
@@ -388,15 +388,19 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
388
assert isinstance(bos_token_id, int)
389
390
def get_replacement_phi3v(item_idx: int):
391
- images = mm_items.get_items("image", ImageProcessorItems)
392
- image_size = images.get_image_size(item_idx)
393
-
394
- num_tokens = self._get_num_image_tokens(
395
- image_width=image_size.width,
396
- image_height=image_size.height,
397
- )
398
-
399
- return [_IMAGE_TOKEN_ID] * num_tokens + [bos_token_id]
391
+ images = mm_items.get_items(
392
+ "image", (ImageEmbeddingItems, ImageProcessorItems))
393
+
394
+ if isinstance(images, ImageEmbeddingItems):
395
+ num_image_tokens = images.get_feature_size(item_idx)
396
+ else:
397
+ image_size = images.get_image_size(item_idx)
398
+ num_image_tokens = self._get_num_image_tokens(
399
+ image_width=image_size.width,
400
+ image_height=image_size.height,
401
+ )
402
+
403
+ return [_IMAGE_TOKEN_ID] * num_image_tokens + [bos_token_id]
404
405
num_images = mm_items.get_count("image", strict=False)
406
vllm/model_executor/models/pixtral.py
+52
-14
@@ -38,6 +38,7 @@ from vllm.sequence import IntermediateTensors, SequenceData
38
from .interfaces import SupportsMultiModal, SupportsPP
39
from .utils import (init_vllm_registered_model, maybe_prefix,
40
merge_multimodal_embeddings)
41
+from .vision import VisionEncoderInfo
42
43
try:
44
from xformers import ops as xops
@@ -697,10 +698,18 @@ def get_pixtral_hf_patch_grid_length(*, image_size: int,
698
return image_size // patch_size
699
700
700
-def get_pixtral_hf_num_patches(*, image_size: int, patch_size: int) -> int:
701
- grid_length = get_pixtral_hf_patch_grid_length(image_size=image_size,
702
- patch_size=patch_size)
703
- return grid_length * grid_length
701
+def get_pixtral_hf_image_feature_size(
702
+ *,
703
+ image_size: int,
704
+ patch_size: int,
705
+) -> int:
706
+ grid_length = get_pixtral_hf_patch_grid_length(
707
+ image_size=image_size,
708
+ patch_size=patch_size,
709
+ )
710
+
711
+ # Consider the image_break_token
712
+ return (grid_length + 1) * grid_length
713
714
715
def get_max_pixtral_hf_image_tokens(hf_config: PixtralVisionConfig) -> int:
@@ -730,13 +739,16 @@ def dummy_image_for_pixtral_hf(
739
return {"image": image if num_images == 1 else [image] * num_images}
740
741
733
-def get_pixtral_hf_image_feature_size(hf_config: PixtralVisionConfig,
734
- image_width: int,
735
- image_height: int) -> Tuple[int, int]:
736
- # Adapted from transformers.models.pixtral.image_processing_pixtral.get_resize_output_image_size # noqa: E501
737
- # https://github.com/huggingface/transformers/blob/2bd4d5897dc73e8b172832070a6f9e567a0df017/src/transformers/models/pixtral/image_processing_pixtral.py#L180 # noqa: E501
738
- max_width, max_height = hf_config.image_size, hf_config.image_size
739
- patch_width, patch_height = hf_config.patch_size, hf_config.patch_size
742
+# Adapted from transformers.models.pixtral.image_processing_pixtral.get_resize_output_image_size # noqa: E501
743
+# https://github.com/huggingface/transformers/blob/2bd4d5897dc73e8b172832070a6f9e567a0df017/src/transformers/models/pixtral/image_processing_pixtral.py#L180
744
+def get_pixtral_hf_image_feature_grid_size(
745
+ hf_config: PixtralVisionConfig,
746
+ *,
747
+ image_width: int,
748
+ image_height: int,
749
+) -> tuple[int, int]:
750
+ max_width = max_height = hf_config.image_size
751
+ patch_width = patch_height = hf_config.patch_size
752
753
ratio = max(image_width / max_width, image_height / max_height)
754
@@ -744,12 +756,38 @@ def get_pixtral_hf_image_feature_size(hf_config: PixtralVisionConfig,
756
image_width = int(math.ceil(image_width / ratio))
757
image_height = int(math.ceil(image_height / ratio))
758
747
- num_height_tokens, num_width_tokens = _get_pixtral_hf_num_image_tokens(
759
+ nrows, ncols = _get_pixtral_hf_num_image_tokens(
760
(image_height, image_width),
761
(patch_height, patch_width),
750
- )
762
+ ) # type: ignore
763
+
764
+ return ncols, nrows
765
+
766
+
767
+class PixtralHFEncoderInfo(VisionEncoderInfo[PixtralVisionConfig]):
768
+
769
+ def get_num_image_tokens(
770
+ self,
771
+ *,
772
+ image_width: int,
773
+ image_height: int,
774
+ ) -> int:
775
+ return get_pixtral_hf_image_feature_size(
776
+ image_size=self.vision_config.image_size,
777
+ patch_size=self.get_image_size(),
778
+ )
779
+
780
+ def get_max_image_tokens(self) -> int:
781
+ return get_max_pixtral_hf_image_tokens(self.vision_config)
782
+
783
+ def get_num_patches(self) -> int:
784
+ return get_pixtral_hf_patch_grid_length(
785
+ image_size=self.vision_config.image_size,
786
+ patch_size=self.vision_config.patch_size,
787
+ )
788
752
- return num_width_tokens, num_height_tokens
789
+ def get_image_size(self) -> int:
790
+ return self.vision_config.image_size
791
792
793
class PixtralHFMLP(nn.Module):
vllm/model_executor/models/siglip.py
+25
@@ -28,6 +28,8 @@ from vllm.multimodal.utils import (cached_get_tokenizer,
28
resolve_visual_encoder_outputs)
29
from vllm.sequence import SequenceData
30
31
+from .vision import VisionEncoderInfo
32
+
33
34
def get_siglip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
35
# Since interpolation is applied, the image size need not be divisible
@@ -156,6 +158,29 @@ def input_processor_for_siglip(
158
multi_modal_placeholders={"image": ranges})
159
160
161
+class SiglipEncoderInfo(VisionEncoderInfo[SiglipVisionConfig]):
162
+
163
+ def get_num_image_tokens(
164
+ self,
165
+ *,
166
+ image_width: int,
167
+ image_height: int,
168
+ ) -> int:
169
+ return get_siglip_image_feature_size(self.vision_config)
170
+
171
+ def get_max_image_tokens(self) -> int:
172
+ return get_max_siglip_image_tokens(self.vision_config)
173
+
174
+ def get_num_patches(self) -> int:
175
+ return get_siglip_patch_grid_length(
176
+ image_size=self.vision_config.image_size,
177
+ patch_size=self.vision_config.patch_size,
178
+ )
179
+
180
+ def get_image_size(self) -> int:
181
+ return self.vision_config.image_size
182
+
183
+
184
# Adapted from https://github.com/huggingface/transformers/blob/v4.43.3/src/transformers/models/siglip/modeling_siglip.py#L249 # noqa
185
class SiglipVisionEmbeddings(nn.Module):
186
vllm/model_executor/models/utils.py
+1
-1
@@ -373,7 +373,7 @@ def embed_multimodal(
373
input_ids: torch.Tensor,
374
multimodal_token_id: int,
375
get_text_embeds: Callable[[torch.Tensor], torch.Tensor],
376
- multimodal_embeds: Union[torch.Tensor, List[torch.Tensor]],
376
+ multimodal_embeds: NestedTensors,
377
) -> torch.Tensor:
378
"""
379
Embed token IDs and multimodal inputs and combine their embeddings.
vllm/model_executor/models/vision.py
new
+52
@@ -0,0 +1,52 @@
1
+from abc import ABC, abstractmethod
2
+from typing import Generic, TypeVar
3
+
4
+from transformers import PretrainedConfig
5
+
6
+_C = TypeVar("_C", bound=PretrainedConfig)
7
+
8
+
9
+class VisionEncoderInfo(ABC, Generic[_C]):
10
+
11
+ def __init__(self, vision_config: _C) -> None:
12
+ super().__init__()
13
+
14
+ self.vision_config = vision_config
15
+
16
+ @abstractmethod
17
+ def get_num_image_tokens(
18
+ self,
19
+ *,
20
+ image_width: int,
21
+ image_height: int,
22
+ ) -> int:
23
+ raise NotImplementedError
24
+
25
+ @abstractmethod
26
+ def get_max_image_tokens(self) -> int:
27
+ raise NotImplementedError
28
+
29
+ @abstractmethod
30
+ def get_num_patches(self) -> int:
31
+ raise NotImplementedError
32
+
33
+ @abstractmethod
34
+ def get_image_size(self) -> int:
35
+ raise NotImplementedError
36
+
37
+
38
+def vision_encoder_info(vision_config: PretrainedConfig) -> VisionEncoderInfo:
39
+ # Avoid circular imports
40
+ from .clip import CLIPEncoderInfo, CLIPVisionConfig
41
+ from .pixtral import PixtralHFEncoderInfo, PixtralVisionConfig
42
+ from .siglip import SiglipEncoderInfo, SiglipVisionConfig
43
+
44
+ if isinstance(vision_config, CLIPVisionConfig):
45
+ return CLIPEncoderInfo(vision_config)
46
+ if isinstance(vision_config, PixtralVisionConfig):
47
+ return PixtralHFEncoderInfo(vision_config)
48
+ if isinstance(vision_config, SiglipVisionConfig):
49
+ return SiglipEncoderInfo(vision_config)
50
+
51
+ msg = f"Unsupported vision config: {type(vision_config)}"
52
+ raise NotImplementedError(msg)
vllm/multimodal/parse.py
+8
-4
@@ -1,7 +1,8 @@
1
from abc import ABC, abstractmethod
2
from collections import UserDict
3
from collections.abc import Callable, Iterator, Mapping, Sequence
4
-from typing import TYPE_CHECKING, Any, Generic, NamedTuple, Optional, TypeVar
4
+from typing import (TYPE_CHECKING, Any, Generic, NamedTuple, Optional, TypeVar,
5
+ Union)
6
7
import numpy as np
8
import torch
@@ -87,7 +88,7 @@ class EmbeddingItems(ModalityDataItems[NestedTensors, torch.Tensor]):
88
def get_count(self) -> int:
89
return len(self.data)
90
90
- def get(self, index: int) -> object:
91
+ def get(self, index: int) -> torch.Tensor:
92
return self.data[index]
93
94
def get_processor_data(self) -> Mapping[str, object]:
@@ -96,6 +97,9 @@ class EmbeddingItems(ModalityDataItems[NestedTensors, torch.Tensor]):
97
def get_passthrough_data(self) -> Mapping[str, object]:
98
return {f"{self.modality}_embeds": self.data}
99
100
+ def get_feature_size(self, item_idx: int) -> int:
101
+ return len(self.get(item_idx))
102
+
103
104
class AudioProcessorItems(ProcessorBatchItems[HfAudioItem]):
105
@@ -182,7 +186,7 @@ class MultiModalDataItems(UserDict[str, ModalityDataItems[Any, Any]]):
186
def get_items(
187
self,
188
modality: str,
185
- typ: type[_D],
189
+ typ: Union[type[_D], tuple[type[_D], ...]],
190
) -> _D:
191
"""
192
Get the data items belonging to a modality,
@@ -199,7 +203,7 @@ class MultiModalDataItems(UserDict[str, ModalityDataItems[Any, Any]]):
203
f"Expected type: {typ}, but "
204
f"found type: {type(items)}")
205
202
- return items
206
+ return items # type: ignore[return-value]
207
208
209
ModalityDataParser: TypeAlias = Callable[[ModalityData[Any]],