@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8c38ee7007
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[VLM] Merged multi-modal processor for LLaVA-NeXT (#11682)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 3, 2025 at 00:39 UTC 8c38ee7007c50ac5aef9ed43ae91c6f031799c40
14 files changed +605 -551
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_llava_next.py deleted
-70
@@ -1,70 +0,0 @@
1 -import pytest
2 -
3 -from vllm.inputs import InputContext
4 -
5 -from ....utils import build_model_context
6 -
7 -
8 -@pytest.fixture()
9 -def get_max_llava_next_image_tokens():
10 - from vllm.model_executor.models.llava_next import (
11 - get_max_llava_next_image_tokens)
12 - return get_max_llava_next_image_tokens
13 -
14 -
15 -@pytest.fixture()
16 -def dummy_data_for_llava_next():
17 - from vllm.model_executor.models.llava_next import dummy_data_for_llava_next
18 - return dummy_data_for_llava_next
19 -
20 -
21 -@pytest.mark.parametrize("gridpoints,expected_max_tokens", [
22 - ([[336, 336]], 1176),
23 - ([[336, 672], [672, 336], [672, 672], [1008, 336], [336, 1008]], 2928),
24 -])
25 -def test_get_max_llava_next_image_tokens(gridpoints, expected_max_tokens,
26 - get_max_llava_next_image_tokens):
27 - ctx = build_model_context(model_name="llava-hf/llava-v1.6-mistral-7b-hf")
28 -
29 - # Update the config image_grid_pinpoints
30 - # and calculate the resulting max tokens
31 - ctx.model_config.hf_config.image_grid_pinpoints = gridpoints
32 -
33 - actual_max_tokens = get_max_llava_next_image_tokens(
34 - InputContext(ctx.model_config))
35 -
36 - assert expected_max_tokens == actual_max_tokens
37 -
38 -
39 -@pytest.mark.parametrize(
40 - "gridpoints,expected_size",
41 - [
42 - # One point; it has to be the largest
43 - ([[336, 336]], (336, 336)),
44 - # Default for most llava next models; the 2x2 tile is the largest
45 - ([[336, 672], [672, 336], [672, 672], [1008, 336], [336, 1008]],
46 - (672, 672)),
47 - # If two rectangular gridpoints are the same, the more vertical
48 - # one has the higher feature count due to newline features
49 - ([[336, 672], [672, 336]], (672, 336))
50 - ])
51 -def test_dummy_data_for_llava_next_feature_size(dummy_data_for_llava_next,
52 - gridpoints, expected_size):
53 - ctx = build_model_context(model_name="llava-hf/llava-v1.6-mistral-7b-hf")
54 -
55 - # Update the config image_grid_pinpoints
56 - ctx.model_config.hf_config.image_grid_pinpoints = gridpoints
57 - seq_len = 5000 # bigger than the max feature size for any image
58 -
59 - dummy_data = dummy_data_for_llava_next(
60 - ctx,
61 - seq_len=seq_len,
62 - mm_counts={"image": 1},
63 - )
64 - seq_data = dummy_data.seq_data
65 - mm_data = dummy_data.multi_modal_data
66 -
67 - # The dummy data dims should match the gridpoint with the biggest feat size
68 - assert mm_data["image"].height == expected_size[0]
69 - assert mm_data["image"].width == expected_size[1]
70 - assert len(seq_data.get_token_ids()) >= seq_len
tests/multimodal/test_mapper.py deleted
-118
@@ -1,118 +0,0 @@
1 -from contextlib import nullcontext
2 -
3 -import numpy as np
4 -import pytest
5 -from transformers import LlavaNextImageProcessor
6 -
7 -from vllm.config import ModelConfig
8 -from vllm.multimodal import MultiModalRegistry
9 -from vllm.multimodal.image import rescale_image_size
10 -
11 -
12 -@pytest.fixture
13 -def mm_registry():
14 - return MultiModalRegistry()
15 -
16 -
17 -@pytest.mark.parametrize("dtype", ["half", "float"])
18 -@pytest.mark.parametrize("size_factor", [0.25, 0.5, 1.0])
19 -def test_llava_next_image_processor(image_assets, mm_registry, dtype,
20 - size_factor):
21 - MODEL_NAME = "llava-hf/llava-v1.6-vicuna-7b-hf"
22 -
23 - hf_processor = LlavaNextImageProcessor.from_pretrained(MODEL_NAME)
24 - assert isinstance(hf_processor, LlavaNextImageProcessor)
25 -
26 - model_config = ModelConfig(
27 - model=MODEL_NAME,
28 - task="auto",
29 - tokenizer=MODEL_NAME,
30 - tokenizer_mode="auto",
31 - trust_remote_code=False,
32 - seed=0,
33 - dtype=dtype,
34 - revision=None,
35 - limit_mm_per_prompt={"image": 1},
36 - )
37 -
38 - mm_registry.init_mm_limits_per_prompt(model_config)
39 -
40 - for asset in image_assets:
41 - image = rescale_image_size(asset.pil_image, size_factor)
42 -
43 - hf_result = hf_processor.preprocess(
44 - image,
45 - return_tensors="pt",
46 - )
47 - vllm_result = mm_registry.map_input(
48 - model_config,
49 - {"image": image},
50 - )
51 -
52 - assert hf_result.keys() == vllm_result.keys()
53 - for key, hf_tensor in hf_result.items():
54 - hf_arr: np.ndarray = hf_tensor.numpy()
55 - vllm_arr: np.ndarray = vllm_result[key].numpy()
56 -
57 - assert hf_arr.shape == vllm_arr.shape, f"Failed for key={key}"
58 - assert np.allclose(hf_arr, vllm_arr), f"Failed for key={key}"
59 -
60 -
61 -@pytest.mark.parametrize(
62 - ("num_images", "limit", "is_valid"),
63 - [(0, 0, True), (0, 1, True), (1, 0, False), (1, 1, True), (1, 2, True),
64 - (2, 1, False), (2, 2, True)],
65 -)
66 -def test_mm_limits(image_assets, mm_registry, num_images, limit, is_valid):
67 - MODEL_NAME = "llava-hf/llava-v1.6-mistral-7b-hf"
68 -
69 - model_config = ModelConfig(
70 - model=MODEL_NAME,
71 - task="auto",
72 - tokenizer=MODEL_NAME,
73 - tokenizer_mode="auto",
74 - trust_remote_code=False,
75 - seed=0,
76 - dtype="half",
77 - revision=None,
78 - limit_mm_per_prompt={"image": limit},
79 - )
80 -
81 - mm_registry.init_mm_limits_per_prompt(model_config)
82 -
83 - image = image_assets[0].pil_image
84 - if num_images == 0:
85 - mm_inputs = {}
86 - elif num_images == 1:
87 - mm_inputs = {"image": image}
88 - else:
89 - mm_inputs = {"image": [image] * num_images}
90 -
91 - with nullcontext() if is_valid else pytest.raises(ValueError):
92 - mm_registry.map_input(model_config, mm_inputs)
93 -
94 -
95 -# NOTE: We don't test zero images since the HF processor doesn't support it
96 -@pytest.mark.parametrize("num_images", [1, 2])
97 -def test_image_mapper_multi(image_assets, mm_registry, num_images):
98 - MODEL_NAME = "llava-hf/llava-v1.6-mistral-7b-hf"
99 -
100 - model_config = ModelConfig(
101 - model=MODEL_NAME,
102 - task="auto",
103 - tokenizer=MODEL_NAME,
104 - tokenizer_mode="auto",
105 - trust_remote_code=False,
106 - seed=0,
107 - dtype="half",
108 - revision=None,
109 - limit_mm_per_prompt={"image": num_images},
110 - )
111 -
112 - mm_registry.init_mm_limits_per_prompt(model_config)
113 -
114 - image = image_assets[0].pil_image
115 - mm_inputs = {"image": [image] * num_images}
116 -
117 - mapped_inputs = mm_registry.map_input(model_config, mm_inputs)
118 - assert len(mapped_inputs["pixel_values"]) == num_images
tests/multimodal/test_processing.py
+97
@@ -1,5 +1,7 @@
1 +from contextlib import nullcontext
2 from functools import partial
3 from typing import cast
4 +from unittest.mock import MagicMock
5
6 import numpy as np
7 import pytest
@@ -526,6 +528,100 @@ def _rand_audio(
528 return rng.rand(audio_len), sr
529
530
531 +@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
532 +@pytest.mark.parametrize(
533 + ("limit", "num_supported", "is_valid"),
534 + [(0, 0, True), (0, 1, True), (1, 0, False), (1, 1, True), (1, 2, True),
535 + (2, 1, False), (2, 2, True)],
536 +)
537 +def test_limit_mm_per_prompt_dummy(model_id, limit, num_supported, is_valid):
538 + limit_mm_per_prompt = {"image": limit}
539 +
540 + model_config = ModelConfig(
541 + model=model_id,
542 + task="auto",
543 + tokenizer=model_id,
544 + tokenizer_mode="auto",
545 + trust_remote_code=False,
546 + seed=0,
547 + dtype="half",
548 + revision=None,
549 + limit_mm_per_prompt=limit_mm_per_prompt,
550 + )
551 + model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
552 +
553 + processor_factory = MULTIMODAL_REGISTRY._processor_factories[model_cls]
554 + ctx = InputProcessingContext(
555 + model_config,
556 + tokenizer=cached_get_tokenizer(model_config.tokenizer),
557 + )
558 +
559 + processor = processor_factory(ctx, cache=None)
560 +
561 + mock_supported_mm_limits = MagicMock(return_value={"image": num_supported})
562 + processor.get_supported_mm_limits = mock_supported_mm_limits
563 +
564 + if is_valid:
565 + exc_ctx = nullcontext()
566 + else:
567 + exc_ctx = pytest.raises(ValueError, match="this model only supports")
568 +
569 + with exc_ctx:
570 + processor._get_and_validate_dummy_mm_counts()
571 +
572 +
573 +@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
574 +@pytest.mark.parametrize(
575 + ("num_images", "limit", "is_valid"),
576 + [(0, 0, True), (0, 1, True), (1, 0, False), (1, 1, True), (1, 2, True),
577 + (2, 1, False), (2, 2, True)],
578 +)
579 +def test_limit_mm_per_prompt_apply(model_id, num_images, limit, is_valid):
580 + limit_mm_per_prompt = {"image": limit}
581 +
582 + model_config = ModelConfig(
583 + model=model_id,
584 + task="auto",
585 + tokenizer=model_id,
586 + tokenizer_mode="auto",
587 + trust_remote_code=False,
588 + seed=0,
589 + dtype="half",
590 + revision=None,
591 + limit_mm_per_prompt=limit_mm_per_prompt,
592 + )
593 + model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
594 +
595 + processor_factory = MULTIMODAL_REGISTRY._processor_factories[model_cls]
596 + ctx = InputProcessingContext(
597 + model_config,
598 + tokenizer=cached_get_tokenizer(model_config.tokenizer),
599 + )
600 +
601 + processor = processor_factory(ctx, cache=None)
602 +
603 + rng = np.random.RandomState(0)
604 + image = _rand_img(rng, min_wh=128, max_wh=256)
605 + if num_images == 0:
606 + mm_data = {}
607 + elif num_images == 1:
608 + mm_data = {"image": image}
609 + else:
610 + mm_data = {"image": [image] * num_images}
611 +
612 + if is_valid:
613 + exc_ctx = nullcontext()
614 + else:
615 + exc_ctx = pytest.raises(ValueError, match=f"passed {num_images} image")
616 +
617 + with exc_ctx:
618 + processor.apply(
619 + "<image>" * num_images,
620 + mm_data=mm_data,
621 + hf_processor_mm_kwargs={},
622 + )
623 +
624 +
625 def _test_processing_cache_correctness(
626 model_id: str,
627 modalities: dict[str, bool],
@@ -631,6 +727,7 @@ def _test_processing_cache_correctness(
727 ("facebook/chameleon-7b", {"image": False}),
728 ("adept/fuyu-8b", {"image": False}),
729 ("llava-hf/llava-1.5-7b-hf", {"image": True}),
730 + ("llava-hf/llava-v1.6-mistral-7b-hf", {"image": True}),
731 ("TIGER-Lab/Mantis-8B-siglip-llama3", {"image": True}),
732 ("mistral-community/pixtral-12b", {"image": True}),
733 ("Qwen/Qwen2-VL-2B-Instruct", {"image": True, "video": True}),
tests/plugins/vllm_add_dummy_model/vllm_add_dummy_model/my_llava.py
+1 -3
@@ -3,13 +3,11 @@ from typing import Optional
3 import torch
4
5 from vllm.model_executor.models.llava import (LlavaForConditionalGeneration,
6 - LlavaMultiModalProcessor,
7 - get_max_llava_image_tokens)
6 + LlavaMultiModalProcessor)
7 from vllm.model_executor.sampling_metadata import SamplingMetadata
8 from vllm.multimodal import MULTIMODAL_REGISTRY
9
10
12 -@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_image_tokens)
11 @MULTIMODAL_REGISTRY.register_processor(LlavaMultiModalProcessor)
12 class MyLlava(LlavaForConditionalGeneration):
13
vllm/model_executor/models/clip.py
+25
@@ -24,6 +24,8 @@ from vllm.multimodal.utils import (cached_get_tokenizer,
24 resolve_visual_encoder_outputs)
25 from vllm.sequence import SequenceData
26
27 +from .vision import VisionEncoderInfo
28 +
29
30 def get_clip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
31 assert image_size % patch_size == 0
@@ -149,6 +151,29 @@ def input_processor_for_clip(
151 multi_modal_placeholders={"image": ranges})
152
153
154 +class CLIPEncoderInfo(VisionEncoderInfo[CLIPVisionConfig]):
155 +
156 + def get_num_image_tokens(
157 + self,
158 + *,
159 + image_width: int,
160 + image_height: int,
161 + ) -> int:
162 + return get_clip_image_feature_size(self.vision_config)
163 +
164 + def get_max_image_tokens(self) -> int:
165 + return get_max_clip_image_tokens(self.vision_config)
166 +
167 + def get_num_patches(self) -> int:
168 + return get_clip_patch_grid_length(
169 + image_size=self.vision_config.image_size,
170 + patch_size=self.vision_config.patch_size,
171 + )
172 +
173 + def get_image_size(self) -> int:
174 + return self.vision_config.image_size
175 +
176 +
177 # Adapted from https://github.com/huggingface/transformers/blob/v4.39.0/src/transformers/models/clip/modeling_clip.py#L164 # noqa
178 class CLIPVisionEmbeddings(nn.Module):
179
vllm/model_executor/models/fuyu.py
+3 -3
@@ -76,7 +76,7 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor):
76 return ImageSize(width=target_size["width"],
77 height=target_size["height"])
78
79 - def _get_image_grid_size(
79 + def _get_image_feature_grid_size(
80 self,
81 *,
82 image_width: int,
@@ -99,7 +99,7 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor):
99 def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
100 target_width, target_height = self._get_image_target_size()
101
102 - max_ncols, max_nrows = self._get_image_grid_size(
102 + max_ncols, max_nrows = self._get_image_feature_grid_size(
103 image_width=target_width,
104 image_height=target_height,
105 )
@@ -172,7 +172,7 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor):
172 images = mm_items.get_items("image", ImageProcessorItems)
173 image_size = images.get_image_size(item_idx)
174
175 - ncols, nrows = self._get_image_grid_size(
175 + ncols, nrows = self._get_image_feature_grid_size(
176 image_width=image_size.width,
177 image_height=image_size.height,
178 )
vllm/model_executor/models/llava.py
+215 -119
@@ -1,6 +1,7 @@
1 +from abc import abstractmethod
2 from functools import cached_property
2 -from typing import (Iterable, List, Literal, Mapping, Optional, Protocol, Set,
3 - Tuple, TypedDict, Union)
3 +from typing import (Final, Iterable, List, Literal, Mapping, Optional,
4 + Protocol, Set, Tuple, TypedDict, Union)
5
6 import torch
7 import torch.nn as nn
@@ -12,7 +13,6 @@ from transformers.models.pixtral import PixtralProcessor
13
14 from vllm.attention import AttentionMetadata
15 from vllm.config import VllmConfig
15 -from vllm.inputs import InputContext
16 from vllm.model_executor.layers.activation import get_act_fn
17 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
18 RowParallelLinear)
@@ -23,23 +23,23 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
23 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
24 MultiModalInputsV2, MultiModalKwargs,
25 NestedTensors)
26 -from vllm.multimodal.parse import ImageProcessorItems
26 +from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
27 + ImageSize)
28 from vllm.multimodal.processing import (BaseMultiModalProcessor,
28 - MultiModalDataItems, ProcessorInputs,
29 - PromptReplacement,
29 + InputProcessingContext,
30 + MultiModalDataItems, ProcessingCache,
31 + ProcessorInputs, PromptReplacement,
32 full_groupby_modality)
33 from vllm.sequence import IntermediateTensors
34
33 -from .clip import (CLIPVisionModel, dummy_image_for_clip,
34 - get_max_clip_image_tokens)
35 +from .clip import CLIPVisionModel
36 from .interfaces import SupportsMultiModal, SupportsPP
36 -from .pixtral import (PixtralHFVisionModel, dummy_image_for_pixtral_hf,
37 - get_max_pixtral_hf_image_tokens,
38 - get_pixtral_hf_image_feature_size)
39 -from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
40 - get_max_siglip_image_tokens)
37 +from .pixtral import (PixtralHFVisionModel,
38 + get_pixtral_hf_image_feature_grid_size)
39 +from .siglip import SiglipVisionModel
40 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
41 maybe_prefix, merge_multimodal_embeddings)
42 +from .vision import vision_encoder_info
43
44
45 class LlavaImagePixelInputs(TypedDict):
@@ -94,39 +94,167 @@ class LlavaMultiModalProjector(nn.Module):
94 return hidden_states
95
96
97 -def get_max_llava_image_tokens(ctx: InputContext):
98 - hf_config = ctx.get_hf_config(LlavaConfig)
99 - vision_config = hf_config.vision_config
97 +class LlavaLikeConfig(Protocol):
98 + vision_config: Final[PretrainedConfig]
99 + vision_feature_select_strategy: Final[str]
100 + vision_feature_layer: Final[Union[int, List[int]]]
101
101 - if isinstance(vision_config, CLIPVisionConfig):
102 - num_image_tokens = get_max_clip_image_tokens(vision_config)
103 - elif isinstance(vision_config, SiglipVisionConfig):
104 - num_image_tokens = get_max_siglip_image_tokens(vision_config)
105 - elif isinstance(vision_config, PixtralVisionConfig):
106 - num_image_tokens = get_max_pixtral_hf_image_tokens(vision_config)
107 - else:
108 - msg = f"Unsupported vision config: {type(vision_config)}"
109 - raise NotImplementedError(msg)
102
111 - strategy = hf_config.vision_feature_select_strategy
112 - if strategy == "default":
113 - return num_image_tokens - 1
114 - elif strategy == "full":
115 - return num_image_tokens
116 - else:
117 - raise ValueError(f"Unexpected select feature strategy: {strategy}")
103 +class BaseLlavaMultiModalProcessor(BaseMultiModalProcessor):
104
105 + def __init__(self,
106 + ctx: InputProcessingContext,
107 + *,
108 + cache: Optional[ProcessingCache] = None,
109 + enable_sanity_checks: bool = True) -> None:
110 + super().__init__(ctx,
111 + cache=cache,
112 + enable_sanity_checks=enable_sanity_checks)
113 +
114 + vision_config = self._get_hf_config().vision_config
115 + self._vision_encoder_info = vision_encoder_info(vision_config)
116
120 -class LlavaMultiModalProcessor(BaseMultiModalProcessor):
117 + @abstractmethod
118 + def _get_hf_config(self) -> LlavaLikeConfig:
119 + raise NotImplementedError
120
121 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
122 return {"image": None}
123
124 + def _apply_feature_select_strategy(
125 + self,
126 + strategy: str,
127 + encoder_num_image_tokens: int,
128 + ) -> int:
129 + if strategy == "default":
130 + return encoder_num_image_tokens - 1
131 + if strategy == "full":
132 + return encoder_num_image_tokens
133 +
134 + msg = f"Unexpected feature select strategy: {strategy!r}"
135 + raise NotImplementedError(msg)
136 +
137 + def _get_max_image_tokens(self) -> int:
138 + hf_config = self._get_hf_config()
139 +
140 + return self._apply_feature_select_strategy(
141 + hf_config.vision_feature_select_strategy,
142 + self._vision_encoder_info.get_max_image_tokens(),
143 + )
144 +
145 def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
126 - return {"image": get_max_llava_image_tokens(self.ctx)}
146 + return {"image": self._get_max_image_tokens()}
147 +
148 + def _get_mm_fields_config(
149 + self,
150 + hf_inputs: BatchFeature,
151 + hf_processor_mm_kwargs: Mapping[str, object],
152 + ) -> Mapping[str, MultiModalFieldConfig]:
153 + return dict(
154 + pixel_values=MultiModalFieldConfig.batched("image"),
155 + image_embeds=MultiModalFieldConfig.batched("image"),
156 + )
157 +
158 + def _get_dummy_image_size(self) -> ImageSize:
159 + image_size = self._vision_encoder_info.get_image_size()
160 + return ImageSize(image_size, image_size)
161 +
162 + @abstractmethod
163 + def _get_image_token(self) -> str:
164 + raise NotImplementedError
165 +
166 + def _get_dummy_mm_inputs(
167 + self,
168 + mm_counts: Mapping[str, int],
169 + ) -> ProcessorInputs:
170 + num_images = mm_counts.get("image", 0)
171 +
172 + image_token = self._get_image_token()
173 + target_width, target_height = self._get_dummy_image_size()
174 +
175 + mm_data = {
176 + "image":
177 + self._get_dummy_images(width=target_width,
178 + height=target_height,
179 + num_images=num_images)
180 + }
181 +
182 + return ProcessorInputs(
183 + prompt_text=image_token * num_images,
184 + mm_data=mm_data,
185 + )
186 +
187 +
188 +class LlavaMultiModalProcessor(BaseLlavaMultiModalProcessor):
189 +
190 + def _get_hf_config(self) -> LlavaConfig:
191 + return self.ctx.get_hf_config(LlavaConfig)
192 +
193 + def _get_hf_processor(self) -> LlavaProcessor:
194 + return self.ctx.get_hf_processor(LlavaProcessor)
195 +
196 + def _get_image_token(self) -> str:
197 + return self._get_hf_processor().image_token
198 +
199 + def _get_num_image_tokens(
200 + self,
201 + *,
202 + image_width: int,
203 + image_height: int,
204 + ) -> int:
205 + hf_config = self._get_hf_config()
206 +
207 + return self._apply_feature_select_strategy(
208 + hf_config.vision_feature_select_strategy,
209 + self._vision_encoder_info.get_num_image_tokens(
210 + image_width=image_width,
211 + image_height=image_height,
212 + ),
213 + )
214 +
215 + def _get_prompt_replacements(
216 + self,
217 + mm_items: MultiModalDataItems,
218 + hf_processor_mm_kwargs: Mapping[str, object],
219 + out_mm_kwargs: MultiModalKwargs,
220 + ) -> list[PromptReplacement]:
221 + hf_config = self._get_hf_config()
222 + image_token_id = hf_config.image_token_index
223
128 - def _get_hf_processor(self) -> Union[LlavaProcessor, PixtralProcessor]:
129 - return self.ctx.get_hf_processor((LlavaProcessor, PixtralProcessor))
224 + def get_replacement(item_idx: int):
225 + images = mm_items.get_items(
226 + "image", (ImageEmbeddingItems, ImageProcessorItems))
227 +
228 + if isinstance(images, ImageEmbeddingItems):
229 + num_image_tokens = images.get_feature_size(item_idx)
230 + else:
231 + image_size = images.get_image_size(item_idx)
232 + num_image_tokens = self._get_num_image_tokens(
233 + image_width=image_size.width,
234 + image_height=image_size.height,
235 + )
236 +
237 + return [image_token_id] * num_image_tokens
238 +
239 + return [
240 + PromptReplacement(
241 + modality="image",
242 + target=[image_token_id],
243 + replacement=get_replacement,
244 + ),
245 + ]
246 +
247 +
248 +class PixtralHFMultiModalProcessor(BaseLlavaMultiModalProcessor):
249 +
250 + def _get_hf_config(self) -> LlavaConfig:
251 + return self.ctx.get_hf_config(LlavaConfig)
252 +
253 + def _get_hf_processor(self) -> PixtralProcessor:
254 + return self.ctx.get_hf_processor(PixtralProcessor)
255 +
256 + def _get_image_token(self) -> str:
257 + return self._get_hf_processor().image_token
258
259 def _call_hf_processor(
260 self,
@@ -140,119 +268,82 @@ class LlavaMultiModalProcessor(BaseMultiModalProcessor):
268 mm_kwargs=mm_kwargs,
269 )
270
143 - # NOTE: pixel_values=None for MLlavaProcessor
271 pixel_values = processed_outputs.get("pixel_values")
272 if pixel_values is not None:
273 images = mm_data["images"]
274 assert isinstance(images, list)
275
149 - if isinstance(self._get_hf_processor(), PixtralProcessor):
150 - # Original output: (1, num_images, C, H, W)
151 - # New output: (num_images, C, H, W)
152 - assert (isinstance(pixel_values, list)
153 - and len(pixel_values) == 1)
154 - assert (isinstance(pixel_values[0], list)
155 - and len(pixel_values[0]) == len(images))
276 + # Original output: (1, num_images, C, H, W)
277 + # New output: (num_images, C, H, W)
278 + assert (isinstance(pixel_values, list) and len(pixel_values) == 1)
279 + assert (isinstance(pixel_values[0], list)
280 + and len(pixel_values[0]) == len(images))
281
157 - processed_outputs["pixel_values"] = pixel_values[0]
282 + processed_outputs["pixel_values"] = pixel_values[0]
283
284 return processed_outputs
285
161 - def _get_mm_fields_config(
162 - self,
163 - hf_inputs: BatchFeature,
164 - hf_processor_mm_kwargs: Mapping[str, object],
165 - ) -> Mapping[str, MultiModalFieldConfig]:
166 - return dict(
167 - pixel_values=MultiModalFieldConfig.batched("image"),
168 - image_embeds=MultiModalFieldConfig.batched("image"),
169 - )
170 -
286 def _get_prompt_replacements(
287 self,
288 mm_items: MultiModalDataItems,
289 hf_processor_mm_kwargs: Mapping[str, object],
290 out_mm_kwargs: MultiModalKwargs,
291 ) -> list[PromptReplacement]:
177 - hf_config = self.ctx.get_hf_config(LlavaConfig)
292 + hf_config = self._get_hf_config()
293 image_token_id = hf_config.image_token_index
294
295 processor = self._get_hf_processor()
181 - if isinstance(processor, PixtralProcessor):
182 - image_token = processor.image_token
183 - image_break_token = processor.image_break_token
184 - image_end_token = processor.image_end_token
185 -
186 - vision_config = hf_config.vision_config
187 - assert isinstance(vision_config, PixtralVisionConfig)
296 + image_token = processor.image_token
297 + image_break_token = processor.image_break_token
298 + image_end_token = processor.image_end_token
299
189 - def get_replacement_pixtral(item_idx: int):
190 - images = mm_items.get_items("image", ImageProcessorItems)
191 - image_size = images.get_image_size(item_idx)
192 -
193 - (
194 - num_width_tokens,
195 - num_height_tokens,
196 - ) = get_pixtral_hf_image_feature_size(
197 - vision_config,
198 - image_width=image_size.width,
199 - image_height=image_size.height,
200 - )
300 + vision_config = hf_config.vision_config
301 + assert isinstance(vision_config, PixtralVisionConfig)
302
202 - tokens = ([image_token] * num_width_tokens +
203 - [image_break_token]) * num_height_tokens
204 - tokens[-1] = image_end_token
303 + def get_replacement(item_idx: int):
304 + images = mm_items.get_items("image", ImageProcessorItems)
305 + image_size = images.get_image_size(item_idx)
306
206 - return "".join(tokens)
307 + ncols, nrows = get_pixtral_hf_image_feature_grid_size(
308 + vision_config,
309 + image_width=image_size.width,
310 + image_height=image_size.height,
311 + )
312
208 - return [
209 - PromptReplacement(
210 - modality="image",
211 - target=[image_token_id],
212 - replacement=get_replacement_pixtral,
213 - ),
214 - ]
313 + tokens = ([image_token] * ncols + [image_break_token]) * nrows
314 + tokens[-1] = image_end_token
315
216 - max_image_tokens = get_max_llava_image_tokens(self.ctx)
316 + return "".join(tokens)
317
318 return [
319 PromptReplacement(
320 modality="image",
321 target=[image_token_id],
222 - replacement=[image_token_id] * max_image_tokens,
223 - )
322 + replacement=get_replacement,
323 + ),
324 ]
325
226 - def _get_dummy_mm_inputs(
227 - self,
228 - mm_counts: Mapping[str, int],
229 - ) -> ProcessorInputs:
230 - hf_config = self.ctx.get_hf_config(LlavaConfig)
231 - vision_config = hf_config.vision_config
232 - num_images = mm_counts.get("image", 0)
233 -
234 - if isinstance(vision_config, CLIPVisionConfig):
235 - data = dummy_image_for_clip(vision_config, num_images)
236 - elif isinstance(vision_config, SiglipVisionConfig):
237 - data = dummy_image_for_siglip(vision_config, num_images)
238 - elif isinstance(vision_config, PixtralVisionConfig):
239 - data = dummy_image_for_pixtral_hf(vision_config, num_images)
240 - else:
241 - msg = f"Unsupported vision config: {type(vision_config)}"
242 - raise NotImplementedError(msg)
326
244 - hf_processor = self._get_hf_processor()
245 - image_token = hf_processor.image_token
327 +def _build_llava_or_pixtral_hf_processor(
328 + ctx: InputProcessingContext,
329 + *,
330 + cache: Optional[ProcessingCache] = None,
331 + enable_sanity_checks: bool = True,
332 +) -> BaseLlavaMultiModalProcessor:
333 + hf_config = ctx.get_hf_config(LlavaConfig)
334
247 - return ProcessorInputs(
248 - prompt_text=image_token * num_images,
249 - mm_data=data,
335 + if isinstance(hf_config.vision_config, PixtralVisionConfig):
336 + return PixtralHFMultiModalProcessor(
337 + ctx,
338 + cache=cache,
339 + enable_sanity_checks=enable_sanity_checks,
340 )
341
252 -
253 -class LlavaLikeConfig(Protocol):
254 - vision_config: PretrainedConfig
255 - vision_feature_layer: Union[int, List[int]]
342 + return LlavaMultiModalProcessor(
343 + ctx,
344 + cache=cache,
345 + enable_sanity_checks=enable_sanity_checks,
346 + )
347
348
349 def _get_num_hidden_layers(hf_config: LlavaLikeConfig) -> int:
@@ -330,7 +421,7 @@ def init_vision_tower_for_llava(
421 raise NotImplementedError(msg)
422
423
333 -@MULTIMODAL_REGISTRY.register_processor(LlavaMultiModalProcessor)
424 +@MULTIMODAL_REGISTRY.register_processor(_build_llava_or_pixtral_hf_processor)
425 class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
426 # BitandBytes specific attributes
427 bitsandbytes_stacked_params_mapping = {
@@ -596,7 +687,12 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
687 ) -> MultiModalInputsV2:
688 hf_config = self.ctx.get_hf_config(LlavaConfig)
689 image_token_id = hf_config.image_token_index
599 - max_image_tokens = get_max_llava_image_tokens(self.ctx)
690 +
691 + # Assume that it doesn't depend on the image size
692 + num_image_tokens = self._get_num_image_tokens(
693 + image_width=-1,
694 + image_height=-1,
695 + )
696
697 result = super().apply(prompt_text, mm_data, hf_processor_mm_kwargs)
698
@@ -609,14 +705,14 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
705 def get_replacement_mantis(item_idx: int):
706 return "".join([
707 f"(image {item_idx+1}: <Image>", # 7 tokens
612 - "<image>" * max_image_tokens,
708 + "<image>" * num_image_tokens,
709 "</Image>)", # 3 tokens
710 ])
711
712 mantis_repls = self._bind_prompt_replacements([
713 PromptReplacement(
714 modality="image",
619 - target=[image_token_id] * max_image_tokens,
715 + target=[image_token_id] * num_image_tokens,
716 replacement=get_replacement_mantis,
717 )
718 ])
vllm/model_executor/models/llava_next.py
+112 -209
@@ -4,31 +4,25 @@ from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
4
5 import torch
6 import torch.nn as nn
7 -from PIL import Image
8 -from transformers import CLIPVisionConfig, LlavaNextConfig, SiglipVisionConfig
7 +from transformers import BatchFeature, LlavaNextConfig, LlavaNextProcessor
8 from transformers.models.llava_next.modeling_llava_next import (
9 get_anyres_image_grid_shape, unpad_image)
10 from typing_extensions import NotRequired
11
12 from vllm.attention import AttentionMetadata
13 from vllm.config import VllmConfig
15 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
16 - InputContext)
14 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
15 from vllm.model_executor.sampling_metadata import SamplingMetadata
16 from vllm.multimodal import MULTIMODAL_REGISTRY
20 -from vllm.multimodal.inputs import NestedTensors
17 +from vllm.multimodal.inputs import MultiModalFieldConfig, NestedTensors
18 +from vllm.multimodal.parse import ImageSize
19 from vllm.sequence import IntermediateTensors
22 -from vllm.utils import is_list_of
20
24 -from .clip import (CLIPVisionModel, dummy_image_for_clip,
25 - dummy_seq_data_for_clip, get_clip_image_feature_size,
26 - get_clip_patch_grid_length, input_processor_for_clip)
21 +from .clip import CLIPVisionModel
22 from .interfaces import SupportsMultiModal, SupportsPP
28 -from .llava import LlavaMultiModalProjector, init_vision_tower_for_llava
29 -from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
30 - dummy_seq_data_for_siglip, get_siglip_image_feature_size,
31 - get_siglip_patch_grid_length, input_processor_for_siglip)
23 +from .llava import (LlavaMultiModalProcessor, LlavaMultiModalProjector,
24 + init_vision_tower_for_llava)
25 +from .siglip import SiglipVisionModel
26 from .utils import (AutoWeightsLoader, embed_multimodal, flatten_bn,
27 init_vllm_registered_model, maybe_prefix)
28
@@ -65,218 +59,127 @@ LlavaNextImageInputs = Union[LlavaNextImagePixelInputs,
59 LlavaNextImageEmbeddingInputs]
60
61
68 -# Based on: https://github.com/huggingface/text-generation-inference/blob/v2.2.0/server/text_generation_server/models/vlm_causal_lm.py#L79
69 -def _get_llava_next_num_unpadded_features(
70 - original_height: int,
71 - original_width: int,
72 - npatches: int,
73 - num_patch_height: int,
74 - num_patch_width: int,
75 -) -> Tuple[int, int]:
76 - current_height = npatches * num_patch_height
77 - current_width = npatches * num_patch_width
78 -
79 - original_aspect_ratio = original_width / original_height
80 - current_aspect_ratio = current_width / current_height
81 -
82 - if original_aspect_ratio > current_aspect_ratio:
83 - scale_factor = current_width / original_width
84 - new_height = int(original_height * scale_factor)
85 - padding = (current_height - new_height) // 2
86 - current_height -= 2 * padding
87 - else:
88 - scale_factor = current_height / original_height
89 - new_width = int(original_width * scale_factor)
90 - padding = (current_width - new_width) // 2
91 - current_width -= 2 * padding
92 -
93 - unpadded_features = current_height * current_width
94 - newline_features = current_height
95 - return (unpadded_features, newline_features)
96 -
97 -
98 -# Based on: https://github.com/huggingface/text-generation-inference/blob/v2.2.0/server/text_generation_server/models/vlm_causal_lm.py#L106
99 -def get_llava_next_image_feature_size(
100 - hf_config: LlavaNextConfig,
101 - *,
102 - input_height: int,
103 - input_width: int,
104 -) -> int:
105 - vision_config = hf_config.vision_config
106 -
107 - if isinstance(vision_config, CLIPVisionConfig):
108 - num_patches = get_clip_patch_grid_length(
109 - image_size=vision_config.image_size,
110 - patch_size=vision_config.patch_size,
111 - )
112 - base_feature_size = get_clip_image_feature_size(vision_config)
113 - elif isinstance(vision_config, SiglipVisionConfig):
114 - num_patches = get_siglip_patch_grid_length(
115 - image_size=vision_config.image_size,
116 - patch_size=vision_config.patch_size,
117 - )
118 - base_feature_size = get_siglip_image_feature_size(vision_config)
119 - else:
120 - msg = f"Unsupported vision config: {type(vision_config)}"
121 - raise NotImplementedError(msg)
122 -
123 - strategy = hf_config.vision_feature_select_strategy
124 - if strategy == "default":
125 - base_feature_size -= 1
126 - elif strategy == "full":
127 - pass
128 - else:
129 - raise ValueError(f"Unexpected select feature strategy: {strategy}")
62 +class LlavaNextMultiModalProcessor(LlavaMultiModalProcessor):
63
131 - num_patch_height, num_patch_width = get_anyres_image_grid_shape(
132 - image_size=(input_height, input_width),
133 - grid_pinpoints=hf_config.image_grid_pinpoints,
134 - patch_size=vision_config.image_size,
135 - )
136 -
137 - (
138 - unpadded_feature_size,
139 - newline_feature_size,
140 - ) = _get_llava_next_num_unpadded_features(input_height, input_width,
141 - num_patches, num_patch_height,
142 - num_patch_width)
143 -
144 - return unpadded_feature_size + newline_feature_size + base_feature_size
145 -
146 -
147 -def get_max_llava_next_image_tokens(ctx: InputContext):
148 - """Compute the max feature size for all possible image grid pinpoints."""
149 - return _get_pinpoint_with_largest_features(ctx)[0]
150 -
151 -
152 -def _get_pinpoint_with_largest_features(
153 - ctx: InputContext) -> Tuple[int, Tuple[int, int]]:
154 - """Get the grid pinpoint with the largest features & its feature size."""
155 - hf_config = ctx.get_hf_config(LlavaNextConfig)
156 - largest_feature_size = 0
157 - largest_feature_pinpoint = None
158 - for (height, width) in hf_config.image_grid_pinpoints:
159 - feat_size = get_llava_next_image_feature_size(
160 - hf_config,
161 - input_height=height,
162 - input_width=width,
163 - )
164 - if feat_size > largest_feature_size:
165 - largest_feature_size = feat_size
166 - largest_feature_pinpoint = (height, width)
167 - if not largest_feature_size or largest_feature_pinpoint is None:
168 - raise ValueError("Cannot have a largest feature size of 0!")
169 - return largest_feature_size, largest_feature_pinpoint
170 -
171 -
172 -def dummy_data_for_llava_next(ctx: InputContext, seq_len: int,
173 - mm_counts: Mapping[str, int]):
174 - hf_config = ctx.get_hf_config(LlavaNextConfig)
175 - vision_config = hf_config.vision_config
176 - num_images = mm_counts["image"]
177 -
178 - image_feature_size, pinpoint = _get_pinpoint_with_largest_features(ctx)
179 - max_feat_height, max_feat_width = pinpoint
180 -
181 - if isinstance(vision_config, CLIPVisionConfig):
182 - seq_data, ranges = dummy_seq_data_for_clip(
183 - vision_config,
184 - seq_len,
185 - num_images,
186 - image_token_id=hf_config.image_token_index,
187 - image_feature_size_override=image_feature_size,
188 - )
64 + def _get_hf_config(self) -> LlavaNextConfig:
65 + return self.ctx.get_hf_config(LlavaNextConfig)
66 +
67 + def _get_hf_processor(self) -> LlavaNextProcessor:
68 + return self.ctx.get_hf_processor(LlavaNextProcessor)
69
190 - mm_data = dummy_image_for_clip(
191 - vision_config,
192 - num_images,
193 - image_width_override=max_feat_width,
194 - image_height_override=max_feat_height,
70 + def _get_image_token(self) -> str:
71 + return self._get_hf_processor().image_token
72 +
73 + def _get_mm_fields_config(
74 + self,
75 + hf_inputs: BatchFeature,
76 + hf_processor_mm_kwargs: Mapping[str, object],
77 + ) -> Mapping[str, MultiModalFieldConfig]:
78 + return dict(
79 + pixel_values=MultiModalFieldConfig.batched("image"),
80 + image_sizes=MultiModalFieldConfig.batched("image"),
81 + image_embeds=MultiModalFieldConfig.batched("image"),
82 )
83
197 - return DummyData(seq_data, mm_data, ranges)
198 - elif isinstance(vision_config, SiglipVisionConfig):
199 - seq_data, ranges = dummy_seq_data_for_siglip(
200 - vision_config,
201 - seq_len,
202 - num_images,
203 - image_token_id=hf_config.image_token_index,
204 - image_feature_size_override=image_feature_size,
84 + def _get_max_image_tokens(self) -> int:
85 + largest_feature_size, _ = self._get_pinpoint_with_most_features()
86 + return largest_feature_size
87 +
88 + def _get_dummy_image_size(self) -> ImageSize:
89 + _, pinpoint = self._get_pinpoint_with_most_features()
90 + return pinpoint
91 +
92 + # Based on: https://github.com/huggingface/text-generation-inference/blob/v2.2.0/server/text_generation_server/models/vlm_causal_lm.py#L106
93 + def _get_num_image_tokens(
94 + self,
95 + *,
96 + image_width: int,
97 + image_height: int,
98 + ) -> int:
99 + hf_config = self._get_hf_config()
100 +
101 + base_feature_size = self._apply_feature_select_strategy(
102 + hf_config.vision_feature_select_strategy,
103 + self._vision_encoder_info.get_num_image_tokens(
104 + image_width=image_width,
105 + image_height=image_height,
106 + ),
107 )
108 + num_patches = self._vision_encoder_info.get_num_patches()
109
207 - mm_data = dummy_image_for_siglip(
208 - vision_config,
209 - num_images,
210 - image_width_override=max_feat_width,
211 - image_height_override=max_feat_height,
110 + num_patch_height, num_patch_width = get_anyres_image_grid_shape(
111 + image_size=(image_height, image_width),
112 + grid_pinpoints=hf_config.image_grid_pinpoints,
113 + patch_size=self._vision_encoder_info.get_image_size(),
114 )
115
214 - return DummyData(seq_data, mm_data, ranges)
116 + (
117 + unpadded_feature_size,
118 + newline_feature_size,
119 + ) = self._get_num_unpadded_features(
120 + original_height=image_height,
121 + original_width=image_width,
122 + npatches=num_patches,
123 + num_patch_height=num_patch_height,
124 + num_patch_width=num_patch_width,
125 + )
126
216 - msg = f"Unsupported vision config: {type(vision_config)}"
217 - raise NotImplementedError(msg)
127 + return unpadded_feature_size + newline_feature_size + base_feature_size
128
129 + # Based on: https://github.com/huggingface/text-generation-inference/blob/v2.2.0/server/text_generation_server/models/vlm_causal_lm.py#L79
130 + def _get_num_unpadded_features(
131 + self,
132 + *,
133 + original_height: int,
134 + original_width: int,
135 + npatches: int,
136 + num_patch_height: int,
137 + num_patch_width: int,
138 + ) -> tuple[int, int]:
139 + current_height = npatches * num_patch_height
140 + current_width = npatches * num_patch_width
141 +
142 + original_aspect_ratio = original_width / original_height
143 + current_aspect_ratio = current_width / current_height
144 +
145 + if original_aspect_ratio > current_aspect_ratio:
146 + scale_factor = current_width / original_width
147 + new_height = int(original_height * scale_factor)
148 + padding = (current_height - new_height) // 2
149 + current_height -= 2 * padding
150 + else:
151 + scale_factor = current_height / original_height
152 + new_width = int(original_width * scale_factor)
153 + padding = (current_width - new_width) // 2
154 + current_width -= 2 * padding
155
220 -def input_processor_for_llava_next(ctx: InputContext,
221 - inputs: DecoderOnlyInputs):
222 - multi_modal_data = inputs.get("multi_modal_data")
223 - if multi_modal_data is None or "image" not in multi_modal_data:
224 - return inputs
156 + unpadded_features = current_height * current_width
157 + newline_features = current_height
158 + return (unpadded_features, newline_features)
159
226 - model_config = ctx.model_config
227 - hf_config = ctx.get_hf_config(LlavaNextConfig)
228 - vision_config = hf_config.vision_config
160 + def _get_pinpoint_with_most_features(self) -> tuple[int, ImageSize]:
161 + """
162 + Get the grid pinpoint with the most features and
163 + the corresponding feature size.
164 + """
165 + hf_config = self._get_hf_config()
166
230 - image_data = multi_modal_data["image"]
231 - if isinstance(image_data, Image.Image):
232 - width, height = image_data.size
167 + largest_feature_size, largest_feature_pinpoint = 0, None
168 + for (height, width) in hf_config.image_grid_pinpoints:
169 + feat_size = self._get_num_image_tokens(image_width=width,
170 + image_height=height)
171 + if feat_size > largest_feature_size:
172 + largest_feature_size = feat_size
173 + largest_feature_pinpoint = ImageSize(width=width,
174 + height=height)
175
234 - image_feature_size = get_llava_next_image_feature_size(
235 - hf_config,
236 - input_height=height,
237 - input_width=width,
238 - )
239 - elif is_list_of(image_data, Image.Image):
240 - image_feature_size = [
241 - get_llava_next_image_feature_size(hf_config,
242 - input_height=img.height,
243 - input_width=img.width)
244 - for img in image_data
245 - ]
246 - elif isinstance(image_data, torch.Tensor):
247 - num_images, image_feature_size, hidden_size = image_data.shape
248 - elif is_list_of(image_data, torch.Tensor):
249 - image_feature_size = [item.shape[1] for item in image_data]
250 - else:
251 - raise TypeError(f"Invalid image type: {type(image_data)}")
252 -
253 - vision_config = hf_config.vision_config
254 -
255 - if isinstance(vision_config, CLIPVisionConfig):
256 - return input_processor_for_clip(
257 - model_config,
258 - vision_config,
259 - inputs,
260 - image_token_id=hf_config.image_token_index,
261 - image_feature_size_override=image_feature_size,
262 - )
263 - elif isinstance(vision_config, SiglipVisionConfig):
264 - return input_processor_for_siglip(
265 - model_config,
266 - vision_config,
267 - inputs,
268 - image_token_id=hf_config.image_token_index,
269 - image_feature_size_override=image_feature_size,
270 - )
176 + if largest_feature_size == 0 or largest_feature_pinpoint is None:
177 + raise ValueError("Cannot have a largest feature size of 0!")
178
272 - msg = f"Unsupported vision config: {type(vision_config)}"
273 - raise NotImplementedError(msg)
179 + return largest_feature_size, largest_feature_pinpoint
180
181
276 -@MULTIMODAL_REGISTRY.register_image_input_mapper()
277 -@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_next_image_tokens)
278 -@INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava_next)
279 -@INPUT_REGISTRY.register_input_processor(input_processor_for_llava_next)
182 +@MULTIMODAL_REGISTRY.register_processor(LlavaNextMultiModalProcessor)
183 class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
184 SupportsPP):
185
@@ -507,7 +410,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
410 def _process_image_pixels(
411 self,
412 inputs: LlavaNextImagePixelInputs,
510 - ) -> Union[torch.Tensor, List[torch.Tensor]]:
413 + ) -> Union[torch.Tensor, tuple[torch.Tensor, ...]]:
414 assert self.vision_tower is not None
415
416 pixel_values = inputs["data"]
vllm/model_executor/models/phi3v.py
+14 -10
@@ -34,7 +34,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
34 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
35 MultiModalInputsV2, MultiModalKwargs,
36 NestedTensors, PlaceholderRange)
37 -from vllm.multimodal.parse import ImageProcessorItems
37 +from vllm.multimodal.parse import ImageEmbeddingItems, ImageProcessorItems
38 from vllm.multimodal.processing import (BaseMultiModalProcessor,
39 MultiModalDataItems, ProcessorInputs,
40 PromptReplacement,
@@ -388,15 +388,19 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
388 assert isinstance(bos_token_id, int)
389
390 def get_replacement_phi3v(item_idx: int):
391 - images = mm_items.get_items("image", ImageProcessorItems)
392 - image_size = images.get_image_size(item_idx)
393 -
394 - num_tokens = self._get_num_image_tokens(
395 - image_width=image_size.width,
396 - image_height=image_size.height,
397 - )
398 -
399 - return [_IMAGE_TOKEN_ID] * num_tokens + [bos_token_id]
391 + images = mm_items.get_items(
392 + "image", (ImageEmbeddingItems, ImageProcessorItems))
393 +
394 + if isinstance(images, ImageEmbeddingItems):
395 + num_image_tokens = images.get_feature_size(item_idx)
396 + else:
397 + image_size = images.get_image_size(item_idx)
398 + num_image_tokens = self._get_num_image_tokens(
399 + image_width=image_size.width,
400 + image_height=image_size.height,
401 + )
402 +
403 + return [_IMAGE_TOKEN_ID] * num_image_tokens + [bos_token_id]
404
405 num_images = mm_items.get_count("image", strict=False)
406
vllm/model_executor/models/pixtral.py
+52 -14
@@ -38,6 +38,7 @@ from vllm.sequence import IntermediateTensors, SequenceData
38 from .interfaces import SupportsMultiModal, SupportsPP
39 from .utils import (init_vllm_registered_model, maybe_prefix,
40 merge_multimodal_embeddings)
41 +from .vision import VisionEncoderInfo
42
43 try:
44 from xformers import ops as xops
@@ -697,10 +698,18 @@ def get_pixtral_hf_patch_grid_length(*, image_size: int,
698 return image_size // patch_size
699
700
700 -def get_pixtral_hf_num_patches(*, image_size: int, patch_size: int) -> int:
701 - grid_length = get_pixtral_hf_patch_grid_length(image_size=image_size,
702 - patch_size=patch_size)
703 - return grid_length * grid_length
701 +def get_pixtral_hf_image_feature_size(
702 + *,
703 + image_size: int,
704 + patch_size: int,
705 +) -> int:
706 + grid_length = get_pixtral_hf_patch_grid_length(
707 + image_size=image_size,
708 + patch_size=patch_size,
709 + )
710 +
711 + # Consider the image_break_token
712 + return (grid_length + 1) * grid_length
713
714
715 def get_max_pixtral_hf_image_tokens(hf_config: PixtralVisionConfig) -> int:
@@ -730,13 +739,16 @@ def dummy_image_for_pixtral_hf(
739 return {"image": image if num_images == 1 else [image] * num_images}
740
741
733 -def get_pixtral_hf_image_feature_size(hf_config: PixtralVisionConfig,
734 - image_width: int,
735 - image_height: int) -> Tuple[int, int]:
736 - # Adapted from transformers.models.pixtral.image_processing_pixtral.get_resize_output_image_size # noqa: E501
737 - # https://github.com/huggingface/transformers/blob/2bd4d5897dc73e8b172832070a6f9e567a0df017/src/transformers/models/pixtral/image_processing_pixtral.py#L180 # noqa: E501
738 - max_width, max_height = hf_config.image_size, hf_config.image_size
739 - patch_width, patch_height = hf_config.patch_size, hf_config.patch_size
742 +# Adapted from transformers.models.pixtral.image_processing_pixtral.get_resize_output_image_size # noqa: E501
743 +# https://github.com/huggingface/transformers/blob/2bd4d5897dc73e8b172832070a6f9e567a0df017/src/transformers/models/pixtral/image_processing_pixtral.py#L180
744 +def get_pixtral_hf_image_feature_grid_size(
745 + hf_config: PixtralVisionConfig,
746 + *,
747 + image_width: int,
748 + image_height: int,
749 +) -> tuple[int, int]:
750 + max_width = max_height = hf_config.image_size
751 + patch_width = patch_height = hf_config.patch_size
752
753 ratio = max(image_width / max_width, image_height / max_height)
754
@@ -744,12 +756,38 @@ def get_pixtral_hf_image_feature_size(hf_config: PixtralVisionConfig,
756 image_width = int(math.ceil(image_width / ratio))
757 image_height = int(math.ceil(image_height / ratio))
758
747 - num_height_tokens, num_width_tokens = _get_pixtral_hf_num_image_tokens(
759 + nrows, ncols = _get_pixtral_hf_num_image_tokens(
760 (image_height, image_width),
761 (patch_height, patch_width),
750 - )
762 + ) # type: ignore
763 +
764 + return ncols, nrows
765 +
766 +
767 +class PixtralHFEncoderInfo(VisionEncoderInfo[PixtralVisionConfig]):
768 +
769 + def get_num_image_tokens(
770 + self,
771 + *,
772 + image_width: int,
773 + image_height: int,
774 + ) -> int:
775 + return get_pixtral_hf_image_feature_size(
776 + image_size=self.vision_config.image_size,
777 + patch_size=self.get_image_size(),
778 + )
779 +
780 + def get_max_image_tokens(self) -> int:
781 + return get_max_pixtral_hf_image_tokens(self.vision_config)
782 +
783 + def get_num_patches(self) -> int:
784 + return get_pixtral_hf_patch_grid_length(
785 + image_size=self.vision_config.image_size,
786 + patch_size=self.vision_config.patch_size,
787 + )
788
752 - return num_width_tokens, num_height_tokens
789 + def get_image_size(self) -> int:
790 + return self.vision_config.image_size
791
792
793 class PixtralHFMLP(nn.Module):
vllm/model_executor/models/siglip.py
+25
@@ -28,6 +28,8 @@ from vllm.multimodal.utils import (cached_get_tokenizer,
28 resolve_visual_encoder_outputs)
29 from vllm.sequence import SequenceData
30
31 +from .vision import VisionEncoderInfo
32 +
33
34 def get_siglip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
35 # Since interpolation is applied, the image size need not be divisible
@@ -156,6 +158,29 @@ def input_processor_for_siglip(
158 multi_modal_placeholders={"image": ranges})
159
160
161 +class SiglipEncoderInfo(VisionEncoderInfo[SiglipVisionConfig]):
162 +
163 + def get_num_image_tokens(
164 + self,
165 + *,
166 + image_width: int,
167 + image_height: int,
168 + ) -> int:
169 + return get_siglip_image_feature_size(self.vision_config)
170 +
171 + def get_max_image_tokens(self) -> int:
172 + return get_max_siglip_image_tokens(self.vision_config)
173 +
174 + def get_num_patches(self) -> int:
175 + return get_siglip_patch_grid_length(
176 + image_size=self.vision_config.image_size,
177 + patch_size=self.vision_config.patch_size,
178 + )
179 +
180 + def get_image_size(self) -> int:
181 + return self.vision_config.image_size
182 +
183 +
184 # Adapted from https://github.com/huggingface/transformers/blob/v4.43.3/src/transformers/models/siglip/modeling_siglip.py#L249 # noqa
185 class SiglipVisionEmbeddings(nn.Module):
186
vllm/model_executor/models/utils.py
+1 -1
@@ -373,7 +373,7 @@ def embed_multimodal(
373 input_ids: torch.Tensor,
374 multimodal_token_id: int,
375 get_text_embeds: Callable[[torch.Tensor], torch.Tensor],
376 - multimodal_embeds: Union[torch.Tensor, List[torch.Tensor]],
376 + multimodal_embeds: NestedTensors,
377 ) -> torch.Tensor:
378 """
379 Embed token IDs and multimodal inputs and combine their embeddings.
vllm/model_executor/models/vision.py new
+52
@@ -0,0 +1,52 @@
1 +from abc import ABC, abstractmethod
2 +from typing import Generic, TypeVar
3 +
4 +from transformers import PretrainedConfig
5 +
6 +_C = TypeVar("_C", bound=PretrainedConfig)
7 +
8 +
9 +class VisionEncoderInfo(ABC, Generic[_C]):
10 +
11 + def __init__(self, vision_config: _C) -> None:
12 + super().__init__()
13 +
14 + self.vision_config = vision_config
15 +
16 + @abstractmethod
17 + def get_num_image_tokens(
18 + self,
19 + *,
20 + image_width: int,
21 + image_height: int,
22 + ) -> int:
23 + raise NotImplementedError
24 +
25 + @abstractmethod
26 + def get_max_image_tokens(self) -> int:
27 + raise NotImplementedError
28 +
29 + @abstractmethod
30 + def get_num_patches(self) -> int:
31 + raise NotImplementedError
32 +
33 + @abstractmethod
34 + def get_image_size(self) -> int:
35 + raise NotImplementedError
36 +
37 +
38 +def vision_encoder_info(vision_config: PretrainedConfig) -> VisionEncoderInfo:
39 + # Avoid circular imports
40 + from .clip import CLIPEncoderInfo, CLIPVisionConfig
41 + from .pixtral import PixtralHFEncoderInfo, PixtralVisionConfig
42 + from .siglip import SiglipEncoderInfo, SiglipVisionConfig
43 +
44 + if isinstance(vision_config, CLIPVisionConfig):
45 + return CLIPEncoderInfo(vision_config)
46 + if isinstance(vision_config, PixtralVisionConfig):
47 + return PixtralHFEncoderInfo(vision_config)
48 + if isinstance(vision_config, SiglipVisionConfig):
49 + return SiglipEncoderInfo(vision_config)
50 +
51 + msg = f"Unsupported vision config: {type(vision_config)}"
52 + raise NotImplementedError(msg)
vllm/multimodal/parse.py
+8 -4
@@ -1,7 +1,8 @@
1 from abc import ABC, abstractmethod
2 from collections import UserDict
3 from collections.abc import Callable, Iterator, Mapping, Sequence
4 -from typing import TYPE_CHECKING, Any, Generic, NamedTuple, Optional, TypeVar
4 +from typing import (TYPE_CHECKING, Any, Generic, NamedTuple, Optional, TypeVar,
5 + Union)
6
7 import numpy as np
8 import torch
@@ -87,7 +88,7 @@ class EmbeddingItems(ModalityDataItems[NestedTensors, torch.Tensor]):
88 def get_count(self) -> int:
89 return len(self.data)
90
90 - def get(self, index: int) -> object:
91 + def get(self, index: int) -> torch.Tensor:
92 return self.data[index]
93
94 def get_processor_data(self) -> Mapping[str, object]:
@@ -96,6 +97,9 @@ class EmbeddingItems(ModalityDataItems[NestedTensors, torch.Tensor]):
97 def get_passthrough_data(self) -> Mapping[str, object]:
98 return {f"{self.modality}_embeds": self.data}
99
100 + def get_feature_size(self, item_idx: int) -> int:
101 + return len(self.get(item_idx))
102 +
103
104 class AudioProcessorItems(ProcessorBatchItems[HfAudioItem]):
105
@@ -182,7 +186,7 @@ class MultiModalDataItems(UserDict[str, ModalityDataItems[Any, Any]]):
186 def get_items(
187 self,
188 modality: str,
185 - typ: type[_D],
189 + typ: Union[type[_D], tuple[type[_D], ...]],
190 ) -> _D:
191 """
192 Get the data items belonging to a modality,
@@ -199,7 +203,7 @@ class MultiModalDataItems(UserDict[str, ModalityDataItems[Any, Any]]):
203 f"Expected type: {typ}, but "
204 f"found type: {type(items)}")
205
202 - return items
206 + return items # type: ignore[return-value]
207
208
209 ModalityDataParser: TypeAlias = Callable[[ModalityData[Any]],