A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Model] Initial support for LLaVA-NeXT (#4199)
Co-authored-by: Roger Wang <ywang@roblox.com>
Cyrus Leung committed
Jun 10, 2024 at 20:47 UTC
6b29d6fe709b91346be102a697bfaea386ed9107
7 files changed
+640
-18
docs/source/models/supported_models.rst
+5
-1
@@ -89,7 +89,11 @@ Alongside each architecture, we include some popular models that use it.
89
- ✅︎
90
* - :code:`LlavaForConditionalGeneration`
91
- LLaVA-1.5
92
- - :code:`llava-hf/llava-1.5-7b-hf`\*, :code:`llava-hf/llava-1.5-13b-hf`\*, etc.
92
+ - :code:`llava-hf/llava-1.5-7b-hf`, :code:`llava-hf/llava-1.5-13b-hf`, etc.
93
+ -
94
+ * - :code:`LlavaNextForConditionalGeneration`
95
+ - LLaVA-NeXT
96
+ - :code:`llava-hf/llava-v1.6-mistral-7b-hf`, :code:`llava-hf/llava-v1.6-vicuna-7b-hf`, etc.
97
-
98
* - :code:`MiniCPMForCausalLM`
99
- MiniCPM
tests/models/test_llava.py
-2
@@ -39,8 +39,6 @@ def iter_llava_configs(model_name: str):
39
40
model_and_vl_config = [
41
*iter_llava_configs("llava-hf/llava-1.5-7b-hf"),
42
- # Not enough memory
43
- # *iter_llava_configs("llava-hf/llava-1.5-13b-hf"),
42
]
43
44
tests/models/test_llava_next.py
new
+123
@@ -0,0 +1,123 @@
1
+from typing import List, Tuple
2
+
3
+import pytest
4
+from transformers import AutoTokenizer
5
+
6
+from vllm.config import VisionLanguageConfig
7
+
8
+from ..conftest import IMAGE_FILES
9
+
10
+pytestmark = pytest.mark.llava
11
+
12
+_PREFACE = (
13
+ "A chat between a curious human and an artificial intelligence assistant. "
14
+ "The assistant gives helpful, detailed, and polite answers to the human's "
15
+ "questions.")
16
+
17
+# The image token is placed before "user" on purpose so that the test can pass
18
+HF_IMAGE_PROMPTS = [
19
+ f"{_PREFACE} <image>\nUSER: What's the content of the image? ASSISTANT:",
20
+ f"{_PREFACE} <image>\nUSER: What is the season? ASSISTANT:",
21
+]
22
+
23
+assert len(HF_IMAGE_PROMPTS) == len(IMAGE_FILES)
24
+
25
+
26
+def iter_llava_next_configs(model_name: str):
27
+ image_hw_to_feature_size = {
28
+ (336, 336): 1176,
29
+ (672, 672): 2928,
30
+ (1344, 336): 1944,
31
+ (336, 1344): 1890,
32
+ }
33
+
34
+ for (h, w), f in image_hw_to_feature_size.items():
35
+ for input_type, input_shape in [
36
+ (VisionLanguageConfig.ImageInputType.PIXEL_VALUES, (1, 3, h, w)),
37
+ ]:
38
+ yield (model_name,
39
+ VisionLanguageConfig(image_input_type=input_type,
40
+ image_feature_size=f,
41
+ image_token_id=32000,
42
+ image_input_shape=input_shape,
43
+ image_processor=model_name,
44
+ image_processor_revision=None))
45
+
46
+
47
+model_and_vl_config = [
48
+ *iter_llava_next_configs("llava-hf/llava-v1.6-vicuna-7b-hf"),
49
+]
50
+
51
+
52
+def vllm_to_hf_output(vllm_output: Tuple[List[int], str],
53
+ vlm_config: VisionLanguageConfig, model_id: str):
54
+ """Sanitize vllm output to be comparable with hf output.
55
+ The function reduces `input_ids` from 1, 32000, 32000, ..., 32000,
56
+ x1, x2, x3 ... to 1, 32000, x1, x2, x3 ...
57
+ It also reduces `output_str` from "<image><image>bla" to "bla".
58
+ """
59
+ input_ids, output_str = vllm_output
60
+ image_token_id = vlm_config.image_token_id
61
+
62
+ tokenizer = AutoTokenizer.from_pretrained(model_id)
63
+ image_token_str = tokenizer.decode(image_token_id)
64
+
65
+ hf_input_ids = [
66
+ input_id for idx, input_id in enumerate(input_ids)
67
+ if input_id != image_token_id or input_ids[idx - 1] != image_token_id
68
+ ]
69
+ hf_output_str = output_str \
70
+ .replace(image_token_str * vlm_config.image_feature_size, " ")
71
+
72
+ return hf_input_ids, hf_output_str
73
+
74
+
75
+@pytest.mark.xfail(
76
+ reason="Inconsistent image processor being used due to lack "
77
+ "of support for dynamic image token replacement")
78
+@pytest.mark.parametrize("model_and_config", model_and_vl_config)
79
+@pytest.mark.parametrize("dtype", ["half"])
80
+@pytest.mark.parametrize("max_tokens", [128])
81
+def test_models(hf_runner, vllm_runner, hf_images, vllm_images,
82
+ model_and_config, dtype: str, max_tokens: int) -> None:
83
+ """Inference result should be the same between hf and vllm.
84
+
85
+ All the image fixtures for the test is under tests/images.
86
+ For huggingface runner, we provide the PIL images as input.
87
+ For vllm runner, we provide MultiModalData objects and corresponding
88
+ vision language config as input.
89
+ Note, the text input is also adjusted to abide by vllm contract.
90
+ The text output is sanitized to be able to compare with hf.
91
+ """
92
+ model_id, vlm_config = model_and_config
93
+
94
+ with hf_runner(model_id, dtype=dtype, is_vision_model=True) as hf_model:
95
+ hf_outputs = hf_model.generate_greedy(HF_IMAGE_PROMPTS,
96
+ max_tokens,
97
+ images=hf_images)
98
+
99
+ vllm_image_prompts = [
100
+ p.replace("<image>", "<image>" * vlm_config.image_feature_size)
101
+ for p in HF_IMAGE_PROMPTS
102
+ ]
103
+
104
+ with vllm_runner(
105
+ model_id,
106
+ dtype=dtype,
107
+ # should be greater than image_feature_size
108
+ max_model_len=4096,
109
+ enforce_eager=True,
110
+ **vlm_config.as_cli_args_dict(),
111
+ ) as vllm_model:
112
+ vllm_outputs = vllm_model.generate_greedy(vllm_image_prompts,
113
+ max_tokens,
114
+ images=vllm_images)
115
+
116
+ for i in range(len(HF_IMAGE_PROMPTS)):
117
+ hf_output_ids, hf_output_str = hf_outputs[i]
118
+ vllm_output_ids, vllm_output_str = vllm_to_hf_output(
119
+ vllm_outputs[i], vlm_config, model_id)
120
+ assert hf_output_str == vllm_output_str, (
121
+ f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
122
+ assert hf_output_ids == vllm_output_ids, (
123
+ f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
tests/multimodal/test_processor.py
+55
-7
@@ -1,6 +1,6 @@
1
import numpy as np
2
import pytest
3
-from transformers import CLIPImageProcessor
3
+from transformers import CLIPImageProcessor, LlavaNextImageProcessor
4
5
from vllm.config import ModelConfig, VisionLanguageConfig
6
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -12,7 +12,7 @@ from ..conftest import _STR_DTYPE_TO_TORCH_DTYPE
12
@pytest.mark.parametrize("dtype", ["half", "float"])
13
def test_clip_image_processor(hf_images, dtype):
14
MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
15
- IMAGE_HEIGHT = IMAGE_WIDTH = 33
15
+ IMAGE_HEIGHT = IMAGE_WIDTH = 560
16
17
hf_processor = CLIPImageProcessor.from_pretrained(MODEL_NAME)
18
assert isinstance(hf_processor, CLIPImageProcessor)
@@ -55,10 +55,61 @@ def test_clip_image_processor(hf_images, dtype):
55
assert np.allclose(hf_arr, vllm_arr), f"Failed for key={key}"
56
57
58
+@pytest.mark.xfail(
59
+ reason="Inconsistent image processor being used due to lack "
60
+ "of support for dynamic image token replacement")
61
+@pytest.mark.parametrize("dtype", ["half", "float"])
62
+def test_llava_next_image_processor(hf_images, dtype):
63
+ MODEL_NAME = "llava-hf/llava-v1.6-34b-hf"
64
+ IMAGE_HEIGHT = IMAGE_WIDTH = 560
65
+
66
+ hf_processor = LlavaNextImageProcessor.from_pretrained(MODEL_NAME)
67
+ assert isinstance(hf_processor, LlavaNextImageProcessor)
68
+
69
+ model_config = ModelConfig(
70
+ model=MODEL_NAME,
71
+ tokenizer=MODEL_NAME,
72
+ tokenizer_mode="auto",
73
+ trust_remote_code=False,
74
+ seed=0,
75
+ dtype=dtype,
76
+ revision=None,
77
+ )
78
+ vlm_config = VisionLanguageConfig(
79
+ image_input_type=VisionLanguageConfig.ImageInputType.PIXEL_VALUES,
80
+ image_token_id=64000,
81
+ image_input_shape=(1, 3, IMAGE_HEIGHT, IMAGE_WIDTH),
82
+ image_feature_size=2928,
83
+ image_processor=MODEL_NAME,
84
+ image_processor_revision=None,
85
+ )
86
+
87
+ for image in hf_images:
88
+ hf_result = hf_processor.preprocess(
89
+ image,
90
+ return_tensors="pt",
91
+ ).to(dtype=_STR_DTYPE_TO_TORCH_DTYPE[dtype])
92
+ vllm_result = MULTIMODAL_REGISTRY.process_input(
93
+ ImagePixelData(image),
94
+ model_config=model_config,
95
+ vlm_config=vlm_config,
96
+ )
97
+
98
+ assert hf_result.keys() == vllm_result.keys()
99
+ for key, hf_tensor in hf_result.items():
100
+ hf_arr: np.ndarray = hf_tensor.numpy()
101
+ vllm_arr: np.ndarray = vllm_result[key].numpy()
102
+
103
+ assert hf_arr.shape == vllm_arr.shape, f"Failed for key={key}"
104
+ assert np.allclose(hf_arr, vllm_arr), f"Failed for key={key}"
105
+
106
+
107
+@pytest.mark.xfail(
108
+ reason="Example image pixels were not processed using HuggingFace")
109
@pytest.mark.parametrize("dtype", ["float"])
110
def test_image_pixel_types(hf_images, vllm_image_tensors, dtype):
111
MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
61
- IMAGE_HEIGHT = IMAGE_WIDTH = 33
112
+ IMAGE_HEIGHT = IMAGE_WIDTH = 560
113
114
model_config = ModelConfig(
115
model=MODEL_NAME,
@@ -95,7 +146,4 @@ def test_image_pixel_types(hf_images, vllm_image_tensors, dtype):
146
tensor_arr: np.ndarray = tensor_result[key].numpy()
147
148
assert image_arr.shape == tensor_arr.shape, f"Failed for key={key}"
98
-
99
- # The examples in PR#3042 have slightly different preprocessing from
100
- # HuggingFace's LlavaProcessor, causing the test to fail.
101
- # assert np.allclose(image_arr, tensor_arr), f"Failed for key={key}"
149
+ assert np.allclose(image_arr, tensor_arr), f"Failed for key={key}"
vllm/model_executor/models/__init__.py
+2
@@ -33,6 +33,8 @@ _GENERATION_MODELS = {
33
"LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
34
"LlavaForConditionalGeneration":
35
("llava", "LlavaForConditionalGeneration"),
36
+ "LlavaNextForConditionalGeneration":
37
+ ("llava_next", "LlavaNextForConditionalGeneration"),
38
# For decapoda-research/llama-*
39
"LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
40
"MistralForCausalLM": ("llama", "LlamaForCausalLM"),
vllm/model_executor/models/llava.py
+10
-8
@@ -1,7 +1,7 @@
1
from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
2
3
import torch
4
-from torch import nn
4
+import torch.nn as nn
5
# TODO(xwjiang): We should port CLIPVisionModel's code over to not depend on
6
# transformers' impl.
7
from transformers import CLIPVisionModel, LlavaConfig
@@ -51,10 +51,10 @@ class LlavaMultiModalProjector(nn.Module):
51
return hidden_states
52
53
54
-def _merge_vision_embeddings(input_ids: torch.Tensor,
55
- inputs_embeds: torch.Tensor,
56
- vision_embeddings: torch.Tensor,
57
- image_token_id: int) -> torch.Tensor:
54
+def merge_vision_embeddings(input_ids: torch.Tensor,
55
+ inputs_embeds: torch.Tensor,
56
+ vision_embeddings: torch.Tensor,
57
+ image_token_id: int) -> torch.Tensor:
58
"""In place merges in vision_embeddings with inputs_embeds."""
59
mask = (input_ids == image_token_id)
60
@@ -151,7 +151,8 @@ class LlavaForConditionalGeneration(VisionLanguageModelBase):
151
return None
152
153
if not isinstance(pixel_values, torch.Tensor):
154
- raise ValueError("Incorrect type of pixel values")
154
+ raise ValueError("Incorrect type of pixel values. "
155
+ f"Got type: {type(pixel_values)}")
156
157
return LlavaImagePixelInputs(
158
type="pixel_values",
@@ -166,7 +167,8 @@ class LlavaForConditionalGeneration(VisionLanguageModelBase):
167
return None
168
169
if not isinstance(image_features, torch.Tensor):
169
- raise ValueError("Incorrect type of image features")
170
+ raise ValueError("Incorrect type of image features. "
171
+ f"Got type: {type(image_features)}")
172
173
return LlavaImageFeatureInputs(
174
type="image_features",
@@ -268,7 +270,7 @@ class LlavaForConditionalGeneration(VisionLanguageModelBase):
270
vision_embeddings = self._process_image_input(image_input)
271
inputs_embeds = self.language_model.get_input_embeddings(input_ids)
272
271
- inputs_embeds = _merge_vision_embeddings(
273
+ inputs_embeds = merge_vision_embeddings(
274
input_ids, inputs_embeds, vision_embeddings,
275
self.vision_language_config.image_token_id)
276
vllm/model_executor/models/llava_next.py
new
+445
@@ -0,0 +1,445 @@
1
+from typing import (Dict, Iterable, List, Literal, Optional, Tuple, TypedDict,
2
+ Union)
3
+
4
+import torch
5
+import torch.nn as nn
6
+from PIL import Image
7
+# TODO(xwjiang): We should port CLIPVisionModel's code over to not depend on
8
+# transformers' impl.
9
+from transformers import CLIPVisionModel, LlavaNextConfig
10
+from transformers.models.llava_next.modeling_llava_next import (
11
+ get_anyres_image_grid_shape, unpad_image)
12
+from typing_extensions import NotRequired
13
+
14
+from vllm.attention import AttentionMetadata
15
+from vllm.config import CacheConfig, ModelConfig, VisionLanguageConfig
16
+from vllm.logger import init_logger
17
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
18
+from vllm.model_executor.layers.quantization.base_config import (
19
+ QuantizationConfig)
20
+from vllm.model_executor.layers.sampler import Sampler
21
+from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
22
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
23
+from vllm.model_executor.models.llama import LlamaModel
24
+from vllm.model_executor.sampling_metadata import SamplingMetadata
25
+from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalData
26
+from vllm.multimodal.image import ImagePixelData, get_dummy_image_data
27
+from vllm.sequence import SamplerOutput, SequenceData
28
+
29
+from .llava import LlavaMultiModalProjector, merge_vision_embeddings
30
+from .vlm_base import VisionLanguageModelBase
31
+
32
+logger = init_logger(__name__)
33
+
34
+_KEYS_TO_MODIFY_MAPPING = {
35
+ "language_model.lm_head": "lm_head",
36
+ "language_model.model": "language_model",
37
+}
38
+
39
+
40
+class LlavaNextImagePixelInputs(TypedDict):
41
+ type: Literal["pixel_values"]
42
+ data: torch.Tensor
43
+ """Shape: (batch_size, 1 + num_patches, num_channels, height, width)"""
44
+
45
+ image_sizes: NotRequired[torch.Tensor]
46
+ """Shape: (batch_size, 2)"""
47
+
48
+
49
+class LlavaNextImageFeatureInputs(TypedDict):
50
+ type: Literal["image_features"]
51
+ data: torch.Tensor
52
+ """Shape: (batch_size, 1 + num_patches, image_feature_size, hidden_size)"""
53
+
54
+ image_sizes: NotRequired[torch.Tensor]
55
+ """Shape: (batch_size, 2)"""
56
+
57
+
58
+LlavaNextImageInputs = Union[LlavaNextImagePixelInputs,
59
+ LlavaNextImageFeatureInputs]
60
+
61
+
62
+def _get_dummy_image_data(
63
+ seq_len: int,
64
+ model_config: ModelConfig,
65
+ vlm_config: VisionLanguageConfig,
66
+) -> Tuple[SequenceData, MultiModalData]:
67
+ seq_data, fake_mm_data = get_dummy_image_data(seq_len, model_config,
68
+ vlm_config)
69
+
70
+ config_input_type = vlm_config.image_input_type
71
+ ImageInputType = VisionLanguageConfig.ImageInputType
72
+
73
+ if config_input_type == ImageInputType.PIXEL_VALUES:
74
+ _, c, h, w = vlm_config.image_input_shape
75
+ mode = {1: "L", 3: "RGB"}[c]
76
+ fake_mm_data = ImagePixelData(Image.new(mode, (w, h), color=0))
77
+
78
+ return seq_data, fake_mm_data
79
+
80
+
81
+def _image_pixel_processor(
82
+ data: ImagePixelData,
83
+ model_config: ModelConfig,
84
+ vlm_config: VisionLanguageConfig,
85
+) -> Dict[str, torch.Tensor]:
86
+ image = data.image
87
+
88
+ if isinstance(image, torch.Tensor):
89
+ pixel_values = image.to(model_config.dtype)
90
+ batch_size, _, _, h, w = pixel_values.shape
91
+ image_sizes = torch.tensor([(w, h) for _ in range(batch_size)])
92
+
93
+ return {"pixel_values": pixel_values, "image_sizes": image_sizes}
94
+
95
+ # Temporary patch before dynamic number of image tokens is supported
96
+ _, _, h, w = vlm_config.image_input_shape
97
+ if (w, h) != (image.width, image.height):
98
+ logger.warning(
99
+ "Dynamic image shape is currently not supported. "
100
+ "Resizing input image to (%d, %d).", w, h)
101
+
102
+ data.image = image.resize((w, h))
103
+
104
+ return MULTIMODAL_REGISTRY._get_plugin_for_data_type(ImagePixelData) \
105
+ ._default_input_processor(data, model_config, vlm_config)
106
+
107
+
108
+@MULTIMODAL_REGISTRY.register_image_pixel_input(_image_pixel_processor)
109
+@MULTIMODAL_REGISTRY.register_dummy_data(_get_dummy_image_data)
110
+class LlavaNextForConditionalGeneration(VisionLanguageModelBase):
111
+ """
112
+ Args to `forward()`:
113
+ input_ids: Flattened (concatenated) input_ids corresponding to a
114
+ batch.
115
+ pixel_values: For PIXEL_VALUES, expects a batch with shape
116
+ [1, num_patches, 3, 336, 336].
117
+ image_features: For IMAGE_FEATURES, expects a batch with shape
118
+ [1, num_patches, 1176, 1024].
119
+ """
120
+
121
+ def __init__(self,
122
+ config: LlavaNextConfig,
123
+ vision_language_config: VisionLanguageConfig,
124
+ cache_config: Optional[CacheConfig] = None,
125
+ quant_config: Optional[QuantizationConfig] = None) -> None:
126
+ super().__init__(vision_language_config)
127
+
128
+ # Update the type annotation from that of its superclass
129
+ self.config = config
130
+
131
+ if self.vision_language_config.image_input_type == (
132
+ VisionLanguageConfig.ImageInputType.PIXEL_VALUES):
133
+ self.vision_tower = CLIPVisionModel(config.vision_config)
134
+ else:
135
+ raise TypeError("Image features are not supported by LLaVA-NeXT")
136
+
137
+ self.multi_modal_projector = LlavaMultiModalProjector(
138
+ vision_hidden_size=config.vision_config.hidden_size,
139
+ text_hidden_size=config.text_config.hidden_size,
140
+ projector_hidden_act=config.projector_hidden_act)
141
+
142
+ self.quant_config = quant_config
143
+ self.language_model = LlamaModel(config.text_config, cache_config,
144
+ quant_config)
145
+ self.unpadded_vocab_size = config.text_config.vocab_size
146
+ self.lm_head = ParallelLMHead(
147
+ self.unpadded_vocab_size,
148
+ config.text_config.hidden_size,
149
+ org_num_embeddings=self.language_model.org_vocab_size)
150
+ logit_scale = getattr(config, "logit_scale", 1.0)
151
+ self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
152
+ config.vocab_size, logit_scale)
153
+ self.sampler = Sampler()
154
+
155
+ self.image_newline = nn.Parameter(
156
+ torch.empty(config.text_config.hidden_size))
157
+
158
+ def _validate_image_pixels(self, data: torch.Tensor) -> torch.Tensor:
159
+ _, num_channels, _, _ = self.vision_language_config.image_input_shape
160
+
161
+ # Note that this is different from that of vLLM vision_language_config
162
+ # since the image is resized by the HuggingFace preprocessor
163
+ height = width = self.config.vision_config.image_size
164
+
165
+ if list(data.shape[2:]) != [num_channels, height, width]:
166
+ raise ValueError(
167
+ f"The expected image tensor shape is batch dimension plus "
168
+ f"num_patches plus {[num_channels, height, width]}. "
169
+ f"You supplied {data.shape}. "
170
+ f"If you are using vLLM's entrypoint, make sure your "
171
+ f"supplied image input is consistent with "
172
+ f"image_input_shape in engine args.")
173
+
174
+ return data
175
+
176
+ def _validate_image_sizes(self, data: torch.Tensor) -> torch.Tensor:
177
+ if list(data.shape[1:]) != [2]:
178
+ raise ValueError(
179
+ f"The expected image sizes shape is batch dimension plus "
180
+ f"{[2]}. You supplied {data.shape}.")
181
+
182
+ return data
183
+
184
+ def _parse_and_validate_image_input(
185
+ self, **kwargs: object) -> Optional[LlavaNextImageInputs]:
186
+ pixel_values = kwargs.pop("pixel_values", None)
187
+ image_sizes = kwargs.pop("image_sizes", None)
188
+ image_features = kwargs.pop("image_features", None)
189
+
190
+ expected_input_type = self.vision_language_config.image_input_type
191
+ ImageInputType = VisionLanguageConfig.ImageInputType
192
+
193
+ if expected_input_type == ImageInputType.PIXEL_VALUES:
194
+ if image_features is not None:
195
+ raise ValueError(
196
+ "Expected pixel values but got image features")
197
+ if pixel_values is None:
198
+ return None
199
+
200
+ if not isinstance(pixel_values, torch.Tensor):
201
+ raise ValueError("Incorrect type of pixel values. "
202
+ f"Got type: {type(pixel_values)}")
203
+
204
+ if not isinstance(image_sizes, torch.Tensor):
205
+ raise ValueError("Incorrect type of image sizes. "
206
+ f"Got type: {type(image_sizes)}")
207
+
208
+ return LlavaNextImagePixelInputs(
209
+ type="pixel_values",
210
+ data=self._validate_image_pixels(pixel_values),
211
+ image_sizes=self._validate_image_sizes(image_sizes),
212
+ )
213
+
214
+ assert expected_input_type != ImageInputType.IMAGE_FEATURES, (
215
+ "Failed to validate this at initialization time")
216
+
217
+ return None
218
+
219
+ def _merge_image_patch_embeddings(self, image_size: torch.Tensor,
220
+ patch_embeddings: torch.Tensor, *,
221
+ strategy: str) -> torch.Tensor:
222
+ # Based on: https://github.com/haotian-liu/LLaVA/blob/main/llava/model/llava_arch.py
223
+ if strategy == "flat":
224
+ return patch_embeddings.flatten(0, 1)
225
+
226
+ if strategy.startswith("spatial"):
227
+ orig_width, orig_height = image_size
228
+ height = width = self.config.vision_config.image_size \
229
+ // self.config.vision_config.patch_size
230
+
231
+ base_patch_embeds = patch_embeddings[0]
232
+ if height * width != base_patch_embeds.shape[0]:
233
+ raise ValueError(
234
+ "The number of patches is not consistent with the "
235
+ "image size.")
236
+
237
+ if patch_embeddings.shape[0] > 1:
238
+ other_patch_embeds = patch_embeddings[1:]
239
+
240
+ # image_aspect_ratio == "anyres"
241
+ num_patch_width, num_patch_height = get_anyres_image_grid_shape(
242
+ (orig_width, orig_height),
243
+ self.config.image_grid_pinpoints,
244
+ self.config.vision_config.image_size,
245
+ )
246
+ other_patch_embeds = other_patch_embeds \
247
+ .view(num_patch_width, num_patch_height, height, width, -1)
248
+
249
+ if "unpad" in strategy:
250
+ other_patch_embeds = other_patch_embeds \
251
+ .permute(4, 0, 2, 1, 3).contiguous() \
252
+ .flatten(1, 2).flatten(2, 3)
253
+ other_patch_embeds = unpad_image(other_patch_embeds,
254
+ image_size)
255
+ other_patch_embeds = torch.cat((
256
+ other_patch_embeds,
257
+ self.image_newline[:, None, None] \
258
+ .expand(*other_patch_embeds.shape[:-1], 1) \
259
+ .to(other_patch_embeds.device),
260
+ ), dim=-1)
261
+ other_patch_embeds = other_patch_embeds \
262
+ .flatten(1, 2).transpose(0, 1)
263
+ else:
264
+ other_patch_embeds = other_patch_embeds \
265
+ .permute(0, 2, 1, 3, 4).contiguous() \
266
+ .flatten(0, 3)
267
+
268
+ merged_patch_embeddings = torch.cat(
269
+ (base_patch_embeds, other_patch_embeds), dim=0)
270
+ else:
271
+ if "unpad" in strategy:
272
+ merged_patch_embeddings = torch.cat(
273
+ (base_patch_embeds,
274
+ self.image_newline[None] \
275
+ .to(base_patch_embeds.device)
276
+ ), dim=0)
277
+ else:
278
+ merged_patch_embeddings = base_patch_embeds
279
+
280
+ return merged_patch_embeddings
281
+
282
+ raise ValueError(f"Unexpected patch merge strategy: {strategy}")
283
+
284
+ def _process_image_pixels(
285
+ self, inputs: LlavaNextImagePixelInputs) -> torch.Tensor:
286
+ assert self.vision_tower is not None
287
+
288
+ pixel_values = inputs["data"]
289
+
290
+ b, num_patches, c, h, w = pixel_values.shape
291
+ stacked_pixel_values = pixel_values.view(b * num_patches, c, h, w)
292
+
293
+ stacked_image_features = self._image_pixels_to_features(
294
+ self.vision_tower, stacked_pixel_values)
295
+
296
+ return stacked_image_features.view(b, num_patches,
297
+ *stacked_image_features.shape[-2:])
298
+
299
+ def _process_image_input(
300
+ self, image_input: LlavaNextImageInputs) -> torch.Tensor:
301
+ if image_input["type"] == "pixel_values":
302
+ assert self.vision_tower is not None
303
+ image_features = self._process_image_pixels(image_input)
304
+ else:
305
+ image_features = image_input["data"]
306
+
307
+ patch_embeddings = self.multi_modal_projector(image_features)
308
+
309
+ image_sizes = image_input.get("image_sizes")
310
+ if image_sizes is None:
311
+ batch_size = image_input["data"].shape[0]
312
+ vision_config = self.config.vision_config
313
+ default_width = default_height = vision_config.image_size
314
+ image_sizes = torch.as_tensor([[default_width, default_height]
315
+ for _ in range(batch_size)])
316
+
317
+ merged_patch_embeddings = [
318
+ self._merge_image_patch_embeddings(image_sizes[i],
319
+ patch_features,
320
+ strategy="spatial_unpad")
321
+ for i, patch_features in enumerate(patch_embeddings)
322
+ ]
323
+
324
+ return torch.stack(merged_patch_embeddings, dim=0)
325
+
326
+ def forward(
327
+ self,
328
+ input_ids: torch.Tensor,
329
+ positions: torch.Tensor,
330
+ kv_caches: List[torch.Tensor],
331
+ attn_metadata: AttentionMetadata,
332
+ **kwargs: object,
333
+ ) -> SamplerOutput:
334
+ """Run forward pass for Llava 1.5.
335
+
336
+ One key thing to understand is the `input_ids` already accounts for the
337
+ positions of the to-be-inserted image embeddings.
338
+ Concretely, consider a text prompt:
339
+ "<image>\nUSER: What's the content of the image?\nASSISTANT:".
340
+ Tokenizer outputs:
341
+ [1, 32000, 29871, 13, 11889, 29901, 1724, 29915, 29879, 278,
342
+ 2793, 310, 278, 1967, 29973, 13, 22933, 9047, 13566, 29901].
343
+ The to-be-inserted image has a size of 576 (24 * 24) along the context
344
+ length dimension.
345
+ `input_ids` is thus [1, 32000, ..., 32000, 29871, 13, 11889, 29901,
346
+ 1724, 29915, 29879, 278, 2793, 310, 278, 1967, 29973, 13, 22933,
347
+ 9047, 13566, 29901].
348
+ There will be 576 `32000` in the `input_ids`.
349
+ (32000 is the token id for `<image>`.)
350
+
351
+ This way, the `positions` and `attn_metadata` are consistent
352
+ with the `input_ids`.
353
+
354
+ The model takes two types of image inputs:
355
+ PIXEL_VALUES and IMAGE_FEATURES.
356
+ The following shows how each maps to huggingface implementation.
357
+ PIXEL_VALUES:
358
+ - https://github.com/huggingface/transformers/blob/07bdbeb/src/transformers/models/llava/modeling_llava.py#L353
359
+ IMAGE_FEATURES:
360
+ - https://github.com/huggingface/transformers/blob/07bdbeb/src/transformers/models/llava/modeling_llava.py#L430
361
+ before going through the multi modal projector.
362
+
363
+ Args:
364
+ input_ids: Flattened (concatenated) input_ids corresponding to a
365
+ batch.
366
+ pixel_values: For PIXEL_VALUES, expects a batch with shape
367
+ [1, 3, 336, 336].
368
+ image_features: For IMAGE_FEATURES, expects a batch with shape
369
+ [1, 576, 1024].
370
+ """
371
+ image_input = self._parse_and_validate_image_input(**kwargs)
372
+
373
+ if image_input is not None:
374
+ vision_embeddings = self._process_image_input(image_input)
375
+ inputs_embeds = self.language_model.get_input_embeddings(input_ids)
376
+
377
+ inputs_embeds = merge_vision_embeddings(
378
+ input_ids, inputs_embeds, vision_embeddings,
379
+ self.vision_language_config.image_token_id)
380
+
381
+ input_ids = None
382
+ else:
383
+ inputs_embeds = None
384
+
385
+ hidden_states = self.language_model(input_ids,
386
+ positions,
387
+ kv_caches,
388
+ attn_metadata,
389
+ inputs_embeds=inputs_embeds)
390
+
391
+ return hidden_states
392
+
393
+ def compute_logits(self, hidden_states: torch.Tensor,
394
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
395
+ logits = self.logits_processor(self.lm_head.weight, hidden_states,
396
+ sampling_metadata)
397
+ return logits
398
+
399
+ def sample(
400
+ self,
401
+ logits: torch.Tensor,
402
+ sampling_metadata: SamplingMetadata,
403
+ ) -> Optional[SamplerOutput]:
404
+ next_tokens = self.sampler(logits, sampling_metadata)
405
+ return next_tokens
406
+
407
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
408
+ # only doing this for language model part for now.
409
+ stacked_params_mapping = [
410
+ # (param_name, shard_name, shard_id)
411
+ ("qkv_proj", "q_proj", "q"),
412
+ ("qkv_proj", "k_proj", "k"),
413
+ ("qkv_proj", "v_proj", "v"),
414
+ ("gate_up_proj", "gate_proj", 0),
415
+ ("gate_up_proj", "up_proj", 1),
416
+ ]
417
+ params_dict = dict(self.named_parameters())
418
+ for name, loaded_weight in weights:
419
+ if "rotary_emb.inv_freq" in name:
420
+ continue
421
+ for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
422
+ if key_to_modify in name:
423
+ name = name.replace(key_to_modify, new_key)
424
+ use_default_weight_loading = False
425
+ if "vision" in name:
426
+ if self.vision_tower is not None:
427
+ # We only do sharding for language model and
428
+ # not vision model for now.
429
+ use_default_weight_loading = True
430
+ else:
431
+ for (param_name, weight_name,
432
+ shard_id) in stacked_params_mapping:
433
+ if weight_name not in name:
434
+ continue
435
+ param = params_dict[name.replace(weight_name, param_name)]
436
+ weight_loader = param.weight_loader
437
+ weight_loader(param, loaded_weight, shard_id)
438
+ break
439
+ else:
440
+ use_default_weight_loading = True
441
+ if use_default_weight_loading:
442
+ param = params_dict[name]
443
+ weight_loader = getattr(param, "weight_loader",
444
+ default_weight_loader)
445
+ weight_loader(param, loaded_weight)