A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Model] Support Pixtral models in the HF Transformers format (#9036)
Michael Goin committed
Oct 18, 2024 at 15:29 UTC
3921a2f29e30df293459d824e20d2e546e4af0c7
7 files changed
+503
-12
docs/source/models/supported_models.rst
+1
-1
@@ -437,7 +437,7 @@ Text Generation
437
* - :code:`PixtralForConditionalGeneration`
438
- Pixtral
439
- T + I\ :sup:`+`
440
- - :code:`mistralai/Pixtral-12B-2409`
440
+ - :code:`mistralai/Pixtral-12B-2409`, :code:`mistral-community/pixtral-12b` etc.
441
-
442
- ✅︎
443
* - :code:`QWenLMHeadModel`
examples/offline_inference_vision_language.py
+17
@@ -277,6 +277,22 @@ def run_qwen2_vl(question: str, modality: str):
277
return llm, prompt, stop_token_ids
278
279
280
+# Pixtral HF-format
281
+def run_pixtral_hf(question: str, modality: str):
282
+ assert modality == "image"
283
+
284
+ model_name = "mistral-community/pixtral-12b"
285
+
286
+ llm = LLM(
287
+ model=model_name,
288
+ max_model_len=8192,
289
+ )
290
+
291
+ prompt = f"<s>[INST]{question}\n[IMG][/INST]"
292
+ stop_token_ids = None
293
+ return llm, prompt, stop_token_ids
294
+
295
+
296
# LLama 3.2
297
def run_mllama(question: str, modality: str):
298
assert modality == "image"
@@ -347,6 +363,7 @@ model_example_map = {
363
"NVLM_D": run_nvlm_d,
364
"qwen_vl": run_qwen_vl,
365
"qwen2_vl": run_qwen2_vl,
366
+ "pixtral_hf": run_pixtral_hf,
367
"mllama": run_mllama,
368
"molmo": run_molmo,
369
"glm4v": run_glm4v,
vllm/model_executor/layers/activation.py
+2
@@ -264,6 +264,8 @@ _ACTIVATION_REGISTRY = LazyDict({
264
lambda: nn.ReLU(),
265
"relu2":
266
lambda: ReLUSquaredActivation(),
267
+ "silu":
268
+ lambda: nn.SiLU(),
269
"quick_gelu":
270
lambda: QuickGELU(),
271
})
vllm/model_executor/models/llava.py
+70
-4
@@ -5,7 +5,8 @@ from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
5
import torch
6
import torch.nn as nn
7
from PIL import Image
8
-from transformers import CLIPVisionConfig, LlavaConfig, SiglipVisionConfig
8
+from transformers import (CLIPVisionConfig, LlavaConfig, PixtralVisionConfig,
9
+ SiglipVisionConfig)
10
11
from vllm.attention import AttentionMetadata
12
from vllm.config import CacheConfig, MultiModalConfig
@@ -22,6 +23,10 @@ from .clip import (CLIPVisionModel, dummy_image_for_clip,
23
dummy_seq_data_for_clip, get_max_clip_image_tokens,
24
input_processor_for_clip)
25
from .interfaces import SupportsMultiModal, SupportsPP
26
+from .pixtral import (PixtralHFVisionModel, dummy_image_for_pixtral_hf,
27
+ dummy_seq_data_for_pixtral_hf,
28
+ get_max_pixtral_hf_image_tokens,
29
+ input_processor_for_pixtral_hf)
30
from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
31
dummy_seq_data_for_siglip, get_max_siglip_image_tokens,
32
input_processor_for_siglip)
@@ -31,8 +36,13 @@ from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
36
37
class LlavaImagePixelInputs(TypedDict):
38
type: Literal["pixel_values"]
34
- data: torch.Tensor
35
- """Shape: `(batch_size * num_images, num_channels, height, width)`"""
39
+ data: Union[torch.Tensor, List[torch.Tensor]]
40
+ """
41
+ Shape: `(batch_size * num_images, num_channels, height, width)`
42
+
43
+ Note that `height` or `width` may be different per batch and image,
44
+ in which case the data is passed as a list instead of a batched tensor.
45
+ """
46
47
48
class LlavaImageEmbeddingInputs(TypedDict):
@@ -77,6 +87,8 @@ def get_max_llava_image_tokens(ctx: InputContext):
87
num_image_tokens = get_max_clip_image_tokens(vision_config)
88
elif isinstance(vision_config, SiglipVisionConfig):
89
num_image_tokens = get_max_siglip_image_tokens(vision_config)
90
+ elif isinstance(vision_config, PixtralVisionConfig):
91
+ num_image_tokens = get_max_pixtral_hf_image_tokens(vision_config)
92
else:
93
msg = f"Unsupported vision config: {type(vision_config)}"
94
raise NotImplementedError(msg)
@@ -120,6 +132,17 @@ def dummy_data_for_llava(ctx: InputContext, seq_len: int,
132
133
mm_data = dummy_image_for_siglip(vision_config, num_images)
134
return seq_data, mm_data
135
+ elif isinstance(vision_config, PixtralVisionConfig):
136
+ seq_data = dummy_seq_data_for_pixtral_hf(
137
+ vision_config,
138
+ seq_len,
139
+ num_images,
140
+ image_token_id=hf_config.image_token_index,
141
+ image_feature_size_override=image_feature_size,
142
+ )
143
+
144
+ mm_data = dummy_image_for_pixtral_hf(vision_config, num_images)
145
+ return seq_data, mm_data
146
147
msg = f"Unsupported vision config: {type(vision_config)}"
148
raise NotImplementedError(msg)
@@ -163,6 +186,15 @@ def input_processor_for_llava(ctx: InputContext, inputs: DecoderOnlyInputs):
186
image_token_id=hf_config.image_token_index,
187
image_feature_size_override=image_feature_size,
188
)
189
+ elif isinstance(vision_config, PixtralVisionConfig):
190
+ # We ignore image_feature_size_override since we have non-uniform
191
+ # image sizes for Pixtral
192
+ return input_processor_for_pixtral_hf(
193
+ model_config,
194
+ vision_config,
195
+ inputs,
196
+ image_token_id=hf_config.image_token_index,
197
+ )
198
199
msg = f"Unsupported vision config: {type(vision_config)}"
200
raise NotImplementedError(msg)
@@ -189,6 +221,9 @@ def _init_vision_tower(hf_config: LlavaConfig):
221
vision_config,
222
num_hidden_layers_override=num_hidden_layers,
223
)
224
+ elif isinstance(vision_config, PixtralVisionConfig):
225
+ # TODO: allow layer override?
226
+ return PixtralHFVisionModel(vision_config)
227
228
msg = f"Unsupported vision config: {type(vision_config)}"
229
raise NotImplementedError(msg)
@@ -210,6 +245,15 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
245
self.config = config
246
self.multimodal_config = multimodal_config
247
248
+ # NOTE: These are special cases for Pixtral-12B in the HF-format
249
+ # https://huggingface.co/mistral-community/pixtral-12b/blob/main/config.json # noqa
250
+ if (config.text_config.architectures is None
251
+ and config.text_config.model_type == "mistral"):
252
+ config.text_config.architectures = ["MistralForCausalLM"]
253
+ if (config.projector_hidden_act is None
254
+ and config.vision_config.hidden_act == "gelu"):
255
+ config.projector_hidden_act = "gelu"
256
+
257
# TODO: Optionally initializes this for supporting embeddings.
258
self.vision_tower = _init_vision_tower(config)
259
self.multi_modal_projector = LlavaMultiModalProjector(
@@ -246,6 +290,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
290
def _parse_and_validate_image_input(
291
self, **kwargs: object) -> Optional[LlavaImageInputs]:
292
pixel_values = kwargs.pop("pixel_values", None)
293
+ image_sizes = kwargs.pop("image_sizes", None)
294
image_embeds = kwargs.pop("image_embeds", None)
295
296
if pixel_values is None and image_embeds is None:
@@ -256,6 +301,26 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
301
raise ValueError("Incorrect type of pixel values. "
302
f"Got type: {type(pixel_values)}")
303
304
+ # Case for models like PixtralHF that have dynamic image sizes
305
+ # so we need to produce a list of tensors
306
+ if image_sizes is not None:
307
+ images = pixel_values
308
+ if isinstance(images, torch.Tensor):
309
+ # if passed as batch take all images
310
+ NN, N, B, C, W, H = images.shape
311
+ images = images.reshape(NN * N * B, C, W, H)
312
+ images = [images[i] for i in range(images.size(0))]
313
+ elif isinstance(images, list):
314
+ # if passed as list flatten lists of tensors
315
+ while isinstance(images, list) and len(images) == 1:
316
+ images = images[0]
317
+
318
+ # TODO: Add validation based on image_sizes
319
+ return LlavaImagePixelInputs(
320
+ type="pixel_values",
321
+ data=images,
322
+ )
323
+
324
return LlavaImagePixelInputs(
325
type="pixel_values",
326
data=self._validate_pixel_values(
@@ -286,7 +351,8 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
351
352
def _image_pixels_to_features(
353
self,
289
- vision_tower: Union[CLIPVisionModel, SiglipVisionModel],
354
+ vision_tower: Union[CLIPVisionModel, SiglipVisionModel,
355
+ PixtralHFVisionModel],
356
pixel_values: torch.Tensor,
357
) -> torch.Tensor:
358
vllm/model_executor/models/pixtral.py
+407
-3
@@ -3,18 +3,26 @@ from functools import cached_property
3
from itertools import tee
4
from typing import Iterable, List, Mapping, Optional, Tuple, Union
5
6
+import numpy
7
import torch
8
import torch.nn as nn
9
import torch.nn.functional as F
10
from mistral_common.protocol.instruct.messages import ImageChunk
11
from PIL import Image
11
-from transformers import PretrainedConfig
12
+from transformers import PixtralVisionConfig, PretrainedConfig
13
+from transformers.models.pixtral.image_processing_pixtral import (
14
+ _num_image_tokens)
15
+from transformers.models.pixtral.modeling_pixtral import (
16
+ PixtralRotaryEmbedding, apply_rotary_pos_emb,
17
+ generate_block_attention_mask, position_ids_in_meshgrid)
18
from xformers.ops.fmha import memory_efficient_attention
19
from xformers.ops.fmha.attn_bias import BlockDiagonalMask
20
21
from vllm.attention import AttentionMetadata
16
-from vllm.config import CacheConfig, MultiModalConfig
17
-from vllm.inputs import INPUT_REGISTRY, DecoderOnlyInputs, InputContext
22
+from vllm.config import CacheConfig, ModelConfig, MultiModalConfig
23
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
24
+ token_inputs)
25
+from vllm.model_executor.layers.activation import get_act_fn
26
from vllm.model_executor.layers.layernorm import RMSNorm
27
from vllm.model_executor.layers.quantization import QuantizationConfig
28
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
@@ -25,6 +33,8 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
33
from vllm.multimodal.base import MultiModalInputs
34
from vllm.multimodal.utils import cached_get_tokenizer
35
from vllm.sequence import IntermediateTensors, SequenceData
36
+from vllm.transformers_utils.processor import cached_get_processor
37
+from vllm.utils import is_list_of
38
39
from .interfaces import SupportsMultiModal, SupportsPP
40
from .utils import init_vllm_registered_model
@@ -576,3 +586,397 @@ class VisionLanguageAdapter(nn.Module):
586
587
def forward(self, x: torch.Tensor) -> torch.Tensor:
588
return self.w_out(self.gelu(self.w_in(x)))
589
+
590
+
591
+#### HF Transformers version of Pixtral ####
592
+# Based off https://github.com/huggingface/transformers/blob/d7950bff82b18c823193d17d72188c5e46d06c83/src/transformers/models/pixtral/modeling_pixtral.py
593
+# This model follows the Llava family, meaning image embeddings are placed
594
+# instead of the `[IMG]` token placeholders.
595
+# The model uses [`PixtralVisionModel`] for its vision encoder,
596
+# and [`MistralForCausalLM`] for its language decoder.
597
+
598
+
599
+def get_pixtral_hf_patch_grid_length(*, image_size: int,
600
+ patch_size: int) -> int:
601
+ # Since interpolation is applied, the image size need not be divisible
602
+ # assert image_size % patch_size == 0
603
+ return image_size // patch_size
604
+
605
+
606
+def get_pixtral_hf_num_patches(*, image_size: int, patch_size: int) -> int:
607
+ grid_length = get_pixtral_hf_patch_grid_length(image_size=image_size,
608
+ patch_size=patch_size)
609
+ return grid_length * grid_length
610
+
611
+
612
+def get_max_pixtral_hf_image_feature_size(
613
+ hf_config: PixtralVisionConfig) -> int:
614
+ return get_pixtral_hf_num_patches(image_size=hf_config.image_size,
615
+ patch_size=hf_config.patch_size)
616
+
617
+
618
+def get_max_pixtral_hf_image_tokens(hf_config: PixtralVisionConfig) -> int:
619
+ return get_max_pixtral_hf_image_feature_size(hf_config)
620
+
621
+
622
+def dummy_seq_data_for_pixtral_hf(
623
+ hf_config: PixtralVisionConfig,
624
+ seq_len: int,
625
+ num_images: int,
626
+ *,
627
+ image_token_id: int,
628
+ image_feature_size_override: Optional[int] = None,
629
+):
630
+ if image_feature_size_override is None:
631
+ image_feature_size = get_max_pixtral_hf_image_feature_size(hf_config)
632
+ else:
633
+ image_feature_size = image_feature_size_override
634
+
635
+ return SequenceData.from_prompt_token_counts(
636
+ (image_token_id, image_feature_size * num_images),
637
+ (0, seq_len - image_feature_size * num_images),
638
+ )
639
+
640
+
641
+def dummy_image_for_pixtral_hf(
642
+ hf_config: PixtralVisionConfig,
643
+ num_images: int,
644
+ *,
645
+ image_width_override: Optional[int] = None,
646
+ image_height_override: Optional[int] = None,
647
+):
648
+ width = height = hf_config.image_size
649
+ if image_width_override is not None:
650
+ width = image_width_override
651
+ if image_height_override is not None:
652
+ height = image_height_override
653
+
654
+ image = Image.new("RGB", (width, height), color=0)
655
+ return {"image": image if num_images == 1 else [image] * num_images}
656
+
657
+
658
+def get_pixtral_hf_image_feature_size(hf_config: PixtralVisionConfig,
659
+ image_width: int,
660
+ image_height: int) -> Tuple[int, int]:
661
+ # Adapted from transformers.models.pixtral.image_processing_pixtral.get_resize_output_image_size # noqa: E501
662
+ # https://github.com/huggingface/transformers/blob/2bd4d5897dc73e8b172832070a6f9e567a0df017/src/transformers/models/pixtral/image_processing_pixtral.py#L180 # noqa: E501
663
+ max_width, max_height = hf_config.image_size, hf_config.image_size
664
+ patch_width, patch_height = hf_config.patch_size, hf_config.patch_size
665
+
666
+ ratio = max(image_width / max_width, image_height / max_height)
667
+
668
+ if ratio > 1:
669
+ image_width = int(numpy.ceil(image_width / ratio))
670
+ image_height = int(numpy.ceil(image_height / ratio))
671
+
672
+ num_height_tokens, num_width_tokens = _num_image_tokens(
673
+ (image_height, image_width), (patch_height, patch_width))
674
+
675
+ return num_width_tokens, num_height_tokens
676
+
677
+
678
+def input_processor_for_pixtral_hf(
679
+ model_config: ModelConfig,
680
+ hf_config: PixtralVisionConfig,
681
+ inputs: DecoderOnlyInputs,
682
+ *,
683
+ image_token_id: int,
684
+ image_feature_size_override: Optional[Union[int, List[int]]] = None,
685
+) -> DecoderOnlyInputs:
686
+ assert image_feature_size_override is None, (
687
+ "image_feature_size_override is not supported for Pixtral")
688
+
689
+ multi_modal_data = inputs.get("multi_modal_data")
690
+ if multi_modal_data is None or "image" not in multi_modal_data:
691
+ return inputs
692
+
693
+ processor = cached_get_processor(model_config.model)
694
+
695
+ image_data = multi_modal_data["image"]
696
+ if isinstance(image_data, Image.Image):
697
+ image_data = [image_data]
698
+ elif not is_list_of(image_data, Image.Image):
699
+ raise TypeError(f"Invalid image type: {type(image_data)}")
700
+
701
+ new_prompt = inputs.get("prompt")
702
+ new_token_ids = inputs["prompt_token_ids"]
703
+
704
+ # Update new_prompt if present
705
+ if new_prompt:
706
+ replace_strings = []
707
+ for image in image_data:
708
+ w, h = image.size
709
+
710
+ (num_width_tokens,
711
+ num_height_tokens) = get_pixtral_hf_image_feature_size(
712
+ hf_config, image_width=w, image_height=h)
713
+
714
+ replace_tokens = [[processor.image_token] * num_width_tokens +
715
+ [processor.image_break_token]
716
+ ] * num_height_tokens
717
+ # Flatten list
718
+ replace_tokens = [
719
+ item for sublist in replace_tokens for item in sublist
720
+ ]
721
+ replace_tokens[-1] = processor.image_end_token
722
+ replace_str = "".join(replace_tokens)
723
+ replace_strings.append(replace_str)
724
+ new_prompt = new_prompt.replace(processor.image_token,
725
+ "<placeholder>", 1)
726
+
727
+ while "<placeholder>" in new_prompt:
728
+ replace_str = replace_strings.pop(0)
729
+ new_prompt = new_prompt.replace("<placeholder>", replace_str, 1)
730
+
731
+ # Update new_token_ids
732
+ image_token_id = 10
733
+ image_break_id = 12
734
+ image_end_id = 13
735
+ placeholder_token_id = -999
736
+ replace_tokens_list = []
737
+ for image in image_data:
738
+ w, h = image.size
739
+
740
+ num_width_tokens, num_height_tokens = get_pixtral_hf_image_feature_size(
741
+ hf_config, image_width=w, image_height=h)
742
+
743
+ replace_tokens = [[image_token_id] * num_width_tokens +
744
+ [image_break_id]] * num_height_tokens
745
+ # Flatten list
746
+ replace_tokens = [
747
+ item for sublist in replace_tokens for item in sublist
748
+ ]
749
+ replace_tokens[-1] = image_end_id
750
+ replace_tokens_list.append(replace_tokens)
751
+ # Replace image id with placeholder id
752
+ next_image_index = new_token_ids.index(image_token_id)
753
+ new_token_ids[next_image_index] = placeholder_token_id
754
+
755
+ while placeholder_token_id in new_token_ids:
756
+ replace_tokens = replace_tokens_list.pop(0)
757
+ next_image_index = new_token_ids.index(placeholder_token_id)
758
+ prefix = new_token_ids[:next_image_index]
759
+ postfix = new_token_ids[next_image_index + 1:]
760
+ new_token_ids = prefix + replace_tokens + postfix
761
+
762
+ # NOTE: Create a defensive copy of the original inputs
763
+ return token_inputs(prompt_token_ids=new_token_ids,
764
+ prompt=new_prompt,
765
+ multi_modal_data=multi_modal_data)
766
+
767
+
768
+class PixtralHFMLP(nn.Module):
769
+
770
+ def __init__(self, config: PixtralVisionConfig):
771
+ super().__init__()
772
+ assert config.intermediate_size is not None
773
+ self.gate_proj = nn.Linear(config.hidden_size,
774
+ config.intermediate_size,
775
+ bias=False)
776
+ self.up_proj = nn.Linear(config.hidden_size,
777
+ config.intermediate_size,
778
+ bias=False)
779
+ self.down_proj = nn.Linear(config.intermediate_size,
780
+ config.hidden_size,
781
+ bias=False)
782
+ self.act = get_act_fn(config.hidden_act)
783
+
784
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
785
+ return self.down_proj(self.act(self.gate_proj(x)) * self.up_proj(x))
786
+
787
+
788
+class PixtralHFAttention(nn.Module):
789
+
790
+ def __init__(self, config: PixtralVisionConfig):
791
+ super().__init__()
792
+ self.config = config
793
+ assert not config.hidden_size % config.num_attention_heads
794
+ self.n_heads = config.num_attention_heads
795
+ self.head_dim = config.hidden_size // config.num_attention_heads
796
+
797
+ self.scale = self.head_dim**-0.5
798
+
799
+ self.q_proj = nn.Linear(config.hidden_size,
800
+ config.hidden_size,
801
+ bias=False)
802
+ self.k_proj = nn.Linear(config.hidden_size,
803
+ config.hidden_size,
804
+ bias=False)
805
+ self.v_proj = nn.Linear(config.hidden_size,
806
+ config.hidden_size,
807
+ bias=False)
808
+ self.o_proj = nn.Linear(config.hidden_size,
809
+ config.hidden_size,
810
+ bias=False)
811
+
812
+ def forward(
813
+ self,
814
+ hidden_states: torch.Tensor,
815
+ attention_mask: torch.Tensor,
816
+ position_embeddings: torch.Tensor,
817
+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
818
+ """Input shape: Batch x Time x Channel"""
819
+
820
+ batch_size, patches, _ = hidden_states.size()
821
+
822
+ query_states = self.q_proj(hidden_states)
823
+ key_states = self.k_proj(hidden_states)
824
+ value_states = self.v_proj(hidden_states)
825
+
826
+ query_states = query_states.view(batch_size, patches, self.n_heads,
827
+ self.head_dim).transpose(1, 2)
828
+ key_states = key_states.view(batch_size, patches, self.n_heads,
829
+ self.head_dim).transpose(1, 2)
830
+ value_states = value_states.view(batch_size, patches, self.n_heads,
831
+ self.head_dim).transpose(1, 2)
832
+
833
+ cos, sin = position_embeddings
834
+ query_states, key_states = apply_rotary_pos_emb(query_states,
835
+ key_states,
836
+ cos,
837
+ sin,
838
+ unsqueeze_dim=0)
839
+
840
+ attn_weights = torch.matmul(query_states, key_states.transpose(
841
+ 2, 3)) * self.scale
842
+
843
+ if attention_mask is not None:
844
+ attn_weights = attn_weights + attention_mask
845
+
846
+ # upcast attention to fp32
847
+ attn_weights = nn.functional.softmax(attn_weights,
848
+ dim=-1,
849
+ dtype=torch.float32).to(
850
+ query_states.dtype)
851
+ attn_output = torch.matmul(attn_weights, value_states)
852
+
853
+ attn_output = attn_output.transpose(1, 2).contiguous()
854
+ attn_output = attn_output.reshape(batch_size, patches, -1)
855
+
856
+ return self.o_proj(attn_output)
857
+
858
+
859
+class PixtralHFTransformerBlock(nn.Module):
860
+
861
+ def __init__(self, config: PixtralVisionConfig):
862
+ super().__init__()
863
+ self.attention_norm = RMSNorm(config.hidden_size, eps=1e-5)
864
+ self.attention = PixtralHFAttention(config)
865
+ self.feed_forward = PixtralHFMLP(config)
866
+ self.ffn_norm = RMSNorm(config.hidden_size, eps=1e-5)
867
+
868
+ def forward(
869
+ self,
870
+ hidden_states: torch.Tensor,
871
+ attention_mask: torch.Tensor,
872
+ position_embeddings: torch.Tensor,
873
+ ) -> torch.Tensor:
874
+ r = self.attention.forward(self.attention_norm(hidden_states),
875
+ attention_mask=attention_mask,
876
+ position_embeddings=position_embeddings)
877
+ h = hidden_states + r
878
+ r = self.feed_forward.forward(self.ffn_norm(h))
879
+ out = h + r
880
+ return out
881
+
882
+
883
+class PixtralHFTransformer(nn.Module):
884
+
885
+ def __init__(self, config: PixtralVisionConfig):
886
+ super().__init__()
887
+ self.layers = torch.nn.ModuleList()
888
+ for _ in range(config.num_hidden_layers):
889
+ self.layers.append(PixtralHFTransformerBlock(config))
890
+
891
+ def forward(
892
+ self,
893
+ x: torch.Tensor,
894
+ attention_mask: torch.Tensor,
895
+ position_embeddings: torch.Tensor,
896
+ ) -> torch.Tensor:
897
+ for layer in self.layers:
898
+ x = layer(x, attention_mask, position_embeddings)
899
+ return x
900
+
901
+
902
+class PixtralHFVisionModel(nn.Module):
903
+
904
+ def __init__(self, config: PixtralVisionConfig):
905
+ super().__init__()
906
+
907
+ self.config = config
908
+ self.patch_conv = nn.Conv2d(
909
+ in_channels=config.num_channels,
910
+ out_channels=config.hidden_size,
911
+ kernel_size=config.patch_size,
912
+ stride=config.patch_size,
913
+ bias=False,
914
+ )
915
+ self.ln_pre = RMSNorm(config.hidden_size, eps=1e-5)
916
+ self.transformer = PixtralHFTransformer(config)
917
+ self.dtype = next(self.parameters()).dtype
918
+ self.device = next(self.parameters()).device
919
+ self.patch_positional_embedding = PixtralRotaryEmbedding(
920
+ config, self.device)
921
+
922
+ def forward(
923
+ self,
924
+ pixel_values: List[torch.Tensor],
925
+ ) -> torch.Tensor:
926
+ """
927
+ Args:
928
+ pixel_values: tensor of token features for
929
+ all tokens of all images of shape (N_toks, D)
930
+ Returns:
931
+ image_features: tensor of token features for
932
+ all tokens of all images of shape (N_toks, D)
933
+ """
934
+ # pass images through initial convolution independently
935
+ patch_embeds_list = [
936
+ self.patch_conv(
937
+ img.reshape(-1, img.shape[-3], img.shape[-2],
938
+ img.shape[-1]).to(self.dtype))
939
+ for img in pixel_values
940
+ ]
941
+
942
+ # flatten to a single sequence
943
+ patch_embeds = torch.cat(
944
+ [p.flatten(2).permute(0, 2, 1) for p in patch_embeds_list], dim=1)
945
+ patch_embeds = self.ln_pre(patch_embeds)
946
+
947
+ # positional embeddings
948
+ position_ids = position_ids_in_meshgrid(
949
+ patch_embeds_list,
950
+ max_width=self.config.image_size // self.config.patch_size).to(
951
+ self.device)
952
+
953
+ position_embedding = self.patch_positional_embedding(
954
+ patch_embeds, position_ids)
955
+ attention_mask = generate_block_attention_mask(
956
+ [p.shape[-2] * p.shape[-1] for p in patch_embeds_list],
957
+ patch_embeds)
958
+ out = self.transformer(patch_embeds, attention_mask,
959
+ position_embedding)
960
+
961
+ return out
962
+
963
+ # (TODO) Add prefix argument for filtering out weights to be loaded
964
+ # ref: https://github.com/vllm-project/vllm/pull/7186#discussion_r1734163986
965
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
966
+ stacked_params_mapping = []
967
+ params_dict = dict(self.named_parameters())
968
+
969
+ for name, loaded_weight in weights:
970
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
971
+ if weight_name not in name:
972
+ continue
973
+
974
+ param = params_dict[name.replace(weight_name, param_name)]
975
+ weight_loader = param.weight_loader
976
+ weight_loader(param, loaded_weight, shard_id)
977
+ break
978
+ else:
979
+ param = params_dict[name]
980
+ weight_loader = getattr(param, "weight_loader",
981
+ default_weight_loader)
982
+ weight_loader(param, loaded_weight)
vllm/model_executor/models/qwen2_vl.py
+2
-4
@@ -22,7 +22,7 @@
22
# See the License for the specific language governing permissions and
23
# limitations under the License.
24
"""Inference-only Qwen2-VL model compatible with HuggingFace weights."""
25
-from functools import lru_cache, partial
25
+from functools import partial
26
from typing import (Any, Callable, Iterable, List, Literal, Mapping, Optional,
27
Tuple, Type, TypedDict, Union)
28
@@ -63,7 +63,7 @@ from vllm.multimodal.base import MultiModalData
63
from vllm.multimodal.image import cached_get_image_processor
64
from vllm.sequence import IntermediateTensors, SequenceData
65
from vllm.transformers_utils.config import uses_mrope
66
-from vllm.transformers_utils.processor import get_processor
66
+from vllm.transformers_utils.processor import cached_get_processor
67
68
from .interfaces import SupportsMultiModal, SupportsPP
69
from .utils import (PPMissingLayer, get_vit_attn_backend,
@@ -544,8 +544,6 @@ class Qwen2VisionTransformer(nn.Module):
544
545
# === Vision input helpers === #
546
547
-cached_get_processor = lru_cache(get_processor)
548
-
547
548
def mm_input_mapper_for_qwen2_vl(
549
ctx: InputContext,
vllm/transformers_utils/processor.py
+4
@@ -1,3 +1,4 @@
1
+from functools import lru_cache
2
from typing import Any, cast
3
4
@@ -37,6 +38,9 @@ def get_processor(
38
return cast(ProcessorMixin, processor)
39
40
41
+cached_get_processor = lru_cache(get_processor)
42
+
43
+
44
def get_image_processor(
45
processor_name: str,
46
*args: Any,