@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3921a2f29e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Model] Support Pixtral models in the HF Transformers format (#9036)

Michael Goin committed Oct 18, 2024 at 15:29 UTC 3921a2f29e30df293459d824e20d2e546e4af0c7
7 files changed +503 -12
docs/source/models/supported_models.rst
+1 -1
@@ -437,7 +437,7 @@ Text Generation
437 * - :code:`PixtralForConditionalGeneration`
438 - Pixtral
439 - T + I\ :sup:`+`
440 - - :code:`mistralai/Pixtral-12B-2409`
440 + - :code:`mistralai/Pixtral-12B-2409`, :code:`mistral-community/pixtral-12b` etc.
441 -
442 - ✅︎
443 * - :code:`QWenLMHeadModel`
examples/offline_inference_vision_language.py
+17
@@ -277,6 +277,22 @@ def run_qwen2_vl(question: str, modality: str):
277 return llm, prompt, stop_token_ids
278
279
280 +# Pixtral HF-format
281 +def run_pixtral_hf(question: str, modality: str):
282 + assert modality == "image"
283 +
284 + model_name = "mistral-community/pixtral-12b"
285 +
286 + llm = LLM(
287 + model=model_name,
288 + max_model_len=8192,
289 + )
290 +
291 + prompt = f"<s>[INST]{question}\n[IMG][/INST]"
292 + stop_token_ids = None
293 + return llm, prompt, stop_token_ids
294 +
295 +
296 # LLama 3.2
297 def run_mllama(question: str, modality: str):
298 assert modality == "image"
@@ -347,6 +363,7 @@ model_example_map = {
363 "NVLM_D": run_nvlm_d,
364 "qwen_vl": run_qwen_vl,
365 "qwen2_vl": run_qwen2_vl,
366 + "pixtral_hf": run_pixtral_hf,
367 "mllama": run_mllama,
368 "molmo": run_molmo,
369 "glm4v": run_glm4v,
vllm/model_executor/layers/activation.py
+2
@@ -264,6 +264,8 @@ _ACTIVATION_REGISTRY = LazyDict({
264 lambda: nn.ReLU(),
265 "relu2":
266 lambda: ReLUSquaredActivation(),
267 + "silu":
268 + lambda: nn.SiLU(),
269 "quick_gelu":
270 lambda: QuickGELU(),
271 })
vllm/model_executor/models/llava.py
+70 -4
@@ -5,7 +5,8 @@ from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
5 import torch
6 import torch.nn as nn
7 from PIL import Image
8 -from transformers import CLIPVisionConfig, LlavaConfig, SiglipVisionConfig
8 +from transformers import (CLIPVisionConfig, LlavaConfig, PixtralVisionConfig,
9 + SiglipVisionConfig)
10
11 from vllm.attention import AttentionMetadata
12 from vllm.config import CacheConfig, MultiModalConfig
@@ -22,6 +23,10 @@ from .clip import (CLIPVisionModel, dummy_image_for_clip,
23 dummy_seq_data_for_clip, get_max_clip_image_tokens,
24 input_processor_for_clip)
25 from .interfaces import SupportsMultiModal, SupportsPP
26 +from .pixtral import (PixtralHFVisionModel, dummy_image_for_pixtral_hf,
27 + dummy_seq_data_for_pixtral_hf,
28 + get_max_pixtral_hf_image_tokens,
29 + input_processor_for_pixtral_hf)
30 from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
31 dummy_seq_data_for_siglip, get_max_siglip_image_tokens,
32 input_processor_for_siglip)
@@ -31,8 +36,13 @@ from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
36
37 class LlavaImagePixelInputs(TypedDict):
38 type: Literal["pixel_values"]
34 - data: torch.Tensor
35 - """Shape: `(batch_size * num_images, num_channels, height, width)`"""
39 + data: Union[torch.Tensor, List[torch.Tensor]]
40 + """
41 + Shape: `(batch_size * num_images, num_channels, height, width)`
42 +
43 + Note that `height` or `width` may be different per batch and image,
44 + in which case the data is passed as a list instead of a batched tensor.
45 + """
46
47
48 class LlavaImageEmbeddingInputs(TypedDict):
@@ -77,6 +87,8 @@ def get_max_llava_image_tokens(ctx: InputContext):
87 num_image_tokens = get_max_clip_image_tokens(vision_config)
88 elif isinstance(vision_config, SiglipVisionConfig):
89 num_image_tokens = get_max_siglip_image_tokens(vision_config)
90 + elif isinstance(vision_config, PixtralVisionConfig):
91 + num_image_tokens = get_max_pixtral_hf_image_tokens(vision_config)
92 else:
93 msg = f"Unsupported vision config: {type(vision_config)}"
94 raise NotImplementedError(msg)
@@ -120,6 +132,17 @@ def dummy_data_for_llava(ctx: InputContext, seq_len: int,
132
133 mm_data = dummy_image_for_siglip(vision_config, num_images)
134 return seq_data, mm_data
135 + elif isinstance(vision_config, PixtralVisionConfig):
136 + seq_data = dummy_seq_data_for_pixtral_hf(
137 + vision_config,
138 + seq_len,
139 + num_images,
140 + image_token_id=hf_config.image_token_index,
141 + image_feature_size_override=image_feature_size,
142 + )
143 +
144 + mm_data = dummy_image_for_pixtral_hf(vision_config, num_images)
145 + return seq_data, mm_data
146
147 msg = f"Unsupported vision config: {type(vision_config)}"
148 raise NotImplementedError(msg)
@@ -163,6 +186,15 @@ def input_processor_for_llava(ctx: InputContext, inputs: DecoderOnlyInputs):
186 image_token_id=hf_config.image_token_index,
187 image_feature_size_override=image_feature_size,
188 )
189 + elif isinstance(vision_config, PixtralVisionConfig):
190 + # We ignore image_feature_size_override since we have non-uniform
191 + # image sizes for Pixtral
192 + return input_processor_for_pixtral_hf(
193 + model_config,
194 + vision_config,
195 + inputs,
196 + image_token_id=hf_config.image_token_index,
197 + )
198
199 msg = f"Unsupported vision config: {type(vision_config)}"
200 raise NotImplementedError(msg)
@@ -189,6 +221,9 @@ def _init_vision_tower(hf_config: LlavaConfig):
221 vision_config,
222 num_hidden_layers_override=num_hidden_layers,
223 )
224 + elif isinstance(vision_config, PixtralVisionConfig):
225 + # TODO: allow layer override?
226 + return PixtralHFVisionModel(vision_config)
227
228 msg = f"Unsupported vision config: {type(vision_config)}"
229 raise NotImplementedError(msg)
@@ -210,6 +245,15 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
245 self.config = config
246 self.multimodal_config = multimodal_config
247
248 + # NOTE: These are special cases for Pixtral-12B in the HF-format
249 + # https://huggingface.co/mistral-community/pixtral-12b/blob/main/config.json # noqa
250 + if (config.text_config.architectures is None
251 + and config.text_config.model_type == "mistral"):
252 + config.text_config.architectures = ["MistralForCausalLM"]
253 + if (config.projector_hidden_act is None
254 + and config.vision_config.hidden_act == "gelu"):
255 + config.projector_hidden_act = "gelu"
256 +
257 # TODO: Optionally initializes this for supporting embeddings.
258 self.vision_tower = _init_vision_tower(config)
259 self.multi_modal_projector = LlavaMultiModalProjector(
@@ -246,6 +290,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
290 def _parse_and_validate_image_input(
291 self, **kwargs: object) -> Optional[LlavaImageInputs]:
292 pixel_values = kwargs.pop("pixel_values", None)
293 + image_sizes = kwargs.pop("image_sizes", None)
294 image_embeds = kwargs.pop("image_embeds", None)
295
296 if pixel_values is None and image_embeds is None:
@@ -256,6 +301,26 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
301 raise ValueError("Incorrect type of pixel values. "
302 f"Got type: {type(pixel_values)}")
303
304 + # Case for models like PixtralHF that have dynamic image sizes
305 + # so we need to produce a list of tensors
306 + if image_sizes is not None:
307 + images = pixel_values
308 + if isinstance(images, torch.Tensor):
309 + # if passed as batch take all images
310 + NN, N, B, C, W, H = images.shape
311 + images = images.reshape(NN * N * B, C, W, H)
312 + images = [images[i] for i in range(images.size(0))]
313 + elif isinstance(images, list):
314 + # if passed as list flatten lists of tensors
315 + while isinstance(images, list) and len(images) == 1:
316 + images = images[0]
317 +
318 + # TODO: Add validation based on image_sizes
319 + return LlavaImagePixelInputs(
320 + type="pixel_values",
321 + data=images,
322 + )
323 +
324 return LlavaImagePixelInputs(
325 type="pixel_values",
326 data=self._validate_pixel_values(
@@ -286,7 +351,8 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
351
352 def _image_pixels_to_features(
353 self,
289 - vision_tower: Union[CLIPVisionModel, SiglipVisionModel],
354 + vision_tower: Union[CLIPVisionModel, SiglipVisionModel,
355 + PixtralHFVisionModel],
356 pixel_values: torch.Tensor,
357 ) -> torch.Tensor:
358
vllm/model_executor/models/pixtral.py
+407 -3
@@ -3,18 +3,26 @@ from functools import cached_property
3 from itertools import tee
4 from typing import Iterable, List, Mapping, Optional, Tuple, Union
5
6 +import numpy
7 import torch
8 import torch.nn as nn
9 import torch.nn.functional as F
10 from mistral_common.protocol.instruct.messages import ImageChunk
11 from PIL import Image
11 -from transformers import PretrainedConfig
12 +from transformers import PixtralVisionConfig, PretrainedConfig
13 +from transformers.models.pixtral.image_processing_pixtral import (
14 + _num_image_tokens)
15 +from transformers.models.pixtral.modeling_pixtral import (
16 + PixtralRotaryEmbedding, apply_rotary_pos_emb,
17 + generate_block_attention_mask, position_ids_in_meshgrid)
18 from xformers.ops.fmha import memory_efficient_attention
19 from xformers.ops.fmha.attn_bias import BlockDiagonalMask
20
21 from vllm.attention import AttentionMetadata
16 -from vllm.config import CacheConfig, MultiModalConfig
17 -from vllm.inputs import INPUT_REGISTRY, DecoderOnlyInputs, InputContext
22 +from vllm.config import CacheConfig, ModelConfig, MultiModalConfig
23 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
24 + token_inputs)
25 +from vllm.model_executor.layers.activation import get_act_fn
26 from vllm.model_executor.layers.layernorm import RMSNorm
27 from vllm.model_executor.layers.quantization import QuantizationConfig
28 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
@@ -25,6 +33,8 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
33 from vllm.multimodal.base import MultiModalInputs
34 from vllm.multimodal.utils import cached_get_tokenizer
35 from vllm.sequence import IntermediateTensors, SequenceData
36 +from vllm.transformers_utils.processor import cached_get_processor
37 +from vllm.utils import is_list_of
38
39 from .interfaces import SupportsMultiModal, SupportsPP
40 from .utils import init_vllm_registered_model
@@ -576,3 +586,397 @@ class VisionLanguageAdapter(nn.Module):
586
587 def forward(self, x: torch.Tensor) -> torch.Tensor:
588 return self.w_out(self.gelu(self.w_in(x)))
589 +
590 +
591 +#### HF Transformers version of Pixtral ####
592 +# Based off https://github.com/huggingface/transformers/blob/d7950bff82b18c823193d17d72188c5e46d06c83/src/transformers/models/pixtral/modeling_pixtral.py
593 +# This model follows the Llava family, meaning image embeddings are placed
594 +# instead of the `[IMG]` token placeholders.
595 +# The model uses [`PixtralVisionModel`] for its vision encoder,
596 +# and [`MistralForCausalLM`] for its language decoder.
597 +
598 +
599 +def get_pixtral_hf_patch_grid_length(*, image_size: int,
600 + patch_size: int) -> int:
601 + # Since interpolation is applied, the image size need not be divisible
602 + # assert image_size % patch_size == 0
603 + return image_size // patch_size
604 +
605 +
606 +def get_pixtral_hf_num_patches(*, image_size: int, patch_size: int) -> int:
607 + grid_length = get_pixtral_hf_patch_grid_length(image_size=image_size,
608 + patch_size=patch_size)
609 + return grid_length * grid_length
610 +
611 +
612 +def get_max_pixtral_hf_image_feature_size(
613 + hf_config: PixtralVisionConfig) -> int:
614 + return get_pixtral_hf_num_patches(image_size=hf_config.image_size,
615 + patch_size=hf_config.patch_size)
616 +
617 +
618 +def get_max_pixtral_hf_image_tokens(hf_config: PixtralVisionConfig) -> int:
619 + return get_max_pixtral_hf_image_feature_size(hf_config)
620 +
621 +
622 +def dummy_seq_data_for_pixtral_hf(
623 + hf_config: PixtralVisionConfig,
624 + seq_len: int,
625 + num_images: int,
626 + *,
627 + image_token_id: int,
628 + image_feature_size_override: Optional[int] = None,
629 +):
630 + if image_feature_size_override is None:
631 + image_feature_size = get_max_pixtral_hf_image_feature_size(hf_config)
632 + else:
633 + image_feature_size = image_feature_size_override
634 +
635 + return SequenceData.from_prompt_token_counts(
636 + (image_token_id, image_feature_size * num_images),
637 + (0, seq_len - image_feature_size * num_images),
638 + )
639 +
640 +
641 +def dummy_image_for_pixtral_hf(
642 + hf_config: PixtralVisionConfig,
643 + num_images: int,
644 + *,
645 + image_width_override: Optional[int] = None,
646 + image_height_override: Optional[int] = None,
647 +):
648 + width = height = hf_config.image_size
649 + if image_width_override is not None:
650 + width = image_width_override
651 + if image_height_override is not None:
652 + height = image_height_override
653 +
654 + image = Image.new("RGB", (width, height), color=0)
655 + return {"image": image if num_images == 1 else [image] * num_images}
656 +
657 +
658 +def get_pixtral_hf_image_feature_size(hf_config: PixtralVisionConfig,
659 + image_width: int,
660 + image_height: int) -> Tuple[int, int]:
661 + # Adapted from transformers.models.pixtral.image_processing_pixtral.get_resize_output_image_size # noqa: E501
662 + # https://github.com/huggingface/transformers/blob/2bd4d5897dc73e8b172832070a6f9e567a0df017/src/transformers/models/pixtral/image_processing_pixtral.py#L180 # noqa: E501
663 + max_width, max_height = hf_config.image_size, hf_config.image_size
664 + patch_width, patch_height = hf_config.patch_size, hf_config.patch_size
665 +
666 + ratio = max(image_width / max_width, image_height / max_height)
667 +
668 + if ratio > 1:
669 + image_width = int(numpy.ceil(image_width / ratio))
670 + image_height = int(numpy.ceil(image_height / ratio))
671 +
672 + num_height_tokens, num_width_tokens = _num_image_tokens(
673 + (image_height, image_width), (patch_height, patch_width))
674 +
675 + return num_width_tokens, num_height_tokens
676 +
677 +
678 +def input_processor_for_pixtral_hf(
679 + model_config: ModelConfig,
680 + hf_config: PixtralVisionConfig,
681 + inputs: DecoderOnlyInputs,
682 + *,
683 + image_token_id: int,
684 + image_feature_size_override: Optional[Union[int, List[int]]] = None,
685 +) -> DecoderOnlyInputs:
686 + assert image_feature_size_override is None, (
687 + "image_feature_size_override is not supported for Pixtral")
688 +
689 + multi_modal_data = inputs.get("multi_modal_data")
690 + if multi_modal_data is None or "image" not in multi_modal_data:
691 + return inputs
692 +
693 + processor = cached_get_processor(model_config.model)
694 +
695 + image_data = multi_modal_data["image"]
696 + if isinstance(image_data, Image.Image):
697 + image_data = [image_data]
698 + elif not is_list_of(image_data, Image.Image):
699 + raise TypeError(f"Invalid image type: {type(image_data)}")
700 +
701 + new_prompt = inputs.get("prompt")
702 + new_token_ids = inputs["prompt_token_ids"]
703 +
704 + # Update new_prompt if present
705 + if new_prompt:
706 + replace_strings = []
707 + for image in image_data:
708 + w, h = image.size
709 +
710 + (num_width_tokens,
711 + num_height_tokens) = get_pixtral_hf_image_feature_size(
712 + hf_config, image_width=w, image_height=h)
713 +
714 + replace_tokens = [[processor.image_token] * num_width_tokens +
715 + [processor.image_break_token]
716 + ] * num_height_tokens
717 + # Flatten list
718 + replace_tokens = [
719 + item for sublist in replace_tokens for item in sublist
720 + ]
721 + replace_tokens[-1] = processor.image_end_token
722 + replace_str = "".join(replace_tokens)
723 + replace_strings.append(replace_str)
724 + new_prompt = new_prompt.replace(processor.image_token,
725 + "<placeholder>", 1)
726 +
727 + while "<placeholder>" in new_prompt:
728 + replace_str = replace_strings.pop(0)
729 + new_prompt = new_prompt.replace("<placeholder>", replace_str, 1)
730 +
731 + # Update new_token_ids
732 + image_token_id = 10
733 + image_break_id = 12
734 + image_end_id = 13
735 + placeholder_token_id = -999
736 + replace_tokens_list = []
737 + for image in image_data:
738 + w, h = image.size
739 +
740 + num_width_tokens, num_height_tokens = get_pixtral_hf_image_feature_size(
741 + hf_config, image_width=w, image_height=h)
742 +
743 + replace_tokens = [[image_token_id] * num_width_tokens +
744 + [image_break_id]] * num_height_tokens
745 + # Flatten list
746 + replace_tokens = [
747 + item for sublist in replace_tokens for item in sublist
748 + ]
749 + replace_tokens[-1] = image_end_id
750 + replace_tokens_list.append(replace_tokens)
751 + # Replace image id with placeholder id
752 + next_image_index = new_token_ids.index(image_token_id)
753 + new_token_ids[next_image_index] = placeholder_token_id
754 +
755 + while placeholder_token_id in new_token_ids:
756 + replace_tokens = replace_tokens_list.pop(0)
757 + next_image_index = new_token_ids.index(placeholder_token_id)
758 + prefix = new_token_ids[:next_image_index]
759 + postfix = new_token_ids[next_image_index + 1:]
760 + new_token_ids = prefix + replace_tokens + postfix
761 +
762 + # NOTE: Create a defensive copy of the original inputs
763 + return token_inputs(prompt_token_ids=new_token_ids,
764 + prompt=new_prompt,
765 + multi_modal_data=multi_modal_data)
766 +
767 +
768 +class PixtralHFMLP(nn.Module):
769 +
770 + def __init__(self, config: PixtralVisionConfig):
771 + super().__init__()
772 + assert config.intermediate_size is not None
773 + self.gate_proj = nn.Linear(config.hidden_size,
774 + config.intermediate_size,
775 + bias=False)
776 + self.up_proj = nn.Linear(config.hidden_size,
777 + config.intermediate_size,
778 + bias=False)
779 + self.down_proj = nn.Linear(config.intermediate_size,
780 + config.hidden_size,
781 + bias=False)
782 + self.act = get_act_fn(config.hidden_act)
783 +
784 + def forward(self, x: torch.Tensor) -> torch.Tensor:
785 + return self.down_proj(self.act(self.gate_proj(x)) * self.up_proj(x))
786 +
787 +
788 +class PixtralHFAttention(nn.Module):
789 +
790 + def __init__(self, config: PixtralVisionConfig):
791 + super().__init__()
792 + self.config = config
793 + assert not config.hidden_size % config.num_attention_heads
794 + self.n_heads = config.num_attention_heads
795 + self.head_dim = config.hidden_size // config.num_attention_heads
796 +
797 + self.scale = self.head_dim**-0.5
798 +
799 + self.q_proj = nn.Linear(config.hidden_size,
800 + config.hidden_size,
801 + bias=False)
802 + self.k_proj = nn.Linear(config.hidden_size,
803 + config.hidden_size,
804 + bias=False)
805 + self.v_proj = nn.Linear(config.hidden_size,
806 + config.hidden_size,
807 + bias=False)
808 + self.o_proj = nn.Linear(config.hidden_size,
809 + config.hidden_size,
810 + bias=False)
811 +
812 + def forward(
813 + self,
814 + hidden_states: torch.Tensor,
815 + attention_mask: torch.Tensor,
816 + position_embeddings: torch.Tensor,
817 + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
818 + """Input shape: Batch x Time x Channel"""
819 +
820 + batch_size, patches, _ = hidden_states.size()
821 +
822 + query_states = self.q_proj(hidden_states)
823 + key_states = self.k_proj(hidden_states)
824 + value_states = self.v_proj(hidden_states)
825 +
826 + query_states = query_states.view(batch_size, patches, self.n_heads,
827 + self.head_dim).transpose(1, 2)
828 + key_states = key_states.view(batch_size, patches, self.n_heads,
829 + self.head_dim).transpose(1, 2)
830 + value_states = value_states.view(batch_size, patches, self.n_heads,
831 + self.head_dim).transpose(1, 2)
832 +
833 + cos, sin = position_embeddings
834 + query_states, key_states = apply_rotary_pos_emb(query_states,
835 + key_states,
836 + cos,
837 + sin,
838 + unsqueeze_dim=0)
839 +
840 + attn_weights = torch.matmul(query_states, key_states.transpose(
841 + 2, 3)) * self.scale
842 +
843 + if attention_mask is not None:
844 + attn_weights = attn_weights + attention_mask
845 +
846 + # upcast attention to fp32
847 + attn_weights = nn.functional.softmax(attn_weights,
848 + dim=-1,
849 + dtype=torch.float32).to(
850 + query_states.dtype)
851 + attn_output = torch.matmul(attn_weights, value_states)
852 +
853 + attn_output = attn_output.transpose(1, 2).contiguous()
854 + attn_output = attn_output.reshape(batch_size, patches, -1)
855 +
856 + return self.o_proj(attn_output)
857 +
858 +
859 +class PixtralHFTransformerBlock(nn.Module):
860 +
861 + def __init__(self, config: PixtralVisionConfig):
862 + super().__init__()
863 + self.attention_norm = RMSNorm(config.hidden_size, eps=1e-5)
864 + self.attention = PixtralHFAttention(config)
865 + self.feed_forward = PixtralHFMLP(config)
866 + self.ffn_norm = RMSNorm(config.hidden_size, eps=1e-5)
867 +
868 + def forward(
869 + self,
870 + hidden_states: torch.Tensor,
871 + attention_mask: torch.Tensor,
872 + position_embeddings: torch.Tensor,
873 + ) -> torch.Tensor:
874 + r = self.attention.forward(self.attention_norm(hidden_states),
875 + attention_mask=attention_mask,
876 + position_embeddings=position_embeddings)
877 + h = hidden_states + r
878 + r = self.feed_forward.forward(self.ffn_norm(h))
879 + out = h + r
880 + return out
881 +
882 +
883 +class PixtralHFTransformer(nn.Module):
884 +
885 + def __init__(self, config: PixtralVisionConfig):
886 + super().__init__()
887 + self.layers = torch.nn.ModuleList()
888 + for _ in range(config.num_hidden_layers):
889 + self.layers.append(PixtralHFTransformerBlock(config))
890 +
891 + def forward(
892 + self,
893 + x: torch.Tensor,
894 + attention_mask: torch.Tensor,
895 + position_embeddings: torch.Tensor,
896 + ) -> torch.Tensor:
897 + for layer in self.layers:
898 + x = layer(x, attention_mask, position_embeddings)
899 + return x
900 +
901 +
902 +class PixtralHFVisionModel(nn.Module):
903 +
904 + def __init__(self, config: PixtralVisionConfig):
905 + super().__init__()
906 +
907 + self.config = config
908 + self.patch_conv = nn.Conv2d(
909 + in_channels=config.num_channels,
910 + out_channels=config.hidden_size,
911 + kernel_size=config.patch_size,
912 + stride=config.patch_size,
913 + bias=False,
914 + )
915 + self.ln_pre = RMSNorm(config.hidden_size, eps=1e-5)
916 + self.transformer = PixtralHFTransformer(config)
917 + self.dtype = next(self.parameters()).dtype
918 + self.device = next(self.parameters()).device
919 + self.patch_positional_embedding = PixtralRotaryEmbedding(
920 + config, self.device)
921 +
922 + def forward(
923 + self,
924 + pixel_values: List[torch.Tensor],
925 + ) -> torch.Tensor:
926 + """
927 + Args:
928 + pixel_values: tensor of token features for
929 + all tokens of all images of shape (N_toks, D)
930 + Returns:
931 + image_features: tensor of token features for
932 + all tokens of all images of shape (N_toks, D)
933 + """
934 + # pass images through initial convolution independently
935 + patch_embeds_list = [
936 + self.patch_conv(
937 + img.reshape(-1, img.shape[-3], img.shape[-2],
938 + img.shape[-1]).to(self.dtype))
939 + for img in pixel_values
940 + ]
941 +
942 + # flatten to a single sequence
943 + patch_embeds = torch.cat(
944 + [p.flatten(2).permute(0, 2, 1) for p in patch_embeds_list], dim=1)
945 + patch_embeds = self.ln_pre(patch_embeds)
946 +
947 + # positional embeddings
948 + position_ids = position_ids_in_meshgrid(
949 + patch_embeds_list,
950 + max_width=self.config.image_size // self.config.patch_size).to(
951 + self.device)
952 +
953 + position_embedding = self.patch_positional_embedding(
954 + patch_embeds, position_ids)
955 + attention_mask = generate_block_attention_mask(
956 + [p.shape[-2] * p.shape[-1] for p in patch_embeds_list],
957 + patch_embeds)
958 + out = self.transformer(patch_embeds, attention_mask,
959 + position_embedding)
960 +
961 + return out
962 +
963 + # (TODO) Add prefix argument for filtering out weights to be loaded
964 + # ref: https://github.com/vllm-project/vllm/pull/7186#discussion_r1734163986
965 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
966 + stacked_params_mapping = []
967 + params_dict = dict(self.named_parameters())
968 +
969 + for name, loaded_weight in weights:
970 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
971 + if weight_name not in name:
972 + continue
973 +
974 + param = params_dict[name.replace(weight_name, param_name)]
975 + weight_loader = param.weight_loader
976 + weight_loader(param, loaded_weight, shard_id)
977 + break
978 + else:
979 + param = params_dict[name]
980 + weight_loader = getattr(param, "weight_loader",
981 + default_weight_loader)
982 + weight_loader(param, loaded_weight)
vllm/model_executor/models/qwen2_vl.py
+2 -4
@@ -22,7 +22,7 @@
22 # See the License for the specific language governing permissions and
23 # limitations under the License.
24 """Inference-only Qwen2-VL model compatible with HuggingFace weights."""
25 -from functools import lru_cache, partial
25 +from functools import partial
26 from typing import (Any, Callable, Iterable, List, Literal, Mapping, Optional,
27 Tuple, Type, TypedDict, Union)
28
@@ -63,7 +63,7 @@ from vllm.multimodal.base import MultiModalData
63 from vllm.multimodal.image import cached_get_image_processor
64 from vllm.sequence import IntermediateTensors, SequenceData
65 from vllm.transformers_utils.config import uses_mrope
66 -from vllm.transformers_utils.processor import get_processor
66 +from vllm.transformers_utils.processor import cached_get_processor
67
68 from .interfaces import SupportsMultiModal, SupportsPP
69 from .utils import (PPMissingLayer, get_vit_attn_backend,
@@ -544,8 +544,6 @@ class Qwen2VisionTransformer(nn.Module):
544
545 # === Vision input helpers === #
546
547 -cached_get_processor = lru_cache(get_processor)
548 -
547
548 def mm_input_mapper_for_qwen2_vl(
549 ctx: InputContext,
vllm/transformers_utils/processor.py
+4
@@ -1,3 +1,4 @@
1 +from functools import lru_cache
2 from typing import Any, cast
3
4
@@ -37,6 +38,9 @@ def get_processor(
38 return cast(ProcessorMixin, processor)
39
40
41 +cached_get_processor = lru_cache(get_processor)
42 +
43 +
44 def get_image_processor(
45 processor_name: str,
46 *args: Any,